1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/frame.h> 4 #include <linux/percpu.h> 5 6 #include <asm/debugreg.h> 7 #include <asm/mmu_context.h> 8 9 #include "cpuid.h" 10 #include "hyperv.h" 11 #include "mmu.h" 12 #include "nested.h" 13 #include "trace.h" 14 #include "x86.h" 15 16 static bool __read_mostly enable_shadow_vmcs = 1; 17 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 18 19 static bool __read_mostly nested_early_check = 0; 20 module_param(nested_early_check, bool, S_IRUGO); 21 22 /* 23 * Hyper-V requires all of these, so mark them as supported even though 24 * they are just treated the same as all-context. 25 */ 26 #define VMX_VPID_EXTENT_SUPPORTED_MASK \ 27 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 28 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 29 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 30 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 31 32 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 33 34 enum { 35 VMX_VMREAD_BITMAP, 36 VMX_VMWRITE_BITMAP, 37 VMX_BITMAP_NR 38 }; 39 static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 40 41 #define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 42 #define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 43 44 static u16 shadow_read_only_fields[] = { 45 #define SHADOW_FIELD_RO(x) x, 46 #include "vmcs_shadow_fields.h" 47 }; 48 static int max_shadow_read_only_fields = 49 ARRAY_SIZE(shadow_read_only_fields); 50 51 static u16 shadow_read_write_fields[] = { 52 #define SHADOW_FIELD_RW(x) x, 53 #include "vmcs_shadow_fields.h" 54 }; 55 static int max_shadow_read_write_fields = 56 ARRAY_SIZE(shadow_read_write_fields); 57 58 static void init_vmcs_shadow_fields(void) 59 { 60 int i, j; 61 62 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 63 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 64 65 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 66 u16 field = shadow_read_only_fields[i]; 67 68 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 69 (i + 1 == max_shadow_read_only_fields || 70 shadow_read_only_fields[i + 1] != field + 1)) 71 pr_err("Missing field from shadow_read_only_field %x\n", 72 field + 1); 73 74 clear_bit(field, vmx_vmread_bitmap); 75 #ifdef CONFIG_X86_64 76 if (field & 1) 77 continue; 78 #endif 79 if (j < i) 80 shadow_read_only_fields[j] = field; 81 j++; 82 } 83 max_shadow_read_only_fields = j; 84 85 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 86 u16 field = shadow_read_write_fields[i]; 87 88 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 89 (i + 1 == max_shadow_read_write_fields || 90 shadow_read_write_fields[i + 1] != field + 1)) 91 pr_err("Missing field from shadow_read_write_field %x\n", 92 field + 1); 93 94 /* 95 * PML and the preemption timer can be emulated, but the 96 * processor cannot vmwrite to fields that don't exist 97 * on bare metal. 98 */ 99 switch (field) { 100 case GUEST_PML_INDEX: 101 if (!cpu_has_vmx_pml()) 102 continue; 103 break; 104 case VMX_PREEMPTION_TIMER_VALUE: 105 if (!cpu_has_vmx_preemption_timer()) 106 continue; 107 break; 108 case GUEST_INTR_STATUS: 109 if (!cpu_has_vmx_apicv()) 110 continue; 111 break; 112 default: 113 break; 114 } 115 116 clear_bit(field, vmx_vmwrite_bitmap); 117 clear_bit(field, vmx_vmread_bitmap); 118 #ifdef CONFIG_X86_64 119 if (field & 1) 120 continue; 121 #endif 122 if (j < i) 123 shadow_read_write_fields[j] = field; 124 j++; 125 } 126 max_shadow_read_write_fields = j; 127 } 128 129 /* 130 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 131 * set the success or error code of an emulated VMX instruction (as specified 132 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 133 * instruction. 134 */ 135 static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 136 { 137 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 138 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 139 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 140 return kvm_skip_emulated_instruction(vcpu); 141 } 142 143 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 144 { 145 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 146 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 147 X86_EFLAGS_SF | X86_EFLAGS_OF)) 148 | X86_EFLAGS_CF); 149 return kvm_skip_emulated_instruction(vcpu); 150 } 151 152 static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 153 u32 vm_instruction_error) 154 { 155 struct vcpu_vmx *vmx = to_vmx(vcpu); 156 157 /* 158 * failValid writes the error number to the current VMCS, which 159 * can't be done if there isn't a current VMCS. 160 */ 161 if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs) 162 return nested_vmx_failInvalid(vcpu); 163 164 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 165 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 166 X86_EFLAGS_SF | X86_EFLAGS_OF)) 167 | X86_EFLAGS_ZF); 168 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 169 /* 170 * We don't need to force a shadow sync because 171 * VM_INSTRUCTION_ERROR is not shadowed 172 */ 173 return kvm_skip_emulated_instruction(vcpu); 174 } 175 176 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 177 { 178 /* TODO: not to reset guest simply here. */ 179 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 180 pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator); 181 } 182 183 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 184 { 185 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); 186 vmcs_write64(VMCS_LINK_POINTER, -1ull); 187 } 188 189 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 190 { 191 struct vcpu_vmx *vmx = to_vmx(vcpu); 192 193 if (!vmx->nested.hv_evmcs) 194 return; 195 196 kunmap(vmx->nested.hv_evmcs_page); 197 kvm_release_page_dirty(vmx->nested.hv_evmcs_page); 198 vmx->nested.hv_evmcs_vmptr = -1ull; 199 vmx->nested.hv_evmcs_page = NULL; 200 vmx->nested.hv_evmcs = NULL; 201 } 202 203 /* 204 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 205 * just stops using VMX. 206 */ 207 static void free_nested(struct kvm_vcpu *vcpu) 208 { 209 struct vcpu_vmx *vmx = to_vmx(vcpu); 210 211 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 212 return; 213 214 vmx->nested.vmxon = false; 215 vmx->nested.smm.vmxon = false; 216 free_vpid(vmx->nested.vpid02); 217 vmx->nested.posted_intr_nv = -1; 218 vmx->nested.current_vmptr = -1ull; 219 if (enable_shadow_vmcs) { 220 vmx_disable_shadow_vmcs(vmx); 221 vmcs_clear(vmx->vmcs01.shadow_vmcs); 222 free_vmcs(vmx->vmcs01.shadow_vmcs); 223 vmx->vmcs01.shadow_vmcs = NULL; 224 } 225 kfree(vmx->nested.cached_vmcs12); 226 kfree(vmx->nested.cached_shadow_vmcs12); 227 /* Unpin physical memory we referred to in the vmcs02 */ 228 if (vmx->nested.apic_access_page) { 229 kvm_release_page_dirty(vmx->nested.apic_access_page); 230 vmx->nested.apic_access_page = NULL; 231 } 232 if (vmx->nested.virtual_apic_page) { 233 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 234 vmx->nested.virtual_apic_page = NULL; 235 } 236 if (vmx->nested.pi_desc_page) { 237 kunmap(vmx->nested.pi_desc_page); 238 kvm_release_page_dirty(vmx->nested.pi_desc_page); 239 vmx->nested.pi_desc_page = NULL; 240 vmx->nested.pi_desc = NULL; 241 } 242 243 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 244 245 nested_release_evmcs(vcpu); 246 247 free_loaded_vmcs(&vmx->nested.vmcs02); 248 } 249 250 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 251 { 252 struct vcpu_vmx *vmx = to_vmx(vcpu); 253 int cpu; 254 255 if (vmx->loaded_vmcs == vmcs) 256 return; 257 258 cpu = get_cpu(); 259 vmx_vcpu_put(vcpu); 260 vmx->loaded_vmcs = vmcs; 261 vmx_vcpu_load(vcpu, cpu); 262 put_cpu(); 263 264 vm_entry_controls_reset_shadow(vmx); 265 vm_exit_controls_reset_shadow(vmx); 266 vmx_segment_cache_clear(vmx); 267 } 268 269 /* 270 * Ensure that the current vmcs of the logical processor is the 271 * vmcs01 of the vcpu before calling free_nested(). 272 */ 273 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 274 { 275 vcpu_load(vcpu); 276 vmx_leave_nested(vcpu); 277 vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01); 278 free_nested(vcpu); 279 vcpu_put(vcpu); 280 } 281 282 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 283 struct x86_exception *fault) 284 { 285 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 286 struct vcpu_vmx *vmx = to_vmx(vcpu); 287 u32 exit_reason; 288 unsigned long exit_qualification = vcpu->arch.exit_qualification; 289 290 if (vmx->nested.pml_full) { 291 exit_reason = EXIT_REASON_PML_FULL; 292 vmx->nested.pml_full = false; 293 exit_qualification &= INTR_INFO_UNBLOCK_NMI; 294 } else if (fault->error_code & PFERR_RSVD_MASK) 295 exit_reason = EXIT_REASON_EPT_MISCONFIG; 296 else 297 exit_reason = EXIT_REASON_EPT_VIOLATION; 298 299 nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification); 300 vmcs12->guest_physical_address = fault->address; 301 } 302 303 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 304 { 305 WARN_ON(mmu_is_nested(vcpu)); 306 307 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 308 kvm_init_shadow_ept_mmu(vcpu, 309 to_vmx(vcpu)->nested.msrs.ept_caps & 310 VMX_EPT_EXECUTE_ONLY_BIT, 311 nested_ept_ad_enabled(vcpu), 312 nested_ept_get_cr3(vcpu)); 313 vcpu->arch.mmu->set_cr3 = vmx_set_cr3; 314 vcpu->arch.mmu->get_cr3 = nested_ept_get_cr3; 315 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 316 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 317 318 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 319 } 320 321 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 322 { 323 vcpu->arch.mmu = &vcpu->arch.root_mmu; 324 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 325 } 326 327 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 328 u16 error_code) 329 { 330 bool inequality, bit; 331 332 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 333 inequality = 334 (error_code & vmcs12->page_fault_error_code_mask) != 335 vmcs12->page_fault_error_code_match; 336 return inequality ^ bit; 337 } 338 339 340 /* 341 * KVM wants to inject page-faults which it got to the guest. This function 342 * checks whether in a nested guest, we need to inject them to L1 or L2. 343 */ 344 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) 345 { 346 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 347 unsigned int nr = vcpu->arch.exception.nr; 348 bool has_payload = vcpu->arch.exception.has_payload; 349 unsigned long payload = vcpu->arch.exception.payload; 350 351 if (nr == PF_VECTOR) { 352 if (vcpu->arch.exception.nested_apf) { 353 *exit_qual = vcpu->arch.apf.nested_apf_token; 354 return 1; 355 } 356 if (nested_vmx_is_page_fault_vmexit(vmcs12, 357 vcpu->arch.exception.error_code)) { 358 *exit_qual = has_payload ? payload : vcpu->arch.cr2; 359 return 1; 360 } 361 } else if (vmcs12->exception_bitmap & (1u << nr)) { 362 if (nr == DB_VECTOR) { 363 if (!has_payload) { 364 payload = vcpu->arch.dr6; 365 payload &= ~(DR6_FIXED_1 | DR6_BT); 366 payload ^= DR6_RTM; 367 } 368 *exit_qual = payload; 369 } else 370 *exit_qual = 0; 371 return 1; 372 } 373 374 return 0; 375 } 376 377 378 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, 379 struct x86_exception *fault) 380 { 381 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 382 383 WARN_ON(!is_guest_mode(vcpu)); 384 385 if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && 386 !to_vmx(vcpu)->nested.nested_run_pending) { 387 vmcs12->vm_exit_intr_error_code = fault->error_code; 388 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 389 PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | 390 INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, 391 fault->address); 392 } else { 393 kvm_inject_page_fault(vcpu, fault); 394 } 395 } 396 397 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa) 398 { 399 return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu)); 400 } 401 402 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 403 struct vmcs12 *vmcs12) 404 { 405 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 406 return 0; 407 408 if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) || 409 !page_address_valid(vcpu, vmcs12->io_bitmap_b)) 410 return -EINVAL; 411 412 return 0; 413 } 414 415 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 416 struct vmcs12 *vmcs12) 417 { 418 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 419 return 0; 420 421 if (!page_address_valid(vcpu, vmcs12->msr_bitmap)) 422 return -EINVAL; 423 424 return 0; 425 } 426 427 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 428 struct vmcs12 *vmcs12) 429 { 430 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 431 return 0; 432 433 if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)) 434 return -EINVAL; 435 436 return 0; 437 } 438 439 /* 440 * Check if MSR is intercepted for L01 MSR bitmap. 441 */ 442 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr) 443 { 444 unsigned long *msr_bitmap; 445 int f = sizeof(unsigned long); 446 447 if (!cpu_has_vmx_msr_bitmap()) 448 return true; 449 450 msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap; 451 452 if (msr <= 0x1fff) { 453 return !!test_bit(msr, msr_bitmap + 0x800 / f); 454 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 455 msr &= 0x1fff; 456 return !!test_bit(msr, msr_bitmap + 0xc00 / f); 457 } 458 459 return true; 460 } 461 462 /* 463 * If a msr is allowed by L0, we should check whether it is allowed by L1. 464 * The corresponding bit will be cleared unless both of L0 and L1 allow it. 465 */ 466 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1, 467 unsigned long *msr_bitmap_nested, 468 u32 msr, int type) 469 { 470 int f = sizeof(unsigned long); 471 472 /* 473 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals 474 * have the write-low and read-high bitmap offsets the wrong way round. 475 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff. 476 */ 477 if (msr <= 0x1fff) { 478 if (type & MSR_TYPE_R && 479 !test_bit(msr, msr_bitmap_l1 + 0x000 / f)) 480 /* read-low */ 481 __clear_bit(msr, msr_bitmap_nested + 0x000 / f); 482 483 if (type & MSR_TYPE_W && 484 !test_bit(msr, msr_bitmap_l1 + 0x800 / f)) 485 /* write-low */ 486 __clear_bit(msr, msr_bitmap_nested + 0x800 / f); 487 488 } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) { 489 msr &= 0x1fff; 490 if (type & MSR_TYPE_R && 491 !test_bit(msr, msr_bitmap_l1 + 0x400 / f)) 492 /* read-high */ 493 __clear_bit(msr, msr_bitmap_nested + 0x400 / f); 494 495 if (type & MSR_TYPE_W && 496 !test_bit(msr, msr_bitmap_l1 + 0xc00 / f)) 497 /* write-high */ 498 __clear_bit(msr, msr_bitmap_nested + 0xc00 / f); 499 500 } 501 } 502 503 /* 504 * Merge L0's and L1's MSR bitmap, return false to indicate that 505 * we do not use the hardware. 506 */ 507 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 508 struct vmcs12 *vmcs12) 509 { 510 int msr; 511 struct page *page; 512 unsigned long *msr_bitmap_l1; 513 unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap; 514 /* 515 * pred_cmd & spec_ctrl are trying to verify two things: 516 * 517 * 1. L0 gave a permission to L1 to actually passthrough the MSR. This 518 * ensures that we do not accidentally generate an L02 MSR bitmap 519 * from the L12 MSR bitmap that is too permissive. 520 * 2. That L1 or L2s have actually used the MSR. This avoids 521 * unnecessarily merging of the bitmap if the MSR is unused. This 522 * works properly because we only update the L01 MSR bitmap lazily. 523 * So even if L0 should pass L1 these MSRs, the L01 bitmap is only 524 * updated to reflect this when L1 (or its L2s) actually write to 525 * the MSR. 526 */ 527 bool pred_cmd = !msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD); 528 bool spec_ctrl = !msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL); 529 530 /* Nothing to do if the MSR bitmap is not in use. */ 531 if (!cpu_has_vmx_msr_bitmap() || 532 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 533 return false; 534 535 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 536 !pred_cmd && !spec_ctrl) 537 return false; 538 539 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap); 540 if (is_error_page(page)) 541 return false; 542 543 msr_bitmap_l1 = (unsigned long *)kmap(page); 544 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 545 /* 546 * L0 need not intercept reads for MSRs between 0x800 and 0x8ff, it 547 * just lets the processor take the value from the virtual-APIC page; 548 * take those 256 bits directly from the L1 bitmap. 549 */ 550 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 551 unsigned word = msr / BITS_PER_LONG; 552 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 553 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 554 } 555 } else { 556 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 557 unsigned word = msr / BITS_PER_LONG; 558 msr_bitmap_l0[word] = ~0; 559 msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0; 560 } 561 } 562 563 nested_vmx_disable_intercept_for_msr( 564 msr_bitmap_l1, msr_bitmap_l0, 565 X2APIC_MSR(APIC_TASKPRI), 566 MSR_TYPE_W); 567 568 if (nested_cpu_has_vid(vmcs12)) { 569 nested_vmx_disable_intercept_for_msr( 570 msr_bitmap_l1, msr_bitmap_l0, 571 X2APIC_MSR(APIC_EOI), 572 MSR_TYPE_W); 573 nested_vmx_disable_intercept_for_msr( 574 msr_bitmap_l1, msr_bitmap_l0, 575 X2APIC_MSR(APIC_SELF_IPI), 576 MSR_TYPE_W); 577 } 578 579 if (spec_ctrl) 580 nested_vmx_disable_intercept_for_msr( 581 msr_bitmap_l1, msr_bitmap_l0, 582 MSR_IA32_SPEC_CTRL, 583 MSR_TYPE_R | MSR_TYPE_W); 584 585 if (pred_cmd) 586 nested_vmx_disable_intercept_for_msr( 587 msr_bitmap_l1, msr_bitmap_l0, 588 MSR_IA32_PRED_CMD, 589 MSR_TYPE_W); 590 591 kunmap(page); 592 kvm_release_page_clean(page); 593 594 return true; 595 } 596 597 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 598 struct vmcs12 *vmcs12) 599 { 600 struct vmcs12 *shadow; 601 struct page *page; 602 603 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 604 vmcs12->vmcs_link_pointer == -1ull) 605 return; 606 607 shadow = get_shadow_vmcs12(vcpu); 608 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 609 610 memcpy(shadow, kmap(page), VMCS12_SIZE); 611 612 kunmap(page); 613 kvm_release_page_clean(page); 614 } 615 616 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 617 struct vmcs12 *vmcs12) 618 { 619 struct vcpu_vmx *vmx = to_vmx(vcpu); 620 621 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 622 vmcs12->vmcs_link_pointer == -1ull) 623 return; 624 625 kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer, 626 get_shadow_vmcs12(vcpu), VMCS12_SIZE); 627 } 628 629 /* 630 * In nested virtualization, check if L1 has set 631 * VM_EXIT_ACK_INTR_ON_EXIT 632 */ 633 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 634 { 635 return get_vmcs12(vcpu)->vm_exit_controls & 636 VM_EXIT_ACK_INTR_ON_EXIT; 637 } 638 639 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu) 640 { 641 return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu)); 642 } 643 644 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 645 struct vmcs12 *vmcs12) 646 { 647 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 648 !page_address_valid(vcpu, vmcs12->apic_access_addr)) 649 return -EINVAL; 650 else 651 return 0; 652 } 653 654 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 655 struct vmcs12 *vmcs12) 656 { 657 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 658 !nested_cpu_has_apic_reg_virt(vmcs12) && 659 !nested_cpu_has_vid(vmcs12) && 660 !nested_cpu_has_posted_intr(vmcs12)) 661 return 0; 662 663 /* 664 * If virtualize x2apic mode is enabled, 665 * virtualize apic access must be disabled. 666 */ 667 if (nested_cpu_has_virt_x2apic_mode(vmcs12) && 668 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 669 return -EINVAL; 670 671 /* 672 * If virtual interrupt delivery is enabled, 673 * we must exit on external interrupts. 674 */ 675 if (nested_cpu_has_vid(vmcs12) && 676 !nested_exit_on_intr(vcpu)) 677 return -EINVAL; 678 679 /* 680 * bits 15:8 should be zero in posted_intr_nv, 681 * the descriptor address has been already checked 682 * in nested_get_vmcs12_pages. 683 * 684 * bits 5:0 of posted_intr_desc_addr should be zero. 685 */ 686 if (nested_cpu_has_posted_intr(vmcs12) && 687 (!nested_cpu_has_vid(vmcs12) || 688 !nested_exit_intr_ack_set(vcpu) || 689 (vmcs12->posted_intr_nv & 0xff00) || 690 (vmcs12->posted_intr_desc_addr & 0x3f) || 691 (vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))) 692 return -EINVAL; 693 694 /* tpr shadow is needed by all apicv features. */ 695 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 696 return -EINVAL; 697 698 return 0; 699 } 700 701 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 702 u32 count, u64 addr) 703 { 704 int maxphyaddr; 705 706 if (count == 0) 707 return 0; 708 maxphyaddr = cpuid_maxphyaddr(vcpu); 709 if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr || 710 (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) 711 return -EINVAL; 712 713 return 0; 714 } 715 716 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 717 struct vmcs12 *vmcs12) 718 { 719 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_load_count, 720 vmcs12->vm_exit_msr_load_addr) || 721 nested_vmx_check_msr_switch(vcpu, vmcs12->vm_exit_msr_store_count, 722 vmcs12->vm_exit_msr_store_addr)) 723 return -EINVAL; 724 725 return 0; 726 } 727 728 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 729 struct vmcs12 *vmcs12) 730 { 731 if (nested_vmx_check_msr_switch(vcpu, vmcs12->vm_entry_msr_load_count, 732 vmcs12->vm_entry_msr_load_addr)) 733 return -EINVAL; 734 735 return 0; 736 } 737 738 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 739 struct vmcs12 *vmcs12) 740 { 741 if (!nested_cpu_has_pml(vmcs12)) 742 return 0; 743 744 if (!nested_cpu_has_ept(vmcs12) || 745 !page_address_valid(vcpu, vmcs12->pml_address)) 746 return -EINVAL; 747 748 return 0; 749 } 750 751 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 752 struct vmcs12 *vmcs12) 753 { 754 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 755 !nested_cpu_has_ept(vmcs12)) 756 return -EINVAL; 757 return 0; 758 } 759 760 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 761 struct vmcs12 *vmcs12) 762 { 763 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 764 !nested_cpu_has_ept(vmcs12)) 765 return -EINVAL; 766 return 0; 767 } 768 769 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 770 struct vmcs12 *vmcs12) 771 { 772 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 773 return 0; 774 775 if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) || 776 !page_address_valid(vcpu, vmcs12->vmwrite_bitmap)) 777 return -EINVAL; 778 779 return 0; 780 } 781 782 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 783 struct vmx_msr_entry *e) 784 { 785 /* x2APIC MSR accesses are not allowed */ 786 if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8) 787 return -EINVAL; 788 if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */ 789 e->index == MSR_IA32_UCODE_REV) 790 return -EINVAL; 791 if (e->reserved != 0) 792 return -EINVAL; 793 return 0; 794 } 795 796 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 797 struct vmx_msr_entry *e) 798 { 799 if (e->index == MSR_FS_BASE || 800 e->index == MSR_GS_BASE || 801 e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */ 802 nested_vmx_msr_check_common(vcpu, e)) 803 return -EINVAL; 804 return 0; 805 } 806 807 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 808 struct vmx_msr_entry *e) 809 { 810 if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */ 811 nested_vmx_msr_check_common(vcpu, e)) 812 return -EINVAL; 813 return 0; 814 } 815 816 /* 817 * Load guest's/host's msr at nested entry/exit. 818 * return 0 for success, entry index for failure. 819 */ 820 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 821 { 822 u32 i; 823 struct vmx_msr_entry e; 824 struct msr_data msr; 825 826 msr.host_initiated = false; 827 for (i = 0; i < count; i++) { 828 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 829 &e, sizeof(e))) { 830 pr_debug_ratelimited( 831 "%s cannot read MSR entry (%u, 0x%08llx)\n", 832 __func__, i, gpa + i * sizeof(e)); 833 goto fail; 834 } 835 if (nested_vmx_load_msr_check(vcpu, &e)) { 836 pr_debug_ratelimited( 837 "%s check failed (%u, 0x%x, 0x%x)\n", 838 __func__, i, e.index, e.reserved); 839 goto fail; 840 } 841 msr.index = e.index; 842 msr.data = e.value; 843 if (kvm_set_msr(vcpu, &msr)) { 844 pr_debug_ratelimited( 845 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 846 __func__, i, e.index, e.value); 847 goto fail; 848 } 849 } 850 return 0; 851 fail: 852 return i + 1; 853 } 854 855 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 856 { 857 u32 i; 858 struct vmx_msr_entry e; 859 860 for (i = 0; i < count; i++) { 861 struct msr_data msr_info; 862 if (kvm_vcpu_read_guest(vcpu, 863 gpa + i * sizeof(e), 864 &e, 2 * sizeof(u32))) { 865 pr_debug_ratelimited( 866 "%s cannot read MSR entry (%u, 0x%08llx)\n", 867 __func__, i, gpa + i * sizeof(e)); 868 return -EINVAL; 869 } 870 if (nested_vmx_store_msr_check(vcpu, &e)) { 871 pr_debug_ratelimited( 872 "%s check failed (%u, 0x%x, 0x%x)\n", 873 __func__, i, e.index, e.reserved); 874 return -EINVAL; 875 } 876 msr_info.host_initiated = false; 877 msr_info.index = e.index; 878 if (kvm_get_msr(vcpu, &msr_info)) { 879 pr_debug_ratelimited( 880 "%s cannot read MSR (%u, 0x%x)\n", 881 __func__, i, e.index); 882 return -EINVAL; 883 } 884 if (kvm_vcpu_write_guest(vcpu, 885 gpa + i * sizeof(e) + 886 offsetof(struct vmx_msr_entry, value), 887 &msr_info.data, sizeof(msr_info.data))) { 888 pr_debug_ratelimited( 889 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 890 __func__, i, e.index, msr_info.data); 891 return -EINVAL; 892 } 893 } 894 return 0; 895 } 896 897 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val) 898 { 899 unsigned long invalid_mask; 900 901 invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu); 902 return (val & invalid_mask) == 0; 903 } 904 905 /* 906 * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are 907 * emulating VM entry into a guest with EPT enabled. 908 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 909 * is assigned to entry_failure_code on failure. 910 */ 911 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, 912 u32 *entry_failure_code) 913 { 914 if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) { 915 if (!nested_cr3_valid(vcpu, cr3)) { 916 *entry_failure_code = ENTRY_FAIL_DEFAULT; 917 return 1; 918 } 919 920 /* 921 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 922 * must not be dereferenced. 923 */ 924 if (!is_long_mode(vcpu) && is_pae(vcpu) && is_paging(vcpu) && 925 !nested_ept) { 926 if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) { 927 *entry_failure_code = ENTRY_FAIL_PDPTE; 928 return 1; 929 } 930 } 931 } 932 933 if (!nested_ept) 934 kvm_mmu_new_cr3(vcpu, cr3, false); 935 936 vcpu->arch.cr3 = cr3; 937 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 938 939 kvm_init_mmu(vcpu, false); 940 941 return 0; 942 } 943 944 /* 945 * Returns if KVM is able to config CPU to tag TLB entries 946 * populated by L2 differently than TLB entries populated 947 * by L1. 948 * 949 * If L1 uses EPT, then TLB entries are tagged with different EPTP. 950 * 951 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 952 * with different VPID (L1 entries are tagged with vmx->vpid 953 * while L2 entries are tagged with vmx->nested.vpid02). 954 */ 955 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 956 { 957 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 958 959 return nested_cpu_has_ept(vmcs12) || 960 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 961 } 962 963 static u16 nested_get_vpid02(struct kvm_vcpu *vcpu) 964 { 965 struct vcpu_vmx *vmx = to_vmx(vcpu); 966 967 return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid; 968 } 969 970 971 static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 972 { 973 return fixed_bits_valid(control, low, high); 974 } 975 976 static inline u64 vmx_control_msr(u32 low, u32 high) 977 { 978 return low | ((u64)high << 32); 979 } 980 981 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 982 { 983 superset &= mask; 984 subset &= mask; 985 986 return (superset | subset) == superset; 987 } 988 989 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 990 { 991 const u64 feature_and_reserved = 992 /* feature (except bit 48; see below) */ 993 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) | 994 /* reserved */ 995 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56); 996 u64 vmx_basic = vmx->nested.msrs.basic; 997 998 if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved)) 999 return -EINVAL; 1000 1001 /* 1002 * KVM does not emulate a version of VMX that constrains physical 1003 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1004 */ 1005 if (data & BIT_ULL(48)) 1006 return -EINVAL; 1007 1008 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1009 vmx_basic_vmcs_revision_id(data)) 1010 return -EINVAL; 1011 1012 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1013 return -EINVAL; 1014 1015 vmx->nested.msrs.basic = data; 1016 return 0; 1017 } 1018 1019 static int 1020 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1021 { 1022 u64 supported; 1023 u32 *lowp, *highp; 1024 1025 switch (msr_index) { 1026 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1027 lowp = &vmx->nested.msrs.pinbased_ctls_low; 1028 highp = &vmx->nested.msrs.pinbased_ctls_high; 1029 break; 1030 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1031 lowp = &vmx->nested.msrs.procbased_ctls_low; 1032 highp = &vmx->nested.msrs.procbased_ctls_high; 1033 break; 1034 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1035 lowp = &vmx->nested.msrs.exit_ctls_low; 1036 highp = &vmx->nested.msrs.exit_ctls_high; 1037 break; 1038 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1039 lowp = &vmx->nested.msrs.entry_ctls_low; 1040 highp = &vmx->nested.msrs.entry_ctls_high; 1041 break; 1042 case MSR_IA32_VMX_PROCBASED_CTLS2: 1043 lowp = &vmx->nested.msrs.secondary_ctls_low; 1044 highp = &vmx->nested.msrs.secondary_ctls_high; 1045 break; 1046 default: 1047 BUG(); 1048 } 1049 1050 supported = vmx_control_msr(*lowp, *highp); 1051 1052 /* Check must-be-1 bits are still 1. */ 1053 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1054 return -EINVAL; 1055 1056 /* Check must-be-0 bits are still 0. */ 1057 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1058 return -EINVAL; 1059 1060 *lowp = data; 1061 *highp = data >> 32; 1062 return 0; 1063 } 1064 1065 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1066 { 1067 const u64 feature_and_reserved_bits = 1068 /* feature */ 1069 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) | 1070 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) | 1071 /* reserved */ 1072 GENMASK_ULL(13, 9) | BIT_ULL(31); 1073 u64 vmx_misc; 1074 1075 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 1076 vmx->nested.msrs.misc_high); 1077 1078 if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits)) 1079 return -EINVAL; 1080 1081 if ((vmx->nested.msrs.pinbased_ctls_high & 1082 PIN_BASED_VMX_PREEMPTION_TIMER) && 1083 vmx_misc_preemption_timer_rate(data) != 1084 vmx_misc_preemption_timer_rate(vmx_misc)) 1085 return -EINVAL; 1086 1087 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1088 return -EINVAL; 1089 1090 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1091 return -EINVAL; 1092 1093 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1094 return -EINVAL; 1095 1096 vmx->nested.msrs.misc_low = data; 1097 vmx->nested.msrs.misc_high = data >> 32; 1098 1099 /* 1100 * If L1 has read-only VM-exit information fields, use the 1101 * less permissive vmx_vmwrite_bitmap to specify write 1102 * permissions for the shadow VMCS. 1103 */ 1104 if (enable_shadow_vmcs && !nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1105 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 1106 1107 return 0; 1108 } 1109 1110 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1111 { 1112 u64 vmx_ept_vpid_cap; 1113 1114 vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps, 1115 vmx->nested.msrs.vpid_caps); 1116 1117 /* Every bit is either reserved or a feature bit. */ 1118 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1119 return -EINVAL; 1120 1121 vmx->nested.msrs.ept_caps = data; 1122 vmx->nested.msrs.vpid_caps = data >> 32; 1123 return 0; 1124 } 1125 1126 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1127 { 1128 u64 *msr; 1129 1130 switch (msr_index) { 1131 case MSR_IA32_VMX_CR0_FIXED0: 1132 msr = &vmx->nested.msrs.cr0_fixed0; 1133 break; 1134 case MSR_IA32_VMX_CR4_FIXED0: 1135 msr = &vmx->nested.msrs.cr4_fixed0; 1136 break; 1137 default: 1138 BUG(); 1139 } 1140 1141 /* 1142 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1143 * must be 1 in the restored value. 1144 */ 1145 if (!is_bitwise_subset(data, *msr, -1ULL)) 1146 return -EINVAL; 1147 1148 *msr = data; 1149 return 0; 1150 } 1151 1152 /* 1153 * Called when userspace is restoring VMX MSRs. 1154 * 1155 * Returns 0 on success, non-0 otherwise. 1156 */ 1157 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1158 { 1159 struct vcpu_vmx *vmx = to_vmx(vcpu); 1160 1161 /* 1162 * Don't allow changes to the VMX capability MSRs while the vCPU 1163 * is in VMX operation. 1164 */ 1165 if (vmx->nested.vmxon) 1166 return -EBUSY; 1167 1168 switch (msr_index) { 1169 case MSR_IA32_VMX_BASIC: 1170 return vmx_restore_vmx_basic(vmx, data); 1171 case MSR_IA32_VMX_PINBASED_CTLS: 1172 case MSR_IA32_VMX_PROCBASED_CTLS: 1173 case MSR_IA32_VMX_EXIT_CTLS: 1174 case MSR_IA32_VMX_ENTRY_CTLS: 1175 /* 1176 * The "non-true" VMX capability MSRs are generated from the 1177 * "true" MSRs, so we do not support restoring them directly. 1178 * 1179 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1180 * should restore the "true" MSRs with the must-be-1 bits 1181 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1182 * DEFAULT SETTINGS". 1183 */ 1184 return -EINVAL; 1185 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1186 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1187 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1188 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1189 case MSR_IA32_VMX_PROCBASED_CTLS2: 1190 return vmx_restore_control_msr(vmx, msr_index, data); 1191 case MSR_IA32_VMX_MISC: 1192 return vmx_restore_vmx_misc(vmx, data); 1193 case MSR_IA32_VMX_CR0_FIXED0: 1194 case MSR_IA32_VMX_CR4_FIXED0: 1195 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1196 case MSR_IA32_VMX_CR0_FIXED1: 1197 case MSR_IA32_VMX_CR4_FIXED1: 1198 /* 1199 * These MSRs are generated based on the vCPU's CPUID, so we 1200 * do not support restoring them directly. 1201 */ 1202 return -EINVAL; 1203 case MSR_IA32_VMX_EPT_VPID_CAP: 1204 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1205 case MSR_IA32_VMX_VMCS_ENUM: 1206 vmx->nested.msrs.vmcs_enum = data; 1207 return 0; 1208 default: 1209 /* 1210 * The rest of the VMX capability MSRs do not support restore. 1211 */ 1212 return -EINVAL; 1213 } 1214 } 1215 1216 /* Returns 0 on success, non-0 otherwise. */ 1217 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1218 { 1219 switch (msr_index) { 1220 case MSR_IA32_VMX_BASIC: 1221 *pdata = msrs->basic; 1222 break; 1223 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1224 case MSR_IA32_VMX_PINBASED_CTLS: 1225 *pdata = vmx_control_msr( 1226 msrs->pinbased_ctls_low, 1227 msrs->pinbased_ctls_high); 1228 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1229 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1230 break; 1231 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1232 case MSR_IA32_VMX_PROCBASED_CTLS: 1233 *pdata = vmx_control_msr( 1234 msrs->procbased_ctls_low, 1235 msrs->procbased_ctls_high); 1236 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1237 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1238 break; 1239 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1240 case MSR_IA32_VMX_EXIT_CTLS: 1241 *pdata = vmx_control_msr( 1242 msrs->exit_ctls_low, 1243 msrs->exit_ctls_high); 1244 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1245 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1246 break; 1247 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1248 case MSR_IA32_VMX_ENTRY_CTLS: 1249 *pdata = vmx_control_msr( 1250 msrs->entry_ctls_low, 1251 msrs->entry_ctls_high); 1252 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1253 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1254 break; 1255 case MSR_IA32_VMX_MISC: 1256 *pdata = vmx_control_msr( 1257 msrs->misc_low, 1258 msrs->misc_high); 1259 break; 1260 case MSR_IA32_VMX_CR0_FIXED0: 1261 *pdata = msrs->cr0_fixed0; 1262 break; 1263 case MSR_IA32_VMX_CR0_FIXED1: 1264 *pdata = msrs->cr0_fixed1; 1265 break; 1266 case MSR_IA32_VMX_CR4_FIXED0: 1267 *pdata = msrs->cr4_fixed0; 1268 break; 1269 case MSR_IA32_VMX_CR4_FIXED1: 1270 *pdata = msrs->cr4_fixed1; 1271 break; 1272 case MSR_IA32_VMX_VMCS_ENUM: 1273 *pdata = msrs->vmcs_enum; 1274 break; 1275 case MSR_IA32_VMX_PROCBASED_CTLS2: 1276 *pdata = vmx_control_msr( 1277 msrs->secondary_ctls_low, 1278 msrs->secondary_ctls_high); 1279 break; 1280 case MSR_IA32_VMX_EPT_VPID_CAP: 1281 *pdata = msrs->ept_caps | 1282 ((u64)msrs->vpid_caps << 32); 1283 break; 1284 case MSR_IA32_VMX_VMFUNC: 1285 *pdata = msrs->vmfunc_controls; 1286 break; 1287 default: 1288 return 1; 1289 } 1290 1291 return 0; 1292 } 1293 1294 /* 1295 * Copy the writable VMCS shadow fields back to the VMCS12, in case 1296 * they have been modified by the L1 guest. Note that the "read-only" 1297 * VM-exit information fields are actually writable if the vCPU is 1298 * configured to support "VMWRITE to any supported field in the VMCS." 1299 */ 1300 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1301 { 1302 const u16 *fields[] = { 1303 shadow_read_write_fields, 1304 shadow_read_only_fields 1305 }; 1306 const int max_fields[] = { 1307 max_shadow_read_write_fields, 1308 max_shadow_read_only_fields 1309 }; 1310 int i, q; 1311 unsigned long field; 1312 u64 field_value; 1313 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1314 1315 preempt_disable(); 1316 1317 vmcs_load(shadow_vmcs); 1318 1319 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1320 for (i = 0; i < max_fields[q]; i++) { 1321 field = fields[q][i]; 1322 field_value = __vmcs_readl(field); 1323 vmcs12_write_any(get_vmcs12(&vmx->vcpu), field, field_value); 1324 } 1325 /* 1326 * Skip the VM-exit information fields if they are read-only. 1327 */ 1328 if (!nested_cpu_has_vmwrite_any_field(&vmx->vcpu)) 1329 break; 1330 } 1331 1332 vmcs_clear(shadow_vmcs); 1333 vmcs_load(vmx->loaded_vmcs->vmcs); 1334 1335 preempt_enable(); 1336 } 1337 1338 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1339 { 1340 const u16 *fields[] = { 1341 shadow_read_write_fields, 1342 shadow_read_only_fields 1343 }; 1344 const int max_fields[] = { 1345 max_shadow_read_write_fields, 1346 max_shadow_read_only_fields 1347 }; 1348 int i, q; 1349 unsigned long field; 1350 u64 field_value = 0; 1351 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1352 1353 vmcs_load(shadow_vmcs); 1354 1355 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1356 for (i = 0; i < max_fields[q]; i++) { 1357 field = fields[q][i]; 1358 vmcs12_read_any(get_vmcs12(&vmx->vcpu), field, &field_value); 1359 __vmcs_writel(field, field_value); 1360 } 1361 } 1362 1363 vmcs_clear(shadow_vmcs); 1364 vmcs_load(vmx->loaded_vmcs->vmcs); 1365 } 1366 1367 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) 1368 { 1369 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1370 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1371 1372 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1373 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1374 vmcs12->guest_rip = evmcs->guest_rip; 1375 1376 if (unlikely(!(evmcs->hv_clean_fields & 1377 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1378 vmcs12->guest_rsp = evmcs->guest_rsp; 1379 vmcs12->guest_rflags = evmcs->guest_rflags; 1380 vmcs12->guest_interruptibility_info = 1381 evmcs->guest_interruptibility_info; 1382 } 1383 1384 if (unlikely(!(evmcs->hv_clean_fields & 1385 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1386 vmcs12->cpu_based_vm_exec_control = 1387 evmcs->cpu_based_vm_exec_control; 1388 } 1389 1390 if (unlikely(!(evmcs->hv_clean_fields & 1391 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1392 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1393 } 1394 1395 if (unlikely(!(evmcs->hv_clean_fields & 1396 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1397 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1398 } 1399 1400 if (unlikely(!(evmcs->hv_clean_fields & 1401 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1402 vmcs12->vm_entry_intr_info_field = 1403 evmcs->vm_entry_intr_info_field; 1404 vmcs12->vm_entry_exception_error_code = 1405 evmcs->vm_entry_exception_error_code; 1406 vmcs12->vm_entry_instruction_len = 1407 evmcs->vm_entry_instruction_len; 1408 } 1409 1410 if (unlikely(!(evmcs->hv_clean_fields & 1411 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1412 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1413 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1414 vmcs12->host_cr0 = evmcs->host_cr0; 1415 vmcs12->host_cr3 = evmcs->host_cr3; 1416 vmcs12->host_cr4 = evmcs->host_cr4; 1417 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1418 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1419 vmcs12->host_rip = evmcs->host_rip; 1420 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1421 vmcs12->host_es_selector = evmcs->host_es_selector; 1422 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1423 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1424 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1425 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1426 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1427 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1428 } 1429 1430 if (unlikely(!(evmcs->hv_clean_fields & 1431 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1432 vmcs12->pin_based_vm_exec_control = 1433 evmcs->pin_based_vm_exec_control; 1434 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1435 vmcs12->secondary_vm_exec_control = 1436 evmcs->secondary_vm_exec_control; 1437 } 1438 1439 if (unlikely(!(evmcs->hv_clean_fields & 1440 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1441 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1442 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1443 } 1444 1445 if (unlikely(!(evmcs->hv_clean_fields & 1446 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1447 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1448 } 1449 1450 if (unlikely(!(evmcs->hv_clean_fields & 1451 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1452 vmcs12->guest_es_base = evmcs->guest_es_base; 1453 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1454 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1455 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1456 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1457 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1458 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1459 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1460 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1461 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1462 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1463 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1464 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1465 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1466 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1467 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1468 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1469 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1470 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1471 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1472 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1473 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1474 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1475 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1476 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1477 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1478 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1479 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1480 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1481 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1482 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1483 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1484 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1485 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1486 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1487 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1488 } 1489 1490 if (unlikely(!(evmcs->hv_clean_fields & 1491 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1492 vmcs12->tsc_offset = evmcs->tsc_offset; 1493 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1494 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1495 } 1496 1497 if (unlikely(!(evmcs->hv_clean_fields & 1498 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1499 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1500 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1501 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1502 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1503 vmcs12->guest_cr0 = evmcs->guest_cr0; 1504 vmcs12->guest_cr3 = evmcs->guest_cr3; 1505 vmcs12->guest_cr4 = evmcs->guest_cr4; 1506 vmcs12->guest_dr7 = evmcs->guest_dr7; 1507 } 1508 1509 if (unlikely(!(evmcs->hv_clean_fields & 1510 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1511 vmcs12->host_fs_base = evmcs->host_fs_base; 1512 vmcs12->host_gs_base = evmcs->host_gs_base; 1513 vmcs12->host_tr_base = evmcs->host_tr_base; 1514 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1515 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1516 vmcs12->host_rsp = evmcs->host_rsp; 1517 } 1518 1519 if (unlikely(!(evmcs->hv_clean_fields & 1520 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1521 vmcs12->ept_pointer = evmcs->ept_pointer; 1522 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1523 } 1524 1525 if (unlikely(!(evmcs->hv_clean_fields & 1526 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1527 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1528 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1529 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1530 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1531 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1532 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1533 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1534 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1535 vmcs12->guest_pending_dbg_exceptions = 1536 evmcs->guest_pending_dbg_exceptions; 1537 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1538 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1539 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1540 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1541 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1542 } 1543 1544 /* 1545 * Not used? 1546 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1547 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1548 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1549 * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0; 1550 * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1; 1551 * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2; 1552 * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3; 1553 * vmcs12->page_fault_error_code_mask = 1554 * evmcs->page_fault_error_code_mask; 1555 * vmcs12->page_fault_error_code_match = 1556 * evmcs->page_fault_error_code_match; 1557 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1558 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1559 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1560 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1561 */ 1562 1563 /* 1564 * Read only fields: 1565 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1566 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1567 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1568 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1569 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1570 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1571 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1572 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1573 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1574 * vmcs12->exit_qualification = evmcs->exit_qualification; 1575 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1576 * 1577 * Not present in struct vmcs12: 1578 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1579 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1580 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1581 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1582 */ 1583 1584 return 0; 1585 } 1586 1587 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1588 { 1589 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1590 struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; 1591 1592 /* 1593 * Should not be changed by KVM: 1594 * 1595 * evmcs->host_es_selector = vmcs12->host_es_selector; 1596 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1597 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1598 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1599 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1600 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1601 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1602 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1603 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1604 * evmcs->host_cr0 = vmcs12->host_cr0; 1605 * evmcs->host_cr3 = vmcs12->host_cr3; 1606 * evmcs->host_cr4 = vmcs12->host_cr4; 1607 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1608 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1609 * evmcs->host_rip = vmcs12->host_rip; 1610 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1611 * evmcs->host_fs_base = vmcs12->host_fs_base; 1612 * evmcs->host_gs_base = vmcs12->host_gs_base; 1613 * evmcs->host_tr_base = vmcs12->host_tr_base; 1614 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1615 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1616 * evmcs->host_rsp = vmcs12->host_rsp; 1617 * sync_vmcs12() doesn't read these: 1618 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1619 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1620 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1621 * evmcs->ept_pointer = vmcs12->ept_pointer; 1622 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1623 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1624 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1625 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1626 * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0; 1627 * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1; 1628 * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2; 1629 * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3; 1630 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1631 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1632 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1633 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1634 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1635 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1636 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1637 * evmcs->page_fault_error_code_mask = 1638 * vmcs12->page_fault_error_code_mask; 1639 * evmcs->page_fault_error_code_match = 1640 * vmcs12->page_fault_error_code_match; 1641 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1642 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1643 * evmcs->tsc_offset = vmcs12->tsc_offset; 1644 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1645 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1646 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1647 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1648 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 1649 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 1650 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 1651 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 1652 * 1653 * Not present in struct vmcs12: 1654 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 1655 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 1656 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 1657 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 1658 */ 1659 1660 evmcs->guest_es_selector = vmcs12->guest_es_selector; 1661 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 1662 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 1663 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 1664 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 1665 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 1666 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 1667 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 1668 1669 evmcs->guest_es_limit = vmcs12->guest_es_limit; 1670 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 1671 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 1672 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 1673 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 1674 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 1675 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 1676 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 1677 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 1678 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 1679 1680 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 1681 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 1682 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 1683 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 1684 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 1685 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 1686 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 1687 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 1688 1689 evmcs->guest_es_base = vmcs12->guest_es_base; 1690 evmcs->guest_cs_base = vmcs12->guest_cs_base; 1691 evmcs->guest_ss_base = vmcs12->guest_ss_base; 1692 evmcs->guest_ds_base = vmcs12->guest_ds_base; 1693 evmcs->guest_fs_base = vmcs12->guest_fs_base; 1694 evmcs->guest_gs_base = vmcs12->guest_gs_base; 1695 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 1696 evmcs->guest_tr_base = vmcs12->guest_tr_base; 1697 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 1698 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 1699 1700 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 1701 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 1702 1703 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 1704 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 1705 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 1706 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 1707 1708 evmcs->guest_pending_dbg_exceptions = 1709 vmcs12->guest_pending_dbg_exceptions; 1710 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 1711 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 1712 1713 evmcs->guest_activity_state = vmcs12->guest_activity_state; 1714 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 1715 1716 evmcs->guest_cr0 = vmcs12->guest_cr0; 1717 evmcs->guest_cr3 = vmcs12->guest_cr3; 1718 evmcs->guest_cr4 = vmcs12->guest_cr4; 1719 evmcs->guest_dr7 = vmcs12->guest_dr7; 1720 1721 evmcs->guest_physical_address = vmcs12->guest_physical_address; 1722 1723 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 1724 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 1725 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 1726 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 1727 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 1728 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 1729 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 1730 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 1731 1732 evmcs->exit_qualification = vmcs12->exit_qualification; 1733 1734 evmcs->guest_linear_address = vmcs12->guest_linear_address; 1735 evmcs->guest_rsp = vmcs12->guest_rsp; 1736 evmcs->guest_rflags = vmcs12->guest_rflags; 1737 1738 evmcs->guest_interruptibility_info = 1739 vmcs12->guest_interruptibility_info; 1740 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 1741 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 1742 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 1743 evmcs->vm_entry_exception_error_code = 1744 vmcs12->vm_entry_exception_error_code; 1745 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 1746 1747 evmcs->guest_rip = vmcs12->guest_rip; 1748 1749 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 1750 1751 return 0; 1752 } 1753 1754 /* 1755 * This is an equivalent of the nested hypervisor executing the vmptrld 1756 * instruction. 1757 */ 1758 static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu, 1759 bool from_launch) 1760 { 1761 struct vcpu_vmx *vmx = to_vmx(vcpu); 1762 struct hv_vp_assist_page assist_page; 1763 1764 if (likely(!vmx->nested.enlightened_vmcs_enabled)) 1765 return 1; 1766 1767 if (unlikely(!kvm_hv_get_assist_page(vcpu, &assist_page))) 1768 return 1; 1769 1770 if (unlikely(!assist_page.enlighten_vmentry)) 1771 return 1; 1772 1773 if (unlikely(assist_page.current_nested_vmcs != 1774 vmx->nested.hv_evmcs_vmptr)) { 1775 1776 if (!vmx->nested.hv_evmcs) 1777 vmx->nested.current_vmptr = -1ull; 1778 1779 nested_release_evmcs(vcpu); 1780 1781 vmx->nested.hv_evmcs_page = kvm_vcpu_gpa_to_page( 1782 vcpu, assist_page.current_nested_vmcs); 1783 1784 if (unlikely(is_error_page(vmx->nested.hv_evmcs_page))) 1785 return 0; 1786 1787 vmx->nested.hv_evmcs = kmap(vmx->nested.hv_evmcs_page); 1788 1789 /* 1790 * Currently, KVM only supports eVMCS version 1 1791 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 1792 * value to first u32 field of eVMCS which should specify eVMCS 1793 * VersionNumber. 1794 * 1795 * Guest should be aware of supported eVMCS versions by host by 1796 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 1797 * expected to set this CPUID leaf according to the value 1798 * returned in vmcs_version from nested_enable_evmcs(). 1799 * 1800 * However, it turns out that Microsoft Hyper-V fails to comply 1801 * to their own invented interface: When Hyper-V use eVMCS, it 1802 * just sets first u32 field of eVMCS to revision_id specified 1803 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 1804 * which is one of the supported versions specified in 1805 * CPUID.0x4000000A.EAX[0:15]. 1806 * 1807 * To overcome Hyper-V bug, we accept here either a supported 1808 * eVMCS version or VMCS12 revision_id as valid values for first 1809 * u32 field of eVMCS. 1810 */ 1811 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 1812 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 1813 nested_release_evmcs(vcpu); 1814 return 0; 1815 } 1816 1817 vmx->nested.dirty_vmcs12 = true; 1818 /* 1819 * As we keep L2 state for one guest only 'hv_clean_fields' mask 1820 * can't be used when we switch between them. Reset it here for 1821 * simplicity. 1822 */ 1823 vmx->nested.hv_evmcs->hv_clean_fields &= 1824 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1825 vmx->nested.hv_evmcs_vmptr = assist_page.current_nested_vmcs; 1826 1827 /* 1828 * Unlike normal vmcs12, enlightened vmcs12 is not fully 1829 * reloaded from guest's memory (read only fields, fields not 1830 * present in struct hv_enlightened_vmcs, ...). Make sure there 1831 * are no leftovers. 1832 */ 1833 if (from_launch) { 1834 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1835 memset(vmcs12, 0, sizeof(*vmcs12)); 1836 vmcs12->hdr.revision_id = VMCS12_REVISION; 1837 } 1838 1839 } 1840 return 1; 1841 } 1842 1843 void nested_sync_from_vmcs12(struct kvm_vcpu *vcpu) 1844 { 1845 struct vcpu_vmx *vmx = to_vmx(vcpu); 1846 1847 /* 1848 * hv_evmcs may end up being not mapped after migration (when 1849 * L2 was running), map it here to make sure vmcs12 changes are 1850 * properly reflected. 1851 */ 1852 if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) 1853 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 1854 1855 if (vmx->nested.hv_evmcs) { 1856 copy_vmcs12_to_enlightened(vmx); 1857 /* All fields are clean */ 1858 vmx->nested.hv_evmcs->hv_clean_fields |= 1859 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 1860 } else { 1861 copy_vmcs12_to_shadow(vmx); 1862 } 1863 1864 vmx->nested.need_vmcs12_sync = false; 1865 } 1866 1867 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 1868 { 1869 struct vcpu_vmx *vmx = 1870 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 1871 1872 vmx->nested.preemption_timer_expired = true; 1873 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 1874 kvm_vcpu_kick(&vmx->vcpu); 1875 1876 return HRTIMER_NORESTART; 1877 } 1878 1879 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) 1880 { 1881 u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value; 1882 struct vcpu_vmx *vmx = to_vmx(vcpu); 1883 1884 /* 1885 * A timer value of zero is architecturally guaranteed to cause 1886 * a VMExit prior to executing any instructions in the guest. 1887 */ 1888 if (preemption_timeout == 0) { 1889 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 1890 return; 1891 } 1892 1893 if (vcpu->arch.virtual_tsc_khz == 0) 1894 return; 1895 1896 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 1897 preemption_timeout *= 1000000; 1898 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 1899 hrtimer_start(&vmx->nested.preemption_timer, 1900 ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL); 1901 } 1902 1903 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1904 { 1905 if (vmx->nested.nested_run_pending && 1906 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 1907 return vmcs12->guest_ia32_efer; 1908 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 1909 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 1910 else 1911 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 1912 } 1913 1914 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 1915 { 1916 /* 1917 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 1918 * according to L0's settings (vmcs12 is irrelevant here). Host 1919 * fields that come from L0 and are not constant, e.g. HOST_CR3, 1920 * will be set as needed prior to VMLAUNCH/VMRESUME. 1921 */ 1922 if (vmx->nested.vmcs02_initialized) 1923 return; 1924 vmx->nested.vmcs02_initialized = true; 1925 1926 /* 1927 * We don't care what the EPTP value is we just need to guarantee 1928 * it's valid so we don't get a false positive when doing early 1929 * consistency checks. 1930 */ 1931 if (enable_ept && nested_early_check) 1932 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0)); 1933 1934 /* All VMFUNCs are currently emulated through L0 vmexits. */ 1935 if (cpu_has_vmx_vmfunc()) 1936 vmcs_write64(VM_FUNCTION_CONTROL, 0); 1937 1938 if (cpu_has_vmx_posted_intr()) 1939 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 1940 1941 if (cpu_has_vmx_msr_bitmap()) 1942 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 1943 1944 if (enable_pml) 1945 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); 1946 1947 /* 1948 * Set the MSR load/store lists to match L0's settings. Only the 1949 * addresses are constant (for vmcs02), the counts can change based 1950 * on L2's behavior, e.g. switching to/from long mode. 1951 */ 1952 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); 1953 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 1954 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 1955 1956 vmx_set_constant_host_state(vmx); 1957 } 1958 1959 static void prepare_vmcs02_early_full(struct vcpu_vmx *vmx, 1960 struct vmcs12 *vmcs12) 1961 { 1962 prepare_vmcs02_constant_state(vmx); 1963 1964 vmcs_write64(VMCS_LINK_POINTER, -1ull); 1965 1966 if (enable_vpid) { 1967 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 1968 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 1969 else 1970 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 1971 } 1972 } 1973 1974 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 1975 { 1976 u32 exec_control, vmcs12_exec_ctrl; 1977 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 1978 1979 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) 1980 prepare_vmcs02_early_full(vmx, vmcs12); 1981 1982 /* 1983 * PIN CONTROLS 1984 */ 1985 exec_control = vmcs12->pin_based_vm_exec_control; 1986 1987 /* Preemption timer setting is computed directly in vmx_vcpu_run. */ 1988 exec_control |= vmcs_config.pin_based_exec_ctrl; 1989 exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER; 1990 vmx->loaded_vmcs->hv_timer_armed = false; 1991 1992 /* Posted interrupts setting is only taken from vmcs12. */ 1993 if (nested_cpu_has_posted_intr(vmcs12)) { 1994 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 1995 vmx->nested.pi_pending = false; 1996 } else { 1997 exec_control &= ~PIN_BASED_POSTED_INTR; 1998 } 1999 vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control); 2000 2001 /* 2002 * EXEC CONTROLS 2003 */ 2004 exec_control = vmx_exec_control(vmx); /* L0's desires */ 2005 exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; 2006 exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING; 2007 exec_control &= ~CPU_BASED_TPR_SHADOW; 2008 exec_control |= vmcs12->cpu_based_vm_exec_control; 2009 2010 /* 2011 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR. Later, if 2012 * nested_get_vmcs12_pages can't fix it up, the illegal value 2013 * will result in a VM entry failure. 2014 */ 2015 if (exec_control & CPU_BASED_TPR_SHADOW) { 2016 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull); 2017 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2018 } else { 2019 #ifdef CONFIG_X86_64 2020 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2021 CPU_BASED_CR8_STORE_EXITING; 2022 #endif 2023 } 2024 2025 /* 2026 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2027 * for I/O port accesses. 2028 */ 2029 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2030 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2031 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); 2032 2033 /* 2034 * SECONDARY EXEC CONTROLS 2035 */ 2036 if (cpu_has_secondary_exec_ctrls()) { 2037 exec_control = vmx->secondary_exec_control; 2038 2039 /* Take the following fields only from vmcs12 */ 2040 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2041 SECONDARY_EXEC_ENABLE_INVPCID | 2042 SECONDARY_EXEC_RDTSCP | 2043 SECONDARY_EXEC_XSAVES | 2044 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2045 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2046 SECONDARY_EXEC_ENABLE_VMFUNC); 2047 if (nested_cpu_has(vmcs12, 2048 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { 2049 vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control & 2050 ~SECONDARY_EXEC_ENABLE_PML; 2051 exec_control |= vmcs12_exec_ctrl; 2052 } 2053 2054 /* VMCS shadowing for L2 is emulated for now */ 2055 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2056 2057 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2058 vmcs_write16(GUEST_INTR_STATUS, 2059 vmcs12->guest_intr_status); 2060 2061 /* 2062 * Write an illegal value to APIC_ACCESS_ADDR. Later, 2063 * nested_get_vmcs12_pages will either fix it up or 2064 * remove the VM execution control. 2065 */ 2066 if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) 2067 vmcs_write64(APIC_ACCESS_ADDR, -1ull); 2068 2069 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2070 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull); 2071 2072 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); 2073 } 2074 2075 /* 2076 * ENTRY CONTROLS 2077 * 2078 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2079 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2080 * on the related bits (if supported by the CPU) in the hope that 2081 * we can avoid VMWrites during vmx_set_efer(). 2082 */ 2083 exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) & 2084 ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER; 2085 if (cpu_has_load_ia32_efer()) { 2086 if (guest_efer & EFER_LMA) 2087 exec_control |= VM_ENTRY_IA32E_MODE; 2088 if (guest_efer != host_efer) 2089 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2090 } 2091 vm_entry_controls_init(vmx, exec_control); 2092 2093 /* 2094 * EXIT CONTROLS 2095 * 2096 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2097 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2098 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2099 */ 2100 exec_control = vmx_vmexit_ctrl(); 2101 if (cpu_has_load_ia32_efer() && guest_efer != host_efer) 2102 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2103 vm_exit_controls_init(vmx, exec_control); 2104 2105 /* 2106 * Conceptually we want to copy the PML address and index from 2107 * vmcs01 here, and then back to vmcs01 on nested vmexit. But, 2108 * since we always flush the log on each vmexit and never change 2109 * the PML address (once set), this happens to be equivalent to 2110 * simply resetting the index in vmcs02. 2111 */ 2112 if (enable_pml) 2113 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); 2114 2115 /* 2116 * Interrupt/Exception Fields 2117 */ 2118 if (vmx->nested.nested_run_pending) { 2119 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2120 vmcs12->vm_entry_intr_info_field); 2121 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2122 vmcs12->vm_entry_exception_error_code); 2123 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2124 vmcs12->vm_entry_instruction_len); 2125 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2126 vmcs12->guest_interruptibility_info); 2127 vmx->loaded_vmcs->nmi_known_unmasked = 2128 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2129 } else { 2130 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2131 } 2132 } 2133 2134 static void prepare_vmcs02_full(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2135 { 2136 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2137 2138 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2139 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2140 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2141 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2142 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2143 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2144 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2145 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2146 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2147 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2148 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2149 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2150 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2151 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2152 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2153 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2154 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2155 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2156 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2157 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2158 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2159 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2160 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2161 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2162 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2163 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2164 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2165 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2166 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2167 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2168 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2169 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2170 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2171 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2172 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2173 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2174 } 2175 2176 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2177 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2178 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2179 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2180 vmcs12->guest_pending_dbg_exceptions); 2181 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2182 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2183 2184 /* 2185 * L1 may access the L2's PDPTR, so save them to construct 2186 * vmcs12 2187 */ 2188 if (enable_ept) { 2189 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2190 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2191 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2192 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2193 } 2194 } 2195 2196 if (nested_cpu_has_xsaves(vmcs12)) 2197 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2198 2199 /* 2200 * Whether page-faults are trapped is determined by a combination of 2201 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. 2202 * If enable_ept, L0 doesn't care about page faults and we should 2203 * set all of these to L1's desires. However, if !enable_ept, L0 does 2204 * care about (at least some) page faults, and because it is not easy 2205 * (if at all possible?) to merge L0 and L1's desires, we simply ask 2206 * to exit on each and every L2 page fault. This is done by setting 2207 * MASK=MATCH=0 and (see below) EB.PF=1. 2208 * Note that below we don't need special code to set EB.PF beyond the 2209 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2210 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2211 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2212 */ 2213 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 2214 enable_ept ? vmcs12->page_fault_error_code_mask : 0); 2215 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 2216 enable_ept ? vmcs12->page_fault_error_code_match : 0); 2217 2218 if (cpu_has_vmx_apicv()) { 2219 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2220 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2221 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2222 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2223 } 2224 2225 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2226 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2227 2228 set_cr4_guest_host_mask(vmx); 2229 2230 if (kvm_mpx_supported()) { 2231 if (vmx->nested.nested_run_pending && 2232 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2233 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2234 else 2235 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs); 2236 } 2237 } 2238 2239 /* 2240 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2241 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2242 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2243 * guest in a way that will both be appropriate to L1's requests, and our 2244 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2245 * function also has additional necessary side-effects, like setting various 2246 * vcpu->arch fields. 2247 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2248 * is assigned to entry_failure_code on failure. 2249 */ 2250 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2251 u32 *entry_failure_code) 2252 { 2253 struct vcpu_vmx *vmx = to_vmx(vcpu); 2254 struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; 2255 2256 if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) { 2257 prepare_vmcs02_full(vmx, vmcs12); 2258 vmx->nested.dirty_vmcs12 = false; 2259 } 2260 2261 /* 2262 * First, the fields that are shadowed. This must be kept in sync 2263 * with vmcs_shadow_fields.h. 2264 */ 2265 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2266 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2267 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2268 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2269 } 2270 2271 if (vmx->nested.nested_run_pending && 2272 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2273 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2274 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl); 2275 } else { 2276 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2277 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl); 2278 } 2279 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2280 2281 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2282 * bitwise-or of what L1 wants to trap for L2, and what we want to 2283 * trap. Note that CR0.TS also needs updating - we do this later. 2284 */ 2285 update_exception_bitmap(vcpu); 2286 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2287 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2288 2289 if (vmx->nested.nested_run_pending && 2290 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2291 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2292 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2293 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2294 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); 2295 } 2296 2297 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2298 2299 if (kvm_has_tsc_control) 2300 decache_tsc_multiplier(vmx); 2301 2302 if (enable_vpid) { 2303 /* 2304 * There is no direct mapping between vpid02 and vpid12, the 2305 * vpid02 is per-vCPU for L0 and reused while the value of 2306 * vpid12 is changed w/ one invvpid during nested vmentry. 2307 * The vpid12 is allocated by L1 for L2, so it will not 2308 * influence global bitmap(for vpid01 and vpid02 allocation) 2309 * even if spawn a lot of nested vCPUs. 2310 */ 2311 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) { 2312 if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 2313 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 2314 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false); 2315 } 2316 } else { 2317 /* 2318 * If L1 use EPT, then L0 needs to execute INVEPT on 2319 * EPTP02 instead of EPTP01. Therefore, delay TLB 2320 * flush until vmcs02->eptp is fully updated by 2321 * KVM_REQ_LOAD_CR3. Note that this assumes 2322 * KVM_REQ_TLB_FLUSH is evaluated after 2323 * KVM_REQ_LOAD_CR3 in vcpu_enter_guest(). 2324 */ 2325 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 2326 } 2327 } 2328 2329 if (nested_cpu_has_ept(vmcs12)) 2330 nested_ept_init_mmu_context(vcpu); 2331 else if (nested_cpu_has2(vmcs12, 2332 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) 2333 vmx_flush_tlb(vcpu, true); 2334 2335 /* 2336 * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those 2337 * bits which we consider mandatory enabled. 2338 * The CR0_READ_SHADOW is what L2 should have expected to read given 2339 * the specifications by L1; It's not enough to take 2340 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we 2341 * have more bits than L1 expected. 2342 */ 2343 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2344 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2345 2346 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2347 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2348 2349 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2350 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2351 vmx_set_efer(vcpu, vcpu->arch.efer); 2352 2353 /* 2354 * Guest state is invalid and unrestricted guest is disabled, 2355 * which means L1 attempted VMEntry to L2 with invalid state. 2356 * Fail the VMEntry. 2357 */ 2358 if (vmx->emulation_required) { 2359 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2360 return 1; 2361 } 2362 2363 /* Shadow page tables on either EPT or shadow page tables. */ 2364 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2365 entry_failure_code)) 2366 return 1; 2367 2368 if (!enable_ept) 2369 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; 2370 2371 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp); 2372 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip); 2373 return 0; 2374 } 2375 2376 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2377 { 2378 if (!nested_cpu_has_nmi_exiting(vmcs12) && 2379 nested_cpu_has_virtual_nmis(vmcs12)) 2380 return -EINVAL; 2381 2382 if (!nested_cpu_has_virtual_nmis(vmcs12) && 2383 nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING)) 2384 return -EINVAL; 2385 2386 return 0; 2387 } 2388 2389 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address) 2390 { 2391 struct vcpu_vmx *vmx = to_vmx(vcpu); 2392 int maxphyaddr = cpuid_maxphyaddr(vcpu); 2393 2394 /* Check for memory type validity */ 2395 switch (address & VMX_EPTP_MT_MASK) { 2396 case VMX_EPTP_MT_UC: 2397 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)) 2398 return false; 2399 break; 2400 case VMX_EPTP_MT_WB: 2401 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)) 2402 return false; 2403 break; 2404 default: 2405 return false; 2406 } 2407 2408 /* only 4 levels page-walk length are valid */ 2409 if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4) 2410 return false; 2411 2412 /* Reserved bits should not be set */ 2413 if (address >> maxphyaddr || ((address >> 7) & 0x1f)) 2414 return false; 2415 2416 /* AD, if set, should be supported */ 2417 if (address & VMX_EPTP_AD_ENABLE_BIT) { 2418 if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)) 2419 return false; 2420 } 2421 2422 return true; 2423 } 2424 2425 /* 2426 * Checks related to VM-Execution Control Fields 2427 */ 2428 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2429 struct vmcs12 *vmcs12) 2430 { 2431 struct vcpu_vmx *vmx = to_vmx(vcpu); 2432 2433 if (!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2434 vmx->nested.msrs.pinbased_ctls_low, 2435 vmx->nested.msrs.pinbased_ctls_high) || 2436 !vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2437 vmx->nested.msrs.procbased_ctls_low, 2438 vmx->nested.msrs.procbased_ctls_high)) 2439 return -EINVAL; 2440 2441 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2442 !vmx_control_verify(vmcs12->secondary_vm_exec_control, 2443 vmx->nested.msrs.secondary_ctls_low, 2444 vmx->nested.msrs.secondary_ctls_high)) 2445 return -EINVAL; 2446 2447 if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu) || 2448 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2449 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2450 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2451 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2452 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2453 nested_vmx_check_nmi_controls(vmcs12) || 2454 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2455 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2456 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2457 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2458 (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2459 return -EINVAL; 2460 2461 if (!nested_cpu_has_preemption_timer(vmcs12) && 2462 nested_cpu_has_save_preemption_timer(vmcs12)) 2463 return -EINVAL; 2464 2465 if (nested_cpu_has_ept(vmcs12) && 2466 !valid_ept_address(vcpu, vmcs12->ept_pointer)) 2467 return -EINVAL; 2468 2469 if (nested_cpu_has_vmfunc(vmcs12)) { 2470 if (vmcs12->vm_function_control & 2471 ~vmx->nested.msrs.vmfunc_controls) 2472 return -EINVAL; 2473 2474 if (nested_cpu_has_eptp_switching(vmcs12)) { 2475 if (!nested_cpu_has_ept(vmcs12) || 2476 !page_address_valid(vcpu, vmcs12->eptp_list_address)) 2477 return -EINVAL; 2478 } 2479 } 2480 2481 return 0; 2482 } 2483 2484 /* 2485 * Checks related to VM-Exit Control Fields 2486 */ 2487 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2488 struct vmcs12 *vmcs12) 2489 { 2490 struct vcpu_vmx *vmx = to_vmx(vcpu); 2491 2492 if (!vmx_control_verify(vmcs12->vm_exit_controls, 2493 vmx->nested.msrs.exit_ctls_low, 2494 vmx->nested.msrs.exit_ctls_high) || 2495 nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)) 2496 return -EINVAL; 2497 2498 return 0; 2499 } 2500 2501 /* 2502 * Checks related to VM-Entry Control Fields 2503 */ 2504 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2505 struct vmcs12 *vmcs12) 2506 { 2507 struct vcpu_vmx *vmx = to_vmx(vcpu); 2508 2509 if (!vmx_control_verify(vmcs12->vm_entry_controls, 2510 vmx->nested.msrs.entry_ctls_low, 2511 vmx->nested.msrs.entry_ctls_high)) 2512 return -EINVAL; 2513 2514 /* 2515 * From the Intel SDM, volume 3: 2516 * Fields relevant to VM-entry event injection must be set properly. 2517 * These fields are the VM-entry interruption-information field, the 2518 * VM-entry exception error code, and the VM-entry instruction length. 2519 */ 2520 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 2521 u32 intr_info = vmcs12->vm_entry_intr_info_field; 2522 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 2523 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 2524 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 2525 bool should_have_error_code; 2526 bool urg = nested_cpu_has2(vmcs12, 2527 SECONDARY_EXEC_UNRESTRICTED_GUEST); 2528 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 2529 2530 /* VM-entry interruption-info field: interruption type */ 2531 if (intr_type == INTR_TYPE_RESERVED || 2532 (intr_type == INTR_TYPE_OTHER_EVENT && 2533 !nested_cpu_supports_monitor_trap_flag(vcpu))) 2534 return -EINVAL; 2535 2536 /* VM-entry interruption-info field: vector */ 2537 if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 2538 (intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 2539 (intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 2540 return -EINVAL; 2541 2542 /* VM-entry interruption-info field: deliver error code */ 2543 should_have_error_code = 2544 intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode && 2545 x86_exception_has_error_code(vector); 2546 if (has_error_code != should_have_error_code) 2547 return -EINVAL; 2548 2549 /* VM-entry exception error code */ 2550 if (has_error_code && 2551 vmcs12->vm_entry_exception_error_code & GENMASK(31, 15)) 2552 return -EINVAL; 2553 2554 /* VM-entry interruption-info field: reserved bits */ 2555 if (intr_info & INTR_INFO_RESVD_BITS_MASK) 2556 return -EINVAL; 2557 2558 /* VM-entry instruction length */ 2559 switch (intr_type) { 2560 case INTR_TYPE_SOFT_EXCEPTION: 2561 case INTR_TYPE_SOFT_INTR: 2562 case INTR_TYPE_PRIV_SW_EXCEPTION: 2563 if ((vmcs12->vm_entry_instruction_len > 15) || 2564 (vmcs12->vm_entry_instruction_len == 0 && 2565 !nested_cpu_has_zero_length_injection(vcpu))) 2566 return -EINVAL; 2567 } 2568 } 2569 2570 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 2571 return -EINVAL; 2572 2573 return 0; 2574 } 2575 2576 /* 2577 * Checks related to Host Control Registers and MSRs 2578 */ 2579 static int nested_check_host_control_regs(struct kvm_vcpu *vcpu, 2580 struct vmcs12 *vmcs12) 2581 { 2582 bool ia32e; 2583 2584 if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) || 2585 !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) || 2586 !nested_cr3_valid(vcpu, vmcs12->host_cr3)) 2587 return -EINVAL; 2588 /* 2589 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 2590 * IA32_EFER MSR must be 0 in the field for that register. In addition, 2591 * the values of the LMA and LME bits in the field must each be that of 2592 * the host address-space size VM-exit control. 2593 */ 2594 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 2595 ia32e = (vmcs12->vm_exit_controls & 2596 VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0; 2597 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) || 2598 ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) || 2599 ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)) 2600 return -EINVAL; 2601 } 2602 2603 return 0; 2604 } 2605 2606 /* 2607 * Checks related to Guest Non-register State 2608 */ 2609 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 2610 { 2611 if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 2612 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT) 2613 return -EINVAL; 2614 2615 return 0; 2616 } 2617 2618 static int nested_vmx_check_vmentry_prereqs(struct kvm_vcpu *vcpu, 2619 struct vmcs12 *vmcs12) 2620 { 2621 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 2622 nested_check_vm_exit_controls(vcpu, vmcs12) || 2623 nested_check_vm_entry_controls(vcpu, vmcs12)) 2624 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2625 2626 if (nested_check_host_control_regs(vcpu, vmcs12)) 2627 return VMXERR_ENTRY_INVALID_HOST_STATE_FIELD; 2628 2629 if (nested_check_guest_non_reg_state(vmcs12)) 2630 return VMXERR_ENTRY_INVALID_CONTROL_FIELD; 2631 2632 return 0; 2633 } 2634 2635 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 2636 struct vmcs12 *vmcs12) 2637 { 2638 int r; 2639 struct page *page; 2640 struct vmcs12 *shadow; 2641 2642 if (vmcs12->vmcs_link_pointer == -1ull) 2643 return 0; 2644 2645 if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)) 2646 return -EINVAL; 2647 2648 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer); 2649 if (is_error_page(page)) 2650 return -EINVAL; 2651 2652 r = 0; 2653 shadow = kmap(page); 2654 if (shadow->hdr.revision_id != VMCS12_REVISION || 2655 shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)) 2656 r = -EINVAL; 2657 kunmap(page); 2658 kvm_release_page_clean(page); 2659 return r; 2660 } 2661 2662 static int nested_vmx_check_vmentry_postreqs(struct kvm_vcpu *vcpu, 2663 struct vmcs12 *vmcs12, 2664 u32 *exit_qual) 2665 { 2666 bool ia32e; 2667 2668 *exit_qual = ENTRY_FAIL_DEFAULT; 2669 2670 if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) || 2671 !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)) 2672 return 1; 2673 2674 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 2675 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR; 2676 return 1; 2677 } 2678 2679 /* 2680 * If the load IA32_EFER VM-entry control is 1, the following checks 2681 * are performed on the field for the IA32_EFER MSR: 2682 * - Bits reserved in the IA32_EFER MSR must be 0. 2683 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 2684 * the IA-32e mode guest VM-exit control. It must also be identical 2685 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 2686 * CR0.PG) is 1. 2687 */ 2688 if (to_vmx(vcpu)->nested.nested_run_pending && 2689 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 2690 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0; 2691 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) || 2692 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) || 2693 ((vmcs12->guest_cr0 & X86_CR0_PG) && 2694 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))) 2695 return 1; 2696 } 2697 2698 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 2699 (is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) || 2700 (vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))) 2701 return 1; 2702 2703 return 0; 2704 } 2705 2706 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) 2707 { 2708 struct vcpu_vmx *vmx = to_vmx(vcpu); 2709 unsigned long cr3, cr4; 2710 bool vm_fail; 2711 2712 if (!nested_early_check) 2713 return 0; 2714 2715 if (vmx->msr_autoload.host.nr) 2716 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); 2717 if (vmx->msr_autoload.guest.nr) 2718 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); 2719 2720 preempt_disable(); 2721 2722 vmx_prepare_switch_to_guest(vcpu); 2723 2724 /* 2725 * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS, 2726 * which is reserved to '1' by hardware. GUEST_RFLAGS is guaranteed to 2727 * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e. 2728 * there is no need to preserve other bits or save/restore the field. 2729 */ 2730 vmcs_writel(GUEST_RFLAGS, 0); 2731 2732 cr3 = __get_current_cr3_fast(); 2733 if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) { 2734 vmcs_writel(HOST_CR3, cr3); 2735 vmx->loaded_vmcs->host_state.cr3 = cr3; 2736 } 2737 2738 cr4 = cr4_read_shadow(); 2739 if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) { 2740 vmcs_writel(HOST_CR4, cr4); 2741 vmx->loaded_vmcs->host_state.cr4 = cr4; 2742 } 2743 2744 asm( 2745 "sub $%c[wordsize], %%" _ASM_SP "\n\t" /* temporarily adjust RSP for CALL */ 2746 "cmp %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" 2747 "je 1f \n\t" 2748 __ex("vmwrite %%" _ASM_SP ", %[HOST_RSP]") "\n\t" 2749 "mov %%" _ASM_SP ", %c[host_state_rsp](%[loaded_vmcs]) \n\t" 2750 "1: \n\t" 2751 "add $%c[wordsize], %%" _ASM_SP "\n\t" /* un-adjust RSP */ 2752 2753 /* Check if vmlaunch or vmresume is needed */ 2754 "cmpb $0, %c[launched](%[loaded_vmcs])\n\t" 2755 2756 /* 2757 * VMLAUNCH and VMRESUME clear RFLAGS.{CF,ZF} on VM-Exit, set 2758 * RFLAGS.CF on VM-Fail Invalid and set RFLAGS.ZF on VM-Fail 2759 * Valid. vmx_vmenter() directly "returns" RFLAGS, and so the 2760 * results of VM-Enter is captured via CC_{SET,OUT} to vm_fail. 2761 */ 2762 "call vmx_vmenter\n\t" 2763 2764 CC_SET(be) 2765 : ASM_CALL_CONSTRAINT, CC_OUT(be) (vm_fail) 2766 : [HOST_RSP]"r"((unsigned long)HOST_RSP), 2767 [loaded_vmcs]"r"(vmx->loaded_vmcs), 2768 [launched]"i"(offsetof(struct loaded_vmcs, launched)), 2769 [host_state_rsp]"i"(offsetof(struct loaded_vmcs, host_state.rsp)), 2770 [wordsize]"i"(sizeof(ulong)) 2771 : "cc", "memory" 2772 ); 2773 2774 preempt_enable(); 2775 2776 if (vmx->msr_autoload.host.nr) 2777 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2778 if (vmx->msr_autoload.guest.nr) 2779 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2780 2781 if (vm_fail) { 2782 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 2783 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 2784 return 1; 2785 } 2786 2787 /* 2788 * VMExit clears RFLAGS.IF and DR7, even on a consistency check. 2789 */ 2790 local_irq_enable(); 2791 if (hw_breakpoint_active()) 2792 set_debugreg(__this_cpu_read(cpu_dr7), 7); 2793 2794 /* 2795 * A non-failing VMEntry means we somehow entered guest mode with 2796 * an illegal RIP, and that's just the tip of the iceberg. There 2797 * is no telling what memory has been modified or what state has 2798 * been exposed to unknown code. Hitting this all but guarantees 2799 * a (very critical) hardware issue. 2800 */ 2801 WARN_ON(!(vmcs_read32(VM_EXIT_REASON) & 2802 VMX_EXIT_REASONS_FAILED_VMENTRY)); 2803 2804 return 0; 2805 } 2806 2807 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 2808 struct vmcs12 *vmcs12); 2809 2810 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 2811 { 2812 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2813 struct vcpu_vmx *vmx = to_vmx(vcpu); 2814 struct page *page; 2815 u64 hpa; 2816 2817 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 2818 /* 2819 * Translate L1 physical address to host physical 2820 * address for vmcs02. Keep the page pinned, so this 2821 * physical address remains valid. We keep a reference 2822 * to it so we can release it later. 2823 */ 2824 if (vmx->nested.apic_access_page) { /* shouldn't happen */ 2825 kvm_release_page_dirty(vmx->nested.apic_access_page); 2826 vmx->nested.apic_access_page = NULL; 2827 } 2828 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr); 2829 /* 2830 * If translation failed, no matter: This feature asks 2831 * to exit when accessing the given address, and if it 2832 * can never be accessed, this feature won't do 2833 * anything anyway. 2834 */ 2835 if (!is_error_page(page)) { 2836 vmx->nested.apic_access_page = page; 2837 hpa = page_to_phys(vmx->nested.apic_access_page); 2838 vmcs_write64(APIC_ACCESS_ADDR, hpa); 2839 } else { 2840 vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, 2841 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); 2842 } 2843 } 2844 2845 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 2846 if (vmx->nested.virtual_apic_page) { /* shouldn't happen */ 2847 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 2848 vmx->nested.virtual_apic_page = NULL; 2849 } 2850 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->virtual_apic_page_addr); 2851 2852 /* 2853 * If translation failed, VM entry will fail because 2854 * prepare_vmcs02 set VIRTUAL_APIC_PAGE_ADDR to -1ull. 2855 * Failing the vm entry is _not_ what the processor 2856 * does but it's basically the only possibility we 2857 * have. We could still enter the guest if CR8 load 2858 * exits are enabled, CR8 store exits are enabled, and 2859 * virtualize APIC access is disabled; in this case 2860 * the processor would never use the TPR shadow and we 2861 * could simply clear the bit from the execution 2862 * control. But such a configuration is useless, so 2863 * let's keep the code simple. 2864 */ 2865 if (!is_error_page(page)) { 2866 vmx->nested.virtual_apic_page = page; 2867 hpa = page_to_phys(vmx->nested.virtual_apic_page); 2868 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, hpa); 2869 } 2870 } 2871 2872 if (nested_cpu_has_posted_intr(vmcs12)) { 2873 if (vmx->nested.pi_desc_page) { /* shouldn't happen */ 2874 kunmap(vmx->nested.pi_desc_page); 2875 kvm_release_page_dirty(vmx->nested.pi_desc_page); 2876 vmx->nested.pi_desc_page = NULL; 2877 vmx->nested.pi_desc = NULL; 2878 vmcs_write64(POSTED_INTR_DESC_ADDR, -1ull); 2879 } 2880 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->posted_intr_desc_addr); 2881 if (is_error_page(page)) 2882 return; 2883 vmx->nested.pi_desc_page = page; 2884 vmx->nested.pi_desc = kmap(vmx->nested.pi_desc_page); 2885 vmx->nested.pi_desc = 2886 (struct pi_desc *)((void *)vmx->nested.pi_desc + 2887 (unsigned long)(vmcs12->posted_intr_desc_addr & 2888 (PAGE_SIZE - 1))); 2889 vmcs_write64(POSTED_INTR_DESC_ADDR, 2890 page_to_phys(vmx->nested.pi_desc_page) + 2891 (unsigned long)(vmcs12->posted_intr_desc_addr & 2892 (PAGE_SIZE - 1))); 2893 } 2894 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 2895 vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, 2896 CPU_BASED_USE_MSR_BITMAPS); 2897 else 2898 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL, 2899 CPU_BASED_USE_MSR_BITMAPS); 2900 } 2901 2902 /* 2903 * Intel's VMX Instruction Reference specifies a common set of prerequisites 2904 * for running VMX instructions (except VMXON, whose prerequisites are 2905 * slightly different). It also specifies what exception to inject otherwise. 2906 * Note that many of these exceptions have priority over VM exits, so they 2907 * don't have to be checked again here. 2908 */ 2909 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 2910 { 2911 if (!to_vmx(vcpu)->nested.vmxon) { 2912 kvm_queue_exception(vcpu, UD_VECTOR); 2913 return 0; 2914 } 2915 2916 if (vmx_get_cpl(vcpu)) { 2917 kvm_inject_gp(vcpu, 0); 2918 return 0; 2919 } 2920 2921 return 1; 2922 } 2923 2924 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu) 2925 { 2926 u8 rvi = vmx_get_rvi(); 2927 u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI); 2928 2929 return ((rvi & 0xf0) > (vppr & 0xf0)); 2930 } 2931 2932 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 2933 struct vmcs12 *vmcs12); 2934 2935 /* 2936 * If from_vmentry is false, this is being called from state restore (either RSM 2937 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 2938 + * 2939 + * Returns: 2940 + * 0 - success, i.e. proceed with actual VMEnter 2941 + * 1 - consistency check VMExit 2942 + * -1 - consistency check VMFail 2943 */ 2944 int nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry) 2945 { 2946 struct vcpu_vmx *vmx = to_vmx(vcpu); 2947 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2948 bool evaluate_pending_interrupts; 2949 u32 exit_reason = EXIT_REASON_INVALID_STATE; 2950 u32 exit_qual; 2951 2952 evaluate_pending_interrupts = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) & 2953 (CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_VIRTUAL_NMI_PENDING); 2954 if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) 2955 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); 2956 2957 if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 2958 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 2959 if (kvm_mpx_supported() && 2960 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2961 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 2962 2963 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 2964 2965 prepare_vmcs02_early(vmx, vmcs12); 2966 2967 if (from_vmentry) { 2968 nested_get_vmcs12_pages(vcpu); 2969 2970 if (nested_vmx_check_vmentry_hw(vcpu)) { 2971 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 2972 return -1; 2973 } 2974 2975 if (nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 2976 goto vmentry_fail_vmexit; 2977 } 2978 2979 enter_guest_mode(vcpu); 2980 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 2981 vcpu->arch.tsc_offset += vmcs12->tsc_offset; 2982 2983 if (prepare_vmcs02(vcpu, vmcs12, &exit_qual)) 2984 goto vmentry_fail_vmexit_guest_mode; 2985 2986 if (from_vmentry) { 2987 exit_reason = EXIT_REASON_MSR_LOAD_FAIL; 2988 exit_qual = nested_vmx_load_msr(vcpu, 2989 vmcs12->vm_entry_msr_load_addr, 2990 vmcs12->vm_entry_msr_load_count); 2991 if (exit_qual) 2992 goto vmentry_fail_vmexit_guest_mode; 2993 } else { 2994 /* 2995 * The MMU is not initialized to point at the right entities yet and 2996 * "get pages" would need to read data from the guest (i.e. we will 2997 * need to perform gpa to hpa translation). Request a call 2998 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 2999 * have already been set at vmentry time and should not be reset. 3000 */ 3001 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu); 3002 } 3003 3004 /* 3005 * If L1 had a pending IRQ/NMI until it executed 3006 * VMLAUNCH/VMRESUME which wasn't delivered because it was 3007 * disallowed (e.g. interrupts disabled), L0 needs to 3008 * evaluate if this pending event should cause an exit from L2 3009 * to L1 or delivered directly to L2 (e.g. In case L1 don't 3010 * intercept EXTERNAL_INTERRUPT). 3011 * 3012 * Usually this would be handled by the processor noticing an 3013 * IRQ/NMI window request, or checking RVI during evaluation of 3014 * pending virtual interrupts. However, this setting was done 3015 * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 3016 * to perform pending event evaluation by requesting a KVM_REQ_EVENT. 3017 */ 3018 if (unlikely(evaluate_pending_interrupts)) 3019 kvm_make_request(KVM_REQ_EVENT, vcpu); 3020 3021 /* 3022 * Do not start the preemption timer hrtimer until after we know 3023 * we are successful, so that only nested_vmx_vmexit needs to cancel 3024 * the timer. 3025 */ 3026 vmx->nested.preemption_timer_expired = false; 3027 if (nested_cpu_has_preemption_timer(vmcs12)) 3028 vmx_start_preemption_timer(vcpu); 3029 3030 /* 3031 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3032 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3033 * returned as far as L1 is concerned. It will only return (and set 3034 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3035 */ 3036 return 0; 3037 3038 /* 3039 * A failed consistency check that leads to a VMExit during L1's 3040 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3041 * 26.7 "VM-entry failures during or after loading guest state". 3042 */ 3043 vmentry_fail_vmexit_guest_mode: 3044 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3045 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3046 leave_guest_mode(vcpu); 3047 3048 vmentry_fail_vmexit: 3049 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3050 3051 if (!from_vmentry) 3052 return 1; 3053 3054 load_vmcs12_host_state(vcpu, vmcs12); 3055 vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY; 3056 vmcs12->exit_qualification = exit_qual; 3057 if (enable_shadow_vmcs || vmx->nested.hv_evmcs) 3058 vmx->nested.need_vmcs12_sync = true; 3059 return 1; 3060 } 3061 3062 /* 3063 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3064 * for running an L2 nested guest. 3065 */ 3066 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3067 { 3068 struct vmcs12 *vmcs12; 3069 struct vcpu_vmx *vmx = to_vmx(vcpu); 3070 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3071 int ret; 3072 3073 if (!nested_vmx_check_permission(vcpu)) 3074 return 1; 3075 3076 if (!nested_vmx_handle_enlightened_vmptrld(vcpu, true)) 3077 return 1; 3078 3079 if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull) 3080 return nested_vmx_failInvalid(vcpu); 3081 3082 vmcs12 = get_vmcs12(vcpu); 3083 3084 /* 3085 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3086 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3087 * rather than RFLAGS.ZF, and no error number is stored to the 3088 * VM-instruction error field. 3089 */ 3090 if (vmcs12->hdr.shadow_vmcs) 3091 return nested_vmx_failInvalid(vcpu); 3092 3093 if (vmx->nested.hv_evmcs) { 3094 copy_enlightened_to_vmcs12(vmx); 3095 /* Enlightened VMCS doesn't have launch state */ 3096 vmcs12->launch_state = !launch; 3097 } else if (enable_shadow_vmcs) { 3098 copy_shadow_to_vmcs12(vmx); 3099 } 3100 3101 /* 3102 * The nested entry process starts with enforcing various prerequisites 3103 * on vmcs12 as required by the Intel SDM, and act appropriately when 3104 * they fail: As the SDM explains, some conditions should cause the 3105 * instruction to fail, while others will cause the instruction to seem 3106 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3107 * To speed up the normal (success) code path, we should avoid checking 3108 * for misconfigurations which will anyway be caught by the processor 3109 * when using the merged vmcs02. 3110 */ 3111 if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) 3112 return nested_vmx_failValid(vcpu, 3113 VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3114 3115 if (vmcs12->launch_state == launch) 3116 return nested_vmx_failValid(vcpu, 3117 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3118 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3119 3120 ret = nested_vmx_check_vmentry_prereqs(vcpu, vmcs12); 3121 if (ret) 3122 return nested_vmx_failValid(vcpu, ret); 3123 3124 /* 3125 * We're finally done with prerequisite checking, and can start with 3126 * the nested entry. 3127 */ 3128 vmx->nested.nested_run_pending = 1; 3129 ret = nested_vmx_enter_non_root_mode(vcpu, true); 3130 vmx->nested.nested_run_pending = !ret; 3131 if (ret > 0) 3132 return 1; 3133 else if (ret) 3134 return nested_vmx_failValid(vcpu, 3135 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3136 3137 /* Hide L1D cache contents from the nested guest. */ 3138 vmx->vcpu.arch.l1tf_flush_l1d = true; 3139 3140 /* 3141 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3142 * also be used as part of restoring nVMX state for 3143 * snapshot restore (migration). 3144 * 3145 * In this flow, it is assumed that vmcs12 cache was 3146 * trasferred as part of captured nVMX state and should 3147 * therefore not be read from guest memory (which may not 3148 * exist on destination host yet). 3149 */ 3150 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3151 3152 /* 3153 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3154 * awakened by event injection or by an NMI-window VM-exit or 3155 * by an interrupt-window VM-exit, halt the vcpu. 3156 */ 3157 if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) && 3158 !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3159 !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_NMI_PENDING) && 3160 !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_INTR_PENDING) && 3161 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3162 vmx->nested.nested_run_pending = 0; 3163 return kvm_vcpu_halt(vcpu); 3164 } 3165 return 1; 3166 } 3167 3168 /* 3169 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3170 * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK). 3171 * This function returns the new value we should put in vmcs12.guest_cr0. 3172 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3173 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3174 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3175 * didn't trap the bit, because if L1 did, so would L0). 3176 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3177 * been modified by L2, and L1 knows it. So just leave the old value of 3178 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3179 * isn't relevant, because if L0 traps this bit it can set it to anything. 3180 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3181 * changed these bits, and therefore they need to be updated, but L0 3182 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3183 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3184 */ 3185 static inline unsigned long 3186 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3187 { 3188 return 3189 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3190 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3191 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3192 vcpu->arch.cr0_guest_owned_bits)); 3193 } 3194 3195 static inline unsigned long 3196 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3197 { 3198 return 3199 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3200 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3201 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3202 vcpu->arch.cr4_guest_owned_bits)); 3203 } 3204 3205 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3206 struct vmcs12 *vmcs12) 3207 { 3208 u32 idt_vectoring; 3209 unsigned int nr; 3210 3211 if (vcpu->arch.exception.injected) { 3212 nr = vcpu->arch.exception.nr; 3213 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3214 3215 if (kvm_exception_is_soft(nr)) { 3216 vmcs12->vm_exit_instruction_len = 3217 vcpu->arch.event_exit_inst_len; 3218 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3219 } else 3220 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3221 3222 if (vcpu->arch.exception.has_error_code) { 3223 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3224 vmcs12->idt_vectoring_error_code = 3225 vcpu->arch.exception.error_code; 3226 } 3227 3228 vmcs12->idt_vectoring_info_field = idt_vectoring; 3229 } else if (vcpu->arch.nmi_injected) { 3230 vmcs12->idt_vectoring_info_field = 3231 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3232 } else if (vcpu->arch.interrupt.injected) { 3233 nr = vcpu->arch.interrupt.nr; 3234 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3235 3236 if (vcpu->arch.interrupt.soft) { 3237 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3238 vmcs12->vm_entry_instruction_len = 3239 vcpu->arch.event_exit_inst_len; 3240 } else 3241 idt_vectoring |= INTR_TYPE_EXT_INTR; 3242 3243 vmcs12->idt_vectoring_info_field = idt_vectoring; 3244 } 3245 } 3246 3247 3248 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 3249 { 3250 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3251 gfn_t gfn; 3252 3253 /* 3254 * Don't need to mark the APIC access page dirty; it is never 3255 * written to by the CPU during APIC virtualization. 3256 */ 3257 3258 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3259 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 3260 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3261 } 3262 3263 if (nested_cpu_has_posted_intr(vmcs12)) { 3264 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 3265 kvm_vcpu_mark_page_dirty(vcpu, gfn); 3266 } 3267 } 3268 3269 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 3270 { 3271 struct vcpu_vmx *vmx = to_vmx(vcpu); 3272 int max_irr; 3273 void *vapic_page; 3274 u16 status; 3275 3276 if (!vmx->nested.pi_desc || !vmx->nested.pi_pending) 3277 return; 3278 3279 vmx->nested.pi_pending = false; 3280 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 3281 return; 3282 3283 max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); 3284 if (max_irr != 256) { 3285 vapic_page = kmap(vmx->nested.virtual_apic_page); 3286 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 3287 vapic_page, &max_irr); 3288 kunmap(vmx->nested.virtual_apic_page); 3289 3290 status = vmcs_read16(GUEST_INTR_STATUS); 3291 if ((u8)max_irr > ((u8)status & 0xff)) { 3292 status &= ~0xff; 3293 status |= (u8)max_irr; 3294 vmcs_write16(GUEST_INTR_STATUS, status); 3295 } 3296 } 3297 3298 nested_mark_vmcs12_pages_dirty(vcpu); 3299 } 3300 3301 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, 3302 unsigned long exit_qual) 3303 { 3304 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3305 unsigned int nr = vcpu->arch.exception.nr; 3306 u32 intr_info = nr | INTR_INFO_VALID_MASK; 3307 3308 if (vcpu->arch.exception.has_error_code) { 3309 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; 3310 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 3311 } 3312 3313 if (kvm_exception_is_soft(nr)) 3314 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 3315 else 3316 intr_info |= INTR_TYPE_HARD_EXCEPTION; 3317 3318 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 3319 vmx_get_nmi_mask(vcpu)) 3320 intr_info |= INTR_INFO_UNBLOCK_NMI; 3321 3322 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 3323 } 3324 3325 static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr) 3326 { 3327 struct vcpu_vmx *vmx = to_vmx(vcpu); 3328 unsigned long exit_qual; 3329 bool block_nested_events = 3330 vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); 3331 3332 if (vcpu->arch.exception.pending && 3333 nested_vmx_check_exception(vcpu, &exit_qual)) { 3334 if (block_nested_events) 3335 return -EBUSY; 3336 nested_vmx_inject_exception_vmexit(vcpu, exit_qual); 3337 return 0; 3338 } 3339 3340 if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 3341 vmx->nested.preemption_timer_expired) { 3342 if (block_nested_events) 3343 return -EBUSY; 3344 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 3345 return 0; 3346 } 3347 3348 if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) { 3349 if (block_nested_events) 3350 return -EBUSY; 3351 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 3352 NMI_VECTOR | INTR_TYPE_NMI_INTR | 3353 INTR_INFO_VALID_MASK, 0); 3354 /* 3355 * The NMI-triggered VM exit counts as injection: 3356 * clear this one and block further NMIs. 3357 */ 3358 vcpu->arch.nmi_pending = 0; 3359 vmx_set_nmi_mask(vcpu, true); 3360 return 0; 3361 } 3362 3363 if ((kvm_cpu_has_interrupt(vcpu) || external_intr) && 3364 nested_exit_on_intr(vcpu)) { 3365 if (block_nested_events) 3366 return -EBUSY; 3367 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 3368 return 0; 3369 } 3370 3371 vmx_complete_nested_posted_interrupt(vcpu); 3372 return 0; 3373 } 3374 3375 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 3376 { 3377 ktime_t remaining = 3378 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 3379 u64 value; 3380 3381 if (ktime_to_ns(remaining) <= 0) 3382 return 0; 3383 3384 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 3385 do_div(value, 1000000); 3386 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 3387 } 3388 3389 /* 3390 * Update the guest state fields of vmcs12 to reflect changes that 3391 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 3392 * VM-entry controls is also updated, since this is really a guest 3393 * state bit.) 3394 */ 3395 static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3396 { 3397 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 3398 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 3399 3400 vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); 3401 vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP); 3402 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 3403 3404 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 3405 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 3406 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 3407 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 3408 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 3409 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 3410 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 3411 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 3412 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 3413 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 3414 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 3415 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 3416 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 3417 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 3418 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 3419 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 3420 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 3421 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 3422 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 3423 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 3424 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 3425 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 3426 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 3427 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 3428 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 3429 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 3430 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 3431 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 3432 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 3433 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 3434 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 3435 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 3436 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 3437 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 3438 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 3439 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 3440 3441 vmcs12->guest_interruptibility_info = 3442 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 3443 vmcs12->guest_pending_dbg_exceptions = 3444 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 3445 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 3446 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 3447 else 3448 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 3449 3450 if (nested_cpu_has_preemption_timer(vmcs12) && 3451 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) 3452 vmcs12->vmx_preemption_timer_value = 3453 vmx_get_preemption_timer_value(vcpu); 3454 3455 /* 3456 * In some cases (usually, nested EPT), L2 is allowed to change its 3457 * own CR3 without exiting. If it has changed it, we must keep it. 3458 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 3459 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 3460 * 3461 * Additionally, restore L2's PDPTR to vmcs12. 3462 */ 3463 if (enable_ept) { 3464 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 3465 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 3466 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 3467 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 3468 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 3469 } 3470 3471 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 3472 3473 if (nested_cpu_has_vid(vmcs12)) 3474 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 3475 3476 vmcs12->vm_entry_controls = 3477 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 3478 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 3479 3480 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) { 3481 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7); 3482 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); 3483 } 3484 3485 /* TODO: These cannot have changed unless we have MSR bitmaps and 3486 * the relevant bit asks not to trap the change */ 3487 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT) 3488 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT); 3489 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 3490 vmcs12->guest_ia32_efer = vcpu->arch.efer; 3491 vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS); 3492 vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP); 3493 vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP); 3494 if (kvm_mpx_supported()) 3495 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3496 } 3497 3498 /* 3499 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 3500 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 3501 * and this function updates it to reflect the changes to the guest state while 3502 * L2 was running (and perhaps made some exits which were handled directly by L0 3503 * without going back to L1), and to reflect the exit reason. 3504 * Note that we do not have to copy here all VMCS fields, just those that 3505 * could have changed by the L2 guest or the exit - i.e., the guest-state and 3506 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 3507 * which already writes to vmcs12 directly. 3508 */ 3509 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 3510 u32 exit_reason, u32 exit_intr_info, 3511 unsigned long exit_qualification) 3512 { 3513 /* update guest state fields: */ 3514 sync_vmcs12(vcpu, vmcs12); 3515 3516 /* update exit information fields: */ 3517 3518 vmcs12->vm_exit_reason = exit_reason; 3519 vmcs12->exit_qualification = exit_qualification; 3520 vmcs12->vm_exit_intr_info = exit_intr_info; 3521 3522 vmcs12->idt_vectoring_info_field = 0; 3523 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); 3524 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 3525 3526 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 3527 vmcs12->launch_state = 1; 3528 3529 /* vm_entry_intr_info_field is cleared on exit. Emulate this 3530 * instead of reading the real value. */ 3531 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 3532 3533 /* 3534 * Transfer the event that L0 or L1 may wanted to inject into 3535 * L2 to IDT_VECTORING_INFO_FIELD. 3536 */ 3537 vmcs12_save_pending_event(vcpu, vmcs12); 3538 3539 /* 3540 * According to spec, there's no need to store the guest's 3541 * MSRs if the exit is due to a VM-entry failure that occurs 3542 * during or after loading the guest state. Since this exit 3543 * does not fall in that category, we need to save the MSRs. 3544 */ 3545 if (nested_vmx_store_msr(vcpu, 3546 vmcs12->vm_exit_msr_store_addr, 3547 vmcs12->vm_exit_msr_store_count)) 3548 nested_vmx_abort(vcpu, 3549 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 3550 } 3551 3552 /* 3553 * Drop what we picked up for L2 via vmx_complete_interrupts. It is 3554 * preserved above and would only end up incorrectly in L1. 3555 */ 3556 vcpu->arch.nmi_injected = false; 3557 kvm_clear_exception_queue(vcpu); 3558 kvm_clear_interrupt_queue(vcpu); 3559 } 3560 3561 /* 3562 * A part of what we need to when the nested L2 guest exits and we want to 3563 * run its L1 parent, is to reset L1's guest state to the host state specified 3564 * in vmcs12. 3565 * This function is to be called not only on normal nested exit, but also on 3566 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 3567 * Failures During or After Loading Guest State"). 3568 * This function should be called when the active VMCS is L1's (vmcs01). 3569 */ 3570 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3571 struct vmcs12 *vmcs12) 3572 { 3573 struct kvm_segment seg; 3574 u32 entry_failure_code; 3575 3576 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 3577 vcpu->arch.efer = vmcs12->host_ia32_efer; 3578 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3579 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 3580 else 3581 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 3582 vmx_set_efer(vcpu, vcpu->arch.efer); 3583 3584 kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp); 3585 kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip); 3586 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 3587 vmx_set_interrupt_shadow(vcpu, 0); 3588 3589 /* 3590 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 3591 * actually changed, because vmx_set_cr0 refers to efer set above. 3592 * 3593 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 3594 * (KVM doesn't change it); 3595 */ 3596 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3597 vmx_set_cr0(vcpu, vmcs12->host_cr0); 3598 3599 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 3600 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3601 vmx_set_cr4(vcpu, vmcs12->host_cr4); 3602 3603 nested_ept_uninit_mmu_context(vcpu); 3604 3605 /* 3606 * Only PDPTE load can fail as the value of cr3 was checked on entry and 3607 * couldn't have changed. 3608 */ 3609 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code)) 3610 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 3611 3612 if (!enable_ept) 3613 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; 3614 3615 /* 3616 * If vmcs01 doesn't use VPID, CPU flushes TLB on every 3617 * VMEntry/VMExit. Thus, no need to flush TLB. 3618 * 3619 * If vmcs12 doesn't use VPID, L1 expects TLB to be 3620 * flushed on every VMEntry/VMExit. 3621 * 3622 * Otherwise, we can preserve TLB entries as long as we are 3623 * able to tag L1 TLB entries differently than L2 TLB entries. 3624 * 3625 * If vmcs12 uses EPT, we need to execute this flush on EPTP01 3626 * and therefore we request the TLB flush to happen only after VMCS EPTP 3627 * has been set by KVM_REQ_LOAD_CR3. 3628 */ 3629 if (enable_vpid && 3630 (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) { 3631 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 3632 } 3633 3634 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 3635 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 3636 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 3637 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 3638 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 3639 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 3640 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 3641 3642 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 3643 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 3644 vmcs_write64(GUEST_BNDCFGS, 0); 3645 3646 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 3647 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 3648 vcpu->arch.pat = vmcs12->host_ia32_pat; 3649 } 3650 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) 3651 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL, 3652 vmcs12->host_ia32_perf_global_ctrl); 3653 3654 /* Set L1 segment info according to Intel SDM 3655 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 3656 seg = (struct kvm_segment) { 3657 .base = 0, 3658 .limit = 0xFFFFFFFF, 3659 .selector = vmcs12->host_cs_selector, 3660 .type = 11, 3661 .present = 1, 3662 .s = 1, 3663 .g = 1 3664 }; 3665 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 3666 seg.l = 1; 3667 else 3668 seg.db = 1; 3669 vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 3670 seg = (struct kvm_segment) { 3671 .base = 0, 3672 .limit = 0xFFFFFFFF, 3673 .type = 3, 3674 .present = 1, 3675 .s = 1, 3676 .db = 1, 3677 .g = 1 3678 }; 3679 seg.selector = vmcs12->host_ds_selector; 3680 vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 3681 seg.selector = vmcs12->host_es_selector; 3682 vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 3683 seg.selector = vmcs12->host_ss_selector; 3684 vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 3685 seg.selector = vmcs12->host_fs_selector; 3686 seg.base = vmcs12->host_fs_base; 3687 vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 3688 seg.selector = vmcs12->host_gs_selector; 3689 seg.base = vmcs12->host_gs_base; 3690 vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 3691 seg = (struct kvm_segment) { 3692 .base = vmcs12->host_tr_base, 3693 .limit = 0x67, 3694 .selector = vmcs12->host_tr_selector, 3695 .type = 11, 3696 .present = 1 3697 }; 3698 vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 3699 3700 kvm_set_dr(vcpu, 7, 0x400); 3701 vmcs_write64(GUEST_IA32_DEBUGCTL, 0); 3702 3703 if (cpu_has_vmx_msr_bitmap()) 3704 vmx_update_msr_bitmap(vcpu); 3705 3706 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 3707 vmcs12->vm_exit_msr_load_count)) 3708 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3709 } 3710 3711 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 3712 { 3713 struct shared_msr_entry *efer_msr; 3714 unsigned int i; 3715 3716 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 3717 return vmcs_read64(GUEST_IA32_EFER); 3718 3719 if (cpu_has_load_ia32_efer()) 3720 return host_efer; 3721 3722 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 3723 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 3724 return vmx->msr_autoload.guest.val[i].value; 3725 } 3726 3727 efer_msr = find_msr_entry(vmx, MSR_EFER); 3728 if (efer_msr) 3729 return efer_msr->data; 3730 3731 return host_efer; 3732 } 3733 3734 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 3735 { 3736 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3737 struct vcpu_vmx *vmx = to_vmx(vcpu); 3738 struct vmx_msr_entry g, h; 3739 struct msr_data msr; 3740 gpa_t gpa; 3741 u32 i, j; 3742 3743 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 3744 3745 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 3746 /* 3747 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 3748 * as vmcs01.GUEST_DR7 contains a userspace defined value 3749 * and vcpu->arch.dr7 is not squirreled away before the 3750 * nested VMENTER (not worth adding a variable in nested_vmx). 3751 */ 3752 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 3753 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 3754 else 3755 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 3756 } 3757 3758 /* 3759 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 3760 * handle a variety of side effects to KVM's software model. 3761 */ 3762 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 3763 3764 vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS; 3765 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 3766 3767 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 3768 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 3769 3770 nested_ept_uninit_mmu_context(vcpu); 3771 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 3772 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 3773 3774 /* 3775 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 3776 * from vmcs01 (if necessary). The PDPTRs are not loaded on 3777 * VMFail, like everything else we just need to ensure our 3778 * software model is up-to-date. 3779 */ 3780 ept_save_pdptrs(vcpu); 3781 3782 kvm_mmu_reset_context(vcpu); 3783 3784 if (cpu_has_vmx_msr_bitmap()) 3785 vmx_update_msr_bitmap(vcpu); 3786 3787 /* 3788 * This nasty bit of open coding is a compromise between blindly 3789 * loading L1's MSRs using the exit load lists (incorrect emulation 3790 * of VMFail), leaving the nested VM's MSRs in the software model 3791 * (incorrect behavior) and snapshotting the modified MSRs (too 3792 * expensive since the lists are unbound by hardware). For each 3793 * MSR that was (prematurely) loaded from the nested VMEntry load 3794 * list, reload it from the exit load list if it exists and differs 3795 * from the guest value. The intent is to stuff host state as 3796 * silently as possible, not to fully process the exit load list. 3797 */ 3798 msr.host_initiated = false; 3799 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 3800 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 3801 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 3802 pr_debug_ratelimited( 3803 "%s read MSR index failed (%u, 0x%08llx)\n", 3804 __func__, i, gpa); 3805 goto vmabort; 3806 } 3807 3808 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 3809 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 3810 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 3811 pr_debug_ratelimited( 3812 "%s read MSR failed (%u, 0x%08llx)\n", 3813 __func__, j, gpa); 3814 goto vmabort; 3815 } 3816 if (h.index != g.index) 3817 continue; 3818 if (h.value == g.value) 3819 break; 3820 3821 if (nested_vmx_load_msr_check(vcpu, &h)) { 3822 pr_debug_ratelimited( 3823 "%s check failed (%u, 0x%x, 0x%x)\n", 3824 __func__, j, h.index, h.reserved); 3825 goto vmabort; 3826 } 3827 3828 msr.index = h.index; 3829 msr.data = h.value; 3830 if (kvm_set_msr(vcpu, &msr)) { 3831 pr_debug_ratelimited( 3832 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 3833 __func__, j, h.index, h.value); 3834 goto vmabort; 3835 } 3836 } 3837 } 3838 3839 return; 3840 3841 vmabort: 3842 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 3843 } 3844 3845 /* 3846 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 3847 * and modify vmcs12 to make it see what it would expect to see there if 3848 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 3849 */ 3850 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, 3851 u32 exit_intr_info, unsigned long exit_qualification) 3852 { 3853 struct vcpu_vmx *vmx = to_vmx(vcpu); 3854 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3855 3856 /* trying to cancel vmlaunch/vmresume is a bug */ 3857 WARN_ON_ONCE(vmx->nested.nested_run_pending); 3858 3859 leave_guest_mode(vcpu); 3860 3861 if (nested_cpu_has_preemption_timer(vmcs12)) 3862 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 3863 3864 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) 3865 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3866 3867 if (likely(!vmx->fail)) { 3868 if (exit_reason == -1) 3869 sync_vmcs12(vcpu, vmcs12); 3870 else 3871 prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info, 3872 exit_qualification); 3873 3874 /* 3875 * Must happen outside of sync_vmcs12() as it will 3876 * also be used to capture vmcs12 cache as part of 3877 * capturing nVMX state for snapshot (migration). 3878 * 3879 * Otherwise, this flush will dirty guest memory at a 3880 * point it is already assumed by user-space to be 3881 * immutable. 3882 */ 3883 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 3884 } else { 3885 /* 3886 * The only expected VM-instruction error is "VM entry with 3887 * invalid control field(s)." Anything else indicates a 3888 * problem with L0. And we should never get here with a 3889 * VMFail of any type if early consistency checks are enabled. 3890 */ 3891 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 3892 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3893 WARN_ON_ONCE(nested_early_check); 3894 } 3895 3896 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3897 3898 /* Update any VMCS fields that might have changed while L2 ran */ 3899 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 3900 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 3901 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 3902 3903 if (kvm_has_tsc_control) 3904 decache_tsc_multiplier(vmx); 3905 3906 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 3907 vmx->nested.change_vmcs01_virtual_apic_mode = false; 3908 vmx_set_virtual_apic_mode(vcpu); 3909 } else if (!nested_cpu_has_ept(vmcs12) && 3910 nested_cpu_has2(vmcs12, 3911 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3912 vmx_flush_tlb(vcpu, true); 3913 } 3914 3915 /* Unpin physical memory we referred to in vmcs02 */ 3916 if (vmx->nested.apic_access_page) { 3917 kvm_release_page_dirty(vmx->nested.apic_access_page); 3918 vmx->nested.apic_access_page = NULL; 3919 } 3920 if (vmx->nested.virtual_apic_page) { 3921 kvm_release_page_dirty(vmx->nested.virtual_apic_page); 3922 vmx->nested.virtual_apic_page = NULL; 3923 } 3924 if (vmx->nested.pi_desc_page) { 3925 kunmap(vmx->nested.pi_desc_page); 3926 kvm_release_page_dirty(vmx->nested.pi_desc_page); 3927 vmx->nested.pi_desc_page = NULL; 3928 vmx->nested.pi_desc = NULL; 3929 } 3930 3931 /* 3932 * We are now running in L2, mmu_notifier will force to reload the 3933 * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1. 3934 */ 3935 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 3936 3937 if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs)) 3938 vmx->nested.need_vmcs12_sync = true; 3939 3940 /* in case we halted in L2 */ 3941 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 3942 3943 if (likely(!vmx->fail)) { 3944 /* 3945 * TODO: SDM says that with acknowledge interrupt on 3946 * exit, bit 31 of the VM-exit interrupt information 3947 * (valid interrupt) is always set to 1 on 3948 * EXIT_REASON_EXTERNAL_INTERRUPT, so we shouldn't 3949 * need kvm_cpu_has_interrupt(). See the commit 3950 * message for details. 3951 */ 3952 if (nested_exit_intr_ack_set(vcpu) && 3953 exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && 3954 kvm_cpu_has_interrupt(vcpu)) { 3955 int irq = kvm_cpu_get_interrupt(vcpu); 3956 WARN_ON(irq < 0); 3957 vmcs12->vm_exit_intr_info = irq | 3958 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; 3959 } 3960 3961 if (exit_reason != -1) 3962 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 3963 vmcs12->exit_qualification, 3964 vmcs12->idt_vectoring_info_field, 3965 vmcs12->vm_exit_intr_info, 3966 vmcs12->vm_exit_intr_error_code, 3967 KVM_ISA_VMX); 3968 3969 load_vmcs12_host_state(vcpu, vmcs12); 3970 3971 return; 3972 } 3973 3974 /* 3975 * After an early L2 VM-entry failure, we're now back 3976 * in L1 which thinks it just finished a VMLAUNCH or 3977 * VMRESUME instruction, so we need to set the failure 3978 * flag and the VM-instruction error field of the VMCS 3979 * accordingly, and skip the emulated instruction. 3980 */ 3981 (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3982 3983 /* 3984 * Restore L1's host state to KVM's software model. We're here 3985 * because a consistency check was caught by hardware, which 3986 * means some amount of guest state has been propagated to KVM's 3987 * model and needs to be unwound to the host's state. 3988 */ 3989 nested_vmx_restore_host_state(vcpu); 3990 3991 vmx->fail = 0; 3992 } 3993 3994 /* 3995 * Decode the memory-address operand of a vmx instruction, as recorded on an 3996 * exit caused by such an instruction (run by a guest hypervisor). 3997 * On success, returns 0. When the operand is invalid, returns 1 and throws 3998 * #UD or #GP. 3999 */ 4000 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 4001 u32 vmx_instruction_info, bool wr, gva_t *ret) 4002 { 4003 gva_t off; 4004 bool exn; 4005 struct kvm_segment s; 4006 4007 /* 4008 * According to Vol. 3B, "Information for VM Exits Due to Instruction 4009 * Execution", on an exit, vmx_instruction_info holds most of the 4010 * addressing components of the operand. Only the displacement part 4011 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 4012 * For how an actual address is calculated from all these components, 4013 * refer to Vol. 1, "Operand Addressing". 4014 */ 4015 int scaling = vmx_instruction_info & 3; 4016 int addr_size = (vmx_instruction_info >> 7) & 7; 4017 bool is_reg = vmx_instruction_info & (1u << 10); 4018 int seg_reg = (vmx_instruction_info >> 15) & 7; 4019 int index_reg = (vmx_instruction_info >> 18) & 0xf; 4020 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 4021 int base_reg = (vmx_instruction_info >> 23) & 0xf; 4022 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 4023 4024 if (is_reg) { 4025 kvm_queue_exception(vcpu, UD_VECTOR); 4026 return 1; 4027 } 4028 4029 /* Addr = segment_base + offset */ 4030 /* offset = base + [index * scale] + displacement */ 4031 off = exit_qualification; /* holds the displacement */ 4032 if (addr_size == 1) 4033 off = (gva_t)sign_extend64(off, 31); 4034 else if (addr_size == 0) 4035 off = (gva_t)sign_extend64(off, 15); 4036 if (base_is_valid) 4037 off += kvm_register_read(vcpu, base_reg); 4038 if (index_is_valid) 4039 off += kvm_register_read(vcpu, index_reg)<<scaling; 4040 vmx_get_segment(vcpu, &s, seg_reg); 4041 4042 /* 4043 * The effective address, i.e. @off, of a memory operand is truncated 4044 * based on the address size of the instruction. Note that this is 4045 * the *effective address*, i.e. the address prior to accounting for 4046 * the segment's base. 4047 */ 4048 if (addr_size == 1) /* 32 bit */ 4049 off &= 0xffffffff; 4050 else if (addr_size == 0) /* 16 bit */ 4051 off &= 0xffff; 4052 4053 /* Checks for #GP/#SS exceptions. */ 4054 exn = false; 4055 if (is_long_mode(vcpu)) { 4056 /* 4057 * The virtual/linear address is never truncated in 64-bit 4058 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 4059 * address when using FS/GS with a non-zero base. 4060 */ 4061 *ret = s.base + off; 4062 4063 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 4064 * non-canonical form. This is the only check on the memory 4065 * destination for long mode! 4066 */ 4067 exn = is_noncanonical_address(*ret, vcpu); 4068 } else { 4069 /* 4070 * When not in long mode, the virtual/linear address is 4071 * unconditionally truncated to 32 bits regardless of the 4072 * address size. 4073 */ 4074 *ret = (s.base + off) & 0xffffffff; 4075 4076 /* Protected mode: apply checks for segment validity in the 4077 * following order: 4078 * - segment type check (#GP(0) may be thrown) 4079 * - usability check (#GP(0)/#SS(0)) 4080 * - limit check (#GP(0)/#SS(0)) 4081 */ 4082 if (wr) 4083 /* #GP(0) if the destination operand is located in a 4084 * read-only data segment or any code segment. 4085 */ 4086 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 4087 else 4088 /* #GP(0) if the source operand is located in an 4089 * execute-only code segment 4090 */ 4091 exn = ((s.type & 0xa) == 8); 4092 if (exn) { 4093 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 4094 return 1; 4095 } 4096 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 4097 */ 4098 exn = (s.unusable != 0); 4099 4100 /* 4101 * Protected mode: #GP(0)/#SS(0) if the memory operand is 4102 * outside the segment limit. All CPUs that support VMX ignore 4103 * limit checks for flat segments, i.e. segments with base==0, 4104 * limit==0xffffffff and of type expand-up data or code. 4105 */ 4106 if (!(s.base == 0 && s.limit == 0xffffffff && 4107 ((s.type & 8) || !(s.type & 4)))) 4108 exn = exn || (off + sizeof(u64) > s.limit); 4109 } 4110 if (exn) { 4111 kvm_queue_exception_e(vcpu, 4112 seg_reg == VCPU_SREG_SS ? 4113 SS_VECTOR : GP_VECTOR, 4114 0); 4115 return 1; 4116 } 4117 4118 return 0; 4119 } 4120 4121 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) 4122 { 4123 gva_t gva; 4124 struct x86_exception e; 4125 4126 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4127 vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva)) 4128 return 1; 4129 4130 if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) { 4131 kvm_inject_page_fault(vcpu, &e); 4132 return 1; 4133 } 4134 4135 return 0; 4136 } 4137 4138 /* 4139 * Allocate a shadow VMCS and associate it with the currently loaded 4140 * VMCS, unless such a shadow VMCS already exists. The newly allocated 4141 * VMCS is also VMCLEARed, so that it is ready for use. 4142 */ 4143 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 4144 { 4145 struct vcpu_vmx *vmx = to_vmx(vcpu); 4146 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 4147 4148 /* 4149 * We should allocate a shadow vmcs for vmcs01 only when L1 4150 * executes VMXON and free it when L1 executes VMXOFF. 4151 * As it is invalid to execute VMXON twice, we shouldn't reach 4152 * here when vmcs01 already have an allocated shadow vmcs. 4153 */ 4154 WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs); 4155 4156 if (!loaded_vmcs->shadow_vmcs) { 4157 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 4158 if (loaded_vmcs->shadow_vmcs) 4159 vmcs_clear(loaded_vmcs->shadow_vmcs); 4160 } 4161 return loaded_vmcs->shadow_vmcs; 4162 } 4163 4164 static int enter_vmx_operation(struct kvm_vcpu *vcpu) 4165 { 4166 struct vcpu_vmx *vmx = to_vmx(vcpu); 4167 int r; 4168 4169 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 4170 if (r < 0) 4171 goto out_vmcs02; 4172 4173 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4174 if (!vmx->nested.cached_vmcs12) 4175 goto out_cached_vmcs12; 4176 4177 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 4178 if (!vmx->nested.cached_shadow_vmcs12) 4179 goto out_cached_shadow_vmcs12; 4180 4181 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 4182 goto out_shadow_vmcs; 4183 4184 hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, 4185 HRTIMER_MODE_REL_PINNED); 4186 vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; 4187 4188 vmx->nested.vpid02 = allocate_vpid(); 4189 4190 vmx->nested.vmcs02_initialized = false; 4191 vmx->nested.vmxon = true; 4192 4193 if (pt_mode == PT_MODE_HOST_GUEST) { 4194 vmx->pt_desc.guest.ctl = 0; 4195 pt_update_intercept_for_msr(vmx); 4196 } 4197 4198 return 0; 4199 4200 out_shadow_vmcs: 4201 kfree(vmx->nested.cached_shadow_vmcs12); 4202 4203 out_cached_shadow_vmcs12: 4204 kfree(vmx->nested.cached_vmcs12); 4205 4206 out_cached_vmcs12: 4207 free_loaded_vmcs(&vmx->nested.vmcs02); 4208 4209 out_vmcs02: 4210 return -ENOMEM; 4211 } 4212 4213 /* 4214 * Emulate the VMXON instruction. 4215 * Currently, we just remember that VMX is active, and do not save or even 4216 * inspect the argument to VMXON (the so-called "VMXON pointer") because we 4217 * do not currently need to store anything in that guest-allocated memory 4218 * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their 4219 * argument is different from the VMXON pointer (which the spec says they do). 4220 */ 4221 static int handle_vmon(struct kvm_vcpu *vcpu) 4222 { 4223 int ret; 4224 gpa_t vmptr; 4225 struct page *page; 4226 struct vcpu_vmx *vmx = to_vmx(vcpu); 4227 const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED 4228 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX; 4229 4230 /* 4231 * The Intel VMX Instruction Reference lists a bunch of bits that are 4232 * prerequisite to running VMXON, most notably cr4.VMXE must be set to 4233 * 1 (see vmx_set_cr4() for when we allow the guest to set this). 4234 * Otherwise, we should fail with #UD. But most faulting conditions 4235 * have already been checked by hardware, prior to the VM-exit for 4236 * VMXON. We do test guest cr4.VMXE because processor CR4 always has 4237 * that bit set to 1 in non-root mode. 4238 */ 4239 if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) { 4240 kvm_queue_exception(vcpu, UD_VECTOR); 4241 return 1; 4242 } 4243 4244 /* CPL=0 must be checked manually. */ 4245 if (vmx_get_cpl(vcpu)) { 4246 kvm_inject_gp(vcpu, 0); 4247 return 1; 4248 } 4249 4250 if (vmx->nested.vmxon) 4251 return nested_vmx_failValid(vcpu, 4252 VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 4253 4254 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 4255 != VMXON_NEEDED_FEATURES) { 4256 kvm_inject_gp(vcpu, 0); 4257 return 1; 4258 } 4259 4260 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4261 return 1; 4262 4263 /* 4264 * SDM 3: 24.11.5 4265 * The first 4 bytes of VMXON region contain the supported 4266 * VMCS revision identifier 4267 * 4268 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 4269 * which replaces physical address width with 32 4270 */ 4271 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4272 return nested_vmx_failInvalid(vcpu); 4273 4274 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4275 if (is_error_page(page)) 4276 return nested_vmx_failInvalid(vcpu); 4277 4278 if (*(u32 *)kmap(page) != VMCS12_REVISION) { 4279 kunmap(page); 4280 kvm_release_page_clean(page); 4281 return nested_vmx_failInvalid(vcpu); 4282 } 4283 kunmap(page); 4284 kvm_release_page_clean(page); 4285 4286 vmx->nested.vmxon_ptr = vmptr; 4287 ret = enter_vmx_operation(vcpu); 4288 if (ret) 4289 return ret; 4290 4291 return nested_vmx_succeed(vcpu); 4292 } 4293 4294 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 4295 { 4296 struct vcpu_vmx *vmx = to_vmx(vcpu); 4297 4298 if (vmx->nested.current_vmptr == -1ull) 4299 return; 4300 4301 if (enable_shadow_vmcs) { 4302 /* copy to memory all shadowed fields in case 4303 they were modified */ 4304 copy_shadow_to_vmcs12(vmx); 4305 vmx->nested.need_vmcs12_sync = false; 4306 vmx_disable_shadow_vmcs(vmx); 4307 } 4308 vmx->nested.posted_intr_nv = -1; 4309 4310 /* Flush VMCS12 to guest memory */ 4311 kvm_vcpu_write_guest_page(vcpu, 4312 vmx->nested.current_vmptr >> PAGE_SHIFT, 4313 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 4314 4315 kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 4316 4317 vmx->nested.current_vmptr = -1ull; 4318 } 4319 4320 /* Emulate the VMXOFF instruction */ 4321 static int handle_vmoff(struct kvm_vcpu *vcpu) 4322 { 4323 if (!nested_vmx_check_permission(vcpu)) 4324 return 1; 4325 free_nested(vcpu); 4326 return nested_vmx_succeed(vcpu); 4327 } 4328 4329 /* Emulate the VMCLEAR instruction */ 4330 static int handle_vmclear(struct kvm_vcpu *vcpu) 4331 { 4332 struct vcpu_vmx *vmx = to_vmx(vcpu); 4333 u32 zero = 0; 4334 gpa_t vmptr; 4335 4336 if (!nested_vmx_check_permission(vcpu)) 4337 return 1; 4338 4339 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4340 return 1; 4341 4342 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4343 return nested_vmx_failValid(vcpu, 4344 VMXERR_VMCLEAR_INVALID_ADDRESS); 4345 4346 if (vmptr == vmx->nested.vmxon_ptr) 4347 return nested_vmx_failValid(vcpu, 4348 VMXERR_VMCLEAR_VMXON_POINTER); 4349 4350 if (vmx->nested.hv_evmcs_page) { 4351 if (vmptr == vmx->nested.hv_evmcs_vmptr) 4352 nested_release_evmcs(vcpu); 4353 } else { 4354 if (vmptr == vmx->nested.current_vmptr) 4355 nested_release_vmcs12(vcpu); 4356 4357 kvm_vcpu_write_guest(vcpu, 4358 vmptr + offsetof(struct vmcs12, 4359 launch_state), 4360 &zero, sizeof(zero)); 4361 } 4362 4363 return nested_vmx_succeed(vcpu); 4364 } 4365 4366 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch); 4367 4368 /* Emulate the VMLAUNCH instruction */ 4369 static int handle_vmlaunch(struct kvm_vcpu *vcpu) 4370 { 4371 return nested_vmx_run(vcpu, true); 4372 } 4373 4374 /* Emulate the VMRESUME instruction */ 4375 static int handle_vmresume(struct kvm_vcpu *vcpu) 4376 { 4377 4378 return nested_vmx_run(vcpu, false); 4379 } 4380 4381 static int handle_vmread(struct kvm_vcpu *vcpu) 4382 { 4383 unsigned long field; 4384 u64 field_value; 4385 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4386 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4387 gva_t gva = 0; 4388 struct vmcs12 *vmcs12; 4389 4390 if (!nested_vmx_check_permission(vcpu)) 4391 return 1; 4392 4393 if (to_vmx(vcpu)->nested.current_vmptr == -1ull) 4394 return nested_vmx_failInvalid(vcpu); 4395 4396 if (!is_guest_mode(vcpu)) 4397 vmcs12 = get_vmcs12(vcpu); 4398 else { 4399 /* 4400 * When vmcs->vmcs_link_pointer is -1ull, any VMREAD 4401 * to shadowed-field sets the ALU flags for VMfailInvalid. 4402 */ 4403 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4404 return nested_vmx_failInvalid(vcpu); 4405 vmcs12 = get_shadow_vmcs12(vcpu); 4406 } 4407 4408 /* Decode instruction info and find the field to read */ 4409 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4410 /* Read the field, zero-extended to a u64 field_value */ 4411 if (vmcs12_read_any(vmcs12, field, &field_value) < 0) 4412 return nested_vmx_failValid(vcpu, 4413 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4414 4415 /* 4416 * Now copy part of this value to register or memory, as requested. 4417 * Note that the number of bits actually copied is 32 or 64 depending 4418 * on the guest's mode (32 or 64 bit), not on the given field's length. 4419 */ 4420 if (vmx_instruction_info & (1u << 10)) { 4421 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf), 4422 field_value); 4423 } else { 4424 if (get_vmx_mem_address(vcpu, exit_qualification, 4425 vmx_instruction_info, true, &gva)) 4426 return 1; 4427 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 4428 kvm_write_guest_virt_system(vcpu, gva, &field_value, 4429 (is_long_mode(vcpu) ? 8 : 4), NULL); 4430 } 4431 4432 return nested_vmx_succeed(vcpu); 4433 } 4434 4435 4436 static int handle_vmwrite(struct kvm_vcpu *vcpu) 4437 { 4438 unsigned long field; 4439 gva_t gva; 4440 struct vcpu_vmx *vmx = to_vmx(vcpu); 4441 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4442 u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4443 4444 /* The value to write might be 32 or 64 bits, depending on L1's long 4445 * mode, and eventually we need to write that into a field of several 4446 * possible lengths. The code below first zero-extends the value to 64 4447 * bit (field_value), and then copies only the appropriate number of 4448 * bits into the vmcs12 field. 4449 */ 4450 u64 field_value = 0; 4451 struct x86_exception e; 4452 struct vmcs12 *vmcs12; 4453 4454 if (!nested_vmx_check_permission(vcpu)) 4455 return 1; 4456 4457 if (vmx->nested.current_vmptr == -1ull) 4458 return nested_vmx_failInvalid(vcpu); 4459 4460 if (vmx_instruction_info & (1u << 10)) 4461 field_value = kvm_register_readl(vcpu, 4462 (((vmx_instruction_info) >> 3) & 0xf)); 4463 else { 4464 if (get_vmx_mem_address(vcpu, exit_qualification, 4465 vmx_instruction_info, false, &gva)) 4466 return 1; 4467 if (kvm_read_guest_virt(vcpu, gva, &field_value, 4468 (is_64_bit_mode(vcpu) ? 8 : 4), &e)) { 4469 kvm_inject_page_fault(vcpu, &e); 4470 return 1; 4471 } 4472 } 4473 4474 4475 field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 4476 /* 4477 * If the vCPU supports "VMWRITE to any supported field in the 4478 * VMCS," then the "read-only" fields are actually read/write. 4479 */ 4480 if (vmcs_field_readonly(field) && 4481 !nested_cpu_has_vmwrite_any_field(vcpu)) 4482 return nested_vmx_failValid(vcpu, 4483 VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 4484 4485 if (!is_guest_mode(vcpu)) 4486 vmcs12 = get_vmcs12(vcpu); 4487 else { 4488 /* 4489 * When vmcs->vmcs_link_pointer is -1ull, any VMWRITE 4490 * to shadowed-field sets the ALU flags for VMfailInvalid. 4491 */ 4492 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) 4493 return nested_vmx_failInvalid(vcpu); 4494 vmcs12 = get_shadow_vmcs12(vcpu); 4495 } 4496 4497 if (vmcs12_write_any(vmcs12, field, field_value) < 0) 4498 return nested_vmx_failValid(vcpu, 4499 VMXERR_UNSUPPORTED_VMCS_COMPONENT); 4500 4501 /* 4502 * Do not track vmcs12 dirty-state if in guest-mode 4503 * as we actually dirty shadow vmcs12 instead of vmcs12. 4504 */ 4505 if (!is_guest_mode(vcpu)) { 4506 switch (field) { 4507 #define SHADOW_FIELD_RW(x) case x: 4508 #include "vmcs_shadow_fields.h" 4509 /* 4510 * The fields that can be updated by L1 without a vmexit are 4511 * always updated in the vmcs02, the others go down the slow 4512 * path of prepare_vmcs02. 4513 */ 4514 break; 4515 default: 4516 vmx->nested.dirty_vmcs12 = true; 4517 break; 4518 } 4519 } 4520 4521 return nested_vmx_succeed(vcpu); 4522 } 4523 4524 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 4525 { 4526 vmx->nested.current_vmptr = vmptr; 4527 if (enable_shadow_vmcs) { 4528 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL, 4529 SECONDARY_EXEC_SHADOW_VMCS); 4530 vmcs_write64(VMCS_LINK_POINTER, 4531 __pa(vmx->vmcs01.shadow_vmcs)); 4532 vmx->nested.need_vmcs12_sync = true; 4533 } 4534 vmx->nested.dirty_vmcs12 = true; 4535 } 4536 4537 /* Emulate the VMPTRLD instruction */ 4538 static int handle_vmptrld(struct kvm_vcpu *vcpu) 4539 { 4540 struct vcpu_vmx *vmx = to_vmx(vcpu); 4541 gpa_t vmptr; 4542 4543 if (!nested_vmx_check_permission(vcpu)) 4544 return 1; 4545 4546 if (nested_vmx_get_vmptr(vcpu, &vmptr)) 4547 return 1; 4548 4549 if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) 4550 return nested_vmx_failValid(vcpu, 4551 VMXERR_VMPTRLD_INVALID_ADDRESS); 4552 4553 if (vmptr == vmx->nested.vmxon_ptr) 4554 return nested_vmx_failValid(vcpu, 4555 VMXERR_VMPTRLD_VMXON_POINTER); 4556 4557 /* Forbid normal VMPTRLD if Enlightened version was used */ 4558 if (vmx->nested.hv_evmcs) 4559 return 1; 4560 4561 if (vmx->nested.current_vmptr != vmptr) { 4562 struct vmcs12 *new_vmcs12; 4563 struct page *page; 4564 4565 page = kvm_vcpu_gpa_to_page(vcpu, vmptr); 4566 if (is_error_page(page)) { 4567 /* 4568 * Reads from an unbacked page return all 1s, 4569 * which means that the 32 bits located at the 4570 * given physical address won't match the required 4571 * VMCS12_REVISION identifier. 4572 */ 4573 return nested_vmx_failValid(vcpu, 4574 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4575 } 4576 new_vmcs12 = kmap(page); 4577 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION || 4578 (new_vmcs12->hdr.shadow_vmcs && 4579 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 4580 kunmap(page); 4581 kvm_release_page_clean(page); 4582 return nested_vmx_failValid(vcpu, 4583 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 4584 } 4585 4586 nested_release_vmcs12(vcpu); 4587 4588 /* 4589 * Load VMCS12 from guest memory since it is not already 4590 * cached. 4591 */ 4592 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE); 4593 kunmap(page); 4594 kvm_release_page_clean(page); 4595 4596 set_current_vmptr(vmx, vmptr); 4597 } 4598 4599 return nested_vmx_succeed(vcpu); 4600 } 4601 4602 /* Emulate the VMPTRST instruction */ 4603 static int handle_vmptrst(struct kvm_vcpu *vcpu) 4604 { 4605 unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION); 4606 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4607 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 4608 struct x86_exception e; 4609 gva_t gva; 4610 4611 if (!nested_vmx_check_permission(vcpu)) 4612 return 1; 4613 4614 if (unlikely(to_vmx(vcpu)->nested.hv_evmcs)) 4615 return 1; 4616 4617 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, &gva)) 4618 return 1; 4619 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 4620 if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 4621 sizeof(gpa_t), &e)) { 4622 kvm_inject_page_fault(vcpu, &e); 4623 return 1; 4624 } 4625 return nested_vmx_succeed(vcpu); 4626 } 4627 4628 /* Emulate the INVEPT instruction */ 4629 static int handle_invept(struct kvm_vcpu *vcpu) 4630 { 4631 struct vcpu_vmx *vmx = to_vmx(vcpu); 4632 u32 vmx_instruction_info, types; 4633 unsigned long type; 4634 gva_t gva; 4635 struct x86_exception e; 4636 struct { 4637 u64 eptp, gpa; 4638 } operand; 4639 4640 if (!(vmx->nested.msrs.secondary_ctls_high & 4641 SECONDARY_EXEC_ENABLE_EPT) || 4642 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 4643 kvm_queue_exception(vcpu, UD_VECTOR); 4644 return 1; 4645 } 4646 4647 if (!nested_vmx_check_permission(vcpu)) 4648 return 1; 4649 4650 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4651 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4652 4653 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 4654 4655 if (type >= 32 || !(types & (1 << type))) 4656 return nested_vmx_failValid(vcpu, 4657 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4658 4659 /* According to the Intel VMX instruction reference, the memory 4660 * operand is read even if it isn't needed (e.g., for type==global) 4661 */ 4662 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4663 vmx_instruction_info, false, &gva)) 4664 return 1; 4665 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4666 kvm_inject_page_fault(vcpu, &e); 4667 return 1; 4668 } 4669 4670 switch (type) { 4671 case VMX_EPT_EXTENT_GLOBAL: 4672 /* 4673 * TODO: track mappings and invalidate 4674 * single context requests appropriately 4675 */ 4676 case VMX_EPT_EXTENT_CONTEXT: 4677 kvm_mmu_sync_roots(vcpu); 4678 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 4679 break; 4680 default: 4681 BUG_ON(1); 4682 break; 4683 } 4684 4685 return nested_vmx_succeed(vcpu); 4686 } 4687 4688 static int handle_invvpid(struct kvm_vcpu *vcpu) 4689 { 4690 struct vcpu_vmx *vmx = to_vmx(vcpu); 4691 u32 vmx_instruction_info; 4692 unsigned long type, types; 4693 gva_t gva; 4694 struct x86_exception e; 4695 struct { 4696 u64 vpid; 4697 u64 gla; 4698 } operand; 4699 u16 vpid02; 4700 4701 if (!(vmx->nested.msrs.secondary_ctls_high & 4702 SECONDARY_EXEC_ENABLE_VPID) || 4703 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 4704 kvm_queue_exception(vcpu, UD_VECTOR); 4705 return 1; 4706 } 4707 4708 if (!nested_vmx_check_permission(vcpu)) 4709 return 1; 4710 4711 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4712 type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf); 4713 4714 types = (vmx->nested.msrs.vpid_caps & 4715 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 4716 4717 if (type >= 32 || !(types & (1 << type))) 4718 return nested_vmx_failValid(vcpu, 4719 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4720 4721 /* according to the intel vmx instruction reference, the memory 4722 * operand is read even if it isn't needed (e.g., for type==global) 4723 */ 4724 if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), 4725 vmx_instruction_info, false, &gva)) 4726 return 1; 4727 if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { 4728 kvm_inject_page_fault(vcpu, &e); 4729 return 1; 4730 } 4731 if (operand.vpid >> 16) 4732 return nested_vmx_failValid(vcpu, 4733 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4734 4735 vpid02 = nested_get_vpid02(vcpu); 4736 switch (type) { 4737 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 4738 if (!operand.vpid || 4739 is_noncanonical_address(operand.gla, vcpu)) 4740 return nested_vmx_failValid(vcpu, 4741 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4742 if (cpu_has_vmx_invvpid_individual_addr()) { 4743 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, 4744 vpid02, operand.gla); 4745 } else 4746 __vmx_flush_tlb(vcpu, vpid02, false); 4747 break; 4748 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 4749 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 4750 if (!operand.vpid) 4751 return nested_vmx_failValid(vcpu, 4752 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 4753 __vmx_flush_tlb(vcpu, vpid02, false); 4754 break; 4755 case VMX_VPID_EXTENT_ALL_CONTEXT: 4756 __vmx_flush_tlb(vcpu, vpid02, false); 4757 break; 4758 default: 4759 WARN_ON_ONCE(1); 4760 return kvm_skip_emulated_instruction(vcpu); 4761 } 4762 4763 return nested_vmx_succeed(vcpu); 4764 } 4765 4766 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 4767 struct vmcs12 *vmcs12) 4768 { 4769 u32 index = vcpu->arch.regs[VCPU_REGS_RCX]; 4770 u64 address; 4771 bool accessed_dirty; 4772 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 4773 4774 if (!nested_cpu_has_eptp_switching(vmcs12) || 4775 !nested_cpu_has_ept(vmcs12)) 4776 return 1; 4777 4778 if (index >= VMFUNC_EPTP_ENTRIES) 4779 return 1; 4780 4781 4782 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 4783 &address, index * 8, 8)) 4784 return 1; 4785 4786 accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT); 4787 4788 /* 4789 * If the (L2) guest does a vmfunc to the currently 4790 * active ept pointer, we don't have to do anything else 4791 */ 4792 if (vmcs12->ept_pointer != address) { 4793 if (!valid_ept_address(vcpu, address)) 4794 return 1; 4795 4796 kvm_mmu_unload(vcpu); 4797 mmu->ept_ad = accessed_dirty; 4798 mmu->mmu_role.base.ad_disabled = !accessed_dirty; 4799 vmcs12->ept_pointer = address; 4800 /* 4801 * TODO: Check what's the correct approach in case 4802 * mmu reload fails. Currently, we just let the next 4803 * reload potentially fail 4804 */ 4805 kvm_mmu_reload(vcpu); 4806 } 4807 4808 return 0; 4809 } 4810 4811 static int handle_vmfunc(struct kvm_vcpu *vcpu) 4812 { 4813 struct vcpu_vmx *vmx = to_vmx(vcpu); 4814 struct vmcs12 *vmcs12; 4815 u32 function = vcpu->arch.regs[VCPU_REGS_RAX]; 4816 4817 /* 4818 * VMFUNC is only supported for nested guests, but we always enable the 4819 * secondary control for simplicity; for non-nested mode, fake that we 4820 * didn't by injecting #UD. 4821 */ 4822 if (!is_guest_mode(vcpu)) { 4823 kvm_queue_exception(vcpu, UD_VECTOR); 4824 return 1; 4825 } 4826 4827 vmcs12 = get_vmcs12(vcpu); 4828 if ((vmcs12->vm_function_control & (1 << function)) == 0) 4829 goto fail; 4830 4831 switch (function) { 4832 case 0: 4833 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 4834 goto fail; 4835 break; 4836 default: 4837 goto fail; 4838 } 4839 return kvm_skip_emulated_instruction(vcpu); 4840 4841 fail: 4842 nested_vmx_vmexit(vcpu, vmx->exit_reason, 4843 vmcs_read32(VM_EXIT_INTR_INFO), 4844 vmcs_readl(EXIT_QUALIFICATION)); 4845 return 1; 4846 } 4847 4848 4849 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 4850 struct vmcs12 *vmcs12) 4851 { 4852 unsigned long exit_qualification; 4853 gpa_t bitmap, last_bitmap; 4854 unsigned int port; 4855 int size; 4856 u8 b; 4857 4858 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 4859 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 4860 4861 exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4862 4863 port = exit_qualification >> 16; 4864 size = (exit_qualification & 7) + 1; 4865 4866 last_bitmap = (gpa_t)-1; 4867 b = -1; 4868 4869 while (size > 0) { 4870 if (port < 0x8000) 4871 bitmap = vmcs12->io_bitmap_a; 4872 else if (port < 0x10000) 4873 bitmap = vmcs12->io_bitmap_b; 4874 else 4875 return true; 4876 bitmap += (port & 0x7fff) / 8; 4877 4878 if (last_bitmap != bitmap) 4879 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 4880 return true; 4881 if (b & (1 << (port & 7))) 4882 return true; 4883 4884 port++; 4885 size--; 4886 last_bitmap = bitmap; 4887 } 4888 4889 return false; 4890 } 4891 4892 /* 4893 * Return 1 if we should exit from L2 to L1 to handle an MSR access access, 4894 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 4895 * disinterest in the current event (read or write a specific MSR) by using an 4896 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 4897 */ 4898 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 4899 struct vmcs12 *vmcs12, u32 exit_reason) 4900 { 4901 u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX]; 4902 gpa_t bitmap; 4903 4904 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 4905 return true; 4906 4907 /* 4908 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 4909 * for the four combinations of read/write and low/high MSR numbers. 4910 * First we need to figure out which of the four to use: 4911 */ 4912 bitmap = vmcs12->msr_bitmap; 4913 if (exit_reason == EXIT_REASON_MSR_WRITE) 4914 bitmap += 2048; 4915 if (msr_index >= 0xc0000000) { 4916 msr_index -= 0xc0000000; 4917 bitmap += 1024; 4918 } 4919 4920 /* Then read the msr_index'th bit from this bitmap: */ 4921 if (msr_index < 1024*8) { 4922 unsigned char b; 4923 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 4924 return true; 4925 return 1 & (b >> (msr_index & 7)); 4926 } else 4927 return true; /* let L1 handle the wrong parameter */ 4928 } 4929 4930 /* 4931 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 4932 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 4933 * intercept (via guest_host_mask etc.) the current event. 4934 */ 4935 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 4936 struct vmcs12 *vmcs12) 4937 { 4938 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); 4939 int cr = exit_qualification & 15; 4940 int reg; 4941 unsigned long val; 4942 4943 switch ((exit_qualification >> 4) & 3) { 4944 case 0: /* mov to cr */ 4945 reg = (exit_qualification >> 8) & 15; 4946 val = kvm_register_readl(vcpu, reg); 4947 switch (cr) { 4948 case 0: 4949 if (vmcs12->cr0_guest_host_mask & 4950 (val ^ vmcs12->cr0_read_shadow)) 4951 return true; 4952 break; 4953 case 3: 4954 if ((vmcs12->cr3_target_count >= 1 && 4955 vmcs12->cr3_target_value0 == val) || 4956 (vmcs12->cr3_target_count >= 2 && 4957 vmcs12->cr3_target_value1 == val) || 4958 (vmcs12->cr3_target_count >= 3 && 4959 vmcs12->cr3_target_value2 == val) || 4960 (vmcs12->cr3_target_count >= 4 && 4961 vmcs12->cr3_target_value3 == val)) 4962 return false; 4963 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 4964 return true; 4965 break; 4966 case 4: 4967 if (vmcs12->cr4_guest_host_mask & 4968 (vmcs12->cr4_read_shadow ^ val)) 4969 return true; 4970 break; 4971 case 8: 4972 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 4973 return true; 4974 break; 4975 } 4976 break; 4977 case 2: /* clts */ 4978 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 4979 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 4980 return true; 4981 break; 4982 case 1: /* mov from cr */ 4983 switch (cr) { 4984 case 3: 4985 if (vmcs12->cpu_based_vm_exec_control & 4986 CPU_BASED_CR3_STORE_EXITING) 4987 return true; 4988 break; 4989 case 8: 4990 if (vmcs12->cpu_based_vm_exec_control & 4991 CPU_BASED_CR8_STORE_EXITING) 4992 return true; 4993 break; 4994 } 4995 break; 4996 case 3: /* lmsw */ 4997 /* 4998 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 4999 * cr0. Other attempted changes are ignored, with no exit. 5000 */ 5001 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 5002 if (vmcs12->cr0_guest_host_mask & 0xe & 5003 (val ^ vmcs12->cr0_read_shadow)) 5004 return true; 5005 if ((vmcs12->cr0_guest_host_mask & 0x1) && 5006 !(vmcs12->cr0_read_shadow & 0x1) && 5007 (val & 0x1)) 5008 return true; 5009 break; 5010 } 5011 return false; 5012 } 5013 5014 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 5015 struct vmcs12 *vmcs12, gpa_t bitmap) 5016 { 5017 u32 vmx_instruction_info; 5018 unsigned long field; 5019 u8 b; 5020 5021 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 5022 return true; 5023 5024 /* Decode instruction info and find the field to access */ 5025 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5026 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 5027 5028 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 5029 if (field >> 15) 5030 return true; 5031 5032 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 5033 return true; 5034 5035 return 1 & (b >> (field & 7)); 5036 } 5037 5038 /* 5039 * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we 5040 * should handle it ourselves in L0 (and then continue L2). Only call this 5041 * when in is_guest_mode (L2). 5042 */ 5043 bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) 5044 { 5045 u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO); 5046 struct vcpu_vmx *vmx = to_vmx(vcpu); 5047 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5048 5049 if (vmx->nested.nested_run_pending) 5050 return false; 5051 5052 if (unlikely(vmx->fail)) { 5053 pr_info_ratelimited("%s failed vm entry %x\n", __func__, 5054 vmcs_read32(VM_INSTRUCTION_ERROR)); 5055 return true; 5056 } 5057 5058 /* 5059 * The host physical addresses of some pages of guest memory 5060 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC 5061 * Page). The CPU may write to these pages via their host 5062 * physical address while L2 is running, bypassing any 5063 * address-translation-based dirty tracking (e.g. EPT write 5064 * protection). 5065 * 5066 * Mark them dirty on every exit from L2 to prevent them from 5067 * getting out of sync with dirty tracking. 5068 */ 5069 nested_mark_vmcs12_pages_dirty(vcpu); 5070 5071 trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, 5072 vmcs_readl(EXIT_QUALIFICATION), 5073 vmx->idt_vectoring_info, 5074 intr_info, 5075 vmcs_read32(VM_EXIT_INTR_ERROR_CODE), 5076 KVM_ISA_VMX); 5077 5078 switch (exit_reason) { 5079 case EXIT_REASON_EXCEPTION_NMI: 5080 if (is_nmi(intr_info)) 5081 return false; 5082 else if (is_page_fault(intr_info)) 5083 return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept; 5084 else if (is_debug(intr_info) && 5085 vcpu->guest_debug & 5086 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 5087 return false; 5088 else if (is_breakpoint(intr_info) && 5089 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 5090 return false; 5091 return vmcs12->exception_bitmap & 5092 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 5093 case EXIT_REASON_EXTERNAL_INTERRUPT: 5094 return false; 5095 case EXIT_REASON_TRIPLE_FAULT: 5096 return true; 5097 case EXIT_REASON_PENDING_INTERRUPT: 5098 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING); 5099 case EXIT_REASON_NMI_WINDOW: 5100 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING); 5101 case EXIT_REASON_TASK_SWITCH: 5102 return true; 5103 case EXIT_REASON_CPUID: 5104 return true; 5105 case EXIT_REASON_HLT: 5106 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 5107 case EXIT_REASON_INVD: 5108 return true; 5109 case EXIT_REASON_INVLPG: 5110 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5111 case EXIT_REASON_RDPMC: 5112 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 5113 case EXIT_REASON_RDRAND: 5114 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 5115 case EXIT_REASON_RDSEED: 5116 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 5117 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 5118 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 5119 case EXIT_REASON_VMREAD: 5120 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5121 vmcs12->vmread_bitmap); 5122 case EXIT_REASON_VMWRITE: 5123 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 5124 vmcs12->vmwrite_bitmap); 5125 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 5126 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 5127 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 5128 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 5129 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 5130 /* 5131 * VMX instructions trap unconditionally. This allows L1 to 5132 * emulate them for its L2 guest, i.e., allows 3-level nesting! 5133 */ 5134 return true; 5135 case EXIT_REASON_CR_ACCESS: 5136 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 5137 case EXIT_REASON_DR_ACCESS: 5138 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 5139 case EXIT_REASON_IO_INSTRUCTION: 5140 return nested_vmx_exit_handled_io(vcpu, vmcs12); 5141 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 5142 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 5143 case EXIT_REASON_MSR_READ: 5144 case EXIT_REASON_MSR_WRITE: 5145 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 5146 case EXIT_REASON_INVALID_STATE: 5147 return true; 5148 case EXIT_REASON_MWAIT_INSTRUCTION: 5149 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 5150 case EXIT_REASON_MONITOR_TRAP_FLAG: 5151 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG); 5152 case EXIT_REASON_MONITOR_INSTRUCTION: 5153 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 5154 case EXIT_REASON_PAUSE_INSTRUCTION: 5155 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 5156 nested_cpu_has2(vmcs12, 5157 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 5158 case EXIT_REASON_MCE_DURING_VMENTRY: 5159 return false; 5160 case EXIT_REASON_TPR_BELOW_THRESHOLD: 5161 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 5162 case EXIT_REASON_APIC_ACCESS: 5163 case EXIT_REASON_APIC_WRITE: 5164 case EXIT_REASON_EOI_INDUCED: 5165 /* 5166 * The controls for "virtualize APIC accesses," "APIC- 5167 * register virtualization," and "virtual-interrupt 5168 * delivery" only come from vmcs12. 5169 */ 5170 return true; 5171 case EXIT_REASON_EPT_VIOLATION: 5172 /* 5173 * L0 always deals with the EPT violation. If nested EPT is 5174 * used, and the nested mmu code discovers that the address is 5175 * missing in the guest EPT table (EPT12), the EPT violation 5176 * will be injected with nested_ept_inject_page_fault() 5177 */ 5178 return false; 5179 case EXIT_REASON_EPT_MISCONFIG: 5180 /* 5181 * L2 never uses directly L1's EPT, but rather L0's own EPT 5182 * table (shadow on EPT) or a merged EPT table that L0 built 5183 * (EPT on EPT). So any problems with the structure of the 5184 * table is L0's fault. 5185 */ 5186 return false; 5187 case EXIT_REASON_INVPCID: 5188 return 5189 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 5190 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 5191 case EXIT_REASON_WBINVD: 5192 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 5193 case EXIT_REASON_XSETBV: 5194 return true; 5195 case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS: 5196 /* 5197 * This should never happen, since it is not possible to 5198 * set XSS to a non-zero value---neither in L1 nor in L2. 5199 * If if it were, XSS would have to be checked against 5200 * the XSS exit bitmap in vmcs12. 5201 */ 5202 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); 5203 case EXIT_REASON_PREEMPTION_TIMER: 5204 return false; 5205 case EXIT_REASON_PML_FULL: 5206 /* We emulate PML support to L1. */ 5207 return false; 5208 case EXIT_REASON_VMFUNC: 5209 /* VM functions are emulated through L2->L0 vmexits. */ 5210 return false; 5211 case EXIT_REASON_ENCLS: 5212 /* SGX is never exposed to L1 */ 5213 return false; 5214 default: 5215 return true; 5216 } 5217 } 5218 5219 5220 static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 5221 struct kvm_nested_state __user *user_kvm_nested_state, 5222 u32 user_data_size) 5223 { 5224 struct vcpu_vmx *vmx; 5225 struct vmcs12 *vmcs12; 5226 struct kvm_nested_state kvm_state = { 5227 .flags = 0, 5228 .format = 0, 5229 .size = sizeof(kvm_state), 5230 .vmx.vmxon_pa = -1ull, 5231 .vmx.vmcs_pa = -1ull, 5232 }; 5233 5234 if (!vcpu) 5235 return kvm_state.size + 2 * VMCS12_SIZE; 5236 5237 vmx = to_vmx(vcpu); 5238 vmcs12 = get_vmcs12(vcpu); 5239 5240 if (nested_vmx_allowed(vcpu) && vmx->nested.enlightened_vmcs_enabled) 5241 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 5242 5243 if (nested_vmx_allowed(vcpu) && 5244 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 5245 kvm_state.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 5246 kvm_state.vmx.vmcs_pa = vmx->nested.current_vmptr; 5247 5248 if (vmx_has_valid_vmcs12(vcpu)) { 5249 kvm_state.size += VMCS12_SIZE; 5250 5251 if (is_guest_mode(vcpu) && 5252 nested_cpu_has_shadow_vmcs(vmcs12) && 5253 vmcs12->vmcs_link_pointer != -1ull) 5254 kvm_state.size += VMCS12_SIZE; 5255 } 5256 5257 if (vmx->nested.smm.vmxon) 5258 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 5259 5260 if (vmx->nested.smm.guest_mode) 5261 kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 5262 5263 if (is_guest_mode(vcpu)) { 5264 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 5265 5266 if (vmx->nested.nested_run_pending) 5267 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 5268 } 5269 } 5270 5271 if (user_data_size < kvm_state.size) 5272 goto out; 5273 5274 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 5275 return -EFAULT; 5276 5277 if (!vmx_has_valid_vmcs12(vcpu)) 5278 goto out; 5279 5280 /* 5281 * When running L2, the authoritative vmcs12 state is in the 5282 * vmcs02. When running L1, the authoritative vmcs12 state is 5283 * in the shadow or enlightened vmcs linked to vmcs01, unless 5284 * need_vmcs12_sync is set, in which case, the authoritative 5285 * vmcs12 state is in the vmcs12 already. 5286 */ 5287 if (is_guest_mode(vcpu)) { 5288 sync_vmcs12(vcpu, vmcs12); 5289 } else if (!vmx->nested.need_vmcs12_sync) { 5290 if (vmx->nested.hv_evmcs) 5291 copy_enlightened_to_vmcs12(vmx); 5292 else if (enable_shadow_vmcs) 5293 copy_shadow_to_vmcs12(vmx); 5294 } 5295 5296 /* 5297 * Copy over the full allocated size of vmcs12 rather than just the size 5298 * of the struct. 5299 */ 5300 if (copy_to_user(user_kvm_nested_state->data, vmcs12, VMCS12_SIZE)) 5301 return -EFAULT; 5302 5303 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5304 vmcs12->vmcs_link_pointer != -1ull) { 5305 if (copy_to_user(user_kvm_nested_state->data + VMCS12_SIZE, 5306 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 5307 return -EFAULT; 5308 } 5309 5310 out: 5311 return kvm_state.size; 5312 } 5313 5314 /* 5315 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 5316 */ 5317 void vmx_leave_nested(struct kvm_vcpu *vcpu) 5318 { 5319 if (is_guest_mode(vcpu)) { 5320 to_vmx(vcpu)->nested.nested_run_pending = 0; 5321 nested_vmx_vmexit(vcpu, -1, 0, 0); 5322 } 5323 free_nested(vcpu); 5324 } 5325 5326 static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 5327 struct kvm_nested_state __user *user_kvm_nested_state, 5328 struct kvm_nested_state *kvm_state) 5329 { 5330 struct vcpu_vmx *vmx = to_vmx(vcpu); 5331 struct vmcs12 *vmcs12; 5332 u32 exit_qual; 5333 int ret; 5334 5335 if (kvm_state->format != 0) 5336 return -EINVAL; 5337 5338 if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) 5339 nested_enable_evmcs(vcpu, NULL); 5340 5341 if (!nested_vmx_allowed(vcpu)) 5342 return kvm_state->vmx.vmxon_pa == -1ull ? 0 : -EINVAL; 5343 5344 if (kvm_state->vmx.vmxon_pa == -1ull) { 5345 if (kvm_state->vmx.smm.flags) 5346 return -EINVAL; 5347 5348 if (kvm_state->vmx.vmcs_pa != -1ull) 5349 return -EINVAL; 5350 5351 vmx_leave_nested(vcpu); 5352 return 0; 5353 } 5354 5355 if (!page_address_valid(vcpu, kvm_state->vmx.vmxon_pa)) 5356 return -EINVAL; 5357 5358 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5359 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5360 return -EINVAL; 5361 5362 if (kvm_state->vmx.smm.flags & 5363 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 5364 return -EINVAL; 5365 5366 /* 5367 * SMM temporarily disables VMX, so we cannot be in guest mode, 5368 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 5369 * must be zero. 5370 */ 5371 if (is_smm(vcpu) ? kvm_state->flags : kvm_state->vmx.smm.flags) 5372 return -EINVAL; 5373 5374 if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 5375 !(kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 5376 return -EINVAL; 5377 5378 vmx_leave_nested(vcpu); 5379 if (kvm_state->vmx.vmxon_pa == -1ull) 5380 return 0; 5381 5382 vmx->nested.vmxon_ptr = kvm_state->vmx.vmxon_pa; 5383 ret = enter_vmx_operation(vcpu); 5384 if (ret) 5385 return ret; 5386 5387 /* Empty 'VMXON' state is permitted */ 5388 if (kvm_state->size < sizeof(kvm_state) + sizeof(*vmcs12)) 5389 return 0; 5390 5391 if (kvm_state->vmx.vmcs_pa != -1ull) { 5392 if (kvm_state->vmx.vmcs_pa == kvm_state->vmx.vmxon_pa || 5393 !page_address_valid(vcpu, kvm_state->vmx.vmcs_pa)) 5394 return -EINVAL; 5395 5396 set_current_vmptr(vmx, kvm_state->vmx.vmcs_pa); 5397 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 5398 /* 5399 * Sync eVMCS upon entry as we may not have 5400 * HV_X64_MSR_VP_ASSIST_PAGE set up yet. 5401 */ 5402 vmx->nested.need_vmcs12_sync = true; 5403 } else { 5404 return -EINVAL; 5405 } 5406 5407 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 5408 vmx->nested.smm.vmxon = true; 5409 vmx->nested.vmxon = false; 5410 5411 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 5412 vmx->nested.smm.guest_mode = true; 5413 } 5414 5415 vmcs12 = get_vmcs12(vcpu); 5416 if (copy_from_user(vmcs12, user_kvm_nested_state->data, sizeof(*vmcs12))) 5417 return -EFAULT; 5418 5419 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 5420 return -EINVAL; 5421 5422 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 5423 return 0; 5424 5425 vmx->nested.nested_run_pending = 5426 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 5427 5428 if (nested_cpu_has_shadow_vmcs(vmcs12) && 5429 vmcs12->vmcs_link_pointer != -1ull) { 5430 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 5431 5432 if (kvm_state->size < sizeof(kvm_state) + 2 * sizeof(*vmcs12)) 5433 return -EINVAL; 5434 5435 if (copy_from_user(shadow_vmcs12, 5436 user_kvm_nested_state->data + VMCS12_SIZE, 5437 sizeof(*vmcs12))) 5438 return -EFAULT; 5439 5440 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 5441 !shadow_vmcs12->hdr.shadow_vmcs) 5442 return -EINVAL; 5443 } 5444 5445 if (nested_vmx_check_vmentry_prereqs(vcpu, vmcs12) || 5446 nested_vmx_check_vmentry_postreqs(vcpu, vmcs12, &exit_qual)) 5447 return -EINVAL; 5448 5449 vmx->nested.dirty_vmcs12 = true; 5450 ret = nested_vmx_enter_non_root_mode(vcpu, false); 5451 if (ret) 5452 return -EINVAL; 5453 5454 return 0; 5455 } 5456 5457 void nested_vmx_vcpu_setup(void) 5458 { 5459 if (enable_shadow_vmcs) { 5460 /* 5461 * At vCPU creation, "VMWRITE to any supported field 5462 * in the VMCS" is supported, so use the more 5463 * permissive vmx_vmread_bitmap to specify both read 5464 * and write permissions for the shadow VMCS. 5465 */ 5466 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 5467 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmread_bitmap)); 5468 } 5469 } 5470 5471 /* 5472 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 5473 * returned for the various VMX controls MSRs when nested VMX is enabled. 5474 * The same values should also be used to verify that vmcs12 control fields are 5475 * valid during nested entry from L1 to L2. 5476 * Each of these control msrs has a low and high 32-bit half: A low bit is on 5477 * if the corresponding bit in the (32-bit) control field *must* be on, and a 5478 * bit in the high half is on if the corresponding bit in the control field 5479 * may be on. See also vmx_control_verify(). 5480 */ 5481 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps, 5482 bool apicv) 5483 { 5484 /* 5485 * Note that as a general rule, the high half of the MSRs (bits in 5486 * the control fields which may be 1) should be initialized by the 5487 * intersection of the underlying hardware's MSR (i.e., features which 5488 * can be supported) and the list of features we want to expose - 5489 * because they are known to be properly supported in our code. 5490 * Also, usually, the low half of the MSRs (bits which must be 1) can 5491 * be set to 0, meaning that L1 may turn off any of these bits. The 5492 * reason is that if one of these bits is necessary, it will appear 5493 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 5494 * fields of vmcs01 and vmcs02, will turn these bits off - and 5495 * nested_vmx_exit_reflected() will not pass related exits to L1. 5496 * These rules have exceptions below. 5497 */ 5498 5499 /* pin-based controls */ 5500 rdmsr(MSR_IA32_VMX_PINBASED_CTLS, 5501 msrs->pinbased_ctls_low, 5502 msrs->pinbased_ctls_high); 5503 msrs->pinbased_ctls_low |= 5504 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5505 msrs->pinbased_ctls_high &= 5506 PIN_BASED_EXT_INTR_MASK | 5507 PIN_BASED_NMI_EXITING | 5508 PIN_BASED_VIRTUAL_NMIS | 5509 (apicv ? PIN_BASED_POSTED_INTR : 0); 5510 msrs->pinbased_ctls_high |= 5511 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5512 PIN_BASED_VMX_PREEMPTION_TIMER; 5513 5514 /* exit controls */ 5515 rdmsr(MSR_IA32_VMX_EXIT_CTLS, 5516 msrs->exit_ctls_low, 5517 msrs->exit_ctls_high); 5518 msrs->exit_ctls_low = 5519 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 5520 5521 msrs->exit_ctls_high &= 5522 #ifdef CONFIG_X86_64 5523 VM_EXIT_HOST_ADDR_SPACE_SIZE | 5524 #endif 5525 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT; 5526 msrs->exit_ctls_high |= 5527 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 5528 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 5529 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; 5530 5531 /* We support free control of debug control saving. */ 5532 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 5533 5534 /* entry controls */ 5535 rdmsr(MSR_IA32_VMX_ENTRY_CTLS, 5536 msrs->entry_ctls_low, 5537 msrs->entry_ctls_high); 5538 msrs->entry_ctls_low = 5539 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 5540 msrs->entry_ctls_high &= 5541 #ifdef CONFIG_X86_64 5542 VM_ENTRY_IA32E_MODE | 5543 #endif 5544 VM_ENTRY_LOAD_IA32_PAT; 5545 msrs->entry_ctls_high |= 5546 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); 5547 5548 /* We support free control of debug control loading. */ 5549 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 5550 5551 /* cpu-based controls */ 5552 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, 5553 msrs->procbased_ctls_low, 5554 msrs->procbased_ctls_high); 5555 msrs->procbased_ctls_low = 5556 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 5557 msrs->procbased_ctls_high &= 5558 CPU_BASED_VIRTUAL_INTR_PENDING | 5559 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING | 5560 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 5561 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 5562 CPU_BASED_CR3_STORE_EXITING | 5563 #ifdef CONFIG_X86_64 5564 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 5565 #endif 5566 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 5567 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 5568 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 5569 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 5570 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 5571 /* 5572 * We can allow some features even when not supported by the 5573 * hardware. For example, L1 can specify an MSR bitmap - and we 5574 * can use it to avoid exits to L1 - even when L0 runs L2 5575 * without MSR bitmaps. 5576 */ 5577 msrs->procbased_ctls_high |= 5578 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 5579 CPU_BASED_USE_MSR_BITMAPS; 5580 5581 /* We support free control of CR3 access interception. */ 5582 msrs->procbased_ctls_low &= 5583 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 5584 5585 /* 5586 * secondary cpu-based controls. Do not include those that 5587 * depend on CPUID bits, they are added later by vmx_cpuid_update. 5588 */ 5589 if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) 5590 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, 5591 msrs->secondary_ctls_low, 5592 msrs->secondary_ctls_high); 5593 5594 msrs->secondary_ctls_low = 0; 5595 msrs->secondary_ctls_high &= 5596 SECONDARY_EXEC_DESC | 5597 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 5598 SECONDARY_EXEC_APIC_REGISTER_VIRT | 5599 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 5600 SECONDARY_EXEC_WBINVD_EXITING; 5601 5602 /* 5603 * We can emulate "VMCS shadowing," even if the hardware 5604 * doesn't support it. 5605 */ 5606 msrs->secondary_ctls_high |= 5607 SECONDARY_EXEC_SHADOW_VMCS; 5608 5609 if (enable_ept) { 5610 /* nested EPT: emulate EPT also to L1 */ 5611 msrs->secondary_ctls_high |= 5612 SECONDARY_EXEC_ENABLE_EPT; 5613 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT | 5614 VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT; 5615 if (cpu_has_vmx_ept_execute_only()) 5616 msrs->ept_caps |= 5617 VMX_EPT_EXECUTE_ONLY_BIT; 5618 msrs->ept_caps &= ept_caps; 5619 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 5620 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 5621 VMX_EPT_1GB_PAGE_BIT; 5622 if (enable_ept_ad_bits) { 5623 msrs->secondary_ctls_high |= 5624 SECONDARY_EXEC_ENABLE_PML; 5625 msrs->ept_caps |= VMX_EPT_AD_BIT; 5626 } 5627 } 5628 5629 if (cpu_has_vmx_vmfunc()) { 5630 msrs->secondary_ctls_high |= 5631 SECONDARY_EXEC_ENABLE_VMFUNC; 5632 /* 5633 * Advertise EPTP switching unconditionally 5634 * since we emulate it 5635 */ 5636 if (enable_ept) 5637 msrs->vmfunc_controls = 5638 VMX_VMFUNC_EPTP_SWITCHING; 5639 } 5640 5641 /* 5642 * Old versions of KVM use the single-context version without 5643 * checking for support, so declare that it is supported even 5644 * though it is treated as global context. The alternative is 5645 * not failing the single-context invvpid, and it is worse. 5646 */ 5647 if (enable_vpid) { 5648 msrs->secondary_ctls_high |= 5649 SECONDARY_EXEC_ENABLE_VPID; 5650 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 5651 VMX_VPID_EXTENT_SUPPORTED_MASK; 5652 } 5653 5654 if (enable_unrestricted_guest) 5655 msrs->secondary_ctls_high |= 5656 SECONDARY_EXEC_UNRESTRICTED_GUEST; 5657 5658 if (flexpriority_enabled) 5659 msrs->secondary_ctls_high |= 5660 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 5661 5662 /* miscellaneous data */ 5663 rdmsr(MSR_IA32_VMX_MISC, 5664 msrs->misc_low, 5665 msrs->misc_high); 5666 msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; 5667 msrs->misc_low |= 5668 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 5669 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 5670 VMX_MISC_ACTIVITY_HLT; 5671 msrs->misc_high = 0; 5672 5673 /* 5674 * This MSR reports some information about VMX support. We 5675 * should return information about the VMX we emulate for the 5676 * guest, and the VMCS structure we give it - not about the 5677 * VMX support of the underlying hardware. 5678 */ 5679 msrs->basic = 5680 VMCS12_REVISION | 5681 VMX_BASIC_TRUE_CTLS | 5682 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) | 5683 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT); 5684 5685 if (cpu_has_vmx_basic_inout()) 5686 msrs->basic |= VMX_BASIC_INOUT; 5687 5688 /* 5689 * These MSRs specify bits which the guest must keep fixed on 5690 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 5691 * We picked the standard core2 setting. 5692 */ 5693 #define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 5694 #define VMXON_CR4_ALWAYSON X86_CR4_VMXE 5695 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 5696 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 5697 5698 /* These MSRs specify bits which the guest must keep fixed off. */ 5699 rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 5700 rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 5701 5702 /* highest index: VMX_PREEMPTION_TIMER_VALUE */ 5703 msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1; 5704 } 5705 5706 void nested_vmx_hardware_unsetup(void) 5707 { 5708 int i; 5709 5710 if (enable_shadow_vmcs) { 5711 for (i = 0; i < VMX_BITMAP_NR; i++) 5712 free_page((unsigned long)vmx_bitmap[i]); 5713 } 5714 } 5715 5716 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 5717 { 5718 int i; 5719 5720 if (!cpu_has_vmx_shadow_vmcs()) 5721 enable_shadow_vmcs = 0; 5722 if (enable_shadow_vmcs) { 5723 for (i = 0; i < VMX_BITMAP_NR; i++) { 5724 /* 5725 * The vmx_bitmap is not tied to a VM and so should 5726 * not be charged to a memcg. 5727 */ 5728 vmx_bitmap[i] = (unsigned long *) 5729 __get_free_page(GFP_KERNEL); 5730 if (!vmx_bitmap[i]) { 5731 nested_vmx_hardware_unsetup(); 5732 return -ENOMEM; 5733 } 5734 } 5735 5736 init_vmcs_shadow_fields(); 5737 } 5738 5739 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear, 5740 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch, 5741 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld, 5742 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst, 5743 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread, 5744 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume, 5745 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite, 5746 exit_handlers[EXIT_REASON_VMOFF] = handle_vmoff, 5747 exit_handlers[EXIT_REASON_VMON] = handle_vmon, 5748 exit_handlers[EXIT_REASON_INVEPT] = handle_invept, 5749 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid, 5750 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc, 5751 5752 kvm_x86_ops->check_nested_events = vmx_check_nested_events; 5753 kvm_x86_ops->get_nested_state = vmx_get_nested_state; 5754 kvm_x86_ops->set_nested_state = vmx_set_nested_state; 5755 kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages, 5756 kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs; 5757 kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version; 5758 5759 return 0; 5760 } 5761