1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * AMD SVM support 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 9 * 10 * Authors: 11 * Yaniv Kamay <yaniv@qumranet.com> 12 * Avi Kivity <avi@qumranet.com> 13 */ 14 15 #define pr_fmt(fmt) "SVM: " fmt 16 17 #include <linux/kvm_types.h> 18 #include <linux/kvm_host.h> 19 #include <linux/kernel.h> 20 21 #include <asm/msr-index.h> 22 #include <asm/debugreg.h> 23 24 #include "kvm_emulate.h" 25 #include "trace.h" 26 #include "mmu.h" 27 #include "x86.h" 28 #include "cpuid.h" 29 #include "lapic.h" 30 #include "svm.h" 31 32 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 33 34 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, 35 struct x86_exception *fault) 36 { 37 struct vcpu_svm *svm = to_svm(vcpu); 38 39 if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { 40 /* 41 * TODO: track the cause of the nested page fault, and 42 * correctly fill in the high bits of exit_info_1. 43 */ 44 svm->vmcb->control.exit_code = SVM_EXIT_NPF; 45 svm->vmcb->control.exit_code_hi = 0; 46 svm->vmcb->control.exit_info_1 = (1ULL << 32); 47 svm->vmcb->control.exit_info_2 = fault->address; 48 } 49 50 svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; 51 svm->vmcb->control.exit_info_1 |= fault->error_code; 52 53 nested_svm_vmexit(svm); 54 } 55 56 static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) 57 { 58 struct vcpu_svm *svm = to_svm(vcpu); 59 WARN_ON(!is_guest_mode(vcpu)); 60 61 if (vmcb12_is_intercept(&svm->nested.ctl, 62 INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && 63 !svm->nested.nested_run_pending) { 64 svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; 65 svm->vmcb->control.exit_code_hi = 0; 66 svm->vmcb->control.exit_info_1 = fault->error_code; 67 svm->vmcb->control.exit_info_2 = fault->address; 68 nested_svm_vmexit(svm); 69 } else { 70 kvm_inject_page_fault(vcpu, fault); 71 } 72 } 73 74 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) 75 { 76 struct vcpu_svm *svm = to_svm(vcpu); 77 u64 cr3 = svm->nested.ctl.nested_cr3; 78 u64 pdpte; 79 int ret; 80 81 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte, 82 offset_in_page(cr3) + index * 8, 8); 83 if (ret) 84 return 0; 85 return pdpte; 86 } 87 88 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) 89 { 90 struct vcpu_svm *svm = to_svm(vcpu); 91 92 return svm->nested.ctl.nested_cr3; 93 } 94 95 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) 96 { 97 struct vcpu_svm *svm = to_svm(vcpu); 98 99 WARN_ON(mmu_is_nested(vcpu)); 100 101 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 102 103 /* 104 * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, 105 * when called via KVM_SET_NESTED_STATE, that state may _not_ match current 106 * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. 107 */ 108 kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4, 109 svm->vmcb01.ptr->save.efer, 110 svm->nested.ctl.nested_cr3); 111 vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; 112 vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; 113 vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; 114 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 115 } 116 117 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) 118 { 119 vcpu->arch.mmu = &vcpu->arch.root_mmu; 120 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 121 } 122 123 void recalc_intercepts(struct vcpu_svm *svm) 124 { 125 struct vmcb_control_area *c, *h; 126 struct vmcb_ctrl_area_cached *g; 127 unsigned int i; 128 129 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 130 131 if (!is_guest_mode(&svm->vcpu)) 132 return; 133 134 c = &svm->vmcb->control; 135 h = &svm->vmcb01.ptr->control; 136 g = &svm->nested.ctl; 137 138 for (i = 0; i < MAX_INTERCEPT; i++) 139 c->intercepts[i] = h->intercepts[i]; 140 141 if (g->int_ctl & V_INTR_MASKING_MASK) { 142 /* We only want the cr8 intercept bits of L1 */ 143 vmcb_clr_intercept(c, INTERCEPT_CR8_READ); 144 vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE); 145 146 /* 147 * Once running L2 with HF_VINTR_MASK, EFLAGS.IF does not 148 * affect any interrupt we may want to inject; therefore, 149 * interrupt window vmexits are irrelevant to L0. 150 */ 151 vmcb_clr_intercept(c, INTERCEPT_VINTR); 152 } 153 154 /* We don't want to see VMMCALLs from a nested guest */ 155 vmcb_clr_intercept(c, INTERCEPT_VMMCALL); 156 157 for (i = 0; i < MAX_INTERCEPT; i++) 158 c->intercepts[i] |= g->intercepts[i]; 159 160 /* If SMI is not intercepted, ignore guest SMI intercept as well */ 161 if (!intercept_smi) 162 vmcb_clr_intercept(c, INTERCEPT_SMI); 163 164 vmcb_set_intercept(c, INTERCEPT_VMLOAD); 165 vmcb_set_intercept(c, INTERCEPT_VMSAVE); 166 } 167 168 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) 169 { 170 /* 171 * This function merges the msr permission bitmaps of kvm and the 172 * nested vmcb. It is optimized in that it only merges the parts where 173 * the kvm msr permission bitmap may contain zero bits 174 */ 175 int i; 176 177 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 178 return true; 179 180 for (i = 0; i < MSRPM_OFFSETS; i++) { 181 u32 value, p; 182 u64 offset; 183 184 if (msrpm_offsets[i] == 0xffffffff) 185 break; 186 187 p = msrpm_offsets[i]; 188 offset = svm->nested.ctl.msrpm_base_pa + (p * 4); 189 190 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) 191 return false; 192 193 svm->nested.msrpm[p] = svm->msrpm[p] | value; 194 } 195 196 svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm)); 197 198 return true; 199 } 200 201 /* 202 * Bits 11:0 of bitmap address are ignored by hardware 203 */ 204 static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size) 205 { 206 u64 addr = PAGE_ALIGN(pa); 207 208 return kvm_vcpu_is_legal_gpa(vcpu, addr) && 209 kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1); 210 } 211 212 static bool nested_svm_check_tlb_ctl(struct kvm_vcpu *vcpu, u8 tlb_ctl) 213 { 214 /* Nested FLUSHBYASID is not supported yet. */ 215 switch(tlb_ctl) { 216 case TLB_CONTROL_DO_NOTHING: 217 case TLB_CONTROL_FLUSH_ALL_ASID: 218 return true; 219 default: 220 return false; 221 } 222 } 223 224 static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, 225 struct vmcb_ctrl_area_cached *control) 226 { 227 if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN))) 228 return false; 229 230 if (CC(control->asid == 0)) 231 return false; 232 233 if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled)) 234 return false; 235 236 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, 237 MSRPM_SIZE))) 238 return false; 239 if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa, 240 IOPM_SIZE))) 241 return false; 242 243 if (CC(!nested_svm_check_tlb_ctl(vcpu, control->tlb_ctl))) 244 return false; 245 246 return true; 247 } 248 249 /* Common checks that apply to both L1 and L2 state. */ 250 static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu, 251 struct vmcb_save_area_cached *save) 252 { 253 if (CC(!(save->efer & EFER_SVME))) 254 return false; 255 256 if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) || 257 CC(save->cr0 & ~0xffffffffULL)) 258 return false; 259 260 if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7))) 261 return false; 262 263 /* 264 * These checks are also performed by KVM_SET_SREGS, 265 * except that EFER.LMA is not checked by SVM against 266 * CR0.PG && EFER.LME. 267 */ 268 if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) { 269 if (CC(!(save->cr4 & X86_CR4_PAE)) || 270 CC(!(save->cr0 & X86_CR0_PE)) || 271 CC(kvm_vcpu_is_illegal_gpa(vcpu, save->cr3))) 272 return false; 273 } 274 275 if (CC(!kvm_is_valid_cr4(vcpu, save->cr4))) 276 return false; 277 278 if (CC(!kvm_valid_efer(vcpu, save->efer))) 279 return false; 280 281 return true; 282 } 283 284 static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu) 285 { 286 struct vcpu_svm *svm = to_svm(vcpu); 287 struct vmcb_save_area_cached *save = &svm->nested.save; 288 289 return __nested_vmcb_check_save(vcpu, save); 290 } 291 292 static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu) 293 { 294 struct vcpu_svm *svm = to_svm(vcpu); 295 struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl; 296 297 return __nested_vmcb_check_controls(vcpu, ctl); 298 } 299 300 static 301 void __nested_copy_vmcb_control_to_cache(struct vmcb_ctrl_area_cached *to, 302 struct vmcb_control_area *from) 303 { 304 unsigned int i; 305 306 for (i = 0; i < MAX_INTERCEPT; i++) 307 to->intercepts[i] = from->intercepts[i]; 308 309 to->iopm_base_pa = from->iopm_base_pa; 310 to->msrpm_base_pa = from->msrpm_base_pa; 311 to->tsc_offset = from->tsc_offset; 312 to->tlb_ctl = from->tlb_ctl; 313 to->int_ctl = from->int_ctl; 314 to->int_vector = from->int_vector; 315 to->int_state = from->int_state; 316 to->exit_code = from->exit_code; 317 to->exit_code_hi = from->exit_code_hi; 318 to->exit_info_1 = from->exit_info_1; 319 to->exit_info_2 = from->exit_info_2; 320 to->exit_int_info = from->exit_int_info; 321 to->exit_int_info_err = from->exit_int_info_err; 322 to->nested_ctl = from->nested_ctl; 323 to->event_inj = from->event_inj; 324 to->event_inj_err = from->event_inj_err; 325 to->nested_cr3 = from->nested_cr3; 326 to->virt_ext = from->virt_ext; 327 to->pause_filter_count = from->pause_filter_count; 328 to->pause_filter_thresh = from->pause_filter_thresh; 329 330 /* Copy asid here because nested_vmcb_check_controls will check it. */ 331 to->asid = from->asid; 332 to->msrpm_base_pa &= ~0x0fffULL; 333 to->iopm_base_pa &= ~0x0fffULL; 334 } 335 336 void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm, 337 struct vmcb_control_area *control) 338 { 339 __nested_copy_vmcb_control_to_cache(&svm->nested.ctl, control); 340 } 341 342 static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, 343 struct vmcb_save_area *from) 344 { 345 /* 346 * Copy only fields that are validated, as we need them 347 * to avoid TOC/TOU races. 348 */ 349 to->efer = from->efer; 350 to->cr0 = from->cr0; 351 to->cr3 = from->cr3; 352 to->cr4 = from->cr4; 353 354 to->dr6 = from->dr6; 355 to->dr7 = from->dr7; 356 } 357 358 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm, 359 struct vmcb_save_area *save) 360 { 361 __nested_copy_vmcb_save_to_cache(&svm->nested.save, save); 362 } 363 364 /* 365 * Synchronize fields that are written by the processor, so that 366 * they can be copied back into the vmcb12. 367 */ 368 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) 369 { 370 u32 mask; 371 svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; 372 svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; 373 374 /* Only a few fields of int_ctl are written by the processor. */ 375 mask = V_IRQ_MASK | V_TPR_MASK; 376 if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && 377 svm_is_intercept(svm, INTERCEPT_VINTR)) { 378 /* 379 * In order to request an interrupt window, L0 is usurping 380 * svm->vmcb->control.int_ctl and possibly setting V_IRQ 381 * even if it was clear in L1's VMCB. Restoring it would be 382 * wrong. However, in this case V_IRQ will remain true until 383 * interrupt_window_interception calls svm_clear_vintr and 384 * restores int_ctl. We can just leave it aside. 385 */ 386 mask &= ~V_IRQ_MASK; 387 } 388 svm->nested.ctl.int_ctl &= ~mask; 389 svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; 390 } 391 392 /* 393 * Transfer any event that L0 or L1 wanted to inject into L2 to 394 * EXIT_INT_INFO. 395 */ 396 static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, 397 struct vmcb *vmcb12) 398 { 399 struct kvm_vcpu *vcpu = &svm->vcpu; 400 u32 exit_int_info = 0; 401 unsigned int nr; 402 403 if (vcpu->arch.exception.injected) { 404 nr = vcpu->arch.exception.nr; 405 exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; 406 407 if (vcpu->arch.exception.has_error_code) { 408 exit_int_info |= SVM_EVTINJ_VALID_ERR; 409 vmcb12->control.exit_int_info_err = 410 vcpu->arch.exception.error_code; 411 } 412 413 } else if (vcpu->arch.nmi_injected) { 414 exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; 415 416 } else if (vcpu->arch.interrupt.injected) { 417 nr = vcpu->arch.interrupt.nr; 418 exit_int_info = nr | SVM_EVTINJ_VALID; 419 420 if (vcpu->arch.interrupt.soft) 421 exit_int_info |= SVM_EVTINJ_TYPE_SOFT; 422 else 423 exit_int_info |= SVM_EVTINJ_TYPE_INTR; 424 } 425 426 vmcb12->control.exit_int_info = exit_int_info; 427 } 428 429 static inline bool nested_npt_enabled(struct vcpu_svm *svm) 430 { 431 return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; 432 } 433 434 static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) 435 { 436 /* 437 * TODO: optimize unconditional TLB flush/MMU sync. A partial list of 438 * things to fix before this can be conditional: 439 * 440 * - Flush TLBs for both L1 and L2 remote TLB flush 441 * - Honor L1's request to flush an ASID on nested VMRUN 442 * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] 443 * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN 444 * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST 445 * 446 * [*] Unlike nested EPT, SVM's ASID management can invalidate nested 447 * NPT guest-physical mappings on VMRUN. 448 */ 449 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 450 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 451 } 452 453 /* 454 * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true 455 * if we are emulating VM-Entry into a guest with NPT enabled. 456 */ 457 static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 458 bool nested_npt, bool reload_pdptrs) 459 { 460 if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) 461 return -EINVAL; 462 463 if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && 464 CC(!load_pdptrs(vcpu, cr3))) 465 return -EINVAL; 466 467 if (!nested_npt) 468 kvm_mmu_new_pgd(vcpu, cr3); 469 470 vcpu->arch.cr3 = cr3; 471 472 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 473 kvm_init_mmu(vcpu); 474 475 return 0; 476 } 477 478 void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) 479 { 480 if (!svm->nested.vmcb02.ptr) 481 return; 482 483 /* FIXME: merge g_pat from vmcb01 and vmcb12. */ 484 svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; 485 } 486 487 static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) 488 { 489 bool new_vmcb12 = false; 490 491 nested_vmcb02_compute_g_pat(svm); 492 493 /* Load the nested guest state */ 494 if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) { 495 new_vmcb12 = true; 496 svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa; 497 } 498 499 if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) { 500 svm->vmcb->save.es = vmcb12->save.es; 501 svm->vmcb->save.cs = vmcb12->save.cs; 502 svm->vmcb->save.ss = vmcb12->save.ss; 503 svm->vmcb->save.ds = vmcb12->save.ds; 504 svm->vmcb->save.cpl = vmcb12->save.cpl; 505 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); 506 } 507 508 if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) { 509 svm->vmcb->save.gdtr = vmcb12->save.gdtr; 510 svm->vmcb->save.idtr = vmcb12->save.idtr; 511 vmcb_mark_dirty(svm->vmcb, VMCB_DT); 512 } 513 514 kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); 515 516 svm_set_efer(&svm->vcpu, svm->nested.save.efer); 517 518 svm_set_cr0(&svm->vcpu, svm->nested.save.cr0); 519 svm_set_cr4(&svm->vcpu, svm->nested.save.cr4); 520 521 svm->vcpu.arch.cr2 = vmcb12->save.cr2; 522 523 kvm_rax_write(&svm->vcpu, vmcb12->save.rax); 524 kvm_rsp_write(&svm->vcpu, vmcb12->save.rsp); 525 kvm_rip_write(&svm->vcpu, vmcb12->save.rip); 526 527 /* In case we don't even reach vcpu_run, the fields are not updated */ 528 svm->vmcb->save.rax = vmcb12->save.rax; 529 svm->vmcb->save.rsp = vmcb12->save.rsp; 530 svm->vmcb->save.rip = vmcb12->save.rip; 531 532 /* These bits will be set properly on the first execution when new_vmc12 is true */ 533 if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { 534 svm->vmcb->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; 535 svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; 536 vmcb_mark_dirty(svm->vmcb, VMCB_DR); 537 } 538 } 539 540 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) 541 { 542 const u32 int_ctl_vmcb01_bits = 543 V_INTR_MASKING_MASK | V_GIF_MASK | V_GIF_ENABLE_MASK; 544 545 const u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; 546 547 struct kvm_vcpu *vcpu = &svm->vcpu; 548 549 /* 550 * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2, 551 * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes. 552 */ 553 554 /* 555 * Also covers avic_vapic_bar, avic_backing_page, avic_logical_id, 556 * avic_physical_id. 557 */ 558 WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); 559 560 /* Copied from vmcb01. msrpm_base can be overwritten later. */ 561 svm->vmcb->control.nested_ctl = svm->vmcb01.ptr->control.nested_ctl; 562 svm->vmcb->control.iopm_base_pa = svm->vmcb01.ptr->control.iopm_base_pa; 563 svm->vmcb->control.msrpm_base_pa = svm->vmcb01.ptr->control.msrpm_base_pa; 564 565 /* Done at vmrun: asid. */ 566 567 /* Also overwritten later if necessary. */ 568 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; 569 570 /* nested_cr3. */ 571 if (nested_npt_enabled(svm)) 572 nested_svm_init_mmu_context(vcpu); 573 574 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 575 vcpu->arch.l1_tsc_offset, 576 svm->nested.ctl.tsc_offset, 577 svm->tsc_ratio_msr); 578 579 svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; 580 581 if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 582 WARN_ON(!svm->tsc_scaling_enabled); 583 nested_svm_update_tsc_ratio_msr(vcpu); 584 } 585 586 svm->vmcb->control.int_ctl = 587 (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) | 588 (svm->vmcb01.ptr->control.int_ctl & int_ctl_vmcb01_bits); 589 590 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; 591 svm->vmcb->control.int_state = svm->nested.ctl.int_state; 592 svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; 593 svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; 594 595 nested_svm_transition_tlb_flush(vcpu); 596 597 /* Enter Guest-Mode */ 598 enter_guest_mode(vcpu); 599 600 /* 601 * Merge guest and host intercepts - must be called with vcpu in 602 * guest-mode to take effect. 603 */ 604 recalc_intercepts(svm); 605 } 606 607 static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb) 608 { 609 /* 610 * Some VMCB state is shared between L1 and L2 and thus has to be 611 * moved at the time of nested vmrun and vmexit. 612 * 613 * VMLOAD/VMSAVE state would also belong in this category, but KVM 614 * always performs VMLOAD and VMSAVE from the VMCB01. 615 */ 616 to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl; 617 } 618 619 int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, 620 struct vmcb *vmcb12, bool from_vmrun) 621 { 622 struct vcpu_svm *svm = to_svm(vcpu); 623 int ret; 624 625 trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, 626 vmcb12->save.rip, 627 vmcb12->control.int_ctl, 628 vmcb12->control.event_inj, 629 vmcb12->control.nested_ctl); 630 631 trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, 632 vmcb12->control.intercepts[INTERCEPT_CR] >> 16, 633 vmcb12->control.intercepts[INTERCEPT_EXCEPTION], 634 vmcb12->control.intercepts[INTERCEPT_WORD3], 635 vmcb12->control.intercepts[INTERCEPT_WORD4], 636 vmcb12->control.intercepts[INTERCEPT_WORD5]); 637 638 639 svm->nested.vmcb12_gpa = vmcb12_gpa; 640 641 WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr); 642 643 nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); 644 645 svm_switch_vmcb(svm, &svm->nested.vmcb02); 646 nested_vmcb02_prepare_control(svm); 647 nested_vmcb02_prepare_save(svm, vmcb12); 648 649 ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, 650 nested_npt_enabled(svm), from_vmrun); 651 if (ret) 652 return ret; 653 654 if (!npt_enabled) 655 vcpu->arch.mmu->inject_page_fault = svm_inject_page_fault_nested; 656 657 if (!from_vmrun) 658 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 659 660 svm_set_gif(svm, true); 661 662 return 0; 663 } 664 665 int nested_svm_vmrun(struct kvm_vcpu *vcpu) 666 { 667 struct vcpu_svm *svm = to_svm(vcpu); 668 int ret; 669 struct vmcb *vmcb12; 670 struct kvm_host_map map; 671 u64 vmcb12_gpa; 672 673 if (!svm->nested.hsave_msr) { 674 kvm_inject_gp(vcpu, 0); 675 return 1; 676 } 677 678 if (is_smm(vcpu)) { 679 kvm_queue_exception(vcpu, UD_VECTOR); 680 return 1; 681 } 682 683 vmcb12_gpa = svm->vmcb->save.rax; 684 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map); 685 if (ret == -EINVAL) { 686 kvm_inject_gp(vcpu, 0); 687 return 1; 688 } else if (ret) { 689 return kvm_skip_emulated_instruction(vcpu); 690 } 691 692 ret = kvm_skip_emulated_instruction(vcpu); 693 694 vmcb12 = map.hva; 695 696 if (WARN_ON_ONCE(!svm->nested.initialized)) 697 return -EINVAL; 698 699 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); 700 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); 701 702 if (!nested_vmcb_check_save(vcpu) || 703 !nested_vmcb_check_controls(vcpu)) { 704 vmcb12->control.exit_code = SVM_EXIT_ERR; 705 vmcb12->control.exit_code_hi = 0; 706 vmcb12->control.exit_info_1 = 0; 707 vmcb12->control.exit_info_2 = 0; 708 goto out; 709 } 710 711 /* 712 * Since vmcb01 is not in use, we can use it to store some of the L1 713 * state. 714 */ 715 svm->vmcb01.ptr->save.efer = vcpu->arch.efer; 716 svm->vmcb01.ptr->save.cr0 = kvm_read_cr0(vcpu); 717 svm->vmcb01.ptr->save.cr4 = vcpu->arch.cr4; 718 svm->vmcb01.ptr->save.rflags = kvm_get_rflags(vcpu); 719 svm->vmcb01.ptr->save.rip = kvm_rip_read(vcpu); 720 721 if (!npt_enabled) 722 svm->vmcb01.ptr->save.cr3 = kvm_read_cr3(vcpu); 723 724 svm->nested.nested_run_pending = 1; 725 726 if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true)) 727 goto out_exit_err; 728 729 if (nested_svm_vmrun_msrpm(svm)) 730 goto out; 731 732 out_exit_err: 733 svm->nested.nested_run_pending = 0; 734 735 svm->vmcb->control.exit_code = SVM_EXIT_ERR; 736 svm->vmcb->control.exit_code_hi = 0; 737 svm->vmcb->control.exit_info_1 = 0; 738 svm->vmcb->control.exit_info_2 = 0; 739 740 nested_svm_vmexit(svm); 741 742 out: 743 kvm_vcpu_unmap(vcpu, &map, true); 744 745 return ret; 746 } 747 748 /* Copy state save area fields which are handled by VMRUN */ 749 void svm_copy_vmrun_state(struct vmcb_save_area *to_save, 750 struct vmcb_save_area *from_save) 751 { 752 to_save->es = from_save->es; 753 to_save->cs = from_save->cs; 754 to_save->ss = from_save->ss; 755 to_save->ds = from_save->ds; 756 to_save->gdtr = from_save->gdtr; 757 to_save->idtr = from_save->idtr; 758 to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED; 759 to_save->efer = from_save->efer; 760 to_save->cr0 = from_save->cr0; 761 to_save->cr3 = from_save->cr3; 762 to_save->cr4 = from_save->cr4; 763 to_save->rax = from_save->rax; 764 to_save->rsp = from_save->rsp; 765 to_save->rip = from_save->rip; 766 to_save->cpl = 0; 767 } 768 769 void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) 770 { 771 to_vmcb->save.fs = from_vmcb->save.fs; 772 to_vmcb->save.gs = from_vmcb->save.gs; 773 to_vmcb->save.tr = from_vmcb->save.tr; 774 to_vmcb->save.ldtr = from_vmcb->save.ldtr; 775 to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; 776 to_vmcb->save.star = from_vmcb->save.star; 777 to_vmcb->save.lstar = from_vmcb->save.lstar; 778 to_vmcb->save.cstar = from_vmcb->save.cstar; 779 to_vmcb->save.sfmask = from_vmcb->save.sfmask; 780 to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; 781 to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; 782 to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; 783 } 784 785 int nested_svm_vmexit(struct vcpu_svm *svm) 786 { 787 struct kvm_vcpu *vcpu = &svm->vcpu; 788 struct vmcb *vmcb12; 789 struct vmcb *vmcb = svm->vmcb; 790 struct kvm_host_map map; 791 int rc; 792 793 /* Triple faults in L2 should never escape. */ 794 WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); 795 796 rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); 797 if (rc) { 798 if (rc == -EINVAL) 799 kvm_inject_gp(vcpu, 0); 800 return 1; 801 } 802 803 vmcb12 = map.hva; 804 805 /* Exit Guest-Mode */ 806 leave_guest_mode(vcpu); 807 svm->nested.vmcb12_gpa = 0; 808 WARN_ON_ONCE(svm->nested.nested_run_pending); 809 810 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 811 812 /* in case we halted in L2 */ 813 svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; 814 815 /* Give the current vmcb to the guest */ 816 817 vmcb12->save.es = vmcb->save.es; 818 vmcb12->save.cs = vmcb->save.cs; 819 vmcb12->save.ss = vmcb->save.ss; 820 vmcb12->save.ds = vmcb->save.ds; 821 vmcb12->save.gdtr = vmcb->save.gdtr; 822 vmcb12->save.idtr = vmcb->save.idtr; 823 vmcb12->save.efer = svm->vcpu.arch.efer; 824 vmcb12->save.cr0 = kvm_read_cr0(vcpu); 825 vmcb12->save.cr3 = kvm_read_cr3(vcpu); 826 vmcb12->save.cr2 = vmcb->save.cr2; 827 vmcb12->save.cr4 = svm->vcpu.arch.cr4; 828 vmcb12->save.rflags = kvm_get_rflags(vcpu); 829 vmcb12->save.rip = kvm_rip_read(vcpu); 830 vmcb12->save.rsp = kvm_rsp_read(vcpu); 831 vmcb12->save.rax = kvm_rax_read(vcpu); 832 vmcb12->save.dr7 = vmcb->save.dr7; 833 vmcb12->save.dr6 = svm->vcpu.arch.dr6; 834 vmcb12->save.cpl = vmcb->save.cpl; 835 836 vmcb12->control.int_state = vmcb->control.int_state; 837 vmcb12->control.exit_code = vmcb->control.exit_code; 838 vmcb12->control.exit_code_hi = vmcb->control.exit_code_hi; 839 vmcb12->control.exit_info_1 = vmcb->control.exit_info_1; 840 vmcb12->control.exit_info_2 = vmcb->control.exit_info_2; 841 842 if (vmcb12->control.exit_code != SVM_EXIT_ERR) 843 nested_save_pending_event_to_vmcb12(svm, vmcb12); 844 845 if (svm->nrips_enabled) 846 vmcb12->control.next_rip = vmcb->control.next_rip; 847 848 vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; 849 vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl; 850 vmcb12->control.event_inj = svm->nested.ctl.event_inj; 851 vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; 852 853 nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); 854 855 svm_switch_vmcb(svm, &svm->vmcb01); 856 857 /* 858 * On vmexit the GIF is set to false and 859 * no event can be injected in L1. 860 */ 861 svm_set_gif(svm, false); 862 svm->vmcb->control.exit_int_info = 0; 863 864 svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; 865 if (svm->vmcb->control.tsc_offset != svm->vcpu.arch.tsc_offset) { 866 svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; 867 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); 868 } 869 870 if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { 871 WARN_ON(!svm->tsc_scaling_enabled); 872 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 873 svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 874 } 875 876 svm->nested.ctl.nested_cr3 = 0; 877 878 /* 879 * Restore processor state that had been saved in vmcb01 880 */ 881 kvm_set_rflags(vcpu, svm->vmcb->save.rflags); 882 svm_set_efer(vcpu, svm->vmcb->save.efer); 883 svm_set_cr0(vcpu, svm->vmcb->save.cr0 | X86_CR0_PE); 884 svm_set_cr4(vcpu, svm->vmcb->save.cr4); 885 kvm_rax_write(vcpu, svm->vmcb->save.rax); 886 kvm_rsp_write(vcpu, svm->vmcb->save.rsp); 887 kvm_rip_write(vcpu, svm->vmcb->save.rip); 888 889 svm->vcpu.arch.dr7 = DR7_FIXED_1; 890 kvm_update_dr7(&svm->vcpu); 891 892 trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, 893 vmcb12->control.exit_info_1, 894 vmcb12->control.exit_info_2, 895 vmcb12->control.exit_int_info, 896 vmcb12->control.exit_int_info_err, 897 KVM_ISA_SVM); 898 899 kvm_vcpu_unmap(vcpu, &map, true); 900 901 nested_svm_transition_tlb_flush(vcpu); 902 903 nested_svm_uninit_mmu_context(vcpu); 904 905 rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false, true); 906 if (rc) 907 return 1; 908 909 /* 910 * Drop what we picked up for L2 via svm_complete_interrupts() so it 911 * doesn't end up in L1. 912 */ 913 svm->vcpu.arch.nmi_injected = false; 914 kvm_clear_exception_queue(vcpu); 915 kvm_clear_interrupt_queue(vcpu); 916 917 /* 918 * If we are here following the completion of a VMRUN that 919 * is being single-stepped, queue the pending #DB intercept 920 * right now so that it an be accounted for before we execute 921 * L1's next instruction. 922 */ 923 if (unlikely(svm->vmcb->save.rflags & X86_EFLAGS_TF)) 924 kvm_queue_exception(&(svm->vcpu), DB_VECTOR); 925 926 return 0; 927 } 928 929 static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) 930 { 931 nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN); 932 } 933 934 int svm_allocate_nested(struct vcpu_svm *svm) 935 { 936 struct page *vmcb02_page; 937 938 if (svm->nested.initialized) 939 return 0; 940 941 vmcb02_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 942 if (!vmcb02_page) 943 return -ENOMEM; 944 svm->nested.vmcb02.ptr = page_address(vmcb02_page); 945 svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT); 946 947 svm->nested.msrpm = svm_vcpu_alloc_msrpm(); 948 if (!svm->nested.msrpm) 949 goto err_free_vmcb02; 950 svm_vcpu_init_msrpm(&svm->vcpu, svm->nested.msrpm); 951 952 svm->nested.initialized = true; 953 return 0; 954 955 err_free_vmcb02: 956 __free_page(vmcb02_page); 957 return -ENOMEM; 958 } 959 960 void svm_free_nested(struct vcpu_svm *svm) 961 { 962 if (!svm->nested.initialized) 963 return; 964 965 svm_vcpu_free_msrpm(svm->nested.msrpm); 966 svm->nested.msrpm = NULL; 967 968 __free_page(virt_to_page(svm->nested.vmcb02.ptr)); 969 svm->nested.vmcb02.ptr = NULL; 970 971 /* 972 * When last_vmcb12_gpa matches the current vmcb12 gpa, 973 * some vmcb12 fields are not loaded if they are marked clean 974 * in the vmcb12, since in this case they are up to date already. 975 * 976 * When the vmcb02 is freed, this optimization becomes invalid. 977 */ 978 svm->nested.last_vmcb12_gpa = INVALID_GPA; 979 980 svm->nested.initialized = false; 981 } 982 983 /* 984 * Forcibly leave nested mode in order to be able to reset the VCPU later on. 985 */ 986 void svm_leave_nested(struct vcpu_svm *svm) 987 { 988 struct kvm_vcpu *vcpu = &svm->vcpu; 989 990 if (is_guest_mode(vcpu)) { 991 svm->nested.nested_run_pending = 0; 992 svm->nested.vmcb12_gpa = INVALID_GPA; 993 994 leave_guest_mode(vcpu); 995 996 svm_switch_vmcb(svm, &svm->vmcb01); 997 998 nested_svm_uninit_mmu_context(vcpu); 999 vmcb_mark_all_dirty(svm->vmcb); 1000 } 1001 1002 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1003 } 1004 1005 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) 1006 { 1007 u32 offset, msr, value; 1008 int write, mask; 1009 1010 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))) 1011 return NESTED_EXIT_HOST; 1012 1013 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; 1014 offset = svm_msrpm_offset(msr); 1015 write = svm->vmcb->control.exit_info_1 & 1; 1016 mask = 1 << ((2 * (msr & 0xf)) + write); 1017 1018 if (offset == MSR_INVALID) 1019 return NESTED_EXIT_DONE; 1020 1021 /* Offset is in 32 bit units but need in 8 bit units */ 1022 offset *= 4; 1023 1024 if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4)) 1025 return NESTED_EXIT_DONE; 1026 1027 return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1028 } 1029 1030 static int nested_svm_intercept_ioio(struct vcpu_svm *svm) 1031 { 1032 unsigned port, size, iopm_len; 1033 u16 val, mask; 1034 u8 start_bit; 1035 u64 gpa; 1036 1037 if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT))) 1038 return NESTED_EXIT_HOST; 1039 1040 port = svm->vmcb->control.exit_info_1 >> 16; 1041 size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> 1042 SVM_IOIO_SIZE_SHIFT; 1043 gpa = svm->nested.ctl.iopm_base_pa + (port / 8); 1044 start_bit = port % 8; 1045 iopm_len = (start_bit + size > 8) ? 2 : 1; 1046 mask = (0xf >> (4 - size)) << start_bit; 1047 val = 0; 1048 1049 if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len)) 1050 return NESTED_EXIT_DONE; 1051 1052 return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; 1053 } 1054 1055 static int nested_svm_intercept(struct vcpu_svm *svm) 1056 { 1057 u32 exit_code = svm->vmcb->control.exit_code; 1058 int vmexit = NESTED_EXIT_HOST; 1059 1060 switch (exit_code) { 1061 case SVM_EXIT_MSR: 1062 vmexit = nested_svm_exit_handled_msr(svm); 1063 break; 1064 case SVM_EXIT_IOIO: 1065 vmexit = nested_svm_intercept_ioio(svm); 1066 break; 1067 case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { 1068 if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1069 vmexit = NESTED_EXIT_DONE; 1070 break; 1071 } 1072 case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { 1073 if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1074 vmexit = NESTED_EXIT_DONE; 1075 break; 1076 } 1077 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 1078 /* 1079 * Host-intercepted exceptions have been checked already in 1080 * nested_svm_exit_special. There is nothing to do here, 1081 * the vmexit is injected by svm_check_nested_events. 1082 */ 1083 vmexit = NESTED_EXIT_DONE; 1084 break; 1085 } 1086 case SVM_EXIT_ERR: { 1087 vmexit = NESTED_EXIT_DONE; 1088 break; 1089 } 1090 default: { 1091 if (vmcb12_is_intercept(&svm->nested.ctl, exit_code)) 1092 vmexit = NESTED_EXIT_DONE; 1093 } 1094 } 1095 1096 return vmexit; 1097 } 1098 1099 int nested_svm_exit_handled(struct vcpu_svm *svm) 1100 { 1101 int vmexit; 1102 1103 vmexit = nested_svm_intercept(svm); 1104 1105 if (vmexit == NESTED_EXIT_DONE) 1106 nested_svm_vmexit(svm); 1107 1108 return vmexit; 1109 } 1110 1111 int nested_svm_check_permissions(struct kvm_vcpu *vcpu) 1112 { 1113 if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) { 1114 kvm_queue_exception(vcpu, UD_VECTOR); 1115 return 1; 1116 } 1117 1118 if (to_svm(vcpu)->vmcb->save.cpl) { 1119 kvm_inject_gp(vcpu, 0); 1120 return 1; 1121 } 1122 1123 return 0; 1124 } 1125 1126 static bool nested_exit_on_exception(struct vcpu_svm *svm) 1127 { 1128 unsigned int nr = svm->vcpu.arch.exception.nr; 1129 1130 return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr)); 1131 } 1132 1133 static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) 1134 { 1135 unsigned int nr = svm->vcpu.arch.exception.nr; 1136 1137 svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; 1138 svm->vmcb->control.exit_code_hi = 0; 1139 1140 if (svm->vcpu.arch.exception.has_error_code) 1141 svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; 1142 1143 /* 1144 * EXITINFO2 is undefined for all exception intercepts other 1145 * than #PF. 1146 */ 1147 if (nr == PF_VECTOR) { 1148 if (svm->vcpu.arch.exception.nested_apf) 1149 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; 1150 else if (svm->vcpu.arch.exception.has_payload) 1151 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; 1152 else 1153 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; 1154 } else if (nr == DB_VECTOR) { 1155 /* See inject_pending_event. */ 1156 kvm_deliver_exception_payload(&svm->vcpu); 1157 if (svm->vcpu.arch.dr7 & DR7_GD) { 1158 svm->vcpu.arch.dr7 &= ~DR7_GD; 1159 kvm_update_dr7(&svm->vcpu); 1160 } 1161 } else 1162 WARN_ON(svm->vcpu.arch.exception.has_payload); 1163 1164 nested_svm_vmexit(svm); 1165 } 1166 1167 static inline bool nested_exit_on_init(struct vcpu_svm *svm) 1168 { 1169 return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT); 1170 } 1171 1172 static int svm_check_nested_events(struct kvm_vcpu *vcpu) 1173 { 1174 struct vcpu_svm *svm = to_svm(vcpu); 1175 bool block_nested_events = 1176 kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; 1177 struct kvm_lapic *apic = vcpu->arch.apic; 1178 1179 if (lapic_in_kernel(vcpu) && 1180 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 1181 if (block_nested_events) 1182 return -EBUSY; 1183 if (!nested_exit_on_init(svm)) 1184 return 0; 1185 nested_svm_simple_vmexit(svm, SVM_EXIT_INIT); 1186 return 0; 1187 } 1188 1189 if (vcpu->arch.exception.pending) { 1190 /* 1191 * Only a pending nested run can block a pending exception. 1192 * Otherwise an injected NMI/interrupt should either be 1193 * lost or delivered to the nested hypervisor in the EXITINTINFO 1194 * vmcb field, while delivering the pending exception. 1195 */ 1196 if (svm->nested.nested_run_pending) 1197 return -EBUSY; 1198 if (!nested_exit_on_exception(svm)) 1199 return 0; 1200 nested_svm_inject_exception_vmexit(svm); 1201 return 0; 1202 } 1203 1204 if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { 1205 if (block_nested_events) 1206 return -EBUSY; 1207 if (!nested_exit_on_smi(svm)) 1208 return 0; 1209 nested_svm_simple_vmexit(svm, SVM_EXIT_SMI); 1210 return 0; 1211 } 1212 1213 if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { 1214 if (block_nested_events) 1215 return -EBUSY; 1216 if (!nested_exit_on_nmi(svm)) 1217 return 0; 1218 nested_svm_simple_vmexit(svm, SVM_EXIT_NMI); 1219 return 0; 1220 } 1221 1222 if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { 1223 if (block_nested_events) 1224 return -EBUSY; 1225 if (!nested_exit_on_intr(svm)) 1226 return 0; 1227 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); 1228 nested_svm_simple_vmexit(svm, SVM_EXIT_INTR); 1229 return 0; 1230 } 1231 1232 return 0; 1233 } 1234 1235 int nested_svm_exit_special(struct vcpu_svm *svm) 1236 { 1237 u32 exit_code = svm->vmcb->control.exit_code; 1238 1239 switch (exit_code) { 1240 case SVM_EXIT_INTR: 1241 case SVM_EXIT_NMI: 1242 case SVM_EXIT_NPF: 1243 return NESTED_EXIT_HOST; 1244 case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { 1245 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); 1246 1247 if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] & 1248 excp_bits) 1249 return NESTED_EXIT_HOST; 1250 else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && 1251 svm->vcpu.arch.apf.host_apf_flags) 1252 /* Trap async PF even if not shadowing */ 1253 return NESTED_EXIT_HOST; 1254 break; 1255 } 1256 default: 1257 break; 1258 } 1259 1260 return NESTED_EXIT_CONTINUE; 1261 } 1262 1263 void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu) 1264 { 1265 struct vcpu_svm *svm = to_svm(vcpu); 1266 1267 vcpu->arch.tsc_scaling_ratio = 1268 kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio, 1269 svm->tsc_ratio_msr); 1270 svm_write_tsc_multiplier(vcpu, vcpu->arch.tsc_scaling_ratio); 1271 } 1272 1273 /* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */ 1274 static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst, 1275 struct vmcb_ctrl_area_cached *from) 1276 { 1277 unsigned int i; 1278 1279 memset(dst, 0, sizeof(struct vmcb_control_area)); 1280 1281 for (i = 0; i < MAX_INTERCEPT; i++) 1282 dst->intercepts[i] = from->intercepts[i]; 1283 1284 dst->iopm_base_pa = from->iopm_base_pa; 1285 dst->msrpm_base_pa = from->msrpm_base_pa; 1286 dst->tsc_offset = from->tsc_offset; 1287 dst->asid = from->asid; 1288 dst->tlb_ctl = from->tlb_ctl; 1289 dst->int_ctl = from->int_ctl; 1290 dst->int_vector = from->int_vector; 1291 dst->int_state = from->int_state; 1292 dst->exit_code = from->exit_code; 1293 dst->exit_code_hi = from->exit_code_hi; 1294 dst->exit_info_1 = from->exit_info_1; 1295 dst->exit_info_2 = from->exit_info_2; 1296 dst->exit_int_info = from->exit_int_info; 1297 dst->exit_int_info_err = from->exit_int_info_err; 1298 dst->nested_ctl = from->nested_ctl; 1299 dst->event_inj = from->event_inj; 1300 dst->event_inj_err = from->event_inj_err; 1301 dst->nested_cr3 = from->nested_cr3; 1302 dst->virt_ext = from->virt_ext; 1303 dst->pause_filter_count = from->pause_filter_count; 1304 dst->pause_filter_thresh = from->pause_filter_thresh; 1305 } 1306 1307 static int svm_get_nested_state(struct kvm_vcpu *vcpu, 1308 struct kvm_nested_state __user *user_kvm_nested_state, 1309 u32 user_data_size) 1310 { 1311 struct vcpu_svm *svm; 1312 struct vmcb_control_area *ctl; 1313 unsigned long r; 1314 struct kvm_nested_state kvm_state = { 1315 .flags = 0, 1316 .format = KVM_STATE_NESTED_FORMAT_SVM, 1317 .size = sizeof(kvm_state), 1318 }; 1319 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1320 &user_kvm_nested_state->data.svm[0]; 1321 1322 if (!vcpu) 1323 return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; 1324 1325 svm = to_svm(vcpu); 1326 1327 if (user_data_size < kvm_state.size) 1328 goto out; 1329 1330 /* First fill in the header and copy it out. */ 1331 if (is_guest_mode(vcpu)) { 1332 kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa; 1333 kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; 1334 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 1335 1336 if (svm->nested.nested_run_pending) 1337 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 1338 } 1339 1340 if (gif_set(svm)) 1341 kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; 1342 1343 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 1344 return -EFAULT; 1345 1346 if (!is_guest_mode(vcpu)) 1347 goto out; 1348 1349 /* 1350 * Copy over the full size of the VMCB rather than just the size 1351 * of the structs. 1352 */ 1353 if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) 1354 return -EFAULT; 1355 1356 ctl = kzalloc(sizeof(*ctl), GFP_KERNEL); 1357 if (!ctl) 1358 return -ENOMEM; 1359 1360 nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl); 1361 r = copy_to_user(&user_vmcb->control, ctl, 1362 sizeof(user_vmcb->control)); 1363 kfree(ctl); 1364 if (r) 1365 return -EFAULT; 1366 1367 if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save, 1368 sizeof(user_vmcb->save))) 1369 return -EFAULT; 1370 out: 1371 return kvm_state.size; 1372 } 1373 1374 static int svm_set_nested_state(struct kvm_vcpu *vcpu, 1375 struct kvm_nested_state __user *user_kvm_nested_state, 1376 struct kvm_nested_state *kvm_state) 1377 { 1378 struct vcpu_svm *svm = to_svm(vcpu); 1379 struct vmcb __user *user_vmcb = (struct vmcb __user *) 1380 &user_kvm_nested_state->data.svm[0]; 1381 struct vmcb_control_area *ctl; 1382 struct vmcb_save_area *save; 1383 struct vmcb_save_area_cached save_cached; 1384 struct vmcb_ctrl_area_cached ctl_cached; 1385 unsigned long cr0; 1386 int ret; 1387 1388 BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > 1389 KVM_STATE_NESTED_SVM_VMCB_SIZE); 1390 1391 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) 1392 return -EINVAL; 1393 1394 if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | 1395 KVM_STATE_NESTED_RUN_PENDING | 1396 KVM_STATE_NESTED_GIF_SET)) 1397 return -EINVAL; 1398 1399 /* 1400 * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's 1401 * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. 1402 */ 1403 if (!(vcpu->arch.efer & EFER_SVME)) { 1404 /* GIF=1 and no guest mode are required if SVME=0. */ 1405 if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET) 1406 return -EINVAL; 1407 } 1408 1409 /* SMM temporarily disables SVM, so we cannot be in guest mode. */ 1410 if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 1411 return -EINVAL; 1412 1413 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { 1414 svm_leave_nested(svm); 1415 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1416 return 0; 1417 } 1418 1419 if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) 1420 return -EINVAL; 1421 if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) 1422 return -EINVAL; 1423 1424 ret = -ENOMEM; 1425 ctl = kzalloc(sizeof(*ctl), GFP_KERNEL_ACCOUNT); 1426 save = kzalloc(sizeof(*save), GFP_KERNEL_ACCOUNT); 1427 if (!ctl || !save) 1428 goto out_free; 1429 1430 ret = -EFAULT; 1431 if (copy_from_user(ctl, &user_vmcb->control, sizeof(*ctl))) 1432 goto out_free; 1433 if (copy_from_user(save, &user_vmcb->save, sizeof(*save))) 1434 goto out_free; 1435 1436 ret = -EINVAL; 1437 __nested_copy_vmcb_control_to_cache(&ctl_cached, ctl); 1438 if (!__nested_vmcb_check_controls(vcpu, &ctl_cached)) 1439 goto out_free; 1440 1441 /* 1442 * Processor state contains L2 state. Check that it is 1443 * valid for guest mode (see nested_vmcb_check_save). 1444 */ 1445 cr0 = kvm_read_cr0(vcpu); 1446 if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) 1447 goto out_free; 1448 1449 /* 1450 * Validate host state saved from before VMRUN (see 1451 * nested_svm_check_permissions). 1452 */ 1453 __nested_copy_vmcb_save_to_cache(&save_cached, save); 1454 if (!(save->cr0 & X86_CR0_PG) || 1455 !(save->cr0 & X86_CR0_PE) || 1456 (save->rflags & X86_EFLAGS_VM) || 1457 !__nested_vmcb_check_save(vcpu, &save_cached)) 1458 goto out_free; 1459 1460 /* 1461 * While the nested guest CR3 is already checked and set by 1462 * KVM_SET_SREGS, it was set when nested state was yet loaded, 1463 * thus MMU might not be initialized correctly. 1464 * Set it again to fix this. 1465 */ 1466 1467 ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, 1468 nested_npt_enabled(svm), false); 1469 if (WARN_ON_ONCE(ret)) 1470 goto out_free; 1471 1472 1473 /* 1474 * All checks done, we can enter guest mode. Userspace provides 1475 * vmcb12.control, which will be combined with L1 and stored into 1476 * vmcb02, and the L1 save state which we store in vmcb01. 1477 * L2 registers if needed are moved from the current VMCB to VMCB02. 1478 */ 1479 1480 if (is_guest_mode(vcpu)) 1481 svm_leave_nested(svm); 1482 else 1483 svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save; 1484 1485 svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); 1486 1487 svm->nested.nested_run_pending = 1488 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 1489 1490 svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; 1491 1492 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); 1493 nested_copy_vmcb_control_to_cache(svm, ctl); 1494 1495 svm_switch_vmcb(svm, &svm->nested.vmcb02); 1496 nested_vmcb02_prepare_control(svm); 1497 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 1498 ret = 0; 1499 out_free: 1500 kfree(save); 1501 kfree(ctl); 1502 1503 return ret; 1504 } 1505 1506 static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) 1507 { 1508 struct vcpu_svm *svm = to_svm(vcpu); 1509 1510 if (WARN_ON(!is_guest_mode(vcpu))) 1511 return true; 1512 1513 if (!vcpu->arch.pdptrs_from_userspace && 1514 !nested_npt_enabled(svm) && is_pae_paging(vcpu)) 1515 /* 1516 * Reload the guest's PDPTRs since after a migration 1517 * the guest CR3 might be restored prior to setting the nested 1518 * state which can lead to a load of wrong PDPTRs. 1519 */ 1520 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 1521 return false; 1522 1523 if (!nested_svm_vmrun_msrpm(svm)) { 1524 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1525 vcpu->run->internal.suberror = 1526 KVM_INTERNAL_ERROR_EMULATION; 1527 vcpu->run->internal.ndata = 0; 1528 return false; 1529 } 1530 1531 return true; 1532 } 1533 1534 struct kvm_x86_nested_ops svm_nested_ops = { 1535 .check_events = svm_check_nested_events, 1536 .triple_fault = nested_svm_triple_fault, 1537 .get_nested_state_pages = svm_get_nested_state_pages, 1538 .get_state = svm_get_nested_state, 1539 .set_state = svm_set_nested_state, 1540 }; 1541