1 /* 2 * Kernel-based Virtual Machine driver for Linux 3 * 4 * derived from drivers/kvm/kvm_main.c 5 * 6 * Copyright (C) 2006 Qumranet, Inc. 7 * Copyright (C) 2008 Qumranet, Inc. 8 * Copyright IBM Corporation, 2008 9 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 10 * 11 * Authors: 12 * Avi Kivity <avi@qumranet.com> 13 * Yaniv Kamay <yaniv@qumranet.com> 14 * Amit Shah <amit.shah@qumranet.com> 15 * Ben-Ami Yassour <benami@il.ibm.com> 16 * 17 * This work is licensed under the terms of the GNU GPL, version 2. See 18 * the COPYING file in the top-level directory. 19 * 20 */ 21 22 #include <linux/kvm_host.h> 23 #include "irq.h" 24 #include "mmu.h" 25 #include "i8254.h" 26 #include "tss.h" 27 #include "kvm_cache_regs.h" 28 #include "x86.h" 29 #include "cpuid.h" 30 #include "assigned-dev.h" 31 #include "pmu.h" 32 #include "hyperv.h" 33 34 #include <linux/clocksource.h> 35 #include <linux/interrupt.h> 36 #include <linux/kvm.h> 37 #include <linux/fs.h> 38 #include <linux/vmalloc.h> 39 #include <linux/module.h> 40 #include <linux/mman.h> 41 #include <linux/highmem.h> 42 #include <linux/iommu.h> 43 #include <linux/intel-iommu.h> 44 #include <linux/cpufreq.h> 45 #include <linux/user-return-notifier.h> 46 #include <linux/srcu.h> 47 #include <linux/slab.h> 48 #include <linux/perf_event.h> 49 #include <linux/uaccess.h> 50 #include <linux/hash.h> 51 #include <linux/pci.h> 52 #include <linux/timekeeper_internal.h> 53 #include <linux/pvclock_gtod.h> 54 #include <linux/kvm_irqfd.h> 55 #include <linux/irqbypass.h> 56 #include <trace/events/kvm.h> 57 58 #define CREATE_TRACE_POINTS 59 #include "trace.h" 60 61 #include <asm/debugreg.h> 62 #include <asm/msr.h> 63 #include <asm/desc.h> 64 #include <asm/mce.h> 65 #include <linux/kernel_stat.h> 66 #include <asm/fpu/internal.h> /* Ugh! */ 67 #include <asm/pvclock.h> 68 #include <asm/div64.h> 69 #include <asm/irq_remapping.h> 70 71 #define MAX_IO_MSRS 256 72 #define KVM_MAX_MCE_BANKS 32 73 #define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P) 74 75 #define emul_to_vcpu(ctxt) \ 76 container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt) 77 78 /* EFER defaults: 79 * - enable syscall per default because its emulated by KVM 80 * - enable LME and LMA per default on 64 bit KVM 81 */ 82 #ifdef CONFIG_X86_64 83 static 84 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA)); 85 #else 86 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); 87 #endif 88 89 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM 90 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 91 92 static void update_cr8_intercept(struct kvm_vcpu *vcpu); 93 static void process_nmi(struct kvm_vcpu *vcpu); 94 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); 95 96 struct kvm_x86_ops *kvm_x86_ops __read_mostly; 97 EXPORT_SYMBOL_GPL(kvm_x86_ops); 98 99 static bool __read_mostly ignore_msrs = 0; 100 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR); 101 102 unsigned int min_timer_period_us = 500; 103 module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR); 104 105 static bool __read_mostly kvmclock_periodic_sync = true; 106 module_param(kvmclock_periodic_sync, bool, S_IRUGO); 107 108 bool __read_mostly kvm_has_tsc_control; 109 EXPORT_SYMBOL_GPL(kvm_has_tsc_control); 110 u32 __read_mostly kvm_max_guest_tsc_khz; 111 EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz); 112 u8 __read_mostly kvm_tsc_scaling_ratio_frac_bits; 113 EXPORT_SYMBOL_GPL(kvm_tsc_scaling_ratio_frac_bits); 114 u64 __read_mostly kvm_max_tsc_scaling_ratio; 115 EXPORT_SYMBOL_GPL(kvm_max_tsc_scaling_ratio); 116 static u64 __read_mostly kvm_default_tsc_scaling_ratio; 117 118 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */ 119 static u32 __read_mostly tsc_tolerance_ppm = 250; 120 module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR); 121 122 /* lapic timer advance (tscdeadline mode only) in nanoseconds */ 123 unsigned int __read_mostly lapic_timer_advance_ns = 0; 124 module_param(lapic_timer_advance_ns, uint, S_IRUGO | S_IWUSR); 125 126 static bool __read_mostly backwards_tsc_observed = false; 127 128 #define KVM_NR_SHARED_MSRS 16 129 130 struct kvm_shared_msrs_global { 131 int nr; 132 u32 msrs[KVM_NR_SHARED_MSRS]; 133 }; 134 135 struct kvm_shared_msrs { 136 struct user_return_notifier urn; 137 bool registered; 138 struct kvm_shared_msr_values { 139 u64 host; 140 u64 curr; 141 } values[KVM_NR_SHARED_MSRS]; 142 }; 143 144 static struct kvm_shared_msrs_global __read_mostly shared_msrs_global; 145 static struct kvm_shared_msrs __percpu *shared_msrs; 146 147 struct kvm_stats_debugfs_item debugfs_entries[] = { 148 { "pf_fixed", VCPU_STAT(pf_fixed) }, 149 { "pf_guest", VCPU_STAT(pf_guest) }, 150 { "tlb_flush", VCPU_STAT(tlb_flush) }, 151 { "invlpg", VCPU_STAT(invlpg) }, 152 { "exits", VCPU_STAT(exits) }, 153 { "io_exits", VCPU_STAT(io_exits) }, 154 { "mmio_exits", VCPU_STAT(mmio_exits) }, 155 { "signal_exits", VCPU_STAT(signal_exits) }, 156 { "irq_window", VCPU_STAT(irq_window_exits) }, 157 { "nmi_window", VCPU_STAT(nmi_window_exits) }, 158 { "halt_exits", VCPU_STAT(halt_exits) }, 159 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, 160 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, 161 { "halt_wakeup", VCPU_STAT(halt_wakeup) }, 162 { "hypercalls", VCPU_STAT(hypercalls) }, 163 { "request_irq", VCPU_STAT(request_irq_exits) }, 164 { "irq_exits", VCPU_STAT(irq_exits) }, 165 { "host_state_reload", VCPU_STAT(host_state_reload) }, 166 { "efer_reload", VCPU_STAT(efer_reload) }, 167 { "fpu_reload", VCPU_STAT(fpu_reload) }, 168 { "insn_emulation", VCPU_STAT(insn_emulation) }, 169 { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, 170 { "irq_injections", VCPU_STAT(irq_injections) }, 171 { "nmi_injections", VCPU_STAT(nmi_injections) }, 172 { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, 173 { "mmu_pte_write", VM_STAT(mmu_pte_write) }, 174 { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, 175 { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, 176 { "mmu_flooded", VM_STAT(mmu_flooded) }, 177 { "mmu_recycled", VM_STAT(mmu_recycled) }, 178 { "mmu_cache_miss", VM_STAT(mmu_cache_miss) }, 179 { "mmu_unsync", VM_STAT(mmu_unsync) }, 180 { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, 181 { "largepages", VM_STAT(lpages) }, 182 { NULL } 183 }; 184 185 u64 __read_mostly host_xcr0; 186 187 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); 188 189 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) 190 { 191 int i; 192 for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++) 193 vcpu->arch.apf.gfns[i] = ~0; 194 } 195 196 static void kvm_on_user_return(struct user_return_notifier *urn) 197 { 198 unsigned slot; 199 struct kvm_shared_msrs *locals 200 = container_of(urn, struct kvm_shared_msrs, urn); 201 struct kvm_shared_msr_values *values; 202 203 for (slot = 0; slot < shared_msrs_global.nr; ++slot) { 204 values = &locals->values[slot]; 205 if (values->host != values->curr) { 206 wrmsrl(shared_msrs_global.msrs[slot], values->host); 207 values->curr = values->host; 208 } 209 } 210 locals->registered = false; 211 user_return_notifier_unregister(urn); 212 } 213 214 static void shared_msr_update(unsigned slot, u32 msr) 215 { 216 u64 value; 217 unsigned int cpu = smp_processor_id(); 218 struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu); 219 220 /* only read, and nobody should modify it at this time, 221 * so don't need lock */ 222 if (slot >= shared_msrs_global.nr) { 223 printk(KERN_ERR "kvm: invalid MSR slot!"); 224 return; 225 } 226 rdmsrl_safe(msr, &value); 227 smsr->values[slot].host = value; 228 smsr->values[slot].curr = value; 229 } 230 231 void kvm_define_shared_msr(unsigned slot, u32 msr) 232 { 233 BUG_ON(slot >= KVM_NR_SHARED_MSRS); 234 shared_msrs_global.msrs[slot] = msr; 235 if (slot >= shared_msrs_global.nr) 236 shared_msrs_global.nr = slot + 1; 237 } 238 EXPORT_SYMBOL_GPL(kvm_define_shared_msr); 239 240 static void kvm_shared_msr_cpu_online(void) 241 { 242 unsigned i; 243 244 for (i = 0; i < shared_msrs_global.nr; ++i) 245 shared_msr_update(i, shared_msrs_global.msrs[i]); 246 } 247 248 int kvm_set_shared_msr(unsigned slot, u64 value, u64 mask) 249 { 250 unsigned int cpu = smp_processor_id(); 251 struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu); 252 int err; 253 254 if (((value ^ smsr->values[slot].curr) & mask) == 0) 255 return 0; 256 smsr->values[slot].curr = value; 257 err = wrmsrl_safe(shared_msrs_global.msrs[slot], value); 258 if (err) 259 return 1; 260 261 if (!smsr->registered) { 262 smsr->urn.on_user_return = kvm_on_user_return; 263 user_return_notifier_register(&smsr->urn); 264 smsr->registered = true; 265 } 266 return 0; 267 } 268 EXPORT_SYMBOL_GPL(kvm_set_shared_msr); 269 270 static void drop_user_return_notifiers(void) 271 { 272 unsigned int cpu = smp_processor_id(); 273 struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu); 274 275 if (smsr->registered) 276 kvm_on_user_return(&smsr->urn); 277 } 278 279 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) 280 { 281 return vcpu->arch.apic_base; 282 } 283 EXPORT_SYMBOL_GPL(kvm_get_apic_base); 284 285 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 286 { 287 u64 old_state = vcpu->arch.apic_base & 288 (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE); 289 u64 new_state = msr_info->data & 290 (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE); 291 u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) | 292 0x2ff | (guest_cpuid_has_x2apic(vcpu) ? 0 : X2APIC_ENABLE); 293 294 if (!msr_info->host_initiated && 295 ((msr_info->data & reserved_bits) != 0 || 296 new_state == X2APIC_ENABLE || 297 (new_state == MSR_IA32_APICBASE_ENABLE && 298 old_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) || 299 (new_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE) && 300 old_state == 0))) 301 return 1; 302 303 kvm_lapic_set_base(vcpu, msr_info->data); 304 return 0; 305 } 306 EXPORT_SYMBOL_GPL(kvm_set_apic_base); 307 308 asmlinkage __visible void kvm_spurious_fault(void) 309 { 310 /* Fault while not rebooting. We want the trace. */ 311 BUG(); 312 } 313 EXPORT_SYMBOL_GPL(kvm_spurious_fault); 314 315 #define EXCPT_BENIGN 0 316 #define EXCPT_CONTRIBUTORY 1 317 #define EXCPT_PF 2 318 319 static int exception_class(int vector) 320 { 321 switch (vector) { 322 case PF_VECTOR: 323 return EXCPT_PF; 324 case DE_VECTOR: 325 case TS_VECTOR: 326 case NP_VECTOR: 327 case SS_VECTOR: 328 case GP_VECTOR: 329 return EXCPT_CONTRIBUTORY; 330 default: 331 break; 332 } 333 return EXCPT_BENIGN; 334 } 335 336 #define EXCPT_FAULT 0 337 #define EXCPT_TRAP 1 338 #define EXCPT_ABORT 2 339 #define EXCPT_INTERRUPT 3 340 341 static int exception_type(int vector) 342 { 343 unsigned int mask; 344 345 if (WARN_ON(vector > 31 || vector == NMI_VECTOR)) 346 return EXCPT_INTERRUPT; 347 348 mask = 1 << vector; 349 350 /* #DB is trap, as instruction watchpoints are handled elsewhere */ 351 if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR))) 352 return EXCPT_TRAP; 353 354 if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR))) 355 return EXCPT_ABORT; 356 357 /* Reserved exceptions will result in fault */ 358 return EXCPT_FAULT; 359 } 360 361 static void kvm_multiple_exception(struct kvm_vcpu *vcpu, 362 unsigned nr, bool has_error, u32 error_code, 363 bool reinject) 364 { 365 u32 prev_nr; 366 int class1, class2; 367 368 kvm_make_request(KVM_REQ_EVENT, vcpu); 369 370 if (!vcpu->arch.exception.pending) { 371 queue: 372 if (has_error && !is_protmode(vcpu)) 373 has_error = false; 374 vcpu->arch.exception.pending = true; 375 vcpu->arch.exception.has_error_code = has_error; 376 vcpu->arch.exception.nr = nr; 377 vcpu->arch.exception.error_code = error_code; 378 vcpu->arch.exception.reinject = reinject; 379 return; 380 } 381 382 /* to check exception */ 383 prev_nr = vcpu->arch.exception.nr; 384 if (prev_nr == DF_VECTOR) { 385 /* triple fault -> shutdown */ 386 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 387 return; 388 } 389 class1 = exception_class(prev_nr); 390 class2 = exception_class(nr); 391 if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) 392 || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) { 393 /* generate double fault per SDM Table 5-5 */ 394 vcpu->arch.exception.pending = true; 395 vcpu->arch.exception.has_error_code = true; 396 vcpu->arch.exception.nr = DF_VECTOR; 397 vcpu->arch.exception.error_code = 0; 398 } else 399 /* replace previous exception with a new one in a hope 400 that instruction re-execution will regenerate lost 401 exception */ 402 goto queue; 403 } 404 405 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) 406 { 407 kvm_multiple_exception(vcpu, nr, false, 0, false); 408 } 409 EXPORT_SYMBOL_GPL(kvm_queue_exception); 410 411 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) 412 { 413 kvm_multiple_exception(vcpu, nr, false, 0, true); 414 } 415 EXPORT_SYMBOL_GPL(kvm_requeue_exception); 416 417 void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) 418 { 419 if (err) 420 kvm_inject_gp(vcpu, 0); 421 else 422 kvm_x86_ops->skip_emulated_instruction(vcpu); 423 } 424 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp); 425 426 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) 427 { 428 ++vcpu->stat.pf_guest; 429 vcpu->arch.cr2 = fault->address; 430 kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code); 431 } 432 EXPORT_SYMBOL_GPL(kvm_inject_page_fault); 433 434 static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) 435 { 436 if (mmu_is_nested(vcpu) && !fault->nested_page_fault) 437 vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault); 438 else 439 vcpu->arch.mmu.inject_page_fault(vcpu, fault); 440 441 return fault->nested_page_fault; 442 } 443 444 void kvm_inject_nmi(struct kvm_vcpu *vcpu) 445 { 446 atomic_inc(&vcpu->arch.nmi_queued); 447 kvm_make_request(KVM_REQ_NMI, vcpu); 448 } 449 EXPORT_SYMBOL_GPL(kvm_inject_nmi); 450 451 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 452 { 453 kvm_multiple_exception(vcpu, nr, true, error_code, false); 454 } 455 EXPORT_SYMBOL_GPL(kvm_queue_exception_e); 456 457 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 458 { 459 kvm_multiple_exception(vcpu, nr, true, error_code, true); 460 } 461 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e); 462 463 /* 464 * Checks if cpl <= required_cpl; if true, return true. Otherwise queue 465 * a #GP and return false. 466 */ 467 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl) 468 { 469 if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl) 470 return true; 471 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 472 return false; 473 } 474 EXPORT_SYMBOL_GPL(kvm_require_cpl); 475 476 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) 477 { 478 if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE)) 479 return true; 480 481 kvm_queue_exception(vcpu, UD_VECTOR); 482 return false; 483 } 484 EXPORT_SYMBOL_GPL(kvm_require_dr); 485 486 /* 487 * This function will be used to read from the physical memory of the currently 488 * running guest. The difference to kvm_vcpu_read_guest_page is that this function 489 * can read from guest physical or from the guest's guest physical memory. 490 */ 491 int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 492 gfn_t ngfn, void *data, int offset, int len, 493 u32 access) 494 { 495 struct x86_exception exception; 496 gfn_t real_gfn; 497 gpa_t ngpa; 498 499 ngpa = gfn_to_gpa(ngfn); 500 real_gfn = mmu->translate_gpa(vcpu, ngpa, access, &exception); 501 if (real_gfn == UNMAPPED_GVA) 502 return -EFAULT; 503 504 real_gfn = gpa_to_gfn(real_gfn); 505 506 return kvm_vcpu_read_guest_page(vcpu, real_gfn, data, offset, len); 507 } 508 EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu); 509 510 static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, 511 void *data, int offset, int len, u32 access) 512 { 513 return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn, 514 data, offset, len, access); 515 } 516 517 /* 518 * Load the pae pdptrs. Return true is they are all valid. 519 */ 520 int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3) 521 { 522 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; 523 unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; 524 int i; 525 int ret; 526 u64 pdpte[ARRAY_SIZE(mmu->pdptrs)]; 527 528 ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte, 529 offset * sizeof(u64), sizeof(pdpte), 530 PFERR_USER_MASK|PFERR_WRITE_MASK); 531 if (ret < 0) { 532 ret = 0; 533 goto out; 534 } 535 for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { 536 if (is_present_gpte(pdpte[i]) && 537 (pdpte[i] & 538 vcpu->arch.mmu.guest_rsvd_check.rsvd_bits_mask[0][2])) { 539 ret = 0; 540 goto out; 541 } 542 } 543 ret = 1; 544 545 memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); 546 __set_bit(VCPU_EXREG_PDPTR, 547 (unsigned long *)&vcpu->arch.regs_avail); 548 __set_bit(VCPU_EXREG_PDPTR, 549 (unsigned long *)&vcpu->arch.regs_dirty); 550 out: 551 552 return ret; 553 } 554 EXPORT_SYMBOL_GPL(load_pdptrs); 555 556 static bool pdptrs_changed(struct kvm_vcpu *vcpu) 557 { 558 u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)]; 559 bool changed = true; 560 int offset; 561 gfn_t gfn; 562 int r; 563 564 if (is_long_mode(vcpu) || !is_pae(vcpu)) 565 return false; 566 567 if (!test_bit(VCPU_EXREG_PDPTR, 568 (unsigned long *)&vcpu->arch.regs_avail)) 569 return true; 570 571 gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT; 572 offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1); 573 r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte), 574 PFERR_USER_MASK | PFERR_WRITE_MASK); 575 if (r < 0) 576 goto out; 577 changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0; 578 out: 579 580 return changed; 581 } 582 583 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 584 { 585 unsigned long old_cr0 = kvm_read_cr0(vcpu); 586 unsigned long update_bits = X86_CR0_PG | X86_CR0_WP; 587 588 cr0 |= X86_CR0_ET; 589 590 #ifdef CONFIG_X86_64 591 if (cr0 & 0xffffffff00000000UL) 592 return 1; 593 #endif 594 595 cr0 &= ~CR0_RESERVED_BITS; 596 597 if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) 598 return 1; 599 600 if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) 601 return 1; 602 603 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { 604 #ifdef CONFIG_X86_64 605 if ((vcpu->arch.efer & EFER_LME)) { 606 int cs_db, cs_l; 607 608 if (!is_pae(vcpu)) 609 return 1; 610 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 611 if (cs_l) 612 return 1; 613 } else 614 #endif 615 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, 616 kvm_read_cr3(vcpu))) 617 return 1; 618 } 619 620 if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) 621 return 1; 622 623 kvm_x86_ops->set_cr0(vcpu, cr0); 624 625 if ((cr0 ^ old_cr0) & X86_CR0_PG) { 626 kvm_clear_async_pf_completion_queue(vcpu); 627 kvm_async_pf_hash_reset(vcpu); 628 } 629 630 if ((cr0 ^ old_cr0) & update_bits) 631 kvm_mmu_reset_context(vcpu); 632 633 if (((cr0 ^ old_cr0) & X86_CR0_CD) && 634 kvm_arch_has_noncoherent_dma(vcpu->kvm) && 635 !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 636 kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL); 637 638 return 0; 639 } 640 EXPORT_SYMBOL_GPL(kvm_set_cr0); 641 642 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) 643 { 644 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); 645 } 646 EXPORT_SYMBOL_GPL(kvm_lmsw); 647 648 static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu) 649 { 650 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) && 651 !vcpu->guest_xcr0_loaded) { 652 /* kvm_set_xcr() also depends on this */ 653 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0); 654 vcpu->guest_xcr0_loaded = 1; 655 } 656 } 657 658 static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu) 659 { 660 if (vcpu->guest_xcr0_loaded) { 661 if (vcpu->arch.xcr0 != host_xcr0) 662 xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0); 663 vcpu->guest_xcr0_loaded = 0; 664 } 665 } 666 667 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) 668 { 669 u64 xcr0 = xcr; 670 u64 old_xcr0 = vcpu->arch.xcr0; 671 u64 valid_bits; 672 673 /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */ 674 if (index != XCR_XFEATURE_ENABLED_MASK) 675 return 1; 676 if (!(xcr0 & XFEATURE_MASK_FP)) 677 return 1; 678 if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE)) 679 return 1; 680 681 /* 682 * Do not allow the guest to set bits that we do not support 683 * saving. However, xcr0 bit 0 is always set, even if the 684 * emulated CPU does not support XSAVE (see fx_init). 685 */ 686 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; 687 if (xcr0 & ~valid_bits) 688 return 1; 689 690 if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) != 691 (!(xcr0 & XFEATURE_MASK_BNDCSR))) 692 return 1; 693 694 if (xcr0 & XFEATURE_MASK_AVX512) { 695 if (!(xcr0 & XFEATURE_MASK_YMM)) 696 return 1; 697 if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512) 698 return 1; 699 } 700 kvm_put_guest_xcr0(vcpu); 701 vcpu->arch.xcr0 = xcr0; 702 703 if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND) 704 kvm_update_cpuid(vcpu); 705 return 0; 706 } 707 708 int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) 709 { 710 if (kvm_x86_ops->get_cpl(vcpu) != 0 || 711 __kvm_set_xcr(vcpu, index, xcr)) { 712 kvm_inject_gp(vcpu, 0); 713 return 1; 714 } 715 return 0; 716 } 717 EXPORT_SYMBOL_GPL(kvm_set_xcr); 718 719 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 720 { 721 unsigned long old_cr4 = kvm_read_cr4(vcpu); 722 unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | 723 X86_CR4_SMEP | X86_CR4_SMAP; 724 725 if (cr4 & CR4_RESERVED_BITS) 726 return 1; 727 728 if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE)) 729 return 1; 730 731 if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP)) 732 return 1; 733 734 if (!guest_cpuid_has_smap(vcpu) && (cr4 & X86_CR4_SMAP)) 735 return 1; 736 737 if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_FSGSBASE)) 738 return 1; 739 740 if (is_long_mode(vcpu)) { 741 if (!(cr4 & X86_CR4_PAE)) 742 return 1; 743 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) 744 && ((cr4 ^ old_cr4) & pdptr_bits) 745 && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, 746 kvm_read_cr3(vcpu))) 747 return 1; 748 749 if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { 750 if (!guest_cpuid_has_pcid(vcpu)) 751 return 1; 752 753 /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ 754 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) 755 return 1; 756 } 757 758 if (kvm_x86_ops->set_cr4(vcpu, cr4)) 759 return 1; 760 761 if (((cr4 ^ old_cr4) & pdptr_bits) || 762 (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) 763 kvm_mmu_reset_context(vcpu); 764 765 if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE) 766 kvm_update_cpuid(vcpu); 767 768 return 0; 769 } 770 EXPORT_SYMBOL_GPL(kvm_set_cr4); 771 772 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 773 { 774 #ifdef CONFIG_X86_64 775 cr3 &= ~CR3_PCID_INVD; 776 #endif 777 778 if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) { 779 kvm_mmu_sync_roots(vcpu); 780 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 781 return 0; 782 } 783 784 if (is_long_mode(vcpu)) { 785 if (cr3 & CR3_L_MODE_RESERVED_BITS) 786 return 1; 787 } else if (is_pae(vcpu) && is_paging(vcpu) && 788 !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) 789 return 1; 790 791 vcpu->arch.cr3 = cr3; 792 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 793 kvm_mmu_new_cr3(vcpu); 794 return 0; 795 } 796 EXPORT_SYMBOL_GPL(kvm_set_cr3); 797 798 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) 799 { 800 if (cr8 & CR8_RESERVED_BITS) 801 return 1; 802 if (lapic_in_kernel(vcpu)) 803 kvm_lapic_set_tpr(vcpu, cr8); 804 else 805 vcpu->arch.cr8 = cr8; 806 return 0; 807 } 808 EXPORT_SYMBOL_GPL(kvm_set_cr8); 809 810 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) 811 { 812 if (lapic_in_kernel(vcpu)) 813 return kvm_lapic_get_cr8(vcpu); 814 else 815 return vcpu->arch.cr8; 816 } 817 EXPORT_SYMBOL_GPL(kvm_get_cr8); 818 819 static void kvm_update_dr0123(struct kvm_vcpu *vcpu) 820 { 821 int i; 822 823 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { 824 for (i = 0; i < KVM_NR_DB_REGS; i++) 825 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 826 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_RELOAD; 827 } 828 } 829 830 static void kvm_update_dr6(struct kvm_vcpu *vcpu) 831 { 832 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) 833 kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6); 834 } 835 836 static void kvm_update_dr7(struct kvm_vcpu *vcpu) 837 { 838 unsigned long dr7; 839 840 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 841 dr7 = vcpu->arch.guest_debug_dr7; 842 else 843 dr7 = vcpu->arch.dr7; 844 kvm_x86_ops->set_dr7(vcpu, dr7); 845 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; 846 if (dr7 & DR7_BP_EN_MASK) 847 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; 848 } 849 850 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) 851 { 852 u64 fixed = DR6_FIXED_1; 853 854 if (!guest_cpuid_has_rtm(vcpu)) 855 fixed |= DR6_RTM; 856 return fixed; 857 } 858 859 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) 860 { 861 switch (dr) { 862 case 0 ... 3: 863 vcpu->arch.db[dr] = val; 864 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) 865 vcpu->arch.eff_db[dr] = val; 866 break; 867 case 4: 868 /* fall through */ 869 case 6: 870 if (val & 0xffffffff00000000ULL) 871 return -1; /* #GP */ 872 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); 873 kvm_update_dr6(vcpu); 874 break; 875 case 5: 876 /* fall through */ 877 default: /* 7 */ 878 if (val & 0xffffffff00000000ULL) 879 return -1; /* #GP */ 880 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; 881 kvm_update_dr7(vcpu); 882 break; 883 } 884 885 return 0; 886 } 887 888 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) 889 { 890 if (__kvm_set_dr(vcpu, dr, val)) { 891 kvm_inject_gp(vcpu, 0); 892 return 1; 893 } 894 return 0; 895 } 896 EXPORT_SYMBOL_GPL(kvm_set_dr); 897 898 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val) 899 { 900 switch (dr) { 901 case 0 ... 3: 902 *val = vcpu->arch.db[dr]; 903 break; 904 case 4: 905 /* fall through */ 906 case 6: 907 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 908 *val = vcpu->arch.dr6; 909 else 910 *val = kvm_x86_ops->get_dr6(vcpu); 911 break; 912 case 5: 913 /* fall through */ 914 default: /* 7 */ 915 *val = vcpu->arch.dr7; 916 break; 917 } 918 return 0; 919 } 920 EXPORT_SYMBOL_GPL(kvm_get_dr); 921 922 bool kvm_rdpmc(struct kvm_vcpu *vcpu) 923 { 924 u32 ecx = kvm_register_read(vcpu, VCPU_REGS_RCX); 925 u64 data; 926 int err; 927 928 err = kvm_pmu_rdpmc(vcpu, ecx, &data); 929 if (err) 930 return err; 931 kvm_register_write(vcpu, VCPU_REGS_RAX, (u32)data); 932 kvm_register_write(vcpu, VCPU_REGS_RDX, data >> 32); 933 return err; 934 } 935 EXPORT_SYMBOL_GPL(kvm_rdpmc); 936 937 /* 938 * List of msr numbers which we expose to userspace through KVM_GET_MSRS 939 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. 940 * 941 * This list is modified at module load time to reflect the 942 * capabilities of the host cpu. This capabilities test skips MSRs that are 943 * kvm-specific. Those are put in emulated_msrs; filtering of emulated_msrs 944 * may depend on host virtualization features rather than host cpu features. 945 */ 946 947 static u32 msrs_to_save[] = { 948 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, 949 MSR_STAR, 950 #ifdef CONFIG_X86_64 951 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, 952 #endif 953 MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA, 954 MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS 955 }; 956 957 static unsigned num_msrs_to_save; 958 959 static u32 emulated_msrs[] = { 960 MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, 961 MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, 962 HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, 963 HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC, 964 HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2, 965 HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL, 966 HV_X64_MSR_RESET, 967 HV_X64_MSR_VP_INDEX, 968 HV_X64_MSR_VP_RUNTIME, 969 HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, 970 MSR_KVM_PV_EOI_EN, 971 972 MSR_IA32_TSC_ADJUST, 973 MSR_IA32_TSCDEADLINE, 974 MSR_IA32_MISC_ENABLE, 975 MSR_IA32_MCG_STATUS, 976 MSR_IA32_MCG_CTL, 977 MSR_IA32_SMBASE, 978 }; 979 980 static unsigned num_emulated_msrs; 981 982 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) 983 { 984 if (efer & efer_reserved_bits) 985 return false; 986 987 if (efer & EFER_FFXSR) { 988 struct kvm_cpuid_entry2 *feat; 989 990 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 991 if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT))) 992 return false; 993 } 994 995 if (efer & EFER_SVME) { 996 struct kvm_cpuid_entry2 *feat; 997 998 feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); 999 if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM))) 1000 return false; 1001 } 1002 1003 return true; 1004 } 1005 EXPORT_SYMBOL_GPL(kvm_valid_efer); 1006 1007 static int set_efer(struct kvm_vcpu *vcpu, u64 efer) 1008 { 1009 u64 old_efer = vcpu->arch.efer; 1010 1011 if (!kvm_valid_efer(vcpu, efer)) 1012 return 1; 1013 1014 if (is_paging(vcpu) 1015 && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) 1016 return 1; 1017 1018 efer &= ~EFER_LMA; 1019 efer |= vcpu->arch.efer & EFER_LMA; 1020 1021 kvm_x86_ops->set_efer(vcpu, efer); 1022 1023 /* Update reserved bits */ 1024 if ((efer ^ old_efer) & EFER_NX) 1025 kvm_mmu_reset_context(vcpu); 1026 1027 return 0; 1028 } 1029 1030 void kvm_enable_efer_bits(u64 mask) 1031 { 1032 efer_reserved_bits &= ~mask; 1033 } 1034 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits); 1035 1036 /* 1037 * Writes msr value into into the appropriate "register". 1038 * Returns 0 on success, non-0 otherwise. 1039 * Assumes vcpu_load() was already called. 1040 */ 1041 int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 1042 { 1043 switch (msr->index) { 1044 case MSR_FS_BASE: 1045 case MSR_GS_BASE: 1046 case MSR_KERNEL_GS_BASE: 1047 case MSR_CSTAR: 1048 case MSR_LSTAR: 1049 if (is_noncanonical_address(msr->data)) 1050 return 1; 1051 break; 1052 case MSR_IA32_SYSENTER_EIP: 1053 case MSR_IA32_SYSENTER_ESP: 1054 /* 1055 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if 1056 * non-canonical address is written on Intel but not on 1057 * AMD (which ignores the top 32-bits, because it does 1058 * not implement 64-bit SYSENTER). 1059 * 1060 * 64-bit code should hence be able to write a non-canonical 1061 * value on AMD. Making the address canonical ensures that 1062 * vmentry does not fail on Intel after writing a non-canonical 1063 * value, and that something deterministic happens if the guest 1064 * invokes 64-bit SYSENTER. 1065 */ 1066 msr->data = get_canonical(msr->data); 1067 } 1068 return kvm_x86_ops->set_msr(vcpu, msr); 1069 } 1070 EXPORT_SYMBOL_GPL(kvm_set_msr); 1071 1072 /* 1073 * Adapt set_msr() to msr_io()'s calling convention 1074 */ 1075 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 1076 { 1077 struct msr_data msr; 1078 int r; 1079 1080 msr.index = index; 1081 msr.host_initiated = true; 1082 r = kvm_get_msr(vcpu, &msr); 1083 if (r) 1084 return r; 1085 1086 *data = msr.data; 1087 return 0; 1088 } 1089 1090 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 1091 { 1092 struct msr_data msr; 1093 1094 msr.data = *data; 1095 msr.index = index; 1096 msr.host_initiated = true; 1097 return kvm_set_msr(vcpu, &msr); 1098 } 1099 1100 #ifdef CONFIG_X86_64 1101 struct pvclock_gtod_data { 1102 seqcount_t seq; 1103 1104 struct { /* extract of a clocksource struct */ 1105 int vclock_mode; 1106 cycle_t cycle_last; 1107 cycle_t mask; 1108 u32 mult; 1109 u32 shift; 1110 } clock; 1111 1112 u64 boot_ns; 1113 u64 nsec_base; 1114 }; 1115 1116 static struct pvclock_gtod_data pvclock_gtod_data; 1117 1118 static void update_pvclock_gtod(struct timekeeper *tk) 1119 { 1120 struct pvclock_gtod_data *vdata = &pvclock_gtod_data; 1121 u64 boot_ns; 1122 1123 boot_ns = ktime_to_ns(ktime_add(tk->tkr_mono.base, tk->offs_boot)); 1124 1125 write_seqcount_begin(&vdata->seq); 1126 1127 /* copy pvclock gtod data */ 1128 vdata->clock.vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; 1129 vdata->clock.cycle_last = tk->tkr_mono.cycle_last; 1130 vdata->clock.mask = tk->tkr_mono.mask; 1131 vdata->clock.mult = tk->tkr_mono.mult; 1132 vdata->clock.shift = tk->tkr_mono.shift; 1133 1134 vdata->boot_ns = boot_ns; 1135 vdata->nsec_base = tk->tkr_mono.xtime_nsec; 1136 1137 write_seqcount_end(&vdata->seq); 1138 } 1139 #endif 1140 1141 void kvm_set_pending_timer(struct kvm_vcpu *vcpu) 1142 { 1143 /* 1144 * Note: KVM_REQ_PENDING_TIMER is implicitly checked in 1145 * vcpu_enter_guest. This function is only called from 1146 * the physical CPU that is running vcpu. 1147 */ 1148 kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu); 1149 } 1150 1151 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock) 1152 { 1153 int version; 1154 int r; 1155 struct pvclock_wall_clock wc; 1156 struct timespec boot; 1157 1158 if (!wall_clock) 1159 return; 1160 1161 r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version)); 1162 if (r) 1163 return; 1164 1165 if (version & 1) 1166 ++version; /* first time write, random junk */ 1167 1168 ++version; 1169 1170 kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); 1171 1172 /* 1173 * The guest calculates current wall clock time by adding 1174 * system time (updated by kvm_guest_time_update below) to the 1175 * wall clock specified here. guest system time equals host 1176 * system time for us, thus we must fill in host boot time here. 1177 */ 1178 getboottime(&boot); 1179 1180 if (kvm->arch.kvmclock_offset) { 1181 struct timespec ts = ns_to_timespec(kvm->arch.kvmclock_offset); 1182 boot = timespec_sub(boot, ts); 1183 } 1184 wc.sec = boot.tv_sec; 1185 wc.nsec = boot.tv_nsec; 1186 wc.version = version; 1187 1188 kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc)); 1189 1190 version++; 1191 kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); 1192 } 1193 1194 static uint32_t div_frac(uint32_t dividend, uint32_t divisor) 1195 { 1196 uint32_t quotient, remainder; 1197 1198 /* Don't try to replace with do_div(), this one calculates 1199 * "(dividend << 32) / divisor" */ 1200 __asm__ ( "divl %4" 1201 : "=a" (quotient), "=d" (remainder) 1202 : "0" (0), "1" (dividend), "r" (divisor) ); 1203 return quotient; 1204 } 1205 1206 static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz, 1207 s8 *pshift, u32 *pmultiplier) 1208 { 1209 uint64_t scaled64; 1210 int32_t shift = 0; 1211 uint64_t tps64; 1212 uint32_t tps32; 1213 1214 tps64 = base_khz * 1000LL; 1215 scaled64 = scaled_khz * 1000LL; 1216 while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) { 1217 tps64 >>= 1; 1218 shift--; 1219 } 1220 1221 tps32 = (uint32_t)tps64; 1222 while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) { 1223 if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000) 1224 scaled64 >>= 1; 1225 else 1226 tps32 <<= 1; 1227 shift++; 1228 } 1229 1230 *pshift = shift; 1231 *pmultiplier = div_frac(scaled64, tps32); 1232 1233 pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n", 1234 __func__, base_khz, scaled_khz, shift, *pmultiplier); 1235 } 1236 1237 #ifdef CONFIG_X86_64 1238 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0); 1239 #endif 1240 1241 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz); 1242 static unsigned long max_tsc_khz; 1243 1244 static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) 1245 { 1246 return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult, 1247 vcpu->arch.virtual_tsc_shift); 1248 } 1249 1250 static u32 adjust_tsc_khz(u32 khz, s32 ppm) 1251 { 1252 u64 v = (u64)khz * (1000000 + ppm); 1253 do_div(v, 1000000); 1254 return v; 1255 } 1256 1257 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale) 1258 { 1259 u64 ratio; 1260 1261 /* Guest TSC same frequency as host TSC? */ 1262 if (!scale) { 1263 vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; 1264 return 0; 1265 } 1266 1267 /* TSC scaling supported? */ 1268 if (!kvm_has_tsc_control) { 1269 if (user_tsc_khz > tsc_khz) { 1270 vcpu->arch.tsc_catchup = 1; 1271 vcpu->arch.tsc_always_catchup = 1; 1272 return 0; 1273 } else { 1274 WARN(1, "user requested TSC rate below hardware speed\n"); 1275 return -1; 1276 } 1277 } 1278 1279 /* TSC scaling required - calculate ratio */ 1280 ratio = mul_u64_u32_div(1ULL << kvm_tsc_scaling_ratio_frac_bits, 1281 user_tsc_khz, tsc_khz); 1282 1283 if (ratio == 0 || ratio >= kvm_max_tsc_scaling_ratio) { 1284 WARN_ONCE(1, "Invalid TSC scaling ratio - virtual-tsc-khz=%u\n", 1285 user_tsc_khz); 1286 return -1; 1287 } 1288 1289 vcpu->arch.tsc_scaling_ratio = ratio; 1290 return 0; 1291 } 1292 1293 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 this_tsc_khz) 1294 { 1295 u32 thresh_lo, thresh_hi; 1296 int use_scaling = 0; 1297 1298 /* tsc_khz can be zero if TSC calibration fails */ 1299 if (this_tsc_khz == 0) { 1300 /* set tsc_scaling_ratio to a safe value */ 1301 vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; 1302 return -1; 1303 } 1304 1305 /* Compute a scale to convert nanoseconds in TSC cycles */ 1306 kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000, 1307 &vcpu->arch.virtual_tsc_shift, 1308 &vcpu->arch.virtual_tsc_mult); 1309 vcpu->arch.virtual_tsc_khz = this_tsc_khz; 1310 1311 /* 1312 * Compute the variation in TSC rate which is acceptable 1313 * within the range of tolerance and decide if the 1314 * rate being applied is within that bounds of the hardware 1315 * rate. If so, no scaling or compensation need be done. 1316 */ 1317 thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm); 1318 thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm); 1319 if (this_tsc_khz < thresh_lo || this_tsc_khz > thresh_hi) { 1320 pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", this_tsc_khz, thresh_lo, thresh_hi); 1321 use_scaling = 1; 1322 } 1323 return set_tsc_khz(vcpu, this_tsc_khz, use_scaling); 1324 } 1325 1326 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) 1327 { 1328 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, 1329 vcpu->arch.virtual_tsc_mult, 1330 vcpu->arch.virtual_tsc_shift); 1331 tsc += vcpu->arch.this_tsc_write; 1332 return tsc; 1333 } 1334 1335 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) 1336 { 1337 #ifdef CONFIG_X86_64 1338 bool vcpus_matched; 1339 struct kvm_arch *ka = &vcpu->kvm->arch; 1340 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 1341 1342 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == 1343 atomic_read(&vcpu->kvm->online_vcpus)); 1344 1345 /* 1346 * Once the masterclock is enabled, always perform request in 1347 * order to update it. 1348 * 1349 * In order to enable masterclock, the host clocksource must be TSC 1350 * and the vcpus need to have matched TSCs. When that happens, 1351 * perform request to enable masterclock. 1352 */ 1353 if (ka->use_master_clock || 1354 (gtod->clock.vclock_mode == VCLOCK_TSC && vcpus_matched)) 1355 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 1356 1357 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc, 1358 atomic_read(&vcpu->kvm->online_vcpus), 1359 ka->use_master_clock, gtod->clock.vclock_mode); 1360 #endif 1361 } 1362 1363 static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset) 1364 { 1365 u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu); 1366 vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset; 1367 } 1368 1369 /* 1370 * Multiply tsc by a fixed point number represented by ratio. 1371 * 1372 * The most significant 64-N bits (mult) of ratio represent the 1373 * integral part of the fixed point number; the remaining N bits 1374 * (frac) represent the fractional part, ie. ratio represents a fixed 1375 * point number (mult + frac * 2^(-N)). 1376 * 1377 * N equals to kvm_tsc_scaling_ratio_frac_bits. 1378 */ 1379 static inline u64 __scale_tsc(u64 ratio, u64 tsc) 1380 { 1381 return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits); 1382 } 1383 1384 u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc) 1385 { 1386 u64 _tsc = tsc; 1387 u64 ratio = vcpu->arch.tsc_scaling_ratio; 1388 1389 if (ratio != kvm_default_tsc_scaling_ratio) 1390 _tsc = __scale_tsc(ratio, tsc); 1391 1392 return _tsc; 1393 } 1394 EXPORT_SYMBOL_GPL(kvm_scale_tsc); 1395 1396 static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) 1397 { 1398 u64 tsc; 1399 1400 tsc = kvm_scale_tsc(vcpu, rdtsc()); 1401 1402 return target_tsc - tsc; 1403 } 1404 1405 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) 1406 { 1407 return kvm_x86_ops->read_l1_tsc(vcpu, kvm_scale_tsc(vcpu, host_tsc)); 1408 } 1409 EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); 1410 1411 void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr) 1412 { 1413 struct kvm *kvm = vcpu->kvm; 1414 u64 offset, ns, elapsed; 1415 unsigned long flags; 1416 s64 usdiff; 1417 bool matched; 1418 bool already_matched; 1419 u64 data = msr->data; 1420 1421 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 1422 offset = kvm_compute_tsc_offset(vcpu, data); 1423 ns = get_kernel_ns(); 1424 elapsed = ns - kvm->arch.last_tsc_nsec; 1425 1426 if (vcpu->arch.virtual_tsc_khz) { 1427 int faulted = 0; 1428 1429 /* n.b - signed multiplication and division required */ 1430 usdiff = data - kvm->arch.last_tsc_write; 1431 #ifdef CONFIG_X86_64 1432 usdiff = (usdiff * 1000) / vcpu->arch.virtual_tsc_khz; 1433 #else 1434 /* do_div() only does unsigned */ 1435 asm("1: idivl %[divisor]\n" 1436 "2: xor %%edx, %%edx\n" 1437 " movl $0, %[faulted]\n" 1438 "3:\n" 1439 ".section .fixup,\"ax\"\n" 1440 "4: movl $1, %[faulted]\n" 1441 " jmp 3b\n" 1442 ".previous\n" 1443 1444 _ASM_EXTABLE(1b, 4b) 1445 1446 : "=A"(usdiff), [faulted] "=r" (faulted) 1447 : "A"(usdiff * 1000), [divisor] "rm"(vcpu->arch.virtual_tsc_khz)); 1448 1449 #endif 1450 do_div(elapsed, 1000); 1451 usdiff -= elapsed; 1452 if (usdiff < 0) 1453 usdiff = -usdiff; 1454 1455 /* idivl overflow => difference is larger than USEC_PER_SEC */ 1456 if (faulted) 1457 usdiff = USEC_PER_SEC; 1458 } else 1459 usdiff = USEC_PER_SEC; /* disable TSC match window below */ 1460 1461 /* 1462 * Special case: TSC write with a small delta (1 second) of virtual 1463 * cycle time against real time is interpreted as an attempt to 1464 * synchronize the CPU. 1465 * 1466 * For a reliable TSC, we can match TSC offsets, and for an unstable 1467 * TSC, we add elapsed time in this computation. We could let the 1468 * compensation code attempt to catch up if we fall behind, but 1469 * it's better to try to match offsets from the beginning. 1470 */ 1471 if (usdiff < USEC_PER_SEC && 1472 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) { 1473 if (!check_tsc_unstable()) { 1474 offset = kvm->arch.cur_tsc_offset; 1475 pr_debug("kvm: matched tsc offset for %llu\n", data); 1476 } else { 1477 u64 delta = nsec_to_cycles(vcpu, elapsed); 1478 data += delta; 1479 offset = kvm_compute_tsc_offset(vcpu, data); 1480 pr_debug("kvm: adjusted tsc offset by %llu\n", delta); 1481 } 1482 matched = true; 1483 already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation); 1484 } else { 1485 /* 1486 * We split periods of matched TSC writes into generations. 1487 * For each generation, we track the original measured 1488 * nanosecond time, offset, and write, so if TSCs are in 1489 * sync, we can match exact offset, and if not, we can match 1490 * exact software computation in compute_guest_tsc() 1491 * 1492 * These values are tracked in kvm->arch.cur_xxx variables. 1493 */ 1494 kvm->arch.cur_tsc_generation++; 1495 kvm->arch.cur_tsc_nsec = ns; 1496 kvm->arch.cur_tsc_write = data; 1497 kvm->arch.cur_tsc_offset = offset; 1498 matched = false; 1499 pr_debug("kvm: new tsc generation %llu, clock %llu\n", 1500 kvm->arch.cur_tsc_generation, data); 1501 } 1502 1503 /* 1504 * We also track th most recent recorded KHZ, write and time to 1505 * allow the matching interval to be extended at each write. 1506 */ 1507 kvm->arch.last_tsc_nsec = ns; 1508 kvm->arch.last_tsc_write = data; 1509 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; 1510 1511 vcpu->arch.last_guest_tsc = data; 1512 1513 /* Keep track of which generation this VCPU has synchronized to */ 1514 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; 1515 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; 1516 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; 1517 1518 if (guest_cpuid_has_tsc_adjust(vcpu) && !msr->host_initiated) 1519 update_ia32_tsc_adjust_msr(vcpu, offset); 1520 kvm_x86_ops->write_tsc_offset(vcpu, offset); 1521 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 1522 1523 spin_lock(&kvm->arch.pvclock_gtod_sync_lock); 1524 if (!matched) { 1525 kvm->arch.nr_vcpus_matched_tsc = 0; 1526 } else if (!already_matched) { 1527 kvm->arch.nr_vcpus_matched_tsc++; 1528 } 1529 1530 kvm_track_tsc_matching(vcpu); 1531 spin_unlock(&kvm->arch.pvclock_gtod_sync_lock); 1532 } 1533 1534 EXPORT_SYMBOL_GPL(kvm_write_tsc); 1535 1536 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, 1537 s64 adjustment) 1538 { 1539 kvm_x86_ops->adjust_tsc_offset_guest(vcpu, adjustment); 1540 } 1541 1542 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) 1543 { 1544 if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio) 1545 WARN_ON(adjustment < 0); 1546 adjustment = kvm_scale_tsc(vcpu, (u64) adjustment); 1547 kvm_x86_ops->adjust_tsc_offset_guest(vcpu, adjustment); 1548 } 1549 1550 #ifdef CONFIG_X86_64 1551 1552 static cycle_t read_tsc(void) 1553 { 1554 cycle_t ret = (cycle_t)rdtsc_ordered(); 1555 u64 last = pvclock_gtod_data.clock.cycle_last; 1556 1557 if (likely(ret >= last)) 1558 return ret; 1559 1560 /* 1561 * GCC likes to generate cmov here, but this branch is extremely 1562 * predictable (it's just a funciton of time and the likely is 1563 * very likely) and there's a data dependence, so force GCC 1564 * to generate a branch instead. I don't barrier() because 1565 * we don't actually need a barrier, and if this function 1566 * ever gets inlined it will generate worse code. 1567 */ 1568 asm volatile (""); 1569 return last; 1570 } 1571 1572 static inline u64 vgettsc(cycle_t *cycle_now) 1573 { 1574 long v; 1575 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 1576 1577 *cycle_now = read_tsc(); 1578 1579 v = (*cycle_now - gtod->clock.cycle_last) & gtod->clock.mask; 1580 return v * gtod->clock.mult; 1581 } 1582 1583 static int do_monotonic_boot(s64 *t, cycle_t *cycle_now) 1584 { 1585 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 1586 unsigned long seq; 1587 int mode; 1588 u64 ns; 1589 1590 do { 1591 seq = read_seqcount_begin(>od->seq); 1592 mode = gtod->clock.vclock_mode; 1593 ns = gtod->nsec_base; 1594 ns += vgettsc(cycle_now); 1595 ns >>= gtod->clock.shift; 1596 ns += gtod->boot_ns; 1597 } while (unlikely(read_seqcount_retry(>od->seq, seq))); 1598 *t = ns; 1599 1600 return mode; 1601 } 1602 1603 /* returns true if host is using tsc clocksource */ 1604 static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now) 1605 { 1606 /* checked again under seqlock below */ 1607 if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC) 1608 return false; 1609 1610 return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC; 1611 } 1612 #endif 1613 1614 /* 1615 * 1616 * Assuming a stable TSC across physical CPUS, and a stable TSC 1617 * across virtual CPUs, the following condition is possible. 1618 * Each numbered line represents an event visible to both 1619 * CPUs at the next numbered event. 1620 * 1621 * "timespecX" represents host monotonic time. "tscX" represents 1622 * RDTSC value. 1623 * 1624 * VCPU0 on CPU0 | VCPU1 on CPU1 1625 * 1626 * 1. read timespec0,tsc0 1627 * 2. | timespec1 = timespec0 + N 1628 * | tsc1 = tsc0 + M 1629 * 3. transition to guest | transition to guest 1630 * 4. ret0 = timespec0 + (rdtsc - tsc0) | 1631 * 5. | ret1 = timespec1 + (rdtsc - tsc1) 1632 * | ret1 = timespec0 + N + (rdtsc - (tsc0 + M)) 1633 * 1634 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity: 1635 * 1636 * - ret0 < ret1 1637 * - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M)) 1638 * ... 1639 * - 0 < N - M => M < N 1640 * 1641 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not 1642 * always the case (the difference between two distinct xtime instances 1643 * might be smaller then the difference between corresponding TSC reads, 1644 * when updating guest vcpus pvclock areas). 1645 * 1646 * To avoid that problem, do not allow visibility of distinct 1647 * system_timestamp/tsc_timestamp values simultaneously: use a master 1648 * copy of host monotonic time values. Update that master copy 1649 * in lockstep. 1650 * 1651 * Rely on synchronization of host TSCs and guest TSCs for monotonicity. 1652 * 1653 */ 1654 1655 static void pvclock_update_vm_gtod_copy(struct kvm *kvm) 1656 { 1657 #ifdef CONFIG_X86_64 1658 struct kvm_arch *ka = &kvm->arch; 1659 int vclock_mode; 1660 bool host_tsc_clocksource, vcpus_matched; 1661 1662 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == 1663 atomic_read(&kvm->online_vcpus)); 1664 1665 /* 1666 * If the host uses TSC clock, then passthrough TSC as stable 1667 * to the guest. 1668 */ 1669 host_tsc_clocksource = kvm_get_time_and_clockread( 1670 &ka->master_kernel_ns, 1671 &ka->master_cycle_now); 1672 1673 ka->use_master_clock = host_tsc_clocksource && vcpus_matched 1674 && !backwards_tsc_observed 1675 && !ka->boot_vcpu_runs_old_kvmclock; 1676 1677 if (ka->use_master_clock) 1678 atomic_set(&kvm_guest_has_master_clock, 1); 1679 1680 vclock_mode = pvclock_gtod_data.clock.vclock_mode; 1681 trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, 1682 vcpus_matched); 1683 #endif 1684 } 1685 1686 static void kvm_gen_update_masterclock(struct kvm *kvm) 1687 { 1688 #ifdef CONFIG_X86_64 1689 int i; 1690 struct kvm_vcpu *vcpu; 1691 struct kvm_arch *ka = &kvm->arch; 1692 1693 spin_lock(&ka->pvclock_gtod_sync_lock); 1694 kvm_make_mclock_inprogress_request(kvm); 1695 /* no guest entries from this point */ 1696 pvclock_update_vm_gtod_copy(kvm); 1697 1698 kvm_for_each_vcpu(i, vcpu, kvm) 1699 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1700 1701 /* guest entries allowed */ 1702 kvm_for_each_vcpu(i, vcpu, kvm) 1703 clear_bit(KVM_REQ_MCLOCK_INPROGRESS, &vcpu->requests); 1704 1705 spin_unlock(&ka->pvclock_gtod_sync_lock); 1706 #endif 1707 } 1708 1709 static int kvm_guest_time_update(struct kvm_vcpu *v) 1710 { 1711 unsigned long flags, this_tsc_khz, tgt_tsc_khz; 1712 struct kvm_vcpu_arch *vcpu = &v->arch; 1713 struct kvm_arch *ka = &v->kvm->arch; 1714 s64 kernel_ns; 1715 u64 tsc_timestamp, host_tsc; 1716 struct pvclock_vcpu_time_info guest_hv_clock; 1717 u8 pvclock_flags; 1718 bool use_master_clock; 1719 1720 kernel_ns = 0; 1721 host_tsc = 0; 1722 1723 /* 1724 * If the host uses TSC clock, then passthrough TSC as stable 1725 * to the guest. 1726 */ 1727 spin_lock(&ka->pvclock_gtod_sync_lock); 1728 use_master_clock = ka->use_master_clock; 1729 if (use_master_clock) { 1730 host_tsc = ka->master_cycle_now; 1731 kernel_ns = ka->master_kernel_ns; 1732 } 1733 spin_unlock(&ka->pvclock_gtod_sync_lock); 1734 1735 /* Keep irq disabled to prevent changes to the clock */ 1736 local_irq_save(flags); 1737 this_tsc_khz = __this_cpu_read(cpu_tsc_khz); 1738 if (unlikely(this_tsc_khz == 0)) { 1739 local_irq_restore(flags); 1740 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); 1741 return 1; 1742 } 1743 if (!use_master_clock) { 1744 host_tsc = rdtsc(); 1745 kernel_ns = get_kernel_ns(); 1746 } 1747 1748 tsc_timestamp = kvm_read_l1_tsc(v, host_tsc); 1749 1750 /* 1751 * We may have to catch up the TSC to match elapsed wall clock 1752 * time for two reasons, even if kvmclock is used. 1753 * 1) CPU could have been running below the maximum TSC rate 1754 * 2) Broken TSC compensation resets the base at each VCPU 1755 * entry to avoid unknown leaps of TSC even when running 1756 * again on the same CPU. This may cause apparent elapsed 1757 * time to disappear, and the guest to stand still or run 1758 * very slowly. 1759 */ 1760 if (vcpu->tsc_catchup) { 1761 u64 tsc = compute_guest_tsc(v, kernel_ns); 1762 if (tsc > tsc_timestamp) { 1763 adjust_tsc_offset_guest(v, tsc - tsc_timestamp); 1764 tsc_timestamp = tsc; 1765 } 1766 } 1767 1768 local_irq_restore(flags); 1769 1770 if (!vcpu->pv_time_enabled) 1771 return 0; 1772 1773 if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) { 1774 tgt_tsc_khz = kvm_has_tsc_control ? 1775 vcpu->virtual_tsc_khz : this_tsc_khz; 1776 kvm_get_time_scale(NSEC_PER_SEC / 1000, tgt_tsc_khz, 1777 &vcpu->hv_clock.tsc_shift, 1778 &vcpu->hv_clock.tsc_to_system_mul); 1779 vcpu->hw_tsc_khz = this_tsc_khz; 1780 } 1781 1782 /* With all the info we got, fill in the values */ 1783 vcpu->hv_clock.tsc_timestamp = tsc_timestamp; 1784 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; 1785 vcpu->last_guest_tsc = tsc_timestamp; 1786 1787 if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time, 1788 &guest_hv_clock, sizeof(guest_hv_clock)))) 1789 return 0; 1790 1791 /* This VCPU is paused, but it's legal for a guest to read another 1792 * VCPU's kvmclock, so we really have to follow the specification where 1793 * it says that version is odd if data is being modified, and even after 1794 * it is consistent. 1795 * 1796 * Version field updates must be kept separate. This is because 1797 * kvm_write_guest_cached might use a "rep movs" instruction, and 1798 * writes within a string instruction are weakly ordered. So there 1799 * are three writes overall. 1800 * 1801 * As a small optimization, only write the version field in the first 1802 * and third write. The vcpu->pv_time cache is still valid, because the 1803 * version field is the first in the struct. 1804 */ 1805 BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0); 1806 1807 vcpu->hv_clock.version = guest_hv_clock.version + 1; 1808 kvm_write_guest_cached(v->kvm, &vcpu->pv_time, 1809 &vcpu->hv_clock, 1810 sizeof(vcpu->hv_clock.version)); 1811 1812 smp_wmb(); 1813 1814 /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */ 1815 pvclock_flags = (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED); 1816 1817 if (vcpu->pvclock_set_guest_stopped_request) { 1818 pvclock_flags |= PVCLOCK_GUEST_STOPPED; 1819 vcpu->pvclock_set_guest_stopped_request = false; 1820 } 1821 1822 /* If the host uses TSC clocksource, then it is stable */ 1823 if (use_master_clock) 1824 pvclock_flags |= PVCLOCK_TSC_STABLE_BIT; 1825 1826 vcpu->hv_clock.flags = pvclock_flags; 1827 1828 trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock); 1829 1830 kvm_write_guest_cached(v->kvm, &vcpu->pv_time, 1831 &vcpu->hv_clock, 1832 sizeof(vcpu->hv_clock)); 1833 1834 smp_wmb(); 1835 1836 vcpu->hv_clock.version++; 1837 kvm_write_guest_cached(v->kvm, &vcpu->pv_time, 1838 &vcpu->hv_clock, 1839 sizeof(vcpu->hv_clock.version)); 1840 return 0; 1841 } 1842 1843 /* 1844 * kvmclock updates which are isolated to a given vcpu, such as 1845 * vcpu->cpu migration, should not allow system_timestamp from 1846 * the rest of the vcpus to remain static. Otherwise ntp frequency 1847 * correction applies to one vcpu's system_timestamp but not 1848 * the others. 1849 * 1850 * So in those cases, request a kvmclock update for all vcpus. 1851 * We need to rate-limit these requests though, as they can 1852 * considerably slow guests that have a large number of vcpus. 1853 * The time for a remote vcpu to update its kvmclock is bound 1854 * by the delay we use to rate-limit the updates. 1855 */ 1856 1857 #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100) 1858 1859 static void kvmclock_update_fn(struct work_struct *work) 1860 { 1861 int i; 1862 struct delayed_work *dwork = to_delayed_work(work); 1863 struct kvm_arch *ka = container_of(dwork, struct kvm_arch, 1864 kvmclock_update_work); 1865 struct kvm *kvm = container_of(ka, struct kvm, arch); 1866 struct kvm_vcpu *vcpu; 1867 1868 kvm_for_each_vcpu(i, vcpu, kvm) { 1869 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 1870 kvm_vcpu_kick(vcpu); 1871 } 1872 } 1873 1874 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v) 1875 { 1876 struct kvm *kvm = v->kvm; 1877 1878 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); 1879 schedule_delayed_work(&kvm->arch.kvmclock_update_work, 1880 KVMCLOCK_UPDATE_DELAY); 1881 } 1882 1883 #define KVMCLOCK_SYNC_PERIOD (300 * HZ) 1884 1885 static void kvmclock_sync_fn(struct work_struct *work) 1886 { 1887 struct delayed_work *dwork = to_delayed_work(work); 1888 struct kvm_arch *ka = container_of(dwork, struct kvm_arch, 1889 kvmclock_sync_work); 1890 struct kvm *kvm = container_of(ka, struct kvm, arch); 1891 1892 if (!kvmclock_periodic_sync) 1893 return; 1894 1895 schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0); 1896 schedule_delayed_work(&kvm->arch.kvmclock_sync_work, 1897 KVMCLOCK_SYNC_PERIOD); 1898 } 1899 1900 static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data) 1901 { 1902 u64 mcg_cap = vcpu->arch.mcg_cap; 1903 unsigned bank_num = mcg_cap & 0xff; 1904 1905 switch (msr) { 1906 case MSR_IA32_MCG_STATUS: 1907 vcpu->arch.mcg_status = data; 1908 break; 1909 case MSR_IA32_MCG_CTL: 1910 if (!(mcg_cap & MCG_CTL_P)) 1911 return 1; 1912 if (data != 0 && data != ~(u64)0) 1913 return -1; 1914 vcpu->arch.mcg_ctl = data; 1915 break; 1916 default: 1917 if (msr >= MSR_IA32_MC0_CTL && 1918 msr < MSR_IA32_MCx_CTL(bank_num)) { 1919 u32 offset = msr - MSR_IA32_MC0_CTL; 1920 /* only 0 or all 1s can be written to IA32_MCi_CTL 1921 * some Linux kernels though clear bit 10 in bank 4 to 1922 * workaround a BIOS/GART TBL issue on AMD K8s, ignore 1923 * this to avoid an uncatched #GP in the guest 1924 */ 1925 if ((offset & 0x3) == 0 && 1926 data != 0 && (data | (1 << 10)) != ~(u64)0) 1927 return -1; 1928 vcpu->arch.mce_banks[offset] = data; 1929 break; 1930 } 1931 return 1; 1932 } 1933 return 0; 1934 } 1935 1936 static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data) 1937 { 1938 struct kvm *kvm = vcpu->kvm; 1939 int lm = is_long_mode(vcpu); 1940 u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64 1941 : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32; 1942 u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64 1943 : kvm->arch.xen_hvm_config.blob_size_32; 1944 u32 page_num = data & ~PAGE_MASK; 1945 u64 page_addr = data & PAGE_MASK; 1946 u8 *page; 1947 int r; 1948 1949 r = -E2BIG; 1950 if (page_num >= blob_size) 1951 goto out; 1952 r = -ENOMEM; 1953 page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE); 1954 if (IS_ERR(page)) { 1955 r = PTR_ERR(page); 1956 goto out; 1957 } 1958 if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) 1959 goto out_free; 1960 r = 0; 1961 out_free: 1962 kfree(page); 1963 out: 1964 return r; 1965 } 1966 1967 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) 1968 { 1969 gpa_t gpa = data & ~0x3f; 1970 1971 /* Bits 2:5 are reserved, Should be zero */ 1972 if (data & 0x3c) 1973 return 1; 1974 1975 vcpu->arch.apf.msr_val = data; 1976 1977 if (!(data & KVM_ASYNC_PF_ENABLED)) { 1978 kvm_clear_async_pf_completion_queue(vcpu); 1979 kvm_async_pf_hash_reset(vcpu); 1980 return 0; 1981 } 1982 1983 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, 1984 sizeof(u32))) 1985 return 1; 1986 1987 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); 1988 kvm_async_pf_wakeup_all(vcpu); 1989 return 0; 1990 } 1991 1992 static void kvmclock_reset(struct kvm_vcpu *vcpu) 1993 { 1994 vcpu->arch.pv_time_enabled = false; 1995 } 1996 1997 static void accumulate_steal_time(struct kvm_vcpu *vcpu) 1998 { 1999 u64 delta; 2000 2001 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 2002 return; 2003 2004 delta = current->sched_info.run_delay - vcpu->arch.st.last_steal; 2005 vcpu->arch.st.last_steal = current->sched_info.run_delay; 2006 vcpu->arch.st.accum_steal = delta; 2007 } 2008 2009 static void record_steal_time(struct kvm_vcpu *vcpu) 2010 { 2011 accumulate_steal_time(vcpu); 2012 2013 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 2014 return; 2015 2016 if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime, 2017 &vcpu->arch.st.steal, sizeof(struct kvm_steal_time)))) 2018 return; 2019 2020 vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal; 2021 vcpu->arch.st.steal.version += 2; 2022 vcpu->arch.st.accum_steal = 0; 2023 2024 kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime, 2025 &vcpu->arch.st.steal, sizeof(struct kvm_steal_time)); 2026 } 2027 2028 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2029 { 2030 bool pr = false; 2031 u32 msr = msr_info->index; 2032 u64 data = msr_info->data; 2033 2034 switch (msr) { 2035 case MSR_AMD64_NB_CFG: 2036 case MSR_IA32_UCODE_REV: 2037 case MSR_IA32_UCODE_WRITE: 2038 case MSR_VM_HSAVE_PA: 2039 case MSR_AMD64_PATCH_LOADER: 2040 case MSR_AMD64_BU_CFG2: 2041 break; 2042 2043 case MSR_EFER: 2044 return set_efer(vcpu, data); 2045 case MSR_K7_HWCR: 2046 data &= ~(u64)0x40; /* ignore flush filter disable */ 2047 data &= ~(u64)0x100; /* ignore ignne emulation enable */ 2048 data &= ~(u64)0x8; /* ignore TLB cache disable */ 2049 data &= ~(u64)0x40000; /* ignore Mc status write enable */ 2050 if (data != 0) { 2051 vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n", 2052 data); 2053 return 1; 2054 } 2055 break; 2056 case MSR_FAM10H_MMIO_CONF_BASE: 2057 if (data != 0) { 2058 vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: " 2059 "0x%llx\n", data); 2060 return 1; 2061 } 2062 break; 2063 case MSR_IA32_DEBUGCTLMSR: 2064 if (!data) { 2065 /* We support the non-activated case already */ 2066 break; 2067 } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) { 2068 /* Values other than LBR and BTF are vendor-specific, 2069 thus reserved and should throw a #GP */ 2070 return 1; 2071 } 2072 vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n", 2073 __func__, data); 2074 break; 2075 case 0x200 ... 0x2ff: 2076 return kvm_mtrr_set_msr(vcpu, msr, data); 2077 case MSR_IA32_APICBASE: 2078 return kvm_set_apic_base(vcpu, msr_info); 2079 case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff: 2080 return kvm_x2apic_msr_write(vcpu, msr, data); 2081 case MSR_IA32_TSCDEADLINE: 2082 kvm_set_lapic_tscdeadline_msr(vcpu, data); 2083 break; 2084 case MSR_IA32_TSC_ADJUST: 2085 if (guest_cpuid_has_tsc_adjust(vcpu)) { 2086 if (!msr_info->host_initiated) { 2087 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr; 2088 adjust_tsc_offset_guest(vcpu, adj); 2089 } 2090 vcpu->arch.ia32_tsc_adjust_msr = data; 2091 } 2092 break; 2093 case MSR_IA32_MISC_ENABLE: 2094 vcpu->arch.ia32_misc_enable_msr = data; 2095 break; 2096 case MSR_IA32_SMBASE: 2097 if (!msr_info->host_initiated) 2098 return 1; 2099 vcpu->arch.smbase = data; 2100 break; 2101 case MSR_KVM_WALL_CLOCK_NEW: 2102 case MSR_KVM_WALL_CLOCK: 2103 vcpu->kvm->arch.wall_clock = data; 2104 kvm_write_wall_clock(vcpu->kvm, data); 2105 break; 2106 case MSR_KVM_SYSTEM_TIME_NEW: 2107 case MSR_KVM_SYSTEM_TIME: { 2108 u64 gpa_offset; 2109 struct kvm_arch *ka = &vcpu->kvm->arch; 2110 2111 kvmclock_reset(vcpu); 2112 2113 if (vcpu->vcpu_id == 0 && !msr_info->host_initiated) { 2114 bool tmp = (msr == MSR_KVM_SYSTEM_TIME); 2115 2116 if (ka->boot_vcpu_runs_old_kvmclock != tmp) 2117 set_bit(KVM_REQ_MASTERCLOCK_UPDATE, 2118 &vcpu->requests); 2119 2120 ka->boot_vcpu_runs_old_kvmclock = tmp; 2121 } 2122 2123 vcpu->arch.time = data; 2124 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); 2125 2126 /* we verify if the enable bit is set... */ 2127 if (!(data & 1)) 2128 break; 2129 2130 gpa_offset = data & ~(PAGE_MASK | 1); 2131 2132 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, 2133 &vcpu->arch.pv_time, data & ~1ULL, 2134 sizeof(struct pvclock_vcpu_time_info))) 2135 vcpu->arch.pv_time_enabled = false; 2136 else 2137 vcpu->arch.pv_time_enabled = true; 2138 2139 break; 2140 } 2141 case MSR_KVM_ASYNC_PF_EN: 2142 if (kvm_pv_enable_async_pf(vcpu, data)) 2143 return 1; 2144 break; 2145 case MSR_KVM_STEAL_TIME: 2146 2147 if (unlikely(!sched_info_on())) 2148 return 1; 2149 2150 if (data & KVM_STEAL_RESERVED_MASK) 2151 return 1; 2152 2153 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime, 2154 data & KVM_STEAL_VALID_BITS, 2155 sizeof(struct kvm_steal_time))) 2156 return 1; 2157 2158 vcpu->arch.st.msr_val = data; 2159 2160 if (!(data & KVM_MSR_ENABLED)) 2161 break; 2162 2163 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 2164 2165 break; 2166 case MSR_KVM_PV_EOI_EN: 2167 if (kvm_lapic_enable_pv_eoi(vcpu, data)) 2168 return 1; 2169 break; 2170 2171 case MSR_IA32_MCG_CTL: 2172 case MSR_IA32_MCG_STATUS: 2173 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 2174 return set_msr_mce(vcpu, msr, data); 2175 2176 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: 2177 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: 2178 pr = true; /* fall through */ 2179 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: 2180 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: 2181 if (kvm_pmu_is_valid_msr(vcpu, msr)) 2182 return kvm_pmu_set_msr(vcpu, msr_info); 2183 2184 if (pr || data != 0) 2185 vcpu_unimpl(vcpu, "disabled perfctr wrmsr: " 2186 "0x%x data 0x%llx\n", msr, data); 2187 break; 2188 case MSR_K7_CLK_CTL: 2189 /* 2190 * Ignore all writes to this no longer documented MSR. 2191 * Writes are only relevant for old K7 processors, 2192 * all pre-dating SVM, but a recommended workaround from 2193 * AMD for these chips. It is possible to specify the 2194 * affected processor models on the command line, hence 2195 * the need to ignore the workaround. 2196 */ 2197 break; 2198 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 2199 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 2200 case HV_X64_MSR_CRASH_CTL: 2201 return kvm_hv_set_msr_common(vcpu, msr, data, 2202 msr_info->host_initiated); 2203 case MSR_IA32_BBL_CR_CTL3: 2204 /* Drop writes to this legacy MSR -- see rdmsr 2205 * counterpart for further detail. 2206 */ 2207 vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data); 2208 break; 2209 case MSR_AMD64_OSVW_ID_LENGTH: 2210 if (!guest_cpuid_has_osvw(vcpu)) 2211 return 1; 2212 vcpu->arch.osvw.length = data; 2213 break; 2214 case MSR_AMD64_OSVW_STATUS: 2215 if (!guest_cpuid_has_osvw(vcpu)) 2216 return 1; 2217 vcpu->arch.osvw.status = data; 2218 break; 2219 default: 2220 if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr)) 2221 return xen_hvm_config(vcpu, data); 2222 if (kvm_pmu_is_valid_msr(vcpu, msr)) 2223 return kvm_pmu_set_msr(vcpu, msr_info); 2224 if (!ignore_msrs) { 2225 vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", 2226 msr, data); 2227 return 1; 2228 } else { 2229 vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", 2230 msr, data); 2231 break; 2232 } 2233 } 2234 return 0; 2235 } 2236 EXPORT_SYMBOL_GPL(kvm_set_msr_common); 2237 2238 2239 /* 2240 * Reads an msr value (of 'msr_index') into 'pdata'. 2241 * Returns 0 on success, non-0 otherwise. 2242 * Assumes vcpu_load() was already called. 2243 */ 2244 int kvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) 2245 { 2246 return kvm_x86_ops->get_msr(vcpu, msr); 2247 } 2248 EXPORT_SYMBOL_GPL(kvm_get_msr); 2249 2250 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) 2251 { 2252 u64 data; 2253 u64 mcg_cap = vcpu->arch.mcg_cap; 2254 unsigned bank_num = mcg_cap & 0xff; 2255 2256 switch (msr) { 2257 case MSR_IA32_P5_MC_ADDR: 2258 case MSR_IA32_P5_MC_TYPE: 2259 data = 0; 2260 break; 2261 case MSR_IA32_MCG_CAP: 2262 data = vcpu->arch.mcg_cap; 2263 break; 2264 case MSR_IA32_MCG_CTL: 2265 if (!(mcg_cap & MCG_CTL_P)) 2266 return 1; 2267 data = vcpu->arch.mcg_ctl; 2268 break; 2269 case MSR_IA32_MCG_STATUS: 2270 data = vcpu->arch.mcg_status; 2271 break; 2272 default: 2273 if (msr >= MSR_IA32_MC0_CTL && 2274 msr < MSR_IA32_MCx_CTL(bank_num)) { 2275 u32 offset = msr - MSR_IA32_MC0_CTL; 2276 data = vcpu->arch.mce_banks[offset]; 2277 break; 2278 } 2279 return 1; 2280 } 2281 *pdata = data; 2282 return 0; 2283 } 2284 2285 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 2286 { 2287 switch (msr_info->index) { 2288 case MSR_IA32_PLATFORM_ID: 2289 case MSR_IA32_EBL_CR_POWERON: 2290 case MSR_IA32_DEBUGCTLMSR: 2291 case MSR_IA32_LASTBRANCHFROMIP: 2292 case MSR_IA32_LASTBRANCHTOIP: 2293 case MSR_IA32_LASTINTFROMIP: 2294 case MSR_IA32_LASTINTTOIP: 2295 case MSR_K8_SYSCFG: 2296 case MSR_K8_TSEG_ADDR: 2297 case MSR_K8_TSEG_MASK: 2298 case MSR_K7_HWCR: 2299 case MSR_VM_HSAVE_PA: 2300 case MSR_K8_INT_PENDING_MSG: 2301 case MSR_AMD64_NB_CFG: 2302 case MSR_FAM10H_MMIO_CONF_BASE: 2303 case MSR_AMD64_BU_CFG2: 2304 msr_info->data = 0; 2305 break; 2306 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: 2307 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: 2308 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: 2309 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: 2310 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) 2311 return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data); 2312 msr_info->data = 0; 2313 break; 2314 case MSR_IA32_UCODE_REV: 2315 msr_info->data = 0x100000000ULL; 2316 break; 2317 case MSR_MTRRcap: 2318 case 0x200 ... 0x2ff: 2319 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data); 2320 case 0xcd: /* fsb frequency */ 2321 msr_info->data = 3; 2322 break; 2323 /* 2324 * MSR_EBC_FREQUENCY_ID 2325 * Conservative value valid for even the basic CPU models. 2326 * Models 0,1: 000 in bits 23:21 indicating a bus speed of 2327 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz, 2328 * and 266MHz for model 3, or 4. Set Core Clock 2329 * Frequency to System Bus Frequency Ratio to 1 (bits 2330 * 31:24) even though these are only valid for CPU 2331 * models > 2, however guests may end up dividing or 2332 * multiplying by zero otherwise. 2333 */ 2334 case MSR_EBC_FREQUENCY_ID: 2335 msr_info->data = 1 << 24; 2336 break; 2337 case MSR_IA32_APICBASE: 2338 msr_info->data = kvm_get_apic_base(vcpu); 2339 break; 2340 case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff: 2341 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data); 2342 break; 2343 case MSR_IA32_TSCDEADLINE: 2344 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu); 2345 break; 2346 case MSR_IA32_TSC_ADJUST: 2347 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr; 2348 break; 2349 case MSR_IA32_MISC_ENABLE: 2350 msr_info->data = vcpu->arch.ia32_misc_enable_msr; 2351 break; 2352 case MSR_IA32_SMBASE: 2353 if (!msr_info->host_initiated) 2354 return 1; 2355 msr_info->data = vcpu->arch.smbase; 2356 break; 2357 case MSR_IA32_PERF_STATUS: 2358 /* TSC increment by tick */ 2359 msr_info->data = 1000ULL; 2360 /* CPU multiplier */ 2361 msr_info->data |= (((uint64_t)4ULL) << 40); 2362 break; 2363 case MSR_EFER: 2364 msr_info->data = vcpu->arch.efer; 2365 break; 2366 case MSR_KVM_WALL_CLOCK: 2367 case MSR_KVM_WALL_CLOCK_NEW: 2368 msr_info->data = vcpu->kvm->arch.wall_clock; 2369 break; 2370 case MSR_KVM_SYSTEM_TIME: 2371 case MSR_KVM_SYSTEM_TIME_NEW: 2372 msr_info->data = vcpu->arch.time; 2373 break; 2374 case MSR_KVM_ASYNC_PF_EN: 2375 msr_info->data = vcpu->arch.apf.msr_val; 2376 break; 2377 case MSR_KVM_STEAL_TIME: 2378 msr_info->data = vcpu->arch.st.msr_val; 2379 break; 2380 case MSR_KVM_PV_EOI_EN: 2381 msr_info->data = vcpu->arch.pv_eoi.msr_val; 2382 break; 2383 case MSR_IA32_P5_MC_ADDR: 2384 case MSR_IA32_P5_MC_TYPE: 2385 case MSR_IA32_MCG_CAP: 2386 case MSR_IA32_MCG_CTL: 2387 case MSR_IA32_MCG_STATUS: 2388 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 2389 return get_msr_mce(vcpu, msr_info->index, &msr_info->data); 2390 case MSR_K7_CLK_CTL: 2391 /* 2392 * Provide expected ramp-up count for K7. All other 2393 * are set to zero, indicating minimum divisors for 2394 * every field. 2395 * 2396 * This prevents guest kernels on AMD host with CPU 2397 * type 6, model 8 and higher from exploding due to 2398 * the rdmsr failing. 2399 */ 2400 msr_info->data = 0x20000000; 2401 break; 2402 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 2403 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 2404 case HV_X64_MSR_CRASH_CTL: 2405 return kvm_hv_get_msr_common(vcpu, 2406 msr_info->index, &msr_info->data); 2407 break; 2408 case MSR_IA32_BBL_CR_CTL3: 2409 /* This legacy MSR exists but isn't fully documented in current 2410 * silicon. It is however accessed by winxp in very narrow 2411 * scenarios where it sets bit #19, itself documented as 2412 * a "reserved" bit. Best effort attempt to source coherent 2413 * read data here should the balance of the register be 2414 * interpreted by the guest: 2415 * 2416 * L2 cache control register 3: 64GB range, 256KB size, 2417 * enabled, latency 0x1, configured 2418 */ 2419 msr_info->data = 0xbe702111; 2420 break; 2421 case MSR_AMD64_OSVW_ID_LENGTH: 2422 if (!guest_cpuid_has_osvw(vcpu)) 2423 return 1; 2424 msr_info->data = vcpu->arch.osvw.length; 2425 break; 2426 case MSR_AMD64_OSVW_STATUS: 2427 if (!guest_cpuid_has_osvw(vcpu)) 2428 return 1; 2429 msr_info->data = vcpu->arch.osvw.status; 2430 break; 2431 default: 2432 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) 2433 return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data); 2434 if (!ignore_msrs) { 2435 vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr_info->index); 2436 return 1; 2437 } else { 2438 vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr_info->index); 2439 msr_info->data = 0; 2440 } 2441 break; 2442 } 2443 return 0; 2444 } 2445 EXPORT_SYMBOL_GPL(kvm_get_msr_common); 2446 2447 /* 2448 * Read or write a bunch of msrs. All parameters are kernel addresses. 2449 * 2450 * @return number of msrs set successfully. 2451 */ 2452 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, 2453 struct kvm_msr_entry *entries, 2454 int (*do_msr)(struct kvm_vcpu *vcpu, 2455 unsigned index, u64 *data)) 2456 { 2457 int i, idx; 2458 2459 idx = srcu_read_lock(&vcpu->kvm->srcu); 2460 for (i = 0; i < msrs->nmsrs; ++i) 2461 if (do_msr(vcpu, entries[i].index, &entries[i].data)) 2462 break; 2463 srcu_read_unlock(&vcpu->kvm->srcu, idx); 2464 2465 return i; 2466 } 2467 2468 /* 2469 * Read or write a bunch of msrs. Parameters are user addresses. 2470 * 2471 * @return number of msrs set successfully. 2472 */ 2473 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, 2474 int (*do_msr)(struct kvm_vcpu *vcpu, 2475 unsigned index, u64 *data), 2476 int writeback) 2477 { 2478 struct kvm_msrs msrs; 2479 struct kvm_msr_entry *entries; 2480 int r, n; 2481 unsigned size; 2482 2483 r = -EFAULT; 2484 if (copy_from_user(&msrs, user_msrs, sizeof msrs)) 2485 goto out; 2486 2487 r = -E2BIG; 2488 if (msrs.nmsrs >= MAX_IO_MSRS) 2489 goto out; 2490 2491 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; 2492 entries = memdup_user(user_msrs->entries, size); 2493 if (IS_ERR(entries)) { 2494 r = PTR_ERR(entries); 2495 goto out; 2496 } 2497 2498 r = n = __msr_io(vcpu, &msrs, entries, do_msr); 2499 if (r < 0) 2500 goto out_free; 2501 2502 r = -EFAULT; 2503 if (writeback && copy_to_user(user_msrs->entries, entries, size)) 2504 goto out_free; 2505 2506 r = n; 2507 2508 out_free: 2509 kfree(entries); 2510 out: 2511 return r; 2512 } 2513 2514 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 2515 { 2516 int r; 2517 2518 switch (ext) { 2519 case KVM_CAP_IRQCHIP: 2520 case KVM_CAP_HLT: 2521 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: 2522 case KVM_CAP_SET_TSS_ADDR: 2523 case KVM_CAP_EXT_CPUID: 2524 case KVM_CAP_EXT_EMUL_CPUID: 2525 case KVM_CAP_CLOCKSOURCE: 2526 case KVM_CAP_PIT: 2527 case KVM_CAP_NOP_IO_DELAY: 2528 case KVM_CAP_MP_STATE: 2529 case KVM_CAP_SYNC_MMU: 2530 case KVM_CAP_USER_NMI: 2531 case KVM_CAP_REINJECT_CONTROL: 2532 case KVM_CAP_IRQ_INJECT_STATUS: 2533 case KVM_CAP_IOEVENTFD: 2534 case KVM_CAP_IOEVENTFD_NO_LENGTH: 2535 case KVM_CAP_PIT2: 2536 case KVM_CAP_PIT_STATE2: 2537 case KVM_CAP_SET_IDENTITY_MAP_ADDR: 2538 case KVM_CAP_XEN_HVM: 2539 case KVM_CAP_ADJUST_CLOCK: 2540 case KVM_CAP_VCPU_EVENTS: 2541 case KVM_CAP_HYPERV: 2542 case KVM_CAP_HYPERV_VAPIC: 2543 case KVM_CAP_HYPERV_SPIN: 2544 case KVM_CAP_PCI_SEGMENT: 2545 case KVM_CAP_DEBUGREGS: 2546 case KVM_CAP_X86_ROBUST_SINGLESTEP: 2547 case KVM_CAP_XSAVE: 2548 case KVM_CAP_ASYNC_PF: 2549 case KVM_CAP_GET_TSC_KHZ: 2550 case KVM_CAP_KVMCLOCK_CTRL: 2551 case KVM_CAP_READONLY_MEM: 2552 case KVM_CAP_HYPERV_TIME: 2553 case KVM_CAP_IOAPIC_POLARITY_IGNORED: 2554 case KVM_CAP_TSC_DEADLINE_TIMER: 2555 case KVM_CAP_ENABLE_CAP_VM: 2556 case KVM_CAP_DISABLE_QUIRKS: 2557 case KVM_CAP_SET_BOOT_CPU_ID: 2558 case KVM_CAP_SPLIT_IRQCHIP: 2559 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT 2560 case KVM_CAP_ASSIGN_DEV_IRQ: 2561 case KVM_CAP_PCI_2_3: 2562 #endif 2563 r = 1; 2564 break; 2565 case KVM_CAP_X86_SMM: 2566 /* SMBASE is usually relocated above 1M on modern chipsets, 2567 * and SMM handlers might indeed rely on 4G segment limits, 2568 * so do not report SMM to be available if real mode is 2569 * emulated via vm86 mode. Still, do not go to great lengths 2570 * to avoid userspace's usage of the feature, because it is a 2571 * fringe case that is not enabled except via specific settings 2572 * of the module parameters. 2573 */ 2574 r = kvm_x86_ops->cpu_has_high_real_mode_segbase(); 2575 break; 2576 case KVM_CAP_COALESCED_MMIO: 2577 r = KVM_COALESCED_MMIO_PAGE_OFFSET; 2578 break; 2579 case KVM_CAP_VAPIC: 2580 r = !kvm_x86_ops->cpu_has_accelerated_tpr(); 2581 break; 2582 case KVM_CAP_NR_VCPUS: 2583 r = KVM_SOFT_MAX_VCPUS; 2584 break; 2585 case KVM_CAP_MAX_VCPUS: 2586 r = KVM_MAX_VCPUS; 2587 break; 2588 case KVM_CAP_NR_MEMSLOTS: 2589 r = KVM_USER_MEM_SLOTS; 2590 break; 2591 case KVM_CAP_PV_MMU: /* obsolete */ 2592 r = 0; 2593 break; 2594 #ifdef CONFIG_KVM_DEVICE_ASSIGNMENT 2595 case KVM_CAP_IOMMU: 2596 r = iommu_present(&pci_bus_type); 2597 break; 2598 #endif 2599 case KVM_CAP_MCE: 2600 r = KVM_MAX_MCE_BANKS; 2601 break; 2602 case KVM_CAP_XCRS: 2603 r = cpu_has_xsave; 2604 break; 2605 case KVM_CAP_TSC_CONTROL: 2606 r = kvm_has_tsc_control; 2607 break; 2608 default: 2609 r = 0; 2610 break; 2611 } 2612 return r; 2613 2614 } 2615 2616 long kvm_arch_dev_ioctl(struct file *filp, 2617 unsigned int ioctl, unsigned long arg) 2618 { 2619 void __user *argp = (void __user *)arg; 2620 long r; 2621 2622 switch (ioctl) { 2623 case KVM_GET_MSR_INDEX_LIST: { 2624 struct kvm_msr_list __user *user_msr_list = argp; 2625 struct kvm_msr_list msr_list; 2626 unsigned n; 2627 2628 r = -EFAULT; 2629 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) 2630 goto out; 2631 n = msr_list.nmsrs; 2632 msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs; 2633 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) 2634 goto out; 2635 r = -E2BIG; 2636 if (n < msr_list.nmsrs) 2637 goto out; 2638 r = -EFAULT; 2639 if (copy_to_user(user_msr_list->indices, &msrs_to_save, 2640 num_msrs_to_save * sizeof(u32))) 2641 goto out; 2642 if (copy_to_user(user_msr_list->indices + num_msrs_to_save, 2643 &emulated_msrs, 2644 num_emulated_msrs * sizeof(u32))) 2645 goto out; 2646 r = 0; 2647 break; 2648 } 2649 case KVM_GET_SUPPORTED_CPUID: 2650 case KVM_GET_EMULATED_CPUID: { 2651 struct kvm_cpuid2 __user *cpuid_arg = argp; 2652 struct kvm_cpuid2 cpuid; 2653 2654 r = -EFAULT; 2655 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 2656 goto out; 2657 2658 r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries, 2659 ioctl); 2660 if (r) 2661 goto out; 2662 2663 r = -EFAULT; 2664 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) 2665 goto out; 2666 r = 0; 2667 break; 2668 } 2669 case KVM_X86_GET_MCE_CAP_SUPPORTED: { 2670 u64 mce_cap; 2671 2672 mce_cap = KVM_MCE_CAP_SUPPORTED; 2673 r = -EFAULT; 2674 if (copy_to_user(argp, &mce_cap, sizeof mce_cap)) 2675 goto out; 2676 r = 0; 2677 break; 2678 } 2679 default: 2680 r = -EINVAL; 2681 } 2682 out: 2683 return r; 2684 } 2685 2686 static void wbinvd_ipi(void *garbage) 2687 { 2688 wbinvd(); 2689 } 2690 2691 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu) 2692 { 2693 return kvm_arch_has_noncoherent_dma(vcpu->kvm); 2694 } 2695 2696 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2697 { 2698 /* Address WBINVD may be executed by guest */ 2699 if (need_emulate_wbinvd(vcpu)) { 2700 if (kvm_x86_ops->has_wbinvd_exit()) 2701 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 2702 else if (vcpu->cpu != -1 && vcpu->cpu != cpu) 2703 smp_call_function_single(vcpu->cpu, 2704 wbinvd_ipi, NULL, 1); 2705 } 2706 2707 kvm_x86_ops->vcpu_load(vcpu, cpu); 2708 2709 /* Apply any externally detected TSC adjustments (due to suspend) */ 2710 if (unlikely(vcpu->arch.tsc_offset_adjustment)) { 2711 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment); 2712 vcpu->arch.tsc_offset_adjustment = 0; 2713 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2714 } 2715 2716 if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) { 2717 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 : 2718 rdtsc() - vcpu->arch.last_host_tsc; 2719 if (tsc_delta < 0) 2720 mark_tsc_unstable("KVM discovered backwards TSC"); 2721 if (check_tsc_unstable()) { 2722 u64 offset = kvm_compute_tsc_offset(vcpu, 2723 vcpu->arch.last_guest_tsc); 2724 kvm_x86_ops->write_tsc_offset(vcpu, offset); 2725 vcpu->arch.tsc_catchup = 1; 2726 } 2727 /* 2728 * On a host with synchronized TSC, there is no need to update 2729 * kvmclock on vcpu->cpu migration 2730 */ 2731 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) 2732 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); 2733 if (vcpu->cpu != cpu) 2734 kvm_migrate_timers(vcpu); 2735 vcpu->cpu = cpu; 2736 } 2737 2738 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 2739 } 2740 2741 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 2742 { 2743 kvm_x86_ops->vcpu_put(vcpu); 2744 kvm_put_guest_fpu(vcpu); 2745 vcpu->arch.last_host_tsc = rdtsc(); 2746 } 2747 2748 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, 2749 struct kvm_lapic_state *s) 2750 { 2751 kvm_x86_ops->sync_pir_to_irr(vcpu); 2752 memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s); 2753 2754 return 0; 2755 } 2756 2757 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, 2758 struct kvm_lapic_state *s) 2759 { 2760 kvm_apic_post_state_restore(vcpu, s); 2761 update_cr8_intercept(vcpu); 2762 2763 return 0; 2764 } 2765 2766 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, 2767 struct kvm_interrupt *irq) 2768 { 2769 if (irq->irq >= KVM_NR_INTERRUPTS) 2770 return -EINVAL; 2771 2772 if (!irqchip_in_kernel(vcpu->kvm)) { 2773 kvm_queue_interrupt(vcpu, irq->irq, false); 2774 kvm_make_request(KVM_REQ_EVENT, vcpu); 2775 return 0; 2776 } 2777 2778 /* 2779 * With in-kernel LAPIC, we only use this to inject EXTINT, so 2780 * fail for in-kernel 8259. 2781 */ 2782 if (pic_in_kernel(vcpu->kvm)) 2783 return -ENXIO; 2784 2785 if (vcpu->arch.pending_external_vector != -1) 2786 return -EEXIST; 2787 2788 vcpu->arch.pending_external_vector = irq->irq; 2789 return 0; 2790 } 2791 2792 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu) 2793 { 2794 kvm_inject_nmi(vcpu); 2795 2796 return 0; 2797 } 2798 2799 static int kvm_vcpu_ioctl_smi(struct kvm_vcpu *vcpu) 2800 { 2801 kvm_make_request(KVM_REQ_SMI, vcpu); 2802 2803 return 0; 2804 } 2805 2806 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu, 2807 struct kvm_tpr_access_ctl *tac) 2808 { 2809 if (tac->flags) 2810 return -EINVAL; 2811 vcpu->arch.tpr_access_reporting = !!tac->enabled; 2812 return 0; 2813 } 2814 2815 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, 2816 u64 mcg_cap) 2817 { 2818 int r; 2819 unsigned bank_num = mcg_cap & 0xff, bank; 2820 2821 r = -EINVAL; 2822 if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS) 2823 goto out; 2824 if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000)) 2825 goto out; 2826 r = 0; 2827 vcpu->arch.mcg_cap = mcg_cap; 2828 /* Init IA32_MCG_CTL to all 1s */ 2829 if (mcg_cap & MCG_CTL_P) 2830 vcpu->arch.mcg_ctl = ~(u64)0; 2831 /* Init IA32_MCi_CTL to all 1s */ 2832 for (bank = 0; bank < bank_num; bank++) 2833 vcpu->arch.mce_banks[bank*4] = ~(u64)0; 2834 out: 2835 return r; 2836 } 2837 2838 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, 2839 struct kvm_x86_mce *mce) 2840 { 2841 u64 mcg_cap = vcpu->arch.mcg_cap; 2842 unsigned bank_num = mcg_cap & 0xff; 2843 u64 *banks = vcpu->arch.mce_banks; 2844 2845 if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL)) 2846 return -EINVAL; 2847 /* 2848 * if IA32_MCG_CTL is not all 1s, the uncorrected error 2849 * reporting is disabled 2850 */ 2851 if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) && 2852 vcpu->arch.mcg_ctl != ~(u64)0) 2853 return 0; 2854 banks += 4 * mce->bank; 2855 /* 2856 * if IA32_MCi_CTL is not all 1s, the uncorrected error 2857 * reporting is disabled for the bank 2858 */ 2859 if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0) 2860 return 0; 2861 if (mce->status & MCI_STATUS_UC) { 2862 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) || 2863 !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) { 2864 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 2865 return 0; 2866 } 2867 if (banks[1] & MCI_STATUS_VAL) 2868 mce->status |= MCI_STATUS_OVER; 2869 banks[2] = mce->addr; 2870 banks[3] = mce->misc; 2871 vcpu->arch.mcg_status = mce->mcg_status; 2872 banks[1] = mce->status; 2873 kvm_queue_exception(vcpu, MC_VECTOR); 2874 } else if (!(banks[1] & MCI_STATUS_VAL) 2875 || !(banks[1] & MCI_STATUS_UC)) { 2876 if (banks[1] & MCI_STATUS_VAL) 2877 mce->status |= MCI_STATUS_OVER; 2878 banks[2] = mce->addr; 2879 banks[3] = mce->misc; 2880 banks[1] = mce->status; 2881 } else 2882 banks[1] |= MCI_STATUS_OVER; 2883 return 0; 2884 } 2885 2886 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, 2887 struct kvm_vcpu_events *events) 2888 { 2889 process_nmi(vcpu); 2890 events->exception.injected = 2891 vcpu->arch.exception.pending && 2892 !kvm_exception_is_soft(vcpu->arch.exception.nr); 2893 events->exception.nr = vcpu->arch.exception.nr; 2894 events->exception.has_error_code = vcpu->arch.exception.has_error_code; 2895 events->exception.pad = 0; 2896 events->exception.error_code = vcpu->arch.exception.error_code; 2897 2898 events->interrupt.injected = 2899 vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft; 2900 events->interrupt.nr = vcpu->arch.interrupt.nr; 2901 events->interrupt.soft = 0; 2902 events->interrupt.shadow = kvm_x86_ops->get_interrupt_shadow(vcpu); 2903 2904 events->nmi.injected = vcpu->arch.nmi_injected; 2905 events->nmi.pending = vcpu->arch.nmi_pending != 0; 2906 events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu); 2907 events->nmi.pad = 0; 2908 2909 events->sipi_vector = 0; /* never valid when reporting to user space */ 2910 2911 events->smi.smm = is_smm(vcpu); 2912 events->smi.pending = vcpu->arch.smi_pending; 2913 events->smi.smm_inside_nmi = 2914 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK); 2915 events->smi.latched_init = kvm_lapic_latched_init(vcpu); 2916 2917 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING 2918 | KVM_VCPUEVENT_VALID_SHADOW 2919 | KVM_VCPUEVENT_VALID_SMM); 2920 memset(&events->reserved, 0, sizeof(events->reserved)); 2921 } 2922 2923 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, 2924 struct kvm_vcpu_events *events) 2925 { 2926 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING 2927 | KVM_VCPUEVENT_VALID_SIPI_VECTOR 2928 | KVM_VCPUEVENT_VALID_SHADOW 2929 | KVM_VCPUEVENT_VALID_SMM)) 2930 return -EINVAL; 2931 2932 process_nmi(vcpu); 2933 vcpu->arch.exception.pending = events->exception.injected; 2934 vcpu->arch.exception.nr = events->exception.nr; 2935 vcpu->arch.exception.has_error_code = events->exception.has_error_code; 2936 vcpu->arch.exception.error_code = events->exception.error_code; 2937 2938 vcpu->arch.interrupt.pending = events->interrupt.injected; 2939 vcpu->arch.interrupt.nr = events->interrupt.nr; 2940 vcpu->arch.interrupt.soft = events->interrupt.soft; 2941 if (events->flags & KVM_VCPUEVENT_VALID_SHADOW) 2942 kvm_x86_ops->set_interrupt_shadow(vcpu, 2943 events->interrupt.shadow); 2944 2945 vcpu->arch.nmi_injected = events->nmi.injected; 2946 if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) 2947 vcpu->arch.nmi_pending = events->nmi.pending; 2948 kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked); 2949 2950 if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR && 2951 kvm_vcpu_has_lapic(vcpu)) 2952 vcpu->arch.apic->sipi_vector = events->sipi_vector; 2953 2954 if (events->flags & KVM_VCPUEVENT_VALID_SMM) { 2955 if (events->smi.smm) 2956 vcpu->arch.hflags |= HF_SMM_MASK; 2957 else 2958 vcpu->arch.hflags &= ~HF_SMM_MASK; 2959 vcpu->arch.smi_pending = events->smi.pending; 2960 if (events->smi.smm_inside_nmi) 2961 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; 2962 else 2963 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK; 2964 if (kvm_vcpu_has_lapic(vcpu)) { 2965 if (events->smi.latched_init) 2966 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 2967 else 2968 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 2969 } 2970 } 2971 2972 kvm_make_request(KVM_REQ_EVENT, vcpu); 2973 2974 return 0; 2975 } 2976 2977 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, 2978 struct kvm_debugregs *dbgregs) 2979 { 2980 unsigned long val; 2981 2982 memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db)); 2983 kvm_get_dr(vcpu, 6, &val); 2984 dbgregs->dr6 = val; 2985 dbgregs->dr7 = vcpu->arch.dr7; 2986 dbgregs->flags = 0; 2987 memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved)); 2988 } 2989 2990 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, 2991 struct kvm_debugregs *dbgregs) 2992 { 2993 if (dbgregs->flags) 2994 return -EINVAL; 2995 2996 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db)); 2997 kvm_update_dr0123(vcpu); 2998 vcpu->arch.dr6 = dbgregs->dr6; 2999 kvm_update_dr6(vcpu); 3000 vcpu->arch.dr7 = dbgregs->dr7; 3001 kvm_update_dr7(vcpu); 3002 3003 return 0; 3004 } 3005 3006 #define XSTATE_COMPACTION_ENABLED (1ULL << 63) 3007 3008 static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu) 3009 { 3010 struct xregs_state *xsave = &vcpu->arch.guest_fpu.state.xsave; 3011 u64 xstate_bv = xsave->header.xfeatures; 3012 u64 valid; 3013 3014 /* 3015 * Copy legacy XSAVE area, to avoid complications with CPUID 3016 * leaves 0 and 1 in the loop below. 3017 */ 3018 memcpy(dest, xsave, XSAVE_HDR_OFFSET); 3019 3020 /* Set XSTATE_BV */ 3021 *(u64 *)(dest + XSAVE_HDR_OFFSET) = xstate_bv; 3022 3023 /* 3024 * Copy each region from the possibly compacted offset to the 3025 * non-compacted offset. 3026 */ 3027 valid = xstate_bv & ~XFEATURE_MASK_FPSSE; 3028 while (valid) { 3029 u64 feature = valid & -valid; 3030 int index = fls64(feature) - 1; 3031 void *src = get_xsave_addr(xsave, feature); 3032 3033 if (src) { 3034 u32 size, offset, ecx, edx; 3035 cpuid_count(XSTATE_CPUID, index, 3036 &size, &offset, &ecx, &edx); 3037 memcpy(dest + offset, src, size); 3038 } 3039 3040 valid -= feature; 3041 } 3042 } 3043 3044 static void load_xsave(struct kvm_vcpu *vcpu, u8 *src) 3045 { 3046 struct xregs_state *xsave = &vcpu->arch.guest_fpu.state.xsave; 3047 u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET); 3048 u64 valid; 3049 3050 /* 3051 * Copy legacy XSAVE area, to avoid complications with CPUID 3052 * leaves 0 and 1 in the loop below. 3053 */ 3054 memcpy(xsave, src, XSAVE_HDR_OFFSET); 3055 3056 /* Set XSTATE_BV and possibly XCOMP_BV. */ 3057 xsave->header.xfeatures = xstate_bv; 3058 if (cpu_has_xsaves) 3059 xsave->header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED; 3060 3061 /* 3062 * Copy each region from the non-compacted offset to the 3063 * possibly compacted offset. 3064 */ 3065 valid = xstate_bv & ~XFEATURE_MASK_FPSSE; 3066 while (valid) { 3067 u64 feature = valid & -valid; 3068 int index = fls64(feature) - 1; 3069 void *dest = get_xsave_addr(xsave, feature); 3070 3071 if (dest) { 3072 u32 size, offset, ecx, edx; 3073 cpuid_count(XSTATE_CPUID, index, 3074 &size, &offset, &ecx, &edx); 3075 memcpy(dest, src + offset, size); 3076 } 3077 3078 valid -= feature; 3079 } 3080 } 3081 3082 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, 3083 struct kvm_xsave *guest_xsave) 3084 { 3085 if (cpu_has_xsave) { 3086 memset(guest_xsave, 0, sizeof(struct kvm_xsave)); 3087 fill_xsave((u8 *) guest_xsave->region, vcpu); 3088 } else { 3089 memcpy(guest_xsave->region, 3090 &vcpu->arch.guest_fpu.state.fxsave, 3091 sizeof(struct fxregs_state)); 3092 *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] = 3093 XFEATURE_MASK_FPSSE; 3094 } 3095 } 3096 3097 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, 3098 struct kvm_xsave *guest_xsave) 3099 { 3100 u64 xstate_bv = 3101 *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)]; 3102 3103 if (cpu_has_xsave) { 3104 /* 3105 * Here we allow setting states that are not present in 3106 * CPUID leaf 0xD, index 0, EDX:EAX. This is for compatibility 3107 * with old userspace. 3108 */ 3109 if (xstate_bv & ~kvm_supported_xcr0()) 3110 return -EINVAL; 3111 load_xsave(vcpu, (u8 *)guest_xsave->region); 3112 } else { 3113 if (xstate_bv & ~XFEATURE_MASK_FPSSE) 3114 return -EINVAL; 3115 memcpy(&vcpu->arch.guest_fpu.state.fxsave, 3116 guest_xsave->region, sizeof(struct fxregs_state)); 3117 } 3118 return 0; 3119 } 3120 3121 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu, 3122 struct kvm_xcrs *guest_xcrs) 3123 { 3124 if (!cpu_has_xsave) { 3125 guest_xcrs->nr_xcrs = 0; 3126 return; 3127 } 3128 3129 guest_xcrs->nr_xcrs = 1; 3130 guest_xcrs->flags = 0; 3131 guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK; 3132 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0; 3133 } 3134 3135 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, 3136 struct kvm_xcrs *guest_xcrs) 3137 { 3138 int i, r = 0; 3139 3140 if (!cpu_has_xsave) 3141 return -EINVAL; 3142 3143 if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags) 3144 return -EINVAL; 3145 3146 for (i = 0; i < guest_xcrs->nr_xcrs; i++) 3147 /* Only support XCR0 currently */ 3148 if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) { 3149 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK, 3150 guest_xcrs->xcrs[i].value); 3151 break; 3152 } 3153 if (r) 3154 r = -EINVAL; 3155 return r; 3156 } 3157 3158 /* 3159 * kvm_set_guest_paused() indicates to the guest kernel that it has been 3160 * stopped by the hypervisor. This function will be called from the host only. 3161 * EINVAL is returned when the host attempts to set the flag for a guest that 3162 * does not support pv clocks. 3163 */ 3164 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) 3165 { 3166 if (!vcpu->arch.pv_time_enabled) 3167 return -EINVAL; 3168 vcpu->arch.pvclock_set_guest_stopped_request = true; 3169 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 3170 return 0; 3171 } 3172 3173 long kvm_arch_vcpu_ioctl(struct file *filp, 3174 unsigned int ioctl, unsigned long arg) 3175 { 3176 struct kvm_vcpu *vcpu = filp->private_data; 3177 void __user *argp = (void __user *)arg; 3178 int r; 3179 union { 3180 struct kvm_lapic_state *lapic; 3181 struct kvm_xsave *xsave; 3182 struct kvm_xcrs *xcrs; 3183 void *buffer; 3184 } u; 3185 3186 u.buffer = NULL; 3187 switch (ioctl) { 3188 case KVM_GET_LAPIC: { 3189 r = -EINVAL; 3190 if (!vcpu->arch.apic) 3191 goto out; 3192 u.lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL); 3193 3194 r = -ENOMEM; 3195 if (!u.lapic) 3196 goto out; 3197 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic); 3198 if (r) 3199 goto out; 3200 r = -EFAULT; 3201 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state))) 3202 goto out; 3203 r = 0; 3204 break; 3205 } 3206 case KVM_SET_LAPIC: { 3207 r = -EINVAL; 3208 if (!vcpu->arch.apic) 3209 goto out; 3210 u.lapic = memdup_user(argp, sizeof(*u.lapic)); 3211 if (IS_ERR(u.lapic)) 3212 return PTR_ERR(u.lapic); 3213 3214 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic); 3215 break; 3216 } 3217 case KVM_INTERRUPT: { 3218 struct kvm_interrupt irq; 3219 3220 r = -EFAULT; 3221 if (copy_from_user(&irq, argp, sizeof irq)) 3222 goto out; 3223 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 3224 break; 3225 } 3226 case KVM_NMI: { 3227 r = kvm_vcpu_ioctl_nmi(vcpu); 3228 break; 3229 } 3230 case KVM_SMI: { 3231 r = kvm_vcpu_ioctl_smi(vcpu); 3232 break; 3233 } 3234 case KVM_SET_CPUID: { 3235 struct kvm_cpuid __user *cpuid_arg = argp; 3236 struct kvm_cpuid cpuid; 3237 3238 r = -EFAULT; 3239 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 3240 goto out; 3241 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); 3242 break; 3243 } 3244 case KVM_SET_CPUID2: { 3245 struct kvm_cpuid2 __user *cpuid_arg = argp; 3246 struct kvm_cpuid2 cpuid; 3247 3248 r = -EFAULT; 3249 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 3250 goto out; 3251 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, 3252 cpuid_arg->entries); 3253 break; 3254 } 3255 case KVM_GET_CPUID2: { 3256 struct kvm_cpuid2 __user *cpuid_arg = argp; 3257 struct kvm_cpuid2 cpuid; 3258 3259 r = -EFAULT; 3260 if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) 3261 goto out; 3262 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, 3263 cpuid_arg->entries); 3264 if (r) 3265 goto out; 3266 r = -EFAULT; 3267 if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) 3268 goto out; 3269 r = 0; 3270 break; 3271 } 3272 case KVM_GET_MSRS: 3273 r = msr_io(vcpu, argp, do_get_msr, 1); 3274 break; 3275 case KVM_SET_MSRS: 3276 r = msr_io(vcpu, argp, do_set_msr, 0); 3277 break; 3278 case KVM_TPR_ACCESS_REPORTING: { 3279 struct kvm_tpr_access_ctl tac; 3280 3281 r = -EFAULT; 3282 if (copy_from_user(&tac, argp, sizeof tac)) 3283 goto out; 3284 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac); 3285 if (r) 3286 goto out; 3287 r = -EFAULT; 3288 if (copy_to_user(argp, &tac, sizeof tac)) 3289 goto out; 3290 r = 0; 3291 break; 3292 }; 3293 case KVM_SET_VAPIC_ADDR: { 3294 struct kvm_vapic_addr va; 3295 3296 r = -EINVAL; 3297 if (!lapic_in_kernel(vcpu)) 3298 goto out; 3299 r = -EFAULT; 3300 if (copy_from_user(&va, argp, sizeof va)) 3301 goto out; 3302 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); 3303 break; 3304 } 3305 case KVM_X86_SETUP_MCE: { 3306 u64 mcg_cap; 3307 3308 r = -EFAULT; 3309 if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap)) 3310 goto out; 3311 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap); 3312 break; 3313 } 3314 case KVM_X86_SET_MCE: { 3315 struct kvm_x86_mce mce; 3316 3317 r = -EFAULT; 3318 if (copy_from_user(&mce, argp, sizeof mce)) 3319 goto out; 3320 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); 3321 break; 3322 } 3323 case KVM_GET_VCPU_EVENTS: { 3324 struct kvm_vcpu_events events; 3325 3326 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events); 3327 3328 r = -EFAULT; 3329 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events))) 3330 break; 3331 r = 0; 3332 break; 3333 } 3334 case KVM_SET_VCPU_EVENTS: { 3335 struct kvm_vcpu_events events; 3336 3337 r = -EFAULT; 3338 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events))) 3339 break; 3340 3341 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events); 3342 break; 3343 } 3344 case KVM_GET_DEBUGREGS: { 3345 struct kvm_debugregs dbgregs; 3346 3347 kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs); 3348 3349 r = -EFAULT; 3350 if (copy_to_user(argp, &dbgregs, 3351 sizeof(struct kvm_debugregs))) 3352 break; 3353 r = 0; 3354 break; 3355 } 3356 case KVM_SET_DEBUGREGS: { 3357 struct kvm_debugregs dbgregs; 3358 3359 r = -EFAULT; 3360 if (copy_from_user(&dbgregs, argp, 3361 sizeof(struct kvm_debugregs))) 3362 break; 3363 3364 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs); 3365 break; 3366 } 3367 case KVM_GET_XSAVE: { 3368 u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL); 3369 r = -ENOMEM; 3370 if (!u.xsave) 3371 break; 3372 3373 kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave); 3374 3375 r = -EFAULT; 3376 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave))) 3377 break; 3378 r = 0; 3379 break; 3380 } 3381 case KVM_SET_XSAVE: { 3382 u.xsave = memdup_user(argp, sizeof(*u.xsave)); 3383 if (IS_ERR(u.xsave)) 3384 return PTR_ERR(u.xsave); 3385 3386 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); 3387 break; 3388 } 3389 case KVM_GET_XCRS: { 3390 u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL); 3391 r = -ENOMEM; 3392 if (!u.xcrs) 3393 break; 3394 3395 kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs); 3396 3397 r = -EFAULT; 3398 if (copy_to_user(argp, u.xcrs, 3399 sizeof(struct kvm_xcrs))) 3400 break; 3401 r = 0; 3402 break; 3403 } 3404 case KVM_SET_XCRS: { 3405 u.xcrs = memdup_user(argp, sizeof(*u.xcrs)); 3406 if (IS_ERR(u.xcrs)) 3407 return PTR_ERR(u.xcrs); 3408 3409 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs); 3410 break; 3411 } 3412 case KVM_SET_TSC_KHZ: { 3413 u32 user_tsc_khz; 3414 3415 r = -EINVAL; 3416 user_tsc_khz = (u32)arg; 3417 3418 if (user_tsc_khz >= kvm_max_guest_tsc_khz) 3419 goto out; 3420 3421 if (user_tsc_khz == 0) 3422 user_tsc_khz = tsc_khz; 3423 3424 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz)) 3425 r = 0; 3426 3427 goto out; 3428 } 3429 case KVM_GET_TSC_KHZ: { 3430 r = vcpu->arch.virtual_tsc_khz; 3431 goto out; 3432 } 3433 case KVM_KVMCLOCK_CTRL: { 3434 r = kvm_set_guest_paused(vcpu); 3435 goto out; 3436 } 3437 default: 3438 r = -EINVAL; 3439 } 3440 out: 3441 kfree(u.buffer); 3442 return r; 3443 } 3444 3445 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 3446 { 3447 return VM_FAULT_SIGBUS; 3448 } 3449 3450 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) 3451 { 3452 int ret; 3453 3454 if (addr > (unsigned int)(-3 * PAGE_SIZE)) 3455 return -EINVAL; 3456 ret = kvm_x86_ops->set_tss_addr(kvm, addr); 3457 return ret; 3458 } 3459 3460 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm, 3461 u64 ident_addr) 3462 { 3463 kvm->arch.ept_identity_map_addr = ident_addr; 3464 return 0; 3465 } 3466 3467 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, 3468 u32 kvm_nr_mmu_pages) 3469 { 3470 if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) 3471 return -EINVAL; 3472 3473 mutex_lock(&kvm->slots_lock); 3474 3475 kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); 3476 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; 3477 3478 mutex_unlock(&kvm->slots_lock); 3479 return 0; 3480 } 3481 3482 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) 3483 { 3484 return kvm->arch.n_max_mmu_pages; 3485 } 3486 3487 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) 3488 { 3489 int r; 3490 3491 r = 0; 3492 switch (chip->chip_id) { 3493 case KVM_IRQCHIP_PIC_MASTER: 3494 memcpy(&chip->chip.pic, 3495 &pic_irqchip(kvm)->pics[0], 3496 sizeof(struct kvm_pic_state)); 3497 break; 3498 case KVM_IRQCHIP_PIC_SLAVE: 3499 memcpy(&chip->chip.pic, 3500 &pic_irqchip(kvm)->pics[1], 3501 sizeof(struct kvm_pic_state)); 3502 break; 3503 case KVM_IRQCHIP_IOAPIC: 3504 r = kvm_get_ioapic(kvm, &chip->chip.ioapic); 3505 break; 3506 default: 3507 r = -EINVAL; 3508 break; 3509 } 3510 return r; 3511 } 3512 3513 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) 3514 { 3515 int r; 3516 3517 r = 0; 3518 switch (chip->chip_id) { 3519 case KVM_IRQCHIP_PIC_MASTER: 3520 spin_lock(&pic_irqchip(kvm)->lock); 3521 memcpy(&pic_irqchip(kvm)->pics[0], 3522 &chip->chip.pic, 3523 sizeof(struct kvm_pic_state)); 3524 spin_unlock(&pic_irqchip(kvm)->lock); 3525 break; 3526 case KVM_IRQCHIP_PIC_SLAVE: 3527 spin_lock(&pic_irqchip(kvm)->lock); 3528 memcpy(&pic_irqchip(kvm)->pics[1], 3529 &chip->chip.pic, 3530 sizeof(struct kvm_pic_state)); 3531 spin_unlock(&pic_irqchip(kvm)->lock); 3532 break; 3533 case KVM_IRQCHIP_IOAPIC: 3534 r = kvm_set_ioapic(kvm, &chip->chip.ioapic); 3535 break; 3536 default: 3537 r = -EINVAL; 3538 break; 3539 } 3540 kvm_pic_update_irq(pic_irqchip(kvm)); 3541 return r; 3542 } 3543 3544 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps) 3545 { 3546 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3547 memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state)); 3548 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3549 return 0; 3550 } 3551 3552 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps) 3553 { 3554 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3555 memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state)); 3556 kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0); 3557 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3558 return 0; 3559 } 3560 3561 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps) 3562 { 3563 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3564 memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels, 3565 sizeof(ps->channels)); 3566 ps->flags = kvm->arch.vpit->pit_state.flags; 3567 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3568 memset(&ps->reserved, 0, sizeof(ps->reserved)); 3569 return 0; 3570 } 3571 3572 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps) 3573 { 3574 int start = 0; 3575 u32 prev_legacy, cur_legacy; 3576 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3577 prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY; 3578 cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY; 3579 if (!prev_legacy && cur_legacy) 3580 start = 1; 3581 memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels, 3582 sizeof(kvm->arch.vpit->pit_state.channels)); 3583 kvm->arch.vpit->pit_state.flags = ps->flags; 3584 kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start); 3585 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3586 return 0; 3587 } 3588 3589 static int kvm_vm_ioctl_reinject(struct kvm *kvm, 3590 struct kvm_reinject_control *control) 3591 { 3592 if (!kvm->arch.vpit) 3593 return -ENXIO; 3594 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3595 kvm->arch.vpit->pit_state.reinject = control->pit_reinject; 3596 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3597 return 0; 3598 } 3599 3600 /** 3601 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot 3602 * @kvm: kvm instance 3603 * @log: slot id and address to which we copy the log 3604 * 3605 * Steps 1-4 below provide general overview of dirty page logging. See 3606 * kvm_get_dirty_log_protect() function description for additional details. 3607 * 3608 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we 3609 * always flush the TLB (step 4) even if previous step failed and the dirty 3610 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API 3611 * does not preclude user space subsequent dirty log read. Flushing TLB ensures 3612 * writes will be marked dirty for next log read. 3613 * 3614 * 1. Take a snapshot of the bit and clear it if needed. 3615 * 2. Write protect the corresponding page. 3616 * 3. Copy the snapshot to the userspace. 3617 * 4. Flush TLB's if needed. 3618 */ 3619 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log) 3620 { 3621 bool is_dirty = false; 3622 int r; 3623 3624 mutex_lock(&kvm->slots_lock); 3625 3626 /* 3627 * Flush potentially hardware-cached dirty pages to dirty_bitmap. 3628 */ 3629 if (kvm_x86_ops->flush_log_dirty) 3630 kvm_x86_ops->flush_log_dirty(kvm); 3631 3632 r = kvm_get_dirty_log_protect(kvm, log, &is_dirty); 3633 3634 /* 3635 * All the TLBs can be flushed out of mmu lock, see the comments in 3636 * kvm_mmu_slot_remove_write_access(). 3637 */ 3638 lockdep_assert_held(&kvm->slots_lock); 3639 if (is_dirty) 3640 kvm_flush_remote_tlbs(kvm); 3641 3642 mutex_unlock(&kvm->slots_lock); 3643 return r; 3644 } 3645 3646 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, 3647 bool line_status) 3648 { 3649 if (!irqchip_in_kernel(kvm)) 3650 return -ENXIO; 3651 3652 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 3653 irq_event->irq, irq_event->level, 3654 line_status); 3655 return 0; 3656 } 3657 3658 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 3659 struct kvm_enable_cap *cap) 3660 { 3661 int r; 3662 3663 if (cap->flags) 3664 return -EINVAL; 3665 3666 switch (cap->cap) { 3667 case KVM_CAP_DISABLE_QUIRKS: 3668 kvm->arch.disabled_quirks = cap->args[0]; 3669 r = 0; 3670 break; 3671 case KVM_CAP_SPLIT_IRQCHIP: { 3672 mutex_lock(&kvm->lock); 3673 r = -EINVAL; 3674 if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS) 3675 goto split_irqchip_unlock; 3676 r = -EEXIST; 3677 if (irqchip_in_kernel(kvm)) 3678 goto split_irqchip_unlock; 3679 if (atomic_read(&kvm->online_vcpus)) 3680 goto split_irqchip_unlock; 3681 r = kvm_setup_empty_irq_routing(kvm); 3682 if (r) 3683 goto split_irqchip_unlock; 3684 /* Pairs with irqchip_in_kernel. */ 3685 smp_wmb(); 3686 kvm->arch.irqchip_split = true; 3687 kvm->arch.nr_reserved_ioapic_pins = cap->args[0]; 3688 r = 0; 3689 split_irqchip_unlock: 3690 mutex_unlock(&kvm->lock); 3691 break; 3692 } 3693 default: 3694 r = -EINVAL; 3695 break; 3696 } 3697 return r; 3698 } 3699 3700 long kvm_arch_vm_ioctl(struct file *filp, 3701 unsigned int ioctl, unsigned long arg) 3702 { 3703 struct kvm *kvm = filp->private_data; 3704 void __user *argp = (void __user *)arg; 3705 int r = -ENOTTY; 3706 /* 3707 * This union makes it completely explicit to gcc-3.x 3708 * that these two variables' stack usage should be 3709 * combined, not added together. 3710 */ 3711 union { 3712 struct kvm_pit_state ps; 3713 struct kvm_pit_state2 ps2; 3714 struct kvm_pit_config pit_config; 3715 } u; 3716 3717 switch (ioctl) { 3718 case KVM_SET_TSS_ADDR: 3719 r = kvm_vm_ioctl_set_tss_addr(kvm, arg); 3720 break; 3721 case KVM_SET_IDENTITY_MAP_ADDR: { 3722 u64 ident_addr; 3723 3724 r = -EFAULT; 3725 if (copy_from_user(&ident_addr, argp, sizeof ident_addr)) 3726 goto out; 3727 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr); 3728 break; 3729 } 3730 case KVM_SET_NR_MMU_PAGES: 3731 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); 3732 break; 3733 case KVM_GET_NR_MMU_PAGES: 3734 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); 3735 break; 3736 case KVM_CREATE_IRQCHIP: { 3737 struct kvm_pic *vpic; 3738 3739 mutex_lock(&kvm->lock); 3740 r = -EEXIST; 3741 if (kvm->arch.vpic) 3742 goto create_irqchip_unlock; 3743 r = -EINVAL; 3744 if (atomic_read(&kvm->online_vcpus)) 3745 goto create_irqchip_unlock; 3746 r = -ENOMEM; 3747 vpic = kvm_create_pic(kvm); 3748 if (vpic) { 3749 r = kvm_ioapic_init(kvm); 3750 if (r) { 3751 mutex_lock(&kvm->slots_lock); 3752 kvm_destroy_pic(vpic); 3753 mutex_unlock(&kvm->slots_lock); 3754 goto create_irqchip_unlock; 3755 } 3756 } else 3757 goto create_irqchip_unlock; 3758 r = kvm_setup_default_irq_routing(kvm); 3759 if (r) { 3760 mutex_lock(&kvm->slots_lock); 3761 mutex_lock(&kvm->irq_lock); 3762 kvm_ioapic_destroy(kvm); 3763 kvm_destroy_pic(vpic); 3764 mutex_unlock(&kvm->irq_lock); 3765 mutex_unlock(&kvm->slots_lock); 3766 goto create_irqchip_unlock; 3767 } 3768 /* Write kvm->irq_routing before kvm->arch.vpic. */ 3769 smp_wmb(); 3770 kvm->arch.vpic = vpic; 3771 create_irqchip_unlock: 3772 mutex_unlock(&kvm->lock); 3773 break; 3774 } 3775 case KVM_CREATE_PIT: 3776 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY; 3777 goto create_pit; 3778 case KVM_CREATE_PIT2: 3779 r = -EFAULT; 3780 if (copy_from_user(&u.pit_config, argp, 3781 sizeof(struct kvm_pit_config))) 3782 goto out; 3783 create_pit: 3784 mutex_lock(&kvm->slots_lock); 3785 r = -EEXIST; 3786 if (kvm->arch.vpit) 3787 goto create_pit_unlock; 3788 r = -ENOMEM; 3789 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags); 3790 if (kvm->arch.vpit) 3791 r = 0; 3792 create_pit_unlock: 3793 mutex_unlock(&kvm->slots_lock); 3794 break; 3795 case KVM_GET_IRQCHIP: { 3796 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 3797 struct kvm_irqchip *chip; 3798 3799 chip = memdup_user(argp, sizeof(*chip)); 3800 if (IS_ERR(chip)) { 3801 r = PTR_ERR(chip); 3802 goto out; 3803 } 3804 3805 r = -ENXIO; 3806 if (!irqchip_in_kernel(kvm) || irqchip_split(kvm)) 3807 goto get_irqchip_out; 3808 r = kvm_vm_ioctl_get_irqchip(kvm, chip); 3809 if (r) 3810 goto get_irqchip_out; 3811 r = -EFAULT; 3812 if (copy_to_user(argp, chip, sizeof *chip)) 3813 goto get_irqchip_out; 3814 r = 0; 3815 get_irqchip_out: 3816 kfree(chip); 3817 break; 3818 } 3819 case KVM_SET_IRQCHIP: { 3820 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 3821 struct kvm_irqchip *chip; 3822 3823 chip = memdup_user(argp, sizeof(*chip)); 3824 if (IS_ERR(chip)) { 3825 r = PTR_ERR(chip); 3826 goto out; 3827 } 3828 3829 r = -ENXIO; 3830 if (!irqchip_in_kernel(kvm) || irqchip_split(kvm)) 3831 goto set_irqchip_out; 3832 r = kvm_vm_ioctl_set_irqchip(kvm, chip); 3833 if (r) 3834 goto set_irqchip_out; 3835 r = 0; 3836 set_irqchip_out: 3837 kfree(chip); 3838 break; 3839 } 3840 case KVM_GET_PIT: { 3841 r = -EFAULT; 3842 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state))) 3843 goto out; 3844 r = -ENXIO; 3845 if (!kvm->arch.vpit) 3846 goto out; 3847 r = kvm_vm_ioctl_get_pit(kvm, &u.ps); 3848 if (r) 3849 goto out; 3850 r = -EFAULT; 3851 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state))) 3852 goto out; 3853 r = 0; 3854 break; 3855 } 3856 case KVM_SET_PIT: { 3857 r = -EFAULT; 3858 if (copy_from_user(&u.ps, argp, sizeof u.ps)) 3859 goto out; 3860 r = -ENXIO; 3861 if (!kvm->arch.vpit) 3862 goto out; 3863 r = kvm_vm_ioctl_set_pit(kvm, &u.ps); 3864 break; 3865 } 3866 case KVM_GET_PIT2: { 3867 r = -ENXIO; 3868 if (!kvm->arch.vpit) 3869 goto out; 3870 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2); 3871 if (r) 3872 goto out; 3873 r = -EFAULT; 3874 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2))) 3875 goto out; 3876 r = 0; 3877 break; 3878 } 3879 case KVM_SET_PIT2: { 3880 r = -EFAULT; 3881 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2))) 3882 goto out; 3883 r = -ENXIO; 3884 if (!kvm->arch.vpit) 3885 goto out; 3886 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2); 3887 break; 3888 } 3889 case KVM_REINJECT_CONTROL: { 3890 struct kvm_reinject_control control; 3891 r = -EFAULT; 3892 if (copy_from_user(&control, argp, sizeof(control))) 3893 goto out; 3894 r = kvm_vm_ioctl_reinject(kvm, &control); 3895 break; 3896 } 3897 case KVM_SET_BOOT_CPU_ID: 3898 r = 0; 3899 mutex_lock(&kvm->lock); 3900 if (atomic_read(&kvm->online_vcpus) != 0) 3901 r = -EBUSY; 3902 else 3903 kvm->arch.bsp_vcpu_id = arg; 3904 mutex_unlock(&kvm->lock); 3905 break; 3906 case KVM_XEN_HVM_CONFIG: { 3907 r = -EFAULT; 3908 if (copy_from_user(&kvm->arch.xen_hvm_config, argp, 3909 sizeof(struct kvm_xen_hvm_config))) 3910 goto out; 3911 r = -EINVAL; 3912 if (kvm->arch.xen_hvm_config.flags) 3913 goto out; 3914 r = 0; 3915 break; 3916 } 3917 case KVM_SET_CLOCK: { 3918 struct kvm_clock_data user_ns; 3919 u64 now_ns; 3920 s64 delta; 3921 3922 r = -EFAULT; 3923 if (copy_from_user(&user_ns, argp, sizeof(user_ns))) 3924 goto out; 3925 3926 r = -EINVAL; 3927 if (user_ns.flags) 3928 goto out; 3929 3930 r = 0; 3931 local_irq_disable(); 3932 now_ns = get_kernel_ns(); 3933 delta = user_ns.clock - now_ns; 3934 local_irq_enable(); 3935 kvm->arch.kvmclock_offset = delta; 3936 kvm_gen_update_masterclock(kvm); 3937 break; 3938 } 3939 case KVM_GET_CLOCK: { 3940 struct kvm_clock_data user_ns; 3941 u64 now_ns; 3942 3943 local_irq_disable(); 3944 now_ns = get_kernel_ns(); 3945 user_ns.clock = kvm->arch.kvmclock_offset + now_ns; 3946 local_irq_enable(); 3947 user_ns.flags = 0; 3948 memset(&user_ns.pad, 0, sizeof(user_ns.pad)); 3949 3950 r = -EFAULT; 3951 if (copy_to_user(argp, &user_ns, sizeof(user_ns))) 3952 goto out; 3953 r = 0; 3954 break; 3955 } 3956 case KVM_ENABLE_CAP: { 3957 struct kvm_enable_cap cap; 3958 3959 r = -EFAULT; 3960 if (copy_from_user(&cap, argp, sizeof(cap))) 3961 goto out; 3962 r = kvm_vm_ioctl_enable_cap(kvm, &cap); 3963 break; 3964 } 3965 default: 3966 r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg); 3967 } 3968 out: 3969 return r; 3970 } 3971 3972 static void kvm_init_msr_list(void) 3973 { 3974 u32 dummy[2]; 3975 unsigned i, j; 3976 3977 for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { 3978 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) 3979 continue; 3980 3981 /* 3982 * Even MSRs that are valid in the host may not be exposed 3983 * to the guests in some cases. We could work around this 3984 * in VMX with the generic MSR save/load machinery, but it 3985 * is not really worthwhile since it will really only 3986 * happen with nested virtualization. 3987 */ 3988 switch (msrs_to_save[i]) { 3989 case MSR_IA32_BNDCFGS: 3990 if (!kvm_x86_ops->mpx_supported()) 3991 continue; 3992 break; 3993 default: 3994 break; 3995 } 3996 3997 if (j < i) 3998 msrs_to_save[j] = msrs_to_save[i]; 3999 j++; 4000 } 4001 num_msrs_to_save = j; 4002 4003 for (i = j = 0; i < ARRAY_SIZE(emulated_msrs); i++) { 4004 switch (emulated_msrs[i]) { 4005 case MSR_IA32_SMBASE: 4006 if (!kvm_x86_ops->cpu_has_high_real_mode_segbase()) 4007 continue; 4008 break; 4009 default: 4010 break; 4011 } 4012 4013 if (j < i) 4014 emulated_msrs[j] = emulated_msrs[i]; 4015 j++; 4016 } 4017 num_emulated_msrs = j; 4018 } 4019 4020 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len, 4021 const void *v) 4022 { 4023 int handled = 0; 4024 int n; 4025 4026 do { 4027 n = min(len, 8); 4028 if (!(vcpu->arch.apic && 4029 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v)) 4030 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v)) 4031 break; 4032 handled += n; 4033 addr += n; 4034 len -= n; 4035 v += n; 4036 } while (len); 4037 4038 return handled; 4039 } 4040 4041 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) 4042 { 4043 int handled = 0; 4044 int n; 4045 4046 do { 4047 n = min(len, 8); 4048 if (!(vcpu->arch.apic && 4049 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev, 4050 addr, n, v)) 4051 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v)) 4052 break; 4053 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, *(u64 *)v); 4054 handled += n; 4055 addr += n; 4056 len -= n; 4057 v += n; 4058 } while (len); 4059 4060 return handled; 4061 } 4062 4063 static void kvm_set_segment(struct kvm_vcpu *vcpu, 4064 struct kvm_segment *var, int seg) 4065 { 4066 kvm_x86_ops->set_segment(vcpu, var, seg); 4067 } 4068 4069 void kvm_get_segment(struct kvm_vcpu *vcpu, 4070 struct kvm_segment *var, int seg) 4071 { 4072 kvm_x86_ops->get_segment(vcpu, var, seg); 4073 } 4074 4075 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, 4076 struct x86_exception *exception) 4077 { 4078 gpa_t t_gpa; 4079 4080 BUG_ON(!mmu_is_nested(vcpu)); 4081 4082 /* NPT walks are always user-walks */ 4083 access |= PFERR_USER_MASK; 4084 t_gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gpa, access, exception); 4085 4086 return t_gpa; 4087 } 4088 4089 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, 4090 struct x86_exception *exception) 4091 { 4092 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 4093 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 4094 } 4095 4096 gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, 4097 struct x86_exception *exception) 4098 { 4099 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 4100 access |= PFERR_FETCH_MASK; 4101 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 4102 } 4103 4104 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, 4105 struct x86_exception *exception) 4106 { 4107 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 4108 access |= PFERR_WRITE_MASK; 4109 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 4110 } 4111 4112 /* uses this to access any guest's mapped memory without checking CPL */ 4113 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, 4114 struct x86_exception *exception) 4115 { 4116 return vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, 0, exception); 4117 } 4118 4119 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 4120 struct kvm_vcpu *vcpu, u32 access, 4121 struct x86_exception *exception) 4122 { 4123 void *data = val; 4124 int r = X86EMUL_CONTINUE; 4125 4126 while (bytes) { 4127 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access, 4128 exception); 4129 unsigned offset = addr & (PAGE_SIZE-1); 4130 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); 4131 int ret; 4132 4133 if (gpa == UNMAPPED_GVA) 4134 return X86EMUL_PROPAGATE_FAULT; 4135 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data, 4136 offset, toread); 4137 if (ret < 0) { 4138 r = X86EMUL_IO_NEEDED; 4139 goto out; 4140 } 4141 4142 bytes -= toread; 4143 data += toread; 4144 addr += toread; 4145 } 4146 out: 4147 return r; 4148 } 4149 4150 /* used for instruction fetching */ 4151 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, 4152 gva_t addr, void *val, unsigned int bytes, 4153 struct x86_exception *exception) 4154 { 4155 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4156 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 4157 unsigned offset; 4158 int ret; 4159 4160 /* Inline kvm_read_guest_virt_helper for speed. */ 4161 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, access|PFERR_FETCH_MASK, 4162 exception); 4163 if (unlikely(gpa == UNMAPPED_GVA)) 4164 return X86EMUL_PROPAGATE_FAULT; 4165 4166 offset = addr & (PAGE_SIZE-1); 4167 if (WARN_ON(offset + bytes > PAGE_SIZE)) 4168 bytes = (unsigned)PAGE_SIZE - offset; 4169 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val, 4170 offset, bytes); 4171 if (unlikely(ret < 0)) 4172 return X86EMUL_IO_NEEDED; 4173 4174 return X86EMUL_CONTINUE; 4175 } 4176 4177 int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt, 4178 gva_t addr, void *val, unsigned int bytes, 4179 struct x86_exception *exception) 4180 { 4181 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4182 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 4183 4184 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, 4185 exception); 4186 } 4187 EXPORT_SYMBOL_GPL(kvm_read_guest_virt); 4188 4189 static int kvm_read_guest_virt_system(struct x86_emulate_ctxt *ctxt, 4190 gva_t addr, void *val, unsigned int bytes, 4191 struct x86_exception *exception) 4192 { 4193 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4194 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, exception); 4195 } 4196 4197 static int kvm_read_guest_phys_system(struct x86_emulate_ctxt *ctxt, 4198 unsigned long addr, void *val, unsigned int bytes) 4199 { 4200 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4201 int r = kvm_vcpu_read_guest(vcpu, addr, val, bytes); 4202 4203 return r < 0 ? X86EMUL_IO_NEEDED : X86EMUL_CONTINUE; 4204 } 4205 4206 int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt, 4207 gva_t addr, void *val, 4208 unsigned int bytes, 4209 struct x86_exception *exception) 4210 { 4211 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4212 void *data = val; 4213 int r = X86EMUL_CONTINUE; 4214 4215 while (bytes) { 4216 gpa_t gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, addr, 4217 PFERR_WRITE_MASK, 4218 exception); 4219 unsigned offset = addr & (PAGE_SIZE-1); 4220 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); 4221 int ret; 4222 4223 if (gpa == UNMAPPED_GVA) 4224 return X86EMUL_PROPAGATE_FAULT; 4225 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite); 4226 if (ret < 0) { 4227 r = X86EMUL_IO_NEEDED; 4228 goto out; 4229 } 4230 4231 bytes -= towrite; 4232 data += towrite; 4233 addr += towrite; 4234 } 4235 out: 4236 return r; 4237 } 4238 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system); 4239 4240 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 4241 gpa_t *gpa, struct x86_exception *exception, 4242 bool write) 4243 { 4244 u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0) 4245 | (write ? PFERR_WRITE_MASK : 0); 4246 4247 if (vcpu_match_mmio_gva(vcpu, gva) 4248 && !permission_fault(vcpu, vcpu->arch.walk_mmu, 4249 vcpu->arch.access, access)) { 4250 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | 4251 (gva & (PAGE_SIZE - 1)); 4252 trace_vcpu_match_mmio(gva, *gpa, write, false); 4253 return 1; 4254 } 4255 4256 *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 4257 4258 if (*gpa == UNMAPPED_GVA) 4259 return -1; 4260 4261 /* For APIC access vmexit */ 4262 if ((*gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 4263 return 1; 4264 4265 if (vcpu_match_mmio_gpa(vcpu, *gpa)) { 4266 trace_vcpu_match_mmio(gva, *gpa, write, true); 4267 return 1; 4268 } 4269 4270 return 0; 4271 } 4272 4273 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, 4274 const void *val, int bytes) 4275 { 4276 int ret; 4277 4278 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes); 4279 if (ret < 0) 4280 return 0; 4281 kvm_mmu_pte_write(vcpu, gpa, val, bytes); 4282 return 1; 4283 } 4284 4285 struct read_write_emulator_ops { 4286 int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val, 4287 int bytes); 4288 int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa, 4289 void *val, int bytes); 4290 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa, 4291 int bytes, void *val); 4292 int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa, 4293 void *val, int bytes); 4294 bool write; 4295 }; 4296 4297 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes) 4298 { 4299 if (vcpu->mmio_read_completed) { 4300 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, 4301 vcpu->mmio_fragments[0].gpa, *(u64 *)val); 4302 vcpu->mmio_read_completed = 0; 4303 return 1; 4304 } 4305 4306 return 0; 4307 } 4308 4309 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa, 4310 void *val, int bytes) 4311 { 4312 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes); 4313 } 4314 4315 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa, 4316 void *val, int bytes) 4317 { 4318 return emulator_write_phys(vcpu, gpa, val, bytes); 4319 } 4320 4321 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val) 4322 { 4323 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val); 4324 return vcpu_mmio_write(vcpu, gpa, bytes, val); 4325 } 4326 4327 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, 4328 void *val, int bytes) 4329 { 4330 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0); 4331 return X86EMUL_IO_NEEDED; 4332 } 4333 4334 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, 4335 void *val, int bytes) 4336 { 4337 struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0]; 4338 4339 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len)); 4340 return X86EMUL_CONTINUE; 4341 } 4342 4343 static const struct read_write_emulator_ops read_emultor = { 4344 .read_write_prepare = read_prepare, 4345 .read_write_emulate = read_emulate, 4346 .read_write_mmio = vcpu_mmio_read, 4347 .read_write_exit_mmio = read_exit_mmio, 4348 }; 4349 4350 static const struct read_write_emulator_ops write_emultor = { 4351 .read_write_emulate = write_emulate, 4352 .read_write_mmio = write_mmio, 4353 .read_write_exit_mmio = write_exit_mmio, 4354 .write = true, 4355 }; 4356 4357 static int emulator_read_write_onepage(unsigned long addr, void *val, 4358 unsigned int bytes, 4359 struct x86_exception *exception, 4360 struct kvm_vcpu *vcpu, 4361 const struct read_write_emulator_ops *ops) 4362 { 4363 gpa_t gpa; 4364 int handled, ret; 4365 bool write = ops->write; 4366 struct kvm_mmio_fragment *frag; 4367 4368 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write); 4369 4370 if (ret < 0) 4371 return X86EMUL_PROPAGATE_FAULT; 4372 4373 /* For APIC access vmexit */ 4374 if (ret) 4375 goto mmio; 4376 4377 if (ops->read_write_emulate(vcpu, gpa, val, bytes)) 4378 return X86EMUL_CONTINUE; 4379 4380 mmio: 4381 /* 4382 * Is this MMIO handled locally? 4383 */ 4384 handled = ops->read_write_mmio(vcpu, gpa, bytes, val); 4385 if (handled == bytes) 4386 return X86EMUL_CONTINUE; 4387 4388 gpa += handled; 4389 bytes -= handled; 4390 val += handled; 4391 4392 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); 4393 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; 4394 frag->gpa = gpa; 4395 frag->data = val; 4396 frag->len = bytes; 4397 return X86EMUL_CONTINUE; 4398 } 4399 4400 static int emulator_read_write(struct x86_emulate_ctxt *ctxt, 4401 unsigned long addr, 4402 void *val, unsigned int bytes, 4403 struct x86_exception *exception, 4404 const struct read_write_emulator_ops *ops) 4405 { 4406 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4407 gpa_t gpa; 4408 int rc; 4409 4410 if (ops->read_write_prepare && 4411 ops->read_write_prepare(vcpu, val, bytes)) 4412 return X86EMUL_CONTINUE; 4413 4414 vcpu->mmio_nr_fragments = 0; 4415 4416 /* Crossing a page boundary? */ 4417 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { 4418 int now; 4419 4420 now = -addr & ~PAGE_MASK; 4421 rc = emulator_read_write_onepage(addr, val, now, exception, 4422 vcpu, ops); 4423 4424 if (rc != X86EMUL_CONTINUE) 4425 return rc; 4426 addr += now; 4427 if (ctxt->mode != X86EMUL_MODE_PROT64) 4428 addr = (u32)addr; 4429 val += now; 4430 bytes -= now; 4431 } 4432 4433 rc = emulator_read_write_onepage(addr, val, bytes, exception, 4434 vcpu, ops); 4435 if (rc != X86EMUL_CONTINUE) 4436 return rc; 4437 4438 if (!vcpu->mmio_nr_fragments) 4439 return rc; 4440 4441 gpa = vcpu->mmio_fragments[0].gpa; 4442 4443 vcpu->mmio_needed = 1; 4444 vcpu->mmio_cur_fragment = 0; 4445 4446 vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len); 4447 vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write; 4448 vcpu->run->exit_reason = KVM_EXIT_MMIO; 4449 vcpu->run->mmio.phys_addr = gpa; 4450 4451 return ops->read_write_exit_mmio(vcpu, gpa, val, bytes); 4452 } 4453 4454 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt, 4455 unsigned long addr, 4456 void *val, 4457 unsigned int bytes, 4458 struct x86_exception *exception) 4459 { 4460 return emulator_read_write(ctxt, addr, val, bytes, 4461 exception, &read_emultor); 4462 } 4463 4464 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 4465 unsigned long addr, 4466 const void *val, 4467 unsigned int bytes, 4468 struct x86_exception *exception) 4469 { 4470 return emulator_read_write(ctxt, addr, (void *)val, bytes, 4471 exception, &write_emultor); 4472 } 4473 4474 #define CMPXCHG_TYPE(t, ptr, old, new) \ 4475 (cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old)) 4476 4477 #ifdef CONFIG_X86_64 4478 # define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new) 4479 #else 4480 # define CMPXCHG64(ptr, old, new) \ 4481 (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old)) 4482 #endif 4483 4484 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, 4485 unsigned long addr, 4486 const void *old, 4487 const void *new, 4488 unsigned int bytes, 4489 struct x86_exception *exception) 4490 { 4491 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4492 gpa_t gpa; 4493 struct page *page; 4494 char *kaddr; 4495 bool exchanged; 4496 4497 /* guests cmpxchg8b have to be emulated atomically */ 4498 if (bytes > 8 || (bytes & (bytes - 1))) 4499 goto emul_write; 4500 4501 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL); 4502 4503 if (gpa == UNMAPPED_GVA || 4504 (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 4505 goto emul_write; 4506 4507 if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK)) 4508 goto emul_write; 4509 4510 page = kvm_vcpu_gfn_to_page(vcpu, gpa >> PAGE_SHIFT); 4511 if (is_error_page(page)) 4512 goto emul_write; 4513 4514 kaddr = kmap_atomic(page); 4515 kaddr += offset_in_page(gpa); 4516 switch (bytes) { 4517 case 1: 4518 exchanged = CMPXCHG_TYPE(u8, kaddr, old, new); 4519 break; 4520 case 2: 4521 exchanged = CMPXCHG_TYPE(u16, kaddr, old, new); 4522 break; 4523 case 4: 4524 exchanged = CMPXCHG_TYPE(u32, kaddr, old, new); 4525 break; 4526 case 8: 4527 exchanged = CMPXCHG64(kaddr, old, new); 4528 break; 4529 default: 4530 BUG(); 4531 } 4532 kunmap_atomic(kaddr); 4533 kvm_release_page_dirty(page); 4534 4535 if (!exchanged) 4536 return X86EMUL_CMPXCHG_FAILED; 4537 4538 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); 4539 kvm_mmu_pte_write(vcpu, gpa, new, bytes); 4540 4541 return X86EMUL_CONTINUE; 4542 4543 emul_write: 4544 printk_once(KERN_WARNING "kvm: emulating exchange as write\n"); 4545 4546 return emulator_write_emulated(ctxt, addr, new, bytes, exception); 4547 } 4548 4549 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd) 4550 { 4551 /* TODO: String I/O for in kernel device */ 4552 int r; 4553 4554 if (vcpu->arch.pio.in) 4555 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, vcpu->arch.pio.port, 4556 vcpu->arch.pio.size, pd); 4557 else 4558 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, 4559 vcpu->arch.pio.port, vcpu->arch.pio.size, 4560 pd); 4561 return r; 4562 } 4563 4564 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size, 4565 unsigned short port, void *val, 4566 unsigned int count, bool in) 4567 { 4568 vcpu->arch.pio.port = port; 4569 vcpu->arch.pio.in = in; 4570 vcpu->arch.pio.count = count; 4571 vcpu->arch.pio.size = size; 4572 4573 if (!kernel_pio(vcpu, vcpu->arch.pio_data)) { 4574 vcpu->arch.pio.count = 0; 4575 return 1; 4576 } 4577 4578 vcpu->run->exit_reason = KVM_EXIT_IO; 4579 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; 4580 vcpu->run->io.size = size; 4581 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; 4582 vcpu->run->io.count = count; 4583 vcpu->run->io.port = port; 4584 4585 return 0; 4586 } 4587 4588 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, 4589 int size, unsigned short port, void *val, 4590 unsigned int count) 4591 { 4592 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4593 int ret; 4594 4595 if (vcpu->arch.pio.count) 4596 goto data_avail; 4597 4598 ret = emulator_pio_in_out(vcpu, size, port, val, count, true); 4599 if (ret) { 4600 data_avail: 4601 memcpy(val, vcpu->arch.pio_data, size * count); 4602 trace_kvm_pio(KVM_PIO_IN, port, size, count, vcpu->arch.pio_data); 4603 vcpu->arch.pio.count = 0; 4604 return 1; 4605 } 4606 4607 return 0; 4608 } 4609 4610 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, 4611 int size, unsigned short port, 4612 const void *val, unsigned int count) 4613 { 4614 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4615 4616 memcpy(vcpu->arch.pio_data, val, size * count); 4617 trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data); 4618 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false); 4619 } 4620 4621 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) 4622 { 4623 return kvm_x86_ops->get_segment_base(vcpu, seg); 4624 } 4625 4626 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) 4627 { 4628 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); 4629 } 4630 4631 int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) 4632 { 4633 if (!need_emulate_wbinvd(vcpu)) 4634 return X86EMUL_CONTINUE; 4635 4636 if (kvm_x86_ops->has_wbinvd_exit()) { 4637 int cpu = get_cpu(); 4638 4639 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 4640 smp_call_function_many(vcpu->arch.wbinvd_dirty_mask, 4641 wbinvd_ipi, NULL, 1); 4642 put_cpu(); 4643 cpumask_clear(vcpu->arch.wbinvd_dirty_mask); 4644 } else 4645 wbinvd(); 4646 return X86EMUL_CONTINUE; 4647 } 4648 4649 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) 4650 { 4651 kvm_x86_ops->skip_emulated_instruction(vcpu); 4652 return kvm_emulate_wbinvd_noskip(vcpu); 4653 } 4654 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); 4655 4656 4657 4658 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) 4659 { 4660 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt)); 4661 } 4662 4663 static int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, 4664 unsigned long *dest) 4665 { 4666 return kvm_get_dr(emul_to_vcpu(ctxt), dr, dest); 4667 } 4668 4669 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, 4670 unsigned long value) 4671 { 4672 4673 return __kvm_set_dr(emul_to_vcpu(ctxt), dr, value); 4674 } 4675 4676 static u64 mk_cr_64(u64 curr_cr, u32 new_val) 4677 { 4678 return (curr_cr & ~((1ULL << 32) - 1)) | new_val; 4679 } 4680 4681 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr) 4682 { 4683 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4684 unsigned long value; 4685 4686 switch (cr) { 4687 case 0: 4688 value = kvm_read_cr0(vcpu); 4689 break; 4690 case 2: 4691 value = vcpu->arch.cr2; 4692 break; 4693 case 3: 4694 value = kvm_read_cr3(vcpu); 4695 break; 4696 case 4: 4697 value = kvm_read_cr4(vcpu); 4698 break; 4699 case 8: 4700 value = kvm_get_cr8(vcpu); 4701 break; 4702 default: 4703 kvm_err("%s: unexpected cr %u\n", __func__, cr); 4704 return 0; 4705 } 4706 4707 return value; 4708 } 4709 4710 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) 4711 { 4712 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4713 int res = 0; 4714 4715 switch (cr) { 4716 case 0: 4717 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val)); 4718 break; 4719 case 2: 4720 vcpu->arch.cr2 = val; 4721 break; 4722 case 3: 4723 res = kvm_set_cr3(vcpu, val); 4724 break; 4725 case 4: 4726 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); 4727 break; 4728 case 8: 4729 res = kvm_set_cr8(vcpu, val); 4730 break; 4731 default: 4732 kvm_err("%s: unexpected cr %u\n", __func__, cr); 4733 res = -1; 4734 } 4735 4736 return res; 4737 } 4738 4739 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt) 4740 { 4741 return kvm_x86_ops->get_cpl(emul_to_vcpu(ctxt)); 4742 } 4743 4744 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4745 { 4746 kvm_x86_ops->get_gdt(emul_to_vcpu(ctxt), dt); 4747 } 4748 4749 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4750 { 4751 kvm_x86_ops->get_idt(emul_to_vcpu(ctxt), dt); 4752 } 4753 4754 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4755 { 4756 kvm_x86_ops->set_gdt(emul_to_vcpu(ctxt), dt); 4757 } 4758 4759 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 4760 { 4761 kvm_x86_ops->set_idt(emul_to_vcpu(ctxt), dt); 4762 } 4763 4764 static unsigned long emulator_get_cached_segment_base( 4765 struct x86_emulate_ctxt *ctxt, int seg) 4766 { 4767 return get_segment_base(emul_to_vcpu(ctxt), seg); 4768 } 4769 4770 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector, 4771 struct desc_struct *desc, u32 *base3, 4772 int seg) 4773 { 4774 struct kvm_segment var; 4775 4776 kvm_get_segment(emul_to_vcpu(ctxt), &var, seg); 4777 *selector = var.selector; 4778 4779 if (var.unusable) { 4780 memset(desc, 0, sizeof(*desc)); 4781 return false; 4782 } 4783 4784 if (var.g) 4785 var.limit >>= 12; 4786 set_desc_limit(desc, var.limit); 4787 set_desc_base(desc, (unsigned long)var.base); 4788 #ifdef CONFIG_X86_64 4789 if (base3) 4790 *base3 = var.base >> 32; 4791 #endif 4792 desc->type = var.type; 4793 desc->s = var.s; 4794 desc->dpl = var.dpl; 4795 desc->p = var.present; 4796 desc->avl = var.avl; 4797 desc->l = var.l; 4798 desc->d = var.db; 4799 desc->g = var.g; 4800 4801 return true; 4802 } 4803 4804 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, 4805 struct desc_struct *desc, u32 base3, 4806 int seg) 4807 { 4808 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4809 struct kvm_segment var; 4810 4811 var.selector = selector; 4812 var.base = get_desc_base(desc); 4813 #ifdef CONFIG_X86_64 4814 var.base |= ((u64)base3) << 32; 4815 #endif 4816 var.limit = get_desc_limit(desc); 4817 if (desc->g) 4818 var.limit = (var.limit << 12) | 0xfff; 4819 var.type = desc->type; 4820 var.dpl = desc->dpl; 4821 var.db = desc->d; 4822 var.s = desc->s; 4823 var.l = desc->l; 4824 var.g = desc->g; 4825 var.avl = desc->avl; 4826 var.present = desc->p; 4827 var.unusable = !var.present; 4828 var.padding = 0; 4829 4830 kvm_set_segment(vcpu, &var, seg); 4831 return; 4832 } 4833 4834 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, 4835 u32 msr_index, u64 *pdata) 4836 { 4837 struct msr_data msr; 4838 int r; 4839 4840 msr.index = msr_index; 4841 msr.host_initiated = false; 4842 r = kvm_get_msr(emul_to_vcpu(ctxt), &msr); 4843 if (r) 4844 return r; 4845 4846 *pdata = msr.data; 4847 return 0; 4848 } 4849 4850 static int emulator_set_msr(struct x86_emulate_ctxt *ctxt, 4851 u32 msr_index, u64 data) 4852 { 4853 struct msr_data msr; 4854 4855 msr.data = data; 4856 msr.index = msr_index; 4857 msr.host_initiated = false; 4858 return kvm_set_msr(emul_to_vcpu(ctxt), &msr); 4859 } 4860 4861 static u64 emulator_get_smbase(struct x86_emulate_ctxt *ctxt) 4862 { 4863 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4864 4865 return vcpu->arch.smbase; 4866 } 4867 4868 static void emulator_set_smbase(struct x86_emulate_ctxt *ctxt, u64 smbase) 4869 { 4870 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 4871 4872 vcpu->arch.smbase = smbase; 4873 } 4874 4875 static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt, 4876 u32 pmc) 4877 { 4878 return kvm_pmu_is_valid_msr_idx(emul_to_vcpu(ctxt), pmc); 4879 } 4880 4881 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt, 4882 u32 pmc, u64 *pdata) 4883 { 4884 return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata); 4885 } 4886 4887 static void emulator_halt(struct x86_emulate_ctxt *ctxt) 4888 { 4889 emul_to_vcpu(ctxt)->arch.halt_request = 1; 4890 } 4891 4892 static void emulator_get_fpu(struct x86_emulate_ctxt *ctxt) 4893 { 4894 preempt_disable(); 4895 kvm_load_guest_fpu(emul_to_vcpu(ctxt)); 4896 /* 4897 * CR0.TS may reference the host fpu state, not the guest fpu state, 4898 * so it may be clear at this point. 4899 */ 4900 clts(); 4901 } 4902 4903 static void emulator_put_fpu(struct x86_emulate_ctxt *ctxt) 4904 { 4905 preempt_enable(); 4906 } 4907 4908 static int emulator_intercept(struct x86_emulate_ctxt *ctxt, 4909 struct x86_instruction_info *info, 4910 enum x86_intercept_stage stage) 4911 { 4912 return kvm_x86_ops->check_intercept(emul_to_vcpu(ctxt), info, stage); 4913 } 4914 4915 static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt, 4916 u32 *eax, u32 *ebx, u32 *ecx, u32 *edx) 4917 { 4918 kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx); 4919 } 4920 4921 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg) 4922 { 4923 return kvm_register_read(emul_to_vcpu(ctxt), reg); 4924 } 4925 4926 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val) 4927 { 4928 kvm_register_write(emul_to_vcpu(ctxt), reg, val); 4929 } 4930 4931 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked) 4932 { 4933 kvm_x86_ops->set_nmi_mask(emul_to_vcpu(ctxt), masked); 4934 } 4935 4936 static const struct x86_emulate_ops emulate_ops = { 4937 .read_gpr = emulator_read_gpr, 4938 .write_gpr = emulator_write_gpr, 4939 .read_std = kvm_read_guest_virt_system, 4940 .write_std = kvm_write_guest_virt_system, 4941 .read_phys = kvm_read_guest_phys_system, 4942 .fetch = kvm_fetch_guest_virt, 4943 .read_emulated = emulator_read_emulated, 4944 .write_emulated = emulator_write_emulated, 4945 .cmpxchg_emulated = emulator_cmpxchg_emulated, 4946 .invlpg = emulator_invlpg, 4947 .pio_in_emulated = emulator_pio_in_emulated, 4948 .pio_out_emulated = emulator_pio_out_emulated, 4949 .get_segment = emulator_get_segment, 4950 .set_segment = emulator_set_segment, 4951 .get_cached_segment_base = emulator_get_cached_segment_base, 4952 .get_gdt = emulator_get_gdt, 4953 .get_idt = emulator_get_idt, 4954 .set_gdt = emulator_set_gdt, 4955 .set_idt = emulator_set_idt, 4956 .get_cr = emulator_get_cr, 4957 .set_cr = emulator_set_cr, 4958 .cpl = emulator_get_cpl, 4959 .get_dr = emulator_get_dr, 4960 .set_dr = emulator_set_dr, 4961 .get_smbase = emulator_get_smbase, 4962 .set_smbase = emulator_set_smbase, 4963 .set_msr = emulator_set_msr, 4964 .get_msr = emulator_get_msr, 4965 .check_pmc = emulator_check_pmc, 4966 .read_pmc = emulator_read_pmc, 4967 .halt = emulator_halt, 4968 .wbinvd = emulator_wbinvd, 4969 .fix_hypercall = emulator_fix_hypercall, 4970 .get_fpu = emulator_get_fpu, 4971 .put_fpu = emulator_put_fpu, 4972 .intercept = emulator_intercept, 4973 .get_cpuid = emulator_get_cpuid, 4974 .set_nmi_mask = emulator_set_nmi_mask, 4975 }; 4976 4977 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) 4978 { 4979 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu); 4980 /* 4981 * an sti; sti; sequence only disable interrupts for the first 4982 * instruction. So, if the last instruction, be it emulated or 4983 * not, left the system with the INT_STI flag enabled, it 4984 * means that the last instruction is an sti. We should not 4985 * leave the flag on in this case. The same goes for mov ss 4986 */ 4987 if (int_shadow & mask) 4988 mask = 0; 4989 if (unlikely(int_shadow || mask)) { 4990 kvm_x86_ops->set_interrupt_shadow(vcpu, mask); 4991 if (!mask) 4992 kvm_make_request(KVM_REQ_EVENT, vcpu); 4993 } 4994 } 4995 4996 static bool inject_emulated_exception(struct kvm_vcpu *vcpu) 4997 { 4998 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 4999 if (ctxt->exception.vector == PF_VECTOR) 5000 return kvm_propagate_fault(vcpu, &ctxt->exception); 5001 5002 if (ctxt->exception.error_code_valid) 5003 kvm_queue_exception_e(vcpu, ctxt->exception.vector, 5004 ctxt->exception.error_code); 5005 else 5006 kvm_queue_exception(vcpu, ctxt->exception.vector); 5007 return false; 5008 } 5009 5010 static void init_emulate_ctxt(struct kvm_vcpu *vcpu) 5011 { 5012 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 5013 int cs_db, cs_l; 5014 5015 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 5016 5017 ctxt->eflags = kvm_get_rflags(vcpu); 5018 ctxt->eip = kvm_rip_read(vcpu); 5019 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : 5020 (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 : 5021 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 : 5022 cs_db ? X86EMUL_MODE_PROT32 : 5023 X86EMUL_MODE_PROT16; 5024 BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK); 5025 BUILD_BUG_ON(HF_SMM_MASK != X86EMUL_SMM_MASK); 5026 BUILD_BUG_ON(HF_SMM_INSIDE_NMI_MASK != X86EMUL_SMM_INSIDE_NMI_MASK); 5027 ctxt->emul_flags = vcpu->arch.hflags; 5028 5029 init_decode_cache(ctxt); 5030 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 5031 } 5032 5033 int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) 5034 { 5035 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 5036 int ret; 5037 5038 init_emulate_ctxt(vcpu); 5039 5040 ctxt->op_bytes = 2; 5041 ctxt->ad_bytes = 2; 5042 ctxt->_eip = ctxt->eip + inc_eip; 5043 ret = emulate_int_real(ctxt, irq); 5044 5045 if (ret != X86EMUL_CONTINUE) 5046 return EMULATE_FAIL; 5047 5048 ctxt->eip = ctxt->_eip; 5049 kvm_rip_write(vcpu, ctxt->eip); 5050 kvm_set_rflags(vcpu, ctxt->eflags); 5051 5052 if (irq == NMI_VECTOR) 5053 vcpu->arch.nmi_pending = 0; 5054 else 5055 vcpu->arch.interrupt.pending = false; 5056 5057 return EMULATE_DONE; 5058 } 5059 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); 5060 5061 static int handle_emulation_failure(struct kvm_vcpu *vcpu) 5062 { 5063 int r = EMULATE_DONE; 5064 5065 ++vcpu->stat.insn_emulation_fail; 5066 trace_kvm_emulate_insn_failed(vcpu); 5067 if (!is_guest_mode(vcpu) && kvm_x86_ops->get_cpl(vcpu) == 0) { 5068 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 5069 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 5070 vcpu->run->internal.ndata = 0; 5071 r = EMULATE_FAIL; 5072 } 5073 kvm_queue_exception(vcpu, UD_VECTOR); 5074 5075 return r; 5076 } 5077 5078 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t cr2, 5079 bool write_fault_to_shadow_pgtable, 5080 int emulation_type) 5081 { 5082 gpa_t gpa = cr2; 5083 pfn_t pfn; 5084 5085 if (emulation_type & EMULTYPE_NO_REEXECUTE) 5086 return false; 5087 5088 if (!vcpu->arch.mmu.direct_map) { 5089 /* 5090 * Write permission should be allowed since only 5091 * write access need to be emulated. 5092 */ 5093 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL); 5094 5095 /* 5096 * If the mapping is invalid in guest, let cpu retry 5097 * it to generate fault. 5098 */ 5099 if (gpa == UNMAPPED_GVA) 5100 return true; 5101 } 5102 5103 /* 5104 * Do not retry the unhandleable instruction if it faults on the 5105 * readonly host memory, otherwise it will goto a infinite loop: 5106 * retry instruction -> write #PF -> emulation fail -> retry 5107 * instruction -> ... 5108 */ 5109 pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa)); 5110 5111 /* 5112 * If the instruction failed on the error pfn, it can not be fixed, 5113 * report the error to userspace. 5114 */ 5115 if (is_error_noslot_pfn(pfn)) 5116 return false; 5117 5118 kvm_release_pfn_clean(pfn); 5119 5120 /* The instructions are well-emulated on direct mmu. */ 5121 if (vcpu->arch.mmu.direct_map) { 5122 unsigned int indirect_shadow_pages; 5123 5124 spin_lock(&vcpu->kvm->mmu_lock); 5125 indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages; 5126 spin_unlock(&vcpu->kvm->mmu_lock); 5127 5128 if (indirect_shadow_pages) 5129 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); 5130 5131 return true; 5132 } 5133 5134 /* 5135 * if emulation was due to access to shadowed page table 5136 * and it failed try to unshadow page and re-enter the 5137 * guest to let CPU execute the instruction. 5138 */ 5139 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); 5140 5141 /* 5142 * If the access faults on its page table, it can not 5143 * be fixed by unprotecting shadow page and it should 5144 * be reported to userspace. 5145 */ 5146 return !write_fault_to_shadow_pgtable; 5147 } 5148 5149 static bool retry_instruction(struct x86_emulate_ctxt *ctxt, 5150 unsigned long cr2, int emulation_type) 5151 { 5152 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5153 unsigned long last_retry_eip, last_retry_addr, gpa = cr2; 5154 5155 last_retry_eip = vcpu->arch.last_retry_eip; 5156 last_retry_addr = vcpu->arch.last_retry_addr; 5157 5158 /* 5159 * If the emulation is caused by #PF and it is non-page_table 5160 * writing instruction, it means the VM-EXIT is caused by shadow 5161 * page protected, we can zap the shadow page and retry this 5162 * instruction directly. 5163 * 5164 * Note: if the guest uses a non-page-table modifying instruction 5165 * on the PDE that points to the instruction, then we will unmap 5166 * the instruction and go to an infinite loop. So, we cache the 5167 * last retried eip and the last fault address, if we meet the eip 5168 * and the address again, we can break out of the potential infinite 5169 * loop. 5170 */ 5171 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0; 5172 5173 if (!(emulation_type & EMULTYPE_RETRY)) 5174 return false; 5175 5176 if (x86_page_table_writing_insn(ctxt)) 5177 return false; 5178 5179 if (ctxt->eip == last_retry_eip && last_retry_addr == cr2) 5180 return false; 5181 5182 vcpu->arch.last_retry_eip = ctxt->eip; 5183 vcpu->arch.last_retry_addr = cr2; 5184 5185 if (!vcpu->arch.mmu.direct_map) 5186 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2, NULL); 5187 5188 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); 5189 5190 return true; 5191 } 5192 5193 static int complete_emulated_mmio(struct kvm_vcpu *vcpu); 5194 static int complete_emulated_pio(struct kvm_vcpu *vcpu); 5195 5196 static void kvm_smm_changed(struct kvm_vcpu *vcpu) 5197 { 5198 if (!(vcpu->arch.hflags & HF_SMM_MASK)) { 5199 /* This is a good place to trace that we are exiting SMM. */ 5200 trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false); 5201 5202 if (unlikely(vcpu->arch.smi_pending)) { 5203 kvm_make_request(KVM_REQ_SMI, vcpu); 5204 vcpu->arch.smi_pending = 0; 5205 } else { 5206 /* Process a latched INIT, if any. */ 5207 kvm_make_request(KVM_REQ_EVENT, vcpu); 5208 } 5209 } 5210 5211 kvm_mmu_reset_context(vcpu); 5212 } 5213 5214 static void kvm_set_hflags(struct kvm_vcpu *vcpu, unsigned emul_flags) 5215 { 5216 unsigned changed = vcpu->arch.hflags ^ emul_flags; 5217 5218 vcpu->arch.hflags = emul_flags; 5219 5220 if (changed & HF_SMM_MASK) 5221 kvm_smm_changed(vcpu); 5222 } 5223 5224 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7, 5225 unsigned long *db) 5226 { 5227 u32 dr6 = 0; 5228 int i; 5229 u32 enable, rwlen; 5230 5231 enable = dr7; 5232 rwlen = dr7 >> 16; 5233 for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4) 5234 if ((enable & 3) && (rwlen & 15) == type && db[i] == addr) 5235 dr6 |= (1 << i); 5236 return dr6; 5237 } 5238 5239 static void kvm_vcpu_check_singlestep(struct kvm_vcpu *vcpu, unsigned long rflags, int *r) 5240 { 5241 struct kvm_run *kvm_run = vcpu->run; 5242 5243 /* 5244 * rflags is the old, "raw" value of the flags. The new value has 5245 * not been saved yet. 5246 * 5247 * This is correct even for TF set by the guest, because "the 5248 * processor will not generate this exception after the instruction 5249 * that sets the TF flag". 5250 */ 5251 if (unlikely(rflags & X86_EFLAGS_TF)) { 5252 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { 5253 kvm_run->debug.arch.dr6 = DR6_BS | DR6_FIXED_1 | 5254 DR6_RTM; 5255 kvm_run->debug.arch.pc = vcpu->arch.singlestep_rip; 5256 kvm_run->debug.arch.exception = DB_VECTOR; 5257 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5258 *r = EMULATE_USER_EXIT; 5259 } else { 5260 vcpu->arch.emulate_ctxt.eflags &= ~X86_EFLAGS_TF; 5261 /* 5262 * "Certain debug exceptions may clear bit 0-3. The 5263 * remaining contents of the DR6 register are never 5264 * cleared by the processor". 5265 */ 5266 vcpu->arch.dr6 &= ~15; 5267 vcpu->arch.dr6 |= DR6_BS | DR6_RTM; 5268 kvm_queue_exception(vcpu, DB_VECTOR); 5269 } 5270 } 5271 } 5272 5273 static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r) 5274 { 5275 if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) && 5276 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) { 5277 struct kvm_run *kvm_run = vcpu->run; 5278 unsigned long eip = kvm_get_linear_rip(vcpu); 5279 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, 5280 vcpu->arch.guest_debug_dr7, 5281 vcpu->arch.eff_db); 5282 5283 if (dr6 != 0) { 5284 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1 | DR6_RTM; 5285 kvm_run->debug.arch.pc = eip; 5286 kvm_run->debug.arch.exception = DB_VECTOR; 5287 kvm_run->exit_reason = KVM_EXIT_DEBUG; 5288 *r = EMULATE_USER_EXIT; 5289 return true; 5290 } 5291 } 5292 5293 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) && 5294 !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) { 5295 unsigned long eip = kvm_get_linear_rip(vcpu); 5296 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, 5297 vcpu->arch.dr7, 5298 vcpu->arch.db); 5299 5300 if (dr6 != 0) { 5301 vcpu->arch.dr6 &= ~15; 5302 vcpu->arch.dr6 |= dr6 | DR6_RTM; 5303 kvm_queue_exception(vcpu, DB_VECTOR); 5304 *r = EMULATE_DONE; 5305 return true; 5306 } 5307 } 5308 5309 return false; 5310 } 5311 5312 int x86_emulate_instruction(struct kvm_vcpu *vcpu, 5313 unsigned long cr2, 5314 int emulation_type, 5315 void *insn, 5316 int insn_len) 5317 { 5318 int r; 5319 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 5320 bool writeback = true; 5321 bool write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable; 5322 5323 /* 5324 * Clear write_fault_to_shadow_pgtable here to ensure it is 5325 * never reused. 5326 */ 5327 vcpu->arch.write_fault_to_shadow_pgtable = false; 5328 kvm_clear_exception_queue(vcpu); 5329 5330 if (!(emulation_type & EMULTYPE_NO_DECODE)) { 5331 init_emulate_ctxt(vcpu); 5332 5333 /* 5334 * We will reenter on the same instruction since 5335 * we do not set complete_userspace_io. This does not 5336 * handle watchpoints yet, those would be handled in 5337 * the emulate_ops. 5338 */ 5339 if (kvm_vcpu_check_breakpoint(vcpu, &r)) 5340 return r; 5341 5342 ctxt->interruptibility = 0; 5343 ctxt->have_exception = false; 5344 ctxt->exception.vector = -1; 5345 ctxt->perm_ok = false; 5346 5347 ctxt->ud = emulation_type & EMULTYPE_TRAP_UD; 5348 5349 r = x86_decode_insn(ctxt, insn, insn_len); 5350 5351 trace_kvm_emulate_insn_start(vcpu); 5352 ++vcpu->stat.insn_emulation; 5353 if (r != EMULATION_OK) { 5354 if (emulation_type & EMULTYPE_TRAP_UD) 5355 return EMULATE_FAIL; 5356 if (reexecute_instruction(vcpu, cr2, write_fault_to_spt, 5357 emulation_type)) 5358 return EMULATE_DONE; 5359 if (emulation_type & EMULTYPE_SKIP) 5360 return EMULATE_FAIL; 5361 return handle_emulation_failure(vcpu); 5362 } 5363 } 5364 5365 if (emulation_type & EMULTYPE_SKIP) { 5366 kvm_rip_write(vcpu, ctxt->_eip); 5367 if (ctxt->eflags & X86_EFLAGS_RF) 5368 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF); 5369 return EMULATE_DONE; 5370 } 5371 5372 if (retry_instruction(ctxt, cr2, emulation_type)) 5373 return EMULATE_DONE; 5374 5375 /* this is needed for vmware backdoor interface to work since it 5376 changes registers values during IO operation */ 5377 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { 5378 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 5379 emulator_invalidate_register_cache(ctxt); 5380 } 5381 5382 restart: 5383 r = x86_emulate_insn(ctxt); 5384 5385 if (r == EMULATION_INTERCEPTED) 5386 return EMULATE_DONE; 5387 5388 if (r == EMULATION_FAILED) { 5389 if (reexecute_instruction(vcpu, cr2, write_fault_to_spt, 5390 emulation_type)) 5391 return EMULATE_DONE; 5392 5393 return handle_emulation_failure(vcpu); 5394 } 5395 5396 if (ctxt->have_exception) { 5397 r = EMULATE_DONE; 5398 if (inject_emulated_exception(vcpu)) 5399 return r; 5400 } else if (vcpu->arch.pio.count) { 5401 if (!vcpu->arch.pio.in) { 5402 /* FIXME: return into emulator if single-stepping. */ 5403 vcpu->arch.pio.count = 0; 5404 } else { 5405 writeback = false; 5406 vcpu->arch.complete_userspace_io = complete_emulated_pio; 5407 } 5408 r = EMULATE_USER_EXIT; 5409 } else if (vcpu->mmio_needed) { 5410 if (!vcpu->mmio_is_write) 5411 writeback = false; 5412 r = EMULATE_USER_EXIT; 5413 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 5414 } else if (r == EMULATION_RESTART) 5415 goto restart; 5416 else 5417 r = EMULATE_DONE; 5418 5419 if (writeback) { 5420 unsigned long rflags = kvm_x86_ops->get_rflags(vcpu); 5421 toggle_interruptibility(vcpu, ctxt->interruptibility); 5422 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 5423 if (vcpu->arch.hflags != ctxt->emul_flags) 5424 kvm_set_hflags(vcpu, ctxt->emul_flags); 5425 kvm_rip_write(vcpu, ctxt->eip); 5426 if (r == EMULATE_DONE) 5427 kvm_vcpu_check_singlestep(vcpu, rflags, &r); 5428 if (!ctxt->have_exception || 5429 exception_type(ctxt->exception.vector) == EXCPT_TRAP) 5430 __kvm_set_rflags(vcpu, ctxt->eflags); 5431 5432 /* 5433 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will 5434 * do nothing, and it will be requested again as soon as 5435 * the shadow expires. But we still need to check here, 5436 * because POPF has no interrupt shadow. 5437 */ 5438 if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF)) 5439 kvm_make_request(KVM_REQ_EVENT, vcpu); 5440 } else 5441 vcpu->arch.emulate_regs_need_sync_to_vcpu = true; 5442 5443 return r; 5444 } 5445 EXPORT_SYMBOL_GPL(x86_emulate_instruction); 5446 5447 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port) 5448 { 5449 unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX); 5450 int ret = emulator_pio_out_emulated(&vcpu->arch.emulate_ctxt, 5451 size, port, &val, 1); 5452 /* do not return to emulator after return from userspace */ 5453 vcpu->arch.pio.count = 0; 5454 return ret; 5455 } 5456 EXPORT_SYMBOL_GPL(kvm_fast_pio_out); 5457 5458 static void tsc_bad(void *info) 5459 { 5460 __this_cpu_write(cpu_tsc_khz, 0); 5461 } 5462 5463 static void tsc_khz_changed(void *data) 5464 { 5465 struct cpufreq_freqs *freq = data; 5466 unsigned long khz = 0; 5467 5468 if (data) 5469 khz = freq->new; 5470 else if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 5471 khz = cpufreq_quick_get(raw_smp_processor_id()); 5472 if (!khz) 5473 khz = tsc_khz; 5474 __this_cpu_write(cpu_tsc_khz, khz); 5475 } 5476 5477 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 5478 void *data) 5479 { 5480 struct cpufreq_freqs *freq = data; 5481 struct kvm *kvm; 5482 struct kvm_vcpu *vcpu; 5483 int i, send_ipi = 0; 5484 5485 /* 5486 * We allow guests to temporarily run on slowing clocks, 5487 * provided we notify them after, or to run on accelerating 5488 * clocks, provided we notify them before. Thus time never 5489 * goes backwards. 5490 * 5491 * However, we have a problem. We can't atomically update 5492 * the frequency of a given CPU from this function; it is 5493 * merely a notifier, which can be called from any CPU. 5494 * Changing the TSC frequency at arbitrary points in time 5495 * requires a recomputation of local variables related to 5496 * the TSC for each VCPU. We must flag these local variables 5497 * to be updated and be sure the update takes place with the 5498 * new frequency before any guests proceed. 5499 * 5500 * Unfortunately, the combination of hotplug CPU and frequency 5501 * change creates an intractable locking scenario; the order 5502 * of when these callouts happen is undefined with respect to 5503 * CPU hotplug, and they can race with each other. As such, 5504 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is 5505 * undefined; you can actually have a CPU frequency change take 5506 * place in between the computation of X and the setting of the 5507 * variable. To protect against this problem, all updates of 5508 * the per_cpu tsc_khz variable are done in an interrupt 5509 * protected IPI, and all callers wishing to update the value 5510 * must wait for a synchronous IPI to complete (which is trivial 5511 * if the caller is on the CPU already). This establishes the 5512 * necessary total order on variable updates. 5513 * 5514 * Note that because a guest time update may take place 5515 * anytime after the setting of the VCPU's request bit, the 5516 * correct TSC value must be set before the request. However, 5517 * to ensure the update actually makes it to any guest which 5518 * starts running in hardware virtualization between the set 5519 * and the acquisition of the spinlock, we must also ping the 5520 * CPU after setting the request bit. 5521 * 5522 */ 5523 5524 if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) 5525 return 0; 5526 if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) 5527 return 0; 5528 5529 smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1); 5530 5531 spin_lock(&kvm_lock); 5532 list_for_each_entry(kvm, &vm_list, vm_list) { 5533 kvm_for_each_vcpu(i, vcpu, kvm) { 5534 if (vcpu->cpu != freq->cpu) 5535 continue; 5536 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 5537 if (vcpu->cpu != smp_processor_id()) 5538 send_ipi = 1; 5539 } 5540 } 5541 spin_unlock(&kvm_lock); 5542 5543 if (freq->old < freq->new && send_ipi) { 5544 /* 5545 * We upscale the frequency. Must make the guest 5546 * doesn't see old kvmclock values while running with 5547 * the new frequency, otherwise we risk the guest sees 5548 * time go backwards. 5549 * 5550 * In case we update the frequency for another cpu 5551 * (which might be in guest context) send an interrupt 5552 * to kick the cpu out of guest context. Next time 5553 * guest context is entered kvmclock will be updated, 5554 * so the guest will not see stale values. 5555 */ 5556 smp_call_function_single(freq->cpu, tsc_khz_changed, freq, 1); 5557 } 5558 return 0; 5559 } 5560 5561 static struct notifier_block kvmclock_cpufreq_notifier_block = { 5562 .notifier_call = kvmclock_cpufreq_notifier 5563 }; 5564 5565 static int kvmclock_cpu_notifier(struct notifier_block *nfb, 5566 unsigned long action, void *hcpu) 5567 { 5568 unsigned int cpu = (unsigned long)hcpu; 5569 5570 switch (action) { 5571 case CPU_ONLINE: 5572 case CPU_DOWN_FAILED: 5573 smp_call_function_single(cpu, tsc_khz_changed, NULL, 1); 5574 break; 5575 case CPU_DOWN_PREPARE: 5576 smp_call_function_single(cpu, tsc_bad, NULL, 1); 5577 break; 5578 } 5579 return NOTIFY_OK; 5580 } 5581 5582 static struct notifier_block kvmclock_cpu_notifier_block = { 5583 .notifier_call = kvmclock_cpu_notifier, 5584 .priority = -INT_MAX 5585 }; 5586 5587 static void kvm_timer_init(void) 5588 { 5589 int cpu; 5590 5591 max_tsc_khz = tsc_khz; 5592 5593 cpu_notifier_register_begin(); 5594 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 5595 #ifdef CONFIG_CPU_FREQ 5596 struct cpufreq_policy policy; 5597 memset(&policy, 0, sizeof(policy)); 5598 cpu = get_cpu(); 5599 cpufreq_get_policy(&policy, cpu); 5600 if (policy.cpuinfo.max_freq) 5601 max_tsc_khz = policy.cpuinfo.max_freq; 5602 put_cpu(); 5603 #endif 5604 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, 5605 CPUFREQ_TRANSITION_NOTIFIER); 5606 } 5607 pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz); 5608 for_each_online_cpu(cpu) 5609 smp_call_function_single(cpu, tsc_khz_changed, NULL, 1); 5610 5611 __register_hotcpu_notifier(&kvmclock_cpu_notifier_block); 5612 cpu_notifier_register_done(); 5613 5614 } 5615 5616 static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu); 5617 5618 int kvm_is_in_guest(void) 5619 { 5620 return __this_cpu_read(current_vcpu) != NULL; 5621 } 5622 5623 static int kvm_is_user_mode(void) 5624 { 5625 int user_mode = 3; 5626 5627 if (__this_cpu_read(current_vcpu)) 5628 user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu)); 5629 5630 return user_mode != 0; 5631 } 5632 5633 static unsigned long kvm_get_guest_ip(void) 5634 { 5635 unsigned long ip = 0; 5636 5637 if (__this_cpu_read(current_vcpu)) 5638 ip = kvm_rip_read(__this_cpu_read(current_vcpu)); 5639 5640 return ip; 5641 } 5642 5643 static struct perf_guest_info_callbacks kvm_guest_cbs = { 5644 .is_in_guest = kvm_is_in_guest, 5645 .is_user_mode = kvm_is_user_mode, 5646 .get_guest_ip = kvm_get_guest_ip, 5647 }; 5648 5649 void kvm_before_handle_nmi(struct kvm_vcpu *vcpu) 5650 { 5651 __this_cpu_write(current_vcpu, vcpu); 5652 } 5653 EXPORT_SYMBOL_GPL(kvm_before_handle_nmi); 5654 5655 void kvm_after_handle_nmi(struct kvm_vcpu *vcpu) 5656 { 5657 __this_cpu_write(current_vcpu, NULL); 5658 } 5659 EXPORT_SYMBOL_GPL(kvm_after_handle_nmi); 5660 5661 static void kvm_set_mmio_spte_mask(void) 5662 { 5663 u64 mask; 5664 int maxphyaddr = boot_cpu_data.x86_phys_bits; 5665 5666 /* 5667 * Set the reserved bits and the present bit of an paging-structure 5668 * entry to generate page fault with PFER.RSV = 1. 5669 */ 5670 /* Mask the reserved physical address bits. */ 5671 mask = rsvd_bits(maxphyaddr, 51); 5672 5673 /* Bit 62 is always reserved for 32bit host. */ 5674 mask |= 0x3ull << 62; 5675 5676 /* Set the present bit. */ 5677 mask |= 1ull; 5678 5679 #ifdef CONFIG_X86_64 5680 /* 5681 * If reserved bit is not supported, clear the present bit to disable 5682 * mmio page fault. 5683 */ 5684 if (maxphyaddr == 52) 5685 mask &= ~1ull; 5686 #endif 5687 5688 kvm_mmu_set_mmio_spte_mask(mask); 5689 } 5690 5691 #ifdef CONFIG_X86_64 5692 static void pvclock_gtod_update_fn(struct work_struct *work) 5693 { 5694 struct kvm *kvm; 5695 5696 struct kvm_vcpu *vcpu; 5697 int i; 5698 5699 spin_lock(&kvm_lock); 5700 list_for_each_entry(kvm, &vm_list, vm_list) 5701 kvm_for_each_vcpu(i, vcpu, kvm) 5702 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 5703 atomic_set(&kvm_guest_has_master_clock, 0); 5704 spin_unlock(&kvm_lock); 5705 } 5706 5707 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn); 5708 5709 /* 5710 * Notification about pvclock gtod data update. 5711 */ 5712 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused, 5713 void *priv) 5714 { 5715 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 5716 struct timekeeper *tk = priv; 5717 5718 update_pvclock_gtod(tk); 5719 5720 /* disable master clock if host does not trust, or does not 5721 * use, TSC clocksource 5722 */ 5723 if (gtod->clock.vclock_mode != VCLOCK_TSC && 5724 atomic_read(&kvm_guest_has_master_clock) != 0) 5725 queue_work(system_long_wq, &pvclock_gtod_work); 5726 5727 return 0; 5728 } 5729 5730 static struct notifier_block pvclock_gtod_notifier = { 5731 .notifier_call = pvclock_gtod_notify, 5732 }; 5733 #endif 5734 5735 int kvm_arch_init(void *opaque) 5736 { 5737 int r; 5738 struct kvm_x86_ops *ops = opaque; 5739 5740 if (kvm_x86_ops) { 5741 printk(KERN_ERR "kvm: already loaded the other module\n"); 5742 r = -EEXIST; 5743 goto out; 5744 } 5745 5746 if (!ops->cpu_has_kvm_support()) { 5747 printk(KERN_ERR "kvm: no hardware support\n"); 5748 r = -EOPNOTSUPP; 5749 goto out; 5750 } 5751 if (ops->disabled_by_bios()) { 5752 printk(KERN_ERR "kvm: disabled by bios\n"); 5753 r = -EOPNOTSUPP; 5754 goto out; 5755 } 5756 5757 r = -ENOMEM; 5758 shared_msrs = alloc_percpu(struct kvm_shared_msrs); 5759 if (!shared_msrs) { 5760 printk(KERN_ERR "kvm: failed to allocate percpu kvm_shared_msrs\n"); 5761 goto out; 5762 } 5763 5764 r = kvm_mmu_module_init(); 5765 if (r) 5766 goto out_free_percpu; 5767 5768 kvm_set_mmio_spte_mask(); 5769 5770 kvm_x86_ops = ops; 5771 5772 kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, 5773 PT_DIRTY_MASK, PT64_NX_MASK, 0); 5774 5775 kvm_timer_init(); 5776 5777 perf_register_guest_info_callbacks(&kvm_guest_cbs); 5778 5779 if (cpu_has_xsave) 5780 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 5781 5782 kvm_lapic_init(); 5783 #ifdef CONFIG_X86_64 5784 pvclock_gtod_register_notifier(&pvclock_gtod_notifier); 5785 #endif 5786 5787 return 0; 5788 5789 out_free_percpu: 5790 free_percpu(shared_msrs); 5791 out: 5792 return r; 5793 } 5794 5795 void kvm_arch_exit(void) 5796 { 5797 perf_unregister_guest_info_callbacks(&kvm_guest_cbs); 5798 5799 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) 5800 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block, 5801 CPUFREQ_TRANSITION_NOTIFIER); 5802 unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block); 5803 #ifdef CONFIG_X86_64 5804 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier); 5805 #endif 5806 kvm_x86_ops = NULL; 5807 kvm_mmu_module_exit(); 5808 free_percpu(shared_msrs); 5809 } 5810 5811 int kvm_vcpu_halt(struct kvm_vcpu *vcpu) 5812 { 5813 ++vcpu->stat.halt_exits; 5814 if (lapic_in_kernel(vcpu)) { 5815 vcpu->arch.mp_state = KVM_MP_STATE_HALTED; 5816 return 1; 5817 } else { 5818 vcpu->run->exit_reason = KVM_EXIT_HLT; 5819 return 0; 5820 } 5821 } 5822 EXPORT_SYMBOL_GPL(kvm_vcpu_halt); 5823 5824 int kvm_emulate_halt(struct kvm_vcpu *vcpu) 5825 { 5826 kvm_x86_ops->skip_emulated_instruction(vcpu); 5827 return kvm_vcpu_halt(vcpu); 5828 } 5829 EXPORT_SYMBOL_GPL(kvm_emulate_halt); 5830 5831 /* 5832 * kvm_pv_kick_cpu_op: Kick a vcpu. 5833 * 5834 * @apicid - apicid of vcpu to be kicked. 5835 */ 5836 static void kvm_pv_kick_cpu_op(struct kvm *kvm, unsigned long flags, int apicid) 5837 { 5838 struct kvm_lapic_irq lapic_irq; 5839 5840 lapic_irq.shorthand = 0; 5841 lapic_irq.dest_mode = 0; 5842 lapic_irq.dest_id = apicid; 5843 lapic_irq.msi_redir_hint = false; 5844 5845 lapic_irq.delivery_mode = APIC_DM_REMRD; 5846 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL); 5847 } 5848 5849 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) 5850 { 5851 unsigned long nr, a0, a1, a2, a3, ret; 5852 int op_64_bit, r = 1; 5853 5854 kvm_x86_ops->skip_emulated_instruction(vcpu); 5855 5856 if (kvm_hv_hypercall_enabled(vcpu->kvm)) 5857 return kvm_hv_hypercall(vcpu); 5858 5859 nr = kvm_register_read(vcpu, VCPU_REGS_RAX); 5860 a0 = kvm_register_read(vcpu, VCPU_REGS_RBX); 5861 a1 = kvm_register_read(vcpu, VCPU_REGS_RCX); 5862 a2 = kvm_register_read(vcpu, VCPU_REGS_RDX); 5863 a3 = kvm_register_read(vcpu, VCPU_REGS_RSI); 5864 5865 trace_kvm_hypercall(nr, a0, a1, a2, a3); 5866 5867 op_64_bit = is_64_bit_mode(vcpu); 5868 if (!op_64_bit) { 5869 nr &= 0xFFFFFFFF; 5870 a0 &= 0xFFFFFFFF; 5871 a1 &= 0xFFFFFFFF; 5872 a2 &= 0xFFFFFFFF; 5873 a3 &= 0xFFFFFFFF; 5874 } 5875 5876 if (kvm_x86_ops->get_cpl(vcpu) != 0) { 5877 ret = -KVM_EPERM; 5878 goto out; 5879 } 5880 5881 switch (nr) { 5882 case KVM_HC_VAPIC_POLL_IRQ: 5883 ret = 0; 5884 break; 5885 case KVM_HC_KICK_CPU: 5886 kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1); 5887 ret = 0; 5888 break; 5889 default: 5890 ret = -KVM_ENOSYS; 5891 break; 5892 } 5893 out: 5894 if (!op_64_bit) 5895 ret = (u32)ret; 5896 kvm_register_write(vcpu, VCPU_REGS_RAX, ret); 5897 ++vcpu->stat.hypercalls; 5898 return r; 5899 } 5900 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); 5901 5902 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) 5903 { 5904 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 5905 char instruction[3]; 5906 unsigned long rip = kvm_rip_read(vcpu); 5907 5908 kvm_x86_ops->patch_hypercall(vcpu, instruction); 5909 5910 return emulator_write_emulated(ctxt, rip, instruction, 3, NULL); 5911 } 5912 5913 /* 5914 * Check if userspace requested an interrupt window, and that the 5915 * interrupt window is open. 5916 * 5917 * No need to exit to userspace if we already have an interrupt queued. 5918 */ 5919 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) 5920 { 5921 if (!vcpu->run->request_interrupt_window || pic_in_kernel(vcpu->kvm)) 5922 return false; 5923 5924 if (kvm_cpu_has_interrupt(vcpu)) 5925 return false; 5926 5927 return (irqchip_split(vcpu->kvm) 5928 ? kvm_apic_accept_pic_intr(vcpu) 5929 : kvm_arch_interrupt_allowed(vcpu)); 5930 } 5931 5932 static void post_kvm_run_save(struct kvm_vcpu *vcpu) 5933 { 5934 struct kvm_run *kvm_run = vcpu->run; 5935 5936 kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0; 5937 kvm_run->flags = is_smm(vcpu) ? KVM_RUN_X86_SMM : 0; 5938 kvm_run->cr8 = kvm_get_cr8(vcpu); 5939 kvm_run->apic_base = kvm_get_apic_base(vcpu); 5940 if (!irqchip_in_kernel(vcpu->kvm)) 5941 kvm_run->ready_for_interrupt_injection = 5942 kvm_arch_interrupt_allowed(vcpu) && 5943 !kvm_cpu_has_interrupt(vcpu) && 5944 !kvm_event_needs_reinjection(vcpu); 5945 else if (!pic_in_kernel(vcpu->kvm)) 5946 kvm_run->ready_for_interrupt_injection = 5947 kvm_apic_accept_pic_intr(vcpu) && 5948 !kvm_cpu_has_interrupt(vcpu); 5949 else 5950 kvm_run->ready_for_interrupt_injection = 1; 5951 } 5952 5953 static void update_cr8_intercept(struct kvm_vcpu *vcpu) 5954 { 5955 int max_irr, tpr; 5956 5957 if (!kvm_x86_ops->update_cr8_intercept) 5958 return; 5959 5960 if (!vcpu->arch.apic) 5961 return; 5962 5963 if (!vcpu->arch.apic->vapic_addr) 5964 max_irr = kvm_lapic_find_highest_irr(vcpu); 5965 else 5966 max_irr = -1; 5967 5968 if (max_irr != -1) 5969 max_irr >>= 4; 5970 5971 tpr = kvm_lapic_get_cr8(vcpu); 5972 5973 kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr); 5974 } 5975 5976 static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win) 5977 { 5978 int r; 5979 5980 /* try to reinject previous events if any */ 5981 if (vcpu->arch.exception.pending) { 5982 trace_kvm_inj_exception(vcpu->arch.exception.nr, 5983 vcpu->arch.exception.has_error_code, 5984 vcpu->arch.exception.error_code); 5985 5986 if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT) 5987 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) | 5988 X86_EFLAGS_RF); 5989 5990 if (vcpu->arch.exception.nr == DB_VECTOR && 5991 (vcpu->arch.dr7 & DR7_GD)) { 5992 vcpu->arch.dr7 &= ~DR7_GD; 5993 kvm_update_dr7(vcpu); 5994 } 5995 5996 kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr, 5997 vcpu->arch.exception.has_error_code, 5998 vcpu->arch.exception.error_code, 5999 vcpu->arch.exception.reinject); 6000 return 0; 6001 } 6002 6003 if (vcpu->arch.nmi_injected) { 6004 kvm_x86_ops->set_nmi(vcpu); 6005 return 0; 6006 } 6007 6008 if (vcpu->arch.interrupt.pending) { 6009 kvm_x86_ops->set_irq(vcpu); 6010 return 0; 6011 } 6012 6013 if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) { 6014 r = kvm_x86_ops->check_nested_events(vcpu, req_int_win); 6015 if (r != 0) 6016 return r; 6017 } 6018 6019 /* try to inject new event if pending */ 6020 if (vcpu->arch.nmi_pending) { 6021 if (kvm_x86_ops->nmi_allowed(vcpu)) { 6022 --vcpu->arch.nmi_pending; 6023 vcpu->arch.nmi_injected = true; 6024 kvm_x86_ops->set_nmi(vcpu); 6025 } 6026 } else if (kvm_cpu_has_injectable_intr(vcpu)) { 6027 /* 6028 * Because interrupts can be injected asynchronously, we are 6029 * calling check_nested_events again here to avoid a race condition. 6030 * See https://lkml.org/lkml/2014/7/2/60 for discussion about this 6031 * proposal and current concerns. Perhaps we should be setting 6032 * KVM_REQ_EVENT only on certain events and not unconditionally? 6033 */ 6034 if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) { 6035 r = kvm_x86_ops->check_nested_events(vcpu, req_int_win); 6036 if (r != 0) 6037 return r; 6038 } 6039 if (kvm_x86_ops->interrupt_allowed(vcpu)) { 6040 kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), 6041 false); 6042 kvm_x86_ops->set_irq(vcpu); 6043 } 6044 } 6045 return 0; 6046 } 6047 6048 static void process_nmi(struct kvm_vcpu *vcpu) 6049 { 6050 unsigned limit = 2; 6051 6052 /* 6053 * x86 is limited to one NMI running, and one NMI pending after it. 6054 * If an NMI is already in progress, limit further NMIs to just one. 6055 * Otherwise, allow two (and we'll inject the first one immediately). 6056 */ 6057 if (kvm_x86_ops->get_nmi_mask(vcpu) || vcpu->arch.nmi_injected) 6058 limit = 1; 6059 6060 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0); 6061 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit); 6062 kvm_make_request(KVM_REQ_EVENT, vcpu); 6063 } 6064 6065 #define put_smstate(type, buf, offset, val) \ 6066 *(type *)((buf) + (offset) - 0x7e00) = val 6067 6068 static u32 process_smi_get_segment_flags(struct kvm_segment *seg) 6069 { 6070 u32 flags = 0; 6071 flags |= seg->g << 23; 6072 flags |= seg->db << 22; 6073 flags |= seg->l << 21; 6074 flags |= seg->avl << 20; 6075 flags |= seg->present << 15; 6076 flags |= seg->dpl << 13; 6077 flags |= seg->s << 12; 6078 flags |= seg->type << 8; 6079 return flags; 6080 } 6081 6082 static void process_smi_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n) 6083 { 6084 struct kvm_segment seg; 6085 int offset; 6086 6087 kvm_get_segment(vcpu, &seg, n); 6088 put_smstate(u32, buf, 0x7fa8 + n * 4, seg.selector); 6089 6090 if (n < 3) 6091 offset = 0x7f84 + n * 12; 6092 else 6093 offset = 0x7f2c + (n - 3) * 12; 6094 6095 put_smstate(u32, buf, offset + 8, seg.base); 6096 put_smstate(u32, buf, offset + 4, seg.limit); 6097 put_smstate(u32, buf, offset, process_smi_get_segment_flags(&seg)); 6098 } 6099 6100 #ifdef CONFIG_X86_64 6101 static void process_smi_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n) 6102 { 6103 struct kvm_segment seg; 6104 int offset; 6105 u16 flags; 6106 6107 kvm_get_segment(vcpu, &seg, n); 6108 offset = 0x7e00 + n * 16; 6109 6110 flags = process_smi_get_segment_flags(&seg) >> 8; 6111 put_smstate(u16, buf, offset, seg.selector); 6112 put_smstate(u16, buf, offset + 2, flags); 6113 put_smstate(u32, buf, offset + 4, seg.limit); 6114 put_smstate(u64, buf, offset + 8, seg.base); 6115 } 6116 #endif 6117 6118 static void process_smi_save_state_32(struct kvm_vcpu *vcpu, char *buf) 6119 { 6120 struct desc_ptr dt; 6121 struct kvm_segment seg; 6122 unsigned long val; 6123 int i; 6124 6125 put_smstate(u32, buf, 0x7ffc, kvm_read_cr0(vcpu)); 6126 put_smstate(u32, buf, 0x7ff8, kvm_read_cr3(vcpu)); 6127 put_smstate(u32, buf, 0x7ff4, kvm_get_rflags(vcpu)); 6128 put_smstate(u32, buf, 0x7ff0, kvm_rip_read(vcpu)); 6129 6130 for (i = 0; i < 8; i++) 6131 put_smstate(u32, buf, 0x7fd0 + i * 4, kvm_register_read(vcpu, i)); 6132 6133 kvm_get_dr(vcpu, 6, &val); 6134 put_smstate(u32, buf, 0x7fcc, (u32)val); 6135 kvm_get_dr(vcpu, 7, &val); 6136 put_smstate(u32, buf, 0x7fc8, (u32)val); 6137 6138 kvm_get_segment(vcpu, &seg, VCPU_SREG_TR); 6139 put_smstate(u32, buf, 0x7fc4, seg.selector); 6140 put_smstate(u32, buf, 0x7f64, seg.base); 6141 put_smstate(u32, buf, 0x7f60, seg.limit); 6142 put_smstate(u32, buf, 0x7f5c, process_smi_get_segment_flags(&seg)); 6143 6144 kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR); 6145 put_smstate(u32, buf, 0x7fc0, seg.selector); 6146 put_smstate(u32, buf, 0x7f80, seg.base); 6147 put_smstate(u32, buf, 0x7f7c, seg.limit); 6148 put_smstate(u32, buf, 0x7f78, process_smi_get_segment_flags(&seg)); 6149 6150 kvm_x86_ops->get_gdt(vcpu, &dt); 6151 put_smstate(u32, buf, 0x7f74, dt.address); 6152 put_smstate(u32, buf, 0x7f70, dt.size); 6153 6154 kvm_x86_ops->get_idt(vcpu, &dt); 6155 put_smstate(u32, buf, 0x7f58, dt.address); 6156 put_smstate(u32, buf, 0x7f54, dt.size); 6157 6158 for (i = 0; i < 6; i++) 6159 process_smi_save_seg_32(vcpu, buf, i); 6160 6161 put_smstate(u32, buf, 0x7f14, kvm_read_cr4(vcpu)); 6162 6163 /* revision id */ 6164 put_smstate(u32, buf, 0x7efc, 0x00020000); 6165 put_smstate(u32, buf, 0x7ef8, vcpu->arch.smbase); 6166 } 6167 6168 static void process_smi_save_state_64(struct kvm_vcpu *vcpu, char *buf) 6169 { 6170 #ifdef CONFIG_X86_64 6171 struct desc_ptr dt; 6172 struct kvm_segment seg; 6173 unsigned long val; 6174 int i; 6175 6176 for (i = 0; i < 16; i++) 6177 put_smstate(u64, buf, 0x7ff8 - i * 8, kvm_register_read(vcpu, i)); 6178 6179 put_smstate(u64, buf, 0x7f78, kvm_rip_read(vcpu)); 6180 put_smstate(u32, buf, 0x7f70, kvm_get_rflags(vcpu)); 6181 6182 kvm_get_dr(vcpu, 6, &val); 6183 put_smstate(u64, buf, 0x7f68, val); 6184 kvm_get_dr(vcpu, 7, &val); 6185 put_smstate(u64, buf, 0x7f60, val); 6186 6187 put_smstate(u64, buf, 0x7f58, kvm_read_cr0(vcpu)); 6188 put_smstate(u64, buf, 0x7f50, kvm_read_cr3(vcpu)); 6189 put_smstate(u64, buf, 0x7f48, kvm_read_cr4(vcpu)); 6190 6191 put_smstate(u32, buf, 0x7f00, vcpu->arch.smbase); 6192 6193 /* revision id */ 6194 put_smstate(u32, buf, 0x7efc, 0x00020064); 6195 6196 put_smstate(u64, buf, 0x7ed0, vcpu->arch.efer); 6197 6198 kvm_get_segment(vcpu, &seg, VCPU_SREG_TR); 6199 put_smstate(u16, buf, 0x7e90, seg.selector); 6200 put_smstate(u16, buf, 0x7e92, process_smi_get_segment_flags(&seg) >> 8); 6201 put_smstate(u32, buf, 0x7e94, seg.limit); 6202 put_smstate(u64, buf, 0x7e98, seg.base); 6203 6204 kvm_x86_ops->get_idt(vcpu, &dt); 6205 put_smstate(u32, buf, 0x7e84, dt.size); 6206 put_smstate(u64, buf, 0x7e88, dt.address); 6207 6208 kvm_get_segment(vcpu, &seg, VCPU_SREG_LDTR); 6209 put_smstate(u16, buf, 0x7e70, seg.selector); 6210 put_smstate(u16, buf, 0x7e72, process_smi_get_segment_flags(&seg) >> 8); 6211 put_smstate(u32, buf, 0x7e74, seg.limit); 6212 put_smstate(u64, buf, 0x7e78, seg.base); 6213 6214 kvm_x86_ops->get_gdt(vcpu, &dt); 6215 put_smstate(u32, buf, 0x7e64, dt.size); 6216 put_smstate(u64, buf, 0x7e68, dt.address); 6217 6218 for (i = 0; i < 6; i++) 6219 process_smi_save_seg_64(vcpu, buf, i); 6220 #else 6221 WARN_ON_ONCE(1); 6222 #endif 6223 } 6224 6225 static void process_smi(struct kvm_vcpu *vcpu) 6226 { 6227 struct kvm_segment cs, ds; 6228 struct desc_ptr dt; 6229 char buf[512]; 6230 u32 cr0; 6231 6232 if (is_smm(vcpu)) { 6233 vcpu->arch.smi_pending = true; 6234 return; 6235 } 6236 6237 trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true); 6238 vcpu->arch.hflags |= HF_SMM_MASK; 6239 memset(buf, 0, 512); 6240 if (guest_cpuid_has_longmode(vcpu)) 6241 process_smi_save_state_64(vcpu, buf); 6242 else 6243 process_smi_save_state_32(vcpu, buf); 6244 6245 kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf)); 6246 6247 if (kvm_x86_ops->get_nmi_mask(vcpu)) 6248 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; 6249 else 6250 kvm_x86_ops->set_nmi_mask(vcpu, true); 6251 6252 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); 6253 kvm_rip_write(vcpu, 0x8000); 6254 6255 cr0 = vcpu->arch.cr0 & ~(X86_CR0_PE | X86_CR0_EM | X86_CR0_TS | X86_CR0_PG); 6256 kvm_x86_ops->set_cr0(vcpu, cr0); 6257 vcpu->arch.cr0 = cr0; 6258 6259 kvm_x86_ops->set_cr4(vcpu, 0); 6260 6261 /* Undocumented: IDT limit is set to zero on entry to SMM. */ 6262 dt.address = dt.size = 0; 6263 kvm_x86_ops->set_idt(vcpu, &dt); 6264 6265 __kvm_set_dr(vcpu, 7, DR7_FIXED_1); 6266 6267 cs.selector = (vcpu->arch.smbase >> 4) & 0xffff; 6268 cs.base = vcpu->arch.smbase; 6269 6270 ds.selector = 0; 6271 ds.base = 0; 6272 6273 cs.limit = ds.limit = 0xffffffff; 6274 cs.type = ds.type = 0x3; 6275 cs.dpl = ds.dpl = 0; 6276 cs.db = ds.db = 0; 6277 cs.s = ds.s = 1; 6278 cs.l = ds.l = 0; 6279 cs.g = ds.g = 1; 6280 cs.avl = ds.avl = 0; 6281 cs.present = ds.present = 1; 6282 cs.unusable = ds.unusable = 0; 6283 cs.padding = ds.padding = 0; 6284 6285 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS); 6286 kvm_set_segment(vcpu, &ds, VCPU_SREG_DS); 6287 kvm_set_segment(vcpu, &ds, VCPU_SREG_ES); 6288 kvm_set_segment(vcpu, &ds, VCPU_SREG_FS); 6289 kvm_set_segment(vcpu, &ds, VCPU_SREG_GS); 6290 kvm_set_segment(vcpu, &ds, VCPU_SREG_SS); 6291 6292 if (guest_cpuid_has_longmode(vcpu)) 6293 kvm_x86_ops->set_efer(vcpu, 0); 6294 6295 kvm_update_cpuid(vcpu); 6296 kvm_mmu_reset_context(vcpu); 6297 } 6298 6299 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) 6300 { 6301 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 6302 return; 6303 6304 memset(vcpu->arch.eoi_exit_bitmap, 0, 256 / 8); 6305 6306 if (irqchip_split(vcpu->kvm)) 6307 kvm_scan_ioapic_routes(vcpu, vcpu->arch.eoi_exit_bitmap); 6308 else { 6309 kvm_x86_ops->sync_pir_to_irr(vcpu); 6310 kvm_ioapic_scan_entry(vcpu, vcpu->arch.eoi_exit_bitmap); 6311 } 6312 kvm_x86_ops->load_eoi_exitmap(vcpu); 6313 } 6314 6315 static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu) 6316 { 6317 ++vcpu->stat.tlb_flush; 6318 kvm_x86_ops->tlb_flush(vcpu); 6319 } 6320 6321 void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) 6322 { 6323 struct page *page = NULL; 6324 6325 if (!lapic_in_kernel(vcpu)) 6326 return; 6327 6328 if (!kvm_x86_ops->set_apic_access_page_addr) 6329 return; 6330 6331 page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 6332 if (is_error_page(page)) 6333 return; 6334 kvm_x86_ops->set_apic_access_page_addr(vcpu, page_to_phys(page)); 6335 6336 /* 6337 * Do not pin apic access page in memory, the MMU notifier 6338 * will call us again if it is migrated or swapped out. 6339 */ 6340 put_page(page); 6341 } 6342 EXPORT_SYMBOL_GPL(kvm_vcpu_reload_apic_access_page); 6343 6344 void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, 6345 unsigned long address) 6346 { 6347 /* 6348 * The physical address of apic access page is stored in the VMCS. 6349 * Update it when it becomes invalid. 6350 */ 6351 if (address == gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT)) 6352 kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD); 6353 } 6354 6355 /* 6356 * Returns 1 to let vcpu_run() continue the guest execution loop without 6357 * exiting to the userspace. Otherwise, the value will be returned to the 6358 * userspace. 6359 */ 6360 static int vcpu_enter_guest(struct kvm_vcpu *vcpu) 6361 { 6362 int r; 6363 bool req_int_win = !lapic_in_kernel(vcpu) && 6364 vcpu->run->request_interrupt_window; 6365 bool req_immediate_exit = false; 6366 6367 if (vcpu->requests) { 6368 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) 6369 kvm_mmu_unload(vcpu); 6370 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu)) 6371 __kvm_migrate_timers(vcpu); 6372 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) 6373 kvm_gen_update_masterclock(vcpu->kvm); 6374 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu)) 6375 kvm_gen_kvmclock_update(vcpu); 6376 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) { 6377 r = kvm_guest_time_update(vcpu); 6378 if (unlikely(r)) 6379 goto out; 6380 } 6381 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu)) 6382 kvm_mmu_sync_roots(vcpu); 6383 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 6384 kvm_vcpu_flush_tlb(vcpu); 6385 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { 6386 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; 6387 r = 0; 6388 goto out; 6389 } 6390 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 6391 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 6392 r = 0; 6393 goto out; 6394 } 6395 if (kvm_check_request(KVM_REQ_DEACTIVATE_FPU, vcpu)) { 6396 vcpu->fpu_active = 0; 6397 kvm_x86_ops->fpu_deactivate(vcpu); 6398 } 6399 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { 6400 /* Page is swapped out. Do synthetic halt */ 6401 vcpu->arch.apf.halted = true; 6402 r = 1; 6403 goto out; 6404 } 6405 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 6406 record_steal_time(vcpu); 6407 if (kvm_check_request(KVM_REQ_SMI, vcpu)) 6408 process_smi(vcpu); 6409 if (kvm_check_request(KVM_REQ_NMI, vcpu)) 6410 process_nmi(vcpu); 6411 if (kvm_check_request(KVM_REQ_PMU, vcpu)) 6412 kvm_pmu_handle_event(vcpu); 6413 if (kvm_check_request(KVM_REQ_PMI, vcpu)) 6414 kvm_pmu_deliver_pmi(vcpu); 6415 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { 6416 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); 6417 if (test_bit(vcpu->arch.pending_ioapic_eoi, 6418 (void *) vcpu->arch.eoi_exit_bitmap)) { 6419 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; 6420 vcpu->run->eoi.vector = 6421 vcpu->arch.pending_ioapic_eoi; 6422 r = 0; 6423 goto out; 6424 } 6425 } 6426 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu)) 6427 vcpu_scan_ioapic(vcpu); 6428 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu)) 6429 kvm_vcpu_reload_apic_access_page(vcpu); 6430 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) { 6431 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 6432 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; 6433 r = 0; 6434 goto out; 6435 } 6436 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) { 6437 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 6438 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET; 6439 r = 0; 6440 goto out; 6441 } 6442 } 6443 6444 /* 6445 * KVM_REQ_EVENT is not set when posted interrupts are set by 6446 * VT-d hardware, so we have to update RVI unconditionally. 6447 */ 6448 if (kvm_lapic_enabled(vcpu)) { 6449 /* 6450 * Update architecture specific hints for APIC 6451 * virtual interrupt delivery. 6452 */ 6453 if (kvm_x86_ops->hwapic_irr_update) 6454 kvm_x86_ops->hwapic_irr_update(vcpu, 6455 kvm_lapic_find_highest_irr(vcpu)); 6456 } 6457 6458 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) { 6459 kvm_apic_accept_events(vcpu); 6460 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 6461 r = 1; 6462 goto out; 6463 } 6464 6465 if (inject_pending_event(vcpu, req_int_win) != 0) 6466 req_immediate_exit = true; 6467 /* enable NMI/IRQ window open exits if needed */ 6468 else if (vcpu->arch.nmi_pending) 6469 kvm_x86_ops->enable_nmi_window(vcpu); 6470 else if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win) 6471 kvm_x86_ops->enable_irq_window(vcpu); 6472 6473 if (kvm_lapic_enabled(vcpu)) { 6474 update_cr8_intercept(vcpu); 6475 kvm_lapic_sync_to_vapic(vcpu); 6476 } 6477 } 6478 6479 r = kvm_mmu_reload(vcpu); 6480 if (unlikely(r)) { 6481 goto cancel_injection; 6482 } 6483 6484 preempt_disable(); 6485 6486 kvm_x86_ops->prepare_guest_switch(vcpu); 6487 if (vcpu->fpu_active) 6488 kvm_load_guest_fpu(vcpu); 6489 kvm_load_guest_xcr0(vcpu); 6490 6491 vcpu->mode = IN_GUEST_MODE; 6492 6493 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 6494 6495 /* We should set ->mode before check ->requests, 6496 * see the comment in make_all_cpus_request. 6497 */ 6498 smp_mb__after_srcu_read_unlock(); 6499 6500 local_irq_disable(); 6501 6502 if (vcpu->mode == EXITING_GUEST_MODE || vcpu->requests 6503 || need_resched() || signal_pending(current)) { 6504 vcpu->mode = OUTSIDE_GUEST_MODE; 6505 smp_wmb(); 6506 local_irq_enable(); 6507 preempt_enable(); 6508 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 6509 r = 1; 6510 goto cancel_injection; 6511 } 6512 6513 if (req_immediate_exit) 6514 smp_send_reschedule(vcpu->cpu); 6515 6516 __kvm_guest_enter(); 6517 6518 if (unlikely(vcpu->arch.switch_db_regs)) { 6519 set_debugreg(0, 7); 6520 set_debugreg(vcpu->arch.eff_db[0], 0); 6521 set_debugreg(vcpu->arch.eff_db[1], 1); 6522 set_debugreg(vcpu->arch.eff_db[2], 2); 6523 set_debugreg(vcpu->arch.eff_db[3], 3); 6524 set_debugreg(vcpu->arch.dr6, 6); 6525 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD; 6526 } 6527 6528 trace_kvm_entry(vcpu->vcpu_id); 6529 wait_lapic_expire(vcpu); 6530 kvm_x86_ops->run(vcpu); 6531 6532 /* 6533 * Do this here before restoring debug registers on the host. And 6534 * since we do this before handling the vmexit, a DR access vmexit 6535 * can (a) read the correct value of the debug registers, (b) set 6536 * KVM_DEBUGREG_WONT_EXIT again. 6537 */ 6538 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) { 6539 int i; 6540 6541 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP); 6542 kvm_x86_ops->sync_dirty_debug_regs(vcpu); 6543 for (i = 0; i < KVM_NR_DB_REGS; i++) 6544 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 6545 } 6546 6547 /* 6548 * If the guest has used debug registers, at least dr7 6549 * will be disabled while returning to the host. 6550 * If we don't have active breakpoints in the host, we don't 6551 * care about the messed up debug address registers. But if 6552 * we have some of them active, restore the old state. 6553 */ 6554 if (hw_breakpoint_active()) 6555 hw_breakpoint_restore(); 6556 6557 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 6558 6559 vcpu->mode = OUTSIDE_GUEST_MODE; 6560 smp_wmb(); 6561 6562 /* Interrupt is enabled by handle_external_intr() */ 6563 kvm_x86_ops->handle_external_intr(vcpu); 6564 6565 ++vcpu->stat.exits; 6566 6567 /* 6568 * We must have an instruction between local_irq_enable() and 6569 * kvm_guest_exit(), so the timer interrupt isn't delayed by 6570 * the interrupt shadow. The stat.exits increment will do nicely. 6571 * But we need to prevent reordering, hence this barrier(): 6572 */ 6573 barrier(); 6574 6575 kvm_guest_exit(); 6576 6577 preempt_enable(); 6578 6579 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 6580 6581 /* 6582 * Profile KVM exit RIPs: 6583 */ 6584 if (unlikely(prof_on == KVM_PROFILING)) { 6585 unsigned long rip = kvm_rip_read(vcpu); 6586 profile_hit(KVM_PROFILING, (void *)rip); 6587 } 6588 6589 if (unlikely(vcpu->arch.tsc_always_catchup)) 6590 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 6591 6592 if (vcpu->arch.apic_attention) 6593 kvm_lapic_sync_from_vapic(vcpu); 6594 6595 r = kvm_x86_ops->handle_exit(vcpu); 6596 return r; 6597 6598 cancel_injection: 6599 kvm_x86_ops->cancel_injection(vcpu); 6600 if (unlikely(vcpu->arch.apic_attention)) 6601 kvm_lapic_sync_from_vapic(vcpu); 6602 out: 6603 return r; 6604 } 6605 6606 static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) 6607 { 6608 if (!kvm_arch_vcpu_runnable(vcpu) && 6609 (!kvm_x86_ops->pre_block || kvm_x86_ops->pre_block(vcpu) == 0)) { 6610 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 6611 kvm_vcpu_block(vcpu); 6612 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 6613 6614 if (kvm_x86_ops->post_block) 6615 kvm_x86_ops->post_block(vcpu); 6616 6617 if (!kvm_check_request(KVM_REQ_UNHALT, vcpu)) 6618 return 1; 6619 } 6620 6621 kvm_apic_accept_events(vcpu); 6622 switch(vcpu->arch.mp_state) { 6623 case KVM_MP_STATE_HALTED: 6624 vcpu->arch.pv.pv_unhalted = false; 6625 vcpu->arch.mp_state = 6626 KVM_MP_STATE_RUNNABLE; 6627 case KVM_MP_STATE_RUNNABLE: 6628 vcpu->arch.apf.halted = false; 6629 break; 6630 case KVM_MP_STATE_INIT_RECEIVED: 6631 break; 6632 default: 6633 return -EINTR; 6634 break; 6635 } 6636 return 1; 6637 } 6638 6639 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) 6640 { 6641 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 6642 !vcpu->arch.apf.halted); 6643 } 6644 6645 static int vcpu_run(struct kvm_vcpu *vcpu) 6646 { 6647 int r; 6648 struct kvm *kvm = vcpu->kvm; 6649 6650 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 6651 6652 for (;;) { 6653 if (kvm_vcpu_running(vcpu)) { 6654 r = vcpu_enter_guest(vcpu); 6655 } else { 6656 r = vcpu_block(kvm, vcpu); 6657 } 6658 6659 if (r <= 0) 6660 break; 6661 6662 clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests); 6663 if (kvm_cpu_has_pending_timer(vcpu)) 6664 kvm_inject_pending_timer_irqs(vcpu); 6665 6666 if (dm_request_for_irq_injection(vcpu)) { 6667 r = 0; 6668 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 6669 ++vcpu->stat.request_irq_exits; 6670 break; 6671 } 6672 6673 kvm_check_async_pf_completion(vcpu); 6674 6675 if (signal_pending(current)) { 6676 r = -EINTR; 6677 vcpu->run->exit_reason = KVM_EXIT_INTR; 6678 ++vcpu->stat.signal_exits; 6679 break; 6680 } 6681 if (need_resched()) { 6682 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 6683 cond_resched(); 6684 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 6685 } 6686 } 6687 6688 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx); 6689 6690 return r; 6691 } 6692 6693 static inline int complete_emulated_io(struct kvm_vcpu *vcpu) 6694 { 6695 int r; 6696 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 6697 r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); 6698 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 6699 if (r != EMULATE_DONE) 6700 return 0; 6701 return 1; 6702 } 6703 6704 static int complete_emulated_pio(struct kvm_vcpu *vcpu) 6705 { 6706 BUG_ON(!vcpu->arch.pio.count); 6707 6708 return complete_emulated_io(vcpu); 6709 } 6710 6711 /* 6712 * Implements the following, as a state machine: 6713 * 6714 * read: 6715 * for each fragment 6716 * for each mmio piece in the fragment 6717 * write gpa, len 6718 * exit 6719 * copy data 6720 * execute insn 6721 * 6722 * write: 6723 * for each fragment 6724 * for each mmio piece in the fragment 6725 * write gpa, len 6726 * copy data 6727 * exit 6728 */ 6729 static int complete_emulated_mmio(struct kvm_vcpu *vcpu) 6730 { 6731 struct kvm_run *run = vcpu->run; 6732 struct kvm_mmio_fragment *frag; 6733 unsigned len; 6734 6735 BUG_ON(!vcpu->mmio_needed); 6736 6737 /* Complete previous fragment */ 6738 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; 6739 len = min(8u, frag->len); 6740 if (!vcpu->mmio_is_write) 6741 memcpy(frag->data, run->mmio.data, len); 6742 6743 if (frag->len <= 8) { 6744 /* Switch to the next fragment. */ 6745 frag++; 6746 vcpu->mmio_cur_fragment++; 6747 } else { 6748 /* Go forward to the next mmio piece. */ 6749 frag->data += len; 6750 frag->gpa += len; 6751 frag->len -= len; 6752 } 6753 6754 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { 6755 vcpu->mmio_needed = 0; 6756 6757 /* FIXME: return into emulator if single-stepping. */ 6758 if (vcpu->mmio_is_write) 6759 return 1; 6760 vcpu->mmio_read_completed = 1; 6761 return complete_emulated_io(vcpu); 6762 } 6763 6764 run->exit_reason = KVM_EXIT_MMIO; 6765 run->mmio.phys_addr = frag->gpa; 6766 if (vcpu->mmio_is_write) 6767 memcpy(run->mmio.data, frag->data, min(8u, frag->len)); 6768 run->mmio.len = min(8u, frag->len); 6769 run->mmio.is_write = vcpu->mmio_is_write; 6770 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 6771 return 0; 6772 } 6773 6774 6775 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 6776 { 6777 struct fpu *fpu = ¤t->thread.fpu; 6778 int r; 6779 sigset_t sigsaved; 6780 6781 fpu__activate_curr(fpu); 6782 6783 if (vcpu->sigset_active) 6784 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); 6785 6786 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { 6787 kvm_vcpu_block(vcpu); 6788 kvm_apic_accept_events(vcpu); 6789 clear_bit(KVM_REQ_UNHALT, &vcpu->requests); 6790 r = -EAGAIN; 6791 goto out; 6792 } 6793 6794 /* re-sync apic's tpr */ 6795 if (!lapic_in_kernel(vcpu)) { 6796 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { 6797 r = -EINVAL; 6798 goto out; 6799 } 6800 } 6801 6802 if (unlikely(vcpu->arch.complete_userspace_io)) { 6803 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io; 6804 vcpu->arch.complete_userspace_io = NULL; 6805 r = cui(vcpu); 6806 if (r <= 0) 6807 goto out; 6808 } else 6809 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed); 6810 6811 r = vcpu_run(vcpu); 6812 6813 out: 6814 post_kvm_run_save(vcpu); 6815 if (vcpu->sigset_active) 6816 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 6817 6818 return r; 6819 } 6820 6821 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 6822 { 6823 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { 6824 /* 6825 * We are here if userspace calls get_regs() in the middle of 6826 * instruction emulation. Registers state needs to be copied 6827 * back from emulation context to vcpu. Userspace shouldn't do 6828 * that usually, but some bad designed PV devices (vmware 6829 * backdoor interface) need this to work 6830 */ 6831 emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt); 6832 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 6833 } 6834 regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX); 6835 regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX); 6836 regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX); 6837 regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX); 6838 regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI); 6839 regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI); 6840 regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP); 6841 regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP); 6842 #ifdef CONFIG_X86_64 6843 regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8); 6844 regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9); 6845 regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10); 6846 regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11); 6847 regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12); 6848 regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13); 6849 regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14); 6850 regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15); 6851 #endif 6852 6853 regs->rip = kvm_rip_read(vcpu); 6854 regs->rflags = kvm_get_rflags(vcpu); 6855 6856 return 0; 6857 } 6858 6859 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 6860 { 6861 vcpu->arch.emulate_regs_need_sync_from_vcpu = true; 6862 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 6863 6864 kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax); 6865 kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx); 6866 kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx); 6867 kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx); 6868 kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi); 6869 kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi); 6870 kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp); 6871 kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp); 6872 #ifdef CONFIG_X86_64 6873 kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8); 6874 kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9); 6875 kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10); 6876 kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11); 6877 kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12); 6878 kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13); 6879 kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14); 6880 kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15); 6881 #endif 6882 6883 kvm_rip_write(vcpu, regs->rip); 6884 kvm_set_rflags(vcpu, regs->rflags); 6885 6886 vcpu->arch.exception.pending = false; 6887 6888 kvm_make_request(KVM_REQ_EVENT, vcpu); 6889 6890 return 0; 6891 } 6892 6893 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) 6894 { 6895 struct kvm_segment cs; 6896 6897 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); 6898 *db = cs.db; 6899 *l = cs.l; 6900 } 6901 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); 6902 6903 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 6904 struct kvm_sregs *sregs) 6905 { 6906 struct desc_ptr dt; 6907 6908 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 6909 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 6910 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); 6911 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 6912 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 6913 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 6914 6915 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 6916 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 6917 6918 kvm_x86_ops->get_idt(vcpu, &dt); 6919 sregs->idt.limit = dt.size; 6920 sregs->idt.base = dt.address; 6921 kvm_x86_ops->get_gdt(vcpu, &dt); 6922 sregs->gdt.limit = dt.size; 6923 sregs->gdt.base = dt.address; 6924 6925 sregs->cr0 = kvm_read_cr0(vcpu); 6926 sregs->cr2 = vcpu->arch.cr2; 6927 sregs->cr3 = kvm_read_cr3(vcpu); 6928 sregs->cr4 = kvm_read_cr4(vcpu); 6929 sregs->cr8 = kvm_get_cr8(vcpu); 6930 sregs->efer = vcpu->arch.efer; 6931 sregs->apic_base = kvm_get_apic_base(vcpu); 6932 6933 memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap); 6934 6935 if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft) 6936 set_bit(vcpu->arch.interrupt.nr, 6937 (unsigned long *)sregs->interrupt_bitmap); 6938 6939 return 0; 6940 } 6941 6942 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 6943 struct kvm_mp_state *mp_state) 6944 { 6945 kvm_apic_accept_events(vcpu); 6946 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED && 6947 vcpu->arch.pv.pv_unhalted) 6948 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 6949 else 6950 mp_state->mp_state = vcpu->arch.mp_state; 6951 6952 return 0; 6953 } 6954 6955 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 6956 struct kvm_mp_state *mp_state) 6957 { 6958 if (!kvm_vcpu_has_lapic(vcpu) && 6959 mp_state->mp_state != KVM_MP_STATE_RUNNABLE) 6960 return -EINVAL; 6961 6962 if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) { 6963 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 6964 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events); 6965 } else 6966 vcpu->arch.mp_state = mp_state->mp_state; 6967 kvm_make_request(KVM_REQ_EVENT, vcpu); 6968 return 0; 6969 } 6970 6971 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, 6972 int reason, bool has_error_code, u32 error_code) 6973 { 6974 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 6975 int ret; 6976 6977 init_emulate_ctxt(vcpu); 6978 6979 ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason, 6980 has_error_code, error_code); 6981 6982 if (ret) 6983 return EMULATE_FAIL; 6984 6985 kvm_rip_write(vcpu, ctxt->eip); 6986 kvm_set_rflags(vcpu, ctxt->eflags); 6987 kvm_make_request(KVM_REQ_EVENT, vcpu); 6988 return EMULATE_DONE; 6989 } 6990 EXPORT_SYMBOL_GPL(kvm_task_switch); 6991 6992 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 6993 struct kvm_sregs *sregs) 6994 { 6995 struct msr_data apic_base_msr; 6996 int mmu_reset_needed = 0; 6997 int pending_vec, max_bits, idx; 6998 struct desc_ptr dt; 6999 7000 if (!guest_cpuid_has_xsave(vcpu) && (sregs->cr4 & X86_CR4_OSXSAVE)) 7001 return -EINVAL; 7002 7003 dt.size = sregs->idt.limit; 7004 dt.address = sregs->idt.base; 7005 kvm_x86_ops->set_idt(vcpu, &dt); 7006 dt.size = sregs->gdt.limit; 7007 dt.address = sregs->gdt.base; 7008 kvm_x86_ops->set_gdt(vcpu, &dt); 7009 7010 vcpu->arch.cr2 = sregs->cr2; 7011 mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; 7012 vcpu->arch.cr3 = sregs->cr3; 7013 __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail); 7014 7015 kvm_set_cr8(vcpu, sregs->cr8); 7016 7017 mmu_reset_needed |= vcpu->arch.efer != sregs->efer; 7018 kvm_x86_ops->set_efer(vcpu, sregs->efer); 7019 apic_base_msr.data = sregs->apic_base; 7020 apic_base_msr.host_initiated = true; 7021 kvm_set_apic_base(vcpu, &apic_base_msr); 7022 7023 mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; 7024 kvm_x86_ops->set_cr0(vcpu, sregs->cr0); 7025 vcpu->arch.cr0 = sregs->cr0; 7026 7027 mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; 7028 kvm_x86_ops->set_cr4(vcpu, sregs->cr4); 7029 if (sregs->cr4 & X86_CR4_OSXSAVE) 7030 kvm_update_cpuid(vcpu); 7031 7032 idx = srcu_read_lock(&vcpu->kvm->srcu); 7033 if (!is_long_mode(vcpu) && is_pae(vcpu)) { 7034 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); 7035 mmu_reset_needed = 1; 7036 } 7037 srcu_read_unlock(&vcpu->kvm->srcu, idx); 7038 7039 if (mmu_reset_needed) 7040 kvm_mmu_reset_context(vcpu); 7041 7042 max_bits = KVM_NR_INTERRUPTS; 7043 pending_vec = find_first_bit( 7044 (const unsigned long *)sregs->interrupt_bitmap, max_bits); 7045 if (pending_vec < max_bits) { 7046 kvm_queue_interrupt(vcpu, pending_vec, false); 7047 pr_debug("Set back pending irq %d\n", pending_vec); 7048 } 7049 7050 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 7051 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 7052 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); 7053 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 7054 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 7055 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 7056 7057 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 7058 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 7059 7060 update_cr8_intercept(vcpu); 7061 7062 /* Older userspace won't unhalt the vcpu on reset. */ 7063 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && 7064 sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && 7065 !is_protmode(vcpu)) 7066 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 7067 7068 kvm_make_request(KVM_REQ_EVENT, vcpu); 7069 7070 return 0; 7071 } 7072 7073 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 7074 struct kvm_guest_debug *dbg) 7075 { 7076 unsigned long rflags; 7077 int i, r; 7078 7079 if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) { 7080 r = -EBUSY; 7081 if (vcpu->arch.exception.pending) 7082 goto out; 7083 if (dbg->control & KVM_GUESTDBG_INJECT_DB) 7084 kvm_queue_exception(vcpu, DB_VECTOR); 7085 else 7086 kvm_queue_exception(vcpu, BP_VECTOR); 7087 } 7088 7089 /* 7090 * Read rflags as long as potentially injected trace flags are still 7091 * filtered out. 7092 */ 7093 rflags = kvm_get_rflags(vcpu); 7094 7095 vcpu->guest_debug = dbg->control; 7096 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) 7097 vcpu->guest_debug = 0; 7098 7099 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 7100 for (i = 0; i < KVM_NR_DB_REGS; ++i) 7101 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; 7102 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7]; 7103 } else { 7104 for (i = 0; i < KVM_NR_DB_REGS; i++) 7105 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 7106 } 7107 kvm_update_dr7(vcpu); 7108 7109 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 7110 vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) + 7111 get_segment_base(vcpu, VCPU_SREG_CS); 7112 7113 /* 7114 * Trigger an rflags update that will inject or remove the trace 7115 * flags. 7116 */ 7117 kvm_set_rflags(vcpu, rflags); 7118 7119 kvm_x86_ops->update_bp_intercept(vcpu); 7120 7121 r = 0; 7122 7123 out: 7124 7125 return r; 7126 } 7127 7128 /* 7129 * Translate a guest virtual address to a guest physical address. 7130 */ 7131 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 7132 struct kvm_translation *tr) 7133 { 7134 unsigned long vaddr = tr->linear_address; 7135 gpa_t gpa; 7136 int idx; 7137 7138 idx = srcu_read_lock(&vcpu->kvm->srcu); 7139 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL); 7140 srcu_read_unlock(&vcpu->kvm->srcu, idx); 7141 tr->physical_address = gpa; 7142 tr->valid = gpa != UNMAPPED_GVA; 7143 tr->writeable = 1; 7144 tr->usermode = 0; 7145 7146 return 0; 7147 } 7148 7149 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 7150 { 7151 struct fxregs_state *fxsave = 7152 &vcpu->arch.guest_fpu.state.fxsave; 7153 7154 memcpy(fpu->fpr, fxsave->st_space, 128); 7155 fpu->fcw = fxsave->cwd; 7156 fpu->fsw = fxsave->swd; 7157 fpu->ftwx = fxsave->twd; 7158 fpu->last_opcode = fxsave->fop; 7159 fpu->last_ip = fxsave->rip; 7160 fpu->last_dp = fxsave->rdp; 7161 memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); 7162 7163 return 0; 7164 } 7165 7166 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 7167 { 7168 struct fxregs_state *fxsave = 7169 &vcpu->arch.guest_fpu.state.fxsave; 7170 7171 memcpy(fxsave->st_space, fpu->fpr, 128); 7172 fxsave->cwd = fpu->fcw; 7173 fxsave->swd = fpu->fsw; 7174 fxsave->twd = fpu->ftwx; 7175 fxsave->fop = fpu->last_opcode; 7176 fxsave->rip = fpu->last_ip; 7177 fxsave->rdp = fpu->last_dp; 7178 memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); 7179 7180 return 0; 7181 } 7182 7183 static void fx_init(struct kvm_vcpu *vcpu) 7184 { 7185 fpstate_init(&vcpu->arch.guest_fpu.state); 7186 if (cpu_has_xsaves) 7187 vcpu->arch.guest_fpu.state.xsave.header.xcomp_bv = 7188 host_xcr0 | XSTATE_COMPACTION_ENABLED; 7189 7190 /* 7191 * Ensure guest xcr0 is valid for loading 7192 */ 7193 vcpu->arch.xcr0 = XFEATURE_MASK_FP; 7194 7195 vcpu->arch.cr0 |= X86_CR0_ET; 7196 } 7197 7198 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) 7199 { 7200 if (vcpu->guest_fpu_loaded) 7201 return; 7202 7203 /* 7204 * Restore all possible states in the guest, 7205 * and assume host would use all available bits. 7206 * Guest xcr0 would be loaded later. 7207 */ 7208 kvm_put_guest_xcr0(vcpu); 7209 vcpu->guest_fpu_loaded = 1; 7210 __kernel_fpu_begin(); 7211 __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu.state); 7212 trace_kvm_fpu(1); 7213 } 7214 7215 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) 7216 { 7217 kvm_put_guest_xcr0(vcpu); 7218 7219 if (!vcpu->guest_fpu_loaded) { 7220 vcpu->fpu_counter = 0; 7221 return; 7222 } 7223 7224 vcpu->guest_fpu_loaded = 0; 7225 copy_fpregs_to_fpstate(&vcpu->arch.guest_fpu); 7226 __kernel_fpu_end(); 7227 ++vcpu->stat.fpu_reload; 7228 /* 7229 * If using eager FPU mode, or if the guest is a frequent user 7230 * of the FPU, just leave the FPU active for next time. 7231 * Every 255 times fpu_counter rolls over to 0; a guest that uses 7232 * the FPU in bursts will revert to loading it on demand. 7233 */ 7234 if (!vcpu->arch.eager_fpu) { 7235 if (++vcpu->fpu_counter < 5) 7236 kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu); 7237 } 7238 trace_kvm_fpu(0); 7239 } 7240 7241 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) 7242 { 7243 kvmclock_reset(vcpu); 7244 7245 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 7246 kvm_x86_ops->vcpu_free(vcpu); 7247 } 7248 7249 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 7250 unsigned int id) 7251 { 7252 struct kvm_vcpu *vcpu; 7253 7254 if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0) 7255 printk_once(KERN_WARNING 7256 "kvm: SMP vm created on host with unstable TSC; " 7257 "guest TSC will not be reliable\n"); 7258 7259 vcpu = kvm_x86_ops->vcpu_create(kvm, id); 7260 7261 return vcpu; 7262 } 7263 7264 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 7265 { 7266 int r; 7267 7268 kvm_vcpu_mtrr_init(vcpu); 7269 r = vcpu_load(vcpu); 7270 if (r) 7271 return r; 7272 kvm_vcpu_reset(vcpu, false); 7273 kvm_mmu_setup(vcpu); 7274 vcpu_put(vcpu); 7275 return r; 7276 } 7277 7278 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 7279 { 7280 struct msr_data msr; 7281 struct kvm *kvm = vcpu->kvm; 7282 7283 if (vcpu_load(vcpu)) 7284 return; 7285 msr.data = 0x0; 7286 msr.index = MSR_IA32_TSC; 7287 msr.host_initiated = true; 7288 kvm_write_tsc(vcpu, &msr); 7289 vcpu_put(vcpu); 7290 7291 if (!kvmclock_periodic_sync) 7292 return; 7293 7294 schedule_delayed_work(&kvm->arch.kvmclock_sync_work, 7295 KVMCLOCK_SYNC_PERIOD); 7296 } 7297 7298 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 7299 { 7300 int r; 7301 vcpu->arch.apf.msr_val = 0; 7302 7303 r = vcpu_load(vcpu); 7304 BUG_ON(r); 7305 kvm_mmu_unload(vcpu); 7306 vcpu_put(vcpu); 7307 7308 kvm_x86_ops->vcpu_free(vcpu); 7309 } 7310 7311 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 7312 { 7313 vcpu->arch.hflags = 0; 7314 7315 atomic_set(&vcpu->arch.nmi_queued, 0); 7316 vcpu->arch.nmi_pending = 0; 7317 vcpu->arch.nmi_injected = false; 7318 kvm_clear_interrupt_queue(vcpu); 7319 kvm_clear_exception_queue(vcpu); 7320 7321 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 7322 kvm_update_dr0123(vcpu); 7323 vcpu->arch.dr6 = DR6_INIT; 7324 kvm_update_dr6(vcpu); 7325 vcpu->arch.dr7 = DR7_FIXED_1; 7326 kvm_update_dr7(vcpu); 7327 7328 vcpu->arch.cr2 = 0; 7329 7330 kvm_make_request(KVM_REQ_EVENT, vcpu); 7331 vcpu->arch.apf.msr_val = 0; 7332 vcpu->arch.st.msr_val = 0; 7333 7334 kvmclock_reset(vcpu); 7335 7336 kvm_clear_async_pf_completion_queue(vcpu); 7337 kvm_async_pf_hash_reset(vcpu); 7338 vcpu->arch.apf.halted = false; 7339 7340 if (!init_event) { 7341 kvm_pmu_reset(vcpu); 7342 vcpu->arch.smbase = 0x30000; 7343 } 7344 7345 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); 7346 vcpu->arch.regs_avail = ~0; 7347 vcpu->arch.regs_dirty = ~0; 7348 7349 kvm_x86_ops->vcpu_reset(vcpu, init_event); 7350 } 7351 7352 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 7353 { 7354 struct kvm_segment cs; 7355 7356 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); 7357 cs.selector = vector << 8; 7358 cs.base = vector << 12; 7359 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS); 7360 kvm_rip_write(vcpu, 0); 7361 } 7362 7363 int kvm_arch_hardware_enable(void) 7364 { 7365 struct kvm *kvm; 7366 struct kvm_vcpu *vcpu; 7367 int i; 7368 int ret; 7369 u64 local_tsc; 7370 u64 max_tsc = 0; 7371 bool stable, backwards_tsc = false; 7372 7373 kvm_shared_msr_cpu_online(); 7374 ret = kvm_x86_ops->hardware_enable(); 7375 if (ret != 0) 7376 return ret; 7377 7378 local_tsc = rdtsc(); 7379 stable = !check_tsc_unstable(); 7380 list_for_each_entry(kvm, &vm_list, vm_list) { 7381 kvm_for_each_vcpu(i, vcpu, kvm) { 7382 if (!stable && vcpu->cpu == smp_processor_id()) 7383 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 7384 if (stable && vcpu->arch.last_host_tsc > local_tsc) { 7385 backwards_tsc = true; 7386 if (vcpu->arch.last_host_tsc > max_tsc) 7387 max_tsc = vcpu->arch.last_host_tsc; 7388 } 7389 } 7390 } 7391 7392 /* 7393 * Sometimes, even reliable TSCs go backwards. This happens on 7394 * platforms that reset TSC during suspend or hibernate actions, but 7395 * maintain synchronization. We must compensate. Fortunately, we can 7396 * detect that condition here, which happens early in CPU bringup, 7397 * before any KVM threads can be running. Unfortunately, we can't 7398 * bring the TSCs fully up to date with real time, as we aren't yet far 7399 * enough into CPU bringup that we know how much real time has actually 7400 * elapsed; our helper function, get_kernel_ns() will be using boot 7401 * variables that haven't been updated yet. 7402 * 7403 * So we simply find the maximum observed TSC above, then record the 7404 * adjustment to TSC in each VCPU. When the VCPU later gets loaded, 7405 * the adjustment will be applied. Note that we accumulate 7406 * adjustments, in case multiple suspend cycles happen before some VCPU 7407 * gets a chance to run again. In the event that no KVM threads get a 7408 * chance to run, we will miss the entire elapsed period, as we'll have 7409 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may 7410 * loose cycle time. This isn't too big a deal, since the loss will be 7411 * uniform across all VCPUs (not to mention the scenario is extremely 7412 * unlikely). It is possible that a second hibernate recovery happens 7413 * much faster than a first, causing the observed TSC here to be 7414 * smaller; this would require additional padding adjustment, which is 7415 * why we set last_host_tsc to the local tsc observed here. 7416 * 7417 * N.B. - this code below runs only on platforms with reliable TSC, 7418 * as that is the only way backwards_tsc is set above. Also note 7419 * that this runs for ALL vcpus, which is not a bug; all VCPUs should 7420 * have the same delta_cyc adjustment applied if backwards_tsc 7421 * is detected. Note further, this adjustment is only done once, 7422 * as we reset last_host_tsc on all VCPUs to stop this from being 7423 * called multiple times (one for each physical CPU bringup). 7424 * 7425 * Platforms with unreliable TSCs don't have to deal with this, they 7426 * will be compensated by the logic in vcpu_load, which sets the TSC to 7427 * catchup mode. This will catchup all VCPUs to real time, but cannot 7428 * guarantee that they stay in perfect synchronization. 7429 */ 7430 if (backwards_tsc) { 7431 u64 delta_cyc = max_tsc - local_tsc; 7432 backwards_tsc_observed = true; 7433 list_for_each_entry(kvm, &vm_list, vm_list) { 7434 kvm_for_each_vcpu(i, vcpu, kvm) { 7435 vcpu->arch.tsc_offset_adjustment += delta_cyc; 7436 vcpu->arch.last_host_tsc = local_tsc; 7437 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 7438 } 7439 7440 /* 7441 * We have to disable TSC offset matching.. if you were 7442 * booting a VM while issuing an S4 host suspend.... 7443 * you may have some problem. Solving this issue is 7444 * left as an exercise to the reader. 7445 */ 7446 kvm->arch.last_tsc_nsec = 0; 7447 kvm->arch.last_tsc_write = 0; 7448 } 7449 7450 } 7451 return 0; 7452 } 7453 7454 void kvm_arch_hardware_disable(void) 7455 { 7456 kvm_x86_ops->hardware_disable(); 7457 drop_user_return_notifiers(); 7458 } 7459 7460 int kvm_arch_hardware_setup(void) 7461 { 7462 int r; 7463 7464 r = kvm_x86_ops->hardware_setup(); 7465 if (r != 0) 7466 return r; 7467 7468 if (kvm_has_tsc_control) { 7469 /* 7470 * Make sure the user can only configure tsc_khz values that 7471 * fit into a signed integer. 7472 * A min value is not calculated needed because it will always 7473 * be 1 on all machines. 7474 */ 7475 u64 max = min(0x7fffffffULL, 7476 __scale_tsc(kvm_max_tsc_scaling_ratio, tsc_khz)); 7477 kvm_max_guest_tsc_khz = max; 7478 7479 kvm_default_tsc_scaling_ratio = 1ULL << kvm_tsc_scaling_ratio_frac_bits; 7480 } 7481 7482 kvm_init_msr_list(); 7483 return 0; 7484 } 7485 7486 void kvm_arch_hardware_unsetup(void) 7487 { 7488 kvm_x86_ops->hardware_unsetup(); 7489 } 7490 7491 void kvm_arch_check_processor_compat(void *rtn) 7492 { 7493 kvm_x86_ops->check_processor_compatibility(rtn); 7494 } 7495 7496 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu) 7497 { 7498 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id; 7499 } 7500 EXPORT_SYMBOL_GPL(kvm_vcpu_is_reset_bsp); 7501 7502 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) 7503 { 7504 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0; 7505 } 7506 7507 bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu) 7508 { 7509 return irqchip_in_kernel(vcpu->kvm) == lapic_in_kernel(vcpu); 7510 } 7511 7512 struct static_key kvm_no_apic_vcpu __read_mostly; 7513 7514 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 7515 { 7516 struct page *page; 7517 struct kvm *kvm; 7518 int r; 7519 7520 BUG_ON(vcpu->kvm == NULL); 7521 kvm = vcpu->kvm; 7522 7523 vcpu->arch.pv.pv_unhalted = false; 7524 vcpu->arch.emulate_ctxt.ops = &emulate_ops; 7525 if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_reset_bsp(vcpu)) 7526 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 7527 else 7528 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED; 7529 7530 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 7531 if (!page) { 7532 r = -ENOMEM; 7533 goto fail; 7534 } 7535 vcpu->arch.pio_data = page_address(page); 7536 7537 kvm_set_tsc_khz(vcpu, max_tsc_khz); 7538 7539 r = kvm_mmu_create(vcpu); 7540 if (r < 0) 7541 goto fail_free_pio_data; 7542 7543 if (irqchip_in_kernel(kvm)) { 7544 r = kvm_create_lapic(vcpu); 7545 if (r < 0) 7546 goto fail_mmu_destroy; 7547 } else 7548 static_key_slow_inc(&kvm_no_apic_vcpu); 7549 7550 vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4, 7551 GFP_KERNEL); 7552 if (!vcpu->arch.mce_banks) { 7553 r = -ENOMEM; 7554 goto fail_free_lapic; 7555 } 7556 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; 7557 7558 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, GFP_KERNEL)) { 7559 r = -ENOMEM; 7560 goto fail_free_mce_banks; 7561 } 7562 7563 fx_init(vcpu); 7564 7565 vcpu->arch.ia32_tsc_adjust_msr = 0x0; 7566 vcpu->arch.pv_time_enabled = false; 7567 7568 vcpu->arch.guest_supported_xcr0 = 0; 7569 vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; 7570 7571 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); 7572 7573 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT; 7574 7575 kvm_async_pf_hash_reset(vcpu); 7576 kvm_pmu_init(vcpu); 7577 7578 vcpu->arch.pending_external_vector = -1; 7579 7580 return 0; 7581 7582 fail_free_mce_banks: 7583 kfree(vcpu->arch.mce_banks); 7584 fail_free_lapic: 7585 kvm_free_lapic(vcpu); 7586 fail_mmu_destroy: 7587 kvm_mmu_destroy(vcpu); 7588 fail_free_pio_data: 7589 free_page((unsigned long)vcpu->arch.pio_data); 7590 fail: 7591 return r; 7592 } 7593 7594 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) 7595 { 7596 int idx; 7597 7598 kvm_pmu_destroy(vcpu); 7599 kfree(vcpu->arch.mce_banks); 7600 kvm_free_lapic(vcpu); 7601 idx = srcu_read_lock(&vcpu->kvm->srcu); 7602 kvm_mmu_destroy(vcpu); 7603 srcu_read_unlock(&vcpu->kvm->srcu, idx); 7604 free_page((unsigned long)vcpu->arch.pio_data); 7605 if (!lapic_in_kernel(vcpu)) 7606 static_key_slow_dec(&kvm_no_apic_vcpu); 7607 } 7608 7609 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) 7610 { 7611 kvm_x86_ops->sched_in(vcpu, cpu); 7612 } 7613 7614 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 7615 { 7616 if (type) 7617 return -EINVAL; 7618 7619 INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list); 7620 INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); 7621 INIT_LIST_HEAD(&kvm->arch.zapped_obsolete_pages); 7622 INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); 7623 atomic_set(&kvm->arch.noncoherent_dma_count, 0); 7624 7625 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ 7626 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); 7627 /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */ 7628 set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 7629 &kvm->arch.irq_sources_bitmap); 7630 7631 raw_spin_lock_init(&kvm->arch.tsc_write_lock); 7632 mutex_init(&kvm->arch.apic_map_lock); 7633 spin_lock_init(&kvm->arch.pvclock_gtod_sync_lock); 7634 7635 pvclock_update_vm_gtod_copy(kvm); 7636 7637 INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn); 7638 INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn); 7639 7640 return 0; 7641 } 7642 7643 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) 7644 { 7645 int r; 7646 r = vcpu_load(vcpu); 7647 BUG_ON(r); 7648 kvm_mmu_unload(vcpu); 7649 vcpu_put(vcpu); 7650 } 7651 7652 static void kvm_free_vcpus(struct kvm *kvm) 7653 { 7654 unsigned int i; 7655 struct kvm_vcpu *vcpu; 7656 7657 /* 7658 * Unpin any mmu pages first. 7659 */ 7660 kvm_for_each_vcpu(i, vcpu, kvm) { 7661 kvm_clear_async_pf_completion_queue(vcpu); 7662 kvm_unload_vcpu_mmu(vcpu); 7663 } 7664 kvm_for_each_vcpu(i, vcpu, kvm) 7665 kvm_arch_vcpu_free(vcpu); 7666 7667 mutex_lock(&kvm->lock); 7668 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 7669 kvm->vcpus[i] = NULL; 7670 7671 atomic_set(&kvm->online_vcpus, 0); 7672 mutex_unlock(&kvm->lock); 7673 } 7674 7675 void kvm_arch_sync_events(struct kvm *kvm) 7676 { 7677 cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work); 7678 cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work); 7679 kvm_free_all_assigned_devices(kvm); 7680 kvm_free_pit(kvm); 7681 } 7682 7683 int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size) 7684 { 7685 int i, r; 7686 unsigned long hva; 7687 struct kvm_memslots *slots = kvm_memslots(kvm); 7688 struct kvm_memory_slot *slot, old; 7689 7690 /* Called with kvm->slots_lock held. */ 7691 if (WARN_ON(id >= KVM_MEM_SLOTS_NUM)) 7692 return -EINVAL; 7693 7694 slot = id_to_memslot(slots, id); 7695 if (size) { 7696 if (WARN_ON(slot->npages)) 7697 return -EEXIST; 7698 7699 /* 7700 * MAP_SHARED to prevent internal slot pages from being moved 7701 * by fork()/COW. 7702 */ 7703 hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE, 7704 MAP_SHARED | MAP_ANONYMOUS, 0); 7705 if (IS_ERR((void *)hva)) 7706 return PTR_ERR((void *)hva); 7707 } else { 7708 if (!slot->npages) 7709 return 0; 7710 7711 hva = 0; 7712 } 7713 7714 old = *slot; 7715 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 7716 struct kvm_userspace_memory_region m; 7717 7718 m.slot = id | (i << 16); 7719 m.flags = 0; 7720 m.guest_phys_addr = gpa; 7721 m.userspace_addr = hva; 7722 m.memory_size = size; 7723 r = __kvm_set_memory_region(kvm, &m); 7724 if (r < 0) 7725 return r; 7726 } 7727 7728 if (!size) { 7729 r = vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE); 7730 WARN_ON(r < 0); 7731 } 7732 7733 return 0; 7734 } 7735 EXPORT_SYMBOL_GPL(__x86_set_memory_region); 7736 7737 int x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size) 7738 { 7739 int r; 7740 7741 mutex_lock(&kvm->slots_lock); 7742 r = __x86_set_memory_region(kvm, id, gpa, size); 7743 mutex_unlock(&kvm->slots_lock); 7744 7745 return r; 7746 } 7747 EXPORT_SYMBOL_GPL(x86_set_memory_region); 7748 7749 void kvm_arch_destroy_vm(struct kvm *kvm) 7750 { 7751 if (current->mm == kvm->mm) { 7752 /* 7753 * Free memory regions allocated on behalf of userspace, 7754 * unless the the memory map has changed due to process exit 7755 * or fd copying. 7756 */ 7757 x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 0, 0); 7758 x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 0, 0); 7759 x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0); 7760 } 7761 kvm_iommu_unmap_guest(kvm); 7762 kfree(kvm->arch.vpic); 7763 kfree(kvm->arch.vioapic); 7764 kvm_free_vcpus(kvm); 7765 kfree(rcu_dereference_check(kvm->arch.apic_map, 1)); 7766 } 7767 7768 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free, 7769 struct kvm_memory_slot *dont) 7770 { 7771 int i; 7772 7773 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 7774 if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) { 7775 kvfree(free->arch.rmap[i]); 7776 free->arch.rmap[i] = NULL; 7777 } 7778 if (i == 0) 7779 continue; 7780 7781 if (!dont || free->arch.lpage_info[i - 1] != 7782 dont->arch.lpage_info[i - 1]) { 7783 kvfree(free->arch.lpage_info[i - 1]); 7784 free->arch.lpage_info[i - 1] = NULL; 7785 } 7786 } 7787 } 7788 7789 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 7790 unsigned long npages) 7791 { 7792 int i; 7793 7794 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 7795 unsigned long ugfn; 7796 int lpages; 7797 int level = i + 1; 7798 7799 lpages = gfn_to_index(slot->base_gfn + npages - 1, 7800 slot->base_gfn, level) + 1; 7801 7802 slot->arch.rmap[i] = 7803 kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i])); 7804 if (!slot->arch.rmap[i]) 7805 goto out_free; 7806 if (i == 0) 7807 continue; 7808 7809 slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages * 7810 sizeof(*slot->arch.lpage_info[i - 1])); 7811 if (!slot->arch.lpage_info[i - 1]) 7812 goto out_free; 7813 7814 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) 7815 slot->arch.lpage_info[i - 1][0].write_count = 1; 7816 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) 7817 slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1; 7818 ugfn = slot->userspace_addr >> PAGE_SHIFT; 7819 /* 7820 * If the gfn and userspace address are not aligned wrt each 7821 * other, or if explicitly asked to, disable large page 7822 * support for this slot 7823 */ 7824 if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) || 7825 !kvm_largepages_enabled()) { 7826 unsigned long j; 7827 7828 for (j = 0; j < lpages; ++j) 7829 slot->arch.lpage_info[i - 1][j].write_count = 1; 7830 } 7831 } 7832 7833 return 0; 7834 7835 out_free: 7836 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 7837 kvfree(slot->arch.rmap[i]); 7838 slot->arch.rmap[i] = NULL; 7839 if (i == 0) 7840 continue; 7841 7842 kvfree(slot->arch.lpage_info[i - 1]); 7843 slot->arch.lpage_info[i - 1] = NULL; 7844 } 7845 return -ENOMEM; 7846 } 7847 7848 void kvm_arch_memslots_updated(struct kvm *kvm, struct kvm_memslots *slots) 7849 { 7850 /* 7851 * memslots->generation has been incremented. 7852 * mmio generation may have reached its maximum value. 7853 */ 7854 kvm_mmu_invalidate_mmio_sptes(kvm, slots); 7855 } 7856 7857 int kvm_arch_prepare_memory_region(struct kvm *kvm, 7858 struct kvm_memory_slot *memslot, 7859 const struct kvm_userspace_memory_region *mem, 7860 enum kvm_mr_change change) 7861 { 7862 return 0; 7863 } 7864 7865 static void kvm_mmu_slot_apply_flags(struct kvm *kvm, 7866 struct kvm_memory_slot *new) 7867 { 7868 /* Still write protect RO slot */ 7869 if (new->flags & KVM_MEM_READONLY) { 7870 kvm_mmu_slot_remove_write_access(kvm, new); 7871 return; 7872 } 7873 7874 /* 7875 * Call kvm_x86_ops dirty logging hooks when they are valid. 7876 * 7877 * kvm_x86_ops->slot_disable_log_dirty is called when: 7878 * 7879 * - KVM_MR_CREATE with dirty logging is disabled 7880 * - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag 7881 * 7882 * The reason is, in case of PML, we need to set D-bit for any slots 7883 * with dirty logging disabled in order to eliminate unnecessary GPA 7884 * logging in PML buffer (and potential PML buffer full VMEXT). This 7885 * guarantees leaving PML enabled during guest's lifetime won't have 7886 * any additonal overhead from PML when guest is running with dirty 7887 * logging disabled for memory slots. 7888 * 7889 * kvm_x86_ops->slot_enable_log_dirty is called when switching new slot 7890 * to dirty logging mode. 7891 * 7892 * If kvm_x86_ops dirty logging hooks are invalid, use write protect. 7893 * 7894 * In case of write protect: 7895 * 7896 * Write protect all pages for dirty logging. 7897 * 7898 * All the sptes including the large sptes which point to this 7899 * slot are set to readonly. We can not create any new large 7900 * spte on this slot until the end of the logging. 7901 * 7902 * See the comments in fast_page_fault(). 7903 */ 7904 if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { 7905 if (kvm_x86_ops->slot_enable_log_dirty) 7906 kvm_x86_ops->slot_enable_log_dirty(kvm, new); 7907 else 7908 kvm_mmu_slot_remove_write_access(kvm, new); 7909 } else { 7910 if (kvm_x86_ops->slot_disable_log_dirty) 7911 kvm_x86_ops->slot_disable_log_dirty(kvm, new); 7912 } 7913 } 7914 7915 void kvm_arch_commit_memory_region(struct kvm *kvm, 7916 const struct kvm_userspace_memory_region *mem, 7917 const struct kvm_memory_slot *old, 7918 const struct kvm_memory_slot *new, 7919 enum kvm_mr_change change) 7920 { 7921 int nr_mmu_pages = 0; 7922 7923 if (!kvm->arch.n_requested_mmu_pages) 7924 nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); 7925 7926 if (nr_mmu_pages) 7927 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); 7928 7929 /* 7930 * Dirty logging tracks sptes in 4k granularity, meaning that large 7931 * sptes have to be split. If live migration is successful, the guest 7932 * in the source machine will be destroyed and large sptes will be 7933 * created in the destination. However, if the guest continues to run 7934 * in the source machine (for example if live migration fails), small 7935 * sptes will remain around and cause bad performance. 7936 * 7937 * Scan sptes if dirty logging has been stopped, dropping those 7938 * which can be collapsed into a single large-page spte. Later 7939 * page faults will create the large-page sptes. 7940 */ 7941 if ((change != KVM_MR_DELETE) && 7942 (old->flags & KVM_MEM_LOG_DIRTY_PAGES) && 7943 !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) 7944 kvm_mmu_zap_collapsible_sptes(kvm, new); 7945 7946 /* 7947 * Set up write protection and/or dirty logging for the new slot. 7948 * 7949 * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have 7950 * been zapped so no dirty logging staff is needed for old slot. For 7951 * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the 7952 * new and it's also covered when dealing with the new slot. 7953 * 7954 * FIXME: const-ify all uses of struct kvm_memory_slot. 7955 */ 7956 if (change != KVM_MR_DELETE) 7957 kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new); 7958 } 7959 7960 void kvm_arch_flush_shadow_all(struct kvm *kvm) 7961 { 7962 kvm_mmu_invalidate_zap_all_pages(kvm); 7963 } 7964 7965 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 7966 struct kvm_memory_slot *slot) 7967 { 7968 kvm_mmu_invalidate_zap_all_pages(kvm); 7969 } 7970 7971 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) 7972 { 7973 if (!list_empty_careful(&vcpu->async_pf.done)) 7974 return true; 7975 7976 if (kvm_apic_has_events(vcpu)) 7977 return true; 7978 7979 if (vcpu->arch.pv.pv_unhalted) 7980 return true; 7981 7982 if (atomic_read(&vcpu->arch.nmi_queued)) 7983 return true; 7984 7985 if (test_bit(KVM_REQ_SMI, &vcpu->requests)) 7986 return true; 7987 7988 if (kvm_arch_interrupt_allowed(vcpu) && 7989 kvm_cpu_has_interrupt(vcpu)) 7990 return true; 7991 7992 return false; 7993 } 7994 7995 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 7996 { 7997 if (is_guest_mode(vcpu) && kvm_x86_ops->check_nested_events) 7998 kvm_x86_ops->check_nested_events(vcpu, false); 7999 8000 return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu); 8001 } 8002 8003 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 8004 { 8005 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 8006 } 8007 8008 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) 8009 { 8010 return kvm_x86_ops->interrupt_allowed(vcpu); 8011 } 8012 8013 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) 8014 { 8015 if (is_64_bit_mode(vcpu)) 8016 return kvm_rip_read(vcpu); 8017 return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) + 8018 kvm_rip_read(vcpu)); 8019 } 8020 EXPORT_SYMBOL_GPL(kvm_get_linear_rip); 8021 8022 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) 8023 { 8024 return kvm_get_linear_rip(vcpu) == linear_rip; 8025 } 8026 EXPORT_SYMBOL_GPL(kvm_is_linear_rip); 8027 8028 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) 8029 { 8030 unsigned long rflags; 8031 8032 rflags = kvm_x86_ops->get_rflags(vcpu); 8033 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 8034 rflags &= ~X86_EFLAGS_TF; 8035 return rflags; 8036 } 8037 EXPORT_SYMBOL_GPL(kvm_get_rflags); 8038 8039 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 8040 { 8041 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && 8042 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) 8043 rflags |= X86_EFLAGS_TF; 8044 kvm_x86_ops->set_rflags(vcpu, rflags); 8045 } 8046 8047 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 8048 { 8049 __kvm_set_rflags(vcpu, rflags); 8050 kvm_make_request(KVM_REQ_EVENT, vcpu); 8051 } 8052 EXPORT_SYMBOL_GPL(kvm_set_rflags); 8053 8054 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) 8055 { 8056 int r; 8057 8058 if ((vcpu->arch.mmu.direct_map != work->arch.direct_map) || 8059 work->wakeup_all) 8060 return; 8061 8062 r = kvm_mmu_reload(vcpu); 8063 if (unlikely(r)) 8064 return; 8065 8066 if (!vcpu->arch.mmu.direct_map && 8067 work->arch.cr3 != vcpu->arch.mmu.get_cr3(vcpu)) 8068 return; 8069 8070 vcpu->arch.mmu.page_fault(vcpu, work->gva, 0, true); 8071 } 8072 8073 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) 8074 { 8075 return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); 8076 } 8077 8078 static inline u32 kvm_async_pf_next_probe(u32 key) 8079 { 8080 return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1); 8081 } 8082 8083 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 8084 { 8085 u32 key = kvm_async_pf_hash_fn(gfn); 8086 8087 while (vcpu->arch.apf.gfns[key] != ~0) 8088 key = kvm_async_pf_next_probe(key); 8089 8090 vcpu->arch.apf.gfns[key] = gfn; 8091 } 8092 8093 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) 8094 { 8095 int i; 8096 u32 key = kvm_async_pf_hash_fn(gfn); 8097 8098 for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) && 8099 (vcpu->arch.apf.gfns[key] != gfn && 8100 vcpu->arch.apf.gfns[key] != ~0); i++) 8101 key = kvm_async_pf_next_probe(key); 8102 8103 return key; 8104 } 8105 8106 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 8107 { 8108 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; 8109 } 8110 8111 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 8112 { 8113 u32 i, j, k; 8114 8115 i = j = kvm_async_pf_gfn_slot(vcpu, gfn); 8116 while (true) { 8117 vcpu->arch.apf.gfns[i] = ~0; 8118 do { 8119 j = kvm_async_pf_next_probe(j); 8120 if (vcpu->arch.apf.gfns[j] == ~0) 8121 return; 8122 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); 8123 /* 8124 * k lies cyclically in ]i,j] 8125 * | i.k.j | 8126 * |....j i.k.| or |.k..j i...| 8127 */ 8128 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j)); 8129 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; 8130 i = j; 8131 } 8132 } 8133 8134 static int apf_put_user(struct kvm_vcpu *vcpu, u32 val) 8135 { 8136 8137 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val, 8138 sizeof(val)); 8139 } 8140 8141 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 8142 struct kvm_async_pf *work) 8143 { 8144 struct x86_exception fault; 8145 8146 trace_kvm_async_pf_not_present(work->arch.token, work->gva); 8147 kvm_add_async_pf_gfn(vcpu, work->arch.gfn); 8148 8149 if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) || 8150 (vcpu->arch.apf.send_user_only && 8151 kvm_x86_ops->get_cpl(vcpu) == 0)) 8152 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 8153 else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) { 8154 fault.vector = PF_VECTOR; 8155 fault.error_code_valid = true; 8156 fault.error_code = 0; 8157 fault.nested_page_fault = false; 8158 fault.address = work->arch.token; 8159 kvm_inject_page_fault(vcpu, &fault); 8160 } 8161 } 8162 8163 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 8164 struct kvm_async_pf *work) 8165 { 8166 struct x86_exception fault; 8167 8168 trace_kvm_async_pf_ready(work->arch.token, work->gva); 8169 if (work->wakeup_all) 8170 work->arch.token = ~0; /* broadcast wakeup */ 8171 else 8172 kvm_del_async_pf_gfn(vcpu, work->arch.gfn); 8173 8174 if ((vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) && 8175 !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) { 8176 fault.vector = PF_VECTOR; 8177 fault.error_code_valid = true; 8178 fault.error_code = 0; 8179 fault.nested_page_fault = false; 8180 fault.address = work->arch.token; 8181 kvm_inject_page_fault(vcpu, &fault); 8182 } 8183 vcpu->arch.apf.halted = false; 8184 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 8185 } 8186 8187 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 8188 { 8189 if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED)) 8190 return true; 8191 else 8192 return !kvm_event_needs_reinjection(vcpu) && 8193 kvm_x86_ops->interrupt_allowed(vcpu); 8194 } 8195 8196 void kvm_arch_start_assignment(struct kvm *kvm) 8197 { 8198 atomic_inc(&kvm->arch.assigned_device_count); 8199 } 8200 EXPORT_SYMBOL_GPL(kvm_arch_start_assignment); 8201 8202 void kvm_arch_end_assignment(struct kvm *kvm) 8203 { 8204 atomic_dec(&kvm->arch.assigned_device_count); 8205 } 8206 EXPORT_SYMBOL_GPL(kvm_arch_end_assignment); 8207 8208 bool kvm_arch_has_assigned_device(struct kvm *kvm) 8209 { 8210 return atomic_read(&kvm->arch.assigned_device_count); 8211 } 8212 EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device); 8213 8214 void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 8215 { 8216 atomic_inc(&kvm->arch.noncoherent_dma_count); 8217 } 8218 EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma); 8219 8220 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 8221 { 8222 atomic_dec(&kvm->arch.noncoherent_dma_count); 8223 } 8224 EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma); 8225 8226 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 8227 { 8228 return atomic_read(&kvm->arch.noncoherent_dma_count); 8229 } 8230 EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma); 8231 8232 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 8233 struct irq_bypass_producer *prod) 8234 { 8235 struct kvm_kernel_irqfd *irqfd = 8236 container_of(cons, struct kvm_kernel_irqfd, consumer); 8237 8238 if (kvm_x86_ops->update_pi_irte) { 8239 irqfd->producer = prod; 8240 return kvm_x86_ops->update_pi_irte(irqfd->kvm, 8241 prod->irq, irqfd->gsi, 1); 8242 } 8243 8244 return -EINVAL; 8245 } 8246 8247 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 8248 struct irq_bypass_producer *prod) 8249 { 8250 int ret; 8251 struct kvm_kernel_irqfd *irqfd = 8252 container_of(cons, struct kvm_kernel_irqfd, consumer); 8253 8254 if (!kvm_x86_ops->update_pi_irte) { 8255 WARN_ON(irqfd->producer != NULL); 8256 return; 8257 } 8258 8259 WARN_ON(irqfd->producer != prod); 8260 irqfd->producer = NULL; 8261 8262 /* 8263 * When producer of consumer is unregistered, we change back to 8264 * remapped mode, so we can re-use the current implementation 8265 * when the irq is masked/disabed or the consumer side (KVM 8266 * int this case doesn't want to receive the interrupts. 8267 */ 8268 ret = kvm_x86_ops->update_pi_irte(irqfd->kvm, prod->irq, irqfd->gsi, 0); 8269 if (ret) 8270 printk(KERN_INFO "irq bypass consumer (token %p) unregistration" 8271 " fails: %d\n", irqfd->consumer.token, ret); 8272 } 8273 8274 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, 8275 uint32_t guest_irq, bool set) 8276 { 8277 if (!kvm_x86_ops->update_pi_irte) 8278 return -EINVAL; 8279 8280 return kvm_x86_ops->update_pi_irte(kvm, host_irq, guest_irq, set); 8281 } 8282 8283 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); 8284 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio); 8285 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); 8286 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); 8287 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); 8288 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); 8289 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun); 8290 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit); 8291 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject); 8292 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit); 8293 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga); 8294 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit); 8295 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts); 8296 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset); 8297 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window); 8298 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full); 8299 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update); 8300