1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Kernel-based Virtual Machine driver for Linux 4 * 5 * derived from drivers/kvm/kvm_main.c 6 * 7 * Copyright (C) 2006 Qumranet, Inc. 8 * Copyright (C) 2008 Qumranet, Inc. 9 * Copyright IBM Corporation, 2008 10 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 11 * 12 * Authors: 13 * Avi Kivity <avi@qumranet.com> 14 * Yaniv Kamay <yaniv@qumranet.com> 15 * Amit Shah <amit.shah@qumranet.com> 16 * Ben-Ami Yassour <benami@il.ibm.com> 17 */ 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/kvm_host.h> 21 #include "irq.h" 22 #include "ioapic.h" 23 #include "mmu.h" 24 #include "i8254.h" 25 #include "tss.h" 26 #include "kvm_cache_regs.h" 27 #include "kvm_emulate.h" 28 #include "x86.h" 29 #include "cpuid.h" 30 #include "pmu.h" 31 #include "hyperv.h" 32 #include "lapic.h" 33 #include "xen.h" 34 #include "smm.h" 35 36 #include <linux/clocksource.h> 37 #include <linux/interrupt.h> 38 #include <linux/kvm.h> 39 #include <linux/fs.h> 40 #include <linux/vmalloc.h> 41 #include <linux/export.h> 42 #include <linux/moduleparam.h> 43 #include <linux/mman.h> 44 #include <linux/highmem.h> 45 #include <linux/iommu.h> 46 #include <linux/cpufreq.h> 47 #include <linux/user-return-notifier.h> 48 #include <linux/srcu.h> 49 #include <linux/slab.h> 50 #include <linux/perf_event.h> 51 #include <linux/uaccess.h> 52 #include <linux/hash.h> 53 #include <linux/pci.h> 54 #include <linux/timekeeper_internal.h> 55 #include <linux/pvclock_gtod.h> 56 #include <linux/kvm_irqfd.h> 57 #include <linux/irqbypass.h> 58 #include <linux/sched/stat.h> 59 #include <linux/sched/isolation.h> 60 #include <linux/mem_encrypt.h> 61 #include <linux/entry-kvm.h> 62 #include <linux/suspend.h> 63 64 #include <trace/events/kvm.h> 65 66 #include <asm/debugreg.h> 67 #include <asm/msr.h> 68 #include <asm/desc.h> 69 #include <asm/mce.h> 70 #include <asm/pkru.h> 71 #include <linux/kernel_stat.h> 72 #include <asm/fpu/api.h> 73 #include <asm/fpu/xcr.h> 74 #include <asm/fpu/xstate.h> 75 #include <asm/pvclock.h> 76 #include <asm/div64.h> 77 #include <asm/irq_remapping.h> 78 #include <asm/mshyperv.h> 79 #include <asm/hypervisor.h> 80 #include <asm/tlbflush.h> 81 #include <asm/intel_pt.h> 82 #include <asm/emulate_prefix.h> 83 #include <asm/sgx.h> 84 #include <clocksource/hyperv_timer.h> 85 86 #define CREATE_TRACE_POINTS 87 #include "trace.h" 88 89 #define MAX_IO_MSRS 256 90 #define KVM_MAX_MCE_BANKS 32 91 92 struct kvm_caps kvm_caps __read_mostly = { 93 .supported_mce_cap = MCG_CTL_P | MCG_SER_P, 94 }; 95 EXPORT_SYMBOL_GPL(kvm_caps); 96 97 #define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e)) 98 99 #define emul_to_vcpu(ctxt) \ 100 ((struct kvm_vcpu *)(ctxt)->vcpu) 101 102 /* EFER defaults: 103 * - enable syscall per default because its emulated by KVM 104 * - enable LME and LMA per default on 64 bit KVM 105 */ 106 #ifdef CONFIG_X86_64 107 static 108 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA)); 109 #else 110 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); 111 #endif 112 113 static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS; 114 115 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) 116 117 #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE 118 119 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ 120 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) 121 122 static void update_cr8_intercept(struct kvm_vcpu *vcpu); 123 static void process_nmi(struct kvm_vcpu *vcpu); 124 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); 125 static void store_regs(struct kvm_vcpu *vcpu); 126 static int sync_regs(struct kvm_vcpu *vcpu); 127 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu); 128 129 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); 130 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); 131 132 static DEFINE_MUTEX(vendor_module_lock); 133 struct kvm_x86_ops kvm_x86_ops __read_mostly; 134 135 #define KVM_X86_OP(func) \ 136 DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ 137 *(((struct kvm_x86_ops *)0)->func)); 138 #define KVM_X86_OP_OPTIONAL KVM_X86_OP 139 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP 140 #include <asm/kvm-x86-ops.h> 141 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits); 142 EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg); 143 144 static bool __read_mostly ignore_msrs = 0; 145 module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR); 146 147 bool __read_mostly report_ignored_msrs = true; 148 module_param(report_ignored_msrs, bool, S_IRUGO | S_IWUSR); 149 EXPORT_SYMBOL_GPL(report_ignored_msrs); 150 151 unsigned int min_timer_period_us = 200; 152 module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR); 153 154 static bool __read_mostly kvmclock_periodic_sync = true; 155 module_param(kvmclock_periodic_sync, bool, S_IRUGO); 156 157 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */ 158 static u32 __read_mostly tsc_tolerance_ppm = 250; 159 module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR); 160 161 /* 162 * lapic timer advance (tscdeadline mode only) in nanoseconds. '-1' enables 163 * adaptive tuning starting from default advancement of 1000ns. '0' disables 164 * advancement entirely. Any other value is used as-is and disables adaptive 165 * tuning, i.e. allows privileged userspace to set an exact advancement time. 166 */ 167 static int __read_mostly lapic_timer_advance_ns = -1; 168 module_param(lapic_timer_advance_ns, int, S_IRUGO | S_IWUSR); 169 170 static bool __read_mostly vector_hashing = true; 171 module_param(vector_hashing, bool, S_IRUGO); 172 173 bool __read_mostly enable_vmware_backdoor = false; 174 module_param(enable_vmware_backdoor, bool, S_IRUGO); 175 EXPORT_SYMBOL_GPL(enable_vmware_backdoor); 176 177 /* 178 * Flags to manipulate forced emulation behavior (any non-zero value will 179 * enable forced emulation). 180 */ 181 #define KVM_FEP_CLEAR_RFLAGS_RF BIT(1) 182 static int __read_mostly force_emulation_prefix; 183 module_param(force_emulation_prefix, int, 0644); 184 185 int __read_mostly pi_inject_timer = -1; 186 module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR); 187 188 /* Enable/disable PMU virtualization */ 189 bool __read_mostly enable_pmu = true; 190 EXPORT_SYMBOL_GPL(enable_pmu); 191 module_param(enable_pmu, bool, 0444); 192 193 bool __read_mostly eager_page_split = true; 194 module_param(eager_page_split, bool, 0644); 195 196 /* 197 * Restoring the host value for MSRs that are only consumed when running in 198 * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU 199 * returns to userspace, i.e. the kernel can run with the guest's value. 200 */ 201 #define KVM_MAX_NR_USER_RETURN_MSRS 16 202 203 struct kvm_user_return_msrs { 204 struct user_return_notifier urn; 205 bool registered; 206 struct kvm_user_return_msr_values { 207 u64 host; 208 u64 curr; 209 } values[KVM_MAX_NR_USER_RETURN_MSRS]; 210 }; 211 212 u32 __read_mostly kvm_nr_uret_msrs; 213 EXPORT_SYMBOL_GPL(kvm_nr_uret_msrs); 214 static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS]; 215 static struct kvm_user_return_msrs __percpu *user_return_msrs; 216 217 #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ 218 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ 219 | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ 220 | XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE) 221 222 u64 __read_mostly host_efer; 223 EXPORT_SYMBOL_GPL(host_efer); 224 225 bool __read_mostly allow_smaller_maxphyaddr = 0; 226 EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr); 227 228 bool __read_mostly enable_apicv = true; 229 EXPORT_SYMBOL_GPL(enable_apicv); 230 231 u64 __read_mostly host_xss; 232 EXPORT_SYMBOL_GPL(host_xss); 233 234 const struct _kvm_stats_desc kvm_vm_stats_desc[] = { 235 KVM_GENERIC_VM_STATS(), 236 STATS_DESC_COUNTER(VM, mmu_shadow_zapped), 237 STATS_DESC_COUNTER(VM, mmu_pte_write), 238 STATS_DESC_COUNTER(VM, mmu_pde_zapped), 239 STATS_DESC_COUNTER(VM, mmu_flooded), 240 STATS_DESC_COUNTER(VM, mmu_recycled), 241 STATS_DESC_COUNTER(VM, mmu_cache_miss), 242 STATS_DESC_ICOUNTER(VM, mmu_unsync), 243 STATS_DESC_ICOUNTER(VM, pages_4k), 244 STATS_DESC_ICOUNTER(VM, pages_2m), 245 STATS_DESC_ICOUNTER(VM, pages_1g), 246 STATS_DESC_ICOUNTER(VM, nx_lpage_splits), 247 STATS_DESC_PCOUNTER(VM, max_mmu_rmap_size), 248 STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions) 249 }; 250 251 const struct kvm_stats_header kvm_vm_stats_header = { 252 .name_size = KVM_STATS_NAME_SIZE, 253 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), 254 .id_offset = sizeof(struct kvm_stats_header), 255 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 256 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 257 sizeof(kvm_vm_stats_desc), 258 }; 259 260 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 261 KVM_GENERIC_VCPU_STATS(), 262 STATS_DESC_COUNTER(VCPU, pf_taken), 263 STATS_DESC_COUNTER(VCPU, pf_fixed), 264 STATS_DESC_COUNTER(VCPU, pf_emulate), 265 STATS_DESC_COUNTER(VCPU, pf_spurious), 266 STATS_DESC_COUNTER(VCPU, pf_fast), 267 STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created), 268 STATS_DESC_COUNTER(VCPU, pf_guest), 269 STATS_DESC_COUNTER(VCPU, tlb_flush), 270 STATS_DESC_COUNTER(VCPU, invlpg), 271 STATS_DESC_COUNTER(VCPU, exits), 272 STATS_DESC_COUNTER(VCPU, io_exits), 273 STATS_DESC_COUNTER(VCPU, mmio_exits), 274 STATS_DESC_COUNTER(VCPU, signal_exits), 275 STATS_DESC_COUNTER(VCPU, irq_window_exits), 276 STATS_DESC_COUNTER(VCPU, nmi_window_exits), 277 STATS_DESC_COUNTER(VCPU, l1d_flush), 278 STATS_DESC_COUNTER(VCPU, halt_exits), 279 STATS_DESC_COUNTER(VCPU, request_irq_exits), 280 STATS_DESC_COUNTER(VCPU, irq_exits), 281 STATS_DESC_COUNTER(VCPU, host_state_reload), 282 STATS_DESC_COUNTER(VCPU, fpu_reload), 283 STATS_DESC_COUNTER(VCPU, insn_emulation), 284 STATS_DESC_COUNTER(VCPU, insn_emulation_fail), 285 STATS_DESC_COUNTER(VCPU, hypercalls), 286 STATS_DESC_COUNTER(VCPU, irq_injections), 287 STATS_DESC_COUNTER(VCPU, nmi_injections), 288 STATS_DESC_COUNTER(VCPU, req_event), 289 STATS_DESC_COUNTER(VCPU, nested_run), 290 STATS_DESC_COUNTER(VCPU, directed_yield_attempted), 291 STATS_DESC_COUNTER(VCPU, directed_yield_successful), 292 STATS_DESC_COUNTER(VCPU, preemption_reported), 293 STATS_DESC_COUNTER(VCPU, preemption_other), 294 STATS_DESC_IBOOLEAN(VCPU, guest_mode), 295 STATS_DESC_COUNTER(VCPU, notify_window_exits), 296 }; 297 298 const struct kvm_stats_header kvm_vcpu_stats_header = { 299 .name_size = KVM_STATS_NAME_SIZE, 300 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 301 .id_offset = sizeof(struct kvm_stats_header), 302 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 303 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 304 sizeof(kvm_vcpu_stats_desc), 305 }; 306 307 u64 __read_mostly host_xcr0; 308 309 static struct kmem_cache *x86_emulator_cache; 310 311 /* 312 * When called, it means the previous get/set msr reached an invalid msr. 313 * Return true if we want to ignore/silent this failed msr access. 314 */ 315 static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write) 316 { 317 const char *op = write ? "wrmsr" : "rdmsr"; 318 319 if (ignore_msrs) { 320 if (report_ignored_msrs) 321 kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n", 322 op, msr, data); 323 /* Mask the error */ 324 return true; 325 } else { 326 kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n", 327 op, msr, data); 328 return false; 329 } 330 } 331 332 static struct kmem_cache *kvm_alloc_emulator_cache(void) 333 { 334 unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src); 335 unsigned int size = sizeof(struct x86_emulate_ctxt); 336 337 return kmem_cache_create_usercopy("x86_emulator", size, 338 __alignof__(struct x86_emulate_ctxt), 339 SLAB_ACCOUNT, useroffset, 340 size - useroffset, NULL); 341 } 342 343 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); 344 345 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) 346 { 347 int i; 348 for (i = 0; i < ASYNC_PF_PER_VCPU; i++) 349 vcpu->arch.apf.gfns[i] = ~0; 350 } 351 352 static void kvm_on_user_return(struct user_return_notifier *urn) 353 { 354 unsigned slot; 355 struct kvm_user_return_msrs *msrs 356 = container_of(urn, struct kvm_user_return_msrs, urn); 357 struct kvm_user_return_msr_values *values; 358 unsigned long flags; 359 360 /* 361 * Disabling irqs at this point since the following code could be 362 * interrupted and executed through kvm_arch_hardware_disable() 363 */ 364 local_irq_save(flags); 365 if (msrs->registered) { 366 msrs->registered = false; 367 user_return_notifier_unregister(urn); 368 } 369 local_irq_restore(flags); 370 for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) { 371 values = &msrs->values[slot]; 372 if (values->host != values->curr) { 373 wrmsrl(kvm_uret_msrs_list[slot], values->host); 374 values->curr = values->host; 375 } 376 } 377 } 378 379 static int kvm_probe_user_return_msr(u32 msr) 380 { 381 u64 val; 382 int ret; 383 384 preempt_disable(); 385 ret = rdmsrl_safe(msr, &val); 386 if (ret) 387 goto out; 388 ret = wrmsrl_safe(msr, val); 389 out: 390 preempt_enable(); 391 return ret; 392 } 393 394 int kvm_add_user_return_msr(u32 msr) 395 { 396 BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS); 397 398 if (kvm_probe_user_return_msr(msr)) 399 return -1; 400 401 kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr; 402 return kvm_nr_uret_msrs++; 403 } 404 EXPORT_SYMBOL_GPL(kvm_add_user_return_msr); 405 406 int kvm_find_user_return_msr(u32 msr) 407 { 408 int i; 409 410 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 411 if (kvm_uret_msrs_list[i] == msr) 412 return i; 413 } 414 return -1; 415 } 416 EXPORT_SYMBOL_GPL(kvm_find_user_return_msr); 417 418 static void kvm_user_return_msr_cpu_online(void) 419 { 420 unsigned int cpu = smp_processor_id(); 421 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu); 422 u64 value; 423 int i; 424 425 for (i = 0; i < kvm_nr_uret_msrs; ++i) { 426 rdmsrl_safe(kvm_uret_msrs_list[i], &value); 427 msrs->values[i].host = value; 428 msrs->values[i].curr = value; 429 } 430 } 431 432 int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) 433 { 434 unsigned int cpu = smp_processor_id(); 435 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu); 436 int err; 437 438 value = (value & mask) | (msrs->values[slot].host & ~mask); 439 if (value == msrs->values[slot].curr) 440 return 0; 441 err = wrmsrl_safe(kvm_uret_msrs_list[slot], value); 442 if (err) 443 return 1; 444 445 msrs->values[slot].curr = value; 446 if (!msrs->registered) { 447 msrs->urn.on_user_return = kvm_on_user_return; 448 user_return_notifier_register(&msrs->urn); 449 msrs->registered = true; 450 } 451 return 0; 452 } 453 EXPORT_SYMBOL_GPL(kvm_set_user_return_msr); 454 455 static void drop_user_return_notifiers(void) 456 { 457 unsigned int cpu = smp_processor_id(); 458 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu); 459 460 if (msrs->registered) 461 kvm_on_user_return(&msrs->urn); 462 } 463 464 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) 465 { 466 return vcpu->arch.apic_base; 467 } 468 469 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu) 470 { 471 return kvm_apic_mode(kvm_get_apic_base(vcpu)); 472 } 473 EXPORT_SYMBOL_GPL(kvm_get_apic_mode); 474 475 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 476 { 477 enum lapic_mode old_mode = kvm_get_apic_mode(vcpu); 478 enum lapic_mode new_mode = kvm_apic_mode(msr_info->data); 479 u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff | 480 (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE); 481 482 if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID) 483 return 1; 484 if (!msr_info->host_initiated) { 485 if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC) 486 return 1; 487 if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC) 488 return 1; 489 } 490 491 kvm_lapic_set_base(vcpu, msr_info->data); 492 kvm_recalculate_apic_map(vcpu->kvm); 493 return 0; 494 } 495 496 /* 497 * Handle a fault on a hardware virtualization (VMX or SVM) instruction. 498 * 499 * Hardware virtualization extension instructions may fault if a reboot turns 500 * off virtualization while processes are running. Usually after catching the 501 * fault we just panic; during reboot instead the instruction is ignored. 502 */ 503 noinstr void kvm_spurious_fault(void) 504 { 505 /* Fault while not rebooting. We want the trace. */ 506 BUG_ON(!kvm_rebooting); 507 } 508 EXPORT_SYMBOL_GPL(kvm_spurious_fault); 509 510 #define EXCPT_BENIGN 0 511 #define EXCPT_CONTRIBUTORY 1 512 #define EXCPT_PF 2 513 514 static int exception_class(int vector) 515 { 516 switch (vector) { 517 case PF_VECTOR: 518 return EXCPT_PF; 519 case DE_VECTOR: 520 case TS_VECTOR: 521 case NP_VECTOR: 522 case SS_VECTOR: 523 case GP_VECTOR: 524 return EXCPT_CONTRIBUTORY; 525 default: 526 break; 527 } 528 return EXCPT_BENIGN; 529 } 530 531 #define EXCPT_FAULT 0 532 #define EXCPT_TRAP 1 533 #define EXCPT_ABORT 2 534 #define EXCPT_INTERRUPT 3 535 #define EXCPT_DB 4 536 537 static int exception_type(int vector) 538 { 539 unsigned int mask; 540 541 if (WARN_ON(vector > 31 || vector == NMI_VECTOR)) 542 return EXCPT_INTERRUPT; 543 544 mask = 1 << vector; 545 546 /* 547 * #DBs can be trap-like or fault-like, the caller must check other CPU 548 * state, e.g. DR6, to determine whether a #DB is a trap or fault. 549 */ 550 if (mask & (1 << DB_VECTOR)) 551 return EXCPT_DB; 552 553 if (mask & ((1 << BP_VECTOR) | (1 << OF_VECTOR))) 554 return EXCPT_TRAP; 555 556 if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR))) 557 return EXCPT_ABORT; 558 559 /* Reserved exceptions will result in fault */ 560 return EXCPT_FAULT; 561 } 562 563 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, 564 struct kvm_queued_exception *ex) 565 { 566 if (!ex->has_payload) 567 return; 568 569 switch (ex->vector) { 570 case DB_VECTOR: 571 /* 572 * "Certain debug exceptions may clear bit 0-3. The 573 * remaining contents of the DR6 register are never 574 * cleared by the processor". 575 */ 576 vcpu->arch.dr6 &= ~DR_TRAP_BITS; 577 /* 578 * In order to reflect the #DB exception payload in guest 579 * dr6, three components need to be considered: active low 580 * bit, FIXED_1 bits and active high bits (e.g. DR6_BD, 581 * DR6_BS and DR6_BT) 582 * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits. 583 * In the target guest dr6: 584 * FIXED_1 bits should always be set. 585 * Active low bits should be cleared if 1-setting in payload. 586 * Active high bits should be set if 1-setting in payload. 587 * 588 * Note, the payload is compatible with the pending debug 589 * exceptions/exit qualification under VMX, that active_low bits 590 * are active high in payload. 591 * So they need to be flipped for DR6. 592 */ 593 vcpu->arch.dr6 |= DR6_ACTIVE_LOW; 594 vcpu->arch.dr6 |= ex->payload; 595 vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; 596 597 /* 598 * The #DB payload is defined as compatible with the 'pending 599 * debug exceptions' field under VMX, not DR6. While bit 12 is 600 * defined in the 'pending debug exceptions' field (enabled 601 * breakpoint), it is reserved and must be zero in DR6. 602 */ 603 vcpu->arch.dr6 &= ~BIT(12); 604 break; 605 case PF_VECTOR: 606 vcpu->arch.cr2 = ex->payload; 607 break; 608 } 609 610 ex->has_payload = false; 611 ex->payload = 0; 612 } 613 EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload); 614 615 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector, 616 bool has_error_code, u32 error_code, 617 bool has_payload, unsigned long payload) 618 { 619 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 620 621 ex->vector = vector; 622 ex->injected = false; 623 ex->pending = true; 624 ex->has_error_code = has_error_code; 625 ex->error_code = error_code; 626 ex->has_payload = has_payload; 627 ex->payload = payload; 628 } 629 630 /* Forcibly leave the nested mode in cases like a vCPU reset */ 631 static void kvm_leave_nested(struct kvm_vcpu *vcpu) 632 { 633 kvm_x86_ops.nested_ops->leave_nested(vcpu); 634 } 635 636 static void kvm_multiple_exception(struct kvm_vcpu *vcpu, 637 unsigned nr, bool has_error, u32 error_code, 638 bool has_payload, unsigned long payload, bool reinject) 639 { 640 u32 prev_nr; 641 int class1, class2; 642 643 kvm_make_request(KVM_REQ_EVENT, vcpu); 644 645 /* 646 * If the exception is destined for L2 and isn't being reinjected, 647 * morph it to a VM-Exit if L1 wants to intercept the exception. A 648 * previously injected exception is not checked because it was checked 649 * when it was original queued, and re-checking is incorrect if _L1_ 650 * injected the exception, in which case it's exempt from interception. 651 */ 652 if (!reinject && is_guest_mode(vcpu) && 653 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) { 654 kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code, 655 has_payload, payload); 656 return; 657 } 658 659 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) { 660 queue: 661 if (reinject) { 662 /* 663 * On VM-Entry, an exception can be pending if and only 664 * if event injection was blocked by nested_run_pending. 665 * In that case, however, vcpu_enter_guest() requests an 666 * immediate exit, and the guest shouldn't proceed far 667 * enough to need reinjection. 668 */ 669 WARN_ON_ONCE(kvm_is_exception_pending(vcpu)); 670 vcpu->arch.exception.injected = true; 671 if (WARN_ON_ONCE(has_payload)) { 672 /* 673 * A reinjected event has already 674 * delivered its payload. 675 */ 676 has_payload = false; 677 payload = 0; 678 } 679 } else { 680 vcpu->arch.exception.pending = true; 681 vcpu->arch.exception.injected = false; 682 } 683 vcpu->arch.exception.has_error_code = has_error; 684 vcpu->arch.exception.vector = nr; 685 vcpu->arch.exception.error_code = error_code; 686 vcpu->arch.exception.has_payload = has_payload; 687 vcpu->arch.exception.payload = payload; 688 if (!is_guest_mode(vcpu)) 689 kvm_deliver_exception_payload(vcpu, 690 &vcpu->arch.exception); 691 return; 692 } 693 694 /* to check exception */ 695 prev_nr = vcpu->arch.exception.vector; 696 if (prev_nr == DF_VECTOR) { 697 /* triple fault -> shutdown */ 698 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 699 return; 700 } 701 class1 = exception_class(prev_nr); 702 class2 = exception_class(nr); 703 if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) || 704 (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) { 705 /* 706 * Synthesize #DF. Clear the previously injected or pending 707 * exception so as not to incorrectly trigger shutdown. 708 */ 709 vcpu->arch.exception.injected = false; 710 vcpu->arch.exception.pending = false; 711 712 kvm_queue_exception_e(vcpu, DF_VECTOR, 0); 713 } else { 714 /* replace previous exception with a new one in a hope 715 that instruction re-execution will regenerate lost 716 exception */ 717 goto queue; 718 } 719 } 720 721 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) 722 { 723 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false); 724 } 725 EXPORT_SYMBOL_GPL(kvm_queue_exception); 726 727 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr) 728 { 729 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true); 730 } 731 EXPORT_SYMBOL_GPL(kvm_requeue_exception); 732 733 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, 734 unsigned long payload) 735 { 736 kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false); 737 } 738 EXPORT_SYMBOL_GPL(kvm_queue_exception_p); 739 740 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr, 741 u32 error_code, unsigned long payload) 742 { 743 kvm_multiple_exception(vcpu, nr, true, error_code, 744 true, payload, false); 745 } 746 747 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) 748 { 749 if (err) 750 kvm_inject_gp(vcpu, 0); 751 else 752 return kvm_skip_emulated_instruction(vcpu); 753 754 return 1; 755 } 756 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp); 757 758 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) 759 { 760 if (err) { 761 kvm_inject_gp(vcpu, 0); 762 return 1; 763 } 764 765 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP | 766 EMULTYPE_COMPLETE_USER_EXIT); 767 } 768 769 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) 770 { 771 ++vcpu->stat.pf_guest; 772 773 /* 774 * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of 775 * whether or not L1 wants to intercept "regular" #PF. 776 */ 777 if (is_guest_mode(vcpu) && fault->async_page_fault) 778 kvm_queue_exception_vmexit(vcpu, PF_VECTOR, 779 true, fault->error_code, 780 true, fault->address); 781 else 782 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code, 783 fault->address); 784 } 785 786 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, 787 struct x86_exception *fault) 788 { 789 struct kvm_mmu *fault_mmu; 790 WARN_ON_ONCE(fault->vector != PF_VECTOR); 791 792 fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu : 793 vcpu->arch.walk_mmu; 794 795 /* 796 * Invalidate the TLB entry for the faulting address, if it exists, 797 * else the access will fault indefinitely (and to emulate hardware). 798 */ 799 if ((fault->error_code & PFERR_PRESENT_MASK) && 800 !(fault->error_code & PFERR_RSVD_MASK)) 801 kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address, 802 fault_mmu->root.hpa); 803 804 fault_mmu->inject_page_fault(vcpu, fault); 805 } 806 EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault); 807 808 void kvm_inject_nmi(struct kvm_vcpu *vcpu) 809 { 810 atomic_inc(&vcpu->arch.nmi_queued); 811 kvm_make_request(KVM_REQ_NMI, vcpu); 812 } 813 814 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 815 { 816 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false); 817 } 818 EXPORT_SYMBOL_GPL(kvm_queue_exception_e); 819 820 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) 821 { 822 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true); 823 } 824 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e); 825 826 /* 827 * Checks if cpl <= required_cpl; if true, return true. Otherwise queue 828 * a #GP and return false. 829 */ 830 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl) 831 { 832 if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl) 833 return true; 834 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 835 return false; 836 } 837 838 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr) 839 { 840 if ((dr != 4 && dr != 5) || !kvm_read_cr4_bits(vcpu, X86_CR4_DE)) 841 return true; 842 843 kvm_queue_exception(vcpu, UD_VECTOR); 844 return false; 845 } 846 EXPORT_SYMBOL_GPL(kvm_require_dr); 847 848 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) 849 { 850 return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); 851 } 852 853 /* 854 * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. 855 */ 856 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) 857 { 858 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 859 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; 860 gpa_t real_gpa; 861 int i; 862 int ret; 863 u64 pdpte[ARRAY_SIZE(mmu->pdptrs)]; 864 865 /* 866 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated 867 * to an L1 GPA. 868 */ 869 real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), 870 PFERR_USER_MASK | PFERR_WRITE_MASK, NULL); 871 if (real_gpa == INVALID_GPA) 872 return 0; 873 874 /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ 875 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, 876 cr3 & GENMASK(11, 5), sizeof(pdpte)); 877 if (ret < 0) 878 return 0; 879 880 for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { 881 if ((pdpte[i] & PT_PRESENT_MASK) && 882 (pdpte[i] & pdptr_rsvd_bits(vcpu))) { 883 return 0; 884 } 885 } 886 887 /* 888 * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled. 889 * Shadow page roots need to be reconstructed instead. 890 */ 891 if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs))) 892 kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); 893 894 memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); 895 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); 896 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); 897 vcpu->arch.pdptrs_from_userspace = false; 898 899 return 1; 900 } 901 EXPORT_SYMBOL_GPL(load_pdptrs); 902 903 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) 904 { 905 if ((cr0 ^ old_cr0) & X86_CR0_PG) { 906 kvm_clear_async_pf_completion_queue(vcpu); 907 kvm_async_pf_hash_reset(vcpu); 908 909 /* 910 * Clearing CR0.PG is defined to flush the TLB from the guest's 911 * perspective. 912 */ 913 if (!(cr0 & X86_CR0_PG)) 914 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 915 } 916 917 if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) 918 kvm_mmu_reset_context(vcpu); 919 920 if (((cr0 ^ old_cr0) & X86_CR0_CD) && 921 kvm_arch_has_noncoherent_dma(vcpu->kvm) && 922 !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) 923 kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL); 924 } 925 EXPORT_SYMBOL_GPL(kvm_post_set_cr0); 926 927 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) 928 { 929 unsigned long old_cr0 = kvm_read_cr0(vcpu); 930 931 cr0 |= X86_CR0_ET; 932 933 #ifdef CONFIG_X86_64 934 if (cr0 & 0xffffffff00000000UL) 935 return 1; 936 #endif 937 938 cr0 &= ~CR0_RESERVED_BITS; 939 940 if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) 941 return 1; 942 943 if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) 944 return 1; 945 946 #ifdef CONFIG_X86_64 947 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && 948 (cr0 & X86_CR0_PG)) { 949 int cs_db, cs_l; 950 951 if (!is_pae(vcpu)) 952 return 1; 953 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); 954 if (cs_l) 955 return 1; 956 } 957 #endif 958 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && 959 is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && 960 !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) 961 return 1; 962 963 if (!(cr0 & X86_CR0_PG) && 964 (is_64_bit_mode(vcpu) || kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))) 965 return 1; 966 967 static_call(kvm_x86_set_cr0)(vcpu, cr0); 968 969 kvm_post_set_cr0(vcpu, old_cr0, cr0); 970 971 return 0; 972 } 973 EXPORT_SYMBOL_GPL(kvm_set_cr0); 974 975 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) 976 { 977 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); 978 } 979 EXPORT_SYMBOL_GPL(kvm_lmsw); 980 981 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu) 982 { 983 if (vcpu->arch.guest_state_protected) 984 return; 985 986 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) { 987 988 if (vcpu->arch.xcr0 != host_xcr0) 989 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0); 990 991 if (vcpu->arch.xsaves_enabled && 992 vcpu->arch.ia32_xss != host_xss) 993 wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss); 994 } 995 996 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 997 if (static_cpu_has(X86_FEATURE_PKU) && 998 vcpu->arch.pkru != vcpu->arch.host_pkru && 999 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || 1000 kvm_read_cr4_bits(vcpu, X86_CR4_PKE))) 1001 write_pkru(vcpu->arch.pkru); 1002 #endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ 1003 } 1004 EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state); 1005 1006 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) 1007 { 1008 if (vcpu->arch.guest_state_protected) 1009 return; 1010 1011 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 1012 if (static_cpu_has(X86_FEATURE_PKU) && 1013 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || 1014 kvm_read_cr4_bits(vcpu, X86_CR4_PKE))) { 1015 vcpu->arch.pkru = rdpkru(); 1016 if (vcpu->arch.pkru != vcpu->arch.host_pkru) 1017 write_pkru(vcpu->arch.host_pkru); 1018 } 1019 #endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ 1020 1021 if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) { 1022 1023 if (vcpu->arch.xcr0 != host_xcr0) 1024 xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0); 1025 1026 if (vcpu->arch.xsaves_enabled && 1027 vcpu->arch.ia32_xss != host_xss) 1028 wrmsrl(MSR_IA32_XSS, host_xss); 1029 } 1030 1031 } 1032 EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state); 1033 1034 #ifdef CONFIG_X86_64 1035 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) 1036 { 1037 return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC; 1038 } 1039 #endif 1040 1041 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) 1042 { 1043 u64 xcr0 = xcr; 1044 u64 old_xcr0 = vcpu->arch.xcr0; 1045 u64 valid_bits; 1046 1047 /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */ 1048 if (index != XCR_XFEATURE_ENABLED_MASK) 1049 return 1; 1050 if (!(xcr0 & XFEATURE_MASK_FP)) 1051 return 1; 1052 if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE)) 1053 return 1; 1054 1055 /* 1056 * Do not allow the guest to set bits that we do not support 1057 * saving. However, xcr0 bit 0 is always set, even if the 1058 * emulated CPU does not support XSAVE (see kvm_vcpu_reset()). 1059 */ 1060 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; 1061 if (xcr0 & ~valid_bits) 1062 return 1; 1063 1064 if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) != 1065 (!(xcr0 & XFEATURE_MASK_BNDCSR))) 1066 return 1; 1067 1068 if (xcr0 & XFEATURE_MASK_AVX512) { 1069 if (!(xcr0 & XFEATURE_MASK_YMM)) 1070 return 1; 1071 if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512) 1072 return 1; 1073 } 1074 1075 if ((xcr0 & XFEATURE_MASK_XTILE) && 1076 ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE)) 1077 return 1; 1078 1079 vcpu->arch.xcr0 = xcr0; 1080 1081 if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND) 1082 kvm_update_cpuid_runtime(vcpu); 1083 return 0; 1084 } 1085 1086 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) 1087 { 1088 /* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */ 1089 if (static_call(kvm_x86_get_cpl)(vcpu) != 0 || 1090 __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) { 1091 kvm_inject_gp(vcpu, 0); 1092 return 1; 1093 } 1094 1095 return kvm_skip_emulated_instruction(vcpu); 1096 } 1097 EXPORT_SYMBOL_GPL(kvm_emulate_xsetbv); 1098 1099 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1100 { 1101 if (cr4 & cr4_reserved_bits) 1102 return false; 1103 1104 if (cr4 & vcpu->arch.cr4_guest_rsvd_bits) 1105 return false; 1106 1107 return true; 1108 } 1109 EXPORT_SYMBOL_GPL(__kvm_is_valid_cr4); 1110 1111 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1112 { 1113 return __kvm_is_valid_cr4(vcpu, cr4) && 1114 static_call(kvm_x86_is_valid_cr4)(vcpu, cr4); 1115 } 1116 1117 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) 1118 { 1119 if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) 1120 kvm_mmu_reset_context(vcpu); 1121 1122 /* 1123 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB 1124 * according to the SDM; however, stale prev_roots could be reused 1125 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we 1126 * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST 1127 * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, 1128 * so fall through. 1129 */ 1130 if (!tdp_enabled && 1131 (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) 1132 kvm_mmu_unload(vcpu); 1133 1134 /* 1135 * The TLB has to be flushed for all PCIDs if any of the following 1136 * (architecturally required) changes happen: 1137 * - CR4.PCIDE is changed from 1 to 0 1138 * - CR4.PGE is toggled 1139 * 1140 * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. 1141 */ 1142 if (((cr4 ^ old_cr4) & X86_CR4_PGE) || 1143 (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) 1144 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1145 1146 /* 1147 * The TLB has to be flushed for the current PCID if any of the 1148 * following (architecturally required) changes happen: 1149 * - CR4.SMEP is changed from 0 to 1 1150 * - CR4.PAE is toggled 1151 */ 1152 else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || 1153 ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) 1154 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1155 1156 } 1157 EXPORT_SYMBOL_GPL(kvm_post_set_cr4); 1158 1159 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) 1160 { 1161 unsigned long old_cr4 = kvm_read_cr4(vcpu); 1162 1163 if (!kvm_is_valid_cr4(vcpu, cr4)) 1164 return 1; 1165 1166 if (is_long_mode(vcpu)) { 1167 if (!(cr4 & X86_CR4_PAE)) 1168 return 1; 1169 if ((cr4 ^ old_cr4) & X86_CR4_LA57) 1170 return 1; 1171 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) 1172 && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) 1173 && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) 1174 return 1; 1175 1176 if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { 1177 if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID)) 1178 return 1; 1179 1180 /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ 1181 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) 1182 return 1; 1183 } 1184 1185 static_call(kvm_x86_set_cr4)(vcpu, cr4); 1186 1187 kvm_post_set_cr4(vcpu, old_cr4, cr4); 1188 1189 return 0; 1190 } 1191 EXPORT_SYMBOL_GPL(kvm_set_cr4); 1192 1193 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) 1194 { 1195 struct kvm_mmu *mmu = vcpu->arch.mmu; 1196 unsigned long roots_to_free = 0; 1197 int i; 1198 1199 /* 1200 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but 1201 * this is reachable when running EPT=1 and unrestricted_guest=0, and 1202 * also via the emulator. KVM's TDP page tables are not in the scope of 1203 * the invalidation, but the guest's TLB entries need to be flushed as 1204 * the CPU may have cached entries in its TLB for the target PCID. 1205 */ 1206 if (unlikely(tdp_enabled)) { 1207 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1208 return; 1209 } 1210 1211 /* 1212 * If neither the current CR3 nor any of the prev_roots use the given 1213 * PCID, then nothing needs to be done here because a resync will 1214 * happen anyway before switching to any other CR3. 1215 */ 1216 if (kvm_get_active_pcid(vcpu) == pcid) { 1217 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); 1218 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1219 } 1220 1221 /* 1222 * If PCID is disabled, there is no need to free prev_roots even if the 1223 * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB 1224 * with PCIDE=0. 1225 */ 1226 if (!kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) 1227 return; 1228 1229 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) 1230 if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid) 1231 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 1232 1233 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 1234 } 1235 1236 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) 1237 { 1238 bool skip_tlb_flush = false; 1239 unsigned long pcid = 0; 1240 #ifdef CONFIG_X86_64 1241 bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE); 1242 1243 if (pcid_enabled) { 1244 skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; 1245 cr3 &= ~X86_CR3_PCID_NOFLUSH; 1246 pcid = cr3 & X86_CR3_PCID_MASK; 1247 } 1248 #endif 1249 1250 /* PDPTRs are always reloaded for PAE paging. */ 1251 if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) 1252 goto handle_tlb_flush; 1253 1254 /* 1255 * Do not condition the GPA check on long mode, this helper is used to 1256 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that 1257 * the current vCPU mode is accurate. 1258 */ 1259 if (kvm_vcpu_is_illegal_gpa(vcpu, cr3)) 1260 return 1; 1261 1262 if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) 1263 return 1; 1264 1265 if (cr3 != kvm_read_cr3(vcpu)) 1266 kvm_mmu_new_pgd(vcpu, cr3); 1267 1268 vcpu->arch.cr3 = cr3; 1269 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 1270 /* Do not call post_set_cr3, we do not get here for confidential guests. */ 1271 1272 handle_tlb_flush: 1273 /* 1274 * A load of CR3 that flushes the TLB flushes only the current PCID, 1275 * even if PCID is disabled, in which case PCID=0 is flushed. It's a 1276 * moot point in the end because _disabling_ PCID will flush all PCIDs, 1277 * and it's impossible to use a non-zero PCID when PCID is disabled, 1278 * i.e. only PCID=0 can be relevant. 1279 */ 1280 if (!skip_tlb_flush) 1281 kvm_invalidate_pcid(vcpu, pcid); 1282 1283 return 0; 1284 } 1285 EXPORT_SYMBOL_GPL(kvm_set_cr3); 1286 1287 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) 1288 { 1289 if (cr8 & CR8_RESERVED_BITS) 1290 return 1; 1291 if (lapic_in_kernel(vcpu)) 1292 kvm_lapic_set_tpr(vcpu, cr8); 1293 else 1294 vcpu->arch.cr8 = cr8; 1295 return 0; 1296 } 1297 EXPORT_SYMBOL_GPL(kvm_set_cr8); 1298 1299 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) 1300 { 1301 if (lapic_in_kernel(vcpu)) 1302 return kvm_lapic_get_cr8(vcpu); 1303 else 1304 return vcpu->arch.cr8; 1305 } 1306 EXPORT_SYMBOL_GPL(kvm_get_cr8); 1307 1308 static void kvm_update_dr0123(struct kvm_vcpu *vcpu) 1309 { 1310 int i; 1311 1312 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { 1313 for (i = 0; i < KVM_NR_DB_REGS; i++) 1314 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 1315 } 1316 } 1317 1318 void kvm_update_dr7(struct kvm_vcpu *vcpu) 1319 { 1320 unsigned long dr7; 1321 1322 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 1323 dr7 = vcpu->arch.guest_debug_dr7; 1324 else 1325 dr7 = vcpu->arch.dr7; 1326 static_call(kvm_x86_set_dr7)(vcpu, dr7); 1327 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; 1328 if (dr7 & DR7_BP_EN_MASK) 1329 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; 1330 } 1331 EXPORT_SYMBOL_GPL(kvm_update_dr7); 1332 1333 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) 1334 { 1335 u64 fixed = DR6_FIXED_1; 1336 1337 if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM)) 1338 fixed |= DR6_RTM; 1339 1340 if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) 1341 fixed |= DR6_BUS_LOCK; 1342 return fixed; 1343 } 1344 1345 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) 1346 { 1347 size_t size = ARRAY_SIZE(vcpu->arch.db); 1348 1349 switch (dr) { 1350 case 0 ... 3: 1351 vcpu->arch.db[array_index_nospec(dr, size)] = val; 1352 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) 1353 vcpu->arch.eff_db[dr] = val; 1354 break; 1355 case 4: 1356 case 6: 1357 if (!kvm_dr6_valid(val)) 1358 return 1; /* #GP */ 1359 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); 1360 break; 1361 case 5: 1362 default: /* 7 */ 1363 if (!kvm_dr7_valid(val)) 1364 return 1; /* #GP */ 1365 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; 1366 kvm_update_dr7(vcpu); 1367 break; 1368 } 1369 1370 return 0; 1371 } 1372 EXPORT_SYMBOL_GPL(kvm_set_dr); 1373 1374 void kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val) 1375 { 1376 size_t size = ARRAY_SIZE(vcpu->arch.db); 1377 1378 switch (dr) { 1379 case 0 ... 3: 1380 *val = vcpu->arch.db[array_index_nospec(dr, size)]; 1381 break; 1382 case 4: 1383 case 6: 1384 *val = vcpu->arch.dr6; 1385 break; 1386 case 5: 1387 default: /* 7 */ 1388 *val = vcpu->arch.dr7; 1389 break; 1390 } 1391 } 1392 EXPORT_SYMBOL_GPL(kvm_get_dr); 1393 1394 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) 1395 { 1396 u32 ecx = kvm_rcx_read(vcpu); 1397 u64 data; 1398 1399 if (kvm_pmu_rdpmc(vcpu, ecx, &data)) { 1400 kvm_inject_gp(vcpu, 0); 1401 return 1; 1402 } 1403 1404 kvm_rax_write(vcpu, (u32)data); 1405 kvm_rdx_write(vcpu, data >> 32); 1406 return kvm_skip_emulated_instruction(vcpu); 1407 } 1408 EXPORT_SYMBOL_GPL(kvm_emulate_rdpmc); 1409 1410 /* 1411 * List of msr numbers which we expose to userspace through KVM_GET_MSRS 1412 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. 1413 * 1414 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) 1415 * extract the supported MSRs from the related const lists. 1416 * msrs_to_save is selected from the msrs_to_save_all to reflect the 1417 * capabilities of the host cpu. This capabilities test skips MSRs that are 1418 * kvm-specific. Those are put in emulated_msrs_all; filtering of emulated_msrs 1419 * may depend on host virtualization features rather than host cpu features. 1420 */ 1421 1422 static const u32 msrs_to_save_base[] = { 1423 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, 1424 MSR_STAR, 1425 #ifdef CONFIG_X86_64 1426 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, 1427 #endif 1428 MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA, 1429 MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX, 1430 MSR_IA32_SPEC_CTRL, 1431 MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH, 1432 MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK, 1433 MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B, 1434 MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B, 1435 MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B, 1436 MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B, 1437 MSR_IA32_UMWAIT_CONTROL, 1438 1439 MSR_IA32_XFD, MSR_IA32_XFD_ERR, 1440 }; 1441 1442 static const u32 msrs_to_save_pmu[] = { 1443 MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1, 1444 MSR_ARCH_PERFMON_FIXED_CTR0 + 2, 1445 MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS, 1446 MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL, 1447 MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG, 1448 1449 /* This part of MSRs should match KVM_INTEL_PMC_MAX_GENERIC. */ 1450 MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1, 1451 MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3, 1452 MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5, 1453 MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7, 1454 MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1, 1455 MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3, 1456 MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5, 1457 MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7, 1458 1459 MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3, 1460 MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3, 1461 1462 /* This part of MSRs should match KVM_AMD_PMC_MAX_GENERIC. */ 1463 MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2, 1464 MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5, 1465 MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2, 1466 MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5, 1467 }; 1468 1469 static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) + 1470 ARRAY_SIZE(msrs_to_save_pmu)]; 1471 static unsigned num_msrs_to_save; 1472 1473 static const u32 emulated_msrs_all[] = { 1474 MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, 1475 MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, 1476 HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, 1477 HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC, 1478 HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY, 1479 HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2, 1480 HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL, 1481 HV_X64_MSR_RESET, 1482 HV_X64_MSR_VP_INDEX, 1483 HV_X64_MSR_VP_RUNTIME, 1484 HV_X64_MSR_SCONTROL, 1485 HV_X64_MSR_STIMER0_CONFIG, 1486 HV_X64_MSR_VP_ASSIST_PAGE, 1487 HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL, 1488 HV_X64_MSR_TSC_EMULATION_STATUS, HV_X64_MSR_TSC_INVARIANT_CONTROL, 1489 HV_X64_MSR_SYNDBG_OPTIONS, 1490 HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS, 1491 HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER, 1492 HV_X64_MSR_SYNDBG_PENDING_BUFFER, 1493 1494 MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, 1495 MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK, 1496 1497 MSR_IA32_TSC_ADJUST, 1498 MSR_IA32_TSC_DEADLINE, 1499 MSR_IA32_ARCH_CAPABILITIES, 1500 MSR_IA32_PERF_CAPABILITIES, 1501 MSR_IA32_MISC_ENABLE, 1502 MSR_IA32_MCG_STATUS, 1503 MSR_IA32_MCG_CTL, 1504 MSR_IA32_MCG_EXT_CTL, 1505 MSR_IA32_SMBASE, 1506 MSR_SMI_COUNT, 1507 MSR_PLATFORM_INFO, 1508 MSR_MISC_FEATURES_ENABLES, 1509 MSR_AMD64_VIRT_SPEC_CTRL, 1510 MSR_AMD64_TSC_RATIO, 1511 MSR_IA32_POWER_CTL, 1512 MSR_IA32_UCODE_REV, 1513 1514 /* 1515 * The following list leaves out MSRs whose values are determined 1516 * by arch/x86/kvm/vmx/nested.c based on CPUID or other MSRs. 1517 * We always support the "true" VMX control MSRs, even if the host 1518 * processor does not, so I am putting these registers here rather 1519 * than in msrs_to_save_all. 1520 */ 1521 MSR_IA32_VMX_BASIC, 1522 MSR_IA32_VMX_TRUE_PINBASED_CTLS, 1523 MSR_IA32_VMX_TRUE_PROCBASED_CTLS, 1524 MSR_IA32_VMX_TRUE_EXIT_CTLS, 1525 MSR_IA32_VMX_TRUE_ENTRY_CTLS, 1526 MSR_IA32_VMX_MISC, 1527 MSR_IA32_VMX_CR0_FIXED0, 1528 MSR_IA32_VMX_CR4_FIXED0, 1529 MSR_IA32_VMX_VMCS_ENUM, 1530 MSR_IA32_VMX_PROCBASED_CTLS2, 1531 MSR_IA32_VMX_EPT_VPID_CAP, 1532 MSR_IA32_VMX_VMFUNC, 1533 1534 MSR_K7_HWCR, 1535 MSR_KVM_POLL_CONTROL, 1536 }; 1537 1538 static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)]; 1539 static unsigned num_emulated_msrs; 1540 1541 /* 1542 * List of msr numbers which are used to expose MSR-based features that 1543 * can be used by a hypervisor to validate requested CPU features. 1544 */ 1545 static const u32 msr_based_features_all[] = { 1546 MSR_IA32_VMX_BASIC, 1547 MSR_IA32_VMX_TRUE_PINBASED_CTLS, 1548 MSR_IA32_VMX_PINBASED_CTLS, 1549 MSR_IA32_VMX_TRUE_PROCBASED_CTLS, 1550 MSR_IA32_VMX_PROCBASED_CTLS, 1551 MSR_IA32_VMX_TRUE_EXIT_CTLS, 1552 MSR_IA32_VMX_EXIT_CTLS, 1553 MSR_IA32_VMX_TRUE_ENTRY_CTLS, 1554 MSR_IA32_VMX_ENTRY_CTLS, 1555 MSR_IA32_VMX_MISC, 1556 MSR_IA32_VMX_CR0_FIXED0, 1557 MSR_IA32_VMX_CR0_FIXED1, 1558 MSR_IA32_VMX_CR4_FIXED0, 1559 MSR_IA32_VMX_CR4_FIXED1, 1560 MSR_IA32_VMX_VMCS_ENUM, 1561 MSR_IA32_VMX_PROCBASED_CTLS2, 1562 MSR_IA32_VMX_EPT_VPID_CAP, 1563 MSR_IA32_VMX_VMFUNC, 1564 1565 MSR_AMD64_DE_CFG, 1566 MSR_IA32_UCODE_REV, 1567 MSR_IA32_ARCH_CAPABILITIES, 1568 MSR_IA32_PERF_CAPABILITIES, 1569 }; 1570 1571 static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)]; 1572 static unsigned int num_msr_based_features; 1573 1574 /* 1575 * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM 1576 * does not yet virtualize. These include: 1577 * 10 - MISC_PACKAGE_CTRLS 1578 * 11 - ENERGY_FILTERING_CTL 1579 * 12 - DOITM 1580 * 18 - FB_CLEAR_CTRL 1581 * 21 - XAPIC_DISABLE_STATUS 1582 * 23 - OVERCLOCKING_STATUS 1583 */ 1584 1585 #define KVM_SUPPORTED_ARCH_CAP \ 1586 (ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \ 1587 ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \ 1588 ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \ 1589 ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \ 1590 ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO) 1591 1592 static u64 kvm_get_arch_capabilities(void) 1593 { 1594 u64 data = 0; 1595 1596 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) { 1597 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data); 1598 data &= KVM_SUPPORTED_ARCH_CAP; 1599 } 1600 1601 /* 1602 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that 1603 * the nested hypervisor runs with NX huge pages. If it is not, 1604 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other 1605 * L1 guests, so it need not worry about its own (L2) guests. 1606 */ 1607 data |= ARCH_CAP_PSCHANGE_MC_NO; 1608 1609 /* 1610 * If we're doing cache flushes (either "always" or "cond") 1611 * we will do one whenever the guest does a vmlaunch/vmresume. 1612 * If an outer hypervisor is doing the cache flush for us 1613 * (VMENTER_L1D_FLUSH_NESTED_VM), we can safely pass that 1614 * capability to the guest too, and if EPT is disabled we're not 1615 * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will 1616 * require a nested hypervisor to do a flush of its own. 1617 */ 1618 if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER) 1619 data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH; 1620 1621 if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN)) 1622 data |= ARCH_CAP_RDCL_NO; 1623 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) 1624 data |= ARCH_CAP_SSB_NO; 1625 if (!boot_cpu_has_bug(X86_BUG_MDS)) 1626 data |= ARCH_CAP_MDS_NO; 1627 1628 if (!boot_cpu_has(X86_FEATURE_RTM)) { 1629 /* 1630 * If RTM=0 because the kernel has disabled TSX, the host might 1631 * have TAA_NO or TSX_CTRL. Clear TAA_NO (the guest sees RTM=0 1632 * and therefore knows that there cannot be TAA) but keep 1633 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts, 1634 * and we want to allow migrating those guests to tsx=off hosts. 1635 */ 1636 data &= ~ARCH_CAP_TAA_NO; 1637 } else if (!boot_cpu_has_bug(X86_BUG_TAA)) { 1638 data |= ARCH_CAP_TAA_NO; 1639 } else { 1640 /* 1641 * Nothing to do here; we emulate TSX_CTRL if present on the 1642 * host so the guest can choose between disabling TSX or 1643 * using VERW to clear CPU buffers. 1644 */ 1645 } 1646 1647 return data; 1648 } 1649 1650 static int kvm_get_msr_feature(struct kvm_msr_entry *msr) 1651 { 1652 switch (msr->index) { 1653 case MSR_IA32_ARCH_CAPABILITIES: 1654 msr->data = kvm_get_arch_capabilities(); 1655 break; 1656 case MSR_IA32_PERF_CAPABILITIES: 1657 msr->data = kvm_caps.supported_perf_cap; 1658 break; 1659 case MSR_IA32_UCODE_REV: 1660 rdmsrl_safe(msr->index, &msr->data); 1661 break; 1662 default: 1663 return static_call(kvm_x86_get_msr_feature)(msr); 1664 } 1665 return 0; 1666 } 1667 1668 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 1669 { 1670 struct kvm_msr_entry msr; 1671 int r; 1672 1673 msr.index = index; 1674 r = kvm_get_msr_feature(&msr); 1675 1676 if (r == KVM_MSR_RET_INVALID) { 1677 /* Unconditionally clear the output for simplicity */ 1678 *data = 0; 1679 if (kvm_msr_ignored_check(index, 0, false)) 1680 r = 0; 1681 } 1682 1683 if (r) 1684 return r; 1685 1686 *data = msr.data; 1687 1688 return 0; 1689 } 1690 1691 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) 1692 { 1693 if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT)) 1694 return false; 1695 1696 if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM)) 1697 return false; 1698 1699 if (efer & (EFER_LME | EFER_LMA) && 1700 !guest_cpuid_has(vcpu, X86_FEATURE_LM)) 1701 return false; 1702 1703 if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX)) 1704 return false; 1705 1706 return true; 1707 1708 } 1709 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) 1710 { 1711 if (efer & efer_reserved_bits) 1712 return false; 1713 1714 return __kvm_valid_efer(vcpu, efer); 1715 } 1716 EXPORT_SYMBOL_GPL(kvm_valid_efer); 1717 1718 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 1719 { 1720 u64 old_efer = vcpu->arch.efer; 1721 u64 efer = msr_info->data; 1722 int r; 1723 1724 if (efer & efer_reserved_bits) 1725 return 1; 1726 1727 if (!msr_info->host_initiated) { 1728 if (!__kvm_valid_efer(vcpu, efer)) 1729 return 1; 1730 1731 if (is_paging(vcpu) && 1732 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) 1733 return 1; 1734 } 1735 1736 efer &= ~EFER_LMA; 1737 efer |= vcpu->arch.efer & EFER_LMA; 1738 1739 r = static_call(kvm_x86_set_efer)(vcpu, efer); 1740 if (r) { 1741 WARN_ON(r > 0); 1742 return r; 1743 } 1744 1745 if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS) 1746 kvm_mmu_reset_context(vcpu); 1747 1748 return 0; 1749 } 1750 1751 void kvm_enable_efer_bits(u64 mask) 1752 { 1753 efer_reserved_bits &= ~mask; 1754 } 1755 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits); 1756 1757 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type) 1758 { 1759 struct kvm_x86_msr_filter *msr_filter; 1760 struct msr_bitmap_range *ranges; 1761 struct kvm *kvm = vcpu->kvm; 1762 bool allowed; 1763 int idx; 1764 u32 i; 1765 1766 /* x2APIC MSRs do not support filtering. */ 1767 if (index >= 0x800 && index <= 0x8ff) 1768 return true; 1769 1770 idx = srcu_read_lock(&kvm->srcu); 1771 1772 msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu); 1773 if (!msr_filter) { 1774 allowed = true; 1775 goto out; 1776 } 1777 1778 allowed = msr_filter->default_allow; 1779 ranges = msr_filter->ranges; 1780 1781 for (i = 0; i < msr_filter->count; i++) { 1782 u32 start = ranges[i].base; 1783 u32 end = start + ranges[i].nmsrs; 1784 u32 flags = ranges[i].flags; 1785 unsigned long *bitmap = ranges[i].bitmap; 1786 1787 if ((index >= start) && (index < end) && (flags & type)) { 1788 allowed = !!test_bit(index - start, bitmap); 1789 break; 1790 } 1791 } 1792 1793 out: 1794 srcu_read_unlock(&kvm->srcu, idx); 1795 1796 return allowed; 1797 } 1798 EXPORT_SYMBOL_GPL(kvm_msr_allowed); 1799 1800 /* 1801 * Write @data into the MSR specified by @index. Select MSR specific fault 1802 * checks are bypassed if @host_initiated is %true. 1803 * Returns 0 on success, non-0 otherwise. 1804 * Assumes vcpu_load() was already called. 1805 */ 1806 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, 1807 bool host_initiated) 1808 { 1809 struct msr_data msr; 1810 1811 switch (index) { 1812 case MSR_FS_BASE: 1813 case MSR_GS_BASE: 1814 case MSR_KERNEL_GS_BASE: 1815 case MSR_CSTAR: 1816 case MSR_LSTAR: 1817 if (is_noncanonical_address(data, vcpu)) 1818 return 1; 1819 break; 1820 case MSR_IA32_SYSENTER_EIP: 1821 case MSR_IA32_SYSENTER_ESP: 1822 /* 1823 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if 1824 * non-canonical address is written on Intel but not on 1825 * AMD (which ignores the top 32-bits, because it does 1826 * not implement 64-bit SYSENTER). 1827 * 1828 * 64-bit code should hence be able to write a non-canonical 1829 * value on AMD. Making the address canonical ensures that 1830 * vmentry does not fail on Intel after writing a non-canonical 1831 * value, and that something deterministic happens if the guest 1832 * invokes 64-bit SYSENTER. 1833 */ 1834 data = __canonical_address(data, vcpu_virt_addr_bits(vcpu)); 1835 break; 1836 case MSR_TSC_AUX: 1837 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) 1838 return 1; 1839 1840 if (!host_initiated && 1841 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && 1842 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) 1843 return 1; 1844 1845 /* 1846 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has 1847 * incomplete and conflicting architectural behavior. Current 1848 * AMD CPUs completely ignore bits 63:32, i.e. they aren't 1849 * reserved and always read as zeros. Enforce Intel's reserved 1850 * bits check if and only if the guest CPU is Intel, and clear 1851 * the bits in all other cases. This ensures cross-vendor 1852 * migration will provide consistent behavior for the guest. 1853 */ 1854 if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0) 1855 return 1; 1856 1857 data = (u32)data; 1858 break; 1859 } 1860 1861 msr.data = data; 1862 msr.index = index; 1863 msr.host_initiated = host_initiated; 1864 1865 return static_call(kvm_x86_set_msr)(vcpu, &msr); 1866 } 1867 1868 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu, 1869 u32 index, u64 data, bool host_initiated) 1870 { 1871 int ret = __kvm_set_msr(vcpu, index, data, host_initiated); 1872 1873 if (ret == KVM_MSR_RET_INVALID) 1874 if (kvm_msr_ignored_check(index, data, true)) 1875 ret = 0; 1876 1877 return ret; 1878 } 1879 1880 /* 1881 * Read the MSR specified by @index into @data. Select MSR specific fault 1882 * checks are bypassed if @host_initiated is %true. 1883 * Returns 0 on success, non-0 otherwise. 1884 * Assumes vcpu_load() was already called. 1885 */ 1886 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, 1887 bool host_initiated) 1888 { 1889 struct msr_data msr; 1890 int ret; 1891 1892 switch (index) { 1893 case MSR_TSC_AUX: 1894 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) 1895 return 1; 1896 1897 if (!host_initiated && 1898 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) && 1899 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID)) 1900 return 1; 1901 break; 1902 } 1903 1904 msr.index = index; 1905 msr.host_initiated = host_initiated; 1906 1907 ret = static_call(kvm_x86_get_msr)(vcpu, &msr); 1908 if (!ret) 1909 *data = msr.data; 1910 return ret; 1911 } 1912 1913 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu, 1914 u32 index, u64 *data, bool host_initiated) 1915 { 1916 int ret = __kvm_get_msr(vcpu, index, data, host_initiated); 1917 1918 if (ret == KVM_MSR_RET_INVALID) { 1919 /* Unconditionally clear *data for simplicity */ 1920 *data = 0; 1921 if (kvm_msr_ignored_check(index, 0, false)) 1922 ret = 0; 1923 } 1924 1925 return ret; 1926 } 1927 1928 static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data) 1929 { 1930 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ)) 1931 return KVM_MSR_RET_FILTERED; 1932 return kvm_get_msr_ignored_check(vcpu, index, data, false); 1933 } 1934 1935 static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data) 1936 { 1937 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE)) 1938 return KVM_MSR_RET_FILTERED; 1939 return kvm_set_msr_ignored_check(vcpu, index, data, false); 1940 } 1941 1942 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data) 1943 { 1944 return kvm_get_msr_ignored_check(vcpu, index, data, false); 1945 } 1946 EXPORT_SYMBOL_GPL(kvm_get_msr); 1947 1948 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data) 1949 { 1950 return kvm_set_msr_ignored_check(vcpu, index, data, false); 1951 } 1952 EXPORT_SYMBOL_GPL(kvm_set_msr); 1953 1954 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu) 1955 { 1956 if (!vcpu->run->msr.error) { 1957 kvm_rax_write(vcpu, (u32)vcpu->run->msr.data); 1958 kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32); 1959 } 1960 } 1961 1962 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu) 1963 { 1964 return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error); 1965 } 1966 1967 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) 1968 { 1969 complete_userspace_rdmsr(vcpu); 1970 return complete_emulated_msr_access(vcpu); 1971 } 1972 1973 static int complete_fast_msr_access(struct kvm_vcpu *vcpu) 1974 { 1975 return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error); 1976 } 1977 1978 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu) 1979 { 1980 complete_userspace_rdmsr(vcpu); 1981 return complete_fast_msr_access(vcpu); 1982 } 1983 1984 static u64 kvm_msr_reason(int r) 1985 { 1986 switch (r) { 1987 case KVM_MSR_RET_INVALID: 1988 return KVM_MSR_EXIT_REASON_UNKNOWN; 1989 case KVM_MSR_RET_FILTERED: 1990 return KVM_MSR_EXIT_REASON_FILTER; 1991 default: 1992 return KVM_MSR_EXIT_REASON_INVAL; 1993 } 1994 } 1995 1996 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index, 1997 u32 exit_reason, u64 data, 1998 int (*completion)(struct kvm_vcpu *vcpu), 1999 int r) 2000 { 2001 u64 msr_reason = kvm_msr_reason(r); 2002 2003 /* Check if the user wanted to know about this MSR fault */ 2004 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason)) 2005 return 0; 2006 2007 vcpu->run->exit_reason = exit_reason; 2008 vcpu->run->msr.error = 0; 2009 memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad)); 2010 vcpu->run->msr.reason = msr_reason; 2011 vcpu->run->msr.index = index; 2012 vcpu->run->msr.data = data; 2013 vcpu->arch.complete_userspace_io = completion; 2014 2015 return 1; 2016 } 2017 2018 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) 2019 { 2020 u32 ecx = kvm_rcx_read(vcpu); 2021 u64 data; 2022 int r; 2023 2024 r = kvm_get_msr_with_filter(vcpu, ecx, &data); 2025 2026 if (!r) { 2027 trace_kvm_msr_read(ecx, data); 2028 2029 kvm_rax_write(vcpu, data & -1u); 2030 kvm_rdx_write(vcpu, (data >> 32) & -1u); 2031 } else { 2032 /* MSR read failed? See if we should ask user space */ 2033 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0, 2034 complete_fast_rdmsr, r)) 2035 return 0; 2036 trace_kvm_msr_read_ex(ecx); 2037 } 2038 2039 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r); 2040 } 2041 EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr); 2042 2043 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) 2044 { 2045 u32 ecx = kvm_rcx_read(vcpu); 2046 u64 data = kvm_read_edx_eax(vcpu); 2047 int r; 2048 2049 r = kvm_set_msr_with_filter(vcpu, ecx, data); 2050 2051 if (!r) { 2052 trace_kvm_msr_write(ecx, data); 2053 } else { 2054 /* MSR write failed? See if we should ask user space */ 2055 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data, 2056 complete_fast_msr_access, r)) 2057 return 0; 2058 /* Signal all other negative errors to userspace */ 2059 if (r < 0) 2060 return r; 2061 trace_kvm_msr_write_ex(ecx, data); 2062 } 2063 2064 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r); 2065 } 2066 EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr); 2067 2068 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu) 2069 { 2070 return kvm_skip_emulated_instruction(vcpu); 2071 } 2072 2073 int kvm_emulate_invd(struct kvm_vcpu *vcpu) 2074 { 2075 /* Treat an INVD instruction as a NOP and just skip it. */ 2076 return kvm_emulate_as_nop(vcpu); 2077 } 2078 EXPORT_SYMBOL_GPL(kvm_emulate_invd); 2079 2080 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu) 2081 { 2082 kvm_queue_exception(vcpu, UD_VECTOR); 2083 return 1; 2084 } 2085 EXPORT_SYMBOL_GPL(kvm_handle_invalid_op); 2086 2087 2088 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn) 2089 { 2090 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) && 2091 !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT)) 2092 return kvm_handle_invalid_op(vcpu); 2093 2094 pr_warn_once("%s instruction emulated as NOP!\n", insn); 2095 return kvm_emulate_as_nop(vcpu); 2096 } 2097 int kvm_emulate_mwait(struct kvm_vcpu *vcpu) 2098 { 2099 return kvm_emulate_monitor_mwait(vcpu, "MWAIT"); 2100 } 2101 EXPORT_SYMBOL_GPL(kvm_emulate_mwait); 2102 2103 int kvm_emulate_monitor(struct kvm_vcpu *vcpu) 2104 { 2105 return kvm_emulate_monitor_mwait(vcpu, "MONITOR"); 2106 } 2107 EXPORT_SYMBOL_GPL(kvm_emulate_monitor); 2108 2109 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu) 2110 { 2111 xfer_to_guest_mode_prepare(); 2112 return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) || 2113 xfer_to_guest_mode_work_pending(); 2114 } 2115 2116 /* 2117 * The fast path for frequent and performance sensitive wrmsr emulation, 2118 * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces 2119 * the latency of virtual IPI by avoiding the expensive bits of transitioning 2120 * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the 2121 * other cases which must be called after interrupts are enabled on the host. 2122 */ 2123 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data) 2124 { 2125 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic)) 2126 return 1; 2127 2128 if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) && 2129 ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) && 2130 ((data & APIC_MODE_MASK) == APIC_DM_FIXED) && 2131 ((u32)(data >> 32) != X2APIC_BROADCAST)) 2132 return kvm_x2apic_icr_write(vcpu->arch.apic, data); 2133 2134 return 1; 2135 } 2136 2137 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data) 2138 { 2139 if (!kvm_can_use_hv_timer(vcpu)) 2140 return 1; 2141 2142 kvm_set_lapic_tscdeadline_msr(vcpu, data); 2143 return 0; 2144 } 2145 2146 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) 2147 { 2148 u32 msr = kvm_rcx_read(vcpu); 2149 u64 data; 2150 fastpath_t ret = EXIT_FASTPATH_NONE; 2151 2152 switch (msr) { 2153 case APIC_BASE_MSR + (APIC_ICR >> 4): 2154 data = kvm_read_edx_eax(vcpu); 2155 if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) { 2156 kvm_skip_emulated_instruction(vcpu); 2157 ret = EXIT_FASTPATH_EXIT_HANDLED; 2158 } 2159 break; 2160 case MSR_IA32_TSC_DEADLINE: 2161 data = kvm_read_edx_eax(vcpu); 2162 if (!handle_fastpath_set_tscdeadline(vcpu, data)) { 2163 kvm_skip_emulated_instruction(vcpu); 2164 ret = EXIT_FASTPATH_REENTER_GUEST; 2165 } 2166 break; 2167 default: 2168 break; 2169 } 2170 2171 if (ret != EXIT_FASTPATH_NONE) 2172 trace_kvm_msr_write(msr, data); 2173 2174 return ret; 2175 } 2176 EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff); 2177 2178 /* 2179 * Adapt set_msr() to msr_io()'s calling convention 2180 */ 2181 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 2182 { 2183 return kvm_get_msr_ignored_check(vcpu, index, data, true); 2184 } 2185 2186 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) 2187 { 2188 return kvm_set_msr_ignored_check(vcpu, index, *data, true); 2189 } 2190 2191 #ifdef CONFIG_X86_64 2192 struct pvclock_clock { 2193 int vclock_mode; 2194 u64 cycle_last; 2195 u64 mask; 2196 u32 mult; 2197 u32 shift; 2198 u64 base_cycles; 2199 u64 offset; 2200 }; 2201 2202 struct pvclock_gtod_data { 2203 seqcount_t seq; 2204 2205 struct pvclock_clock clock; /* extract of a clocksource struct */ 2206 struct pvclock_clock raw_clock; /* extract of a clocksource struct */ 2207 2208 ktime_t offs_boot; 2209 u64 wall_time_sec; 2210 }; 2211 2212 static struct pvclock_gtod_data pvclock_gtod_data; 2213 2214 static void update_pvclock_gtod(struct timekeeper *tk) 2215 { 2216 struct pvclock_gtod_data *vdata = &pvclock_gtod_data; 2217 2218 write_seqcount_begin(&vdata->seq); 2219 2220 /* copy pvclock gtod data */ 2221 vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode; 2222 vdata->clock.cycle_last = tk->tkr_mono.cycle_last; 2223 vdata->clock.mask = tk->tkr_mono.mask; 2224 vdata->clock.mult = tk->tkr_mono.mult; 2225 vdata->clock.shift = tk->tkr_mono.shift; 2226 vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec; 2227 vdata->clock.offset = tk->tkr_mono.base; 2228 2229 vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode; 2230 vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last; 2231 vdata->raw_clock.mask = tk->tkr_raw.mask; 2232 vdata->raw_clock.mult = tk->tkr_raw.mult; 2233 vdata->raw_clock.shift = tk->tkr_raw.shift; 2234 vdata->raw_clock.base_cycles = tk->tkr_raw.xtime_nsec; 2235 vdata->raw_clock.offset = tk->tkr_raw.base; 2236 2237 vdata->wall_time_sec = tk->xtime_sec; 2238 2239 vdata->offs_boot = tk->offs_boot; 2240 2241 write_seqcount_end(&vdata->seq); 2242 } 2243 2244 static s64 get_kvmclock_base_ns(void) 2245 { 2246 /* Count up from boot time, but with the frequency of the raw clock. */ 2247 return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot)); 2248 } 2249 #else 2250 static s64 get_kvmclock_base_ns(void) 2251 { 2252 /* Master clock not used, so we can just use CLOCK_BOOTTIME. */ 2253 return ktime_get_boottime_ns(); 2254 } 2255 #endif 2256 2257 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs) 2258 { 2259 int version; 2260 int r; 2261 struct pvclock_wall_clock wc; 2262 u32 wc_sec_hi; 2263 u64 wall_nsec; 2264 2265 if (!wall_clock) 2266 return; 2267 2268 r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version)); 2269 if (r) 2270 return; 2271 2272 if (version & 1) 2273 ++version; /* first time write, random junk */ 2274 2275 ++version; 2276 2277 if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version))) 2278 return; 2279 2280 /* 2281 * The guest calculates current wall clock time by adding 2282 * system time (updated by kvm_guest_time_update below) to the 2283 * wall clock specified here. We do the reverse here. 2284 */ 2285 wall_nsec = ktime_get_real_ns() - get_kvmclock_ns(kvm); 2286 2287 wc.nsec = do_div(wall_nsec, 1000000000); 2288 wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */ 2289 wc.version = version; 2290 2291 kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc)); 2292 2293 if (sec_hi_ofs) { 2294 wc_sec_hi = wall_nsec >> 32; 2295 kvm_write_guest(kvm, wall_clock + sec_hi_ofs, 2296 &wc_sec_hi, sizeof(wc_sec_hi)); 2297 } 2298 2299 version++; 2300 kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); 2301 } 2302 2303 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time, 2304 bool old_msr, bool host_initiated) 2305 { 2306 struct kvm_arch *ka = &vcpu->kvm->arch; 2307 2308 if (vcpu->vcpu_id == 0 && !host_initiated) { 2309 if (ka->boot_vcpu_runs_old_kvmclock != old_msr) 2310 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 2311 2312 ka->boot_vcpu_runs_old_kvmclock = old_msr; 2313 } 2314 2315 vcpu->arch.time = system_time; 2316 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); 2317 2318 /* we verify if the enable bit is set... */ 2319 if (system_time & 1) 2320 kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL, 2321 sizeof(struct pvclock_vcpu_time_info)); 2322 else 2323 kvm_gpc_deactivate(&vcpu->arch.pv_time); 2324 2325 return; 2326 } 2327 2328 static uint32_t div_frac(uint32_t dividend, uint32_t divisor) 2329 { 2330 do_shl32_div32(dividend, divisor); 2331 return dividend; 2332 } 2333 2334 static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz, 2335 s8 *pshift, u32 *pmultiplier) 2336 { 2337 uint64_t scaled64; 2338 int32_t shift = 0; 2339 uint64_t tps64; 2340 uint32_t tps32; 2341 2342 tps64 = base_hz; 2343 scaled64 = scaled_hz; 2344 while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) { 2345 tps64 >>= 1; 2346 shift--; 2347 } 2348 2349 tps32 = (uint32_t)tps64; 2350 while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) { 2351 if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000) 2352 scaled64 >>= 1; 2353 else 2354 tps32 <<= 1; 2355 shift++; 2356 } 2357 2358 *pshift = shift; 2359 *pmultiplier = div_frac(scaled64, tps32); 2360 } 2361 2362 #ifdef CONFIG_X86_64 2363 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0); 2364 #endif 2365 2366 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz); 2367 static unsigned long max_tsc_khz; 2368 2369 static u32 adjust_tsc_khz(u32 khz, s32 ppm) 2370 { 2371 u64 v = (u64)khz * (1000000 + ppm); 2372 do_div(v, 1000000); 2373 return v; 2374 } 2375 2376 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier); 2377 2378 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale) 2379 { 2380 u64 ratio; 2381 2382 /* Guest TSC same frequency as host TSC? */ 2383 if (!scale) { 2384 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); 2385 return 0; 2386 } 2387 2388 /* TSC scaling supported? */ 2389 if (!kvm_caps.has_tsc_control) { 2390 if (user_tsc_khz > tsc_khz) { 2391 vcpu->arch.tsc_catchup = 1; 2392 vcpu->arch.tsc_always_catchup = 1; 2393 return 0; 2394 } else { 2395 pr_warn_ratelimited("user requested TSC rate below hardware speed\n"); 2396 return -1; 2397 } 2398 } 2399 2400 /* TSC scaling required - calculate ratio */ 2401 ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits, 2402 user_tsc_khz, tsc_khz); 2403 2404 if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) { 2405 pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n", 2406 user_tsc_khz); 2407 return -1; 2408 } 2409 2410 kvm_vcpu_write_tsc_multiplier(vcpu, ratio); 2411 return 0; 2412 } 2413 2414 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) 2415 { 2416 u32 thresh_lo, thresh_hi; 2417 int use_scaling = 0; 2418 2419 /* tsc_khz can be zero if TSC calibration fails */ 2420 if (user_tsc_khz == 0) { 2421 /* set tsc_scaling_ratio to a safe value */ 2422 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio); 2423 return -1; 2424 } 2425 2426 /* Compute a scale to convert nanoseconds in TSC cycles */ 2427 kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC, 2428 &vcpu->arch.virtual_tsc_shift, 2429 &vcpu->arch.virtual_tsc_mult); 2430 vcpu->arch.virtual_tsc_khz = user_tsc_khz; 2431 2432 /* 2433 * Compute the variation in TSC rate which is acceptable 2434 * within the range of tolerance and decide if the 2435 * rate being applied is within that bounds of the hardware 2436 * rate. If so, no scaling or compensation need be done. 2437 */ 2438 thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm); 2439 thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm); 2440 if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) { 2441 pr_debug("requested TSC rate %u falls outside tolerance [%u,%u]\n", 2442 user_tsc_khz, thresh_lo, thresh_hi); 2443 use_scaling = 1; 2444 } 2445 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling); 2446 } 2447 2448 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) 2449 { 2450 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, 2451 vcpu->arch.virtual_tsc_mult, 2452 vcpu->arch.virtual_tsc_shift); 2453 tsc += vcpu->arch.this_tsc_write; 2454 return tsc; 2455 } 2456 2457 #ifdef CONFIG_X86_64 2458 static inline int gtod_is_based_on_tsc(int mode) 2459 { 2460 return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK; 2461 } 2462 #endif 2463 2464 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) 2465 { 2466 #ifdef CONFIG_X86_64 2467 bool vcpus_matched; 2468 struct kvm_arch *ka = &vcpu->kvm->arch; 2469 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 2470 2471 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == 2472 atomic_read(&vcpu->kvm->online_vcpus)); 2473 2474 /* 2475 * Once the masterclock is enabled, always perform request in 2476 * order to update it. 2477 * 2478 * In order to enable masterclock, the host clocksource must be TSC 2479 * and the vcpus need to have matched TSCs. When that happens, 2480 * perform request to enable masterclock. 2481 */ 2482 if (ka->use_master_clock || 2483 (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched)) 2484 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 2485 2486 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc, 2487 atomic_read(&vcpu->kvm->online_vcpus), 2488 ka->use_master_clock, gtod->clock.vclock_mode); 2489 #endif 2490 } 2491 2492 /* 2493 * Multiply tsc by a fixed point number represented by ratio. 2494 * 2495 * The most significant 64-N bits (mult) of ratio represent the 2496 * integral part of the fixed point number; the remaining N bits 2497 * (frac) represent the fractional part, ie. ratio represents a fixed 2498 * point number (mult + frac * 2^(-N)). 2499 * 2500 * N equals to kvm_caps.tsc_scaling_ratio_frac_bits. 2501 */ 2502 static inline u64 __scale_tsc(u64 ratio, u64 tsc) 2503 { 2504 return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits); 2505 } 2506 2507 u64 kvm_scale_tsc(u64 tsc, u64 ratio) 2508 { 2509 u64 _tsc = tsc; 2510 2511 if (ratio != kvm_caps.default_tsc_scaling_ratio) 2512 _tsc = __scale_tsc(ratio, tsc); 2513 2514 return _tsc; 2515 } 2516 2517 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) 2518 { 2519 u64 tsc; 2520 2521 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio); 2522 2523 return target_tsc - tsc; 2524 } 2525 2526 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) 2527 { 2528 return vcpu->arch.l1_tsc_offset + 2529 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); 2530 } 2531 EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); 2532 2533 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier) 2534 { 2535 u64 nested_offset; 2536 2537 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) 2538 nested_offset = l1_offset; 2539 else 2540 nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier, 2541 kvm_caps.tsc_scaling_ratio_frac_bits); 2542 2543 nested_offset += l2_offset; 2544 return nested_offset; 2545 } 2546 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset); 2547 2548 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) 2549 { 2550 if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio) 2551 return mul_u64_u64_shr(l1_multiplier, l2_multiplier, 2552 kvm_caps.tsc_scaling_ratio_frac_bits); 2553 2554 return l1_multiplier; 2555 } 2556 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier); 2557 2558 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset) 2559 { 2560 trace_kvm_write_tsc_offset(vcpu->vcpu_id, 2561 vcpu->arch.l1_tsc_offset, 2562 l1_offset); 2563 2564 vcpu->arch.l1_tsc_offset = l1_offset; 2565 2566 /* 2567 * If we are here because L1 chose not to trap WRMSR to TSC then 2568 * according to the spec this should set L1's TSC (as opposed to 2569 * setting L1's offset for L2). 2570 */ 2571 if (is_guest_mode(vcpu)) 2572 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 2573 l1_offset, 2574 static_call(kvm_x86_get_l2_tsc_offset)(vcpu), 2575 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu)); 2576 else 2577 vcpu->arch.tsc_offset = l1_offset; 2578 2579 static_call(kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset); 2580 } 2581 2582 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier) 2583 { 2584 vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier; 2585 2586 /* Userspace is changing the multiplier while L2 is active */ 2587 if (is_guest_mode(vcpu)) 2588 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 2589 l1_multiplier, 2590 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu)); 2591 else 2592 vcpu->arch.tsc_scaling_ratio = l1_multiplier; 2593 2594 if (kvm_caps.has_tsc_control) 2595 static_call(kvm_x86_write_tsc_multiplier)( 2596 vcpu, vcpu->arch.tsc_scaling_ratio); 2597 } 2598 2599 static inline bool kvm_check_tsc_unstable(void) 2600 { 2601 #ifdef CONFIG_X86_64 2602 /* 2603 * TSC is marked unstable when we're running on Hyper-V, 2604 * 'TSC page' clocksource is good. 2605 */ 2606 if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK) 2607 return false; 2608 #endif 2609 return check_tsc_unstable(); 2610 } 2611 2612 /* 2613 * Infers attempts to synchronize the guest's tsc from host writes. Sets the 2614 * offset for the vcpu and tracks the TSC matching generation that the vcpu 2615 * participates in. 2616 */ 2617 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, 2618 u64 ns, bool matched) 2619 { 2620 struct kvm *kvm = vcpu->kvm; 2621 2622 lockdep_assert_held(&kvm->arch.tsc_write_lock); 2623 2624 /* 2625 * We also track th most recent recorded KHZ, write and time to 2626 * allow the matching interval to be extended at each write. 2627 */ 2628 kvm->arch.last_tsc_nsec = ns; 2629 kvm->arch.last_tsc_write = tsc; 2630 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; 2631 kvm->arch.last_tsc_offset = offset; 2632 2633 vcpu->arch.last_guest_tsc = tsc; 2634 2635 kvm_vcpu_write_tsc_offset(vcpu, offset); 2636 2637 if (!matched) { 2638 /* 2639 * We split periods of matched TSC writes into generations. 2640 * For each generation, we track the original measured 2641 * nanosecond time, offset, and write, so if TSCs are in 2642 * sync, we can match exact offset, and if not, we can match 2643 * exact software computation in compute_guest_tsc() 2644 * 2645 * These values are tracked in kvm->arch.cur_xxx variables. 2646 */ 2647 kvm->arch.cur_tsc_generation++; 2648 kvm->arch.cur_tsc_nsec = ns; 2649 kvm->arch.cur_tsc_write = tsc; 2650 kvm->arch.cur_tsc_offset = offset; 2651 kvm->arch.nr_vcpus_matched_tsc = 0; 2652 } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) { 2653 kvm->arch.nr_vcpus_matched_tsc++; 2654 } 2655 2656 /* Keep track of which generation this VCPU has synchronized to */ 2657 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation; 2658 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec; 2659 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write; 2660 2661 kvm_track_tsc_matching(vcpu); 2662 } 2663 2664 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data) 2665 { 2666 struct kvm *kvm = vcpu->kvm; 2667 u64 offset, ns, elapsed; 2668 unsigned long flags; 2669 bool matched = false; 2670 bool synchronizing = false; 2671 2672 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 2673 offset = kvm_compute_l1_tsc_offset(vcpu, data); 2674 ns = get_kvmclock_base_ns(); 2675 elapsed = ns - kvm->arch.last_tsc_nsec; 2676 2677 if (vcpu->arch.virtual_tsc_khz) { 2678 if (data == 0) { 2679 /* 2680 * detection of vcpu initialization -- need to sync 2681 * with other vCPUs. This particularly helps to keep 2682 * kvm_clock stable after CPU hotplug 2683 */ 2684 synchronizing = true; 2685 } else { 2686 u64 tsc_exp = kvm->arch.last_tsc_write + 2687 nsec_to_cycles(vcpu, elapsed); 2688 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL; 2689 /* 2690 * Special case: TSC write with a small delta (1 second) 2691 * of virtual cycle time against real time is 2692 * interpreted as an attempt to synchronize the CPU. 2693 */ 2694 synchronizing = data < tsc_exp + tsc_hz && 2695 data + tsc_hz > tsc_exp; 2696 } 2697 } 2698 2699 /* 2700 * For a reliable TSC, we can match TSC offsets, and for an unstable 2701 * TSC, we add elapsed time in this computation. We could let the 2702 * compensation code attempt to catch up if we fall behind, but 2703 * it's better to try to match offsets from the beginning. 2704 */ 2705 if (synchronizing && 2706 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) { 2707 if (!kvm_check_tsc_unstable()) { 2708 offset = kvm->arch.cur_tsc_offset; 2709 } else { 2710 u64 delta = nsec_to_cycles(vcpu, elapsed); 2711 data += delta; 2712 offset = kvm_compute_l1_tsc_offset(vcpu, data); 2713 } 2714 matched = true; 2715 } 2716 2717 __kvm_synchronize_tsc(vcpu, offset, data, ns, matched); 2718 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 2719 } 2720 2721 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, 2722 s64 adjustment) 2723 { 2724 u64 tsc_offset = vcpu->arch.l1_tsc_offset; 2725 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment); 2726 } 2727 2728 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) 2729 { 2730 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio) 2731 WARN_ON(adjustment < 0); 2732 adjustment = kvm_scale_tsc((u64) adjustment, 2733 vcpu->arch.l1_tsc_scaling_ratio); 2734 adjust_tsc_offset_guest(vcpu, adjustment); 2735 } 2736 2737 #ifdef CONFIG_X86_64 2738 2739 static u64 read_tsc(void) 2740 { 2741 u64 ret = (u64)rdtsc_ordered(); 2742 u64 last = pvclock_gtod_data.clock.cycle_last; 2743 2744 if (likely(ret >= last)) 2745 return ret; 2746 2747 /* 2748 * GCC likes to generate cmov here, but this branch is extremely 2749 * predictable (it's just a function of time and the likely is 2750 * very likely) and there's a data dependence, so force GCC 2751 * to generate a branch instead. I don't barrier() because 2752 * we don't actually need a barrier, and if this function 2753 * ever gets inlined it will generate worse code. 2754 */ 2755 asm volatile (""); 2756 return last; 2757 } 2758 2759 static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, 2760 int *mode) 2761 { 2762 long v; 2763 u64 tsc_pg_val; 2764 2765 switch (clock->vclock_mode) { 2766 case VDSO_CLOCKMODE_HVCLOCK: 2767 tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(), 2768 tsc_timestamp); 2769 if (tsc_pg_val != U64_MAX) { 2770 /* TSC page valid */ 2771 *mode = VDSO_CLOCKMODE_HVCLOCK; 2772 v = (tsc_pg_val - clock->cycle_last) & 2773 clock->mask; 2774 } else { 2775 /* TSC page invalid */ 2776 *mode = VDSO_CLOCKMODE_NONE; 2777 } 2778 break; 2779 case VDSO_CLOCKMODE_TSC: 2780 *mode = VDSO_CLOCKMODE_TSC; 2781 *tsc_timestamp = read_tsc(); 2782 v = (*tsc_timestamp - clock->cycle_last) & 2783 clock->mask; 2784 break; 2785 default: 2786 *mode = VDSO_CLOCKMODE_NONE; 2787 } 2788 2789 if (*mode == VDSO_CLOCKMODE_NONE) 2790 *tsc_timestamp = v = 0; 2791 2792 return v * clock->mult; 2793 } 2794 2795 static int do_monotonic_raw(s64 *t, u64 *tsc_timestamp) 2796 { 2797 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 2798 unsigned long seq; 2799 int mode; 2800 u64 ns; 2801 2802 do { 2803 seq = read_seqcount_begin(>od->seq); 2804 ns = gtod->raw_clock.base_cycles; 2805 ns += vgettsc(>od->raw_clock, tsc_timestamp, &mode); 2806 ns >>= gtod->raw_clock.shift; 2807 ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot)); 2808 } while (unlikely(read_seqcount_retry(>od->seq, seq))); 2809 *t = ns; 2810 2811 return mode; 2812 } 2813 2814 static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) 2815 { 2816 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 2817 unsigned long seq; 2818 int mode; 2819 u64 ns; 2820 2821 do { 2822 seq = read_seqcount_begin(>od->seq); 2823 ts->tv_sec = gtod->wall_time_sec; 2824 ns = gtod->clock.base_cycles; 2825 ns += vgettsc(>od->clock, tsc_timestamp, &mode); 2826 ns >>= gtod->clock.shift; 2827 } while (unlikely(read_seqcount_retry(>od->seq, seq))); 2828 2829 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 2830 ts->tv_nsec = ns; 2831 2832 return mode; 2833 } 2834 2835 /* returns true if host is using TSC based clocksource */ 2836 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) 2837 { 2838 /* checked again under seqlock below */ 2839 if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) 2840 return false; 2841 2842 return gtod_is_based_on_tsc(do_monotonic_raw(kernel_ns, 2843 tsc_timestamp)); 2844 } 2845 2846 /* returns true if host is using TSC based clocksource */ 2847 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, 2848 u64 *tsc_timestamp) 2849 { 2850 /* checked again under seqlock below */ 2851 if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) 2852 return false; 2853 2854 return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp)); 2855 } 2856 #endif 2857 2858 /* 2859 * 2860 * Assuming a stable TSC across physical CPUS, and a stable TSC 2861 * across virtual CPUs, the following condition is possible. 2862 * Each numbered line represents an event visible to both 2863 * CPUs at the next numbered event. 2864 * 2865 * "timespecX" represents host monotonic time. "tscX" represents 2866 * RDTSC value. 2867 * 2868 * VCPU0 on CPU0 | VCPU1 on CPU1 2869 * 2870 * 1. read timespec0,tsc0 2871 * 2. | timespec1 = timespec0 + N 2872 * | tsc1 = tsc0 + M 2873 * 3. transition to guest | transition to guest 2874 * 4. ret0 = timespec0 + (rdtsc - tsc0) | 2875 * 5. | ret1 = timespec1 + (rdtsc - tsc1) 2876 * | ret1 = timespec0 + N + (rdtsc - (tsc0 + M)) 2877 * 2878 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity: 2879 * 2880 * - ret0 < ret1 2881 * - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M)) 2882 * ... 2883 * - 0 < N - M => M < N 2884 * 2885 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not 2886 * always the case (the difference between two distinct xtime instances 2887 * might be smaller then the difference between corresponding TSC reads, 2888 * when updating guest vcpus pvclock areas). 2889 * 2890 * To avoid that problem, do not allow visibility of distinct 2891 * system_timestamp/tsc_timestamp values simultaneously: use a master 2892 * copy of host monotonic time values. Update that master copy 2893 * in lockstep. 2894 * 2895 * Rely on synchronization of host TSCs and guest TSCs for monotonicity. 2896 * 2897 */ 2898 2899 static void pvclock_update_vm_gtod_copy(struct kvm *kvm) 2900 { 2901 #ifdef CONFIG_X86_64 2902 struct kvm_arch *ka = &kvm->arch; 2903 int vclock_mode; 2904 bool host_tsc_clocksource, vcpus_matched; 2905 2906 lockdep_assert_held(&kvm->arch.tsc_write_lock); 2907 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 == 2908 atomic_read(&kvm->online_vcpus)); 2909 2910 /* 2911 * If the host uses TSC clock, then passthrough TSC as stable 2912 * to the guest. 2913 */ 2914 host_tsc_clocksource = kvm_get_time_and_clockread( 2915 &ka->master_kernel_ns, 2916 &ka->master_cycle_now); 2917 2918 ka->use_master_clock = host_tsc_clocksource && vcpus_matched 2919 && !ka->backwards_tsc_observed 2920 && !ka->boot_vcpu_runs_old_kvmclock; 2921 2922 if (ka->use_master_clock) 2923 atomic_set(&kvm_guest_has_master_clock, 1); 2924 2925 vclock_mode = pvclock_gtod_data.clock.vclock_mode; 2926 trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, 2927 vcpus_matched); 2928 #endif 2929 } 2930 2931 static void kvm_make_mclock_inprogress_request(struct kvm *kvm) 2932 { 2933 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS); 2934 } 2935 2936 static void __kvm_start_pvclock_update(struct kvm *kvm) 2937 { 2938 raw_spin_lock_irq(&kvm->arch.tsc_write_lock); 2939 write_seqcount_begin(&kvm->arch.pvclock_sc); 2940 } 2941 2942 static void kvm_start_pvclock_update(struct kvm *kvm) 2943 { 2944 kvm_make_mclock_inprogress_request(kvm); 2945 2946 /* no guest entries from this point */ 2947 __kvm_start_pvclock_update(kvm); 2948 } 2949 2950 static void kvm_end_pvclock_update(struct kvm *kvm) 2951 { 2952 struct kvm_arch *ka = &kvm->arch; 2953 struct kvm_vcpu *vcpu; 2954 unsigned long i; 2955 2956 write_seqcount_end(&ka->pvclock_sc); 2957 raw_spin_unlock_irq(&ka->tsc_write_lock); 2958 kvm_for_each_vcpu(i, vcpu, kvm) 2959 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2960 2961 /* guest entries allowed */ 2962 kvm_for_each_vcpu(i, vcpu, kvm) 2963 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu); 2964 } 2965 2966 static void kvm_update_masterclock(struct kvm *kvm) 2967 { 2968 kvm_hv_request_tsc_page_update(kvm); 2969 kvm_start_pvclock_update(kvm); 2970 pvclock_update_vm_gtod_copy(kvm); 2971 kvm_end_pvclock_update(kvm); 2972 } 2973 2974 /* 2975 * Use the kernel's tsc_khz directly if the TSC is constant, otherwise use KVM's 2976 * per-CPU value (which may be zero if a CPU is going offline). Note, tsc_khz 2977 * can change during boot even if the TSC is constant, as it's possible for KVM 2978 * to be loaded before TSC calibration completes. Ideally, KVM would get a 2979 * notification when calibration completes, but practically speaking calibration 2980 * will complete before userspace is alive enough to create VMs. 2981 */ 2982 static unsigned long get_cpu_tsc_khz(void) 2983 { 2984 if (static_cpu_has(X86_FEATURE_CONSTANT_TSC)) 2985 return tsc_khz; 2986 else 2987 return __this_cpu_read(cpu_tsc_khz); 2988 } 2989 2990 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ 2991 static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) 2992 { 2993 struct kvm_arch *ka = &kvm->arch; 2994 struct pvclock_vcpu_time_info hv_clock; 2995 2996 /* both __this_cpu_read() and rdtsc() should be on the same cpu */ 2997 get_cpu(); 2998 2999 data->flags = 0; 3000 if (ka->use_master_clock && 3001 (static_cpu_has(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) { 3002 #ifdef CONFIG_X86_64 3003 struct timespec64 ts; 3004 3005 if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { 3006 data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; 3007 data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; 3008 } else 3009 #endif 3010 data->host_tsc = rdtsc(); 3011 3012 data->flags |= KVM_CLOCK_TSC_STABLE; 3013 hv_clock.tsc_timestamp = ka->master_cycle_now; 3014 hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; 3015 kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL, 3016 &hv_clock.tsc_shift, 3017 &hv_clock.tsc_to_system_mul); 3018 data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); 3019 } else { 3020 data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; 3021 } 3022 3023 put_cpu(); 3024 } 3025 3026 static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) 3027 { 3028 struct kvm_arch *ka = &kvm->arch; 3029 unsigned seq; 3030 3031 do { 3032 seq = read_seqcount_begin(&ka->pvclock_sc); 3033 __get_kvmclock(kvm, data); 3034 } while (read_seqcount_retry(&ka->pvclock_sc, seq)); 3035 } 3036 3037 u64 get_kvmclock_ns(struct kvm *kvm) 3038 { 3039 struct kvm_clock_data data; 3040 3041 get_kvmclock(kvm, &data); 3042 return data.clock; 3043 } 3044 3045 static void kvm_setup_guest_pvclock(struct kvm_vcpu *v, 3046 struct gfn_to_pfn_cache *gpc, 3047 unsigned int offset) 3048 { 3049 struct kvm_vcpu_arch *vcpu = &v->arch; 3050 struct pvclock_vcpu_time_info *guest_hv_clock; 3051 unsigned long flags; 3052 3053 read_lock_irqsave(&gpc->lock, flags); 3054 while (!kvm_gpc_check(gpc, offset + sizeof(*guest_hv_clock))) { 3055 read_unlock_irqrestore(&gpc->lock, flags); 3056 3057 if (kvm_gpc_refresh(gpc, offset + sizeof(*guest_hv_clock))) 3058 return; 3059 3060 read_lock_irqsave(&gpc->lock, flags); 3061 } 3062 3063 guest_hv_clock = (void *)(gpc->khva + offset); 3064 3065 /* 3066 * This VCPU is paused, but it's legal for a guest to read another 3067 * VCPU's kvmclock, so we really have to follow the specification where 3068 * it says that version is odd if data is being modified, and even after 3069 * it is consistent. 3070 */ 3071 3072 guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1; 3073 smp_wmb(); 3074 3075 /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */ 3076 vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED); 3077 3078 if (vcpu->pvclock_set_guest_stopped_request) { 3079 vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED; 3080 vcpu->pvclock_set_guest_stopped_request = false; 3081 } 3082 3083 memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock)); 3084 smp_wmb(); 3085 3086 guest_hv_clock->version = ++vcpu->hv_clock.version; 3087 3088 mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); 3089 read_unlock_irqrestore(&gpc->lock, flags); 3090 3091 trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock); 3092 } 3093 3094 static int kvm_guest_time_update(struct kvm_vcpu *v) 3095 { 3096 unsigned long flags, tgt_tsc_khz; 3097 unsigned seq; 3098 struct kvm_vcpu_arch *vcpu = &v->arch; 3099 struct kvm_arch *ka = &v->kvm->arch; 3100 s64 kernel_ns; 3101 u64 tsc_timestamp, host_tsc; 3102 u8 pvclock_flags; 3103 bool use_master_clock; 3104 3105 kernel_ns = 0; 3106 host_tsc = 0; 3107 3108 /* 3109 * If the host uses TSC clock, then passthrough TSC as stable 3110 * to the guest. 3111 */ 3112 do { 3113 seq = read_seqcount_begin(&ka->pvclock_sc); 3114 use_master_clock = ka->use_master_clock; 3115 if (use_master_clock) { 3116 host_tsc = ka->master_cycle_now; 3117 kernel_ns = ka->master_kernel_ns; 3118 } 3119 } while (read_seqcount_retry(&ka->pvclock_sc, seq)); 3120 3121 /* Keep irq disabled to prevent changes to the clock */ 3122 local_irq_save(flags); 3123 tgt_tsc_khz = get_cpu_tsc_khz(); 3124 if (unlikely(tgt_tsc_khz == 0)) { 3125 local_irq_restore(flags); 3126 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); 3127 return 1; 3128 } 3129 if (!use_master_clock) { 3130 host_tsc = rdtsc(); 3131 kernel_ns = get_kvmclock_base_ns(); 3132 } 3133 3134 tsc_timestamp = kvm_read_l1_tsc(v, host_tsc); 3135 3136 /* 3137 * We may have to catch up the TSC to match elapsed wall clock 3138 * time for two reasons, even if kvmclock is used. 3139 * 1) CPU could have been running below the maximum TSC rate 3140 * 2) Broken TSC compensation resets the base at each VCPU 3141 * entry to avoid unknown leaps of TSC even when running 3142 * again on the same CPU. This may cause apparent elapsed 3143 * time to disappear, and the guest to stand still or run 3144 * very slowly. 3145 */ 3146 if (vcpu->tsc_catchup) { 3147 u64 tsc = compute_guest_tsc(v, kernel_ns); 3148 if (tsc > tsc_timestamp) { 3149 adjust_tsc_offset_guest(v, tsc - tsc_timestamp); 3150 tsc_timestamp = tsc; 3151 } 3152 } 3153 3154 local_irq_restore(flags); 3155 3156 /* With all the info we got, fill in the values */ 3157 3158 if (kvm_caps.has_tsc_control) 3159 tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz, 3160 v->arch.l1_tsc_scaling_ratio); 3161 3162 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) { 3163 kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL, 3164 &vcpu->hv_clock.tsc_shift, 3165 &vcpu->hv_clock.tsc_to_system_mul); 3166 vcpu->hw_tsc_khz = tgt_tsc_khz; 3167 } 3168 3169 vcpu->hv_clock.tsc_timestamp = tsc_timestamp; 3170 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; 3171 vcpu->last_guest_tsc = tsc_timestamp; 3172 3173 /* If the host uses TSC clocksource, then it is stable */ 3174 pvclock_flags = 0; 3175 if (use_master_clock) 3176 pvclock_flags |= PVCLOCK_TSC_STABLE_BIT; 3177 3178 vcpu->hv_clock.flags = pvclock_flags; 3179 3180 if (vcpu->pv_time.active) 3181 kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0); 3182 if (vcpu->xen.vcpu_info_cache.active) 3183 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache, 3184 offsetof(struct compat_vcpu_info, time)); 3185 if (vcpu->xen.vcpu_time_info_cache.active) 3186 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0); 3187 kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock); 3188 return 0; 3189 } 3190 3191 /* 3192 * kvmclock updates which are isolated to a given vcpu, such as 3193 * vcpu->cpu migration, should not allow system_timestamp from 3194 * the rest of the vcpus to remain static. Otherwise ntp frequency 3195 * correction applies to one vcpu's system_timestamp but not 3196 * the others. 3197 * 3198 * So in those cases, request a kvmclock update for all vcpus. 3199 * We need to rate-limit these requests though, as they can 3200 * considerably slow guests that have a large number of vcpus. 3201 * The time for a remote vcpu to update its kvmclock is bound 3202 * by the delay we use to rate-limit the updates. 3203 */ 3204 3205 #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100) 3206 3207 static void kvmclock_update_fn(struct work_struct *work) 3208 { 3209 unsigned long i; 3210 struct delayed_work *dwork = to_delayed_work(work); 3211 struct kvm_arch *ka = container_of(dwork, struct kvm_arch, 3212 kvmclock_update_work); 3213 struct kvm *kvm = container_of(ka, struct kvm, arch); 3214 struct kvm_vcpu *vcpu; 3215 3216 kvm_for_each_vcpu(i, vcpu, kvm) { 3217 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 3218 kvm_vcpu_kick(vcpu); 3219 } 3220 } 3221 3222 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v) 3223 { 3224 struct kvm *kvm = v->kvm; 3225 3226 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); 3227 schedule_delayed_work(&kvm->arch.kvmclock_update_work, 3228 KVMCLOCK_UPDATE_DELAY); 3229 } 3230 3231 #define KVMCLOCK_SYNC_PERIOD (300 * HZ) 3232 3233 static void kvmclock_sync_fn(struct work_struct *work) 3234 { 3235 struct delayed_work *dwork = to_delayed_work(work); 3236 struct kvm_arch *ka = container_of(dwork, struct kvm_arch, 3237 kvmclock_sync_work); 3238 struct kvm *kvm = container_of(ka, struct kvm, arch); 3239 3240 if (!kvmclock_periodic_sync) 3241 return; 3242 3243 schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0); 3244 schedule_delayed_work(&kvm->arch.kvmclock_sync_work, 3245 KVMCLOCK_SYNC_PERIOD); 3246 } 3247 3248 /* These helpers are safe iff @msr is known to be an MCx bank MSR. */ 3249 static bool is_mci_control_msr(u32 msr) 3250 { 3251 return (msr & 3) == 0; 3252 } 3253 static bool is_mci_status_msr(u32 msr) 3254 { 3255 return (msr & 3) == 1; 3256 } 3257 3258 /* 3259 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP. 3260 */ 3261 static bool can_set_mci_status(struct kvm_vcpu *vcpu) 3262 { 3263 /* McStatusWrEn enabled? */ 3264 if (guest_cpuid_is_amd_or_hygon(vcpu)) 3265 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18)); 3266 3267 return false; 3268 } 3269 3270 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 3271 { 3272 u64 mcg_cap = vcpu->arch.mcg_cap; 3273 unsigned bank_num = mcg_cap & 0xff; 3274 u32 msr = msr_info->index; 3275 u64 data = msr_info->data; 3276 u32 offset, last_msr; 3277 3278 switch (msr) { 3279 case MSR_IA32_MCG_STATUS: 3280 vcpu->arch.mcg_status = data; 3281 break; 3282 case MSR_IA32_MCG_CTL: 3283 if (!(mcg_cap & MCG_CTL_P) && 3284 (data || !msr_info->host_initiated)) 3285 return 1; 3286 if (data != 0 && data != ~(u64)0) 3287 return 1; 3288 vcpu->arch.mcg_ctl = data; 3289 break; 3290 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 3291 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; 3292 if (msr > last_msr) 3293 return 1; 3294 3295 if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated)) 3296 return 1; 3297 /* An attempt to write a 1 to a reserved bit raises #GP */ 3298 if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK)) 3299 return 1; 3300 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, 3301 last_msr + 1 - MSR_IA32_MC0_CTL2); 3302 vcpu->arch.mci_ctl2_banks[offset] = data; 3303 break; 3304 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 3305 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; 3306 if (msr > last_msr) 3307 return 1; 3308 3309 /* 3310 * Only 0 or all 1s can be written to IA32_MCi_CTL, all other 3311 * values are architecturally undefined. But, some Linux 3312 * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB 3313 * issue on AMD K8s, allow bit 10 to be clear when setting all 3314 * other bits in order to avoid an uncaught #GP in the guest. 3315 * 3316 * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable, 3317 * single-bit ECC data errors. 3318 */ 3319 if (is_mci_control_msr(msr) && 3320 data != 0 && (data | (1 << 10) | 1) != ~(u64)0) 3321 return 1; 3322 3323 /* 3324 * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR. 3325 * AMD-based CPUs allow non-zero values, but if and only if 3326 * HWCR[McStatusWrEn] is set. 3327 */ 3328 if (!msr_info->host_initiated && is_mci_status_msr(msr) && 3329 data != 0 && !can_set_mci_status(vcpu)) 3330 return 1; 3331 3332 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, 3333 last_msr + 1 - MSR_IA32_MC0_CTL); 3334 vcpu->arch.mce_banks[offset] = data; 3335 break; 3336 default: 3337 return 1; 3338 } 3339 return 0; 3340 } 3341 3342 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) 3343 { 3344 u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; 3345 3346 return (vcpu->arch.apf.msr_en_val & mask) == mask; 3347 } 3348 3349 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) 3350 { 3351 gpa_t gpa = data & ~0x3f; 3352 3353 /* Bits 4:5 are reserved, Should be zero */ 3354 if (data & 0x30) 3355 return 1; 3356 3357 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) && 3358 (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT)) 3359 return 1; 3360 3361 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) && 3362 (data & KVM_ASYNC_PF_DELIVERY_AS_INT)) 3363 return 1; 3364 3365 if (!lapic_in_kernel(vcpu)) 3366 return data ? 1 : 0; 3367 3368 vcpu->arch.apf.msr_en_val = data; 3369 3370 if (!kvm_pv_async_pf_enabled(vcpu)) { 3371 kvm_clear_async_pf_completion_queue(vcpu); 3372 kvm_async_pf_hash_reset(vcpu); 3373 return 0; 3374 } 3375 3376 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, 3377 sizeof(u64))) 3378 return 1; 3379 3380 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); 3381 vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; 3382 3383 kvm_async_pf_wakeup_all(vcpu); 3384 3385 return 0; 3386 } 3387 3388 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) 3389 { 3390 /* Bits 8-63 are reserved */ 3391 if (data >> 8) 3392 return 1; 3393 3394 if (!lapic_in_kernel(vcpu)) 3395 return 1; 3396 3397 vcpu->arch.apf.msr_int_val = data; 3398 3399 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK; 3400 3401 return 0; 3402 } 3403 3404 static void kvmclock_reset(struct kvm_vcpu *vcpu) 3405 { 3406 kvm_gpc_deactivate(&vcpu->arch.pv_time); 3407 vcpu->arch.time = 0; 3408 } 3409 3410 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu) 3411 { 3412 ++vcpu->stat.tlb_flush; 3413 static_call(kvm_x86_flush_tlb_all)(vcpu); 3414 3415 /* Flushing all ASIDs flushes the current ASID... */ 3416 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 3417 } 3418 3419 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) 3420 { 3421 ++vcpu->stat.tlb_flush; 3422 3423 if (!tdp_enabled) { 3424 /* 3425 * A TLB flush on behalf of the guest is equivalent to 3426 * INVPCID(all), toggling CR4.PGE, etc., which requires 3427 * a forced sync of the shadow page tables. Ensure all the 3428 * roots are synced and the guest TLB in hardware is clean. 3429 */ 3430 kvm_mmu_sync_roots(vcpu); 3431 kvm_mmu_sync_prev_roots(vcpu); 3432 } 3433 3434 static_call(kvm_x86_flush_tlb_guest)(vcpu); 3435 3436 /* 3437 * Flushing all "guest" TLB is always a superset of Hyper-V's fine 3438 * grained flushing. 3439 */ 3440 kvm_hv_vcpu_purge_flush_tlb(vcpu); 3441 } 3442 3443 3444 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu) 3445 { 3446 ++vcpu->stat.tlb_flush; 3447 static_call(kvm_x86_flush_tlb_current)(vcpu); 3448 } 3449 3450 /* 3451 * Service "local" TLB flush requests, which are specific to the current MMU 3452 * context. In addition to the generic event handling in vcpu_enter_guest(), 3453 * TLB flushes that are targeted at an MMU context also need to be serviced 3454 * prior before nested VM-Enter/VM-Exit. 3455 */ 3456 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu) 3457 { 3458 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) 3459 kvm_vcpu_flush_tlb_current(vcpu); 3460 3461 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu)) 3462 kvm_vcpu_flush_tlb_guest(vcpu); 3463 } 3464 EXPORT_SYMBOL_GPL(kvm_service_local_tlb_flush_requests); 3465 3466 static void record_steal_time(struct kvm_vcpu *vcpu) 3467 { 3468 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; 3469 struct kvm_steal_time __user *st; 3470 struct kvm_memslots *slots; 3471 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; 3472 u64 steal; 3473 u32 version; 3474 3475 if (kvm_xen_msr_enabled(vcpu->kvm)) { 3476 kvm_xen_runstate_set_running(vcpu); 3477 return; 3478 } 3479 3480 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 3481 return; 3482 3483 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm)) 3484 return; 3485 3486 slots = kvm_memslots(vcpu->kvm); 3487 3488 if (unlikely(slots->generation != ghc->generation || 3489 gpa != ghc->gpa || 3490 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) { 3491 /* We rely on the fact that it fits in a single page. */ 3492 BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS); 3493 3494 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) || 3495 kvm_is_error_hva(ghc->hva) || !ghc->memslot) 3496 return; 3497 } 3498 3499 st = (struct kvm_steal_time __user *)ghc->hva; 3500 /* 3501 * Doing a TLB flush here, on the guest's behalf, can avoid 3502 * expensive IPIs. 3503 */ 3504 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) { 3505 u8 st_preempted = 0; 3506 int err = -EFAULT; 3507 3508 if (!user_access_begin(st, sizeof(*st))) 3509 return; 3510 3511 asm volatile("1: xchgb %0, %2\n" 3512 "xor %1, %1\n" 3513 "2:\n" 3514 _ASM_EXTABLE_UA(1b, 2b) 3515 : "+q" (st_preempted), 3516 "+&r" (err), 3517 "+m" (st->preempted)); 3518 if (err) 3519 goto out; 3520 3521 user_access_end(); 3522 3523 vcpu->arch.st.preempted = 0; 3524 3525 trace_kvm_pv_tlb_flush(vcpu->vcpu_id, 3526 st_preempted & KVM_VCPU_FLUSH_TLB); 3527 if (st_preempted & KVM_VCPU_FLUSH_TLB) 3528 kvm_vcpu_flush_tlb_guest(vcpu); 3529 3530 if (!user_access_begin(st, sizeof(*st))) 3531 goto dirty; 3532 } else { 3533 if (!user_access_begin(st, sizeof(*st))) 3534 return; 3535 3536 unsafe_put_user(0, &st->preempted, out); 3537 vcpu->arch.st.preempted = 0; 3538 } 3539 3540 unsafe_get_user(version, &st->version, out); 3541 if (version & 1) 3542 version += 1; /* first time write, random junk */ 3543 3544 version += 1; 3545 unsafe_put_user(version, &st->version, out); 3546 3547 smp_wmb(); 3548 3549 unsafe_get_user(steal, &st->steal, out); 3550 steal += current->sched_info.run_delay - 3551 vcpu->arch.st.last_steal; 3552 vcpu->arch.st.last_steal = current->sched_info.run_delay; 3553 unsafe_put_user(steal, &st->steal, out); 3554 3555 version += 1; 3556 unsafe_put_user(version, &st->version, out); 3557 3558 out: 3559 user_access_end(); 3560 dirty: 3561 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 3562 } 3563 3564 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 3565 { 3566 u32 msr = msr_info->index; 3567 u64 data = msr_info->data; 3568 3569 if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr) 3570 return kvm_xen_write_hypercall_page(vcpu, data); 3571 3572 switch (msr) { 3573 case MSR_AMD64_NB_CFG: 3574 case MSR_IA32_UCODE_WRITE: 3575 case MSR_VM_HSAVE_PA: 3576 case MSR_AMD64_PATCH_LOADER: 3577 case MSR_AMD64_BU_CFG2: 3578 case MSR_AMD64_DC_CFG: 3579 case MSR_F15H_EX_CFG: 3580 break; 3581 3582 case MSR_IA32_UCODE_REV: 3583 if (msr_info->host_initiated) 3584 vcpu->arch.microcode_version = data; 3585 break; 3586 case MSR_IA32_ARCH_CAPABILITIES: 3587 if (!msr_info->host_initiated) 3588 return 1; 3589 vcpu->arch.arch_capabilities = data; 3590 break; 3591 case MSR_IA32_PERF_CAPABILITIES: 3592 if (!msr_info->host_initiated) 3593 return 1; 3594 if (data & ~kvm_caps.supported_perf_cap) 3595 return 1; 3596 3597 vcpu->arch.perf_capabilities = data; 3598 kvm_pmu_refresh(vcpu); 3599 return 0; 3600 case MSR_EFER: 3601 return set_efer(vcpu, msr_info); 3602 case MSR_K7_HWCR: 3603 data &= ~(u64)0x40; /* ignore flush filter disable */ 3604 data &= ~(u64)0x100; /* ignore ignne emulation enable */ 3605 data &= ~(u64)0x8; /* ignore TLB cache disable */ 3606 3607 /* Handle McStatusWrEn */ 3608 if (data == BIT_ULL(18)) { 3609 vcpu->arch.msr_hwcr = data; 3610 } else if (data != 0) { 3611 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 3612 return 1; 3613 } 3614 break; 3615 case MSR_FAM10H_MMIO_CONF_BASE: 3616 if (data != 0) { 3617 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 3618 return 1; 3619 } 3620 break; 3621 case 0x200 ... MSR_IA32_MC0_CTL2 - 1: 3622 case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff: 3623 return kvm_mtrr_set_msr(vcpu, msr, data); 3624 case MSR_IA32_APICBASE: 3625 return kvm_set_apic_base(vcpu, msr_info); 3626 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: 3627 return kvm_x2apic_msr_write(vcpu, msr, data); 3628 case MSR_IA32_TSC_DEADLINE: 3629 kvm_set_lapic_tscdeadline_msr(vcpu, data); 3630 break; 3631 case MSR_IA32_TSC_ADJUST: 3632 if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) { 3633 if (!msr_info->host_initiated) { 3634 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr; 3635 adjust_tsc_offset_guest(vcpu, adj); 3636 /* Before back to guest, tsc_timestamp must be adjusted 3637 * as well, otherwise guest's percpu pvclock time could jump. 3638 */ 3639 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 3640 } 3641 vcpu->arch.ia32_tsc_adjust_msr = data; 3642 } 3643 break; 3644 case MSR_IA32_MISC_ENABLE: { 3645 u64 old_val = vcpu->arch.ia32_misc_enable_msr; 3646 3647 if (!msr_info->host_initiated) { 3648 /* RO bits */ 3649 if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK) 3650 return 1; 3651 3652 /* R bits, i.e. writes are ignored, but don't fault. */ 3653 data = data & ~MSR_IA32_MISC_ENABLE_EMON; 3654 data |= old_val & MSR_IA32_MISC_ENABLE_EMON; 3655 } 3656 3657 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) && 3658 ((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) { 3659 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3)) 3660 return 1; 3661 vcpu->arch.ia32_misc_enable_msr = data; 3662 kvm_update_cpuid_runtime(vcpu); 3663 } else { 3664 vcpu->arch.ia32_misc_enable_msr = data; 3665 } 3666 break; 3667 } 3668 case MSR_IA32_SMBASE: 3669 if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) 3670 return 1; 3671 vcpu->arch.smbase = data; 3672 break; 3673 case MSR_IA32_POWER_CTL: 3674 vcpu->arch.msr_ia32_power_ctl = data; 3675 break; 3676 case MSR_IA32_TSC: 3677 if (msr_info->host_initiated) { 3678 kvm_synchronize_tsc(vcpu, data); 3679 } else { 3680 u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; 3681 adjust_tsc_offset_guest(vcpu, adj); 3682 vcpu->arch.ia32_tsc_adjust_msr += adj; 3683 } 3684 break; 3685 case MSR_IA32_XSS: 3686 if (!msr_info->host_initiated && 3687 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) 3688 return 1; 3689 /* 3690 * KVM supports exposing PT to the guest, but does not support 3691 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than 3692 * XSAVES/XRSTORS to save/restore PT MSRs. 3693 */ 3694 if (data & ~kvm_caps.supported_xss) 3695 return 1; 3696 vcpu->arch.ia32_xss = data; 3697 kvm_update_cpuid_runtime(vcpu); 3698 break; 3699 case MSR_SMI_COUNT: 3700 if (!msr_info->host_initiated) 3701 return 1; 3702 vcpu->arch.smi_count = data; 3703 break; 3704 case MSR_KVM_WALL_CLOCK_NEW: 3705 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 3706 return 1; 3707 3708 vcpu->kvm->arch.wall_clock = data; 3709 kvm_write_wall_clock(vcpu->kvm, data, 0); 3710 break; 3711 case MSR_KVM_WALL_CLOCK: 3712 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 3713 return 1; 3714 3715 vcpu->kvm->arch.wall_clock = data; 3716 kvm_write_wall_clock(vcpu->kvm, data, 0); 3717 break; 3718 case MSR_KVM_SYSTEM_TIME_NEW: 3719 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 3720 return 1; 3721 3722 kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); 3723 break; 3724 case MSR_KVM_SYSTEM_TIME: 3725 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 3726 return 1; 3727 3728 kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); 3729 break; 3730 case MSR_KVM_ASYNC_PF_EN: 3731 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) 3732 return 1; 3733 3734 if (kvm_pv_enable_async_pf(vcpu, data)) 3735 return 1; 3736 break; 3737 case MSR_KVM_ASYNC_PF_INT: 3738 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 3739 return 1; 3740 3741 if (kvm_pv_enable_async_pf_int(vcpu, data)) 3742 return 1; 3743 break; 3744 case MSR_KVM_ASYNC_PF_ACK: 3745 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 3746 return 1; 3747 if (data & 0x1) { 3748 vcpu->arch.apf.pageready_pending = false; 3749 kvm_check_async_pf_completion(vcpu); 3750 } 3751 break; 3752 case MSR_KVM_STEAL_TIME: 3753 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) 3754 return 1; 3755 3756 if (unlikely(!sched_info_on())) 3757 return 1; 3758 3759 if (data & KVM_STEAL_RESERVED_MASK) 3760 return 1; 3761 3762 vcpu->arch.st.msr_val = data; 3763 3764 if (!(data & KVM_MSR_ENABLED)) 3765 break; 3766 3767 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 3768 3769 break; 3770 case MSR_KVM_PV_EOI_EN: 3771 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) 3772 return 1; 3773 3774 if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) 3775 return 1; 3776 break; 3777 3778 case MSR_KVM_POLL_CONTROL: 3779 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) 3780 return 1; 3781 3782 /* only enable bit supported */ 3783 if (data & (-1ULL << 1)) 3784 return 1; 3785 3786 vcpu->arch.msr_kvm_poll_control = data; 3787 break; 3788 3789 case MSR_IA32_MCG_CTL: 3790 case MSR_IA32_MCG_STATUS: 3791 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 3792 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 3793 return set_msr_mce(vcpu, msr_info); 3794 3795 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: 3796 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: 3797 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: 3798 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: 3799 if (kvm_pmu_is_valid_msr(vcpu, msr)) 3800 return kvm_pmu_set_msr(vcpu, msr_info); 3801 3802 if (data) 3803 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 3804 break; 3805 case MSR_K7_CLK_CTL: 3806 /* 3807 * Ignore all writes to this no longer documented MSR. 3808 * Writes are only relevant for old K7 processors, 3809 * all pre-dating SVM, but a recommended workaround from 3810 * AMD for these chips. It is possible to specify the 3811 * affected processor models on the command line, hence 3812 * the need to ignore the workaround. 3813 */ 3814 break; 3815 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 3816 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 3817 case HV_X64_MSR_SYNDBG_OPTIONS: 3818 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 3819 case HV_X64_MSR_CRASH_CTL: 3820 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: 3821 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 3822 case HV_X64_MSR_TSC_EMULATION_CONTROL: 3823 case HV_X64_MSR_TSC_EMULATION_STATUS: 3824 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 3825 return kvm_hv_set_msr_common(vcpu, msr, data, 3826 msr_info->host_initiated); 3827 case MSR_IA32_BBL_CR_CTL3: 3828 /* Drop writes to this legacy MSR -- see rdmsr 3829 * counterpart for further detail. 3830 */ 3831 kvm_pr_unimpl_wrmsr(vcpu, msr, data); 3832 break; 3833 case MSR_AMD64_OSVW_ID_LENGTH: 3834 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) 3835 return 1; 3836 vcpu->arch.osvw.length = data; 3837 break; 3838 case MSR_AMD64_OSVW_STATUS: 3839 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) 3840 return 1; 3841 vcpu->arch.osvw.status = data; 3842 break; 3843 case MSR_PLATFORM_INFO: 3844 if (!msr_info->host_initiated || 3845 (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) && 3846 cpuid_fault_enabled(vcpu))) 3847 return 1; 3848 vcpu->arch.msr_platform_info = data; 3849 break; 3850 case MSR_MISC_FEATURES_ENABLES: 3851 if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT || 3852 (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT && 3853 !supports_cpuid_fault(vcpu))) 3854 return 1; 3855 vcpu->arch.msr_misc_features_enables = data; 3856 break; 3857 #ifdef CONFIG_X86_64 3858 case MSR_IA32_XFD: 3859 if (!msr_info->host_initiated && 3860 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) 3861 return 1; 3862 3863 if (data & ~kvm_guest_supported_xfd(vcpu)) 3864 return 1; 3865 3866 fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data); 3867 break; 3868 case MSR_IA32_XFD_ERR: 3869 if (!msr_info->host_initiated && 3870 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) 3871 return 1; 3872 3873 if (data & ~kvm_guest_supported_xfd(vcpu)) 3874 return 1; 3875 3876 vcpu->arch.guest_fpu.xfd_err = data; 3877 break; 3878 #endif 3879 case MSR_IA32_PEBS_ENABLE: 3880 case MSR_IA32_DS_AREA: 3881 case MSR_PEBS_DATA_CFG: 3882 case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5: 3883 if (kvm_pmu_is_valid_msr(vcpu, msr)) 3884 return kvm_pmu_set_msr(vcpu, msr_info); 3885 /* 3886 * Userspace is allowed to write '0' to MSRs that KVM reports 3887 * as to-be-saved, even if an MSRs isn't fully supported. 3888 */ 3889 return !msr_info->host_initiated || data; 3890 default: 3891 if (kvm_pmu_is_valid_msr(vcpu, msr)) 3892 return kvm_pmu_set_msr(vcpu, msr_info); 3893 return KVM_MSR_RET_INVALID; 3894 } 3895 return 0; 3896 } 3897 EXPORT_SYMBOL_GPL(kvm_set_msr_common); 3898 3899 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) 3900 { 3901 u64 data; 3902 u64 mcg_cap = vcpu->arch.mcg_cap; 3903 unsigned bank_num = mcg_cap & 0xff; 3904 u32 offset, last_msr; 3905 3906 switch (msr) { 3907 case MSR_IA32_P5_MC_ADDR: 3908 case MSR_IA32_P5_MC_TYPE: 3909 data = 0; 3910 break; 3911 case MSR_IA32_MCG_CAP: 3912 data = vcpu->arch.mcg_cap; 3913 break; 3914 case MSR_IA32_MCG_CTL: 3915 if (!(mcg_cap & MCG_CTL_P) && !host) 3916 return 1; 3917 data = vcpu->arch.mcg_ctl; 3918 break; 3919 case MSR_IA32_MCG_STATUS: 3920 data = vcpu->arch.mcg_status; 3921 break; 3922 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 3923 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; 3924 if (msr > last_msr) 3925 return 1; 3926 3927 if (!(mcg_cap & MCG_CMCI_P) && !host) 3928 return 1; 3929 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, 3930 last_msr + 1 - MSR_IA32_MC0_CTL2); 3931 data = vcpu->arch.mci_ctl2_banks[offset]; 3932 break; 3933 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 3934 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; 3935 if (msr > last_msr) 3936 return 1; 3937 3938 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, 3939 last_msr + 1 - MSR_IA32_MC0_CTL); 3940 data = vcpu->arch.mce_banks[offset]; 3941 break; 3942 default: 3943 return 1; 3944 } 3945 *pdata = data; 3946 return 0; 3947 } 3948 3949 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) 3950 { 3951 switch (msr_info->index) { 3952 case MSR_IA32_PLATFORM_ID: 3953 case MSR_IA32_EBL_CR_POWERON: 3954 case MSR_IA32_LASTBRANCHFROMIP: 3955 case MSR_IA32_LASTBRANCHTOIP: 3956 case MSR_IA32_LASTINTFROMIP: 3957 case MSR_IA32_LASTINTTOIP: 3958 case MSR_AMD64_SYSCFG: 3959 case MSR_K8_TSEG_ADDR: 3960 case MSR_K8_TSEG_MASK: 3961 case MSR_VM_HSAVE_PA: 3962 case MSR_K8_INT_PENDING_MSG: 3963 case MSR_AMD64_NB_CFG: 3964 case MSR_FAM10H_MMIO_CONF_BASE: 3965 case MSR_AMD64_BU_CFG2: 3966 case MSR_IA32_PERF_CTL: 3967 case MSR_AMD64_DC_CFG: 3968 case MSR_F15H_EX_CFG: 3969 /* 3970 * Intel Sandy Bridge CPUs must support the RAPL (running average power 3971 * limit) MSRs. Just return 0, as we do not want to expose the host 3972 * data here. Do not conditionalize this on CPUID, as KVM does not do 3973 * so for existing CPU-specific MSRs. 3974 */ 3975 case MSR_RAPL_POWER_UNIT: 3976 case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */ 3977 case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */ 3978 case MSR_PKG_ENERGY_STATUS: /* Total package */ 3979 case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */ 3980 msr_info->data = 0; 3981 break; 3982 case MSR_IA32_PEBS_ENABLE: 3983 case MSR_IA32_DS_AREA: 3984 case MSR_PEBS_DATA_CFG: 3985 case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5: 3986 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) 3987 return kvm_pmu_get_msr(vcpu, msr_info); 3988 /* 3989 * Userspace is allowed to read MSRs that KVM reports as 3990 * to-be-saved, even if an MSR isn't fully supported. 3991 */ 3992 if (!msr_info->host_initiated) 3993 return 1; 3994 msr_info->data = 0; 3995 break; 3996 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: 3997 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: 3998 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: 3999 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: 4000 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) 4001 return kvm_pmu_get_msr(vcpu, msr_info); 4002 msr_info->data = 0; 4003 break; 4004 case MSR_IA32_UCODE_REV: 4005 msr_info->data = vcpu->arch.microcode_version; 4006 break; 4007 case MSR_IA32_ARCH_CAPABILITIES: 4008 if (!msr_info->host_initiated && 4009 !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) 4010 return 1; 4011 msr_info->data = vcpu->arch.arch_capabilities; 4012 break; 4013 case MSR_IA32_PERF_CAPABILITIES: 4014 if (!msr_info->host_initiated && 4015 !guest_cpuid_has(vcpu, X86_FEATURE_PDCM)) 4016 return 1; 4017 msr_info->data = vcpu->arch.perf_capabilities; 4018 break; 4019 case MSR_IA32_POWER_CTL: 4020 msr_info->data = vcpu->arch.msr_ia32_power_ctl; 4021 break; 4022 case MSR_IA32_TSC: { 4023 /* 4024 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset 4025 * even when not intercepted. AMD manual doesn't explicitly 4026 * state this but appears to behave the same. 4027 * 4028 * On userspace reads and writes, however, we unconditionally 4029 * return L1's TSC value to ensure backwards-compatible 4030 * behavior for migration. 4031 */ 4032 u64 offset, ratio; 4033 4034 if (msr_info->host_initiated) { 4035 offset = vcpu->arch.l1_tsc_offset; 4036 ratio = vcpu->arch.l1_tsc_scaling_ratio; 4037 } else { 4038 offset = vcpu->arch.tsc_offset; 4039 ratio = vcpu->arch.tsc_scaling_ratio; 4040 } 4041 4042 msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset; 4043 break; 4044 } 4045 case MSR_MTRRcap: 4046 case 0x200 ... MSR_IA32_MC0_CTL2 - 1: 4047 case MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) ... 0x2ff: 4048 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data); 4049 case 0xcd: /* fsb frequency */ 4050 msr_info->data = 3; 4051 break; 4052 /* 4053 * MSR_EBC_FREQUENCY_ID 4054 * Conservative value valid for even the basic CPU models. 4055 * Models 0,1: 000 in bits 23:21 indicating a bus speed of 4056 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz, 4057 * and 266MHz for model 3, or 4. Set Core Clock 4058 * Frequency to System Bus Frequency Ratio to 1 (bits 4059 * 31:24) even though these are only valid for CPU 4060 * models > 2, however guests may end up dividing or 4061 * multiplying by zero otherwise. 4062 */ 4063 case MSR_EBC_FREQUENCY_ID: 4064 msr_info->data = 1 << 24; 4065 break; 4066 case MSR_IA32_APICBASE: 4067 msr_info->data = kvm_get_apic_base(vcpu); 4068 break; 4069 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: 4070 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data); 4071 case MSR_IA32_TSC_DEADLINE: 4072 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu); 4073 break; 4074 case MSR_IA32_TSC_ADJUST: 4075 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr; 4076 break; 4077 case MSR_IA32_MISC_ENABLE: 4078 msr_info->data = vcpu->arch.ia32_misc_enable_msr; 4079 break; 4080 case MSR_IA32_SMBASE: 4081 if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) 4082 return 1; 4083 msr_info->data = vcpu->arch.smbase; 4084 break; 4085 case MSR_SMI_COUNT: 4086 msr_info->data = vcpu->arch.smi_count; 4087 break; 4088 case MSR_IA32_PERF_STATUS: 4089 /* TSC increment by tick */ 4090 msr_info->data = 1000ULL; 4091 /* CPU multiplier */ 4092 msr_info->data |= (((uint64_t)4ULL) << 40); 4093 break; 4094 case MSR_EFER: 4095 msr_info->data = vcpu->arch.efer; 4096 break; 4097 case MSR_KVM_WALL_CLOCK: 4098 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 4099 return 1; 4100 4101 msr_info->data = vcpu->kvm->arch.wall_clock; 4102 break; 4103 case MSR_KVM_WALL_CLOCK_NEW: 4104 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 4105 return 1; 4106 4107 msr_info->data = vcpu->kvm->arch.wall_clock; 4108 break; 4109 case MSR_KVM_SYSTEM_TIME: 4110 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) 4111 return 1; 4112 4113 msr_info->data = vcpu->arch.time; 4114 break; 4115 case MSR_KVM_SYSTEM_TIME_NEW: 4116 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) 4117 return 1; 4118 4119 msr_info->data = vcpu->arch.time; 4120 break; 4121 case MSR_KVM_ASYNC_PF_EN: 4122 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) 4123 return 1; 4124 4125 msr_info->data = vcpu->arch.apf.msr_en_val; 4126 break; 4127 case MSR_KVM_ASYNC_PF_INT: 4128 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 4129 return 1; 4130 4131 msr_info->data = vcpu->arch.apf.msr_int_val; 4132 break; 4133 case MSR_KVM_ASYNC_PF_ACK: 4134 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) 4135 return 1; 4136 4137 msr_info->data = 0; 4138 break; 4139 case MSR_KVM_STEAL_TIME: 4140 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) 4141 return 1; 4142 4143 msr_info->data = vcpu->arch.st.msr_val; 4144 break; 4145 case MSR_KVM_PV_EOI_EN: 4146 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) 4147 return 1; 4148 4149 msr_info->data = vcpu->arch.pv_eoi.msr_val; 4150 break; 4151 case MSR_KVM_POLL_CONTROL: 4152 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) 4153 return 1; 4154 4155 msr_info->data = vcpu->arch.msr_kvm_poll_control; 4156 break; 4157 case MSR_IA32_P5_MC_ADDR: 4158 case MSR_IA32_P5_MC_TYPE: 4159 case MSR_IA32_MCG_CAP: 4160 case MSR_IA32_MCG_CTL: 4161 case MSR_IA32_MCG_STATUS: 4162 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: 4163 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: 4164 return get_msr_mce(vcpu, msr_info->index, &msr_info->data, 4165 msr_info->host_initiated); 4166 case MSR_IA32_XSS: 4167 if (!msr_info->host_initiated && 4168 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) 4169 return 1; 4170 msr_info->data = vcpu->arch.ia32_xss; 4171 break; 4172 case MSR_K7_CLK_CTL: 4173 /* 4174 * Provide expected ramp-up count for K7. All other 4175 * are set to zero, indicating minimum divisors for 4176 * every field. 4177 * 4178 * This prevents guest kernels on AMD host with CPU 4179 * type 6, model 8 and higher from exploding due to 4180 * the rdmsr failing. 4181 */ 4182 msr_info->data = 0x20000000; 4183 break; 4184 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: 4185 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: 4186 case HV_X64_MSR_SYNDBG_OPTIONS: 4187 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 4188 case HV_X64_MSR_CRASH_CTL: 4189 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: 4190 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 4191 case HV_X64_MSR_TSC_EMULATION_CONTROL: 4192 case HV_X64_MSR_TSC_EMULATION_STATUS: 4193 case HV_X64_MSR_TSC_INVARIANT_CONTROL: 4194 return kvm_hv_get_msr_common(vcpu, 4195 msr_info->index, &msr_info->data, 4196 msr_info->host_initiated); 4197 case MSR_IA32_BBL_CR_CTL3: 4198 /* This legacy MSR exists but isn't fully documented in current 4199 * silicon. It is however accessed by winxp in very narrow 4200 * scenarios where it sets bit #19, itself documented as 4201 * a "reserved" bit. Best effort attempt to source coherent 4202 * read data here should the balance of the register be 4203 * interpreted by the guest: 4204 * 4205 * L2 cache control register 3: 64GB range, 256KB size, 4206 * enabled, latency 0x1, configured 4207 */ 4208 msr_info->data = 0xbe702111; 4209 break; 4210 case MSR_AMD64_OSVW_ID_LENGTH: 4211 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) 4212 return 1; 4213 msr_info->data = vcpu->arch.osvw.length; 4214 break; 4215 case MSR_AMD64_OSVW_STATUS: 4216 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW)) 4217 return 1; 4218 msr_info->data = vcpu->arch.osvw.status; 4219 break; 4220 case MSR_PLATFORM_INFO: 4221 if (!msr_info->host_initiated && 4222 !vcpu->kvm->arch.guest_can_read_msr_platform_info) 4223 return 1; 4224 msr_info->data = vcpu->arch.msr_platform_info; 4225 break; 4226 case MSR_MISC_FEATURES_ENABLES: 4227 msr_info->data = vcpu->arch.msr_misc_features_enables; 4228 break; 4229 case MSR_K7_HWCR: 4230 msr_info->data = vcpu->arch.msr_hwcr; 4231 break; 4232 #ifdef CONFIG_X86_64 4233 case MSR_IA32_XFD: 4234 if (!msr_info->host_initiated && 4235 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) 4236 return 1; 4237 4238 msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd; 4239 break; 4240 case MSR_IA32_XFD_ERR: 4241 if (!msr_info->host_initiated && 4242 !guest_cpuid_has(vcpu, X86_FEATURE_XFD)) 4243 return 1; 4244 4245 msr_info->data = vcpu->arch.guest_fpu.xfd_err; 4246 break; 4247 #endif 4248 default: 4249 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) 4250 return kvm_pmu_get_msr(vcpu, msr_info); 4251 return KVM_MSR_RET_INVALID; 4252 } 4253 return 0; 4254 } 4255 EXPORT_SYMBOL_GPL(kvm_get_msr_common); 4256 4257 /* 4258 * Read or write a bunch of msrs. All parameters are kernel addresses. 4259 * 4260 * @return number of msrs set successfully. 4261 */ 4262 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, 4263 struct kvm_msr_entry *entries, 4264 int (*do_msr)(struct kvm_vcpu *vcpu, 4265 unsigned index, u64 *data)) 4266 { 4267 int i; 4268 4269 for (i = 0; i < msrs->nmsrs; ++i) 4270 if (do_msr(vcpu, entries[i].index, &entries[i].data)) 4271 break; 4272 4273 return i; 4274 } 4275 4276 /* 4277 * Read or write a bunch of msrs. Parameters are user addresses. 4278 * 4279 * @return number of msrs set successfully. 4280 */ 4281 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, 4282 int (*do_msr)(struct kvm_vcpu *vcpu, 4283 unsigned index, u64 *data), 4284 int writeback) 4285 { 4286 struct kvm_msrs msrs; 4287 struct kvm_msr_entry *entries; 4288 int r, n; 4289 unsigned size; 4290 4291 r = -EFAULT; 4292 if (copy_from_user(&msrs, user_msrs, sizeof(msrs))) 4293 goto out; 4294 4295 r = -E2BIG; 4296 if (msrs.nmsrs >= MAX_IO_MSRS) 4297 goto out; 4298 4299 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; 4300 entries = memdup_user(user_msrs->entries, size); 4301 if (IS_ERR(entries)) { 4302 r = PTR_ERR(entries); 4303 goto out; 4304 } 4305 4306 r = n = __msr_io(vcpu, &msrs, entries, do_msr); 4307 if (r < 0) 4308 goto out_free; 4309 4310 r = -EFAULT; 4311 if (writeback && copy_to_user(user_msrs->entries, entries, size)) 4312 goto out_free; 4313 4314 r = n; 4315 4316 out_free: 4317 kfree(entries); 4318 out: 4319 return r; 4320 } 4321 4322 static inline bool kvm_can_mwait_in_guest(void) 4323 { 4324 return boot_cpu_has(X86_FEATURE_MWAIT) && 4325 !boot_cpu_has_bug(X86_BUG_MONITOR) && 4326 boot_cpu_has(X86_FEATURE_ARAT); 4327 } 4328 4329 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu, 4330 struct kvm_cpuid2 __user *cpuid_arg) 4331 { 4332 struct kvm_cpuid2 cpuid; 4333 int r; 4334 4335 r = -EFAULT; 4336 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 4337 return r; 4338 4339 r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries); 4340 if (r) 4341 return r; 4342 4343 r = -EFAULT; 4344 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 4345 return r; 4346 4347 return 0; 4348 } 4349 4350 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 4351 { 4352 int r = 0; 4353 4354 switch (ext) { 4355 case KVM_CAP_IRQCHIP: 4356 case KVM_CAP_HLT: 4357 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: 4358 case KVM_CAP_SET_TSS_ADDR: 4359 case KVM_CAP_EXT_CPUID: 4360 case KVM_CAP_EXT_EMUL_CPUID: 4361 case KVM_CAP_CLOCKSOURCE: 4362 case KVM_CAP_PIT: 4363 case KVM_CAP_NOP_IO_DELAY: 4364 case KVM_CAP_MP_STATE: 4365 case KVM_CAP_SYNC_MMU: 4366 case KVM_CAP_USER_NMI: 4367 case KVM_CAP_REINJECT_CONTROL: 4368 case KVM_CAP_IRQ_INJECT_STATUS: 4369 case KVM_CAP_IOEVENTFD: 4370 case KVM_CAP_IOEVENTFD_NO_LENGTH: 4371 case KVM_CAP_PIT2: 4372 case KVM_CAP_PIT_STATE2: 4373 case KVM_CAP_SET_IDENTITY_MAP_ADDR: 4374 case KVM_CAP_VCPU_EVENTS: 4375 case KVM_CAP_HYPERV: 4376 case KVM_CAP_HYPERV_VAPIC: 4377 case KVM_CAP_HYPERV_SPIN: 4378 case KVM_CAP_HYPERV_SYNIC: 4379 case KVM_CAP_HYPERV_SYNIC2: 4380 case KVM_CAP_HYPERV_VP_INDEX: 4381 case KVM_CAP_HYPERV_EVENTFD: 4382 case KVM_CAP_HYPERV_TLBFLUSH: 4383 case KVM_CAP_HYPERV_SEND_IPI: 4384 case KVM_CAP_HYPERV_CPUID: 4385 case KVM_CAP_HYPERV_ENFORCE_CPUID: 4386 case KVM_CAP_SYS_HYPERV_CPUID: 4387 case KVM_CAP_PCI_SEGMENT: 4388 case KVM_CAP_DEBUGREGS: 4389 case KVM_CAP_X86_ROBUST_SINGLESTEP: 4390 case KVM_CAP_XSAVE: 4391 case KVM_CAP_ASYNC_PF: 4392 case KVM_CAP_ASYNC_PF_INT: 4393 case KVM_CAP_GET_TSC_KHZ: 4394 case KVM_CAP_KVMCLOCK_CTRL: 4395 case KVM_CAP_READONLY_MEM: 4396 case KVM_CAP_HYPERV_TIME: 4397 case KVM_CAP_IOAPIC_POLARITY_IGNORED: 4398 case KVM_CAP_TSC_DEADLINE_TIMER: 4399 case KVM_CAP_DISABLE_QUIRKS: 4400 case KVM_CAP_SET_BOOT_CPU_ID: 4401 case KVM_CAP_SPLIT_IRQCHIP: 4402 case KVM_CAP_IMMEDIATE_EXIT: 4403 case KVM_CAP_PMU_EVENT_FILTER: 4404 case KVM_CAP_PMU_EVENT_MASKED_EVENTS: 4405 case KVM_CAP_GET_MSR_FEATURES: 4406 case KVM_CAP_MSR_PLATFORM_INFO: 4407 case KVM_CAP_EXCEPTION_PAYLOAD: 4408 case KVM_CAP_X86_TRIPLE_FAULT_EVENT: 4409 case KVM_CAP_SET_GUEST_DEBUG: 4410 case KVM_CAP_LAST_CPU: 4411 case KVM_CAP_X86_USER_SPACE_MSR: 4412 case KVM_CAP_X86_MSR_FILTER: 4413 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: 4414 #ifdef CONFIG_X86_SGX_KVM 4415 case KVM_CAP_SGX_ATTRIBUTE: 4416 #endif 4417 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: 4418 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: 4419 case KVM_CAP_SREGS2: 4420 case KVM_CAP_EXIT_ON_EMULATION_FAILURE: 4421 case KVM_CAP_VCPU_ATTRIBUTES: 4422 case KVM_CAP_SYS_ATTRIBUTES: 4423 case KVM_CAP_VAPIC: 4424 case KVM_CAP_ENABLE_CAP: 4425 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES: 4426 r = 1; 4427 break; 4428 case KVM_CAP_EXIT_HYPERCALL: 4429 r = KVM_EXIT_HYPERCALL_VALID_MASK; 4430 break; 4431 case KVM_CAP_SET_GUEST_DEBUG2: 4432 return KVM_GUESTDBG_VALID_MASK; 4433 #ifdef CONFIG_KVM_XEN 4434 case KVM_CAP_XEN_HVM: 4435 r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR | 4436 KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | 4437 KVM_XEN_HVM_CONFIG_SHARED_INFO | 4438 KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL | 4439 KVM_XEN_HVM_CONFIG_EVTCHN_SEND; 4440 if (sched_info_on()) 4441 r |= KVM_XEN_HVM_CONFIG_RUNSTATE | 4442 KVM_XEN_HVM_CONFIG_RUNSTATE_UPDATE_FLAG; 4443 break; 4444 #endif 4445 case KVM_CAP_SYNC_REGS: 4446 r = KVM_SYNC_X86_VALID_FIELDS; 4447 break; 4448 case KVM_CAP_ADJUST_CLOCK: 4449 r = KVM_CLOCK_VALID_FLAGS; 4450 break; 4451 case KVM_CAP_X86_DISABLE_EXITS: 4452 r |= KVM_X86_DISABLE_EXITS_HLT | KVM_X86_DISABLE_EXITS_PAUSE | 4453 KVM_X86_DISABLE_EXITS_CSTATE; 4454 if(kvm_can_mwait_in_guest()) 4455 r |= KVM_X86_DISABLE_EXITS_MWAIT; 4456 break; 4457 case KVM_CAP_X86_SMM: 4458 if (!IS_ENABLED(CONFIG_KVM_SMM)) 4459 break; 4460 4461 /* SMBASE is usually relocated above 1M on modern chipsets, 4462 * and SMM handlers might indeed rely on 4G segment limits, 4463 * so do not report SMM to be available if real mode is 4464 * emulated via vm86 mode. Still, do not go to great lengths 4465 * to avoid userspace's usage of the feature, because it is a 4466 * fringe case that is not enabled except via specific settings 4467 * of the module parameters. 4468 */ 4469 r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_IA32_SMBASE); 4470 break; 4471 case KVM_CAP_NR_VCPUS: 4472 r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); 4473 break; 4474 case KVM_CAP_MAX_VCPUS: 4475 r = KVM_MAX_VCPUS; 4476 break; 4477 case KVM_CAP_MAX_VCPU_ID: 4478 r = KVM_MAX_VCPU_IDS; 4479 break; 4480 case KVM_CAP_PV_MMU: /* obsolete */ 4481 r = 0; 4482 break; 4483 case KVM_CAP_MCE: 4484 r = KVM_MAX_MCE_BANKS; 4485 break; 4486 case KVM_CAP_XCRS: 4487 r = boot_cpu_has(X86_FEATURE_XSAVE); 4488 break; 4489 case KVM_CAP_TSC_CONTROL: 4490 case KVM_CAP_VM_TSC_CONTROL: 4491 r = kvm_caps.has_tsc_control; 4492 break; 4493 case KVM_CAP_X2APIC_API: 4494 r = KVM_X2APIC_API_VALID_FLAGS; 4495 break; 4496 case KVM_CAP_NESTED_STATE: 4497 r = kvm_x86_ops.nested_ops->get_state ? 4498 kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0; 4499 break; 4500 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: 4501 r = kvm_x86_ops.enable_l2_tlb_flush != NULL; 4502 break; 4503 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: 4504 r = kvm_x86_ops.nested_ops->enable_evmcs != NULL; 4505 break; 4506 case KVM_CAP_SMALLER_MAXPHYADDR: 4507 r = (int) allow_smaller_maxphyaddr; 4508 break; 4509 case KVM_CAP_STEAL_TIME: 4510 r = sched_info_on(); 4511 break; 4512 case KVM_CAP_X86_BUS_LOCK_EXIT: 4513 if (kvm_caps.has_bus_lock_exit) 4514 r = KVM_BUS_LOCK_DETECTION_OFF | 4515 KVM_BUS_LOCK_DETECTION_EXIT; 4516 else 4517 r = 0; 4518 break; 4519 case KVM_CAP_XSAVE2: { 4520 u64 guest_perm = xstate_get_guest_group_perm(); 4521 4522 r = xstate_required_size(kvm_caps.supported_xcr0 & guest_perm, false); 4523 if (r < sizeof(struct kvm_xsave)) 4524 r = sizeof(struct kvm_xsave); 4525 break; 4526 } 4527 case KVM_CAP_PMU_CAPABILITY: 4528 r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0; 4529 break; 4530 case KVM_CAP_DISABLE_QUIRKS2: 4531 r = KVM_X86_VALID_QUIRKS; 4532 break; 4533 case KVM_CAP_X86_NOTIFY_VMEXIT: 4534 r = kvm_caps.has_notify_vmexit; 4535 break; 4536 default: 4537 break; 4538 } 4539 return r; 4540 } 4541 4542 static inline void __user *kvm_get_attr_addr(struct kvm_device_attr *attr) 4543 { 4544 void __user *uaddr = (void __user*)(unsigned long)attr->addr; 4545 4546 if ((u64)(unsigned long)uaddr != attr->addr) 4547 return ERR_PTR_USR(-EFAULT); 4548 return uaddr; 4549 } 4550 4551 static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr) 4552 { 4553 u64 __user *uaddr = kvm_get_attr_addr(attr); 4554 4555 if (attr->group) 4556 return -ENXIO; 4557 4558 if (IS_ERR(uaddr)) 4559 return PTR_ERR(uaddr); 4560 4561 switch (attr->attr) { 4562 case KVM_X86_XCOMP_GUEST_SUPP: 4563 if (put_user(kvm_caps.supported_xcr0, uaddr)) 4564 return -EFAULT; 4565 return 0; 4566 default: 4567 return -ENXIO; 4568 break; 4569 } 4570 } 4571 4572 static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr) 4573 { 4574 if (attr->group) 4575 return -ENXIO; 4576 4577 switch (attr->attr) { 4578 case KVM_X86_XCOMP_GUEST_SUPP: 4579 return 0; 4580 default: 4581 return -ENXIO; 4582 } 4583 } 4584 4585 long kvm_arch_dev_ioctl(struct file *filp, 4586 unsigned int ioctl, unsigned long arg) 4587 { 4588 void __user *argp = (void __user *)arg; 4589 long r; 4590 4591 switch (ioctl) { 4592 case KVM_GET_MSR_INDEX_LIST: { 4593 struct kvm_msr_list __user *user_msr_list = argp; 4594 struct kvm_msr_list msr_list; 4595 unsigned n; 4596 4597 r = -EFAULT; 4598 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) 4599 goto out; 4600 n = msr_list.nmsrs; 4601 msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs; 4602 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) 4603 goto out; 4604 r = -E2BIG; 4605 if (n < msr_list.nmsrs) 4606 goto out; 4607 r = -EFAULT; 4608 if (copy_to_user(user_msr_list->indices, &msrs_to_save, 4609 num_msrs_to_save * sizeof(u32))) 4610 goto out; 4611 if (copy_to_user(user_msr_list->indices + num_msrs_to_save, 4612 &emulated_msrs, 4613 num_emulated_msrs * sizeof(u32))) 4614 goto out; 4615 r = 0; 4616 break; 4617 } 4618 case KVM_GET_SUPPORTED_CPUID: 4619 case KVM_GET_EMULATED_CPUID: { 4620 struct kvm_cpuid2 __user *cpuid_arg = argp; 4621 struct kvm_cpuid2 cpuid; 4622 4623 r = -EFAULT; 4624 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 4625 goto out; 4626 4627 r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries, 4628 ioctl); 4629 if (r) 4630 goto out; 4631 4632 r = -EFAULT; 4633 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 4634 goto out; 4635 r = 0; 4636 break; 4637 } 4638 case KVM_X86_GET_MCE_CAP_SUPPORTED: 4639 r = -EFAULT; 4640 if (copy_to_user(argp, &kvm_caps.supported_mce_cap, 4641 sizeof(kvm_caps.supported_mce_cap))) 4642 goto out; 4643 r = 0; 4644 break; 4645 case KVM_GET_MSR_FEATURE_INDEX_LIST: { 4646 struct kvm_msr_list __user *user_msr_list = argp; 4647 struct kvm_msr_list msr_list; 4648 unsigned int n; 4649 4650 r = -EFAULT; 4651 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) 4652 goto out; 4653 n = msr_list.nmsrs; 4654 msr_list.nmsrs = num_msr_based_features; 4655 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) 4656 goto out; 4657 r = -E2BIG; 4658 if (n < msr_list.nmsrs) 4659 goto out; 4660 r = -EFAULT; 4661 if (copy_to_user(user_msr_list->indices, &msr_based_features, 4662 num_msr_based_features * sizeof(u32))) 4663 goto out; 4664 r = 0; 4665 break; 4666 } 4667 case KVM_GET_MSRS: 4668 r = msr_io(NULL, argp, do_get_msr_feature, 1); 4669 break; 4670 case KVM_GET_SUPPORTED_HV_CPUID: 4671 r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp); 4672 break; 4673 case KVM_GET_DEVICE_ATTR: { 4674 struct kvm_device_attr attr; 4675 r = -EFAULT; 4676 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 4677 break; 4678 r = kvm_x86_dev_get_attr(&attr); 4679 break; 4680 } 4681 case KVM_HAS_DEVICE_ATTR: { 4682 struct kvm_device_attr attr; 4683 r = -EFAULT; 4684 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 4685 break; 4686 r = kvm_x86_dev_has_attr(&attr); 4687 break; 4688 } 4689 default: 4690 r = -EINVAL; 4691 break; 4692 } 4693 out: 4694 return r; 4695 } 4696 4697 static void wbinvd_ipi(void *garbage) 4698 { 4699 wbinvd(); 4700 } 4701 4702 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu) 4703 { 4704 return kvm_arch_has_noncoherent_dma(vcpu->kvm); 4705 } 4706 4707 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 4708 { 4709 /* Address WBINVD may be executed by guest */ 4710 if (need_emulate_wbinvd(vcpu)) { 4711 if (static_call(kvm_x86_has_wbinvd_exit)()) 4712 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 4713 else if (vcpu->cpu != -1 && vcpu->cpu != cpu) 4714 smp_call_function_single(vcpu->cpu, 4715 wbinvd_ipi, NULL, 1); 4716 } 4717 4718 static_call(kvm_x86_vcpu_load)(vcpu, cpu); 4719 4720 /* Save host pkru register if supported */ 4721 vcpu->arch.host_pkru = read_pkru(); 4722 4723 /* Apply any externally detected TSC adjustments (due to suspend) */ 4724 if (unlikely(vcpu->arch.tsc_offset_adjustment)) { 4725 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment); 4726 vcpu->arch.tsc_offset_adjustment = 0; 4727 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 4728 } 4729 4730 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) { 4731 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 : 4732 rdtsc() - vcpu->arch.last_host_tsc; 4733 if (tsc_delta < 0) 4734 mark_tsc_unstable("KVM discovered backwards TSC"); 4735 4736 if (kvm_check_tsc_unstable()) { 4737 u64 offset = kvm_compute_l1_tsc_offset(vcpu, 4738 vcpu->arch.last_guest_tsc); 4739 kvm_vcpu_write_tsc_offset(vcpu, offset); 4740 vcpu->arch.tsc_catchup = 1; 4741 } 4742 4743 if (kvm_lapic_hv_timer_in_use(vcpu)) 4744 kvm_lapic_restart_hv_timer(vcpu); 4745 4746 /* 4747 * On a host with synchronized TSC, there is no need to update 4748 * kvmclock on vcpu->cpu migration 4749 */ 4750 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) 4751 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); 4752 if (vcpu->cpu != cpu) 4753 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu); 4754 vcpu->cpu = cpu; 4755 } 4756 4757 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 4758 } 4759 4760 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu) 4761 { 4762 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache; 4763 struct kvm_steal_time __user *st; 4764 struct kvm_memslots *slots; 4765 static const u8 preempted = KVM_VCPU_PREEMPTED; 4766 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS; 4767 4768 /* 4769 * The vCPU can be marked preempted if and only if the VM-Exit was on 4770 * an instruction boundary and will not trigger guest emulation of any 4771 * kind (see vcpu_run). Vendor specific code controls (conservatively) 4772 * when this is true, for example allowing the vCPU to be marked 4773 * preempted if and only if the VM-Exit was due to a host interrupt. 4774 */ 4775 if (!vcpu->arch.at_instruction_boundary) { 4776 vcpu->stat.preemption_other++; 4777 return; 4778 } 4779 4780 vcpu->stat.preemption_reported++; 4781 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED)) 4782 return; 4783 4784 if (vcpu->arch.st.preempted) 4785 return; 4786 4787 /* This happens on process exit */ 4788 if (unlikely(current->mm != vcpu->kvm->mm)) 4789 return; 4790 4791 slots = kvm_memslots(vcpu->kvm); 4792 4793 if (unlikely(slots->generation != ghc->generation || 4794 gpa != ghc->gpa || 4795 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) 4796 return; 4797 4798 st = (struct kvm_steal_time __user *)ghc->hva; 4799 BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted)); 4800 4801 if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted))) 4802 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED; 4803 4804 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 4805 } 4806 4807 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 4808 { 4809 int idx; 4810 4811 if (vcpu->preempted) { 4812 if (!vcpu->arch.guest_state_protected) 4813 vcpu->arch.preempted_in_kernel = !static_call(kvm_x86_get_cpl)(vcpu); 4814 4815 /* 4816 * Take the srcu lock as memslots will be accessed to check the gfn 4817 * cache generation against the memslots generation. 4818 */ 4819 idx = srcu_read_lock(&vcpu->kvm->srcu); 4820 if (kvm_xen_msr_enabled(vcpu->kvm)) 4821 kvm_xen_runstate_set_preempted(vcpu); 4822 else 4823 kvm_steal_time_set_preempted(vcpu); 4824 srcu_read_unlock(&vcpu->kvm->srcu, idx); 4825 } 4826 4827 static_call(kvm_x86_vcpu_put)(vcpu); 4828 vcpu->arch.last_host_tsc = rdtsc(); 4829 } 4830 4831 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, 4832 struct kvm_lapic_state *s) 4833 { 4834 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu); 4835 4836 return kvm_apic_get_state(vcpu, s); 4837 } 4838 4839 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, 4840 struct kvm_lapic_state *s) 4841 { 4842 int r; 4843 4844 r = kvm_apic_set_state(vcpu, s); 4845 if (r) 4846 return r; 4847 update_cr8_intercept(vcpu); 4848 4849 return 0; 4850 } 4851 4852 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu) 4853 { 4854 /* 4855 * We can accept userspace's request for interrupt injection 4856 * as long as we have a place to store the interrupt number. 4857 * The actual injection will happen when the CPU is able to 4858 * deliver the interrupt. 4859 */ 4860 if (kvm_cpu_has_extint(vcpu)) 4861 return false; 4862 4863 /* Acknowledging ExtINT does not happen if LINT0 is masked. */ 4864 return (!lapic_in_kernel(vcpu) || 4865 kvm_apic_accept_pic_intr(vcpu)); 4866 } 4867 4868 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu) 4869 { 4870 /* 4871 * Do not cause an interrupt window exit if an exception 4872 * is pending or an event needs reinjection; userspace 4873 * might want to inject the interrupt manually using KVM_SET_REGS 4874 * or KVM_SET_SREGS. For that to work, we must be at an 4875 * instruction boundary and with no events half-injected. 4876 */ 4877 return (kvm_arch_interrupt_allowed(vcpu) && 4878 kvm_cpu_accept_dm_intr(vcpu) && 4879 !kvm_event_needs_reinjection(vcpu) && 4880 !kvm_is_exception_pending(vcpu)); 4881 } 4882 4883 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, 4884 struct kvm_interrupt *irq) 4885 { 4886 if (irq->irq >= KVM_NR_INTERRUPTS) 4887 return -EINVAL; 4888 4889 if (!irqchip_in_kernel(vcpu->kvm)) { 4890 kvm_queue_interrupt(vcpu, irq->irq, false); 4891 kvm_make_request(KVM_REQ_EVENT, vcpu); 4892 return 0; 4893 } 4894 4895 /* 4896 * With in-kernel LAPIC, we only use this to inject EXTINT, so 4897 * fail for in-kernel 8259. 4898 */ 4899 if (pic_in_kernel(vcpu->kvm)) 4900 return -ENXIO; 4901 4902 if (vcpu->arch.pending_external_vector != -1) 4903 return -EEXIST; 4904 4905 vcpu->arch.pending_external_vector = irq->irq; 4906 kvm_make_request(KVM_REQ_EVENT, vcpu); 4907 return 0; 4908 } 4909 4910 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu) 4911 { 4912 kvm_inject_nmi(vcpu); 4913 4914 return 0; 4915 } 4916 4917 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu, 4918 struct kvm_tpr_access_ctl *tac) 4919 { 4920 if (tac->flags) 4921 return -EINVAL; 4922 vcpu->arch.tpr_access_reporting = !!tac->enabled; 4923 return 0; 4924 } 4925 4926 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu, 4927 u64 mcg_cap) 4928 { 4929 int r; 4930 unsigned bank_num = mcg_cap & 0xff, bank; 4931 4932 r = -EINVAL; 4933 if (!bank_num || bank_num > KVM_MAX_MCE_BANKS) 4934 goto out; 4935 if (mcg_cap & ~(kvm_caps.supported_mce_cap | 0xff | 0xff0000)) 4936 goto out; 4937 r = 0; 4938 vcpu->arch.mcg_cap = mcg_cap; 4939 /* Init IA32_MCG_CTL to all 1s */ 4940 if (mcg_cap & MCG_CTL_P) 4941 vcpu->arch.mcg_ctl = ~(u64)0; 4942 /* Init IA32_MCi_CTL to all 1s, IA32_MCi_CTL2 to all 0s */ 4943 for (bank = 0; bank < bank_num; bank++) { 4944 vcpu->arch.mce_banks[bank*4] = ~(u64)0; 4945 if (mcg_cap & MCG_CMCI_P) 4946 vcpu->arch.mci_ctl2_banks[bank] = 0; 4947 } 4948 4949 kvm_apic_after_set_mcg_cap(vcpu); 4950 4951 static_call(kvm_x86_setup_mce)(vcpu); 4952 out: 4953 return r; 4954 } 4955 4956 /* 4957 * Validate this is an UCNA (uncorrectable no action) error by checking the 4958 * MCG_STATUS and MCi_STATUS registers: 4959 * - none of the bits for Machine Check Exceptions are set 4960 * - both the VAL (valid) and UC (uncorrectable) bits are set 4961 * MCI_STATUS_PCC - Processor Context Corrupted 4962 * MCI_STATUS_S - Signaled as a Machine Check Exception 4963 * MCI_STATUS_AR - Software recoverable Action Required 4964 */ 4965 static bool is_ucna(struct kvm_x86_mce *mce) 4966 { 4967 return !mce->mcg_status && 4968 !(mce->status & (MCI_STATUS_PCC | MCI_STATUS_S | MCI_STATUS_AR)) && 4969 (mce->status & MCI_STATUS_VAL) && 4970 (mce->status & MCI_STATUS_UC); 4971 } 4972 4973 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks) 4974 { 4975 u64 mcg_cap = vcpu->arch.mcg_cap; 4976 4977 banks[1] = mce->status; 4978 banks[2] = mce->addr; 4979 banks[3] = mce->misc; 4980 vcpu->arch.mcg_status = mce->mcg_status; 4981 4982 if (!(mcg_cap & MCG_CMCI_P) || 4983 !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN)) 4984 return 0; 4985 4986 if (lapic_in_kernel(vcpu)) 4987 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI); 4988 4989 return 0; 4990 } 4991 4992 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, 4993 struct kvm_x86_mce *mce) 4994 { 4995 u64 mcg_cap = vcpu->arch.mcg_cap; 4996 unsigned bank_num = mcg_cap & 0xff; 4997 u64 *banks = vcpu->arch.mce_banks; 4998 4999 if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL)) 5000 return -EINVAL; 5001 5002 banks += array_index_nospec(4 * mce->bank, 4 * bank_num); 5003 5004 if (is_ucna(mce)) 5005 return kvm_vcpu_x86_set_ucna(vcpu, mce, banks); 5006 5007 /* 5008 * if IA32_MCG_CTL is not all 1s, the uncorrected error 5009 * reporting is disabled 5010 */ 5011 if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) && 5012 vcpu->arch.mcg_ctl != ~(u64)0) 5013 return 0; 5014 /* 5015 * if IA32_MCi_CTL is not all 1s, the uncorrected error 5016 * reporting is disabled for the bank 5017 */ 5018 if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0) 5019 return 0; 5020 if (mce->status & MCI_STATUS_UC) { 5021 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) || 5022 !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) { 5023 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 5024 return 0; 5025 } 5026 if (banks[1] & MCI_STATUS_VAL) 5027 mce->status |= MCI_STATUS_OVER; 5028 banks[2] = mce->addr; 5029 banks[3] = mce->misc; 5030 vcpu->arch.mcg_status = mce->mcg_status; 5031 banks[1] = mce->status; 5032 kvm_queue_exception(vcpu, MC_VECTOR); 5033 } else if (!(banks[1] & MCI_STATUS_VAL) 5034 || !(banks[1] & MCI_STATUS_UC)) { 5035 if (banks[1] & MCI_STATUS_VAL) 5036 mce->status |= MCI_STATUS_OVER; 5037 banks[2] = mce->addr; 5038 banks[3] = mce->misc; 5039 banks[1] = mce->status; 5040 } else 5041 banks[1] |= MCI_STATUS_OVER; 5042 return 0; 5043 } 5044 5045 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, 5046 struct kvm_vcpu_events *events) 5047 { 5048 struct kvm_queued_exception *ex; 5049 5050 process_nmi(vcpu); 5051 5052 #ifdef CONFIG_KVM_SMM 5053 if (kvm_check_request(KVM_REQ_SMI, vcpu)) 5054 process_smi(vcpu); 5055 #endif 5056 5057 /* 5058 * KVM's ABI only allows for one exception to be migrated. Luckily, 5059 * the only time there can be two queued exceptions is if there's a 5060 * non-exiting _injected_ exception, and a pending exiting exception. 5061 * In that case, ignore the VM-Exiting exception as it's an extension 5062 * of the injected exception. 5063 */ 5064 if (vcpu->arch.exception_vmexit.pending && 5065 !vcpu->arch.exception.pending && 5066 !vcpu->arch.exception.injected) 5067 ex = &vcpu->arch.exception_vmexit; 5068 else 5069 ex = &vcpu->arch.exception; 5070 5071 /* 5072 * In guest mode, payload delivery should be deferred if the exception 5073 * will be intercepted by L1, e.g. KVM should not modifying CR2 if L1 5074 * intercepts #PF, ditto for DR6 and #DBs. If the per-VM capability, 5075 * KVM_CAP_EXCEPTION_PAYLOAD, is not set, userspace may or may not 5076 * propagate the payload and so it cannot be safely deferred. Deliver 5077 * the payload if the capability hasn't been requested. 5078 */ 5079 if (!vcpu->kvm->arch.exception_payload_enabled && 5080 ex->pending && ex->has_payload) 5081 kvm_deliver_exception_payload(vcpu, ex); 5082 5083 memset(events, 0, sizeof(*events)); 5084 5085 /* 5086 * The API doesn't provide the instruction length for software 5087 * exceptions, so don't report them. As long as the guest RIP 5088 * isn't advanced, we should expect to encounter the exception 5089 * again. 5090 */ 5091 if (!kvm_exception_is_soft(ex->vector)) { 5092 events->exception.injected = ex->injected; 5093 events->exception.pending = ex->pending; 5094 /* 5095 * For ABI compatibility, deliberately conflate 5096 * pending and injected exceptions when 5097 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled. 5098 */ 5099 if (!vcpu->kvm->arch.exception_payload_enabled) 5100 events->exception.injected |= ex->pending; 5101 } 5102 events->exception.nr = ex->vector; 5103 events->exception.has_error_code = ex->has_error_code; 5104 events->exception.error_code = ex->error_code; 5105 events->exception_has_payload = ex->has_payload; 5106 events->exception_payload = ex->payload; 5107 5108 events->interrupt.injected = 5109 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft; 5110 events->interrupt.nr = vcpu->arch.interrupt.nr; 5111 events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu); 5112 5113 events->nmi.injected = vcpu->arch.nmi_injected; 5114 events->nmi.pending = vcpu->arch.nmi_pending != 0; 5115 events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu); 5116 5117 /* events->sipi_vector is never valid when reporting to user space */ 5118 5119 #ifdef CONFIG_KVM_SMM 5120 events->smi.smm = is_smm(vcpu); 5121 events->smi.pending = vcpu->arch.smi_pending; 5122 events->smi.smm_inside_nmi = 5123 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK); 5124 #endif 5125 events->smi.latched_init = kvm_lapic_latched_init(vcpu); 5126 5127 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING 5128 | KVM_VCPUEVENT_VALID_SHADOW 5129 | KVM_VCPUEVENT_VALID_SMM); 5130 if (vcpu->kvm->arch.exception_payload_enabled) 5131 events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD; 5132 if (vcpu->kvm->arch.triple_fault_event) { 5133 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu); 5134 events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT; 5135 } 5136 } 5137 5138 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, 5139 struct kvm_vcpu_events *events) 5140 { 5141 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING 5142 | KVM_VCPUEVENT_VALID_SIPI_VECTOR 5143 | KVM_VCPUEVENT_VALID_SHADOW 5144 | KVM_VCPUEVENT_VALID_SMM 5145 | KVM_VCPUEVENT_VALID_PAYLOAD 5146 | KVM_VCPUEVENT_VALID_TRIPLE_FAULT)) 5147 return -EINVAL; 5148 5149 if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) { 5150 if (!vcpu->kvm->arch.exception_payload_enabled) 5151 return -EINVAL; 5152 if (events->exception.pending) 5153 events->exception.injected = 0; 5154 else 5155 events->exception_has_payload = 0; 5156 } else { 5157 events->exception.pending = 0; 5158 events->exception_has_payload = 0; 5159 } 5160 5161 if ((events->exception.injected || events->exception.pending) && 5162 (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR)) 5163 return -EINVAL; 5164 5165 /* INITs are latched while in SMM */ 5166 if (events->flags & KVM_VCPUEVENT_VALID_SMM && 5167 (events->smi.smm || events->smi.pending) && 5168 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 5169 return -EINVAL; 5170 5171 process_nmi(vcpu); 5172 5173 /* 5174 * Flag that userspace is stuffing an exception, the next KVM_RUN will 5175 * morph the exception to a VM-Exit if appropriate. Do this only for 5176 * pending exceptions, already-injected exceptions are not subject to 5177 * intercpetion. Note, userspace that conflates pending and injected 5178 * is hosed, and will incorrectly convert an injected exception into a 5179 * pending exception, which in turn may cause a spurious VM-Exit. 5180 */ 5181 vcpu->arch.exception_from_userspace = events->exception.pending; 5182 5183 vcpu->arch.exception_vmexit.pending = false; 5184 5185 vcpu->arch.exception.injected = events->exception.injected; 5186 vcpu->arch.exception.pending = events->exception.pending; 5187 vcpu->arch.exception.vector = events->exception.nr; 5188 vcpu->arch.exception.has_error_code = events->exception.has_error_code; 5189 vcpu->arch.exception.error_code = events->exception.error_code; 5190 vcpu->arch.exception.has_payload = events->exception_has_payload; 5191 vcpu->arch.exception.payload = events->exception_payload; 5192 5193 vcpu->arch.interrupt.injected = events->interrupt.injected; 5194 vcpu->arch.interrupt.nr = events->interrupt.nr; 5195 vcpu->arch.interrupt.soft = events->interrupt.soft; 5196 if (events->flags & KVM_VCPUEVENT_VALID_SHADOW) 5197 static_call(kvm_x86_set_interrupt_shadow)(vcpu, 5198 events->interrupt.shadow); 5199 5200 vcpu->arch.nmi_injected = events->nmi.injected; 5201 if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) 5202 vcpu->arch.nmi_pending = events->nmi.pending; 5203 static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked); 5204 5205 if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR && 5206 lapic_in_kernel(vcpu)) 5207 vcpu->arch.apic->sipi_vector = events->sipi_vector; 5208 5209 if (events->flags & KVM_VCPUEVENT_VALID_SMM) { 5210 #ifdef CONFIG_KVM_SMM 5211 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) { 5212 kvm_leave_nested(vcpu); 5213 kvm_smm_changed(vcpu, events->smi.smm); 5214 } 5215 5216 vcpu->arch.smi_pending = events->smi.pending; 5217 5218 if (events->smi.smm) { 5219 if (events->smi.smm_inside_nmi) 5220 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK; 5221 else 5222 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK; 5223 } 5224 5225 #else 5226 if (events->smi.smm || events->smi.pending || 5227 events->smi.smm_inside_nmi) 5228 return -EINVAL; 5229 #endif 5230 5231 if (lapic_in_kernel(vcpu)) { 5232 if (events->smi.latched_init) 5233 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 5234 else 5235 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); 5236 } 5237 } 5238 5239 if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) { 5240 if (!vcpu->kvm->arch.triple_fault_event) 5241 return -EINVAL; 5242 if (events->triple_fault.pending) 5243 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 5244 else 5245 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 5246 } 5247 5248 kvm_make_request(KVM_REQ_EVENT, vcpu); 5249 5250 return 0; 5251 } 5252 5253 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, 5254 struct kvm_debugregs *dbgregs) 5255 { 5256 unsigned long val; 5257 5258 memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db)); 5259 kvm_get_dr(vcpu, 6, &val); 5260 dbgregs->dr6 = val; 5261 dbgregs->dr7 = vcpu->arch.dr7; 5262 dbgregs->flags = 0; 5263 memset(&dbgregs->reserved, 0, sizeof(dbgregs->reserved)); 5264 } 5265 5266 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, 5267 struct kvm_debugregs *dbgregs) 5268 { 5269 if (dbgregs->flags) 5270 return -EINVAL; 5271 5272 if (!kvm_dr6_valid(dbgregs->dr6)) 5273 return -EINVAL; 5274 if (!kvm_dr7_valid(dbgregs->dr7)) 5275 return -EINVAL; 5276 5277 memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db)); 5278 kvm_update_dr0123(vcpu); 5279 vcpu->arch.dr6 = dbgregs->dr6; 5280 vcpu->arch.dr7 = dbgregs->dr7; 5281 kvm_update_dr7(vcpu); 5282 5283 return 0; 5284 } 5285 5286 static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, 5287 struct kvm_xsave *guest_xsave) 5288 { 5289 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 5290 return; 5291 5292 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, 5293 guest_xsave->region, 5294 sizeof(guest_xsave->region), 5295 vcpu->arch.pkru); 5296 } 5297 5298 static void kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, 5299 u8 *state, unsigned int size) 5300 { 5301 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 5302 return; 5303 5304 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, 5305 state, size, vcpu->arch.pkru); 5306 } 5307 5308 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, 5309 struct kvm_xsave *guest_xsave) 5310 { 5311 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 5312 return 0; 5313 5314 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu, 5315 guest_xsave->region, 5316 kvm_caps.supported_xcr0, 5317 &vcpu->arch.pkru); 5318 } 5319 5320 static void kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu, 5321 struct kvm_xcrs *guest_xcrs) 5322 { 5323 if (!boot_cpu_has(X86_FEATURE_XSAVE)) { 5324 guest_xcrs->nr_xcrs = 0; 5325 return; 5326 } 5327 5328 guest_xcrs->nr_xcrs = 1; 5329 guest_xcrs->flags = 0; 5330 guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK; 5331 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0; 5332 } 5333 5334 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, 5335 struct kvm_xcrs *guest_xcrs) 5336 { 5337 int i, r = 0; 5338 5339 if (!boot_cpu_has(X86_FEATURE_XSAVE)) 5340 return -EINVAL; 5341 5342 if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags) 5343 return -EINVAL; 5344 5345 for (i = 0; i < guest_xcrs->nr_xcrs; i++) 5346 /* Only support XCR0 currently */ 5347 if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) { 5348 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK, 5349 guest_xcrs->xcrs[i].value); 5350 break; 5351 } 5352 if (r) 5353 r = -EINVAL; 5354 return r; 5355 } 5356 5357 /* 5358 * kvm_set_guest_paused() indicates to the guest kernel that it has been 5359 * stopped by the hypervisor. This function will be called from the host only. 5360 * EINVAL is returned when the host attempts to set the flag for a guest that 5361 * does not support pv clocks. 5362 */ 5363 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) 5364 { 5365 if (!vcpu->arch.pv_time.active) 5366 return -EINVAL; 5367 vcpu->arch.pvclock_set_guest_stopped_request = true; 5368 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 5369 return 0; 5370 } 5371 5372 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu, 5373 struct kvm_device_attr *attr) 5374 { 5375 int r; 5376 5377 switch (attr->attr) { 5378 case KVM_VCPU_TSC_OFFSET: 5379 r = 0; 5380 break; 5381 default: 5382 r = -ENXIO; 5383 } 5384 5385 return r; 5386 } 5387 5388 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu, 5389 struct kvm_device_attr *attr) 5390 { 5391 u64 __user *uaddr = kvm_get_attr_addr(attr); 5392 int r; 5393 5394 if (IS_ERR(uaddr)) 5395 return PTR_ERR(uaddr); 5396 5397 switch (attr->attr) { 5398 case KVM_VCPU_TSC_OFFSET: 5399 r = -EFAULT; 5400 if (put_user(vcpu->arch.l1_tsc_offset, uaddr)) 5401 break; 5402 r = 0; 5403 break; 5404 default: 5405 r = -ENXIO; 5406 } 5407 5408 return r; 5409 } 5410 5411 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu, 5412 struct kvm_device_attr *attr) 5413 { 5414 u64 __user *uaddr = kvm_get_attr_addr(attr); 5415 struct kvm *kvm = vcpu->kvm; 5416 int r; 5417 5418 if (IS_ERR(uaddr)) 5419 return PTR_ERR(uaddr); 5420 5421 switch (attr->attr) { 5422 case KVM_VCPU_TSC_OFFSET: { 5423 u64 offset, tsc, ns; 5424 unsigned long flags; 5425 bool matched; 5426 5427 r = -EFAULT; 5428 if (get_user(offset, uaddr)) 5429 break; 5430 5431 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 5432 5433 matched = (vcpu->arch.virtual_tsc_khz && 5434 kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz && 5435 kvm->arch.last_tsc_offset == offset); 5436 5437 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset; 5438 ns = get_kvmclock_base_ns(); 5439 5440 __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched); 5441 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 5442 5443 r = 0; 5444 break; 5445 } 5446 default: 5447 r = -ENXIO; 5448 } 5449 5450 return r; 5451 } 5452 5453 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu, 5454 unsigned int ioctl, 5455 void __user *argp) 5456 { 5457 struct kvm_device_attr attr; 5458 int r; 5459 5460 if (copy_from_user(&attr, argp, sizeof(attr))) 5461 return -EFAULT; 5462 5463 if (attr.group != KVM_VCPU_TSC_CTRL) 5464 return -ENXIO; 5465 5466 switch (ioctl) { 5467 case KVM_HAS_DEVICE_ATTR: 5468 r = kvm_arch_tsc_has_attr(vcpu, &attr); 5469 break; 5470 case KVM_GET_DEVICE_ATTR: 5471 r = kvm_arch_tsc_get_attr(vcpu, &attr); 5472 break; 5473 case KVM_SET_DEVICE_ATTR: 5474 r = kvm_arch_tsc_set_attr(vcpu, &attr); 5475 break; 5476 } 5477 5478 return r; 5479 } 5480 5481 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 5482 struct kvm_enable_cap *cap) 5483 { 5484 int r; 5485 uint16_t vmcs_version; 5486 void __user *user_ptr; 5487 5488 if (cap->flags) 5489 return -EINVAL; 5490 5491 switch (cap->cap) { 5492 case KVM_CAP_HYPERV_SYNIC2: 5493 if (cap->args[0]) 5494 return -EINVAL; 5495 fallthrough; 5496 5497 case KVM_CAP_HYPERV_SYNIC: 5498 if (!irqchip_in_kernel(vcpu->kvm)) 5499 return -EINVAL; 5500 return kvm_hv_activate_synic(vcpu, cap->cap == 5501 KVM_CAP_HYPERV_SYNIC2); 5502 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: 5503 if (!kvm_x86_ops.nested_ops->enable_evmcs) 5504 return -ENOTTY; 5505 r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); 5506 if (!r) { 5507 user_ptr = (void __user *)(uintptr_t)cap->args[0]; 5508 if (copy_to_user(user_ptr, &vmcs_version, 5509 sizeof(vmcs_version))) 5510 r = -EFAULT; 5511 } 5512 return r; 5513 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: 5514 if (!kvm_x86_ops.enable_l2_tlb_flush) 5515 return -ENOTTY; 5516 5517 return static_call(kvm_x86_enable_l2_tlb_flush)(vcpu); 5518 5519 case KVM_CAP_HYPERV_ENFORCE_CPUID: 5520 return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]); 5521 5522 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: 5523 vcpu->arch.pv_cpuid.enforce = cap->args[0]; 5524 if (vcpu->arch.pv_cpuid.enforce) 5525 kvm_update_pv_runtime(vcpu); 5526 5527 return 0; 5528 default: 5529 return -EINVAL; 5530 } 5531 } 5532 5533 long kvm_arch_vcpu_ioctl(struct file *filp, 5534 unsigned int ioctl, unsigned long arg) 5535 { 5536 struct kvm_vcpu *vcpu = filp->private_data; 5537 void __user *argp = (void __user *)arg; 5538 int r; 5539 union { 5540 struct kvm_sregs2 *sregs2; 5541 struct kvm_lapic_state *lapic; 5542 struct kvm_xsave *xsave; 5543 struct kvm_xcrs *xcrs; 5544 void *buffer; 5545 } u; 5546 5547 vcpu_load(vcpu); 5548 5549 u.buffer = NULL; 5550 switch (ioctl) { 5551 case KVM_GET_LAPIC: { 5552 r = -EINVAL; 5553 if (!lapic_in_kernel(vcpu)) 5554 goto out; 5555 u.lapic = kzalloc(sizeof(struct kvm_lapic_state), 5556 GFP_KERNEL_ACCOUNT); 5557 5558 r = -ENOMEM; 5559 if (!u.lapic) 5560 goto out; 5561 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic); 5562 if (r) 5563 goto out; 5564 r = -EFAULT; 5565 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state))) 5566 goto out; 5567 r = 0; 5568 break; 5569 } 5570 case KVM_SET_LAPIC: { 5571 r = -EINVAL; 5572 if (!lapic_in_kernel(vcpu)) 5573 goto out; 5574 u.lapic = memdup_user(argp, sizeof(*u.lapic)); 5575 if (IS_ERR(u.lapic)) { 5576 r = PTR_ERR(u.lapic); 5577 goto out_nofree; 5578 } 5579 5580 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic); 5581 break; 5582 } 5583 case KVM_INTERRUPT: { 5584 struct kvm_interrupt irq; 5585 5586 r = -EFAULT; 5587 if (copy_from_user(&irq, argp, sizeof(irq))) 5588 goto out; 5589 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); 5590 break; 5591 } 5592 case KVM_NMI: { 5593 r = kvm_vcpu_ioctl_nmi(vcpu); 5594 break; 5595 } 5596 case KVM_SMI: { 5597 r = kvm_inject_smi(vcpu); 5598 break; 5599 } 5600 case KVM_SET_CPUID: { 5601 struct kvm_cpuid __user *cpuid_arg = argp; 5602 struct kvm_cpuid cpuid; 5603 5604 r = -EFAULT; 5605 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 5606 goto out; 5607 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); 5608 break; 5609 } 5610 case KVM_SET_CPUID2: { 5611 struct kvm_cpuid2 __user *cpuid_arg = argp; 5612 struct kvm_cpuid2 cpuid; 5613 5614 r = -EFAULT; 5615 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 5616 goto out; 5617 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, 5618 cpuid_arg->entries); 5619 break; 5620 } 5621 case KVM_GET_CPUID2: { 5622 struct kvm_cpuid2 __user *cpuid_arg = argp; 5623 struct kvm_cpuid2 cpuid; 5624 5625 r = -EFAULT; 5626 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid))) 5627 goto out; 5628 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, 5629 cpuid_arg->entries); 5630 if (r) 5631 goto out; 5632 r = -EFAULT; 5633 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid))) 5634 goto out; 5635 r = 0; 5636 break; 5637 } 5638 case KVM_GET_MSRS: { 5639 int idx = srcu_read_lock(&vcpu->kvm->srcu); 5640 r = msr_io(vcpu, argp, do_get_msr, 1); 5641 srcu_read_unlock(&vcpu->kvm->srcu, idx); 5642 break; 5643 } 5644 case KVM_SET_MSRS: { 5645 int idx = srcu_read_lock(&vcpu->kvm->srcu); 5646 r = msr_io(vcpu, argp, do_set_msr, 0); 5647 srcu_read_unlock(&vcpu->kvm->srcu, idx); 5648 break; 5649 } 5650 case KVM_TPR_ACCESS_REPORTING: { 5651 struct kvm_tpr_access_ctl tac; 5652 5653 r = -EFAULT; 5654 if (copy_from_user(&tac, argp, sizeof(tac))) 5655 goto out; 5656 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac); 5657 if (r) 5658 goto out; 5659 r = -EFAULT; 5660 if (copy_to_user(argp, &tac, sizeof(tac))) 5661 goto out; 5662 r = 0; 5663 break; 5664 }; 5665 case KVM_SET_VAPIC_ADDR: { 5666 struct kvm_vapic_addr va; 5667 int idx; 5668 5669 r = -EINVAL; 5670 if (!lapic_in_kernel(vcpu)) 5671 goto out; 5672 r = -EFAULT; 5673 if (copy_from_user(&va, argp, sizeof(va))) 5674 goto out; 5675 idx = srcu_read_lock(&vcpu->kvm->srcu); 5676 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr); 5677 srcu_read_unlock(&vcpu->kvm->srcu, idx); 5678 break; 5679 } 5680 case KVM_X86_SETUP_MCE: { 5681 u64 mcg_cap; 5682 5683 r = -EFAULT; 5684 if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap))) 5685 goto out; 5686 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap); 5687 break; 5688 } 5689 case KVM_X86_SET_MCE: { 5690 struct kvm_x86_mce mce; 5691 5692 r = -EFAULT; 5693 if (copy_from_user(&mce, argp, sizeof(mce))) 5694 goto out; 5695 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce); 5696 break; 5697 } 5698 case KVM_GET_VCPU_EVENTS: { 5699 struct kvm_vcpu_events events; 5700 5701 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events); 5702 5703 r = -EFAULT; 5704 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events))) 5705 break; 5706 r = 0; 5707 break; 5708 } 5709 case KVM_SET_VCPU_EVENTS: { 5710 struct kvm_vcpu_events events; 5711 5712 r = -EFAULT; 5713 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events))) 5714 break; 5715 5716 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events); 5717 break; 5718 } 5719 case KVM_GET_DEBUGREGS: { 5720 struct kvm_debugregs dbgregs; 5721 5722 kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs); 5723 5724 r = -EFAULT; 5725 if (copy_to_user(argp, &dbgregs, 5726 sizeof(struct kvm_debugregs))) 5727 break; 5728 r = 0; 5729 break; 5730 } 5731 case KVM_SET_DEBUGREGS: { 5732 struct kvm_debugregs dbgregs; 5733 5734 r = -EFAULT; 5735 if (copy_from_user(&dbgregs, argp, 5736 sizeof(struct kvm_debugregs))) 5737 break; 5738 5739 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs); 5740 break; 5741 } 5742 case KVM_GET_XSAVE: { 5743 r = -EINVAL; 5744 if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave)) 5745 break; 5746 5747 u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT); 5748 r = -ENOMEM; 5749 if (!u.xsave) 5750 break; 5751 5752 kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave); 5753 5754 r = -EFAULT; 5755 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave))) 5756 break; 5757 r = 0; 5758 break; 5759 } 5760 case KVM_SET_XSAVE: { 5761 int size = vcpu->arch.guest_fpu.uabi_size; 5762 5763 u.xsave = memdup_user(argp, size); 5764 if (IS_ERR(u.xsave)) { 5765 r = PTR_ERR(u.xsave); 5766 goto out_nofree; 5767 } 5768 5769 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave); 5770 break; 5771 } 5772 5773 case KVM_GET_XSAVE2: { 5774 int size = vcpu->arch.guest_fpu.uabi_size; 5775 5776 u.xsave = kzalloc(size, GFP_KERNEL_ACCOUNT); 5777 r = -ENOMEM; 5778 if (!u.xsave) 5779 break; 5780 5781 kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size); 5782 5783 r = -EFAULT; 5784 if (copy_to_user(argp, u.xsave, size)) 5785 break; 5786 5787 r = 0; 5788 break; 5789 } 5790 5791 case KVM_GET_XCRS: { 5792 u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT); 5793 r = -ENOMEM; 5794 if (!u.xcrs) 5795 break; 5796 5797 kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs); 5798 5799 r = -EFAULT; 5800 if (copy_to_user(argp, u.xcrs, 5801 sizeof(struct kvm_xcrs))) 5802 break; 5803 r = 0; 5804 break; 5805 } 5806 case KVM_SET_XCRS: { 5807 u.xcrs = memdup_user(argp, sizeof(*u.xcrs)); 5808 if (IS_ERR(u.xcrs)) { 5809 r = PTR_ERR(u.xcrs); 5810 goto out_nofree; 5811 } 5812 5813 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs); 5814 break; 5815 } 5816 case KVM_SET_TSC_KHZ: { 5817 u32 user_tsc_khz; 5818 5819 r = -EINVAL; 5820 user_tsc_khz = (u32)arg; 5821 5822 if (kvm_caps.has_tsc_control && 5823 user_tsc_khz >= kvm_caps.max_guest_tsc_khz) 5824 goto out; 5825 5826 if (user_tsc_khz == 0) 5827 user_tsc_khz = tsc_khz; 5828 5829 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz)) 5830 r = 0; 5831 5832 goto out; 5833 } 5834 case KVM_GET_TSC_KHZ: { 5835 r = vcpu->arch.virtual_tsc_khz; 5836 goto out; 5837 } 5838 case KVM_KVMCLOCK_CTRL: { 5839 r = kvm_set_guest_paused(vcpu); 5840 goto out; 5841 } 5842 case KVM_ENABLE_CAP: { 5843 struct kvm_enable_cap cap; 5844 5845 r = -EFAULT; 5846 if (copy_from_user(&cap, argp, sizeof(cap))) 5847 goto out; 5848 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 5849 break; 5850 } 5851 case KVM_GET_NESTED_STATE: { 5852 struct kvm_nested_state __user *user_kvm_nested_state = argp; 5853 u32 user_data_size; 5854 5855 r = -EINVAL; 5856 if (!kvm_x86_ops.nested_ops->get_state) 5857 break; 5858 5859 BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); 5860 r = -EFAULT; 5861 if (get_user(user_data_size, &user_kvm_nested_state->size)) 5862 break; 5863 5864 r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state, 5865 user_data_size); 5866 if (r < 0) 5867 break; 5868 5869 if (r > user_data_size) { 5870 if (put_user(r, &user_kvm_nested_state->size)) 5871 r = -EFAULT; 5872 else 5873 r = -E2BIG; 5874 break; 5875 } 5876 5877 r = 0; 5878 break; 5879 } 5880 case KVM_SET_NESTED_STATE: { 5881 struct kvm_nested_state __user *user_kvm_nested_state = argp; 5882 struct kvm_nested_state kvm_state; 5883 int idx; 5884 5885 r = -EINVAL; 5886 if (!kvm_x86_ops.nested_ops->set_state) 5887 break; 5888 5889 r = -EFAULT; 5890 if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state))) 5891 break; 5892 5893 r = -EINVAL; 5894 if (kvm_state.size < sizeof(kvm_state)) 5895 break; 5896 5897 if (kvm_state.flags & 5898 ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE 5899 | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING 5900 | KVM_STATE_NESTED_GIF_SET)) 5901 break; 5902 5903 /* nested_run_pending implies guest_mode. */ 5904 if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING) 5905 && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE)) 5906 break; 5907 5908 idx = srcu_read_lock(&vcpu->kvm->srcu); 5909 r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state); 5910 srcu_read_unlock(&vcpu->kvm->srcu, idx); 5911 break; 5912 } 5913 case KVM_GET_SUPPORTED_HV_CPUID: 5914 r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp); 5915 break; 5916 #ifdef CONFIG_KVM_XEN 5917 case KVM_XEN_VCPU_GET_ATTR: { 5918 struct kvm_xen_vcpu_attr xva; 5919 5920 r = -EFAULT; 5921 if (copy_from_user(&xva, argp, sizeof(xva))) 5922 goto out; 5923 r = kvm_xen_vcpu_get_attr(vcpu, &xva); 5924 if (!r && copy_to_user(argp, &xva, sizeof(xva))) 5925 r = -EFAULT; 5926 break; 5927 } 5928 case KVM_XEN_VCPU_SET_ATTR: { 5929 struct kvm_xen_vcpu_attr xva; 5930 5931 r = -EFAULT; 5932 if (copy_from_user(&xva, argp, sizeof(xva))) 5933 goto out; 5934 r = kvm_xen_vcpu_set_attr(vcpu, &xva); 5935 break; 5936 } 5937 #endif 5938 case KVM_GET_SREGS2: { 5939 u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL); 5940 r = -ENOMEM; 5941 if (!u.sregs2) 5942 goto out; 5943 __get_sregs2(vcpu, u.sregs2); 5944 r = -EFAULT; 5945 if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) 5946 goto out; 5947 r = 0; 5948 break; 5949 } 5950 case KVM_SET_SREGS2: { 5951 u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2)); 5952 if (IS_ERR(u.sregs2)) { 5953 r = PTR_ERR(u.sregs2); 5954 u.sregs2 = NULL; 5955 goto out; 5956 } 5957 r = __set_sregs2(vcpu, u.sregs2); 5958 break; 5959 } 5960 case KVM_HAS_DEVICE_ATTR: 5961 case KVM_GET_DEVICE_ATTR: 5962 case KVM_SET_DEVICE_ATTR: 5963 r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp); 5964 break; 5965 default: 5966 r = -EINVAL; 5967 } 5968 out: 5969 kfree(u.buffer); 5970 out_nofree: 5971 vcpu_put(vcpu); 5972 return r; 5973 } 5974 5975 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 5976 { 5977 return VM_FAULT_SIGBUS; 5978 } 5979 5980 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) 5981 { 5982 int ret; 5983 5984 if (addr > (unsigned int)(-3 * PAGE_SIZE)) 5985 return -EINVAL; 5986 ret = static_call(kvm_x86_set_tss_addr)(kvm, addr); 5987 return ret; 5988 } 5989 5990 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm, 5991 u64 ident_addr) 5992 { 5993 return static_call(kvm_x86_set_identity_map_addr)(kvm, ident_addr); 5994 } 5995 5996 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, 5997 unsigned long kvm_nr_mmu_pages) 5998 { 5999 if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) 6000 return -EINVAL; 6001 6002 mutex_lock(&kvm->slots_lock); 6003 6004 kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); 6005 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; 6006 6007 mutex_unlock(&kvm->slots_lock); 6008 return 0; 6009 } 6010 6011 static unsigned long kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) 6012 { 6013 return kvm->arch.n_max_mmu_pages; 6014 } 6015 6016 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) 6017 { 6018 struct kvm_pic *pic = kvm->arch.vpic; 6019 int r; 6020 6021 r = 0; 6022 switch (chip->chip_id) { 6023 case KVM_IRQCHIP_PIC_MASTER: 6024 memcpy(&chip->chip.pic, &pic->pics[0], 6025 sizeof(struct kvm_pic_state)); 6026 break; 6027 case KVM_IRQCHIP_PIC_SLAVE: 6028 memcpy(&chip->chip.pic, &pic->pics[1], 6029 sizeof(struct kvm_pic_state)); 6030 break; 6031 case KVM_IRQCHIP_IOAPIC: 6032 kvm_get_ioapic(kvm, &chip->chip.ioapic); 6033 break; 6034 default: 6035 r = -EINVAL; 6036 break; 6037 } 6038 return r; 6039 } 6040 6041 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) 6042 { 6043 struct kvm_pic *pic = kvm->arch.vpic; 6044 int r; 6045 6046 r = 0; 6047 switch (chip->chip_id) { 6048 case KVM_IRQCHIP_PIC_MASTER: 6049 spin_lock(&pic->lock); 6050 memcpy(&pic->pics[0], &chip->chip.pic, 6051 sizeof(struct kvm_pic_state)); 6052 spin_unlock(&pic->lock); 6053 break; 6054 case KVM_IRQCHIP_PIC_SLAVE: 6055 spin_lock(&pic->lock); 6056 memcpy(&pic->pics[1], &chip->chip.pic, 6057 sizeof(struct kvm_pic_state)); 6058 spin_unlock(&pic->lock); 6059 break; 6060 case KVM_IRQCHIP_IOAPIC: 6061 kvm_set_ioapic(kvm, &chip->chip.ioapic); 6062 break; 6063 default: 6064 r = -EINVAL; 6065 break; 6066 } 6067 kvm_pic_update_irq(pic); 6068 return r; 6069 } 6070 6071 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps) 6072 { 6073 struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state; 6074 6075 BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels)); 6076 6077 mutex_lock(&kps->lock); 6078 memcpy(ps, &kps->channels, sizeof(*ps)); 6079 mutex_unlock(&kps->lock); 6080 return 0; 6081 } 6082 6083 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps) 6084 { 6085 int i; 6086 struct kvm_pit *pit = kvm->arch.vpit; 6087 6088 mutex_lock(&pit->pit_state.lock); 6089 memcpy(&pit->pit_state.channels, ps, sizeof(*ps)); 6090 for (i = 0; i < 3; i++) 6091 kvm_pit_load_count(pit, i, ps->channels[i].count, 0); 6092 mutex_unlock(&pit->pit_state.lock); 6093 return 0; 6094 } 6095 6096 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps) 6097 { 6098 mutex_lock(&kvm->arch.vpit->pit_state.lock); 6099 memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels, 6100 sizeof(ps->channels)); 6101 ps->flags = kvm->arch.vpit->pit_state.flags; 6102 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 6103 memset(&ps->reserved, 0, sizeof(ps->reserved)); 6104 return 0; 6105 } 6106 6107 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps) 6108 { 6109 int start = 0; 6110 int i; 6111 u32 prev_legacy, cur_legacy; 6112 struct kvm_pit *pit = kvm->arch.vpit; 6113 6114 mutex_lock(&pit->pit_state.lock); 6115 prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY; 6116 cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY; 6117 if (!prev_legacy && cur_legacy) 6118 start = 1; 6119 memcpy(&pit->pit_state.channels, &ps->channels, 6120 sizeof(pit->pit_state.channels)); 6121 pit->pit_state.flags = ps->flags; 6122 for (i = 0; i < 3; i++) 6123 kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count, 6124 start && i == 0); 6125 mutex_unlock(&pit->pit_state.lock); 6126 return 0; 6127 } 6128 6129 static int kvm_vm_ioctl_reinject(struct kvm *kvm, 6130 struct kvm_reinject_control *control) 6131 { 6132 struct kvm_pit *pit = kvm->arch.vpit; 6133 6134 /* pit->pit_state.lock was overloaded to prevent userspace from getting 6135 * an inconsistent state after running multiple KVM_REINJECT_CONTROL 6136 * ioctls in parallel. Use a separate lock if that ioctl isn't rare. 6137 */ 6138 mutex_lock(&pit->pit_state.lock); 6139 kvm_pit_set_reinject(pit, control->pit_reinject); 6140 mutex_unlock(&pit->pit_state.lock); 6141 6142 return 0; 6143 } 6144 6145 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) 6146 { 6147 6148 /* 6149 * Flush all CPUs' dirty log buffers to the dirty_bitmap. Called 6150 * before reporting dirty_bitmap to userspace. KVM flushes the buffers 6151 * on all VM-Exits, thus we only need to kick running vCPUs to force a 6152 * VM-Exit. 6153 */ 6154 struct kvm_vcpu *vcpu; 6155 unsigned long i; 6156 6157 kvm_for_each_vcpu(i, vcpu, kvm) 6158 kvm_vcpu_kick(vcpu); 6159 } 6160 6161 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, 6162 bool line_status) 6163 { 6164 if (!irqchip_in_kernel(kvm)) 6165 return -ENXIO; 6166 6167 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 6168 irq_event->irq, irq_event->level, 6169 line_status); 6170 return 0; 6171 } 6172 6173 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, 6174 struct kvm_enable_cap *cap) 6175 { 6176 int r; 6177 6178 if (cap->flags) 6179 return -EINVAL; 6180 6181 switch (cap->cap) { 6182 case KVM_CAP_DISABLE_QUIRKS2: 6183 r = -EINVAL; 6184 if (cap->args[0] & ~KVM_X86_VALID_QUIRKS) 6185 break; 6186 fallthrough; 6187 case KVM_CAP_DISABLE_QUIRKS: 6188 kvm->arch.disabled_quirks = cap->args[0]; 6189 r = 0; 6190 break; 6191 case KVM_CAP_SPLIT_IRQCHIP: { 6192 mutex_lock(&kvm->lock); 6193 r = -EINVAL; 6194 if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS) 6195 goto split_irqchip_unlock; 6196 r = -EEXIST; 6197 if (irqchip_in_kernel(kvm)) 6198 goto split_irqchip_unlock; 6199 if (kvm->created_vcpus) 6200 goto split_irqchip_unlock; 6201 r = kvm_setup_empty_irq_routing(kvm); 6202 if (r) 6203 goto split_irqchip_unlock; 6204 /* Pairs with irqchip_in_kernel. */ 6205 smp_wmb(); 6206 kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT; 6207 kvm->arch.nr_reserved_ioapic_pins = cap->args[0]; 6208 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); 6209 r = 0; 6210 split_irqchip_unlock: 6211 mutex_unlock(&kvm->lock); 6212 break; 6213 } 6214 case KVM_CAP_X2APIC_API: 6215 r = -EINVAL; 6216 if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS) 6217 break; 6218 6219 if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS) 6220 kvm->arch.x2apic_format = true; 6221 if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) 6222 kvm->arch.x2apic_broadcast_quirk_disabled = true; 6223 6224 r = 0; 6225 break; 6226 case KVM_CAP_X86_DISABLE_EXITS: 6227 r = -EINVAL; 6228 if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS) 6229 break; 6230 6231 if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) && 6232 kvm_can_mwait_in_guest()) 6233 kvm->arch.mwait_in_guest = true; 6234 if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT) 6235 kvm->arch.hlt_in_guest = true; 6236 if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE) 6237 kvm->arch.pause_in_guest = true; 6238 if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE) 6239 kvm->arch.cstate_in_guest = true; 6240 r = 0; 6241 break; 6242 case KVM_CAP_MSR_PLATFORM_INFO: 6243 kvm->arch.guest_can_read_msr_platform_info = cap->args[0]; 6244 r = 0; 6245 break; 6246 case KVM_CAP_EXCEPTION_PAYLOAD: 6247 kvm->arch.exception_payload_enabled = cap->args[0]; 6248 r = 0; 6249 break; 6250 case KVM_CAP_X86_TRIPLE_FAULT_EVENT: 6251 kvm->arch.triple_fault_event = cap->args[0]; 6252 r = 0; 6253 break; 6254 case KVM_CAP_X86_USER_SPACE_MSR: 6255 r = -EINVAL; 6256 if (cap->args[0] & ~KVM_MSR_EXIT_REASON_VALID_MASK) 6257 break; 6258 kvm->arch.user_space_msr_mask = cap->args[0]; 6259 r = 0; 6260 break; 6261 case KVM_CAP_X86_BUS_LOCK_EXIT: 6262 r = -EINVAL; 6263 if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE) 6264 break; 6265 6266 if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) && 6267 (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)) 6268 break; 6269 6270 if (kvm_caps.has_bus_lock_exit && 6271 cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT) 6272 kvm->arch.bus_lock_detection_enabled = true; 6273 r = 0; 6274 break; 6275 #ifdef CONFIG_X86_SGX_KVM 6276 case KVM_CAP_SGX_ATTRIBUTE: { 6277 unsigned long allowed_attributes = 0; 6278 6279 r = sgx_set_attribute(&allowed_attributes, cap->args[0]); 6280 if (r) 6281 break; 6282 6283 /* KVM only supports the PROVISIONKEY privileged attribute. */ 6284 if ((allowed_attributes & SGX_ATTR_PROVISIONKEY) && 6285 !(allowed_attributes & ~SGX_ATTR_PROVISIONKEY)) 6286 kvm->arch.sgx_provisioning_allowed = true; 6287 else 6288 r = -EINVAL; 6289 break; 6290 } 6291 #endif 6292 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: 6293 r = -EINVAL; 6294 if (!kvm_x86_ops.vm_copy_enc_context_from) 6295 break; 6296 6297 r = static_call(kvm_x86_vm_copy_enc_context_from)(kvm, cap->args[0]); 6298 break; 6299 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM: 6300 r = -EINVAL; 6301 if (!kvm_x86_ops.vm_move_enc_context_from) 6302 break; 6303 6304 r = static_call(kvm_x86_vm_move_enc_context_from)(kvm, cap->args[0]); 6305 break; 6306 case KVM_CAP_EXIT_HYPERCALL: 6307 if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) { 6308 r = -EINVAL; 6309 break; 6310 } 6311 kvm->arch.hypercall_exit_enabled = cap->args[0]; 6312 r = 0; 6313 break; 6314 case KVM_CAP_EXIT_ON_EMULATION_FAILURE: 6315 r = -EINVAL; 6316 if (cap->args[0] & ~1) 6317 break; 6318 kvm->arch.exit_on_emulation_error = cap->args[0]; 6319 r = 0; 6320 break; 6321 case KVM_CAP_PMU_CAPABILITY: 6322 r = -EINVAL; 6323 if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK)) 6324 break; 6325 6326 mutex_lock(&kvm->lock); 6327 if (!kvm->created_vcpus) { 6328 kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE); 6329 r = 0; 6330 } 6331 mutex_unlock(&kvm->lock); 6332 break; 6333 case KVM_CAP_MAX_VCPU_ID: 6334 r = -EINVAL; 6335 if (cap->args[0] > KVM_MAX_VCPU_IDS) 6336 break; 6337 6338 mutex_lock(&kvm->lock); 6339 if (kvm->arch.max_vcpu_ids == cap->args[0]) { 6340 r = 0; 6341 } else if (!kvm->arch.max_vcpu_ids) { 6342 kvm->arch.max_vcpu_ids = cap->args[0]; 6343 r = 0; 6344 } 6345 mutex_unlock(&kvm->lock); 6346 break; 6347 case KVM_CAP_X86_NOTIFY_VMEXIT: 6348 r = -EINVAL; 6349 if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS) 6350 break; 6351 if (!kvm_caps.has_notify_vmexit) 6352 break; 6353 if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED)) 6354 break; 6355 mutex_lock(&kvm->lock); 6356 if (!kvm->created_vcpus) { 6357 kvm->arch.notify_window = cap->args[0] >> 32; 6358 kvm->arch.notify_vmexit_flags = (u32)cap->args[0]; 6359 r = 0; 6360 } 6361 mutex_unlock(&kvm->lock); 6362 break; 6363 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES: 6364 r = -EINVAL; 6365 6366 /* 6367 * Since the risk of disabling NX hugepages is a guest crashing 6368 * the system, ensure the userspace process has permission to 6369 * reboot the system. 6370 * 6371 * Note that unlike the reboot() syscall, the process must have 6372 * this capability in the root namespace because exposing 6373 * /dev/kvm into a container does not limit the scope of the 6374 * iTLB multihit bug to that container. In other words, 6375 * this must use capable(), not ns_capable(). 6376 */ 6377 if (!capable(CAP_SYS_BOOT)) { 6378 r = -EPERM; 6379 break; 6380 } 6381 6382 if (cap->args[0]) 6383 break; 6384 6385 mutex_lock(&kvm->lock); 6386 if (!kvm->created_vcpus) { 6387 kvm->arch.disable_nx_huge_pages = true; 6388 r = 0; 6389 } 6390 mutex_unlock(&kvm->lock); 6391 break; 6392 default: 6393 r = -EINVAL; 6394 break; 6395 } 6396 return r; 6397 } 6398 6399 static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow) 6400 { 6401 struct kvm_x86_msr_filter *msr_filter; 6402 6403 msr_filter = kzalloc(sizeof(*msr_filter), GFP_KERNEL_ACCOUNT); 6404 if (!msr_filter) 6405 return NULL; 6406 6407 msr_filter->default_allow = default_allow; 6408 return msr_filter; 6409 } 6410 6411 static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter) 6412 { 6413 u32 i; 6414 6415 if (!msr_filter) 6416 return; 6417 6418 for (i = 0; i < msr_filter->count; i++) 6419 kfree(msr_filter->ranges[i].bitmap); 6420 6421 kfree(msr_filter); 6422 } 6423 6424 static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter, 6425 struct kvm_msr_filter_range *user_range) 6426 { 6427 unsigned long *bitmap = NULL; 6428 size_t bitmap_size; 6429 6430 if (!user_range->nmsrs) 6431 return 0; 6432 6433 if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK) 6434 return -EINVAL; 6435 6436 if (!user_range->flags) 6437 return -EINVAL; 6438 6439 bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long); 6440 if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE) 6441 return -EINVAL; 6442 6443 bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size); 6444 if (IS_ERR(bitmap)) 6445 return PTR_ERR(bitmap); 6446 6447 msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) { 6448 .flags = user_range->flags, 6449 .base = user_range->base, 6450 .nmsrs = user_range->nmsrs, 6451 .bitmap = bitmap, 6452 }; 6453 6454 msr_filter->count++; 6455 return 0; 6456 } 6457 6458 static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, 6459 struct kvm_msr_filter *filter) 6460 { 6461 struct kvm_x86_msr_filter *new_filter, *old_filter; 6462 bool default_allow; 6463 bool empty = true; 6464 int r = 0; 6465 u32 i; 6466 6467 if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK) 6468 return -EINVAL; 6469 6470 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) 6471 empty &= !filter->ranges[i].nmsrs; 6472 6473 default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY); 6474 if (empty && !default_allow) 6475 return -EINVAL; 6476 6477 new_filter = kvm_alloc_msr_filter(default_allow); 6478 if (!new_filter) 6479 return -ENOMEM; 6480 6481 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) { 6482 r = kvm_add_msr_filter(new_filter, &filter->ranges[i]); 6483 if (r) { 6484 kvm_free_msr_filter(new_filter); 6485 return r; 6486 } 6487 } 6488 6489 mutex_lock(&kvm->lock); 6490 6491 /* The per-VM filter is protected by kvm->lock... */ 6492 old_filter = srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1); 6493 6494 rcu_assign_pointer(kvm->arch.msr_filter, new_filter); 6495 synchronize_srcu(&kvm->srcu); 6496 6497 kvm_free_msr_filter(old_filter); 6498 6499 kvm_make_all_cpus_request(kvm, KVM_REQ_MSR_FILTER_CHANGED); 6500 mutex_unlock(&kvm->lock); 6501 6502 return 0; 6503 } 6504 6505 #ifdef CONFIG_KVM_COMPAT 6506 /* for KVM_X86_SET_MSR_FILTER */ 6507 struct kvm_msr_filter_range_compat { 6508 __u32 flags; 6509 __u32 nmsrs; 6510 __u32 base; 6511 __u32 bitmap; 6512 }; 6513 6514 struct kvm_msr_filter_compat { 6515 __u32 flags; 6516 struct kvm_msr_filter_range_compat ranges[KVM_MSR_FILTER_MAX_RANGES]; 6517 }; 6518 6519 #define KVM_X86_SET_MSR_FILTER_COMPAT _IOW(KVMIO, 0xc6, struct kvm_msr_filter_compat) 6520 6521 long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl, 6522 unsigned long arg) 6523 { 6524 void __user *argp = (void __user *)arg; 6525 struct kvm *kvm = filp->private_data; 6526 long r = -ENOTTY; 6527 6528 switch (ioctl) { 6529 case KVM_X86_SET_MSR_FILTER_COMPAT: { 6530 struct kvm_msr_filter __user *user_msr_filter = argp; 6531 struct kvm_msr_filter_compat filter_compat; 6532 struct kvm_msr_filter filter; 6533 int i; 6534 6535 if (copy_from_user(&filter_compat, user_msr_filter, 6536 sizeof(filter_compat))) 6537 return -EFAULT; 6538 6539 filter.flags = filter_compat.flags; 6540 for (i = 0; i < ARRAY_SIZE(filter.ranges); i++) { 6541 struct kvm_msr_filter_range_compat *cr; 6542 6543 cr = &filter_compat.ranges[i]; 6544 filter.ranges[i] = (struct kvm_msr_filter_range) { 6545 .flags = cr->flags, 6546 .nmsrs = cr->nmsrs, 6547 .base = cr->base, 6548 .bitmap = (__u8 *)(ulong)cr->bitmap, 6549 }; 6550 } 6551 6552 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter); 6553 break; 6554 } 6555 } 6556 6557 return r; 6558 } 6559 #endif 6560 6561 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER 6562 static int kvm_arch_suspend_notifier(struct kvm *kvm) 6563 { 6564 struct kvm_vcpu *vcpu; 6565 unsigned long i; 6566 int ret = 0; 6567 6568 mutex_lock(&kvm->lock); 6569 kvm_for_each_vcpu(i, vcpu, kvm) { 6570 if (!vcpu->arch.pv_time.active) 6571 continue; 6572 6573 ret = kvm_set_guest_paused(vcpu); 6574 if (ret) { 6575 kvm_err("Failed to pause guest VCPU%d: %d\n", 6576 vcpu->vcpu_id, ret); 6577 break; 6578 } 6579 } 6580 mutex_unlock(&kvm->lock); 6581 6582 return ret ? NOTIFY_BAD : NOTIFY_DONE; 6583 } 6584 6585 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state) 6586 { 6587 switch (state) { 6588 case PM_HIBERNATION_PREPARE: 6589 case PM_SUSPEND_PREPARE: 6590 return kvm_arch_suspend_notifier(kvm); 6591 } 6592 6593 return NOTIFY_DONE; 6594 } 6595 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ 6596 6597 static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp) 6598 { 6599 struct kvm_clock_data data = { 0 }; 6600 6601 get_kvmclock(kvm, &data); 6602 if (copy_to_user(argp, &data, sizeof(data))) 6603 return -EFAULT; 6604 6605 return 0; 6606 } 6607 6608 static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp) 6609 { 6610 struct kvm_arch *ka = &kvm->arch; 6611 struct kvm_clock_data data; 6612 u64 now_raw_ns; 6613 6614 if (copy_from_user(&data, argp, sizeof(data))) 6615 return -EFAULT; 6616 6617 /* 6618 * Only KVM_CLOCK_REALTIME is used, but allow passing the 6619 * result of KVM_GET_CLOCK back to KVM_SET_CLOCK. 6620 */ 6621 if (data.flags & ~KVM_CLOCK_VALID_FLAGS) 6622 return -EINVAL; 6623 6624 kvm_hv_request_tsc_page_update(kvm); 6625 kvm_start_pvclock_update(kvm); 6626 pvclock_update_vm_gtod_copy(kvm); 6627 6628 /* 6629 * This pairs with kvm_guest_time_update(): when masterclock is 6630 * in use, we use master_kernel_ns + kvmclock_offset to set 6631 * unsigned 'system_time' so if we use get_kvmclock_ns() (which 6632 * is slightly ahead) here we risk going negative on unsigned 6633 * 'system_time' when 'data.clock' is very small. 6634 */ 6635 if (data.flags & KVM_CLOCK_REALTIME) { 6636 u64 now_real_ns = ktime_get_real_ns(); 6637 6638 /* 6639 * Avoid stepping the kvmclock backwards. 6640 */ 6641 if (now_real_ns > data.realtime) 6642 data.clock += now_real_ns - data.realtime; 6643 } 6644 6645 if (ka->use_master_clock) 6646 now_raw_ns = ka->master_kernel_ns; 6647 else 6648 now_raw_ns = get_kvmclock_base_ns(); 6649 ka->kvmclock_offset = data.clock - now_raw_ns; 6650 kvm_end_pvclock_update(kvm); 6651 return 0; 6652 } 6653 6654 long kvm_arch_vm_ioctl(struct file *filp, 6655 unsigned int ioctl, unsigned long arg) 6656 { 6657 struct kvm *kvm = filp->private_data; 6658 void __user *argp = (void __user *)arg; 6659 int r = -ENOTTY; 6660 /* 6661 * This union makes it completely explicit to gcc-3.x 6662 * that these two variables' stack usage should be 6663 * combined, not added together. 6664 */ 6665 union { 6666 struct kvm_pit_state ps; 6667 struct kvm_pit_state2 ps2; 6668 struct kvm_pit_config pit_config; 6669 } u; 6670 6671 switch (ioctl) { 6672 case KVM_SET_TSS_ADDR: 6673 r = kvm_vm_ioctl_set_tss_addr(kvm, arg); 6674 break; 6675 case KVM_SET_IDENTITY_MAP_ADDR: { 6676 u64 ident_addr; 6677 6678 mutex_lock(&kvm->lock); 6679 r = -EINVAL; 6680 if (kvm->created_vcpus) 6681 goto set_identity_unlock; 6682 r = -EFAULT; 6683 if (copy_from_user(&ident_addr, argp, sizeof(ident_addr))) 6684 goto set_identity_unlock; 6685 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr); 6686 set_identity_unlock: 6687 mutex_unlock(&kvm->lock); 6688 break; 6689 } 6690 case KVM_SET_NR_MMU_PAGES: 6691 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); 6692 break; 6693 case KVM_GET_NR_MMU_PAGES: 6694 r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); 6695 break; 6696 case KVM_CREATE_IRQCHIP: { 6697 mutex_lock(&kvm->lock); 6698 6699 r = -EEXIST; 6700 if (irqchip_in_kernel(kvm)) 6701 goto create_irqchip_unlock; 6702 6703 r = -EINVAL; 6704 if (kvm->created_vcpus) 6705 goto create_irqchip_unlock; 6706 6707 r = kvm_pic_init(kvm); 6708 if (r) 6709 goto create_irqchip_unlock; 6710 6711 r = kvm_ioapic_init(kvm); 6712 if (r) { 6713 kvm_pic_destroy(kvm); 6714 goto create_irqchip_unlock; 6715 } 6716 6717 r = kvm_setup_default_irq_routing(kvm); 6718 if (r) { 6719 kvm_ioapic_destroy(kvm); 6720 kvm_pic_destroy(kvm); 6721 goto create_irqchip_unlock; 6722 } 6723 /* Write kvm->irq_routing before enabling irqchip_in_kernel. */ 6724 smp_wmb(); 6725 kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL; 6726 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT); 6727 create_irqchip_unlock: 6728 mutex_unlock(&kvm->lock); 6729 break; 6730 } 6731 case KVM_CREATE_PIT: 6732 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY; 6733 goto create_pit; 6734 case KVM_CREATE_PIT2: 6735 r = -EFAULT; 6736 if (copy_from_user(&u.pit_config, argp, 6737 sizeof(struct kvm_pit_config))) 6738 goto out; 6739 create_pit: 6740 mutex_lock(&kvm->lock); 6741 r = -EEXIST; 6742 if (kvm->arch.vpit) 6743 goto create_pit_unlock; 6744 r = -ENOMEM; 6745 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags); 6746 if (kvm->arch.vpit) 6747 r = 0; 6748 create_pit_unlock: 6749 mutex_unlock(&kvm->lock); 6750 break; 6751 case KVM_GET_IRQCHIP: { 6752 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 6753 struct kvm_irqchip *chip; 6754 6755 chip = memdup_user(argp, sizeof(*chip)); 6756 if (IS_ERR(chip)) { 6757 r = PTR_ERR(chip); 6758 goto out; 6759 } 6760 6761 r = -ENXIO; 6762 if (!irqchip_kernel(kvm)) 6763 goto get_irqchip_out; 6764 r = kvm_vm_ioctl_get_irqchip(kvm, chip); 6765 if (r) 6766 goto get_irqchip_out; 6767 r = -EFAULT; 6768 if (copy_to_user(argp, chip, sizeof(*chip))) 6769 goto get_irqchip_out; 6770 r = 0; 6771 get_irqchip_out: 6772 kfree(chip); 6773 break; 6774 } 6775 case KVM_SET_IRQCHIP: { 6776 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 6777 struct kvm_irqchip *chip; 6778 6779 chip = memdup_user(argp, sizeof(*chip)); 6780 if (IS_ERR(chip)) { 6781 r = PTR_ERR(chip); 6782 goto out; 6783 } 6784 6785 r = -ENXIO; 6786 if (!irqchip_kernel(kvm)) 6787 goto set_irqchip_out; 6788 r = kvm_vm_ioctl_set_irqchip(kvm, chip); 6789 set_irqchip_out: 6790 kfree(chip); 6791 break; 6792 } 6793 case KVM_GET_PIT: { 6794 r = -EFAULT; 6795 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state))) 6796 goto out; 6797 r = -ENXIO; 6798 if (!kvm->arch.vpit) 6799 goto out; 6800 r = kvm_vm_ioctl_get_pit(kvm, &u.ps); 6801 if (r) 6802 goto out; 6803 r = -EFAULT; 6804 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state))) 6805 goto out; 6806 r = 0; 6807 break; 6808 } 6809 case KVM_SET_PIT: { 6810 r = -EFAULT; 6811 if (copy_from_user(&u.ps, argp, sizeof(u.ps))) 6812 goto out; 6813 mutex_lock(&kvm->lock); 6814 r = -ENXIO; 6815 if (!kvm->arch.vpit) 6816 goto set_pit_out; 6817 r = kvm_vm_ioctl_set_pit(kvm, &u.ps); 6818 set_pit_out: 6819 mutex_unlock(&kvm->lock); 6820 break; 6821 } 6822 case KVM_GET_PIT2: { 6823 r = -ENXIO; 6824 if (!kvm->arch.vpit) 6825 goto out; 6826 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2); 6827 if (r) 6828 goto out; 6829 r = -EFAULT; 6830 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2))) 6831 goto out; 6832 r = 0; 6833 break; 6834 } 6835 case KVM_SET_PIT2: { 6836 r = -EFAULT; 6837 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2))) 6838 goto out; 6839 mutex_lock(&kvm->lock); 6840 r = -ENXIO; 6841 if (!kvm->arch.vpit) 6842 goto set_pit2_out; 6843 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2); 6844 set_pit2_out: 6845 mutex_unlock(&kvm->lock); 6846 break; 6847 } 6848 case KVM_REINJECT_CONTROL: { 6849 struct kvm_reinject_control control; 6850 r = -EFAULT; 6851 if (copy_from_user(&control, argp, sizeof(control))) 6852 goto out; 6853 r = -ENXIO; 6854 if (!kvm->arch.vpit) 6855 goto out; 6856 r = kvm_vm_ioctl_reinject(kvm, &control); 6857 break; 6858 } 6859 case KVM_SET_BOOT_CPU_ID: 6860 r = 0; 6861 mutex_lock(&kvm->lock); 6862 if (kvm->created_vcpus) 6863 r = -EBUSY; 6864 else 6865 kvm->arch.bsp_vcpu_id = arg; 6866 mutex_unlock(&kvm->lock); 6867 break; 6868 #ifdef CONFIG_KVM_XEN 6869 case KVM_XEN_HVM_CONFIG: { 6870 struct kvm_xen_hvm_config xhc; 6871 r = -EFAULT; 6872 if (copy_from_user(&xhc, argp, sizeof(xhc))) 6873 goto out; 6874 r = kvm_xen_hvm_config(kvm, &xhc); 6875 break; 6876 } 6877 case KVM_XEN_HVM_GET_ATTR: { 6878 struct kvm_xen_hvm_attr xha; 6879 6880 r = -EFAULT; 6881 if (copy_from_user(&xha, argp, sizeof(xha))) 6882 goto out; 6883 r = kvm_xen_hvm_get_attr(kvm, &xha); 6884 if (!r && copy_to_user(argp, &xha, sizeof(xha))) 6885 r = -EFAULT; 6886 break; 6887 } 6888 case KVM_XEN_HVM_SET_ATTR: { 6889 struct kvm_xen_hvm_attr xha; 6890 6891 r = -EFAULT; 6892 if (copy_from_user(&xha, argp, sizeof(xha))) 6893 goto out; 6894 r = kvm_xen_hvm_set_attr(kvm, &xha); 6895 break; 6896 } 6897 case KVM_XEN_HVM_EVTCHN_SEND: { 6898 struct kvm_irq_routing_xen_evtchn uxe; 6899 6900 r = -EFAULT; 6901 if (copy_from_user(&uxe, argp, sizeof(uxe))) 6902 goto out; 6903 r = kvm_xen_hvm_evtchn_send(kvm, &uxe); 6904 break; 6905 } 6906 #endif 6907 case KVM_SET_CLOCK: 6908 r = kvm_vm_ioctl_set_clock(kvm, argp); 6909 break; 6910 case KVM_GET_CLOCK: 6911 r = kvm_vm_ioctl_get_clock(kvm, argp); 6912 break; 6913 case KVM_SET_TSC_KHZ: { 6914 u32 user_tsc_khz; 6915 6916 r = -EINVAL; 6917 user_tsc_khz = (u32)arg; 6918 6919 if (kvm_caps.has_tsc_control && 6920 user_tsc_khz >= kvm_caps.max_guest_tsc_khz) 6921 goto out; 6922 6923 if (user_tsc_khz == 0) 6924 user_tsc_khz = tsc_khz; 6925 6926 WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz); 6927 r = 0; 6928 6929 goto out; 6930 } 6931 case KVM_GET_TSC_KHZ: { 6932 r = READ_ONCE(kvm->arch.default_tsc_khz); 6933 goto out; 6934 } 6935 case KVM_MEMORY_ENCRYPT_OP: { 6936 r = -ENOTTY; 6937 if (!kvm_x86_ops.mem_enc_ioctl) 6938 goto out; 6939 6940 r = static_call(kvm_x86_mem_enc_ioctl)(kvm, argp); 6941 break; 6942 } 6943 case KVM_MEMORY_ENCRYPT_REG_REGION: { 6944 struct kvm_enc_region region; 6945 6946 r = -EFAULT; 6947 if (copy_from_user(®ion, argp, sizeof(region))) 6948 goto out; 6949 6950 r = -ENOTTY; 6951 if (!kvm_x86_ops.mem_enc_register_region) 6952 goto out; 6953 6954 r = static_call(kvm_x86_mem_enc_register_region)(kvm, ®ion); 6955 break; 6956 } 6957 case KVM_MEMORY_ENCRYPT_UNREG_REGION: { 6958 struct kvm_enc_region region; 6959 6960 r = -EFAULT; 6961 if (copy_from_user(®ion, argp, sizeof(region))) 6962 goto out; 6963 6964 r = -ENOTTY; 6965 if (!kvm_x86_ops.mem_enc_unregister_region) 6966 goto out; 6967 6968 r = static_call(kvm_x86_mem_enc_unregister_region)(kvm, ®ion); 6969 break; 6970 } 6971 case KVM_HYPERV_EVENTFD: { 6972 struct kvm_hyperv_eventfd hvevfd; 6973 6974 r = -EFAULT; 6975 if (copy_from_user(&hvevfd, argp, sizeof(hvevfd))) 6976 goto out; 6977 r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd); 6978 break; 6979 } 6980 case KVM_SET_PMU_EVENT_FILTER: 6981 r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp); 6982 break; 6983 case KVM_X86_SET_MSR_FILTER: { 6984 struct kvm_msr_filter __user *user_msr_filter = argp; 6985 struct kvm_msr_filter filter; 6986 6987 if (copy_from_user(&filter, user_msr_filter, sizeof(filter))) 6988 return -EFAULT; 6989 6990 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter); 6991 break; 6992 } 6993 default: 6994 r = -ENOTTY; 6995 } 6996 out: 6997 return r; 6998 } 6999 7000 static void kvm_probe_msr_to_save(u32 msr_index) 7001 { 7002 u32 dummy[2]; 7003 7004 if (rdmsr_safe(msr_index, &dummy[0], &dummy[1])) 7005 return; 7006 7007 /* 7008 * Even MSRs that are valid in the host may not be exposed to guests in 7009 * some cases. 7010 */ 7011 switch (msr_index) { 7012 case MSR_IA32_BNDCFGS: 7013 if (!kvm_mpx_supported()) 7014 return; 7015 break; 7016 case MSR_TSC_AUX: 7017 if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) && 7018 !kvm_cpu_cap_has(X86_FEATURE_RDPID)) 7019 return; 7020 break; 7021 case MSR_IA32_UMWAIT_CONTROL: 7022 if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG)) 7023 return; 7024 break; 7025 case MSR_IA32_RTIT_CTL: 7026 case MSR_IA32_RTIT_STATUS: 7027 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) 7028 return; 7029 break; 7030 case MSR_IA32_RTIT_CR3_MATCH: 7031 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || 7032 !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering)) 7033 return; 7034 break; 7035 case MSR_IA32_RTIT_OUTPUT_BASE: 7036 case MSR_IA32_RTIT_OUTPUT_MASK: 7037 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || 7038 (!intel_pt_validate_hw_cap(PT_CAP_topa_output) && 7039 !intel_pt_validate_hw_cap(PT_CAP_single_range_output))) 7040 return; 7041 break; 7042 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 7043 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || 7044 (msr_index - MSR_IA32_RTIT_ADDR0_A >= 7045 intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)) 7046 return; 7047 break; 7048 case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR_MAX: 7049 if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >= 7050 kvm_pmu_cap.num_counters_gp) 7051 return; 7052 break; 7053 case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL_MAX: 7054 if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >= 7055 kvm_pmu_cap.num_counters_gp) 7056 return; 7057 break; 7058 case MSR_IA32_XFD: 7059 case MSR_IA32_XFD_ERR: 7060 if (!kvm_cpu_cap_has(X86_FEATURE_XFD)) 7061 return; 7062 break; 7063 default: 7064 break; 7065 } 7066 7067 msrs_to_save[num_msrs_to_save++] = msr_index; 7068 } 7069 7070 static void kvm_init_msr_list(void) 7071 { 7072 unsigned i; 7073 7074 BUILD_BUG_ON_MSG(KVM_PMC_MAX_FIXED != 3, 7075 "Please update the fixed PMCs in msrs_to_save_pmu[]"); 7076 7077 num_msrs_to_save = 0; 7078 num_emulated_msrs = 0; 7079 num_msr_based_features = 0; 7080 7081 for (i = 0; i < ARRAY_SIZE(msrs_to_save_base); i++) 7082 kvm_probe_msr_to_save(msrs_to_save_base[i]); 7083 7084 if (enable_pmu) { 7085 for (i = 0; i < ARRAY_SIZE(msrs_to_save_pmu); i++) 7086 kvm_probe_msr_to_save(msrs_to_save_pmu[i]); 7087 } 7088 7089 for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) { 7090 if (!static_call(kvm_x86_has_emulated_msr)(NULL, emulated_msrs_all[i])) 7091 continue; 7092 7093 emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i]; 7094 } 7095 7096 for (i = 0; i < ARRAY_SIZE(msr_based_features_all); i++) { 7097 struct kvm_msr_entry msr; 7098 7099 msr.index = msr_based_features_all[i]; 7100 if (kvm_get_msr_feature(&msr)) 7101 continue; 7102 7103 msr_based_features[num_msr_based_features++] = msr_based_features_all[i]; 7104 } 7105 } 7106 7107 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len, 7108 const void *v) 7109 { 7110 int handled = 0; 7111 int n; 7112 7113 do { 7114 n = min(len, 8); 7115 if (!(lapic_in_kernel(vcpu) && 7116 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v)) 7117 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v)) 7118 break; 7119 handled += n; 7120 addr += n; 7121 len -= n; 7122 v += n; 7123 } while (len); 7124 7125 return handled; 7126 } 7127 7128 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v) 7129 { 7130 int handled = 0; 7131 int n; 7132 7133 do { 7134 n = min(len, 8); 7135 if (!(lapic_in_kernel(vcpu) && 7136 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev, 7137 addr, n, v)) 7138 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v)) 7139 break; 7140 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v); 7141 handled += n; 7142 addr += n; 7143 len -= n; 7144 v += n; 7145 } while (len); 7146 7147 return handled; 7148 } 7149 7150 void kvm_set_segment(struct kvm_vcpu *vcpu, 7151 struct kvm_segment *var, int seg) 7152 { 7153 static_call(kvm_x86_set_segment)(vcpu, var, seg); 7154 } 7155 7156 void kvm_get_segment(struct kvm_vcpu *vcpu, 7157 struct kvm_segment *var, int seg) 7158 { 7159 static_call(kvm_x86_get_segment)(vcpu, var, seg); 7160 } 7161 7162 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access, 7163 struct x86_exception *exception) 7164 { 7165 struct kvm_mmu *mmu = vcpu->arch.mmu; 7166 gpa_t t_gpa; 7167 7168 BUG_ON(!mmu_is_nested(vcpu)); 7169 7170 /* NPT walks are always user-walks */ 7171 access |= PFERR_USER_MASK; 7172 t_gpa = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception); 7173 7174 return t_gpa; 7175 } 7176 7177 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, 7178 struct x86_exception *exception) 7179 { 7180 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 7181 7182 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 7183 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); 7184 } 7185 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read); 7186 7187 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, 7188 struct x86_exception *exception) 7189 { 7190 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 7191 7192 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 7193 access |= PFERR_WRITE_MASK; 7194 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); 7195 } 7196 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_write); 7197 7198 /* uses this to access any guest's mapped memory without checking CPL */ 7199 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, 7200 struct x86_exception *exception) 7201 { 7202 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 7203 7204 return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception); 7205 } 7206 7207 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 7208 struct kvm_vcpu *vcpu, u64 access, 7209 struct x86_exception *exception) 7210 { 7211 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 7212 void *data = val; 7213 int r = X86EMUL_CONTINUE; 7214 7215 while (bytes) { 7216 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception); 7217 unsigned offset = addr & (PAGE_SIZE-1); 7218 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); 7219 int ret; 7220 7221 if (gpa == INVALID_GPA) 7222 return X86EMUL_PROPAGATE_FAULT; 7223 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data, 7224 offset, toread); 7225 if (ret < 0) { 7226 r = X86EMUL_IO_NEEDED; 7227 goto out; 7228 } 7229 7230 bytes -= toread; 7231 data += toread; 7232 addr += toread; 7233 } 7234 out: 7235 return r; 7236 } 7237 7238 /* used for instruction fetching */ 7239 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt, 7240 gva_t addr, void *val, unsigned int bytes, 7241 struct x86_exception *exception) 7242 { 7243 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7244 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 7245 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 7246 unsigned offset; 7247 int ret; 7248 7249 /* Inline kvm_read_guest_virt_helper for speed. */ 7250 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK, 7251 exception); 7252 if (unlikely(gpa == INVALID_GPA)) 7253 return X86EMUL_PROPAGATE_FAULT; 7254 7255 offset = addr & (PAGE_SIZE-1); 7256 if (WARN_ON(offset + bytes > PAGE_SIZE)) 7257 bytes = (unsigned)PAGE_SIZE - offset; 7258 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val, 7259 offset, bytes); 7260 if (unlikely(ret < 0)) 7261 return X86EMUL_IO_NEEDED; 7262 7263 return X86EMUL_CONTINUE; 7264 } 7265 7266 int kvm_read_guest_virt(struct kvm_vcpu *vcpu, 7267 gva_t addr, void *val, unsigned int bytes, 7268 struct x86_exception *exception) 7269 { 7270 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0; 7271 7272 /* 7273 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED 7274 * is returned, but our callers are not ready for that and they blindly 7275 * call kvm_inject_page_fault. Ensure that they at least do not leak 7276 * uninitialized kernel stack memory into cr2 and error code. 7277 */ 7278 memset(exception, 0, sizeof(*exception)); 7279 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, 7280 exception); 7281 } 7282 EXPORT_SYMBOL_GPL(kvm_read_guest_virt); 7283 7284 static int emulator_read_std(struct x86_emulate_ctxt *ctxt, 7285 gva_t addr, void *val, unsigned int bytes, 7286 struct x86_exception *exception, bool system) 7287 { 7288 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7289 u64 access = 0; 7290 7291 if (system) 7292 access |= PFERR_IMPLICIT_ACCESS; 7293 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3) 7294 access |= PFERR_USER_MASK; 7295 7296 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception); 7297 } 7298 7299 static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes, 7300 struct kvm_vcpu *vcpu, u64 access, 7301 struct x86_exception *exception) 7302 { 7303 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 7304 void *data = val; 7305 int r = X86EMUL_CONTINUE; 7306 7307 while (bytes) { 7308 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception); 7309 unsigned offset = addr & (PAGE_SIZE-1); 7310 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); 7311 int ret; 7312 7313 if (gpa == INVALID_GPA) 7314 return X86EMUL_PROPAGATE_FAULT; 7315 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite); 7316 if (ret < 0) { 7317 r = X86EMUL_IO_NEEDED; 7318 goto out; 7319 } 7320 7321 bytes -= towrite; 7322 data += towrite; 7323 addr += towrite; 7324 } 7325 out: 7326 return r; 7327 } 7328 7329 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val, 7330 unsigned int bytes, struct x86_exception *exception, 7331 bool system) 7332 { 7333 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7334 u64 access = PFERR_WRITE_MASK; 7335 7336 if (system) 7337 access |= PFERR_IMPLICIT_ACCESS; 7338 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3) 7339 access |= PFERR_USER_MASK; 7340 7341 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, 7342 access, exception); 7343 } 7344 7345 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val, 7346 unsigned int bytes, struct x86_exception *exception) 7347 { 7348 /* kvm_write_guest_virt_system can pull in tons of pages. */ 7349 vcpu->arch.l1tf_flush_l1d = true; 7350 7351 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, 7352 PFERR_WRITE_MASK, exception); 7353 } 7354 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system); 7355 7356 static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type, 7357 void *insn, int insn_len) 7358 { 7359 return static_call(kvm_x86_can_emulate_instruction)(vcpu, emul_type, 7360 insn, insn_len); 7361 } 7362 7363 int handle_ud(struct kvm_vcpu *vcpu) 7364 { 7365 static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX }; 7366 int fep_flags = READ_ONCE(force_emulation_prefix); 7367 int emul_type = EMULTYPE_TRAP_UD; 7368 char sig[5]; /* ud2; .ascii "kvm" */ 7369 struct x86_exception e; 7370 7371 if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0))) 7372 return 1; 7373 7374 if (fep_flags && 7375 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu), 7376 sig, sizeof(sig), &e) == 0 && 7377 memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) { 7378 if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF) 7379 kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF); 7380 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig)); 7381 emul_type = EMULTYPE_TRAP_UD_FORCED; 7382 } 7383 7384 return kvm_emulate_instruction(vcpu, emul_type); 7385 } 7386 EXPORT_SYMBOL_GPL(handle_ud); 7387 7388 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 7389 gpa_t gpa, bool write) 7390 { 7391 /* For APIC access vmexit */ 7392 if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 7393 return 1; 7394 7395 if (vcpu_match_mmio_gpa(vcpu, gpa)) { 7396 trace_vcpu_match_mmio(gva, gpa, write, true); 7397 return 1; 7398 } 7399 7400 return 0; 7401 } 7402 7403 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva, 7404 gpa_t *gpa, struct x86_exception *exception, 7405 bool write) 7406 { 7407 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 7408 u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0) 7409 | (write ? PFERR_WRITE_MASK : 0); 7410 7411 /* 7412 * currently PKRU is only applied to ept enabled guest so 7413 * there is no pkey in EPT page table for L1 guest or EPT 7414 * shadow page table for L2 guest. 7415 */ 7416 if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) || 7417 !permission_fault(vcpu, vcpu->arch.walk_mmu, 7418 vcpu->arch.mmio_access, 0, access))) { 7419 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | 7420 (gva & (PAGE_SIZE - 1)); 7421 trace_vcpu_match_mmio(gva, *gpa, write, false); 7422 return 1; 7423 } 7424 7425 *gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception); 7426 7427 if (*gpa == INVALID_GPA) 7428 return -1; 7429 7430 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write); 7431 } 7432 7433 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, 7434 const void *val, int bytes) 7435 { 7436 int ret; 7437 7438 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes); 7439 if (ret < 0) 7440 return 0; 7441 kvm_page_track_write(vcpu, gpa, val, bytes); 7442 return 1; 7443 } 7444 7445 struct read_write_emulator_ops { 7446 int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val, 7447 int bytes); 7448 int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa, 7449 void *val, int bytes); 7450 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa, 7451 int bytes, void *val); 7452 int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa, 7453 void *val, int bytes); 7454 bool write; 7455 }; 7456 7457 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes) 7458 { 7459 if (vcpu->mmio_read_completed) { 7460 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, 7461 vcpu->mmio_fragments[0].gpa, val); 7462 vcpu->mmio_read_completed = 0; 7463 return 1; 7464 } 7465 7466 return 0; 7467 } 7468 7469 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa, 7470 void *val, int bytes) 7471 { 7472 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes); 7473 } 7474 7475 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa, 7476 void *val, int bytes) 7477 { 7478 return emulator_write_phys(vcpu, gpa, val, bytes); 7479 } 7480 7481 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val) 7482 { 7483 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val); 7484 return vcpu_mmio_write(vcpu, gpa, bytes, val); 7485 } 7486 7487 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, 7488 void *val, int bytes) 7489 { 7490 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL); 7491 return X86EMUL_IO_NEEDED; 7492 } 7493 7494 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, 7495 void *val, int bytes) 7496 { 7497 struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0]; 7498 7499 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len)); 7500 return X86EMUL_CONTINUE; 7501 } 7502 7503 static const struct read_write_emulator_ops read_emultor = { 7504 .read_write_prepare = read_prepare, 7505 .read_write_emulate = read_emulate, 7506 .read_write_mmio = vcpu_mmio_read, 7507 .read_write_exit_mmio = read_exit_mmio, 7508 }; 7509 7510 static const struct read_write_emulator_ops write_emultor = { 7511 .read_write_emulate = write_emulate, 7512 .read_write_mmio = write_mmio, 7513 .read_write_exit_mmio = write_exit_mmio, 7514 .write = true, 7515 }; 7516 7517 static int emulator_read_write_onepage(unsigned long addr, void *val, 7518 unsigned int bytes, 7519 struct x86_exception *exception, 7520 struct kvm_vcpu *vcpu, 7521 const struct read_write_emulator_ops *ops) 7522 { 7523 gpa_t gpa; 7524 int handled, ret; 7525 bool write = ops->write; 7526 struct kvm_mmio_fragment *frag; 7527 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 7528 7529 /* 7530 * If the exit was due to a NPF we may already have a GPA. 7531 * If the GPA is present, use it to avoid the GVA to GPA table walk. 7532 * Note, this cannot be used on string operations since string 7533 * operation using rep will only have the initial GPA from the NPF 7534 * occurred. 7535 */ 7536 if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) && 7537 (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { 7538 gpa = ctxt->gpa_val; 7539 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); 7540 } else { 7541 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write); 7542 if (ret < 0) 7543 return X86EMUL_PROPAGATE_FAULT; 7544 } 7545 7546 if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes)) 7547 return X86EMUL_CONTINUE; 7548 7549 /* 7550 * Is this MMIO handled locally? 7551 */ 7552 handled = ops->read_write_mmio(vcpu, gpa, bytes, val); 7553 if (handled == bytes) 7554 return X86EMUL_CONTINUE; 7555 7556 gpa += handled; 7557 bytes -= handled; 7558 val += handled; 7559 7560 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); 7561 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; 7562 frag->gpa = gpa; 7563 frag->data = val; 7564 frag->len = bytes; 7565 return X86EMUL_CONTINUE; 7566 } 7567 7568 static int emulator_read_write(struct x86_emulate_ctxt *ctxt, 7569 unsigned long addr, 7570 void *val, unsigned int bytes, 7571 struct x86_exception *exception, 7572 const struct read_write_emulator_ops *ops) 7573 { 7574 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7575 gpa_t gpa; 7576 int rc; 7577 7578 if (ops->read_write_prepare && 7579 ops->read_write_prepare(vcpu, val, bytes)) 7580 return X86EMUL_CONTINUE; 7581 7582 vcpu->mmio_nr_fragments = 0; 7583 7584 /* Crossing a page boundary? */ 7585 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { 7586 int now; 7587 7588 now = -addr & ~PAGE_MASK; 7589 rc = emulator_read_write_onepage(addr, val, now, exception, 7590 vcpu, ops); 7591 7592 if (rc != X86EMUL_CONTINUE) 7593 return rc; 7594 addr += now; 7595 if (ctxt->mode != X86EMUL_MODE_PROT64) 7596 addr = (u32)addr; 7597 val += now; 7598 bytes -= now; 7599 } 7600 7601 rc = emulator_read_write_onepage(addr, val, bytes, exception, 7602 vcpu, ops); 7603 if (rc != X86EMUL_CONTINUE) 7604 return rc; 7605 7606 if (!vcpu->mmio_nr_fragments) 7607 return rc; 7608 7609 gpa = vcpu->mmio_fragments[0].gpa; 7610 7611 vcpu->mmio_needed = 1; 7612 vcpu->mmio_cur_fragment = 0; 7613 7614 vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len); 7615 vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write; 7616 vcpu->run->exit_reason = KVM_EXIT_MMIO; 7617 vcpu->run->mmio.phys_addr = gpa; 7618 7619 return ops->read_write_exit_mmio(vcpu, gpa, val, bytes); 7620 } 7621 7622 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt, 7623 unsigned long addr, 7624 void *val, 7625 unsigned int bytes, 7626 struct x86_exception *exception) 7627 { 7628 return emulator_read_write(ctxt, addr, val, bytes, 7629 exception, &read_emultor); 7630 } 7631 7632 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, 7633 unsigned long addr, 7634 const void *val, 7635 unsigned int bytes, 7636 struct x86_exception *exception) 7637 { 7638 return emulator_read_write(ctxt, addr, (void *)val, bytes, 7639 exception, &write_emultor); 7640 } 7641 7642 #define emulator_try_cmpxchg_user(t, ptr, old, new) \ 7643 (__try_cmpxchg_user((t __user *)(ptr), (t *)(old), *(t *)(new), efault ## t)) 7644 7645 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, 7646 unsigned long addr, 7647 const void *old, 7648 const void *new, 7649 unsigned int bytes, 7650 struct x86_exception *exception) 7651 { 7652 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7653 u64 page_line_mask; 7654 unsigned long hva; 7655 gpa_t gpa; 7656 int r; 7657 7658 /* guests cmpxchg8b have to be emulated atomically */ 7659 if (bytes > 8 || (bytes & (bytes - 1))) 7660 goto emul_write; 7661 7662 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL); 7663 7664 if (gpa == INVALID_GPA || 7665 (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) 7666 goto emul_write; 7667 7668 /* 7669 * Emulate the atomic as a straight write to avoid #AC if SLD is 7670 * enabled in the host and the access splits a cache line. 7671 */ 7672 if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) 7673 page_line_mask = ~(cache_line_size() - 1); 7674 else 7675 page_line_mask = PAGE_MASK; 7676 7677 if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask)) 7678 goto emul_write; 7679 7680 hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa)); 7681 if (kvm_is_error_hva(hva)) 7682 goto emul_write; 7683 7684 hva += offset_in_page(gpa); 7685 7686 switch (bytes) { 7687 case 1: 7688 r = emulator_try_cmpxchg_user(u8, hva, old, new); 7689 break; 7690 case 2: 7691 r = emulator_try_cmpxchg_user(u16, hva, old, new); 7692 break; 7693 case 4: 7694 r = emulator_try_cmpxchg_user(u32, hva, old, new); 7695 break; 7696 case 8: 7697 r = emulator_try_cmpxchg_user(u64, hva, old, new); 7698 break; 7699 default: 7700 BUG(); 7701 } 7702 7703 if (r < 0) 7704 return X86EMUL_UNHANDLEABLE; 7705 if (r) 7706 return X86EMUL_CMPXCHG_FAILED; 7707 7708 kvm_page_track_write(vcpu, gpa, new, bytes); 7709 7710 return X86EMUL_CONTINUE; 7711 7712 emul_write: 7713 pr_warn_once("emulating exchange as write\n"); 7714 7715 return emulator_write_emulated(ctxt, addr, new, bytes, exception); 7716 } 7717 7718 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size, 7719 unsigned short port, void *data, 7720 unsigned int count, bool in) 7721 { 7722 unsigned i; 7723 int r; 7724 7725 WARN_ON_ONCE(vcpu->arch.pio.count); 7726 for (i = 0; i < count; i++) { 7727 if (in) 7728 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data); 7729 else 7730 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data); 7731 7732 if (r) { 7733 if (i == 0) 7734 goto userspace_io; 7735 7736 /* 7737 * Userspace must have unregistered the device while PIO 7738 * was running. Drop writes / read as 0. 7739 */ 7740 if (in) 7741 memset(data, 0, size * (count - i)); 7742 break; 7743 } 7744 7745 data += size; 7746 } 7747 return 1; 7748 7749 userspace_io: 7750 vcpu->arch.pio.port = port; 7751 vcpu->arch.pio.in = in; 7752 vcpu->arch.pio.count = count; 7753 vcpu->arch.pio.size = size; 7754 7755 if (in) 7756 memset(vcpu->arch.pio_data, 0, size * count); 7757 else 7758 memcpy(vcpu->arch.pio_data, data, size * count); 7759 7760 vcpu->run->exit_reason = KVM_EXIT_IO; 7761 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; 7762 vcpu->run->io.size = size; 7763 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; 7764 vcpu->run->io.count = count; 7765 vcpu->run->io.port = port; 7766 return 0; 7767 } 7768 7769 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size, 7770 unsigned short port, void *val, unsigned int count) 7771 { 7772 int r = emulator_pio_in_out(vcpu, size, port, val, count, true); 7773 if (r) 7774 trace_kvm_pio(KVM_PIO_IN, port, size, count, val); 7775 7776 return r; 7777 } 7778 7779 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val) 7780 { 7781 int size = vcpu->arch.pio.size; 7782 unsigned int count = vcpu->arch.pio.count; 7783 memcpy(val, vcpu->arch.pio_data, size * count); 7784 trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data); 7785 vcpu->arch.pio.count = 0; 7786 } 7787 7788 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt, 7789 int size, unsigned short port, void *val, 7790 unsigned int count) 7791 { 7792 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7793 if (vcpu->arch.pio.count) { 7794 /* 7795 * Complete a previous iteration that required userspace I/O. 7796 * Note, @count isn't guaranteed to match pio.count as userspace 7797 * can modify ECX before rerunning the vCPU. Ignore any such 7798 * shenanigans as KVM doesn't support modifying the rep count, 7799 * and the emulator ensures @count doesn't overflow the buffer. 7800 */ 7801 complete_emulator_pio_in(vcpu, val); 7802 return 1; 7803 } 7804 7805 return emulator_pio_in(vcpu, size, port, val, count); 7806 } 7807 7808 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size, 7809 unsigned short port, const void *val, 7810 unsigned int count) 7811 { 7812 trace_kvm_pio(KVM_PIO_OUT, port, size, count, val); 7813 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false); 7814 } 7815 7816 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt, 7817 int size, unsigned short port, 7818 const void *val, unsigned int count) 7819 { 7820 return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count); 7821 } 7822 7823 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) 7824 { 7825 return static_call(kvm_x86_get_segment_base)(vcpu, seg); 7826 } 7827 7828 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address) 7829 { 7830 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address); 7831 } 7832 7833 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu) 7834 { 7835 if (!need_emulate_wbinvd(vcpu)) 7836 return X86EMUL_CONTINUE; 7837 7838 if (static_call(kvm_x86_has_wbinvd_exit)()) { 7839 int cpu = get_cpu(); 7840 7841 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask); 7842 on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask, 7843 wbinvd_ipi, NULL, 1); 7844 put_cpu(); 7845 cpumask_clear(vcpu->arch.wbinvd_dirty_mask); 7846 } else 7847 wbinvd(); 7848 return X86EMUL_CONTINUE; 7849 } 7850 7851 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu) 7852 { 7853 kvm_emulate_wbinvd_noskip(vcpu); 7854 return kvm_skip_emulated_instruction(vcpu); 7855 } 7856 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd); 7857 7858 7859 7860 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt) 7861 { 7862 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt)); 7863 } 7864 7865 static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, 7866 unsigned long *dest) 7867 { 7868 kvm_get_dr(emul_to_vcpu(ctxt), dr, dest); 7869 } 7870 7871 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, 7872 unsigned long value) 7873 { 7874 7875 return kvm_set_dr(emul_to_vcpu(ctxt), dr, value); 7876 } 7877 7878 static u64 mk_cr_64(u64 curr_cr, u32 new_val) 7879 { 7880 return (curr_cr & ~((1ULL << 32) - 1)) | new_val; 7881 } 7882 7883 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr) 7884 { 7885 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7886 unsigned long value; 7887 7888 switch (cr) { 7889 case 0: 7890 value = kvm_read_cr0(vcpu); 7891 break; 7892 case 2: 7893 value = vcpu->arch.cr2; 7894 break; 7895 case 3: 7896 value = kvm_read_cr3(vcpu); 7897 break; 7898 case 4: 7899 value = kvm_read_cr4(vcpu); 7900 break; 7901 case 8: 7902 value = kvm_get_cr8(vcpu); 7903 break; 7904 default: 7905 kvm_err("%s: unexpected cr %u\n", __func__, cr); 7906 return 0; 7907 } 7908 7909 return value; 7910 } 7911 7912 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val) 7913 { 7914 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 7915 int res = 0; 7916 7917 switch (cr) { 7918 case 0: 7919 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val)); 7920 break; 7921 case 2: 7922 vcpu->arch.cr2 = val; 7923 break; 7924 case 3: 7925 res = kvm_set_cr3(vcpu, val); 7926 break; 7927 case 4: 7928 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val)); 7929 break; 7930 case 8: 7931 res = kvm_set_cr8(vcpu, val); 7932 break; 7933 default: 7934 kvm_err("%s: unexpected cr %u\n", __func__, cr); 7935 res = -1; 7936 } 7937 7938 return res; 7939 } 7940 7941 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt) 7942 { 7943 return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt)); 7944 } 7945 7946 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 7947 { 7948 static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt); 7949 } 7950 7951 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 7952 { 7953 static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt); 7954 } 7955 7956 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 7957 { 7958 static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt); 7959 } 7960 7961 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt) 7962 { 7963 static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt); 7964 } 7965 7966 static unsigned long emulator_get_cached_segment_base( 7967 struct x86_emulate_ctxt *ctxt, int seg) 7968 { 7969 return get_segment_base(emul_to_vcpu(ctxt), seg); 7970 } 7971 7972 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector, 7973 struct desc_struct *desc, u32 *base3, 7974 int seg) 7975 { 7976 struct kvm_segment var; 7977 7978 kvm_get_segment(emul_to_vcpu(ctxt), &var, seg); 7979 *selector = var.selector; 7980 7981 if (var.unusable) { 7982 memset(desc, 0, sizeof(*desc)); 7983 if (base3) 7984 *base3 = 0; 7985 return false; 7986 } 7987 7988 if (var.g) 7989 var.limit >>= 12; 7990 set_desc_limit(desc, var.limit); 7991 set_desc_base(desc, (unsigned long)var.base); 7992 #ifdef CONFIG_X86_64 7993 if (base3) 7994 *base3 = var.base >> 32; 7995 #endif 7996 desc->type = var.type; 7997 desc->s = var.s; 7998 desc->dpl = var.dpl; 7999 desc->p = var.present; 8000 desc->avl = var.avl; 8001 desc->l = var.l; 8002 desc->d = var.db; 8003 desc->g = var.g; 8004 8005 return true; 8006 } 8007 8008 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector, 8009 struct desc_struct *desc, u32 base3, 8010 int seg) 8011 { 8012 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 8013 struct kvm_segment var; 8014 8015 var.selector = selector; 8016 var.base = get_desc_base(desc); 8017 #ifdef CONFIG_X86_64 8018 var.base |= ((u64)base3) << 32; 8019 #endif 8020 var.limit = get_desc_limit(desc); 8021 if (desc->g) 8022 var.limit = (var.limit << 12) | 0xfff; 8023 var.type = desc->type; 8024 var.dpl = desc->dpl; 8025 var.db = desc->d; 8026 var.s = desc->s; 8027 var.l = desc->l; 8028 var.g = desc->g; 8029 var.avl = desc->avl; 8030 var.present = desc->p; 8031 var.unusable = !var.present; 8032 var.padding = 0; 8033 8034 kvm_set_segment(vcpu, &var, seg); 8035 return; 8036 } 8037 8038 static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt, 8039 u32 msr_index, u64 *pdata) 8040 { 8041 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 8042 int r; 8043 8044 r = kvm_get_msr_with_filter(vcpu, msr_index, pdata); 8045 if (r < 0) 8046 return X86EMUL_UNHANDLEABLE; 8047 8048 if (r) { 8049 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, 8050 complete_emulated_rdmsr, r)) 8051 return X86EMUL_IO_NEEDED; 8052 8053 trace_kvm_msr_read_ex(msr_index); 8054 return X86EMUL_PROPAGATE_FAULT; 8055 } 8056 8057 trace_kvm_msr_read(msr_index, *pdata); 8058 return X86EMUL_CONTINUE; 8059 } 8060 8061 static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt, 8062 u32 msr_index, u64 data) 8063 { 8064 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 8065 int r; 8066 8067 r = kvm_set_msr_with_filter(vcpu, msr_index, data); 8068 if (r < 0) 8069 return X86EMUL_UNHANDLEABLE; 8070 8071 if (r) { 8072 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, 8073 complete_emulated_msr_access, r)) 8074 return X86EMUL_IO_NEEDED; 8075 8076 trace_kvm_msr_write_ex(msr_index, data); 8077 return X86EMUL_PROPAGATE_FAULT; 8078 } 8079 8080 trace_kvm_msr_write(msr_index, data); 8081 return X86EMUL_CONTINUE; 8082 } 8083 8084 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, 8085 u32 msr_index, u64 *pdata) 8086 { 8087 return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata); 8088 } 8089 8090 static int emulator_check_pmc(struct x86_emulate_ctxt *ctxt, 8091 u32 pmc) 8092 { 8093 if (kvm_pmu_is_valid_rdpmc_ecx(emul_to_vcpu(ctxt), pmc)) 8094 return 0; 8095 return -EINVAL; 8096 } 8097 8098 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt, 8099 u32 pmc, u64 *pdata) 8100 { 8101 return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata); 8102 } 8103 8104 static void emulator_halt(struct x86_emulate_ctxt *ctxt) 8105 { 8106 emul_to_vcpu(ctxt)->arch.halt_request = 1; 8107 } 8108 8109 static int emulator_intercept(struct x86_emulate_ctxt *ctxt, 8110 struct x86_instruction_info *info, 8111 enum x86_intercept_stage stage) 8112 { 8113 return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage, 8114 &ctxt->exception); 8115 } 8116 8117 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt, 8118 u32 *eax, u32 *ebx, u32 *ecx, u32 *edx, 8119 bool exact_only) 8120 { 8121 return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only); 8122 } 8123 8124 static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt) 8125 { 8126 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM); 8127 } 8128 8129 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt) 8130 { 8131 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE); 8132 } 8133 8134 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt) 8135 { 8136 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR); 8137 } 8138 8139 static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt) 8140 { 8141 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID); 8142 } 8143 8144 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg) 8145 { 8146 return kvm_register_read_raw(emul_to_vcpu(ctxt), reg); 8147 } 8148 8149 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val) 8150 { 8151 kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val); 8152 } 8153 8154 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked) 8155 { 8156 static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked); 8157 } 8158 8159 static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt) 8160 { 8161 return emul_to_vcpu(ctxt)->arch.hflags; 8162 } 8163 8164 #ifndef CONFIG_KVM_SMM 8165 static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt) 8166 { 8167 WARN_ON_ONCE(1); 8168 return X86EMUL_UNHANDLEABLE; 8169 } 8170 #endif 8171 8172 static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt) 8173 { 8174 kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt)); 8175 } 8176 8177 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr) 8178 { 8179 return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr); 8180 } 8181 8182 static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt) 8183 { 8184 struct kvm *kvm = emul_to_vcpu(ctxt)->kvm; 8185 8186 if (!kvm->vm_bugged) 8187 kvm_vm_bugged(kvm); 8188 } 8189 8190 static const struct x86_emulate_ops emulate_ops = { 8191 .vm_bugged = emulator_vm_bugged, 8192 .read_gpr = emulator_read_gpr, 8193 .write_gpr = emulator_write_gpr, 8194 .read_std = emulator_read_std, 8195 .write_std = emulator_write_std, 8196 .fetch = kvm_fetch_guest_virt, 8197 .read_emulated = emulator_read_emulated, 8198 .write_emulated = emulator_write_emulated, 8199 .cmpxchg_emulated = emulator_cmpxchg_emulated, 8200 .invlpg = emulator_invlpg, 8201 .pio_in_emulated = emulator_pio_in_emulated, 8202 .pio_out_emulated = emulator_pio_out_emulated, 8203 .get_segment = emulator_get_segment, 8204 .set_segment = emulator_set_segment, 8205 .get_cached_segment_base = emulator_get_cached_segment_base, 8206 .get_gdt = emulator_get_gdt, 8207 .get_idt = emulator_get_idt, 8208 .set_gdt = emulator_set_gdt, 8209 .set_idt = emulator_set_idt, 8210 .get_cr = emulator_get_cr, 8211 .set_cr = emulator_set_cr, 8212 .cpl = emulator_get_cpl, 8213 .get_dr = emulator_get_dr, 8214 .set_dr = emulator_set_dr, 8215 .set_msr_with_filter = emulator_set_msr_with_filter, 8216 .get_msr_with_filter = emulator_get_msr_with_filter, 8217 .get_msr = emulator_get_msr, 8218 .check_pmc = emulator_check_pmc, 8219 .read_pmc = emulator_read_pmc, 8220 .halt = emulator_halt, 8221 .wbinvd = emulator_wbinvd, 8222 .fix_hypercall = emulator_fix_hypercall, 8223 .intercept = emulator_intercept, 8224 .get_cpuid = emulator_get_cpuid, 8225 .guest_has_long_mode = emulator_guest_has_long_mode, 8226 .guest_has_movbe = emulator_guest_has_movbe, 8227 .guest_has_fxsr = emulator_guest_has_fxsr, 8228 .guest_has_rdpid = emulator_guest_has_rdpid, 8229 .set_nmi_mask = emulator_set_nmi_mask, 8230 .get_hflags = emulator_get_hflags, 8231 .leave_smm = emulator_leave_smm, 8232 .triple_fault = emulator_triple_fault, 8233 .set_xcr = emulator_set_xcr, 8234 }; 8235 8236 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) 8237 { 8238 u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu); 8239 /* 8240 * an sti; sti; sequence only disable interrupts for the first 8241 * instruction. So, if the last instruction, be it emulated or 8242 * not, left the system with the INT_STI flag enabled, it 8243 * means that the last instruction is an sti. We should not 8244 * leave the flag on in this case. The same goes for mov ss 8245 */ 8246 if (int_shadow & mask) 8247 mask = 0; 8248 if (unlikely(int_shadow || mask)) { 8249 static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask); 8250 if (!mask) 8251 kvm_make_request(KVM_REQ_EVENT, vcpu); 8252 } 8253 } 8254 8255 static void inject_emulated_exception(struct kvm_vcpu *vcpu) 8256 { 8257 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 8258 8259 if (ctxt->exception.vector == PF_VECTOR) 8260 kvm_inject_emulated_page_fault(vcpu, &ctxt->exception); 8261 else if (ctxt->exception.error_code_valid) 8262 kvm_queue_exception_e(vcpu, ctxt->exception.vector, 8263 ctxt->exception.error_code); 8264 else 8265 kvm_queue_exception(vcpu, ctxt->exception.vector); 8266 } 8267 8268 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu) 8269 { 8270 struct x86_emulate_ctxt *ctxt; 8271 8272 ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT); 8273 if (!ctxt) { 8274 pr_err("failed to allocate vcpu's emulator\n"); 8275 return NULL; 8276 } 8277 8278 ctxt->vcpu = vcpu; 8279 ctxt->ops = &emulate_ops; 8280 vcpu->arch.emulate_ctxt = ctxt; 8281 8282 return ctxt; 8283 } 8284 8285 static void init_emulate_ctxt(struct kvm_vcpu *vcpu) 8286 { 8287 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 8288 int cs_db, cs_l; 8289 8290 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); 8291 8292 ctxt->gpa_available = false; 8293 ctxt->eflags = kvm_get_rflags(vcpu); 8294 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; 8295 8296 ctxt->eip = kvm_rip_read(vcpu); 8297 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL : 8298 (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 : 8299 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 : 8300 cs_db ? X86EMUL_MODE_PROT32 : 8301 X86EMUL_MODE_PROT16; 8302 BUILD_BUG_ON(HF_GUEST_MASK != X86EMUL_GUEST_MASK); 8303 8304 ctxt->interruptibility = 0; 8305 ctxt->have_exception = false; 8306 ctxt->exception.vector = -1; 8307 ctxt->perm_ok = false; 8308 8309 init_decode_cache(ctxt); 8310 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 8311 } 8312 8313 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) 8314 { 8315 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 8316 int ret; 8317 8318 init_emulate_ctxt(vcpu); 8319 8320 ctxt->op_bytes = 2; 8321 ctxt->ad_bytes = 2; 8322 ctxt->_eip = ctxt->eip + inc_eip; 8323 ret = emulate_int_real(ctxt, irq); 8324 8325 if (ret != X86EMUL_CONTINUE) { 8326 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 8327 } else { 8328 ctxt->eip = ctxt->_eip; 8329 kvm_rip_write(vcpu, ctxt->eip); 8330 kvm_set_rflags(vcpu, ctxt->eflags); 8331 } 8332 } 8333 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); 8334 8335 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, 8336 u8 ndata, u8 *insn_bytes, u8 insn_size) 8337 { 8338 struct kvm_run *run = vcpu->run; 8339 u64 info[5]; 8340 u8 info_start; 8341 8342 /* 8343 * Zero the whole array used to retrieve the exit info, as casting to 8344 * u32 for select entries will leave some chunks uninitialized. 8345 */ 8346 memset(&info, 0, sizeof(info)); 8347 8348 static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1], 8349 &info[2], (u32 *)&info[3], 8350 (u32 *)&info[4]); 8351 8352 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 8353 run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION; 8354 8355 /* 8356 * There's currently space for 13 entries, but 5 are used for the exit 8357 * reason and info. Restrict to 4 to reduce the maintenance burden 8358 * when expanding kvm_run.emulation_failure in the future. 8359 */ 8360 if (WARN_ON_ONCE(ndata > 4)) 8361 ndata = 4; 8362 8363 /* Always include the flags as a 'data' entry. */ 8364 info_start = 1; 8365 run->emulation_failure.flags = 0; 8366 8367 if (insn_size) { 8368 BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) + 8369 sizeof(run->emulation_failure.insn_bytes) != 16)); 8370 info_start += 2; 8371 run->emulation_failure.flags |= 8372 KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES; 8373 run->emulation_failure.insn_size = insn_size; 8374 memset(run->emulation_failure.insn_bytes, 0x90, 8375 sizeof(run->emulation_failure.insn_bytes)); 8376 memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size); 8377 } 8378 8379 memcpy(&run->internal.data[info_start], info, sizeof(info)); 8380 memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data, 8381 ndata * sizeof(data[0])); 8382 8383 run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata; 8384 } 8385 8386 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu) 8387 { 8388 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 8389 8390 prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data, 8391 ctxt->fetch.end - ctxt->fetch.data); 8392 } 8393 8394 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data, 8395 u8 ndata) 8396 { 8397 prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0); 8398 } 8399 EXPORT_SYMBOL_GPL(__kvm_prepare_emulation_failure_exit); 8400 8401 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) 8402 { 8403 __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0); 8404 } 8405 EXPORT_SYMBOL_GPL(kvm_prepare_emulation_failure_exit); 8406 8407 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) 8408 { 8409 struct kvm *kvm = vcpu->kvm; 8410 8411 ++vcpu->stat.insn_emulation_fail; 8412 trace_kvm_emulate_insn_failed(vcpu); 8413 8414 if (emulation_type & EMULTYPE_VMWARE_GP) { 8415 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 8416 return 1; 8417 } 8418 8419 if (kvm->arch.exit_on_emulation_error || 8420 (emulation_type & EMULTYPE_SKIP)) { 8421 prepare_emulation_ctxt_failure_exit(vcpu); 8422 return 0; 8423 } 8424 8425 kvm_queue_exception(vcpu, UD_VECTOR); 8426 8427 if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) { 8428 prepare_emulation_ctxt_failure_exit(vcpu); 8429 return 0; 8430 } 8431 8432 return 1; 8433 } 8434 8435 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 8436 bool write_fault_to_shadow_pgtable, 8437 int emulation_type) 8438 { 8439 gpa_t gpa = cr2_or_gpa; 8440 kvm_pfn_t pfn; 8441 8442 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF)) 8443 return false; 8444 8445 if (WARN_ON_ONCE(is_guest_mode(vcpu)) || 8446 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF))) 8447 return false; 8448 8449 if (!vcpu->arch.mmu->root_role.direct) { 8450 /* 8451 * Write permission should be allowed since only 8452 * write access need to be emulated. 8453 */ 8454 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL); 8455 8456 /* 8457 * If the mapping is invalid in guest, let cpu retry 8458 * it to generate fault. 8459 */ 8460 if (gpa == INVALID_GPA) 8461 return true; 8462 } 8463 8464 /* 8465 * Do not retry the unhandleable instruction if it faults on the 8466 * readonly host memory, otherwise it will goto a infinite loop: 8467 * retry instruction -> write #PF -> emulation fail -> retry 8468 * instruction -> ... 8469 */ 8470 pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa)); 8471 8472 /* 8473 * If the instruction failed on the error pfn, it can not be fixed, 8474 * report the error to userspace. 8475 */ 8476 if (is_error_noslot_pfn(pfn)) 8477 return false; 8478 8479 kvm_release_pfn_clean(pfn); 8480 8481 /* The instructions are well-emulated on direct mmu. */ 8482 if (vcpu->arch.mmu->root_role.direct) { 8483 unsigned int indirect_shadow_pages; 8484 8485 write_lock(&vcpu->kvm->mmu_lock); 8486 indirect_shadow_pages = vcpu->kvm->arch.indirect_shadow_pages; 8487 write_unlock(&vcpu->kvm->mmu_lock); 8488 8489 if (indirect_shadow_pages) 8490 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); 8491 8492 return true; 8493 } 8494 8495 /* 8496 * if emulation was due to access to shadowed page table 8497 * and it failed try to unshadow page and re-enter the 8498 * guest to let CPU execute the instruction. 8499 */ 8500 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); 8501 8502 /* 8503 * If the access faults on its page table, it can not 8504 * be fixed by unprotecting shadow page and it should 8505 * be reported to userspace. 8506 */ 8507 return !write_fault_to_shadow_pgtable; 8508 } 8509 8510 static bool retry_instruction(struct x86_emulate_ctxt *ctxt, 8511 gpa_t cr2_or_gpa, int emulation_type) 8512 { 8513 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 8514 unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa; 8515 8516 last_retry_eip = vcpu->arch.last_retry_eip; 8517 last_retry_addr = vcpu->arch.last_retry_addr; 8518 8519 /* 8520 * If the emulation is caused by #PF and it is non-page_table 8521 * writing instruction, it means the VM-EXIT is caused by shadow 8522 * page protected, we can zap the shadow page and retry this 8523 * instruction directly. 8524 * 8525 * Note: if the guest uses a non-page-table modifying instruction 8526 * on the PDE that points to the instruction, then we will unmap 8527 * the instruction and go to an infinite loop. So, we cache the 8528 * last retried eip and the last fault address, if we meet the eip 8529 * and the address again, we can break out of the potential infinite 8530 * loop. 8531 */ 8532 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0; 8533 8534 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF)) 8535 return false; 8536 8537 if (WARN_ON_ONCE(is_guest_mode(vcpu)) || 8538 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF))) 8539 return false; 8540 8541 if (x86_page_table_writing_insn(ctxt)) 8542 return false; 8543 8544 if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa) 8545 return false; 8546 8547 vcpu->arch.last_retry_eip = ctxt->eip; 8548 vcpu->arch.last_retry_addr = cr2_or_gpa; 8549 8550 if (!vcpu->arch.mmu->root_role.direct) 8551 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL); 8552 8553 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); 8554 8555 return true; 8556 } 8557 8558 static int complete_emulated_mmio(struct kvm_vcpu *vcpu); 8559 static int complete_emulated_pio(struct kvm_vcpu *vcpu); 8560 8561 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7, 8562 unsigned long *db) 8563 { 8564 u32 dr6 = 0; 8565 int i; 8566 u32 enable, rwlen; 8567 8568 enable = dr7; 8569 rwlen = dr7 >> 16; 8570 for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4) 8571 if ((enable & 3) && (rwlen & 15) == type && db[i] == addr) 8572 dr6 |= (1 << i); 8573 return dr6; 8574 } 8575 8576 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu) 8577 { 8578 struct kvm_run *kvm_run = vcpu->run; 8579 8580 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { 8581 kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW; 8582 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu); 8583 kvm_run->debug.arch.exception = DB_VECTOR; 8584 kvm_run->exit_reason = KVM_EXIT_DEBUG; 8585 return 0; 8586 } 8587 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS); 8588 return 1; 8589 } 8590 8591 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) 8592 { 8593 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu); 8594 int r; 8595 8596 r = static_call(kvm_x86_skip_emulated_instruction)(vcpu); 8597 if (unlikely(!r)) 8598 return 0; 8599 8600 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS); 8601 8602 /* 8603 * rflags is the old, "raw" value of the flags. The new value has 8604 * not been saved yet. 8605 * 8606 * This is correct even for TF set by the guest, because "the 8607 * processor will not generate this exception after the instruction 8608 * that sets the TF flag". 8609 */ 8610 if (unlikely(rflags & X86_EFLAGS_TF)) 8611 r = kvm_vcpu_do_singlestep(vcpu); 8612 return r; 8613 } 8614 EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction); 8615 8616 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu) 8617 { 8618 u32 shadow; 8619 8620 if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF) 8621 return true; 8622 8623 /* 8624 * Intel CPUs inhibit code #DBs when MOV/POP SS blocking is active, 8625 * but AMD CPUs do not. MOV/POP SS blocking is rare, check that first 8626 * to avoid the relatively expensive CPUID lookup. 8627 */ 8628 shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu); 8629 return (shadow & KVM_X86_SHADOW_INT_MOV_SS) && 8630 guest_cpuid_is_intel(vcpu); 8631 } 8632 8633 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, 8634 int emulation_type, int *r) 8635 { 8636 WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE); 8637 8638 /* 8639 * Do not check for code breakpoints if hardware has already done the 8640 * checks, as inferred from the emulation type. On NO_DECODE and SKIP, 8641 * the instruction has passed all exception checks, and all intercepted 8642 * exceptions that trigger emulation have lower priority than code 8643 * breakpoints, i.e. the fact that the intercepted exception occurred 8644 * means any code breakpoints have already been serviced. 8645 * 8646 * Note, KVM needs to check for code #DBs on EMULTYPE_TRAP_UD_FORCED as 8647 * hardware has checked the RIP of the magic prefix, but not the RIP of 8648 * the instruction being emulated. The intent of forced emulation is 8649 * to behave as if KVM intercepted the instruction without an exception 8650 * and without a prefix. 8651 */ 8652 if (emulation_type & (EMULTYPE_NO_DECODE | EMULTYPE_SKIP | 8653 EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF)) 8654 return false; 8655 8656 if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) && 8657 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) { 8658 struct kvm_run *kvm_run = vcpu->run; 8659 unsigned long eip = kvm_get_linear_rip(vcpu); 8660 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, 8661 vcpu->arch.guest_debug_dr7, 8662 vcpu->arch.eff_db); 8663 8664 if (dr6 != 0) { 8665 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW; 8666 kvm_run->debug.arch.pc = eip; 8667 kvm_run->debug.arch.exception = DB_VECTOR; 8668 kvm_run->exit_reason = KVM_EXIT_DEBUG; 8669 *r = 0; 8670 return true; 8671 } 8672 } 8673 8674 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) && 8675 !kvm_is_code_breakpoint_inhibited(vcpu)) { 8676 unsigned long eip = kvm_get_linear_rip(vcpu); 8677 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, 8678 vcpu->arch.dr7, 8679 vcpu->arch.db); 8680 8681 if (dr6 != 0) { 8682 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6); 8683 *r = 1; 8684 return true; 8685 } 8686 } 8687 8688 return false; 8689 } 8690 8691 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) 8692 { 8693 switch (ctxt->opcode_len) { 8694 case 1: 8695 switch (ctxt->b) { 8696 case 0xe4: /* IN */ 8697 case 0xe5: 8698 case 0xec: 8699 case 0xed: 8700 case 0xe6: /* OUT */ 8701 case 0xe7: 8702 case 0xee: 8703 case 0xef: 8704 case 0x6c: /* INS */ 8705 case 0x6d: 8706 case 0x6e: /* OUTS */ 8707 case 0x6f: 8708 return true; 8709 } 8710 break; 8711 case 2: 8712 switch (ctxt->b) { 8713 case 0x33: /* RDPMC */ 8714 return true; 8715 } 8716 break; 8717 } 8718 8719 return false; 8720 } 8721 8722 /* 8723 * Decode an instruction for emulation. The caller is responsible for handling 8724 * code breakpoints. Note, manually detecting code breakpoints is unnecessary 8725 * (and wrong) when emulating on an intercepted fault-like exception[*], as 8726 * code breakpoints have higher priority and thus have already been done by 8727 * hardware. 8728 * 8729 * [*] Except #MC, which is higher priority, but KVM should never emulate in 8730 * response to a machine check. 8731 */ 8732 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type, 8733 void *insn, int insn_len) 8734 { 8735 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 8736 int r; 8737 8738 init_emulate_ctxt(vcpu); 8739 8740 r = x86_decode_insn(ctxt, insn, insn_len, emulation_type); 8741 8742 trace_kvm_emulate_insn_start(vcpu); 8743 ++vcpu->stat.insn_emulation; 8744 8745 return r; 8746 } 8747 EXPORT_SYMBOL_GPL(x86_decode_emulated_instruction); 8748 8749 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, 8750 int emulation_type, void *insn, int insn_len) 8751 { 8752 int r; 8753 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 8754 bool writeback = true; 8755 bool write_fault_to_spt; 8756 8757 if (unlikely(!kvm_can_emulate_insn(vcpu, emulation_type, insn, insn_len))) 8758 return 1; 8759 8760 vcpu->arch.l1tf_flush_l1d = true; 8761 8762 /* 8763 * Clear write_fault_to_shadow_pgtable here to ensure it is 8764 * never reused. 8765 */ 8766 write_fault_to_spt = vcpu->arch.write_fault_to_shadow_pgtable; 8767 vcpu->arch.write_fault_to_shadow_pgtable = false; 8768 8769 if (!(emulation_type & EMULTYPE_NO_DECODE)) { 8770 kvm_clear_exception_queue(vcpu); 8771 8772 /* 8773 * Return immediately if RIP hits a code breakpoint, such #DBs 8774 * are fault-like and are higher priority than any faults on 8775 * the code fetch itself. 8776 */ 8777 if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r)) 8778 return r; 8779 8780 r = x86_decode_emulated_instruction(vcpu, emulation_type, 8781 insn, insn_len); 8782 if (r != EMULATION_OK) { 8783 if ((emulation_type & EMULTYPE_TRAP_UD) || 8784 (emulation_type & EMULTYPE_TRAP_UD_FORCED)) { 8785 kvm_queue_exception(vcpu, UD_VECTOR); 8786 return 1; 8787 } 8788 if (reexecute_instruction(vcpu, cr2_or_gpa, 8789 write_fault_to_spt, 8790 emulation_type)) 8791 return 1; 8792 8793 if (ctxt->have_exception && 8794 !(emulation_type & EMULTYPE_SKIP)) { 8795 /* 8796 * #UD should result in just EMULATION_FAILED, and trap-like 8797 * exception should not be encountered during decode. 8798 */ 8799 WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR || 8800 exception_type(ctxt->exception.vector) == EXCPT_TRAP); 8801 inject_emulated_exception(vcpu); 8802 return 1; 8803 } 8804 return handle_emulation_failure(vcpu, emulation_type); 8805 } 8806 } 8807 8808 if ((emulation_type & EMULTYPE_VMWARE_GP) && 8809 !is_vmware_backdoor_opcode(ctxt)) { 8810 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 8811 return 1; 8812 } 8813 8814 /* 8815 * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for 8816 * use *only* by vendor callbacks for kvm_skip_emulated_instruction(). 8817 * The caller is responsible for updating interruptibility state and 8818 * injecting single-step #DBs. 8819 */ 8820 if (emulation_type & EMULTYPE_SKIP) { 8821 if (ctxt->mode != X86EMUL_MODE_PROT64) 8822 ctxt->eip = (u32)ctxt->_eip; 8823 else 8824 ctxt->eip = ctxt->_eip; 8825 8826 if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) { 8827 r = 1; 8828 goto writeback; 8829 } 8830 8831 kvm_rip_write(vcpu, ctxt->eip); 8832 if (ctxt->eflags & X86_EFLAGS_RF) 8833 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF); 8834 return 1; 8835 } 8836 8837 if (retry_instruction(ctxt, cr2_or_gpa, emulation_type)) 8838 return 1; 8839 8840 /* this is needed for vmware backdoor interface to work since it 8841 changes registers values during IO operation */ 8842 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { 8843 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 8844 emulator_invalidate_register_cache(ctxt); 8845 } 8846 8847 restart: 8848 if (emulation_type & EMULTYPE_PF) { 8849 /* Save the faulting GPA (cr2) in the address field */ 8850 ctxt->exception.address = cr2_or_gpa; 8851 8852 /* With shadow page tables, cr2 contains a GVA or nGPA. */ 8853 if (vcpu->arch.mmu->root_role.direct) { 8854 ctxt->gpa_available = true; 8855 ctxt->gpa_val = cr2_or_gpa; 8856 } 8857 } else { 8858 /* Sanitize the address out of an abundance of paranoia. */ 8859 ctxt->exception.address = 0; 8860 } 8861 8862 r = x86_emulate_insn(ctxt); 8863 8864 if (r == EMULATION_INTERCEPTED) 8865 return 1; 8866 8867 if (r == EMULATION_FAILED) { 8868 if (reexecute_instruction(vcpu, cr2_or_gpa, write_fault_to_spt, 8869 emulation_type)) 8870 return 1; 8871 8872 return handle_emulation_failure(vcpu, emulation_type); 8873 } 8874 8875 if (ctxt->have_exception) { 8876 r = 1; 8877 inject_emulated_exception(vcpu); 8878 } else if (vcpu->arch.pio.count) { 8879 if (!vcpu->arch.pio.in) { 8880 /* FIXME: return into emulator if single-stepping. */ 8881 vcpu->arch.pio.count = 0; 8882 } else { 8883 writeback = false; 8884 vcpu->arch.complete_userspace_io = complete_emulated_pio; 8885 } 8886 r = 0; 8887 } else if (vcpu->mmio_needed) { 8888 ++vcpu->stat.mmio_exits; 8889 8890 if (!vcpu->mmio_is_write) 8891 writeback = false; 8892 r = 0; 8893 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 8894 } else if (vcpu->arch.complete_userspace_io) { 8895 writeback = false; 8896 r = 0; 8897 } else if (r == EMULATION_RESTART) 8898 goto restart; 8899 else 8900 r = 1; 8901 8902 writeback: 8903 if (writeback) { 8904 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu); 8905 toggle_interruptibility(vcpu, ctxt->interruptibility); 8906 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 8907 8908 /* 8909 * Note, EXCPT_DB is assumed to be fault-like as the emulator 8910 * only supports code breakpoints and general detect #DB, both 8911 * of which are fault-like. 8912 */ 8913 if (!ctxt->have_exception || 8914 exception_type(ctxt->exception.vector) == EXCPT_TRAP) { 8915 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS); 8916 if (ctxt->is_branch) 8917 kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_BRANCH_INSTRUCTIONS); 8918 kvm_rip_write(vcpu, ctxt->eip); 8919 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP))) 8920 r = kvm_vcpu_do_singlestep(vcpu); 8921 static_call_cond(kvm_x86_update_emulated_instruction)(vcpu); 8922 __kvm_set_rflags(vcpu, ctxt->eflags); 8923 } 8924 8925 /* 8926 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will 8927 * do nothing, and it will be requested again as soon as 8928 * the shadow expires. But we still need to check here, 8929 * because POPF has no interrupt shadow. 8930 */ 8931 if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF)) 8932 kvm_make_request(KVM_REQ_EVENT, vcpu); 8933 } else 8934 vcpu->arch.emulate_regs_need_sync_to_vcpu = true; 8935 8936 return r; 8937 } 8938 8939 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type) 8940 { 8941 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0); 8942 } 8943 EXPORT_SYMBOL_GPL(kvm_emulate_instruction); 8944 8945 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu, 8946 void *insn, int insn_len) 8947 { 8948 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len); 8949 } 8950 EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer); 8951 8952 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu) 8953 { 8954 vcpu->arch.pio.count = 0; 8955 return 1; 8956 } 8957 8958 static int complete_fast_pio_out(struct kvm_vcpu *vcpu) 8959 { 8960 vcpu->arch.pio.count = 0; 8961 8962 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) 8963 return 1; 8964 8965 return kvm_skip_emulated_instruction(vcpu); 8966 } 8967 8968 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, 8969 unsigned short port) 8970 { 8971 unsigned long val = kvm_rax_read(vcpu); 8972 int ret = emulator_pio_out(vcpu, size, port, &val, 1); 8973 8974 if (ret) 8975 return ret; 8976 8977 /* 8978 * Workaround userspace that relies on old KVM behavior of %rip being 8979 * incremented prior to exiting to userspace to handle "OUT 0x7e". 8980 */ 8981 if (port == 0x7e && 8982 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) { 8983 vcpu->arch.complete_userspace_io = 8984 complete_fast_pio_out_port_0x7e; 8985 kvm_skip_emulated_instruction(vcpu); 8986 } else { 8987 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu); 8988 vcpu->arch.complete_userspace_io = complete_fast_pio_out; 8989 } 8990 return 0; 8991 } 8992 8993 static int complete_fast_pio_in(struct kvm_vcpu *vcpu) 8994 { 8995 unsigned long val; 8996 8997 /* We should only ever be called with arch.pio.count equal to 1 */ 8998 BUG_ON(vcpu->arch.pio.count != 1); 8999 9000 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) { 9001 vcpu->arch.pio.count = 0; 9002 return 1; 9003 } 9004 9005 /* For size less than 4 we merge, else we zero extend */ 9006 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0; 9007 9008 complete_emulator_pio_in(vcpu, &val); 9009 kvm_rax_write(vcpu, val); 9010 9011 return kvm_skip_emulated_instruction(vcpu); 9012 } 9013 9014 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size, 9015 unsigned short port) 9016 { 9017 unsigned long val; 9018 int ret; 9019 9020 /* For size less than 4 we merge, else we zero extend */ 9021 val = (size < 4) ? kvm_rax_read(vcpu) : 0; 9022 9023 ret = emulator_pio_in(vcpu, size, port, &val, 1); 9024 if (ret) { 9025 kvm_rax_write(vcpu, val); 9026 return ret; 9027 } 9028 9029 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu); 9030 vcpu->arch.complete_userspace_io = complete_fast_pio_in; 9031 9032 return 0; 9033 } 9034 9035 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in) 9036 { 9037 int ret; 9038 9039 if (in) 9040 ret = kvm_fast_pio_in(vcpu, size, port); 9041 else 9042 ret = kvm_fast_pio_out(vcpu, size, port); 9043 return ret && kvm_skip_emulated_instruction(vcpu); 9044 } 9045 EXPORT_SYMBOL_GPL(kvm_fast_pio); 9046 9047 static int kvmclock_cpu_down_prep(unsigned int cpu) 9048 { 9049 __this_cpu_write(cpu_tsc_khz, 0); 9050 return 0; 9051 } 9052 9053 static void tsc_khz_changed(void *data) 9054 { 9055 struct cpufreq_freqs *freq = data; 9056 unsigned long khz = 0; 9057 9058 WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_CONSTANT_TSC)); 9059 9060 if (data) 9061 khz = freq->new; 9062 else 9063 khz = cpufreq_quick_get(raw_smp_processor_id()); 9064 if (!khz) 9065 khz = tsc_khz; 9066 __this_cpu_write(cpu_tsc_khz, khz); 9067 } 9068 9069 #ifdef CONFIG_X86_64 9070 static void kvm_hyperv_tsc_notifier(void) 9071 { 9072 struct kvm *kvm; 9073 int cpu; 9074 9075 mutex_lock(&kvm_lock); 9076 list_for_each_entry(kvm, &vm_list, vm_list) 9077 kvm_make_mclock_inprogress_request(kvm); 9078 9079 /* no guest entries from this point */ 9080 hyperv_stop_tsc_emulation(); 9081 9082 /* TSC frequency always matches when on Hyper-V */ 9083 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 9084 for_each_present_cpu(cpu) 9085 per_cpu(cpu_tsc_khz, cpu) = tsc_khz; 9086 } 9087 kvm_caps.max_guest_tsc_khz = tsc_khz; 9088 9089 list_for_each_entry(kvm, &vm_list, vm_list) { 9090 __kvm_start_pvclock_update(kvm); 9091 pvclock_update_vm_gtod_copy(kvm); 9092 kvm_end_pvclock_update(kvm); 9093 } 9094 9095 mutex_unlock(&kvm_lock); 9096 } 9097 #endif 9098 9099 static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu) 9100 { 9101 struct kvm *kvm; 9102 struct kvm_vcpu *vcpu; 9103 int send_ipi = 0; 9104 unsigned long i; 9105 9106 /* 9107 * We allow guests to temporarily run on slowing clocks, 9108 * provided we notify them after, or to run on accelerating 9109 * clocks, provided we notify them before. Thus time never 9110 * goes backwards. 9111 * 9112 * However, we have a problem. We can't atomically update 9113 * the frequency of a given CPU from this function; it is 9114 * merely a notifier, which can be called from any CPU. 9115 * Changing the TSC frequency at arbitrary points in time 9116 * requires a recomputation of local variables related to 9117 * the TSC for each VCPU. We must flag these local variables 9118 * to be updated and be sure the update takes place with the 9119 * new frequency before any guests proceed. 9120 * 9121 * Unfortunately, the combination of hotplug CPU and frequency 9122 * change creates an intractable locking scenario; the order 9123 * of when these callouts happen is undefined with respect to 9124 * CPU hotplug, and they can race with each other. As such, 9125 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is 9126 * undefined; you can actually have a CPU frequency change take 9127 * place in between the computation of X and the setting of the 9128 * variable. To protect against this problem, all updates of 9129 * the per_cpu tsc_khz variable are done in an interrupt 9130 * protected IPI, and all callers wishing to update the value 9131 * must wait for a synchronous IPI to complete (which is trivial 9132 * if the caller is on the CPU already). This establishes the 9133 * necessary total order on variable updates. 9134 * 9135 * Note that because a guest time update may take place 9136 * anytime after the setting of the VCPU's request bit, the 9137 * correct TSC value must be set before the request. However, 9138 * to ensure the update actually makes it to any guest which 9139 * starts running in hardware virtualization between the set 9140 * and the acquisition of the spinlock, we must also ping the 9141 * CPU after setting the request bit. 9142 * 9143 */ 9144 9145 smp_call_function_single(cpu, tsc_khz_changed, freq, 1); 9146 9147 mutex_lock(&kvm_lock); 9148 list_for_each_entry(kvm, &vm_list, vm_list) { 9149 kvm_for_each_vcpu(i, vcpu, kvm) { 9150 if (vcpu->cpu != cpu) 9151 continue; 9152 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 9153 if (vcpu->cpu != raw_smp_processor_id()) 9154 send_ipi = 1; 9155 } 9156 } 9157 mutex_unlock(&kvm_lock); 9158 9159 if (freq->old < freq->new && send_ipi) { 9160 /* 9161 * We upscale the frequency. Must make the guest 9162 * doesn't see old kvmclock values while running with 9163 * the new frequency, otherwise we risk the guest sees 9164 * time go backwards. 9165 * 9166 * In case we update the frequency for another cpu 9167 * (which might be in guest context) send an interrupt 9168 * to kick the cpu out of guest context. Next time 9169 * guest context is entered kvmclock will be updated, 9170 * so the guest will not see stale values. 9171 */ 9172 smp_call_function_single(cpu, tsc_khz_changed, freq, 1); 9173 } 9174 } 9175 9176 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, 9177 void *data) 9178 { 9179 struct cpufreq_freqs *freq = data; 9180 int cpu; 9181 9182 if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) 9183 return 0; 9184 if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) 9185 return 0; 9186 9187 for_each_cpu(cpu, freq->policy->cpus) 9188 __kvmclock_cpufreq_notifier(freq, cpu); 9189 9190 return 0; 9191 } 9192 9193 static struct notifier_block kvmclock_cpufreq_notifier_block = { 9194 .notifier_call = kvmclock_cpufreq_notifier 9195 }; 9196 9197 static int kvmclock_cpu_online(unsigned int cpu) 9198 { 9199 tsc_khz_changed(NULL); 9200 return 0; 9201 } 9202 9203 static void kvm_timer_init(void) 9204 { 9205 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 9206 max_tsc_khz = tsc_khz; 9207 9208 if (IS_ENABLED(CONFIG_CPU_FREQ)) { 9209 struct cpufreq_policy *policy; 9210 int cpu; 9211 9212 cpu = get_cpu(); 9213 policy = cpufreq_cpu_get(cpu); 9214 if (policy) { 9215 if (policy->cpuinfo.max_freq) 9216 max_tsc_khz = policy->cpuinfo.max_freq; 9217 cpufreq_cpu_put(policy); 9218 } 9219 put_cpu(); 9220 } 9221 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, 9222 CPUFREQ_TRANSITION_NOTIFIER); 9223 9224 cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online", 9225 kvmclock_cpu_online, kvmclock_cpu_down_prep); 9226 } 9227 } 9228 9229 #ifdef CONFIG_X86_64 9230 static void pvclock_gtod_update_fn(struct work_struct *work) 9231 { 9232 struct kvm *kvm; 9233 struct kvm_vcpu *vcpu; 9234 unsigned long i; 9235 9236 mutex_lock(&kvm_lock); 9237 list_for_each_entry(kvm, &vm_list, vm_list) 9238 kvm_for_each_vcpu(i, vcpu, kvm) 9239 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 9240 atomic_set(&kvm_guest_has_master_clock, 0); 9241 mutex_unlock(&kvm_lock); 9242 } 9243 9244 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn); 9245 9246 /* 9247 * Indirection to move queue_work() out of the tk_core.seq write held 9248 * region to prevent possible deadlocks against time accessors which 9249 * are invoked with work related locks held. 9250 */ 9251 static void pvclock_irq_work_fn(struct irq_work *w) 9252 { 9253 queue_work(system_long_wq, &pvclock_gtod_work); 9254 } 9255 9256 static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn); 9257 9258 /* 9259 * Notification about pvclock gtod data update. 9260 */ 9261 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused, 9262 void *priv) 9263 { 9264 struct pvclock_gtod_data *gtod = &pvclock_gtod_data; 9265 struct timekeeper *tk = priv; 9266 9267 update_pvclock_gtod(tk); 9268 9269 /* 9270 * Disable master clock if host does not trust, or does not use, 9271 * TSC based clocksource. Delegate queue_work() to irq_work as 9272 * this is invoked with tk_core.seq write held. 9273 */ 9274 if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) && 9275 atomic_read(&kvm_guest_has_master_clock) != 0) 9276 irq_work_queue(&pvclock_irq_work); 9277 return 0; 9278 } 9279 9280 static struct notifier_block pvclock_gtod_notifier = { 9281 .notifier_call = pvclock_gtod_notify, 9282 }; 9283 #endif 9284 9285 static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) 9286 { 9287 memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); 9288 9289 #define __KVM_X86_OP(func) \ 9290 static_call_update(kvm_x86_##func, kvm_x86_ops.func); 9291 #define KVM_X86_OP(func) \ 9292 WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func) 9293 #define KVM_X86_OP_OPTIONAL __KVM_X86_OP 9294 #define KVM_X86_OP_OPTIONAL_RET0(func) \ 9295 static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \ 9296 (void *)__static_call_return0); 9297 #include <asm/kvm-x86-ops.h> 9298 #undef __KVM_X86_OP 9299 9300 kvm_pmu_ops_update(ops->pmu_ops); 9301 } 9302 9303 static int kvm_x86_check_processor_compatibility(void) 9304 { 9305 int cpu = smp_processor_id(); 9306 struct cpuinfo_x86 *c = &cpu_data(cpu); 9307 9308 /* 9309 * Compatibility checks are done when loading KVM and when enabling 9310 * hardware, e.g. during CPU hotplug, to ensure all online CPUs are 9311 * compatible, i.e. KVM should never perform a compatibility check on 9312 * an offline CPU. 9313 */ 9314 WARN_ON(!cpu_online(cpu)); 9315 9316 if (__cr4_reserved_bits(cpu_has, c) != 9317 __cr4_reserved_bits(cpu_has, &boot_cpu_data)) 9318 return -EIO; 9319 9320 return static_call(kvm_x86_check_processor_compatibility)(); 9321 } 9322 9323 static void kvm_x86_check_cpu_compat(void *ret) 9324 { 9325 *(int *)ret = kvm_x86_check_processor_compatibility(); 9326 } 9327 9328 static int __kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) 9329 { 9330 u64 host_pat; 9331 int r, cpu; 9332 9333 if (kvm_x86_ops.hardware_enable) { 9334 pr_err("already loaded vendor module '%s'\n", kvm_x86_ops.name); 9335 return -EEXIST; 9336 } 9337 9338 /* 9339 * KVM explicitly assumes that the guest has an FPU and 9340 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the 9341 * vCPU's FPU state as a fxregs_state struct. 9342 */ 9343 if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) { 9344 pr_err("inadequate fpu\n"); 9345 return -EOPNOTSUPP; 9346 } 9347 9348 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 9349 pr_err("RT requires X86_FEATURE_CONSTANT_TSC\n"); 9350 return -EOPNOTSUPP; 9351 } 9352 9353 /* 9354 * KVM assumes that PAT entry '0' encodes WB memtype and simply zeroes 9355 * the PAT bits in SPTEs. Bail if PAT[0] is programmed to something 9356 * other than WB. Note, EPT doesn't utilize the PAT, but don't bother 9357 * with an exception. PAT[0] is set to WB on RESET and also by the 9358 * kernel, i.e. failure indicates a kernel bug or broken firmware. 9359 */ 9360 if (rdmsrl_safe(MSR_IA32_CR_PAT, &host_pat) || 9361 (host_pat & GENMASK(2, 0)) != 6) { 9362 pr_err("host PAT[0] is not WB\n"); 9363 return -EIO; 9364 } 9365 9366 x86_emulator_cache = kvm_alloc_emulator_cache(); 9367 if (!x86_emulator_cache) { 9368 pr_err("failed to allocate cache for x86 emulator\n"); 9369 return -ENOMEM; 9370 } 9371 9372 user_return_msrs = alloc_percpu(struct kvm_user_return_msrs); 9373 if (!user_return_msrs) { 9374 pr_err("failed to allocate percpu kvm_user_return_msrs\n"); 9375 r = -ENOMEM; 9376 goto out_free_x86_emulator_cache; 9377 } 9378 kvm_nr_uret_msrs = 0; 9379 9380 r = kvm_mmu_vendor_module_init(); 9381 if (r) 9382 goto out_free_percpu; 9383 9384 if (boot_cpu_has(X86_FEATURE_XSAVE)) { 9385 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 9386 kvm_caps.supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0; 9387 } 9388 9389 rdmsrl_safe(MSR_EFER, &host_efer); 9390 9391 if (boot_cpu_has(X86_FEATURE_XSAVES)) 9392 rdmsrl(MSR_IA32_XSS, host_xss); 9393 9394 kvm_init_pmu_capability(ops->pmu_ops); 9395 9396 r = ops->hardware_setup(); 9397 if (r != 0) 9398 goto out_mmu_exit; 9399 9400 kvm_ops_update(ops); 9401 9402 for_each_online_cpu(cpu) { 9403 smp_call_function_single(cpu, kvm_x86_check_cpu_compat, &r, 1); 9404 if (r < 0) 9405 goto out_unwind_ops; 9406 } 9407 9408 /* 9409 * Point of no return! DO NOT add error paths below this point unless 9410 * absolutely necessary, as most operations from this point forward 9411 * require unwinding. 9412 */ 9413 kvm_timer_init(); 9414 9415 if (pi_inject_timer == -1) 9416 pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER); 9417 #ifdef CONFIG_X86_64 9418 pvclock_gtod_register_notifier(&pvclock_gtod_notifier); 9419 9420 if (hypervisor_is_type(X86_HYPER_MS_HYPERV)) 9421 set_hv_tscchange_cb(kvm_hyperv_tsc_notifier); 9422 #endif 9423 9424 kvm_register_perf_callbacks(ops->handle_intel_pt_intr); 9425 9426 if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES)) 9427 kvm_caps.supported_xss = 0; 9428 9429 #define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f) 9430 cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_); 9431 #undef __kvm_cpu_cap_has 9432 9433 if (kvm_caps.has_tsc_control) { 9434 /* 9435 * Make sure the user can only configure tsc_khz values that 9436 * fit into a signed integer. 9437 * A min value is not calculated because it will always 9438 * be 1 on all machines. 9439 */ 9440 u64 max = min(0x7fffffffULL, 9441 __scale_tsc(kvm_caps.max_tsc_scaling_ratio, tsc_khz)); 9442 kvm_caps.max_guest_tsc_khz = max; 9443 } 9444 kvm_caps.default_tsc_scaling_ratio = 1ULL << kvm_caps.tsc_scaling_ratio_frac_bits; 9445 kvm_init_msr_list(); 9446 return 0; 9447 9448 out_unwind_ops: 9449 kvm_x86_ops.hardware_enable = NULL; 9450 static_call(kvm_x86_hardware_unsetup)(); 9451 out_mmu_exit: 9452 kvm_mmu_vendor_module_exit(); 9453 out_free_percpu: 9454 free_percpu(user_return_msrs); 9455 out_free_x86_emulator_cache: 9456 kmem_cache_destroy(x86_emulator_cache); 9457 return r; 9458 } 9459 9460 int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) 9461 { 9462 int r; 9463 9464 mutex_lock(&vendor_module_lock); 9465 r = __kvm_x86_vendor_init(ops); 9466 mutex_unlock(&vendor_module_lock); 9467 9468 return r; 9469 } 9470 EXPORT_SYMBOL_GPL(kvm_x86_vendor_init); 9471 9472 void kvm_x86_vendor_exit(void) 9473 { 9474 kvm_unregister_perf_callbacks(); 9475 9476 #ifdef CONFIG_X86_64 9477 if (hypervisor_is_type(X86_HYPER_MS_HYPERV)) 9478 clear_hv_tscchange_cb(); 9479 #endif 9480 kvm_lapic_exit(); 9481 9482 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { 9483 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block, 9484 CPUFREQ_TRANSITION_NOTIFIER); 9485 cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE); 9486 } 9487 #ifdef CONFIG_X86_64 9488 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier); 9489 irq_work_sync(&pvclock_irq_work); 9490 cancel_work_sync(&pvclock_gtod_work); 9491 #endif 9492 static_call(kvm_x86_hardware_unsetup)(); 9493 kvm_mmu_vendor_module_exit(); 9494 free_percpu(user_return_msrs); 9495 kmem_cache_destroy(x86_emulator_cache); 9496 #ifdef CONFIG_KVM_XEN 9497 static_key_deferred_flush(&kvm_xen_enabled); 9498 WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key)); 9499 #endif 9500 mutex_lock(&vendor_module_lock); 9501 kvm_x86_ops.hardware_enable = NULL; 9502 mutex_unlock(&vendor_module_lock); 9503 } 9504 EXPORT_SYMBOL_GPL(kvm_x86_vendor_exit); 9505 9506 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason) 9507 { 9508 /* 9509 * The vCPU has halted, e.g. executed HLT. Update the run state if the 9510 * local APIC is in-kernel, the run loop will detect the non-runnable 9511 * state and halt the vCPU. Exit to userspace if the local APIC is 9512 * managed by userspace, in which case userspace is responsible for 9513 * handling wake events. 9514 */ 9515 ++vcpu->stat.halt_exits; 9516 if (lapic_in_kernel(vcpu)) { 9517 vcpu->arch.mp_state = state; 9518 return 1; 9519 } else { 9520 vcpu->run->exit_reason = reason; 9521 return 0; 9522 } 9523 } 9524 9525 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu) 9526 { 9527 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT); 9528 } 9529 EXPORT_SYMBOL_GPL(kvm_emulate_halt_noskip); 9530 9531 int kvm_emulate_halt(struct kvm_vcpu *vcpu) 9532 { 9533 int ret = kvm_skip_emulated_instruction(vcpu); 9534 /* 9535 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered 9536 * KVM_EXIT_DEBUG here. 9537 */ 9538 return kvm_emulate_halt_noskip(vcpu) && ret; 9539 } 9540 EXPORT_SYMBOL_GPL(kvm_emulate_halt); 9541 9542 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu) 9543 { 9544 int ret = kvm_skip_emulated_instruction(vcpu); 9545 9546 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD, 9547 KVM_EXIT_AP_RESET_HOLD) && ret; 9548 } 9549 EXPORT_SYMBOL_GPL(kvm_emulate_ap_reset_hold); 9550 9551 #ifdef CONFIG_X86_64 9552 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr, 9553 unsigned long clock_type) 9554 { 9555 struct kvm_clock_pairing clock_pairing; 9556 struct timespec64 ts; 9557 u64 cycle; 9558 int ret; 9559 9560 if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK) 9561 return -KVM_EOPNOTSUPP; 9562 9563 /* 9564 * When tsc is in permanent catchup mode guests won't be able to use 9565 * pvclock_read_retry loop to get consistent view of pvclock 9566 */ 9567 if (vcpu->arch.tsc_always_catchup) 9568 return -KVM_EOPNOTSUPP; 9569 9570 if (!kvm_get_walltime_and_clockread(&ts, &cycle)) 9571 return -KVM_EOPNOTSUPP; 9572 9573 clock_pairing.sec = ts.tv_sec; 9574 clock_pairing.nsec = ts.tv_nsec; 9575 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle); 9576 clock_pairing.flags = 0; 9577 memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad)); 9578 9579 ret = 0; 9580 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing, 9581 sizeof(struct kvm_clock_pairing))) 9582 ret = -KVM_EFAULT; 9583 9584 return ret; 9585 } 9586 #endif 9587 9588 /* 9589 * kvm_pv_kick_cpu_op: Kick a vcpu. 9590 * 9591 * @apicid - apicid of vcpu to be kicked. 9592 */ 9593 static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid) 9594 { 9595 /* 9596 * All other fields are unused for APIC_DM_REMRD, but may be consumed by 9597 * common code, e.g. for tracing. Defer initialization to the compiler. 9598 */ 9599 struct kvm_lapic_irq lapic_irq = { 9600 .delivery_mode = APIC_DM_REMRD, 9601 .dest_mode = APIC_DEST_PHYSICAL, 9602 .shorthand = APIC_DEST_NOSHORT, 9603 .dest_id = apicid, 9604 }; 9605 9606 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL); 9607 } 9608 9609 bool kvm_apicv_activated(struct kvm *kvm) 9610 { 9611 return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0); 9612 } 9613 EXPORT_SYMBOL_GPL(kvm_apicv_activated); 9614 9615 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu) 9616 { 9617 ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons); 9618 ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu); 9619 9620 return (vm_reasons | vcpu_reasons) == 0; 9621 } 9622 EXPORT_SYMBOL_GPL(kvm_vcpu_apicv_activated); 9623 9624 static void set_or_clear_apicv_inhibit(unsigned long *inhibits, 9625 enum kvm_apicv_inhibit reason, bool set) 9626 { 9627 if (set) 9628 __set_bit(reason, inhibits); 9629 else 9630 __clear_bit(reason, inhibits); 9631 9632 trace_kvm_apicv_inhibit_changed(reason, set, *inhibits); 9633 } 9634 9635 static void kvm_apicv_init(struct kvm *kvm) 9636 { 9637 unsigned long *inhibits = &kvm->arch.apicv_inhibit_reasons; 9638 9639 init_rwsem(&kvm->arch.apicv_update_lock); 9640 9641 set_or_clear_apicv_inhibit(inhibits, APICV_INHIBIT_REASON_ABSENT, true); 9642 9643 if (!enable_apicv) 9644 set_or_clear_apicv_inhibit(inhibits, 9645 APICV_INHIBIT_REASON_DISABLE, true); 9646 } 9647 9648 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) 9649 { 9650 struct kvm_vcpu *target = NULL; 9651 struct kvm_apic_map *map; 9652 9653 vcpu->stat.directed_yield_attempted++; 9654 9655 if (single_task_running()) 9656 goto no_yield; 9657 9658 rcu_read_lock(); 9659 map = rcu_dereference(vcpu->kvm->arch.apic_map); 9660 9661 if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id]) 9662 target = map->phys_map[dest_id]->vcpu; 9663 9664 rcu_read_unlock(); 9665 9666 if (!target || !READ_ONCE(target->ready)) 9667 goto no_yield; 9668 9669 /* Ignore requests to yield to self */ 9670 if (vcpu == target) 9671 goto no_yield; 9672 9673 if (kvm_vcpu_yield_to(target) <= 0) 9674 goto no_yield; 9675 9676 vcpu->stat.directed_yield_successful++; 9677 9678 no_yield: 9679 return; 9680 } 9681 9682 static int complete_hypercall_exit(struct kvm_vcpu *vcpu) 9683 { 9684 u64 ret = vcpu->run->hypercall.ret; 9685 9686 if (!is_64_bit_mode(vcpu)) 9687 ret = (u32)ret; 9688 kvm_rax_write(vcpu, ret); 9689 ++vcpu->stat.hypercalls; 9690 return kvm_skip_emulated_instruction(vcpu); 9691 } 9692 9693 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) 9694 { 9695 unsigned long nr, a0, a1, a2, a3, ret; 9696 int op_64_bit; 9697 9698 if (kvm_xen_hypercall_enabled(vcpu->kvm)) 9699 return kvm_xen_hypercall(vcpu); 9700 9701 if (kvm_hv_hypercall_enabled(vcpu)) 9702 return kvm_hv_hypercall(vcpu); 9703 9704 nr = kvm_rax_read(vcpu); 9705 a0 = kvm_rbx_read(vcpu); 9706 a1 = kvm_rcx_read(vcpu); 9707 a2 = kvm_rdx_read(vcpu); 9708 a3 = kvm_rsi_read(vcpu); 9709 9710 trace_kvm_hypercall(nr, a0, a1, a2, a3); 9711 9712 op_64_bit = is_64_bit_hypercall(vcpu); 9713 if (!op_64_bit) { 9714 nr &= 0xFFFFFFFF; 9715 a0 &= 0xFFFFFFFF; 9716 a1 &= 0xFFFFFFFF; 9717 a2 &= 0xFFFFFFFF; 9718 a3 &= 0xFFFFFFFF; 9719 } 9720 9721 if (static_call(kvm_x86_get_cpl)(vcpu) != 0) { 9722 ret = -KVM_EPERM; 9723 goto out; 9724 } 9725 9726 ret = -KVM_ENOSYS; 9727 9728 switch (nr) { 9729 case KVM_HC_VAPIC_POLL_IRQ: 9730 ret = 0; 9731 break; 9732 case KVM_HC_KICK_CPU: 9733 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT)) 9734 break; 9735 9736 kvm_pv_kick_cpu_op(vcpu->kvm, a1); 9737 kvm_sched_yield(vcpu, a1); 9738 ret = 0; 9739 break; 9740 #ifdef CONFIG_X86_64 9741 case KVM_HC_CLOCK_PAIRING: 9742 ret = kvm_pv_clock_pairing(vcpu, a0, a1); 9743 break; 9744 #endif 9745 case KVM_HC_SEND_IPI: 9746 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI)) 9747 break; 9748 9749 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit); 9750 break; 9751 case KVM_HC_SCHED_YIELD: 9752 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD)) 9753 break; 9754 9755 kvm_sched_yield(vcpu, a0); 9756 ret = 0; 9757 break; 9758 case KVM_HC_MAP_GPA_RANGE: { 9759 u64 gpa = a0, npages = a1, attrs = a2; 9760 9761 ret = -KVM_ENOSYS; 9762 if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE))) 9763 break; 9764 9765 if (!PAGE_ALIGNED(gpa) || !npages || 9766 gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) { 9767 ret = -KVM_EINVAL; 9768 break; 9769 } 9770 9771 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; 9772 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; 9773 vcpu->run->hypercall.args[0] = gpa; 9774 vcpu->run->hypercall.args[1] = npages; 9775 vcpu->run->hypercall.args[2] = attrs; 9776 vcpu->run->hypercall.longmode = op_64_bit; 9777 vcpu->arch.complete_userspace_io = complete_hypercall_exit; 9778 return 0; 9779 } 9780 default: 9781 ret = -KVM_ENOSYS; 9782 break; 9783 } 9784 out: 9785 if (!op_64_bit) 9786 ret = (u32)ret; 9787 kvm_rax_write(vcpu, ret); 9788 9789 ++vcpu->stat.hypercalls; 9790 return kvm_skip_emulated_instruction(vcpu); 9791 } 9792 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); 9793 9794 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) 9795 { 9796 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 9797 char instruction[3]; 9798 unsigned long rip = kvm_rip_read(vcpu); 9799 9800 /* 9801 * If the quirk is disabled, synthesize a #UD and let the guest pick up 9802 * the pieces. 9803 */ 9804 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) { 9805 ctxt->exception.error_code_valid = false; 9806 ctxt->exception.vector = UD_VECTOR; 9807 ctxt->have_exception = true; 9808 return X86EMUL_PROPAGATE_FAULT; 9809 } 9810 9811 static_call(kvm_x86_patch_hypercall)(vcpu, instruction); 9812 9813 return emulator_write_emulated(ctxt, rip, instruction, 3, 9814 &ctxt->exception); 9815 } 9816 9817 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu) 9818 { 9819 return vcpu->run->request_interrupt_window && 9820 likely(!pic_in_kernel(vcpu->kvm)); 9821 } 9822 9823 /* Called within kvm->srcu read side. */ 9824 static void post_kvm_run_save(struct kvm_vcpu *vcpu) 9825 { 9826 struct kvm_run *kvm_run = vcpu->run; 9827 9828 kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu); 9829 kvm_run->cr8 = kvm_get_cr8(vcpu); 9830 kvm_run->apic_base = kvm_get_apic_base(vcpu); 9831 9832 kvm_run->ready_for_interrupt_injection = 9833 pic_in_kernel(vcpu->kvm) || 9834 kvm_vcpu_ready_for_interrupt_injection(vcpu); 9835 9836 if (is_smm(vcpu)) 9837 kvm_run->flags |= KVM_RUN_X86_SMM; 9838 } 9839 9840 static void update_cr8_intercept(struct kvm_vcpu *vcpu) 9841 { 9842 int max_irr, tpr; 9843 9844 if (!kvm_x86_ops.update_cr8_intercept) 9845 return; 9846 9847 if (!lapic_in_kernel(vcpu)) 9848 return; 9849 9850 if (vcpu->arch.apic->apicv_active) 9851 return; 9852 9853 if (!vcpu->arch.apic->vapic_addr) 9854 max_irr = kvm_lapic_find_highest_irr(vcpu); 9855 else 9856 max_irr = -1; 9857 9858 if (max_irr != -1) 9859 max_irr >>= 4; 9860 9861 tpr = kvm_lapic_get_cr8(vcpu); 9862 9863 static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr); 9864 } 9865 9866 9867 int kvm_check_nested_events(struct kvm_vcpu *vcpu) 9868 { 9869 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 9870 kvm_x86_ops.nested_ops->triple_fault(vcpu); 9871 return 1; 9872 } 9873 9874 return kvm_x86_ops.nested_ops->check_events(vcpu); 9875 } 9876 9877 static void kvm_inject_exception(struct kvm_vcpu *vcpu) 9878 { 9879 trace_kvm_inj_exception(vcpu->arch.exception.vector, 9880 vcpu->arch.exception.has_error_code, 9881 vcpu->arch.exception.error_code, 9882 vcpu->arch.exception.injected); 9883 9884 if (vcpu->arch.exception.error_code && !is_protmode(vcpu)) 9885 vcpu->arch.exception.error_code = false; 9886 static_call(kvm_x86_inject_exception)(vcpu); 9887 } 9888 9889 /* 9890 * Check for any event (interrupt or exception) that is ready to be injected, 9891 * and if there is at least one event, inject the event with the highest 9892 * priority. This handles both "pending" events, i.e. events that have never 9893 * been injected into the guest, and "injected" events, i.e. events that were 9894 * injected as part of a previous VM-Enter, but weren't successfully delivered 9895 * and need to be re-injected. 9896 * 9897 * Note, this is not guaranteed to be invoked on a guest instruction boundary, 9898 * i.e. doesn't guarantee that there's an event window in the guest. KVM must 9899 * be able to inject exceptions in the "middle" of an instruction, and so must 9900 * also be able to re-inject NMIs and IRQs in the middle of an instruction. 9901 * I.e. for exceptions and re-injected events, NOT invoking this on instruction 9902 * boundaries is necessary and correct. 9903 * 9904 * For simplicity, KVM uses a single path to inject all events (except events 9905 * that are injected directly from L1 to L2) and doesn't explicitly track 9906 * instruction boundaries for asynchronous events. However, because VM-Exits 9907 * that can occur during instruction execution typically result in KVM skipping 9908 * the instruction or injecting an exception, e.g. instruction and exception 9909 * intercepts, and because pending exceptions have higher priority than pending 9910 * interrupts, KVM still honors instruction boundaries in most scenarios. 9911 * 9912 * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip 9913 * the instruction or inject an exception, then KVM can incorrecty inject a new 9914 * asynchrounous event if the event became pending after the CPU fetched the 9915 * instruction (in the guest). E.g. if a page fault (#PF, #NPF, EPT violation) 9916 * occurs and is resolved by KVM, a coincident NMI, SMI, IRQ, etc... can be 9917 * injected on the restarted instruction instead of being deferred until the 9918 * instruction completes. 9919 * 9920 * In practice, this virtualization hole is unlikely to be observed by the 9921 * guest, and even less likely to cause functional problems. To detect the 9922 * hole, the guest would have to trigger an event on a side effect of an early 9923 * phase of instruction execution, e.g. on the instruction fetch from memory. 9924 * And for it to be a functional problem, the guest would need to depend on the 9925 * ordering between that side effect, the instruction completing, _and_ the 9926 * delivery of the asynchronous event. 9927 */ 9928 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, 9929 bool *req_immediate_exit) 9930 { 9931 bool can_inject; 9932 int r; 9933 9934 /* 9935 * Process nested events first, as nested VM-Exit supercedes event 9936 * re-injection. If there's an event queued for re-injection, it will 9937 * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit. 9938 */ 9939 if (is_guest_mode(vcpu)) 9940 r = kvm_check_nested_events(vcpu); 9941 else 9942 r = 0; 9943 9944 /* 9945 * Re-inject exceptions and events *especially* if immediate entry+exit 9946 * to/from L2 is needed, as any event that has already been injected 9947 * into L2 needs to complete its lifecycle before injecting a new event. 9948 * 9949 * Don't re-inject an NMI or interrupt if there is a pending exception. 9950 * This collision arises if an exception occurred while vectoring the 9951 * injected event, KVM intercepted said exception, and KVM ultimately 9952 * determined the fault belongs to the guest and queues the exception 9953 * for injection back into the guest. 9954 * 9955 * "Injected" interrupts can also collide with pending exceptions if 9956 * userspace ignores the "ready for injection" flag and blindly queues 9957 * an interrupt. In that case, prioritizing the exception is correct, 9958 * as the exception "occurred" before the exit to userspace. Trap-like 9959 * exceptions, e.g. most #DBs, have higher priority than interrupts. 9960 * And while fault-like exceptions, e.g. #GP and #PF, are the lowest 9961 * priority, they're only generated (pended) during instruction 9962 * execution, and interrupts are recognized at instruction boundaries. 9963 * Thus a pending fault-like exception means the fault occurred on the 9964 * *previous* instruction and must be serviced prior to recognizing any 9965 * new events in order to fully complete the previous instruction. 9966 */ 9967 if (vcpu->arch.exception.injected) 9968 kvm_inject_exception(vcpu); 9969 else if (kvm_is_exception_pending(vcpu)) 9970 ; /* see above */ 9971 else if (vcpu->arch.nmi_injected) 9972 static_call(kvm_x86_inject_nmi)(vcpu); 9973 else if (vcpu->arch.interrupt.injected) 9974 static_call(kvm_x86_inject_irq)(vcpu, true); 9975 9976 /* 9977 * Exceptions that morph to VM-Exits are handled above, and pending 9978 * exceptions on top of injected exceptions that do not VM-Exit should 9979 * either morph to #DF or, sadly, override the injected exception. 9980 */ 9981 WARN_ON_ONCE(vcpu->arch.exception.injected && 9982 vcpu->arch.exception.pending); 9983 9984 /* 9985 * Bail if immediate entry+exit to/from the guest is needed to complete 9986 * nested VM-Enter or event re-injection so that a different pending 9987 * event can be serviced (or if KVM needs to exit to userspace). 9988 * 9989 * Otherwise, continue processing events even if VM-Exit occurred. The 9990 * VM-Exit will have cleared exceptions that were meant for L2, but 9991 * there may now be events that can be injected into L1. 9992 */ 9993 if (r < 0) 9994 goto out; 9995 9996 /* 9997 * A pending exception VM-Exit should either result in nested VM-Exit 9998 * or force an immediate re-entry and exit to/from L2, and exception 9999 * VM-Exits cannot be injected (flag should _never_ be set). 10000 */ 10001 WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected || 10002 vcpu->arch.exception_vmexit.pending); 10003 10004 /* 10005 * New events, other than exceptions, cannot be injected if KVM needs 10006 * to re-inject a previous event. See above comments on re-injecting 10007 * for why pending exceptions get priority. 10008 */ 10009 can_inject = !kvm_event_needs_reinjection(vcpu); 10010 10011 if (vcpu->arch.exception.pending) { 10012 /* 10013 * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS 10014 * value pushed on the stack. Trap-like exception and all #DBs 10015 * leave RF as-is (KVM follows Intel's behavior in this regard; 10016 * AMD states that code breakpoint #DBs excplitly clear RF=0). 10017 * 10018 * Note, most versions of Intel's SDM and AMD's APM incorrectly 10019 * describe the behavior of General Detect #DBs, which are 10020 * fault-like. They do _not_ set RF, a la code breakpoints. 10021 */ 10022 if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT) 10023 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) | 10024 X86_EFLAGS_RF); 10025 10026 if (vcpu->arch.exception.vector == DB_VECTOR) { 10027 kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception); 10028 if (vcpu->arch.dr7 & DR7_GD) { 10029 vcpu->arch.dr7 &= ~DR7_GD; 10030 kvm_update_dr7(vcpu); 10031 } 10032 } 10033 10034 kvm_inject_exception(vcpu); 10035 10036 vcpu->arch.exception.pending = false; 10037 vcpu->arch.exception.injected = true; 10038 10039 can_inject = false; 10040 } 10041 10042 /* Don't inject interrupts if the user asked to avoid doing so */ 10043 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) 10044 return 0; 10045 10046 /* 10047 * Finally, inject interrupt events. If an event cannot be injected 10048 * due to architectural conditions (e.g. IF=0) a window-open exit 10049 * will re-request KVM_REQ_EVENT. Sometimes however an event is pending 10050 * and can architecturally be injected, but we cannot do it right now: 10051 * an interrupt could have arrived just now and we have to inject it 10052 * as a vmexit, or there could already an event in the queue, which is 10053 * indicated by can_inject. In that case we request an immediate exit 10054 * in order to make progress and get back here for another iteration. 10055 * The kvm_x86_ops hooks communicate this by returning -EBUSY. 10056 */ 10057 #ifdef CONFIG_KVM_SMM 10058 if (vcpu->arch.smi_pending) { 10059 r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY; 10060 if (r < 0) 10061 goto out; 10062 if (r) { 10063 vcpu->arch.smi_pending = false; 10064 ++vcpu->arch.smi_count; 10065 enter_smm(vcpu); 10066 can_inject = false; 10067 } else 10068 static_call(kvm_x86_enable_smi_window)(vcpu); 10069 } 10070 #endif 10071 10072 if (vcpu->arch.nmi_pending) { 10073 r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY; 10074 if (r < 0) 10075 goto out; 10076 if (r) { 10077 --vcpu->arch.nmi_pending; 10078 vcpu->arch.nmi_injected = true; 10079 static_call(kvm_x86_inject_nmi)(vcpu); 10080 can_inject = false; 10081 WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0); 10082 } 10083 if (vcpu->arch.nmi_pending) 10084 static_call(kvm_x86_enable_nmi_window)(vcpu); 10085 } 10086 10087 if (kvm_cpu_has_injectable_intr(vcpu)) { 10088 r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY; 10089 if (r < 0) 10090 goto out; 10091 if (r) { 10092 kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false); 10093 static_call(kvm_x86_inject_irq)(vcpu, false); 10094 WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0); 10095 } 10096 if (kvm_cpu_has_injectable_intr(vcpu)) 10097 static_call(kvm_x86_enable_irq_window)(vcpu); 10098 } 10099 10100 if (is_guest_mode(vcpu) && 10101 kvm_x86_ops.nested_ops->has_events && 10102 kvm_x86_ops.nested_ops->has_events(vcpu)) 10103 *req_immediate_exit = true; 10104 10105 /* 10106 * KVM must never queue a new exception while injecting an event; KVM 10107 * is done emulating and should only propagate the to-be-injected event 10108 * to the VMCS/VMCB. Queueing a new exception can put the vCPU into an 10109 * infinite loop as KVM will bail from VM-Enter to inject the pending 10110 * exception and start the cycle all over. 10111 * 10112 * Exempt triple faults as they have special handling and won't put the 10113 * vCPU into an infinite loop. Triple fault can be queued when running 10114 * VMX without unrestricted guest, as that requires KVM to emulate Real 10115 * Mode events (see kvm_inject_realmode_interrupt()). 10116 */ 10117 WARN_ON_ONCE(vcpu->arch.exception.pending || 10118 vcpu->arch.exception_vmexit.pending); 10119 return 0; 10120 10121 out: 10122 if (r == -EBUSY) { 10123 *req_immediate_exit = true; 10124 r = 0; 10125 } 10126 return r; 10127 } 10128 10129 static void process_nmi(struct kvm_vcpu *vcpu) 10130 { 10131 unsigned limit = 2; 10132 10133 /* 10134 * x86 is limited to one NMI running, and one NMI pending after it. 10135 * If an NMI is already in progress, limit further NMIs to just one. 10136 * Otherwise, allow two (and we'll inject the first one immediately). 10137 */ 10138 if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected) 10139 limit = 1; 10140 10141 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0); 10142 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit); 10143 kvm_make_request(KVM_REQ_EVENT, vcpu); 10144 } 10145 10146 void kvm_make_scan_ioapic_request_mask(struct kvm *kvm, 10147 unsigned long *vcpu_bitmap) 10148 { 10149 kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap); 10150 } 10151 10152 void kvm_make_scan_ioapic_request(struct kvm *kvm) 10153 { 10154 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC); 10155 } 10156 10157 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) 10158 { 10159 struct kvm_lapic *apic = vcpu->arch.apic; 10160 bool activate; 10161 10162 if (!lapic_in_kernel(vcpu)) 10163 return; 10164 10165 down_read(&vcpu->kvm->arch.apicv_update_lock); 10166 preempt_disable(); 10167 10168 /* Do not activate APICV when APIC is disabled */ 10169 activate = kvm_vcpu_apicv_activated(vcpu) && 10170 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED); 10171 10172 if (apic->apicv_active == activate) 10173 goto out; 10174 10175 apic->apicv_active = activate; 10176 kvm_apic_update_apicv(vcpu); 10177 static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu); 10178 10179 /* 10180 * When APICv gets disabled, we may still have injected interrupts 10181 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was 10182 * still active when the interrupt got accepted. Make sure 10183 * kvm_check_and_inject_events() is called to check for that. 10184 */ 10185 if (!apic->apicv_active) 10186 kvm_make_request(KVM_REQ_EVENT, vcpu); 10187 10188 out: 10189 preempt_enable(); 10190 up_read(&vcpu->kvm->arch.apicv_update_lock); 10191 } 10192 EXPORT_SYMBOL_GPL(__kvm_vcpu_update_apicv); 10193 10194 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) 10195 { 10196 if (!lapic_in_kernel(vcpu)) 10197 return; 10198 10199 /* 10200 * Due to sharing page tables across vCPUs, the xAPIC memslot must be 10201 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but 10202 * and hardware doesn't support x2APIC virtualization. E.g. some AMD 10203 * CPUs support AVIC but not x2APIC. KVM still allows enabling AVIC in 10204 * this case so that KVM can the AVIC doorbell to inject interrupts to 10205 * running vCPUs, but KVM must not create SPTEs for the APIC base as 10206 * the vCPU would incorrectly be able to access the vAPIC page via MMIO 10207 * despite being in x2APIC mode. For simplicity, inhibiting the APIC 10208 * access page is sticky. 10209 */ 10210 if (apic_x2apic_mode(vcpu->arch.apic) && 10211 kvm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization) 10212 kvm_inhibit_apic_access_page(vcpu); 10213 10214 __kvm_vcpu_update_apicv(vcpu); 10215 } 10216 10217 void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, 10218 enum kvm_apicv_inhibit reason, bool set) 10219 { 10220 unsigned long old, new; 10221 10222 lockdep_assert_held_write(&kvm->arch.apicv_update_lock); 10223 10224 if (!(kvm_x86_ops.required_apicv_inhibits & BIT(reason))) 10225 return; 10226 10227 old = new = kvm->arch.apicv_inhibit_reasons; 10228 10229 set_or_clear_apicv_inhibit(&new, reason, set); 10230 10231 if (!!old != !!new) { 10232 /* 10233 * Kick all vCPUs before setting apicv_inhibit_reasons to avoid 10234 * false positives in the sanity check WARN in svm_vcpu_run(). 10235 * This task will wait for all vCPUs to ack the kick IRQ before 10236 * updating apicv_inhibit_reasons, and all other vCPUs will 10237 * block on acquiring apicv_update_lock so that vCPUs can't 10238 * redo svm_vcpu_run() without seeing the new inhibit state. 10239 * 10240 * Note, holding apicv_update_lock and taking it in the read 10241 * side (handling the request) also prevents other vCPUs from 10242 * servicing the request with a stale apicv_inhibit_reasons. 10243 */ 10244 kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE); 10245 kvm->arch.apicv_inhibit_reasons = new; 10246 if (new) { 10247 unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); 10248 int idx = srcu_read_lock(&kvm->srcu); 10249 10250 kvm_zap_gfn_range(kvm, gfn, gfn+1); 10251 srcu_read_unlock(&kvm->srcu, idx); 10252 } 10253 } else { 10254 kvm->arch.apicv_inhibit_reasons = new; 10255 } 10256 } 10257 10258 void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, 10259 enum kvm_apicv_inhibit reason, bool set) 10260 { 10261 if (!enable_apicv) 10262 return; 10263 10264 down_write(&kvm->arch.apicv_update_lock); 10265 __kvm_set_or_clear_apicv_inhibit(kvm, reason, set); 10266 up_write(&kvm->arch.apicv_update_lock); 10267 } 10268 EXPORT_SYMBOL_GPL(kvm_set_or_clear_apicv_inhibit); 10269 10270 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu) 10271 { 10272 if (!kvm_apic_present(vcpu)) 10273 return; 10274 10275 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256); 10276 10277 if (irqchip_split(vcpu->kvm)) 10278 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors); 10279 else { 10280 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu); 10281 if (ioapic_in_kernel(vcpu->kvm)) 10282 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors); 10283 } 10284 10285 if (is_guest_mode(vcpu)) 10286 vcpu->arch.load_eoi_exitmap_pending = true; 10287 else 10288 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu); 10289 } 10290 10291 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) 10292 { 10293 u64 eoi_exit_bitmap[4]; 10294 10295 if (!kvm_apic_hw_enabled(vcpu->arch.apic)) 10296 return; 10297 10298 if (to_hv_vcpu(vcpu)) { 10299 bitmap_or((ulong *)eoi_exit_bitmap, 10300 vcpu->arch.ioapic_handled_vectors, 10301 to_hv_synic(vcpu)->vec_bitmap, 256); 10302 static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap); 10303 return; 10304 } 10305 10306 static_call_cond(kvm_x86_load_eoi_exitmap)( 10307 vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors); 10308 } 10309 10310 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, 10311 unsigned long start, unsigned long end) 10312 { 10313 unsigned long apic_address; 10314 10315 /* 10316 * The physical address of apic access page is stored in the VMCS. 10317 * Update it when it becomes invalid. 10318 */ 10319 apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); 10320 if (start <= apic_address && apic_address < end) 10321 kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD); 10322 } 10323 10324 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm) 10325 { 10326 static_call_cond(kvm_x86_guest_memory_reclaimed)(kvm); 10327 } 10328 10329 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) 10330 { 10331 if (!lapic_in_kernel(vcpu)) 10332 return; 10333 10334 static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu); 10335 } 10336 10337 void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu) 10338 { 10339 smp_send_reschedule(vcpu->cpu); 10340 } 10341 EXPORT_SYMBOL_GPL(__kvm_request_immediate_exit); 10342 10343 /* 10344 * Called within kvm->srcu read side. 10345 * Returns 1 to let vcpu_run() continue the guest execution loop without 10346 * exiting to the userspace. Otherwise, the value will be returned to the 10347 * userspace. 10348 */ 10349 static int vcpu_enter_guest(struct kvm_vcpu *vcpu) 10350 { 10351 int r; 10352 bool req_int_win = 10353 dm_request_for_irq_injection(vcpu) && 10354 kvm_cpu_accept_dm_intr(vcpu); 10355 fastpath_t exit_fastpath; 10356 10357 bool req_immediate_exit = false; 10358 10359 if (kvm_request_pending(vcpu)) { 10360 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { 10361 r = -EIO; 10362 goto out; 10363 } 10364 10365 if (kvm_dirty_ring_check_request(vcpu)) { 10366 r = 0; 10367 goto out; 10368 } 10369 10370 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 10371 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) { 10372 r = 0; 10373 goto out; 10374 } 10375 } 10376 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)) 10377 kvm_mmu_free_obsolete_roots(vcpu); 10378 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu)) 10379 __kvm_migrate_timers(vcpu); 10380 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) 10381 kvm_update_masterclock(vcpu->kvm); 10382 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu)) 10383 kvm_gen_kvmclock_update(vcpu); 10384 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) { 10385 r = kvm_guest_time_update(vcpu); 10386 if (unlikely(r)) 10387 goto out; 10388 } 10389 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu)) 10390 kvm_mmu_sync_roots(vcpu); 10391 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) 10392 kvm_mmu_load_pgd(vcpu); 10393 10394 /* 10395 * Note, the order matters here, as flushing "all" TLB entries 10396 * also flushes the "current" TLB entries, i.e. servicing the 10397 * flush "all" will clear any request to flush "current". 10398 */ 10399 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 10400 kvm_vcpu_flush_tlb_all(vcpu); 10401 10402 kvm_service_local_tlb_flush_requests(vcpu); 10403 10404 /* 10405 * Fall back to a "full" guest flush if Hyper-V's precise 10406 * flushing fails. Note, Hyper-V's flushing is per-vCPU, but 10407 * the flushes are considered "remote" and not "local" because 10408 * the requests can be initiated from other vCPUs. 10409 */ 10410 if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) && 10411 kvm_hv_vcpu_flush_tlb(vcpu)) 10412 kvm_vcpu_flush_tlb_guest(vcpu); 10413 10414 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { 10415 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; 10416 r = 0; 10417 goto out; 10418 } 10419 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 10420 if (is_guest_mode(vcpu)) 10421 kvm_x86_ops.nested_ops->triple_fault(vcpu); 10422 10423 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { 10424 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; 10425 vcpu->mmio_needed = 0; 10426 r = 0; 10427 goto out; 10428 } 10429 } 10430 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) { 10431 /* Page is swapped out. Do synthetic halt */ 10432 vcpu->arch.apf.halted = true; 10433 r = 1; 10434 goto out; 10435 } 10436 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 10437 record_steal_time(vcpu); 10438 #ifdef CONFIG_KVM_SMM 10439 if (kvm_check_request(KVM_REQ_SMI, vcpu)) 10440 process_smi(vcpu); 10441 #endif 10442 if (kvm_check_request(KVM_REQ_NMI, vcpu)) 10443 process_nmi(vcpu); 10444 if (kvm_check_request(KVM_REQ_PMU, vcpu)) 10445 kvm_pmu_handle_event(vcpu); 10446 if (kvm_check_request(KVM_REQ_PMI, vcpu)) 10447 kvm_pmu_deliver_pmi(vcpu); 10448 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { 10449 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); 10450 if (test_bit(vcpu->arch.pending_ioapic_eoi, 10451 vcpu->arch.ioapic_handled_vectors)) { 10452 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; 10453 vcpu->run->eoi.vector = 10454 vcpu->arch.pending_ioapic_eoi; 10455 r = 0; 10456 goto out; 10457 } 10458 } 10459 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu)) 10460 vcpu_scan_ioapic(vcpu); 10461 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu)) 10462 vcpu_load_eoi_exitmap(vcpu); 10463 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu)) 10464 kvm_vcpu_reload_apic_access_page(vcpu); 10465 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) { 10466 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 10467 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; 10468 vcpu->run->system_event.ndata = 0; 10469 r = 0; 10470 goto out; 10471 } 10472 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) { 10473 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 10474 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET; 10475 vcpu->run->system_event.ndata = 0; 10476 r = 0; 10477 goto out; 10478 } 10479 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) { 10480 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 10481 10482 vcpu->run->exit_reason = KVM_EXIT_HYPERV; 10483 vcpu->run->hyperv = hv_vcpu->exit; 10484 r = 0; 10485 goto out; 10486 } 10487 10488 /* 10489 * KVM_REQ_HV_STIMER has to be processed after 10490 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers 10491 * depend on the guest clock being up-to-date 10492 */ 10493 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu)) 10494 kvm_hv_process_stimers(vcpu); 10495 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) 10496 kvm_vcpu_update_apicv(vcpu); 10497 if (kvm_check_request(KVM_REQ_APF_READY, vcpu)) 10498 kvm_check_async_pf_completion(vcpu); 10499 if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu)) 10500 static_call(kvm_x86_msr_filter_changed)(vcpu); 10501 10502 if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) 10503 static_call(kvm_x86_update_cpu_dirty_logging)(vcpu); 10504 } 10505 10506 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || 10507 kvm_xen_has_interrupt(vcpu)) { 10508 ++vcpu->stat.req_event; 10509 r = kvm_apic_accept_events(vcpu); 10510 if (r < 0) { 10511 r = 0; 10512 goto out; 10513 } 10514 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 10515 r = 1; 10516 goto out; 10517 } 10518 10519 r = kvm_check_and_inject_events(vcpu, &req_immediate_exit); 10520 if (r < 0) { 10521 r = 0; 10522 goto out; 10523 } 10524 if (req_int_win) 10525 static_call(kvm_x86_enable_irq_window)(vcpu); 10526 10527 if (kvm_lapic_enabled(vcpu)) { 10528 update_cr8_intercept(vcpu); 10529 kvm_lapic_sync_to_vapic(vcpu); 10530 } 10531 } 10532 10533 r = kvm_mmu_reload(vcpu); 10534 if (unlikely(r)) { 10535 goto cancel_injection; 10536 } 10537 10538 preempt_disable(); 10539 10540 static_call(kvm_x86_prepare_switch_to_guest)(vcpu); 10541 10542 /* 10543 * Disable IRQs before setting IN_GUEST_MODE. Posted interrupt 10544 * IPI are then delayed after guest entry, which ensures that they 10545 * result in virtual interrupt delivery. 10546 */ 10547 local_irq_disable(); 10548 10549 /* Store vcpu->apicv_active before vcpu->mode. */ 10550 smp_store_release(&vcpu->mode, IN_GUEST_MODE); 10551 10552 kvm_vcpu_srcu_read_unlock(vcpu); 10553 10554 /* 10555 * 1) We should set ->mode before checking ->requests. Please see 10556 * the comment in kvm_vcpu_exiting_guest_mode(). 10557 * 10558 * 2) For APICv, we should set ->mode before checking PID.ON. This 10559 * pairs with the memory barrier implicit in pi_test_and_set_on 10560 * (see vmx_deliver_posted_interrupt). 10561 * 10562 * 3) This also orders the write to mode from any reads to the page 10563 * tables done while the VCPU is running. Please see the comment 10564 * in kvm_flush_remote_tlbs. 10565 */ 10566 smp_mb__after_srcu_read_unlock(); 10567 10568 /* 10569 * Process pending posted interrupts to handle the case where the 10570 * notification IRQ arrived in the host, or was never sent (because the 10571 * target vCPU wasn't running). Do this regardless of the vCPU's APICv 10572 * status, KVM doesn't update assigned devices when APICv is inhibited, 10573 * i.e. they can post interrupts even if APICv is temporarily disabled. 10574 */ 10575 if (kvm_lapic_enabled(vcpu)) 10576 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu); 10577 10578 if (kvm_vcpu_exit_request(vcpu)) { 10579 vcpu->mode = OUTSIDE_GUEST_MODE; 10580 smp_wmb(); 10581 local_irq_enable(); 10582 preempt_enable(); 10583 kvm_vcpu_srcu_read_lock(vcpu); 10584 r = 1; 10585 goto cancel_injection; 10586 } 10587 10588 if (req_immediate_exit) { 10589 kvm_make_request(KVM_REQ_EVENT, vcpu); 10590 static_call(kvm_x86_request_immediate_exit)(vcpu); 10591 } 10592 10593 fpregs_assert_state_consistent(); 10594 if (test_thread_flag(TIF_NEED_FPU_LOAD)) 10595 switch_fpu_return(); 10596 10597 if (vcpu->arch.guest_fpu.xfd_err) 10598 wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); 10599 10600 if (unlikely(vcpu->arch.switch_db_regs)) { 10601 set_debugreg(0, 7); 10602 set_debugreg(vcpu->arch.eff_db[0], 0); 10603 set_debugreg(vcpu->arch.eff_db[1], 1); 10604 set_debugreg(vcpu->arch.eff_db[2], 2); 10605 set_debugreg(vcpu->arch.eff_db[3], 3); 10606 } else if (unlikely(hw_breakpoint_active())) { 10607 set_debugreg(0, 7); 10608 } 10609 10610 guest_timing_enter_irqoff(); 10611 10612 for (;;) { 10613 /* 10614 * Assert that vCPU vs. VM APICv state is consistent. An APICv 10615 * update must kick and wait for all vCPUs before toggling the 10616 * per-VM state, and responsing vCPUs must wait for the update 10617 * to complete before servicing KVM_REQ_APICV_UPDATE. 10618 */ 10619 WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) && 10620 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED)); 10621 10622 exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu); 10623 if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST)) 10624 break; 10625 10626 if (kvm_lapic_enabled(vcpu)) 10627 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu); 10628 10629 if (unlikely(kvm_vcpu_exit_request(vcpu))) { 10630 exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED; 10631 break; 10632 } 10633 } 10634 10635 /* 10636 * Do this here before restoring debug registers on the host. And 10637 * since we do this before handling the vmexit, a DR access vmexit 10638 * can (a) read the correct value of the debug registers, (b) set 10639 * KVM_DEBUGREG_WONT_EXIT again. 10640 */ 10641 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) { 10642 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP); 10643 static_call(kvm_x86_sync_dirty_debug_regs)(vcpu); 10644 kvm_update_dr0123(vcpu); 10645 kvm_update_dr7(vcpu); 10646 } 10647 10648 /* 10649 * If the guest has used debug registers, at least dr7 10650 * will be disabled while returning to the host. 10651 * If we don't have active breakpoints in the host, we don't 10652 * care about the messed up debug address registers. But if 10653 * we have some of them active, restore the old state. 10654 */ 10655 if (hw_breakpoint_active()) 10656 hw_breakpoint_restore(); 10657 10658 vcpu->arch.last_vmentry_cpu = vcpu->cpu; 10659 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); 10660 10661 vcpu->mode = OUTSIDE_GUEST_MODE; 10662 smp_wmb(); 10663 10664 /* 10665 * Sync xfd before calling handle_exit_irqoff() which may 10666 * rely on the fact that guest_fpu::xfd is up-to-date (e.g. 10667 * in #NM irqoff handler). 10668 */ 10669 if (vcpu->arch.xfd_no_write_intercept) 10670 fpu_sync_guest_vmexit_xfd_state(); 10671 10672 static_call(kvm_x86_handle_exit_irqoff)(vcpu); 10673 10674 if (vcpu->arch.guest_fpu.xfd_err) 10675 wrmsrl(MSR_IA32_XFD_ERR, 0); 10676 10677 /* 10678 * Consume any pending interrupts, including the possible source of 10679 * VM-Exit on SVM and any ticks that occur between VM-Exit and now. 10680 * An instruction is required after local_irq_enable() to fully unblock 10681 * interrupts on processors that implement an interrupt shadow, the 10682 * stat.exits increment will do nicely. 10683 */ 10684 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); 10685 local_irq_enable(); 10686 ++vcpu->stat.exits; 10687 local_irq_disable(); 10688 kvm_after_interrupt(vcpu); 10689 10690 /* 10691 * Wait until after servicing IRQs to account guest time so that any 10692 * ticks that occurred while running the guest are properly accounted 10693 * to the guest. Waiting until IRQs are enabled degrades the accuracy 10694 * of accounting via context tracking, but the loss of accuracy is 10695 * acceptable for all known use cases. 10696 */ 10697 guest_timing_exit_irqoff(); 10698 10699 local_irq_enable(); 10700 preempt_enable(); 10701 10702 kvm_vcpu_srcu_read_lock(vcpu); 10703 10704 /* 10705 * Profile KVM exit RIPs: 10706 */ 10707 if (unlikely(prof_on == KVM_PROFILING)) { 10708 unsigned long rip = kvm_rip_read(vcpu); 10709 profile_hit(KVM_PROFILING, (void *)rip); 10710 } 10711 10712 if (unlikely(vcpu->arch.tsc_always_catchup)) 10713 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 10714 10715 if (vcpu->arch.apic_attention) 10716 kvm_lapic_sync_from_vapic(vcpu); 10717 10718 r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath); 10719 return r; 10720 10721 cancel_injection: 10722 if (req_immediate_exit) 10723 kvm_make_request(KVM_REQ_EVENT, vcpu); 10724 static_call(kvm_x86_cancel_injection)(vcpu); 10725 if (unlikely(vcpu->arch.apic_attention)) 10726 kvm_lapic_sync_from_vapic(vcpu); 10727 out: 10728 return r; 10729 } 10730 10731 /* Called within kvm->srcu read side. */ 10732 static inline int vcpu_block(struct kvm_vcpu *vcpu) 10733 { 10734 bool hv_timer; 10735 10736 if (!kvm_arch_vcpu_runnable(vcpu)) { 10737 /* 10738 * Switch to the software timer before halt-polling/blocking as 10739 * the guest's timer may be a break event for the vCPU, and the 10740 * hypervisor timer runs only when the CPU is in guest mode. 10741 * Switch before halt-polling so that KVM recognizes an expired 10742 * timer before blocking. 10743 */ 10744 hv_timer = kvm_lapic_hv_timer_in_use(vcpu); 10745 if (hv_timer) 10746 kvm_lapic_switch_to_sw_timer(vcpu); 10747 10748 kvm_vcpu_srcu_read_unlock(vcpu); 10749 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 10750 kvm_vcpu_halt(vcpu); 10751 else 10752 kvm_vcpu_block(vcpu); 10753 kvm_vcpu_srcu_read_lock(vcpu); 10754 10755 if (hv_timer) 10756 kvm_lapic_switch_to_hv_timer(vcpu); 10757 10758 /* 10759 * If the vCPU is not runnable, a signal or another host event 10760 * of some kind is pending; service it without changing the 10761 * vCPU's activity state. 10762 */ 10763 if (!kvm_arch_vcpu_runnable(vcpu)) 10764 return 1; 10765 } 10766 10767 /* 10768 * Evaluate nested events before exiting the halted state. This allows 10769 * the halt state to be recorded properly in the VMCS12's activity 10770 * state field (AMD does not have a similar field and a VM-Exit always 10771 * causes a spurious wakeup from HLT). 10772 */ 10773 if (is_guest_mode(vcpu)) { 10774 if (kvm_check_nested_events(vcpu) < 0) 10775 return 0; 10776 } 10777 10778 if (kvm_apic_accept_events(vcpu) < 0) 10779 return 0; 10780 switch(vcpu->arch.mp_state) { 10781 case KVM_MP_STATE_HALTED: 10782 case KVM_MP_STATE_AP_RESET_HOLD: 10783 vcpu->arch.pv.pv_unhalted = false; 10784 vcpu->arch.mp_state = 10785 KVM_MP_STATE_RUNNABLE; 10786 fallthrough; 10787 case KVM_MP_STATE_RUNNABLE: 10788 vcpu->arch.apf.halted = false; 10789 break; 10790 case KVM_MP_STATE_INIT_RECEIVED: 10791 break; 10792 default: 10793 WARN_ON_ONCE(1); 10794 break; 10795 } 10796 return 1; 10797 } 10798 10799 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) 10800 { 10801 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 10802 !vcpu->arch.apf.halted); 10803 } 10804 10805 /* Called within kvm->srcu read side. */ 10806 static int vcpu_run(struct kvm_vcpu *vcpu) 10807 { 10808 int r; 10809 10810 vcpu->arch.l1tf_flush_l1d = true; 10811 10812 for (;;) { 10813 /* 10814 * If another guest vCPU requests a PV TLB flush in the middle 10815 * of instruction emulation, the rest of the emulation could 10816 * use a stale page translation. Assume that any code after 10817 * this point can start executing an instruction. 10818 */ 10819 vcpu->arch.at_instruction_boundary = false; 10820 if (kvm_vcpu_running(vcpu)) { 10821 r = vcpu_enter_guest(vcpu); 10822 } else { 10823 r = vcpu_block(vcpu); 10824 } 10825 10826 if (r <= 0) 10827 break; 10828 10829 kvm_clear_request(KVM_REQ_UNBLOCK, vcpu); 10830 if (kvm_xen_has_pending_events(vcpu)) 10831 kvm_xen_inject_pending_events(vcpu); 10832 10833 if (kvm_cpu_has_pending_timer(vcpu)) 10834 kvm_inject_pending_timer_irqs(vcpu); 10835 10836 if (dm_request_for_irq_injection(vcpu) && 10837 kvm_vcpu_ready_for_interrupt_injection(vcpu)) { 10838 r = 0; 10839 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 10840 ++vcpu->stat.request_irq_exits; 10841 break; 10842 } 10843 10844 if (__xfer_to_guest_mode_work_pending()) { 10845 kvm_vcpu_srcu_read_unlock(vcpu); 10846 r = xfer_to_guest_mode_handle_work(vcpu); 10847 kvm_vcpu_srcu_read_lock(vcpu); 10848 if (r) 10849 return r; 10850 } 10851 } 10852 10853 return r; 10854 } 10855 10856 static inline int complete_emulated_io(struct kvm_vcpu *vcpu) 10857 { 10858 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE); 10859 } 10860 10861 static int complete_emulated_pio(struct kvm_vcpu *vcpu) 10862 { 10863 BUG_ON(!vcpu->arch.pio.count); 10864 10865 return complete_emulated_io(vcpu); 10866 } 10867 10868 /* 10869 * Implements the following, as a state machine: 10870 * 10871 * read: 10872 * for each fragment 10873 * for each mmio piece in the fragment 10874 * write gpa, len 10875 * exit 10876 * copy data 10877 * execute insn 10878 * 10879 * write: 10880 * for each fragment 10881 * for each mmio piece in the fragment 10882 * write gpa, len 10883 * copy data 10884 * exit 10885 */ 10886 static int complete_emulated_mmio(struct kvm_vcpu *vcpu) 10887 { 10888 struct kvm_run *run = vcpu->run; 10889 struct kvm_mmio_fragment *frag; 10890 unsigned len; 10891 10892 BUG_ON(!vcpu->mmio_needed); 10893 10894 /* Complete previous fragment */ 10895 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; 10896 len = min(8u, frag->len); 10897 if (!vcpu->mmio_is_write) 10898 memcpy(frag->data, run->mmio.data, len); 10899 10900 if (frag->len <= 8) { 10901 /* Switch to the next fragment. */ 10902 frag++; 10903 vcpu->mmio_cur_fragment++; 10904 } else { 10905 /* Go forward to the next mmio piece. */ 10906 frag->data += len; 10907 frag->gpa += len; 10908 frag->len -= len; 10909 } 10910 10911 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { 10912 vcpu->mmio_needed = 0; 10913 10914 /* FIXME: return into emulator if single-stepping. */ 10915 if (vcpu->mmio_is_write) 10916 return 1; 10917 vcpu->mmio_read_completed = 1; 10918 return complete_emulated_io(vcpu); 10919 } 10920 10921 run->exit_reason = KVM_EXIT_MMIO; 10922 run->mmio.phys_addr = frag->gpa; 10923 if (vcpu->mmio_is_write) 10924 memcpy(run->mmio.data, frag->data, min(8u, frag->len)); 10925 run->mmio.len = min(8u, frag->len); 10926 run->mmio.is_write = vcpu->mmio_is_write; 10927 vcpu->arch.complete_userspace_io = complete_emulated_mmio; 10928 return 0; 10929 } 10930 10931 /* Swap (qemu) user FPU context for the guest FPU context. */ 10932 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) 10933 { 10934 /* Exclude PKRU, it's restored separately immediately after VM-Exit. */ 10935 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true); 10936 trace_kvm_fpu(1); 10937 } 10938 10939 /* When vcpu_run ends, restore user space FPU context. */ 10940 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) 10941 { 10942 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false); 10943 ++vcpu->stat.fpu_reload; 10944 trace_kvm_fpu(0); 10945 } 10946 10947 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 10948 { 10949 struct kvm_queued_exception *ex = &vcpu->arch.exception; 10950 struct kvm_run *kvm_run = vcpu->run; 10951 int r; 10952 10953 vcpu_load(vcpu); 10954 kvm_sigset_activate(vcpu); 10955 kvm_run->flags = 0; 10956 kvm_load_guest_fpu(vcpu); 10957 10958 kvm_vcpu_srcu_read_lock(vcpu); 10959 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) { 10960 if (kvm_run->immediate_exit) { 10961 r = -EINTR; 10962 goto out; 10963 } 10964 /* 10965 * It should be impossible for the hypervisor timer to be in 10966 * use before KVM has ever run the vCPU. 10967 */ 10968 WARN_ON_ONCE(kvm_lapic_hv_timer_in_use(vcpu)); 10969 10970 kvm_vcpu_srcu_read_unlock(vcpu); 10971 kvm_vcpu_block(vcpu); 10972 kvm_vcpu_srcu_read_lock(vcpu); 10973 10974 if (kvm_apic_accept_events(vcpu) < 0) { 10975 r = 0; 10976 goto out; 10977 } 10978 r = -EAGAIN; 10979 if (signal_pending(current)) { 10980 r = -EINTR; 10981 kvm_run->exit_reason = KVM_EXIT_INTR; 10982 ++vcpu->stat.signal_exits; 10983 } 10984 goto out; 10985 } 10986 10987 if ((kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) || 10988 (kvm_run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)) { 10989 r = -EINVAL; 10990 goto out; 10991 } 10992 10993 if (kvm_run->kvm_dirty_regs) { 10994 r = sync_regs(vcpu); 10995 if (r != 0) 10996 goto out; 10997 } 10998 10999 /* re-sync apic's tpr */ 11000 if (!lapic_in_kernel(vcpu)) { 11001 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) { 11002 r = -EINVAL; 11003 goto out; 11004 } 11005 } 11006 11007 /* 11008 * If userspace set a pending exception and L2 is active, convert it to 11009 * a pending VM-Exit if L1 wants to intercept the exception. 11010 */ 11011 if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) && 11012 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector, 11013 ex->error_code)) { 11014 kvm_queue_exception_vmexit(vcpu, ex->vector, 11015 ex->has_error_code, ex->error_code, 11016 ex->has_payload, ex->payload); 11017 ex->injected = false; 11018 ex->pending = false; 11019 } 11020 vcpu->arch.exception_from_userspace = false; 11021 11022 if (unlikely(vcpu->arch.complete_userspace_io)) { 11023 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io; 11024 vcpu->arch.complete_userspace_io = NULL; 11025 r = cui(vcpu); 11026 if (r <= 0) 11027 goto out; 11028 } else { 11029 WARN_ON_ONCE(vcpu->arch.pio.count); 11030 WARN_ON_ONCE(vcpu->mmio_needed); 11031 } 11032 11033 if (kvm_run->immediate_exit) { 11034 r = -EINTR; 11035 goto out; 11036 } 11037 11038 r = static_call(kvm_x86_vcpu_pre_run)(vcpu); 11039 if (r <= 0) 11040 goto out; 11041 11042 r = vcpu_run(vcpu); 11043 11044 out: 11045 kvm_put_guest_fpu(vcpu); 11046 if (kvm_run->kvm_valid_regs) 11047 store_regs(vcpu); 11048 post_kvm_run_save(vcpu); 11049 kvm_vcpu_srcu_read_unlock(vcpu); 11050 11051 kvm_sigset_deactivate(vcpu); 11052 vcpu_put(vcpu); 11053 return r; 11054 } 11055 11056 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 11057 { 11058 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { 11059 /* 11060 * We are here if userspace calls get_regs() in the middle of 11061 * instruction emulation. Registers state needs to be copied 11062 * back from emulation context to vcpu. Userspace shouldn't do 11063 * that usually, but some bad designed PV devices (vmware 11064 * backdoor interface) need this to work 11065 */ 11066 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); 11067 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 11068 } 11069 regs->rax = kvm_rax_read(vcpu); 11070 regs->rbx = kvm_rbx_read(vcpu); 11071 regs->rcx = kvm_rcx_read(vcpu); 11072 regs->rdx = kvm_rdx_read(vcpu); 11073 regs->rsi = kvm_rsi_read(vcpu); 11074 regs->rdi = kvm_rdi_read(vcpu); 11075 regs->rsp = kvm_rsp_read(vcpu); 11076 regs->rbp = kvm_rbp_read(vcpu); 11077 #ifdef CONFIG_X86_64 11078 regs->r8 = kvm_r8_read(vcpu); 11079 regs->r9 = kvm_r9_read(vcpu); 11080 regs->r10 = kvm_r10_read(vcpu); 11081 regs->r11 = kvm_r11_read(vcpu); 11082 regs->r12 = kvm_r12_read(vcpu); 11083 regs->r13 = kvm_r13_read(vcpu); 11084 regs->r14 = kvm_r14_read(vcpu); 11085 regs->r15 = kvm_r15_read(vcpu); 11086 #endif 11087 11088 regs->rip = kvm_rip_read(vcpu); 11089 regs->rflags = kvm_get_rflags(vcpu); 11090 } 11091 11092 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 11093 { 11094 vcpu_load(vcpu); 11095 __get_regs(vcpu, regs); 11096 vcpu_put(vcpu); 11097 return 0; 11098 } 11099 11100 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 11101 { 11102 vcpu->arch.emulate_regs_need_sync_from_vcpu = true; 11103 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 11104 11105 kvm_rax_write(vcpu, regs->rax); 11106 kvm_rbx_write(vcpu, regs->rbx); 11107 kvm_rcx_write(vcpu, regs->rcx); 11108 kvm_rdx_write(vcpu, regs->rdx); 11109 kvm_rsi_write(vcpu, regs->rsi); 11110 kvm_rdi_write(vcpu, regs->rdi); 11111 kvm_rsp_write(vcpu, regs->rsp); 11112 kvm_rbp_write(vcpu, regs->rbp); 11113 #ifdef CONFIG_X86_64 11114 kvm_r8_write(vcpu, regs->r8); 11115 kvm_r9_write(vcpu, regs->r9); 11116 kvm_r10_write(vcpu, regs->r10); 11117 kvm_r11_write(vcpu, regs->r11); 11118 kvm_r12_write(vcpu, regs->r12); 11119 kvm_r13_write(vcpu, regs->r13); 11120 kvm_r14_write(vcpu, regs->r14); 11121 kvm_r15_write(vcpu, regs->r15); 11122 #endif 11123 11124 kvm_rip_write(vcpu, regs->rip); 11125 kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); 11126 11127 vcpu->arch.exception.pending = false; 11128 vcpu->arch.exception_vmexit.pending = false; 11129 11130 kvm_make_request(KVM_REQ_EVENT, vcpu); 11131 } 11132 11133 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 11134 { 11135 vcpu_load(vcpu); 11136 __set_regs(vcpu, regs); 11137 vcpu_put(vcpu); 11138 return 0; 11139 } 11140 11141 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 11142 { 11143 struct desc_ptr dt; 11144 11145 if (vcpu->arch.guest_state_protected) 11146 goto skip_protected_regs; 11147 11148 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 11149 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 11150 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); 11151 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 11152 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 11153 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 11154 11155 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 11156 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 11157 11158 static_call(kvm_x86_get_idt)(vcpu, &dt); 11159 sregs->idt.limit = dt.size; 11160 sregs->idt.base = dt.address; 11161 static_call(kvm_x86_get_gdt)(vcpu, &dt); 11162 sregs->gdt.limit = dt.size; 11163 sregs->gdt.base = dt.address; 11164 11165 sregs->cr2 = vcpu->arch.cr2; 11166 sregs->cr3 = kvm_read_cr3(vcpu); 11167 11168 skip_protected_regs: 11169 sregs->cr0 = kvm_read_cr0(vcpu); 11170 sregs->cr4 = kvm_read_cr4(vcpu); 11171 sregs->cr8 = kvm_get_cr8(vcpu); 11172 sregs->efer = vcpu->arch.efer; 11173 sregs->apic_base = kvm_get_apic_base(vcpu); 11174 } 11175 11176 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 11177 { 11178 __get_sregs_common(vcpu, sregs); 11179 11180 if (vcpu->arch.guest_state_protected) 11181 return; 11182 11183 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) 11184 set_bit(vcpu->arch.interrupt.nr, 11185 (unsigned long *)sregs->interrupt_bitmap); 11186 } 11187 11188 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) 11189 { 11190 int i; 11191 11192 __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); 11193 11194 if (vcpu->arch.guest_state_protected) 11195 return; 11196 11197 if (is_pae_paging(vcpu)) { 11198 for (i = 0 ; i < 4 ; i++) 11199 sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); 11200 sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; 11201 } 11202 } 11203 11204 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 11205 struct kvm_sregs *sregs) 11206 { 11207 vcpu_load(vcpu); 11208 __get_sregs(vcpu, sregs); 11209 vcpu_put(vcpu); 11210 return 0; 11211 } 11212 11213 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 11214 struct kvm_mp_state *mp_state) 11215 { 11216 int r; 11217 11218 vcpu_load(vcpu); 11219 if (kvm_mpx_supported()) 11220 kvm_load_guest_fpu(vcpu); 11221 11222 r = kvm_apic_accept_events(vcpu); 11223 if (r < 0) 11224 goto out; 11225 r = 0; 11226 11227 if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED || 11228 vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) && 11229 vcpu->arch.pv.pv_unhalted) 11230 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 11231 else 11232 mp_state->mp_state = vcpu->arch.mp_state; 11233 11234 out: 11235 if (kvm_mpx_supported()) 11236 kvm_put_guest_fpu(vcpu); 11237 vcpu_put(vcpu); 11238 return r; 11239 } 11240 11241 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 11242 struct kvm_mp_state *mp_state) 11243 { 11244 int ret = -EINVAL; 11245 11246 vcpu_load(vcpu); 11247 11248 switch (mp_state->mp_state) { 11249 case KVM_MP_STATE_UNINITIALIZED: 11250 case KVM_MP_STATE_HALTED: 11251 case KVM_MP_STATE_AP_RESET_HOLD: 11252 case KVM_MP_STATE_INIT_RECEIVED: 11253 case KVM_MP_STATE_SIPI_RECEIVED: 11254 if (!lapic_in_kernel(vcpu)) 11255 goto out; 11256 break; 11257 11258 case KVM_MP_STATE_RUNNABLE: 11259 break; 11260 11261 default: 11262 goto out; 11263 } 11264 11265 /* 11266 * Pending INITs are reported using KVM_SET_VCPU_EVENTS, disallow 11267 * forcing the guest into INIT/SIPI if those events are supposed to be 11268 * blocked. KVM prioritizes SMI over INIT, so reject INIT/SIPI state 11269 * if an SMI is pending as well. 11270 */ 11271 if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) && 11272 (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED || 11273 mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED)) 11274 goto out; 11275 11276 if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) { 11277 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; 11278 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events); 11279 } else 11280 vcpu->arch.mp_state = mp_state->mp_state; 11281 kvm_make_request(KVM_REQ_EVENT, vcpu); 11282 11283 ret = 0; 11284 out: 11285 vcpu_put(vcpu); 11286 return ret; 11287 } 11288 11289 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, 11290 int reason, bool has_error_code, u32 error_code) 11291 { 11292 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; 11293 int ret; 11294 11295 init_emulate_ctxt(vcpu); 11296 11297 ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason, 11298 has_error_code, error_code); 11299 if (ret) { 11300 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 11301 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; 11302 vcpu->run->internal.ndata = 0; 11303 return 0; 11304 } 11305 11306 kvm_rip_write(vcpu, ctxt->eip); 11307 kvm_set_rflags(vcpu, ctxt->eflags); 11308 return 1; 11309 } 11310 EXPORT_SYMBOL_GPL(kvm_task_switch); 11311 11312 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 11313 { 11314 if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { 11315 /* 11316 * When EFER.LME and CR0.PG are set, the processor is in 11317 * 64-bit mode (though maybe in a 32-bit code segment). 11318 * CR4.PAE and EFER.LMA must be set. 11319 */ 11320 if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) 11321 return false; 11322 if (kvm_vcpu_is_illegal_gpa(vcpu, sregs->cr3)) 11323 return false; 11324 } else { 11325 /* 11326 * Not in 64-bit mode: EFER.LMA is clear and the code 11327 * segment cannot be 64-bit. 11328 */ 11329 if (sregs->efer & EFER_LMA || sregs->cs.l) 11330 return false; 11331 } 11332 11333 return kvm_is_valid_cr4(vcpu, sregs->cr4); 11334 } 11335 11336 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, 11337 int *mmu_reset_needed, bool update_pdptrs) 11338 { 11339 struct msr_data apic_base_msr; 11340 int idx; 11341 struct desc_ptr dt; 11342 11343 if (!kvm_is_valid_sregs(vcpu, sregs)) 11344 return -EINVAL; 11345 11346 apic_base_msr.data = sregs->apic_base; 11347 apic_base_msr.host_initiated = true; 11348 if (kvm_set_apic_base(vcpu, &apic_base_msr)) 11349 return -EINVAL; 11350 11351 if (vcpu->arch.guest_state_protected) 11352 return 0; 11353 11354 dt.size = sregs->idt.limit; 11355 dt.address = sregs->idt.base; 11356 static_call(kvm_x86_set_idt)(vcpu, &dt); 11357 dt.size = sregs->gdt.limit; 11358 dt.address = sregs->gdt.base; 11359 static_call(kvm_x86_set_gdt)(vcpu, &dt); 11360 11361 vcpu->arch.cr2 = sregs->cr2; 11362 *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; 11363 vcpu->arch.cr3 = sregs->cr3; 11364 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 11365 static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3); 11366 11367 kvm_set_cr8(vcpu, sregs->cr8); 11368 11369 *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; 11370 static_call(kvm_x86_set_efer)(vcpu, sregs->efer); 11371 11372 *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; 11373 static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0); 11374 vcpu->arch.cr0 = sregs->cr0; 11375 11376 *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; 11377 static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4); 11378 11379 if (update_pdptrs) { 11380 idx = srcu_read_lock(&vcpu->kvm->srcu); 11381 if (is_pae_paging(vcpu)) { 11382 load_pdptrs(vcpu, kvm_read_cr3(vcpu)); 11383 *mmu_reset_needed = 1; 11384 } 11385 srcu_read_unlock(&vcpu->kvm->srcu, idx); 11386 } 11387 11388 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); 11389 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); 11390 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); 11391 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); 11392 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); 11393 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); 11394 11395 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); 11396 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); 11397 11398 update_cr8_intercept(vcpu); 11399 11400 /* Older userspace won't unhalt the vcpu on reset. */ 11401 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && 11402 sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && 11403 !is_protmode(vcpu)) 11404 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 11405 11406 return 0; 11407 } 11408 11409 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 11410 { 11411 int pending_vec, max_bits; 11412 int mmu_reset_needed = 0; 11413 int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); 11414 11415 if (ret) 11416 return ret; 11417 11418 if (mmu_reset_needed) 11419 kvm_mmu_reset_context(vcpu); 11420 11421 max_bits = KVM_NR_INTERRUPTS; 11422 pending_vec = find_first_bit( 11423 (const unsigned long *)sregs->interrupt_bitmap, max_bits); 11424 11425 if (pending_vec < max_bits) { 11426 kvm_queue_interrupt(vcpu, pending_vec, false); 11427 pr_debug("Set back pending irq %d\n", pending_vec); 11428 kvm_make_request(KVM_REQ_EVENT, vcpu); 11429 } 11430 return 0; 11431 } 11432 11433 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) 11434 { 11435 int mmu_reset_needed = 0; 11436 bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; 11437 bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && 11438 !(sregs2->efer & EFER_LMA); 11439 int i, ret; 11440 11441 if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) 11442 return -EINVAL; 11443 11444 if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) 11445 return -EINVAL; 11446 11447 ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, 11448 &mmu_reset_needed, !valid_pdptrs); 11449 if (ret) 11450 return ret; 11451 11452 if (valid_pdptrs) { 11453 for (i = 0; i < 4 ; i++) 11454 kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); 11455 11456 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); 11457 mmu_reset_needed = 1; 11458 vcpu->arch.pdptrs_from_userspace = true; 11459 } 11460 if (mmu_reset_needed) 11461 kvm_mmu_reset_context(vcpu); 11462 return 0; 11463 } 11464 11465 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 11466 struct kvm_sregs *sregs) 11467 { 11468 int ret; 11469 11470 vcpu_load(vcpu); 11471 ret = __set_sregs(vcpu, sregs); 11472 vcpu_put(vcpu); 11473 return ret; 11474 } 11475 11476 static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) 11477 { 11478 bool set = false; 11479 struct kvm_vcpu *vcpu; 11480 unsigned long i; 11481 11482 if (!enable_apicv) 11483 return; 11484 11485 down_write(&kvm->arch.apicv_update_lock); 11486 11487 kvm_for_each_vcpu(i, vcpu, kvm) { 11488 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) { 11489 set = true; 11490 break; 11491 } 11492 } 11493 __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set); 11494 up_write(&kvm->arch.apicv_update_lock); 11495 } 11496 11497 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 11498 struct kvm_guest_debug *dbg) 11499 { 11500 unsigned long rflags; 11501 int i, r; 11502 11503 if (vcpu->arch.guest_state_protected) 11504 return -EINVAL; 11505 11506 vcpu_load(vcpu); 11507 11508 if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) { 11509 r = -EBUSY; 11510 if (kvm_is_exception_pending(vcpu)) 11511 goto out; 11512 if (dbg->control & KVM_GUESTDBG_INJECT_DB) 11513 kvm_queue_exception(vcpu, DB_VECTOR); 11514 else 11515 kvm_queue_exception(vcpu, BP_VECTOR); 11516 } 11517 11518 /* 11519 * Read rflags as long as potentially injected trace flags are still 11520 * filtered out. 11521 */ 11522 rflags = kvm_get_rflags(vcpu); 11523 11524 vcpu->guest_debug = dbg->control; 11525 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) 11526 vcpu->guest_debug = 0; 11527 11528 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 11529 for (i = 0; i < KVM_NR_DB_REGS; ++i) 11530 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; 11531 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7]; 11532 } else { 11533 for (i = 0; i < KVM_NR_DB_REGS; i++) 11534 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 11535 } 11536 kvm_update_dr7(vcpu); 11537 11538 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 11539 vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu); 11540 11541 /* 11542 * Trigger an rflags update that will inject or remove the trace 11543 * flags. 11544 */ 11545 kvm_set_rflags(vcpu, rflags); 11546 11547 static_call(kvm_x86_update_exception_bitmap)(vcpu); 11548 11549 kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm); 11550 11551 r = 0; 11552 11553 out: 11554 vcpu_put(vcpu); 11555 return r; 11556 } 11557 11558 /* 11559 * Translate a guest virtual address to a guest physical address. 11560 */ 11561 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 11562 struct kvm_translation *tr) 11563 { 11564 unsigned long vaddr = tr->linear_address; 11565 gpa_t gpa; 11566 int idx; 11567 11568 vcpu_load(vcpu); 11569 11570 idx = srcu_read_lock(&vcpu->kvm->srcu); 11571 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL); 11572 srcu_read_unlock(&vcpu->kvm->srcu, idx); 11573 tr->physical_address = gpa; 11574 tr->valid = gpa != INVALID_GPA; 11575 tr->writeable = 1; 11576 tr->usermode = 0; 11577 11578 vcpu_put(vcpu); 11579 return 0; 11580 } 11581 11582 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 11583 { 11584 struct fxregs_state *fxsave; 11585 11586 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 11587 return 0; 11588 11589 vcpu_load(vcpu); 11590 11591 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; 11592 memcpy(fpu->fpr, fxsave->st_space, 128); 11593 fpu->fcw = fxsave->cwd; 11594 fpu->fsw = fxsave->swd; 11595 fpu->ftwx = fxsave->twd; 11596 fpu->last_opcode = fxsave->fop; 11597 fpu->last_ip = fxsave->rip; 11598 fpu->last_dp = fxsave->rdp; 11599 memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space)); 11600 11601 vcpu_put(vcpu); 11602 return 0; 11603 } 11604 11605 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 11606 { 11607 struct fxregs_state *fxsave; 11608 11609 if (fpstate_is_confidential(&vcpu->arch.guest_fpu)) 11610 return 0; 11611 11612 vcpu_load(vcpu); 11613 11614 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave; 11615 11616 memcpy(fxsave->st_space, fpu->fpr, 128); 11617 fxsave->cwd = fpu->fcw; 11618 fxsave->swd = fpu->fsw; 11619 fxsave->twd = fpu->ftwx; 11620 fxsave->fop = fpu->last_opcode; 11621 fxsave->rip = fpu->last_ip; 11622 fxsave->rdp = fpu->last_dp; 11623 memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space)); 11624 11625 vcpu_put(vcpu); 11626 return 0; 11627 } 11628 11629 static void store_regs(struct kvm_vcpu *vcpu) 11630 { 11631 BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); 11632 11633 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) 11634 __get_regs(vcpu, &vcpu->run->s.regs.regs); 11635 11636 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) 11637 __get_sregs(vcpu, &vcpu->run->s.regs.sregs); 11638 11639 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS) 11640 kvm_vcpu_ioctl_x86_get_vcpu_events( 11641 vcpu, &vcpu->run->s.regs.events); 11642 } 11643 11644 static int sync_regs(struct kvm_vcpu *vcpu) 11645 { 11646 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { 11647 __set_regs(vcpu, &vcpu->run->s.regs.regs); 11648 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; 11649 } 11650 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { 11651 if (__set_sregs(vcpu, &vcpu->run->s.regs.sregs)) 11652 return -EINVAL; 11653 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; 11654 } 11655 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) { 11656 if (kvm_vcpu_ioctl_x86_set_vcpu_events( 11657 vcpu, &vcpu->run->s.regs.events)) 11658 return -EINVAL; 11659 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS; 11660 } 11661 11662 return 0; 11663 } 11664 11665 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 11666 { 11667 if (kvm_check_tsc_unstable() && kvm->created_vcpus) 11668 pr_warn_once("SMP vm created on host with unstable TSC; " 11669 "guest TSC will not be reliable\n"); 11670 11671 if (!kvm->arch.max_vcpu_ids) 11672 kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS; 11673 11674 if (id >= kvm->arch.max_vcpu_ids) 11675 return -EINVAL; 11676 11677 return static_call(kvm_x86_vcpu_precreate)(kvm); 11678 } 11679 11680 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 11681 { 11682 struct page *page; 11683 int r; 11684 11685 vcpu->arch.last_vmentry_cpu = -1; 11686 vcpu->arch.regs_avail = ~0; 11687 vcpu->arch.regs_dirty = ~0; 11688 11689 kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm, vcpu, KVM_HOST_USES_PFN); 11690 11691 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) 11692 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 11693 else 11694 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED; 11695 11696 r = kvm_mmu_create(vcpu); 11697 if (r < 0) 11698 return r; 11699 11700 if (irqchip_in_kernel(vcpu->kvm)) { 11701 r = kvm_create_lapic(vcpu, lapic_timer_advance_ns); 11702 if (r < 0) 11703 goto fail_mmu_destroy; 11704 11705 /* 11706 * Defer evaluating inhibits until the vCPU is first run, as 11707 * this vCPU will not get notified of any changes until this 11708 * vCPU is visible to other vCPUs (marked online and added to 11709 * the set of vCPUs). Opportunistically mark APICv active as 11710 * VMX in particularly is highly unlikely to have inhibits. 11711 * Ignore the current per-VM APICv state so that vCPU creation 11712 * is guaranteed to run with a deterministic value, the request 11713 * will ensure the vCPU gets the correct state before VM-Entry. 11714 */ 11715 if (enable_apicv) { 11716 vcpu->arch.apic->apicv_active = true; 11717 kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); 11718 } 11719 } else 11720 static_branch_inc(&kvm_has_noapic_vcpu); 11721 11722 r = -ENOMEM; 11723 11724 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); 11725 if (!page) 11726 goto fail_free_lapic; 11727 vcpu->arch.pio_data = page_address(page); 11728 11729 vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64), 11730 GFP_KERNEL_ACCOUNT); 11731 vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64), 11732 GFP_KERNEL_ACCOUNT); 11733 if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks) 11734 goto fail_free_mce_banks; 11735 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; 11736 11737 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, 11738 GFP_KERNEL_ACCOUNT)) 11739 goto fail_free_mce_banks; 11740 11741 if (!alloc_emulate_ctxt(vcpu)) 11742 goto free_wbinvd_dirty_mask; 11743 11744 if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) { 11745 pr_err("failed to allocate vcpu's fpu\n"); 11746 goto free_emulate_ctxt; 11747 } 11748 11749 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); 11750 vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu); 11751 11752 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT; 11753 11754 kvm_async_pf_hash_reset(vcpu); 11755 11756 vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap; 11757 kvm_pmu_init(vcpu); 11758 11759 vcpu->arch.pending_external_vector = -1; 11760 vcpu->arch.preempted_in_kernel = false; 11761 11762 #if IS_ENABLED(CONFIG_HYPERV) 11763 vcpu->arch.hv_root_tdp = INVALID_PAGE; 11764 #endif 11765 11766 r = static_call(kvm_x86_vcpu_create)(vcpu); 11767 if (r) 11768 goto free_guest_fpu; 11769 11770 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities(); 11771 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT; 11772 kvm_xen_init_vcpu(vcpu); 11773 kvm_vcpu_mtrr_init(vcpu); 11774 vcpu_load(vcpu); 11775 kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz); 11776 kvm_vcpu_reset(vcpu, false); 11777 kvm_init_mmu(vcpu); 11778 vcpu_put(vcpu); 11779 return 0; 11780 11781 free_guest_fpu: 11782 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); 11783 free_emulate_ctxt: 11784 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); 11785 free_wbinvd_dirty_mask: 11786 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 11787 fail_free_mce_banks: 11788 kfree(vcpu->arch.mce_banks); 11789 kfree(vcpu->arch.mci_ctl2_banks); 11790 free_page((unsigned long)vcpu->arch.pio_data); 11791 fail_free_lapic: 11792 kvm_free_lapic(vcpu); 11793 fail_mmu_destroy: 11794 kvm_mmu_destroy(vcpu); 11795 return r; 11796 } 11797 11798 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 11799 { 11800 struct kvm *kvm = vcpu->kvm; 11801 11802 if (mutex_lock_killable(&vcpu->mutex)) 11803 return; 11804 vcpu_load(vcpu); 11805 kvm_synchronize_tsc(vcpu, 0); 11806 vcpu_put(vcpu); 11807 11808 /* poll control enabled by default */ 11809 vcpu->arch.msr_kvm_poll_control = 1; 11810 11811 mutex_unlock(&vcpu->mutex); 11812 11813 if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0) 11814 schedule_delayed_work(&kvm->arch.kvmclock_sync_work, 11815 KVMCLOCK_SYNC_PERIOD); 11816 } 11817 11818 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 11819 { 11820 int idx; 11821 11822 kvmclock_reset(vcpu); 11823 11824 static_call(kvm_x86_vcpu_free)(vcpu); 11825 11826 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt); 11827 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); 11828 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); 11829 11830 kvm_xen_destroy_vcpu(vcpu); 11831 kvm_hv_vcpu_uninit(vcpu); 11832 kvm_pmu_destroy(vcpu); 11833 kfree(vcpu->arch.mce_banks); 11834 kfree(vcpu->arch.mci_ctl2_banks); 11835 kvm_free_lapic(vcpu); 11836 idx = srcu_read_lock(&vcpu->kvm->srcu); 11837 kvm_mmu_destroy(vcpu); 11838 srcu_read_unlock(&vcpu->kvm->srcu, idx); 11839 free_page((unsigned long)vcpu->arch.pio_data); 11840 kvfree(vcpu->arch.cpuid_entries); 11841 if (!lapic_in_kernel(vcpu)) 11842 static_branch_dec(&kvm_has_noapic_vcpu); 11843 } 11844 11845 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) 11846 { 11847 struct kvm_cpuid_entry2 *cpuid_0x1; 11848 unsigned long old_cr0 = kvm_read_cr0(vcpu); 11849 unsigned long new_cr0; 11850 11851 /* 11852 * Several of the "set" flows, e.g. ->set_cr0(), read other registers 11853 * to handle side effects. RESET emulation hits those flows and relies 11854 * on emulated/virtualized registers, including those that are loaded 11855 * into hardware, to be zeroed at vCPU creation. Use CRs as a sentinel 11856 * to detect improper or missing initialization. 11857 */ 11858 WARN_ON_ONCE(!init_event && 11859 (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu))); 11860 11861 /* 11862 * SVM doesn't unconditionally VM-Exit on INIT and SHUTDOWN, thus it's 11863 * possible to INIT the vCPU while L2 is active. Force the vCPU back 11864 * into L1 as EFER.SVME is cleared on INIT (along with all other EFER 11865 * bits), i.e. virtualization is disabled. 11866 */ 11867 if (is_guest_mode(vcpu)) 11868 kvm_leave_nested(vcpu); 11869 11870 kvm_lapic_reset(vcpu, init_event); 11871 11872 WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu)); 11873 vcpu->arch.hflags = 0; 11874 11875 vcpu->arch.smi_pending = 0; 11876 vcpu->arch.smi_count = 0; 11877 atomic_set(&vcpu->arch.nmi_queued, 0); 11878 vcpu->arch.nmi_pending = 0; 11879 vcpu->arch.nmi_injected = false; 11880 kvm_clear_interrupt_queue(vcpu); 11881 kvm_clear_exception_queue(vcpu); 11882 11883 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 11884 kvm_update_dr0123(vcpu); 11885 vcpu->arch.dr6 = DR6_ACTIVE_LOW; 11886 vcpu->arch.dr7 = DR7_FIXED_1; 11887 kvm_update_dr7(vcpu); 11888 11889 vcpu->arch.cr2 = 0; 11890 11891 kvm_make_request(KVM_REQ_EVENT, vcpu); 11892 vcpu->arch.apf.msr_en_val = 0; 11893 vcpu->arch.apf.msr_int_val = 0; 11894 vcpu->arch.st.msr_val = 0; 11895 11896 kvmclock_reset(vcpu); 11897 11898 kvm_clear_async_pf_completion_queue(vcpu); 11899 kvm_async_pf_hash_reset(vcpu); 11900 vcpu->arch.apf.halted = false; 11901 11902 if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) { 11903 struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate; 11904 11905 /* 11906 * All paths that lead to INIT are required to load the guest's 11907 * FPU state (because most paths are buried in KVM_RUN). 11908 */ 11909 if (init_event) 11910 kvm_put_guest_fpu(vcpu); 11911 11912 fpstate_clear_xstate_component(fpstate, XFEATURE_BNDREGS); 11913 fpstate_clear_xstate_component(fpstate, XFEATURE_BNDCSR); 11914 11915 if (init_event) 11916 kvm_load_guest_fpu(vcpu); 11917 } 11918 11919 if (!init_event) { 11920 kvm_pmu_reset(vcpu); 11921 vcpu->arch.smbase = 0x30000; 11922 11923 vcpu->arch.msr_misc_features_enables = 0; 11924 vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL | 11925 MSR_IA32_MISC_ENABLE_BTS_UNAVAIL; 11926 11927 __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP); 11928 __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true); 11929 } 11930 11931 /* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */ 11932 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs)); 11933 kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP); 11934 11935 /* 11936 * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon) 11937 * if no CPUID match is found. Note, it's impossible to get a match at 11938 * RESET since KVM emulates RESET before exposing the vCPU to userspace, 11939 * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry 11940 * on RESET. But, go through the motions in case that's ever remedied. 11941 */ 11942 cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1); 11943 kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600); 11944 11945 static_call(kvm_x86_vcpu_reset)(vcpu, init_event); 11946 11947 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED); 11948 kvm_rip_write(vcpu, 0xfff0); 11949 11950 vcpu->arch.cr3 = 0; 11951 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 11952 11953 /* 11954 * CR0.CD/NW are set on RESET, preserved on INIT. Note, some versions 11955 * of Intel's SDM list CD/NW as being set on INIT, but they contradict 11956 * (or qualify) that with a footnote stating that CD/NW are preserved. 11957 */ 11958 new_cr0 = X86_CR0_ET; 11959 if (init_event) 11960 new_cr0 |= (old_cr0 & (X86_CR0_NW | X86_CR0_CD)); 11961 else 11962 new_cr0 |= X86_CR0_NW | X86_CR0_CD; 11963 11964 static_call(kvm_x86_set_cr0)(vcpu, new_cr0); 11965 static_call(kvm_x86_set_cr4)(vcpu, 0); 11966 static_call(kvm_x86_set_efer)(vcpu, 0); 11967 static_call(kvm_x86_update_exception_bitmap)(vcpu); 11968 11969 /* 11970 * On the standard CR0/CR4/EFER modification paths, there are several 11971 * complex conditions determining whether the MMU has to be reset and/or 11972 * which PCIDs have to be flushed. However, CR0.WP and the paging-related 11973 * bits in CR4 and EFER are irrelevant if CR0.PG was '0'; and a reset+flush 11974 * is needed anyway if CR0.PG was '1' (which can only happen for INIT, as 11975 * CR0 will be '0' prior to RESET). So we only need to check CR0.PG here. 11976 */ 11977 if (old_cr0 & X86_CR0_PG) { 11978 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 11979 kvm_mmu_reset_context(vcpu); 11980 } 11981 11982 /* 11983 * Intel's SDM states that all TLB entries are flushed on INIT. AMD's 11984 * APM states the TLBs are untouched by INIT, but it also states that 11985 * the TLBs are flushed on "External initialization of the processor." 11986 * Flush the guest TLB regardless of vendor, there is no meaningful 11987 * benefit in relying on the guest to flush the TLB immediately after 11988 * INIT. A spurious TLB flush is benign and likely negligible from a 11989 * performance perspective. 11990 */ 11991 if (init_event) 11992 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 11993 } 11994 EXPORT_SYMBOL_GPL(kvm_vcpu_reset); 11995 11996 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) 11997 { 11998 struct kvm_segment cs; 11999 12000 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS); 12001 cs.selector = vector << 8; 12002 cs.base = vector << 12; 12003 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS); 12004 kvm_rip_write(vcpu, 0); 12005 } 12006 EXPORT_SYMBOL_GPL(kvm_vcpu_deliver_sipi_vector); 12007 12008 int kvm_arch_hardware_enable(void) 12009 { 12010 struct kvm *kvm; 12011 struct kvm_vcpu *vcpu; 12012 unsigned long i; 12013 int ret; 12014 u64 local_tsc; 12015 u64 max_tsc = 0; 12016 bool stable, backwards_tsc = false; 12017 12018 kvm_user_return_msr_cpu_online(); 12019 12020 ret = kvm_x86_check_processor_compatibility(); 12021 if (ret) 12022 return ret; 12023 12024 ret = static_call(kvm_x86_hardware_enable)(); 12025 if (ret != 0) 12026 return ret; 12027 12028 local_tsc = rdtsc(); 12029 stable = !kvm_check_tsc_unstable(); 12030 list_for_each_entry(kvm, &vm_list, vm_list) { 12031 kvm_for_each_vcpu(i, vcpu, kvm) { 12032 if (!stable && vcpu->cpu == smp_processor_id()) 12033 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 12034 if (stable && vcpu->arch.last_host_tsc > local_tsc) { 12035 backwards_tsc = true; 12036 if (vcpu->arch.last_host_tsc > max_tsc) 12037 max_tsc = vcpu->arch.last_host_tsc; 12038 } 12039 } 12040 } 12041 12042 /* 12043 * Sometimes, even reliable TSCs go backwards. This happens on 12044 * platforms that reset TSC during suspend or hibernate actions, but 12045 * maintain synchronization. We must compensate. Fortunately, we can 12046 * detect that condition here, which happens early in CPU bringup, 12047 * before any KVM threads can be running. Unfortunately, we can't 12048 * bring the TSCs fully up to date with real time, as we aren't yet far 12049 * enough into CPU bringup that we know how much real time has actually 12050 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot 12051 * variables that haven't been updated yet. 12052 * 12053 * So we simply find the maximum observed TSC above, then record the 12054 * adjustment to TSC in each VCPU. When the VCPU later gets loaded, 12055 * the adjustment will be applied. Note that we accumulate 12056 * adjustments, in case multiple suspend cycles happen before some VCPU 12057 * gets a chance to run again. In the event that no KVM threads get a 12058 * chance to run, we will miss the entire elapsed period, as we'll have 12059 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may 12060 * loose cycle time. This isn't too big a deal, since the loss will be 12061 * uniform across all VCPUs (not to mention the scenario is extremely 12062 * unlikely). It is possible that a second hibernate recovery happens 12063 * much faster than a first, causing the observed TSC here to be 12064 * smaller; this would require additional padding adjustment, which is 12065 * why we set last_host_tsc to the local tsc observed here. 12066 * 12067 * N.B. - this code below runs only on platforms with reliable TSC, 12068 * as that is the only way backwards_tsc is set above. Also note 12069 * that this runs for ALL vcpus, which is not a bug; all VCPUs should 12070 * have the same delta_cyc adjustment applied if backwards_tsc 12071 * is detected. Note further, this adjustment is only done once, 12072 * as we reset last_host_tsc on all VCPUs to stop this from being 12073 * called multiple times (one for each physical CPU bringup). 12074 * 12075 * Platforms with unreliable TSCs don't have to deal with this, they 12076 * will be compensated by the logic in vcpu_load, which sets the TSC to 12077 * catchup mode. This will catchup all VCPUs to real time, but cannot 12078 * guarantee that they stay in perfect synchronization. 12079 */ 12080 if (backwards_tsc) { 12081 u64 delta_cyc = max_tsc - local_tsc; 12082 list_for_each_entry(kvm, &vm_list, vm_list) { 12083 kvm->arch.backwards_tsc_observed = true; 12084 kvm_for_each_vcpu(i, vcpu, kvm) { 12085 vcpu->arch.tsc_offset_adjustment += delta_cyc; 12086 vcpu->arch.last_host_tsc = local_tsc; 12087 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); 12088 } 12089 12090 /* 12091 * We have to disable TSC offset matching.. if you were 12092 * booting a VM while issuing an S4 host suspend.... 12093 * you may have some problem. Solving this issue is 12094 * left as an exercise to the reader. 12095 */ 12096 kvm->arch.last_tsc_nsec = 0; 12097 kvm->arch.last_tsc_write = 0; 12098 } 12099 12100 } 12101 return 0; 12102 } 12103 12104 void kvm_arch_hardware_disable(void) 12105 { 12106 static_call(kvm_x86_hardware_disable)(); 12107 drop_user_return_notifiers(); 12108 } 12109 12110 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu) 12111 { 12112 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id; 12113 } 12114 12115 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) 12116 { 12117 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0; 12118 } 12119 12120 __read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); 12121 EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu); 12122 12123 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) 12124 { 12125 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); 12126 12127 vcpu->arch.l1tf_flush_l1d = true; 12128 if (pmu->version && unlikely(pmu->event_count)) { 12129 pmu->need_cleanup = true; 12130 kvm_make_request(KVM_REQ_PMU, vcpu); 12131 } 12132 static_call(kvm_x86_sched_in)(vcpu, cpu); 12133 } 12134 12135 void kvm_arch_free_vm(struct kvm *kvm) 12136 { 12137 kfree(to_kvm_hv(kvm)->hv_pa_pg); 12138 __kvm_arch_free_vm(kvm); 12139 } 12140 12141 12142 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 12143 { 12144 int ret; 12145 unsigned long flags; 12146 12147 if (type) 12148 return -EINVAL; 12149 12150 ret = kvm_page_track_init(kvm); 12151 if (ret) 12152 goto out; 12153 12154 ret = kvm_mmu_init_vm(kvm); 12155 if (ret) 12156 goto out_page_track; 12157 12158 ret = static_call(kvm_x86_vm_init)(kvm); 12159 if (ret) 12160 goto out_uninit_mmu; 12161 12162 INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list); 12163 INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); 12164 atomic_set(&kvm->arch.noncoherent_dma_count, 0); 12165 12166 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ 12167 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); 12168 /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */ 12169 set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 12170 &kvm->arch.irq_sources_bitmap); 12171 12172 raw_spin_lock_init(&kvm->arch.tsc_write_lock); 12173 mutex_init(&kvm->arch.apic_map_lock); 12174 seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock); 12175 kvm->arch.kvmclock_offset = -get_kvmclock_base_ns(); 12176 12177 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); 12178 pvclock_update_vm_gtod_copy(kvm); 12179 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); 12180 12181 kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; 12182 kvm->arch.guest_can_read_msr_platform_info = true; 12183 kvm->arch.enable_pmu = enable_pmu; 12184 12185 #if IS_ENABLED(CONFIG_HYPERV) 12186 spin_lock_init(&kvm->arch.hv_root_tdp_lock); 12187 kvm->arch.hv_root_tdp = INVALID_PAGE; 12188 #endif 12189 12190 INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn); 12191 INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn); 12192 12193 kvm_apicv_init(kvm); 12194 kvm_hv_init_vm(kvm); 12195 kvm_xen_init_vm(kvm); 12196 12197 return 0; 12198 12199 out_uninit_mmu: 12200 kvm_mmu_uninit_vm(kvm); 12201 out_page_track: 12202 kvm_page_track_cleanup(kvm); 12203 out: 12204 return ret; 12205 } 12206 12207 int kvm_arch_post_init_vm(struct kvm *kvm) 12208 { 12209 return kvm_mmu_post_init_vm(kvm); 12210 } 12211 12212 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) 12213 { 12214 vcpu_load(vcpu); 12215 kvm_mmu_unload(vcpu); 12216 vcpu_put(vcpu); 12217 } 12218 12219 static void kvm_unload_vcpu_mmus(struct kvm *kvm) 12220 { 12221 unsigned long i; 12222 struct kvm_vcpu *vcpu; 12223 12224 kvm_for_each_vcpu(i, vcpu, kvm) { 12225 kvm_clear_async_pf_completion_queue(vcpu); 12226 kvm_unload_vcpu_mmu(vcpu); 12227 } 12228 } 12229 12230 void kvm_arch_sync_events(struct kvm *kvm) 12231 { 12232 cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work); 12233 cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work); 12234 kvm_free_pit(kvm); 12235 } 12236 12237 /** 12238 * __x86_set_memory_region: Setup KVM internal memory slot 12239 * 12240 * @kvm: the kvm pointer to the VM. 12241 * @id: the slot ID to setup. 12242 * @gpa: the GPA to install the slot (unused when @size == 0). 12243 * @size: the size of the slot. Set to zero to uninstall a slot. 12244 * 12245 * This function helps to setup a KVM internal memory slot. Specify 12246 * @size > 0 to install a new slot, while @size == 0 to uninstall a 12247 * slot. The return code can be one of the following: 12248 * 12249 * HVA: on success (uninstall will return a bogus HVA) 12250 * -errno: on error 12251 * 12252 * The caller should always use IS_ERR() to check the return value 12253 * before use. Note, the KVM internal memory slots are guaranteed to 12254 * remain valid and unchanged until the VM is destroyed, i.e., the 12255 * GPA->HVA translation will not change. However, the HVA is a user 12256 * address, i.e. its accessibility is not guaranteed, and must be 12257 * accessed via __copy_{to,from}_user(). 12258 */ 12259 void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, 12260 u32 size) 12261 { 12262 int i, r; 12263 unsigned long hva, old_npages; 12264 struct kvm_memslots *slots = kvm_memslots(kvm); 12265 struct kvm_memory_slot *slot; 12266 12267 /* Called with kvm->slots_lock held. */ 12268 if (WARN_ON(id >= KVM_MEM_SLOTS_NUM)) 12269 return ERR_PTR_USR(-EINVAL); 12270 12271 slot = id_to_memslot(slots, id); 12272 if (size) { 12273 if (slot && slot->npages) 12274 return ERR_PTR_USR(-EEXIST); 12275 12276 /* 12277 * MAP_SHARED to prevent internal slot pages from being moved 12278 * by fork()/COW. 12279 */ 12280 hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE, 12281 MAP_SHARED | MAP_ANONYMOUS, 0); 12282 if (IS_ERR((void *)hva)) 12283 return (void __user *)hva; 12284 } else { 12285 if (!slot || !slot->npages) 12286 return NULL; 12287 12288 old_npages = slot->npages; 12289 hva = slot->userspace_addr; 12290 } 12291 12292 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { 12293 struct kvm_userspace_memory_region m; 12294 12295 m.slot = id | (i << 16); 12296 m.flags = 0; 12297 m.guest_phys_addr = gpa; 12298 m.userspace_addr = hva; 12299 m.memory_size = size; 12300 r = __kvm_set_memory_region(kvm, &m); 12301 if (r < 0) 12302 return ERR_PTR_USR(r); 12303 } 12304 12305 if (!size) 12306 vm_munmap(hva, old_npages * PAGE_SIZE); 12307 12308 return (void __user *)hva; 12309 } 12310 EXPORT_SYMBOL_GPL(__x86_set_memory_region); 12311 12312 void kvm_arch_pre_destroy_vm(struct kvm *kvm) 12313 { 12314 kvm_mmu_pre_destroy_vm(kvm); 12315 } 12316 12317 void kvm_arch_destroy_vm(struct kvm *kvm) 12318 { 12319 if (current->mm == kvm->mm) { 12320 /* 12321 * Free memory regions allocated on behalf of userspace, 12322 * unless the memory map has changed due to process exit 12323 * or fd copying. 12324 */ 12325 mutex_lock(&kvm->slots_lock); 12326 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, 12327 0, 0); 12328 __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT, 12329 0, 0); 12330 __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0); 12331 mutex_unlock(&kvm->slots_lock); 12332 } 12333 kvm_unload_vcpu_mmus(kvm); 12334 static_call_cond(kvm_x86_vm_destroy)(kvm); 12335 kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1)); 12336 kvm_pic_destroy(kvm); 12337 kvm_ioapic_destroy(kvm); 12338 kvm_destroy_vcpus(kvm); 12339 kvfree(rcu_dereference_check(kvm->arch.apic_map, 1)); 12340 kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1)); 12341 kvm_mmu_uninit_vm(kvm); 12342 kvm_page_track_cleanup(kvm); 12343 kvm_xen_destroy_vm(kvm); 12344 kvm_hv_destroy_vm(kvm); 12345 } 12346 12347 static void memslot_rmap_free(struct kvm_memory_slot *slot) 12348 { 12349 int i; 12350 12351 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 12352 kvfree(slot->arch.rmap[i]); 12353 slot->arch.rmap[i] = NULL; 12354 } 12355 } 12356 12357 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) 12358 { 12359 int i; 12360 12361 memslot_rmap_free(slot); 12362 12363 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 12364 kvfree(slot->arch.lpage_info[i - 1]); 12365 slot->arch.lpage_info[i - 1] = NULL; 12366 } 12367 12368 kvm_page_track_free_memslot(slot); 12369 } 12370 12371 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages) 12372 { 12373 const int sz = sizeof(*slot->arch.rmap[0]); 12374 int i; 12375 12376 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { 12377 int level = i + 1; 12378 int lpages = __kvm_mmu_slot_lpages(slot, npages, level); 12379 12380 if (slot->arch.rmap[i]) 12381 continue; 12382 12383 slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT); 12384 if (!slot->arch.rmap[i]) { 12385 memslot_rmap_free(slot); 12386 return -ENOMEM; 12387 } 12388 } 12389 12390 return 0; 12391 } 12392 12393 static int kvm_alloc_memslot_metadata(struct kvm *kvm, 12394 struct kvm_memory_slot *slot) 12395 { 12396 unsigned long npages = slot->npages; 12397 int i, r; 12398 12399 /* 12400 * Clear out the previous array pointers for the KVM_MR_MOVE case. The 12401 * old arrays will be freed by __kvm_set_memory_region() if installing 12402 * the new memslot is successful. 12403 */ 12404 memset(&slot->arch, 0, sizeof(slot->arch)); 12405 12406 if (kvm_memslots_have_rmaps(kvm)) { 12407 r = memslot_rmap_alloc(slot, npages); 12408 if (r) 12409 return r; 12410 } 12411 12412 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 12413 struct kvm_lpage_info *linfo; 12414 unsigned long ugfn; 12415 int lpages; 12416 int level = i + 1; 12417 12418 lpages = __kvm_mmu_slot_lpages(slot, npages, level); 12419 12420 linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT); 12421 if (!linfo) 12422 goto out_free; 12423 12424 slot->arch.lpage_info[i - 1] = linfo; 12425 12426 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) 12427 linfo[0].disallow_lpage = 1; 12428 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) 12429 linfo[lpages - 1].disallow_lpage = 1; 12430 ugfn = slot->userspace_addr >> PAGE_SHIFT; 12431 /* 12432 * If the gfn and userspace address are not aligned wrt each 12433 * other, disable large page support for this slot. 12434 */ 12435 if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) { 12436 unsigned long j; 12437 12438 for (j = 0; j < lpages; ++j) 12439 linfo[j].disallow_lpage = 1; 12440 } 12441 } 12442 12443 if (kvm_page_track_create_memslot(kvm, slot, npages)) 12444 goto out_free; 12445 12446 return 0; 12447 12448 out_free: 12449 memslot_rmap_free(slot); 12450 12451 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { 12452 kvfree(slot->arch.lpage_info[i - 1]); 12453 slot->arch.lpage_info[i - 1] = NULL; 12454 } 12455 return -ENOMEM; 12456 } 12457 12458 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) 12459 { 12460 struct kvm_vcpu *vcpu; 12461 unsigned long i; 12462 12463 /* 12464 * memslots->generation has been incremented. 12465 * mmio generation may have reached its maximum value. 12466 */ 12467 kvm_mmu_invalidate_mmio_sptes(kvm, gen); 12468 12469 /* Force re-initialization of steal_time cache */ 12470 kvm_for_each_vcpu(i, vcpu, kvm) 12471 kvm_vcpu_kick(vcpu); 12472 } 12473 12474 int kvm_arch_prepare_memory_region(struct kvm *kvm, 12475 const struct kvm_memory_slot *old, 12476 struct kvm_memory_slot *new, 12477 enum kvm_mr_change change) 12478 { 12479 if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) { 12480 if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn()) 12481 return -EINVAL; 12482 12483 return kvm_alloc_memslot_metadata(kvm, new); 12484 } 12485 12486 if (change == KVM_MR_FLAGS_ONLY) 12487 memcpy(&new->arch, &old->arch, sizeof(old->arch)); 12488 else if (WARN_ON_ONCE(change != KVM_MR_DELETE)) 12489 return -EIO; 12490 12491 return 0; 12492 } 12493 12494 12495 static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable) 12496 { 12497 struct kvm_arch *ka = &kvm->arch; 12498 12499 if (!kvm_x86_ops.cpu_dirty_log_size) 12500 return; 12501 12502 if ((enable && ++ka->cpu_dirty_logging_count == 1) || 12503 (!enable && --ka->cpu_dirty_logging_count == 0)) 12504 kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING); 12505 12506 WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0); 12507 } 12508 12509 static void kvm_mmu_slot_apply_flags(struct kvm *kvm, 12510 struct kvm_memory_slot *old, 12511 const struct kvm_memory_slot *new, 12512 enum kvm_mr_change change) 12513 { 12514 u32 old_flags = old ? old->flags : 0; 12515 u32 new_flags = new ? new->flags : 0; 12516 bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES; 12517 12518 /* 12519 * Update CPU dirty logging if dirty logging is being toggled. This 12520 * applies to all operations. 12521 */ 12522 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) 12523 kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages); 12524 12525 /* 12526 * Nothing more to do for RO slots (which can't be dirtied and can't be 12527 * made writable) or CREATE/MOVE/DELETE of a slot. 12528 * 12529 * For a memslot with dirty logging disabled: 12530 * CREATE: No dirty mappings will already exist. 12531 * MOVE/DELETE: The old mappings will already have been cleaned up by 12532 * kvm_arch_flush_shadow_memslot() 12533 * 12534 * For a memslot with dirty logging enabled: 12535 * CREATE: No shadow pages exist, thus nothing to write-protect 12536 * and no dirty bits to clear. 12537 * MOVE/DELETE: The old mappings will already have been cleaned up by 12538 * kvm_arch_flush_shadow_memslot(). 12539 */ 12540 if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY)) 12541 return; 12542 12543 /* 12544 * READONLY and non-flags changes were filtered out above, and the only 12545 * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty 12546 * logging isn't being toggled on or off. 12547 */ 12548 if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES))) 12549 return; 12550 12551 if (!log_dirty_pages) { 12552 /* 12553 * Dirty logging tracks sptes in 4k granularity, meaning that 12554 * large sptes have to be split. If live migration succeeds, 12555 * the guest in the source machine will be destroyed and large 12556 * sptes will be created in the destination. However, if the 12557 * guest continues to run in the source machine (for example if 12558 * live migration fails), small sptes will remain around and 12559 * cause bad performance. 12560 * 12561 * Scan sptes if dirty logging has been stopped, dropping those 12562 * which can be collapsed into a single large-page spte. Later 12563 * page faults will create the large-page sptes. 12564 */ 12565 kvm_mmu_zap_collapsible_sptes(kvm, new); 12566 } else { 12567 /* 12568 * Initially-all-set does not require write protecting any page, 12569 * because they're all assumed to be dirty. 12570 */ 12571 if (kvm_dirty_log_manual_protect_and_init_set(kvm)) 12572 return; 12573 12574 if (READ_ONCE(eager_page_split)) 12575 kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K); 12576 12577 if (kvm_x86_ops.cpu_dirty_log_size) { 12578 kvm_mmu_slot_leaf_clear_dirty(kvm, new); 12579 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M); 12580 } else { 12581 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K); 12582 } 12583 12584 /* 12585 * Unconditionally flush the TLBs after enabling dirty logging. 12586 * A flush is almost always going to be necessary (see below), 12587 * and unconditionally flushing allows the helpers to omit 12588 * the subtly complex checks when removing write access. 12589 * 12590 * Do the flush outside of mmu_lock to reduce the amount of 12591 * time mmu_lock is held. Flushing after dropping mmu_lock is 12592 * safe as KVM only needs to guarantee the slot is fully 12593 * write-protected before returning to userspace, i.e. before 12594 * userspace can consume the dirty status. 12595 * 12596 * Flushing outside of mmu_lock requires KVM to be careful when 12597 * making decisions based on writable status of an SPTE, e.g. a 12598 * !writable SPTE doesn't guarantee a CPU can't perform writes. 12599 * 12600 * Specifically, KVM also write-protects guest page tables to 12601 * monitor changes when using shadow paging, and must guarantee 12602 * no CPUs can write to those page before mmu_lock is dropped. 12603 * Because CPUs may have stale TLB entries at this point, a 12604 * !writable SPTE doesn't guarantee CPUs can't perform writes. 12605 * 12606 * KVM also allows making SPTES writable outside of mmu_lock, 12607 * e.g. to allow dirty logging without taking mmu_lock. 12608 * 12609 * To handle these scenarios, KVM uses a separate software-only 12610 * bit (MMU-writable) to track if a SPTE is !writable due to 12611 * a guest page table being write-protected (KVM clears the 12612 * MMU-writable flag when write-protecting for shadow paging). 12613 * 12614 * The use of MMU-writable is also the primary motivation for 12615 * the unconditional flush. Because KVM must guarantee that a 12616 * CPU doesn't contain stale, writable TLB entries for a 12617 * !MMU-writable SPTE, KVM must flush if it encounters any 12618 * MMU-writable SPTE regardless of whether the actual hardware 12619 * writable bit was set. I.e. KVM is almost guaranteed to need 12620 * to flush, while unconditionally flushing allows the "remove 12621 * write access" helpers to ignore MMU-writable entirely. 12622 * 12623 * See is_writable_pte() for more details (the case involving 12624 * access-tracked SPTEs is particularly relevant). 12625 */ 12626 kvm_arch_flush_remote_tlbs_memslot(kvm, new); 12627 } 12628 } 12629 12630 void kvm_arch_commit_memory_region(struct kvm *kvm, 12631 struct kvm_memory_slot *old, 12632 const struct kvm_memory_slot *new, 12633 enum kvm_mr_change change) 12634 { 12635 if (!kvm->arch.n_requested_mmu_pages && 12636 (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) { 12637 unsigned long nr_mmu_pages; 12638 12639 nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO; 12640 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES); 12641 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); 12642 } 12643 12644 kvm_mmu_slot_apply_flags(kvm, old, new, change); 12645 12646 /* Free the arrays associated with the old memslot. */ 12647 if (change == KVM_MR_MOVE) 12648 kvm_arch_free_memslot(kvm, old); 12649 } 12650 12651 void kvm_arch_flush_shadow_all(struct kvm *kvm) 12652 { 12653 kvm_mmu_zap_all(kvm); 12654 } 12655 12656 void kvm_arch_flush_shadow_memslot(struct kvm *kvm, 12657 struct kvm_memory_slot *slot) 12658 { 12659 kvm_page_track_flush_slot(kvm, slot); 12660 } 12661 12662 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu) 12663 { 12664 return (is_guest_mode(vcpu) && 12665 static_call(kvm_x86_guest_apic_has_interrupt)(vcpu)); 12666 } 12667 12668 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) 12669 { 12670 if (!list_empty_careful(&vcpu->async_pf.done)) 12671 return true; 12672 12673 if (kvm_apic_has_pending_init_or_sipi(vcpu) && 12674 kvm_apic_init_sipi_allowed(vcpu)) 12675 return true; 12676 12677 if (vcpu->arch.pv.pv_unhalted) 12678 return true; 12679 12680 if (kvm_is_exception_pending(vcpu)) 12681 return true; 12682 12683 if (kvm_test_request(KVM_REQ_NMI, vcpu) || 12684 (vcpu->arch.nmi_pending && 12685 static_call(kvm_x86_nmi_allowed)(vcpu, false))) 12686 return true; 12687 12688 #ifdef CONFIG_KVM_SMM 12689 if (kvm_test_request(KVM_REQ_SMI, vcpu) || 12690 (vcpu->arch.smi_pending && 12691 static_call(kvm_x86_smi_allowed)(vcpu, false))) 12692 return true; 12693 #endif 12694 12695 if (kvm_arch_interrupt_allowed(vcpu) && 12696 (kvm_cpu_has_interrupt(vcpu) || 12697 kvm_guest_apic_has_interrupt(vcpu))) 12698 return true; 12699 12700 if (kvm_hv_has_stimer_pending(vcpu)) 12701 return true; 12702 12703 if (is_guest_mode(vcpu) && 12704 kvm_x86_ops.nested_ops->has_events && 12705 kvm_x86_ops.nested_ops->has_events(vcpu)) 12706 return true; 12707 12708 if (kvm_xen_has_pending_events(vcpu)) 12709 return true; 12710 12711 return false; 12712 } 12713 12714 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 12715 { 12716 return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu); 12717 } 12718 12719 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu) 12720 { 12721 if (kvm_vcpu_apicv_active(vcpu) && 12722 static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu)) 12723 return true; 12724 12725 return false; 12726 } 12727 12728 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu) 12729 { 12730 if (READ_ONCE(vcpu->arch.pv.pv_unhalted)) 12731 return true; 12732 12733 if (kvm_test_request(KVM_REQ_NMI, vcpu) || 12734 #ifdef CONFIG_KVM_SMM 12735 kvm_test_request(KVM_REQ_SMI, vcpu) || 12736 #endif 12737 kvm_test_request(KVM_REQ_EVENT, vcpu)) 12738 return true; 12739 12740 return kvm_arch_dy_has_pending_interrupt(vcpu); 12741 } 12742 12743 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 12744 { 12745 if (vcpu->arch.guest_state_protected) 12746 return true; 12747 12748 return vcpu->arch.preempted_in_kernel; 12749 } 12750 12751 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) 12752 { 12753 return kvm_rip_read(vcpu); 12754 } 12755 12756 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 12757 { 12758 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 12759 } 12760 12761 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) 12762 { 12763 return static_call(kvm_x86_interrupt_allowed)(vcpu, false); 12764 } 12765 12766 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) 12767 { 12768 /* Can't read the RIP when guest state is protected, just return 0 */ 12769 if (vcpu->arch.guest_state_protected) 12770 return 0; 12771 12772 if (is_64_bit_mode(vcpu)) 12773 return kvm_rip_read(vcpu); 12774 return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) + 12775 kvm_rip_read(vcpu)); 12776 } 12777 EXPORT_SYMBOL_GPL(kvm_get_linear_rip); 12778 12779 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip) 12780 { 12781 return kvm_get_linear_rip(vcpu) == linear_rip; 12782 } 12783 EXPORT_SYMBOL_GPL(kvm_is_linear_rip); 12784 12785 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu) 12786 { 12787 unsigned long rflags; 12788 12789 rflags = static_call(kvm_x86_get_rflags)(vcpu); 12790 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 12791 rflags &= ~X86_EFLAGS_TF; 12792 return rflags; 12793 } 12794 EXPORT_SYMBOL_GPL(kvm_get_rflags); 12795 12796 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 12797 { 12798 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP && 12799 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip)) 12800 rflags |= X86_EFLAGS_TF; 12801 static_call(kvm_x86_set_rflags)(vcpu, rflags); 12802 } 12803 12804 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) 12805 { 12806 __kvm_set_rflags(vcpu, rflags); 12807 kvm_make_request(KVM_REQ_EVENT, vcpu); 12808 } 12809 EXPORT_SYMBOL_GPL(kvm_set_rflags); 12810 12811 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) 12812 { 12813 BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU)); 12814 12815 return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); 12816 } 12817 12818 static inline u32 kvm_async_pf_next_probe(u32 key) 12819 { 12820 return (key + 1) & (ASYNC_PF_PER_VCPU - 1); 12821 } 12822 12823 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 12824 { 12825 u32 key = kvm_async_pf_hash_fn(gfn); 12826 12827 while (vcpu->arch.apf.gfns[key] != ~0) 12828 key = kvm_async_pf_next_probe(key); 12829 12830 vcpu->arch.apf.gfns[key] = gfn; 12831 } 12832 12833 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) 12834 { 12835 int i; 12836 u32 key = kvm_async_pf_hash_fn(gfn); 12837 12838 for (i = 0; i < ASYNC_PF_PER_VCPU && 12839 (vcpu->arch.apf.gfns[key] != gfn && 12840 vcpu->arch.apf.gfns[key] != ~0); i++) 12841 key = kvm_async_pf_next_probe(key); 12842 12843 return key; 12844 } 12845 12846 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 12847 { 12848 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn; 12849 } 12850 12851 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) 12852 { 12853 u32 i, j, k; 12854 12855 i = j = kvm_async_pf_gfn_slot(vcpu, gfn); 12856 12857 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn)) 12858 return; 12859 12860 while (true) { 12861 vcpu->arch.apf.gfns[i] = ~0; 12862 do { 12863 j = kvm_async_pf_next_probe(j); 12864 if (vcpu->arch.apf.gfns[j] == ~0) 12865 return; 12866 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]); 12867 /* 12868 * k lies cyclically in ]i,j] 12869 * | i.k.j | 12870 * |....j i.k.| or |.k..j i...| 12871 */ 12872 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j)); 12873 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j]; 12874 i = j; 12875 } 12876 } 12877 12878 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu) 12879 { 12880 u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT; 12881 12882 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason, 12883 sizeof(reason)); 12884 } 12885 12886 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token) 12887 { 12888 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); 12889 12890 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, 12891 &token, offset, sizeof(token)); 12892 } 12893 12894 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu) 12895 { 12896 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); 12897 u32 val; 12898 12899 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, 12900 &val, offset, sizeof(val))) 12901 return false; 12902 12903 return !val; 12904 } 12905 12906 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) 12907 { 12908 12909 if (!kvm_pv_async_pf_enabled(vcpu)) 12910 return false; 12911 12912 if (vcpu->arch.apf.send_user_only && 12913 static_call(kvm_x86_get_cpl)(vcpu) == 0) 12914 return false; 12915 12916 if (is_guest_mode(vcpu)) { 12917 /* 12918 * L1 needs to opt into the special #PF vmexits that are 12919 * used to deliver async page faults. 12920 */ 12921 return vcpu->arch.apf.delivery_as_pf_vmexit; 12922 } else { 12923 /* 12924 * Play it safe in case the guest temporarily disables paging. 12925 * The real mode IDT in particular is unlikely to have a #PF 12926 * exception setup. 12927 */ 12928 return is_paging(vcpu); 12929 } 12930 } 12931 12932 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) 12933 { 12934 if (unlikely(!lapic_in_kernel(vcpu) || 12935 kvm_event_needs_reinjection(vcpu) || 12936 kvm_is_exception_pending(vcpu))) 12937 return false; 12938 12939 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu)) 12940 return false; 12941 12942 /* 12943 * If interrupts are off we cannot even use an artificial 12944 * halt state. 12945 */ 12946 return kvm_arch_interrupt_allowed(vcpu); 12947 } 12948 12949 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 12950 struct kvm_async_pf *work) 12951 { 12952 struct x86_exception fault; 12953 12954 trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa); 12955 kvm_add_async_pf_gfn(vcpu, work->arch.gfn); 12956 12957 if (kvm_can_deliver_async_pf(vcpu) && 12958 !apf_put_user_notpresent(vcpu)) { 12959 fault.vector = PF_VECTOR; 12960 fault.error_code_valid = true; 12961 fault.error_code = 0; 12962 fault.nested_page_fault = false; 12963 fault.address = work->arch.token; 12964 fault.async_page_fault = true; 12965 kvm_inject_page_fault(vcpu, &fault); 12966 return true; 12967 } else { 12968 /* 12969 * It is not possible to deliver a paravirtualized asynchronous 12970 * page fault, but putting the guest in an artificial halt state 12971 * can be beneficial nevertheless: if an interrupt arrives, we 12972 * can deliver it timely and perhaps the guest will schedule 12973 * another process. When the instruction that triggered a page 12974 * fault is retried, hopefully the page will be ready in the host. 12975 */ 12976 kvm_make_request(KVM_REQ_APF_HALT, vcpu); 12977 return false; 12978 } 12979 } 12980 12981 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 12982 struct kvm_async_pf *work) 12983 { 12984 struct kvm_lapic_irq irq = { 12985 .delivery_mode = APIC_DM_FIXED, 12986 .vector = vcpu->arch.apf.vec 12987 }; 12988 12989 if (work->wakeup_all) 12990 work->arch.token = ~0; /* broadcast wakeup */ 12991 else 12992 kvm_del_async_pf_gfn(vcpu, work->arch.gfn); 12993 trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); 12994 12995 if ((work->wakeup_all || work->notpresent_injected) && 12996 kvm_pv_async_pf_enabled(vcpu) && 12997 !apf_put_user_ready(vcpu, work->arch.token)) { 12998 vcpu->arch.apf.pageready_pending = true; 12999 kvm_apic_set_irq(vcpu, &irq, NULL); 13000 } 13001 13002 vcpu->arch.apf.halted = false; 13003 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; 13004 } 13005 13006 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) 13007 { 13008 kvm_make_request(KVM_REQ_APF_READY, vcpu); 13009 if (!vcpu->arch.apf.pageready_pending) 13010 kvm_vcpu_kick(vcpu); 13011 } 13012 13013 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) 13014 { 13015 if (!kvm_pv_async_pf_enabled(vcpu)) 13016 return true; 13017 else 13018 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu); 13019 } 13020 13021 void kvm_arch_start_assignment(struct kvm *kvm) 13022 { 13023 if (atomic_inc_return(&kvm->arch.assigned_device_count) == 1) 13024 static_call_cond(kvm_x86_pi_start_assignment)(kvm); 13025 } 13026 EXPORT_SYMBOL_GPL(kvm_arch_start_assignment); 13027 13028 void kvm_arch_end_assignment(struct kvm *kvm) 13029 { 13030 atomic_dec(&kvm->arch.assigned_device_count); 13031 } 13032 EXPORT_SYMBOL_GPL(kvm_arch_end_assignment); 13033 13034 bool noinstr kvm_arch_has_assigned_device(struct kvm *kvm) 13035 { 13036 return arch_atomic_read(&kvm->arch.assigned_device_count); 13037 } 13038 EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device); 13039 13040 void kvm_arch_register_noncoherent_dma(struct kvm *kvm) 13041 { 13042 atomic_inc(&kvm->arch.noncoherent_dma_count); 13043 } 13044 EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma); 13045 13046 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm) 13047 { 13048 atomic_dec(&kvm->arch.noncoherent_dma_count); 13049 } 13050 EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma); 13051 13052 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm) 13053 { 13054 return atomic_read(&kvm->arch.noncoherent_dma_count); 13055 } 13056 EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma); 13057 13058 bool kvm_arch_has_irq_bypass(void) 13059 { 13060 return true; 13061 } 13062 13063 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons, 13064 struct irq_bypass_producer *prod) 13065 { 13066 struct kvm_kernel_irqfd *irqfd = 13067 container_of(cons, struct kvm_kernel_irqfd, consumer); 13068 int ret; 13069 13070 irqfd->producer = prod; 13071 kvm_arch_start_assignment(irqfd->kvm); 13072 ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, 13073 prod->irq, irqfd->gsi, 1); 13074 13075 if (ret) 13076 kvm_arch_end_assignment(irqfd->kvm); 13077 13078 return ret; 13079 } 13080 13081 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons, 13082 struct irq_bypass_producer *prod) 13083 { 13084 int ret; 13085 struct kvm_kernel_irqfd *irqfd = 13086 container_of(cons, struct kvm_kernel_irqfd, consumer); 13087 13088 WARN_ON(irqfd->producer != prod); 13089 irqfd->producer = NULL; 13090 13091 /* 13092 * When producer of consumer is unregistered, we change back to 13093 * remapped mode, so we can re-use the current implementation 13094 * when the irq is masked/disabled or the consumer side (KVM 13095 * int this case doesn't want to receive the interrupts. 13096 */ 13097 ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0); 13098 if (ret) 13099 printk(KERN_INFO "irq bypass consumer (token %p) unregistration" 13100 " fails: %d\n", irqfd->consumer.token, ret); 13101 13102 kvm_arch_end_assignment(irqfd->kvm); 13103 } 13104 13105 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq, 13106 uint32_t guest_irq, bool set) 13107 { 13108 return static_call(kvm_x86_pi_update_irte)(kvm, host_irq, guest_irq, set); 13109 } 13110 13111 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old, 13112 struct kvm_kernel_irq_routing_entry *new) 13113 { 13114 if (new->type != KVM_IRQ_ROUTING_MSI) 13115 return true; 13116 13117 return !!memcmp(&old->msi, &new->msi, sizeof(new->msi)); 13118 } 13119 13120 bool kvm_vector_hashing_enabled(void) 13121 { 13122 return vector_hashing; 13123 } 13124 13125 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) 13126 { 13127 return (vcpu->arch.msr_kvm_poll_control & 1) == 0; 13128 } 13129 EXPORT_SYMBOL_GPL(kvm_arch_no_poll); 13130 13131 13132 int kvm_spec_ctrl_test_value(u64 value) 13133 { 13134 /* 13135 * test that setting IA32_SPEC_CTRL to given value 13136 * is allowed by the host processor 13137 */ 13138 13139 u64 saved_value; 13140 unsigned long flags; 13141 int ret = 0; 13142 13143 local_irq_save(flags); 13144 13145 if (rdmsrl_safe(MSR_IA32_SPEC_CTRL, &saved_value)) 13146 ret = 1; 13147 else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value)) 13148 ret = 1; 13149 else 13150 wrmsrl(MSR_IA32_SPEC_CTRL, saved_value); 13151 13152 local_irq_restore(flags); 13153 13154 return ret; 13155 } 13156 EXPORT_SYMBOL_GPL(kvm_spec_ctrl_test_value); 13157 13158 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code) 13159 { 13160 struct kvm_mmu *mmu = vcpu->arch.walk_mmu; 13161 struct x86_exception fault; 13162 u64 access = error_code & 13163 (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK); 13164 13165 if (!(error_code & PFERR_PRESENT_MASK) || 13166 mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) { 13167 /* 13168 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page 13169 * tables probably do not match the TLB. Just proceed 13170 * with the error code that the processor gave. 13171 */ 13172 fault.vector = PF_VECTOR; 13173 fault.error_code_valid = true; 13174 fault.error_code = error_code; 13175 fault.nested_page_fault = false; 13176 fault.address = gva; 13177 fault.async_page_fault = false; 13178 } 13179 vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault); 13180 } 13181 EXPORT_SYMBOL_GPL(kvm_fixup_and_inject_pf_error); 13182 13183 /* 13184 * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns 13185 * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value 13186 * indicates whether exit to userspace is needed. 13187 */ 13188 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, 13189 struct x86_exception *e) 13190 { 13191 if (r == X86EMUL_PROPAGATE_FAULT) { 13192 if (KVM_BUG_ON(!e, vcpu->kvm)) 13193 return -EIO; 13194 13195 kvm_inject_emulated_page_fault(vcpu, e); 13196 return 1; 13197 } 13198 13199 /* 13200 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED 13201 * while handling a VMX instruction KVM could've handled the request 13202 * correctly by exiting to userspace and performing I/O but there 13203 * doesn't seem to be a real use-case behind such requests, just return 13204 * KVM_EXIT_INTERNAL_ERROR for now. 13205 */ 13206 kvm_prepare_emulation_failure_exit(vcpu); 13207 13208 return 0; 13209 } 13210 EXPORT_SYMBOL_GPL(kvm_handle_memory_failure); 13211 13212 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) 13213 { 13214 bool pcid_enabled; 13215 struct x86_exception e; 13216 struct { 13217 u64 pcid; 13218 u64 gla; 13219 } operand; 13220 int r; 13221 13222 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 13223 if (r != X86EMUL_CONTINUE) 13224 return kvm_handle_memory_failure(vcpu, r, &e); 13225 13226 if (operand.pcid >> 12 != 0) { 13227 kvm_inject_gp(vcpu, 0); 13228 return 1; 13229 } 13230 13231 pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE); 13232 13233 switch (type) { 13234 case INVPCID_TYPE_INDIV_ADDR: 13235 if ((!pcid_enabled && (operand.pcid != 0)) || 13236 is_noncanonical_address(operand.gla, vcpu)) { 13237 kvm_inject_gp(vcpu, 0); 13238 return 1; 13239 } 13240 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid); 13241 return kvm_skip_emulated_instruction(vcpu); 13242 13243 case INVPCID_TYPE_SINGLE_CTXT: 13244 if (!pcid_enabled && (operand.pcid != 0)) { 13245 kvm_inject_gp(vcpu, 0); 13246 return 1; 13247 } 13248 13249 kvm_invalidate_pcid(vcpu, operand.pcid); 13250 return kvm_skip_emulated_instruction(vcpu); 13251 13252 case INVPCID_TYPE_ALL_NON_GLOBAL: 13253 /* 13254 * Currently, KVM doesn't mark global entries in the shadow 13255 * page tables, so a non-global flush just degenerates to a 13256 * global flush. If needed, we could optimize this later by 13257 * keeping track of global entries in shadow page tables. 13258 */ 13259 13260 fallthrough; 13261 case INVPCID_TYPE_ALL_INCL_GLOBAL: 13262 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 13263 return kvm_skip_emulated_instruction(vcpu); 13264 13265 default: 13266 kvm_inject_gp(vcpu, 0); 13267 return 1; 13268 } 13269 } 13270 EXPORT_SYMBOL_GPL(kvm_handle_invpcid); 13271 13272 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu) 13273 { 13274 struct kvm_run *run = vcpu->run; 13275 struct kvm_mmio_fragment *frag; 13276 unsigned int len; 13277 13278 BUG_ON(!vcpu->mmio_needed); 13279 13280 /* Complete previous fragment */ 13281 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment]; 13282 len = min(8u, frag->len); 13283 if (!vcpu->mmio_is_write) 13284 memcpy(frag->data, run->mmio.data, len); 13285 13286 if (frag->len <= 8) { 13287 /* Switch to the next fragment. */ 13288 frag++; 13289 vcpu->mmio_cur_fragment++; 13290 } else { 13291 /* Go forward to the next mmio piece. */ 13292 frag->data += len; 13293 frag->gpa += len; 13294 frag->len -= len; 13295 } 13296 13297 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) { 13298 vcpu->mmio_needed = 0; 13299 13300 // VMG change, at this point, we're always done 13301 // RIP has already been advanced 13302 return 1; 13303 } 13304 13305 // More MMIO is needed 13306 run->mmio.phys_addr = frag->gpa; 13307 run->mmio.len = min(8u, frag->len); 13308 run->mmio.is_write = vcpu->mmio_is_write; 13309 if (run->mmio.is_write) 13310 memcpy(run->mmio.data, frag->data, min(8u, frag->len)); 13311 run->exit_reason = KVM_EXIT_MMIO; 13312 13313 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; 13314 13315 return 0; 13316 } 13317 13318 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes, 13319 void *data) 13320 { 13321 int handled; 13322 struct kvm_mmio_fragment *frag; 13323 13324 if (!data) 13325 return -EINVAL; 13326 13327 handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data); 13328 if (handled == bytes) 13329 return 1; 13330 13331 bytes -= handled; 13332 gpa += handled; 13333 data += handled; 13334 13335 /*TODO: Check if need to increment number of frags */ 13336 frag = vcpu->mmio_fragments; 13337 vcpu->mmio_nr_fragments = 1; 13338 frag->len = bytes; 13339 frag->gpa = gpa; 13340 frag->data = data; 13341 13342 vcpu->mmio_needed = 1; 13343 vcpu->mmio_cur_fragment = 0; 13344 13345 vcpu->run->mmio.phys_addr = gpa; 13346 vcpu->run->mmio.len = min(8u, frag->len); 13347 vcpu->run->mmio.is_write = 1; 13348 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len)); 13349 vcpu->run->exit_reason = KVM_EXIT_MMIO; 13350 13351 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; 13352 13353 return 0; 13354 } 13355 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_write); 13356 13357 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes, 13358 void *data) 13359 { 13360 int handled; 13361 struct kvm_mmio_fragment *frag; 13362 13363 if (!data) 13364 return -EINVAL; 13365 13366 handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data); 13367 if (handled == bytes) 13368 return 1; 13369 13370 bytes -= handled; 13371 gpa += handled; 13372 data += handled; 13373 13374 /*TODO: Check if need to increment number of frags */ 13375 frag = vcpu->mmio_fragments; 13376 vcpu->mmio_nr_fragments = 1; 13377 frag->len = bytes; 13378 frag->gpa = gpa; 13379 frag->data = data; 13380 13381 vcpu->mmio_needed = 1; 13382 vcpu->mmio_cur_fragment = 0; 13383 13384 vcpu->run->mmio.phys_addr = gpa; 13385 vcpu->run->mmio.len = min(8u, frag->len); 13386 vcpu->run->mmio.is_write = 0; 13387 vcpu->run->exit_reason = KVM_EXIT_MMIO; 13388 13389 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio; 13390 13391 return 0; 13392 } 13393 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_read); 13394 13395 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size) 13396 { 13397 vcpu->arch.sev_pio_count -= count; 13398 vcpu->arch.sev_pio_data += count * size; 13399 } 13400 13401 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size, 13402 unsigned int port); 13403 13404 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu) 13405 { 13406 int size = vcpu->arch.pio.size; 13407 int port = vcpu->arch.pio.port; 13408 13409 vcpu->arch.pio.count = 0; 13410 if (vcpu->arch.sev_pio_count) 13411 return kvm_sev_es_outs(vcpu, size, port); 13412 return 1; 13413 } 13414 13415 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size, 13416 unsigned int port) 13417 { 13418 for (;;) { 13419 unsigned int count = 13420 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); 13421 int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count); 13422 13423 /* memcpy done already by emulator_pio_out. */ 13424 advance_sev_es_emulated_pio(vcpu, count, size); 13425 if (!ret) 13426 break; 13427 13428 /* Emulation done by the kernel. */ 13429 if (!vcpu->arch.sev_pio_count) 13430 return 1; 13431 } 13432 13433 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs; 13434 return 0; 13435 } 13436 13437 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size, 13438 unsigned int port); 13439 13440 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu) 13441 { 13442 unsigned count = vcpu->arch.pio.count; 13443 int size = vcpu->arch.pio.size; 13444 int port = vcpu->arch.pio.port; 13445 13446 complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data); 13447 advance_sev_es_emulated_pio(vcpu, count, size); 13448 if (vcpu->arch.sev_pio_count) 13449 return kvm_sev_es_ins(vcpu, size, port); 13450 return 1; 13451 } 13452 13453 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size, 13454 unsigned int port) 13455 { 13456 for (;;) { 13457 unsigned int count = 13458 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count); 13459 if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count)) 13460 break; 13461 13462 /* Emulation done by the kernel. */ 13463 advance_sev_es_emulated_pio(vcpu, count, size); 13464 if (!vcpu->arch.sev_pio_count) 13465 return 1; 13466 } 13467 13468 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins; 13469 return 0; 13470 } 13471 13472 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size, 13473 unsigned int port, void *data, unsigned int count, 13474 int in) 13475 { 13476 vcpu->arch.sev_pio_data = data; 13477 vcpu->arch.sev_pio_count = count; 13478 return in ? kvm_sev_es_ins(vcpu, size, port) 13479 : kvm_sev_es_outs(vcpu, size, port); 13480 } 13481 EXPORT_SYMBOL_GPL(kvm_sev_es_string_io); 13482 13483 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry); 13484 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit); 13485 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio); 13486 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); 13487 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); 13488 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); 13489 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); 13490 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter); 13491 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit); 13492 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject); 13493 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit); 13494 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed); 13495 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga); 13496 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit); 13497 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts); 13498 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset); 13499 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update); 13500 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full); 13501 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update); 13502 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access); 13503 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi); 13504 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log); 13505 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_kick_vcpu_slowpath); 13506 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_doorbell); 13507 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq); 13508 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter); 13509 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit); 13510 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter); 13511 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit); 13512 13513 static int __init kvm_x86_init(void) 13514 { 13515 kvm_mmu_x86_module_init(); 13516 return 0; 13517 } 13518 module_init(kvm_x86_init); 13519 13520 static void __exit kvm_x86_exit(void) 13521 { 13522 /* 13523 * If module_init() is implemented, module_exit() must also be 13524 * implemented to allow module unload. 13525 */ 13526 } 13527 module_exit(kvm_x86_exit); 13528