1 /* 2 * QEMU KVM support 3 * 4 * Copyright (C) 2006-2008 Qumranet Technologies 5 * Copyright IBM, Corp. 2008 6 * 7 * Authors: 8 * Anthony Liguori <aliguori@us.ibm.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2 or later. 11 * See the COPYING file in the top-level directory. 12 * 13 */ 14 15 #include "qemu/osdep.h" 16 #include "qapi/qapi-events-run-state.h" 17 #include "qapi/error.h" 18 #include <sys/ioctl.h> 19 #include <sys/utsname.h> 20 21 #include <linux/kvm.h> 22 #include "standard-headers/asm-x86/kvm_para.h" 23 24 #include "cpu.h" 25 #include "sysemu/sysemu.h" 26 #include "sysemu/hw_accel.h" 27 #include "sysemu/kvm_int.h" 28 #include "sysemu/runstate.h" 29 #include "kvm_i386.h" 30 #include "hyperv.h" 31 #include "hyperv-proto.h" 32 33 #include "exec/gdbstub.h" 34 #include "qemu/host-utils.h" 35 #include "qemu/main-loop.h" 36 #include "qemu/config-file.h" 37 #include "qemu/error-report.h" 38 #include "hw/i386/x86.h" 39 #include "hw/i386/apic.h" 40 #include "hw/i386/apic_internal.h" 41 #include "hw/i386/apic-msidef.h" 42 #include "hw/i386/intel_iommu.h" 43 #include "hw/i386/x86-iommu.h" 44 #include "hw/i386/e820_memory_layout.h" 45 #include "sysemu/sev.h" 46 47 #include "hw/pci/pci.h" 48 #include "hw/pci/msi.h" 49 #include "hw/pci/msix.h" 50 #include "migration/blocker.h" 51 #include "exec/memattrs.h" 52 #include "trace.h" 53 54 //#define DEBUG_KVM 55 56 #ifdef DEBUG_KVM 57 #define DPRINTF(fmt, ...) \ 58 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0) 59 #else 60 #define DPRINTF(fmt, ...) \ 61 do { } while (0) 62 #endif 63 64 /* From arch/x86/kvm/lapic.h */ 65 #define KVM_APIC_BUS_CYCLE_NS 1 66 #define KVM_APIC_BUS_FREQUENCY (1000000000ULL / KVM_APIC_BUS_CYCLE_NS) 67 68 #define MSR_KVM_WALL_CLOCK 0x11 69 #define MSR_KVM_SYSTEM_TIME 0x12 70 71 /* A 4096-byte buffer can hold the 8-byte kvm_msrs header, plus 72 * 255 kvm_msr_entry structs */ 73 #define MSR_BUF_SIZE 4096 74 75 static void kvm_init_msrs(X86CPU *cpu); 76 77 const KVMCapabilityInfo kvm_arch_required_capabilities[] = { 78 KVM_CAP_INFO(SET_TSS_ADDR), 79 KVM_CAP_INFO(EXT_CPUID), 80 KVM_CAP_INFO(MP_STATE), 81 KVM_CAP_LAST_INFO 82 }; 83 84 static bool has_msr_star; 85 static bool has_msr_hsave_pa; 86 static bool has_msr_tsc_aux; 87 static bool has_msr_tsc_adjust; 88 static bool has_msr_tsc_deadline; 89 static bool has_msr_feature_control; 90 static bool has_msr_misc_enable; 91 static bool has_msr_smbase; 92 static bool has_msr_bndcfgs; 93 static int lm_capable_kernel; 94 static bool has_msr_hv_hypercall; 95 static bool has_msr_hv_crash; 96 static bool has_msr_hv_reset; 97 static bool has_msr_hv_vpindex; 98 static bool hv_vpindex_settable; 99 static bool has_msr_hv_runtime; 100 static bool has_msr_hv_synic; 101 static bool has_msr_hv_stimer; 102 static bool has_msr_hv_frequencies; 103 static bool has_msr_hv_reenlightenment; 104 static bool has_msr_xss; 105 static bool has_msr_umwait; 106 static bool has_msr_spec_ctrl; 107 static bool has_msr_tsx_ctrl; 108 static bool has_msr_virt_ssbd; 109 static bool has_msr_smi_count; 110 static bool has_msr_arch_capabs; 111 static bool has_msr_core_capabs; 112 static bool has_msr_vmx_vmfunc; 113 static bool has_msr_ucode_rev; 114 static bool has_msr_vmx_procbased_ctls2; 115 static bool has_msr_perf_capabs; 116 static bool has_msr_pkrs; 117 118 static uint32_t has_architectural_pmu_version; 119 static uint32_t num_architectural_pmu_gp_counters; 120 static uint32_t num_architectural_pmu_fixed_counters; 121 122 static int has_xsave; 123 static int has_xcrs; 124 static int has_pit_state2; 125 static int has_exception_payload; 126 127 static bool has_msr_mcg_ext_ctl; 128 129 static struct kvm_cpuid2 *cpuid_cache; 130 static struct kvm_msr_list *kvm_feature_msrs; 131 132 int kvm_has_pit_state2(void) 133 { 134 return has_pit_state2; 135 } 136 137 bool kvm_has_smm(void) 138 { 139 return kvm_check_extension(kvm_state, KVM_CAP_X86_SMM); 140 } 141 142 bool kvm_has_adjust_clock_stable(void) 143 { 144 int ret = kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK); 145 146 return (ret == KVM_CLOCK_TSC_STABLE); 147 } 148 149 bool kvm_has_adjust_clock(void) 150 { 151 return kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK); 152 } 153 154 bool kvm_has_exception_payload(void) 155 { 156 return has_exception_payload; 157 } 158 159 static bool kvm_x2apic_api_set_flags(uint64_t flags) 160 { 161 KVMState *s = KVM_STATE(current_accel()); 162 163 return !kvm_vm_enable_cap(s, KVM_CAP_X2APIC_API, 0, flags); 164 } 165 166 #define MEMORIZE(fn, _result) \ 167 ({ \ 168 static bool _memorized; \ 169 \ 170 if (_memorized) { \ 171 return _result; \ 172 } \ 173 _memorized = true; \ 174 _result = fn; \ 175 }) 176 177 static bool has_x2apic_api; 178 179 bool kvm_has_x2apic_api(void) 180 { 181 return has_x2apic_api; 182 } 183 184 bool kvm_enable_x2apic(void) 185 { 186 return MEMORIZE( 187 kvm_x2apic_api_set_flags(KVM_X2APIC_API_USE_32BIT_IDS | 188 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK), 189 has_x2apic_api); 190 } 191 192 bool kvm_hv_vpindex_settable(void) 193 { 194 return hv_vpindex_settable; 195 } 196 197 static int kvm_get_tsc(CPUState *cs) 198 { 199 X86CPU *cpu = X86_CPU(cs); 200 CPUX86State *env = &cpu->env; 201 struct { 202 struct kvm_msrs info; 203 struct kvm_msr_entry entries[1]; 204 } msr_data = {}; 205 int ret; 206 207 if (env->tsc_valid) { 208 return 0; 209 } 210 211 memset(&msr_data, 0, sizeof(msr_data)); 212 msr_data.info.nmsrs = 1; 213 msr_data.entries[0].index = MSR_IA32_TSC; 214 env->tsc_valid = !runstate_is_running(); 215 216 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data); 217 if (ret < 0) { 218 return ret; 219 } 220 221 assert(ret == 1); 222 env->tsc = msr_data.entries[0].data; 223 return 0; 224 } 225 226 static inline void do_kvm_synchronize_tsc(CPUState *cpu, run_on_cpu_data arg) 227 { 228 kvm_get_tsc(cpu); 229 } 230 231 void kvm_synchronize_all_tsc(void) 232 { 233 CPUState *cpu; 234 235 if (kvm_enabled()) { 236 CPU_FOREACH(cpu) { 237 run_on_cpu(cpu, do_kvm_synchronize_tsc, RUN_ON_CPU_NULL); 238 } 239 } 240 } 241 242 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max) 243 { 244 struct kvm_cpuid2 *cpuid; 245 int r, size; 246 247 size = sizeof(*cpuid) + max * sizeof(*cpuid->entries); 248 cpuid = g_malloc0(size); 249 cpuid->nent = max; 250 r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid); 251 if (r == 0 && cpuid->nent >= max) { 252 r = -E2BIG; 253 } 254 if (r < 0) { 255 if (r == -E2BIG) { 256 g_free(cpuid); 257 return NULL; 258 } else { 259 fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n", 260 strerror(-r)); 261 exit(1); 262 } 263 } 264 return cpuid; 265 } 266 267 /* Run KVM_GET_SUPPORTED_CPUID ioctl(), allocating a buffer large enough 268 * for all entries. 269 */ 270 static struct kvm_cpuid2 *get_supported_cpuid(KVMState *s) 271 { 272 struct kvm_cpuid2 *cpuid; 273 int max = 1; 274 275 if (cpuid_cache != NULL) { 276 return cpuid_cache; 277 } 278 while ((cpuid = try_get_cpuid(s, max)) == NULL) { 279 max *= 2; 280 } 281 cpuid_cache = cpuid; 282 return cpuid; 283 } 284 285 static bool host_tsx_broken(void) 286 { 287 int family, model, stepping;\ 288 char vendor[CPUID_VENDOR_SZ + 1]; 289 290 host_vendor_fms(vendor, &family, &model, &stepping); 291 292 /* Check if we are running on a Haswell host known to have broken TSX */ 293 return !strcmp(vendor, CPUID_VENDOR_INTEL) && 294 (family == 6) && 295 ((model == 63 && stepping < 4) || 296 model == 60 || model == 69 || model == 70); 297 } 298 299 /* Returns the value for a specific register on the cpuid entry 300 */ 301 static uint32_t cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, int reg) 302 { 303 uint32_t ret = 0; 304 switch (reg) { 305 case R_EAX: 306 ret = entry->eax; 307 break; 308 case R_EBX: 309 ret = entry->ebx; 310 break; 311 case R_ECX: 312 ret = entry->ecx; 313 break; 314 case R_EDX: 315 ret = entry->edx; 316 break; 317 } 318 return ret; 319 } 320 321 /* Find matching entry for function/index on kvm_cpuid2 struct 322 */ 323 static struct kvm_cpuid_entry2 *cpuid_find_entry(struct kvm_cpuid2 *cpuid, 324 uint32_t function, 325 uint32_t index) 326 { 327 int i; 328 for (i = 0; i < cpuid->nent; ++i) { 329 if (cpuid->entries[i].function == function && 330 cpuid->entries[i].index == index) { 331 return &cpuid->entries[i]; 332 } 333 } 334 /* not found: */ 335 return NULL; 336 } 337 338 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function, 339 uint32_t index, int reg) 340 { 341 struct kvm_cpuid2 *cpuid; 342 uint32_t ret = 0; 343 uint32_t cpuid_1_edx; 344 345 cpuid = get_supported_cpuid(s); 346 347 struct kvm_cpuid_entry2 *entry = cpuid_find_entry(cpuid, function, index); 348 if (entry) { 349 ret = cpuid_entry_get_reg(entry, reg); 350 } 351 352 /* Fixups for the data returned by KVM, below */ 353 354 if (function == 1 && reg == R_EDX) { 355 /* KVM before 2.6.30 misreports the following features */ 356 ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA; 357 } else if (function == 1 && reg == R_ECX) { 358 /* We can set the hypervisor flag, even if KVM does not return it on 359 * GET_SUPPORTED_CPUID 360 */ 361 ret |= CPUID_EXT_HYPERVISOR; 362 /* tsc-deadline flag is not returned by GET_SUPPORTED_CPUID, but it 363 * can be enabled if the kernel has KVM_CAP_TSC_DEADLINE_TIMER, 364 * and the irqchip is in the kernel. 365 */ 366 if (kvm_irqchip_in_kernel() && 367 kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) { 368 ret |= CPUID_EXT_TSC_DEADLINE_TIMER; 369 } 370 371 /* x2apic is reported by GET_SUPPORTED_CPUID, but it can't be enabled 372 * without the in-kernel irqchip 373 */ 374 if (!kvm_irqchip_in_kernel()) { 375 ret &= ~CPUID_EXT_X2APIC; 376 } 377 378 if (enable_cpu_pm) { 379 int disable_exits = kvm_check_extension(s, 380 KVM_CAP_X86_DISABLE_EXITS); 381 382 if (disable_exits & KVM_X86_DISABLE_EXITS_MWAIT) { 383 ret |= CPUID_EXT_MONITOR; 384 } 385 } 386 } else if (function == 6 && reg == R_EAX) { 387 ret |= CPUID_6_EAX_ARAT; /* safe to allow because of emulated APIC */ 388 } else if (function == 7 && index == 0 && reg == R_EBX) { 389 if (host_tsx_broken()) { 390 ret &= ~(CPUID_7_0_EBX_RTM | CPUID_7_0_EBX_HLE); 391 } 392 } else if (function == 7 && index == 0 && reg == R_EDX) { 393 /* 394 * Linux v4.17-v4.20 incorrectly return ARCH_CAPABILITIES on SVM hosts. 395 * We can detect the bug by checking if MSR_IA32_ARCH_CAPABILITIES is 396 * returned by KVM_GET_MSR_INDEX_LIST. 397 */ 398 if (!has_msr_arch_capabs) { 399 ret &= ~CPUID_7_0_EDX_ARCH_CAPABILITIES; 400 } 401 } else if (function == 0x80000001 && reg == R_ECX) { 402 /* 403 * It's safe to enable TOPOEXT even if it's not returned by 404 * GET_SUPPORTED_CPUID. Unconditionally enabling TOPOEXT here allows 405 * us to keep CPU models including TOPOEXT runnable on older kernels. 406 */ 407 ret |= CPUID_EXT3_TOPOEXT; 408 } else if (function == 0x80000001 && reg == R_EDX) { 409 /* On Intel, kvm returns cpuid according to the Intel spec, 410 * so add missing bits according to the AMD spec: 411 */ 412 cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX); 413 ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES; 414 } else if (function == KVM_CPUID_FEATURES && reg == R_EAX) { 415 /* kvm_pv_unhalt is reported by GET_SUPPORTED_CPUID, but it can't 416 * be enabled without the in-kernel irqchip 417 */ 418 if (!kvm_irqchip_in_kernel()) { 419 ret &= ~(1U << KVM_FEATURE_PV_UNHALT); 420 } 421 if (kvm_irqchip_is_split()) { 422 ret |= 1U << KVM_FEATURE_MSI_EXT_DEST_ID; 423 } 424 } else if (function == KVM_CPUID_FEATURES && reg == R_EDX) { 425 ret |= 1U << KVM_HINTS_REALTIME; 426 } 427 428 return ret; 429 } 430 431 uint64_t kvm_arch_get_supported_msr_feature(KVMState *s, uint32_t index) 432 { 433 struct { 434 struct kvm_msrs info; 435 struct kvm_msr_entry entries[1]; 436 } msr_data = {}; 437 uint64_t value; 438 uint32_t ret, can_be_one, must_be_one; 439 440 if (kvm_feature_msrs == NULL) { /* Host doesn't support feature MSRs */ 441 return 0; 442 } 443 444 /* Check if requested MSR is supported feature MSR */ 445 int i; 446 for (i = 0; i < kvm_feature_msrs->nmsrs; i++) 447 if (kvm_feature_msrs->indices[i] == index) { 448 break; 449 } 450 if (i == kvm_feature_msrs->nmsrs) { 451 return 0; /* if the feature MSR is not supported, simply return 0 */ 452 } 453 454 msr_data.info.nmsrs = 1; 455 msr_data.entries[0].index = index; 456 457 ret = kvm_ioctl(s, KVM_GET_MSRS, &msr_data); 458 if (ret != 1) { 459 error_report("KVM get MSR (index=0x%x) feature failed, %s", 460 index, strerror(-ret)); 461 exit(1); 462 } 463 464 value = msr_data.entries[0].data; 465 switch (index) { 466 case MSR_IA32_VMX_PROCBASED_CTLS2: 467 if (!has_msr_vmx_procbased_ctls2) { 468 /* KVM forgot to add these bits for some time, do this ourselves. */ 469 if (kvm_arch_get_supported_cpuid(s, 0xD, 1, R_ECX) & 470 CPUID_XSAVE_XSAVES) { 471 value |= (uint64_t)VMX_SECONDARY_EXEC_XSAVES << 32; 472 } 473 if (kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX) & 474 CPUID_EXT_RDRAND) { 475 value |= (uint64_t)VMX_SECONDARY_EXEC_RDRAND_EXITING << 32; 476 } 477 if (kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX) & 478 CPUID_7_0_EBX_INVPCID) { 479 value |= (uint64_t)VMX_SECONDARY_EXEC_ENABLE_INVPCID << 32; 480 } 481 if (kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX) & 482 CPUID_7_0_EBX_RDSEED) { 483 value |= (uint64_t)VMX_SECONDARY_EXEC_RDSEED_EXITING << 32; 484 } 485 if (kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_EDX) & 486 CPUID_EXT2_RDTSCP) { 487 value |= (uint64_t)VMX_SECONDARY_EXEC_RDTSCP << 32; 488 } 489 } 490 /* fall through */ 491 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 492 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 493 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 494 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 495 /* 496 * Return true for bits that can be one, but do not have to be one. 497 * The SDM tells us which bits could have a "must be one" setting, 498 * so we can do the opposite transformation in make_vmx_msr_value. 499 */ 500 must_be_one = (uint32_t)value; 501 can_be_one = (uint32_t)(value >> 32); 502 return can_be_one & ~must_be_one; 503 504 default: 505 return value; 506 } 507 } 508 509 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap, 510 int *max_banks) 511 { 512 int r; 513 514 r = kvm_check_extension(s, KVM_CAP_MCE); 515 if (r > 0) { 516 *max_banks = r; 517 return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap); 518 } 519 return -ENOSYS; 520 } 521 522 static void kvm_mce_inject(X86CPU *cpu, hwaddr paddr, int code) 523 { 524 CPUState *cs = CPU(cpu); 525 CPUX86State *env = &cpu->env; 526 uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN | 527 MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S; 528 uint64_t mcg_status = MCG_STATUS_MCIP; 529 int flags = 0; 530 531 if (code == BUS_MCEERR_AR) { 532 status |= MCI_STATUS_AR | 0x134; 533 mcg_status |= MCG_STATUS_EIPV; 534 } else { 535 status |= 0xc0; 536 mcg_status |= MCG_STATUS_RIPV; 537 } 538 539 flags = cpu_x86_support_mca_broadcast(env) ? MCE_INJECT_BROADCAST : 0; 540 /* We need to read back the value of MSR_EXT_MCG_CTL that was set by the 541 * guest kernel back into env->mcg_ext_ctl. 542 */ 543 cpu_synchronize_state(cs); 544 if (env->mcg_ext_ctl & MCG_EXT_CTL_LMCE_EN) { 545 mcg_status |= MCG_STATUS_LMCE; 546 flags = 0; 547 } 548 549 cpu_x86_inject_mce(NULL, cpu, 9, status, mcg_status, paddr, 550 (MCM_ADDR_PHYS << 6) | 0xc, flags); 551 } 552 553 static void emit_hypervisor_memory_failure(MemoryFailureAction action, bool ar) 554 { 555 MemoryFailureFlags mff = {.action_required = ar, .recursive = false}; 556 557 qapi_event_send_memory_failure(MEMORY_FAILURE_RECIPIENT_HYPERVISOR, action, 558 &mff); 559 } 560 561 static void hardware_memory_error(void *host_addr) 562 { 563 emit_hypervisor_memory_failure(MEMORY_FAILURE_ACTION_FATAL, true); 564 error_report("QEMU got Hardware memory error at addr %p", host_addr); 565 exit(1); 566 } 567 568 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr) 569 { 570 X86CPU *cpu = X86_CPU(c); 571 CPUX86State *env = &cpu->env; 572 ram_addr_t ram_addr; 573 hwaddr paddr; 574 575 /* If we get an action required MCE, it has been injected by KVM 576 * while the VM was running. An action optional MCE instead should 577 * be coming from the main thread, which qemu_init_sigbus identifies 578 * as the "early kill" thread. 579 */ 580 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO); 581 582 if ((env->mcg_cap & MCG_SER_P) && addr) { 583 ram_addr = qemu_ram_addr_from_host(addr); 584 if (ram_addr != RAM_ADDR_INVALID && 585 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) { 586 kvm_hwpoison_page_add(ram_addr); 587 kvm_mce_inject(cpu, paddr, code); 588 589 /* 590 * Use different logging severity based on error type. 591 * If there is additional MCE reporting on the hypervisor, QEMU VA 592 * could be another source to identify the PA and MCE details. 593 */ 594 if (code == BUS_MCEERR_AR) { 595 error_report("Guest MCE Memory Error at QEMU addr %p and " 596 "GUEST addr 0x%" HWADDR_PRIx " of type %s injected", 597 addr, paddr, "BUS_MCEERR_AR"); 598 } else { 599 warn_report("Guest MCE Memory Error at QEMU addr %p and " 600 "GUEST addr 0x%" HWADDR_PRIx " of type %s injected", 601 addr, paddr, "BUS_MCEERR_AO"); 602 } 603 604 return; 605 } 606 607 if (code == BUS_MCEERR_AO) { 608 warn_report("Hardware memory error at addr %p of type %s " 609 "for memory used by QEMU itself instead of guest system!", 610 addr, "BUS_MCEERR_AO"); 611 } 612 } 613 614 if (code == BUS_MCEERR_AR) { 615 hardware_memory_error(addr); 616 } 617 618 /* Hope we are lucky for AO MCE, just notify a event */ 619 emit_hypervisor_memory_failure(MEMORY_FAILURE_ACTION_IGNORE, false); 620 } 621 622 static void kvm_reset_exception(CPUX86State *env) 623 { 624 env->exception_nr = -1; 625 env->exception_pending = 0; 626 env->exception_injected = 0; 627 env->exception_has_payload = false; 628 env->exception_payload = 0; 629 } 630 631 static void kvm_queue_exception(CPUX86State *env, 632 int32_t exception_nr, 633 uint8_t exception_has_payload, 634 uint64_t exception_payload) 635 { 636 assert(env->exception_nr == -1); 637 assert(!env->exception_pending); 638 assert(!env->exception_injected); 639 assert(!env->exception_has_payload); 640 641 env->exception_nr = exception_nr; 642 643 if (has_exception_payload) { 644 env->exception_pending = 1; 645 646 env->exception_has_payload = exception_has_payload; 647 env->exception_payload = exception_payload; 648 } else { 649 env->exception_injected = 1; 650 651 if (exception_nr == EXCP01_DB) { 652 assert(exception_has_payload); 653 env->dr[6] = exception_payload; 654 } else if (exception_nr == EXCP0E_PAGE) { 655 assert(exception_has_payload); 656 env->cr[2] = exception_payload; 657 } else { 658 assert(!exception_has_payload); 659 } 660 } 661 } 662 663 static int kvm_inject_mce_oldstyle(X86CPU *cpu) 664 { 665 CPUX86State *env = &cpu->env; 666 667 if (!kvm_has_vcpu_events() && env->exception_nr == EXCP12_MCHK) { 668 unsigned int bank, bank_num = env->mcg_cap & 0xff; 669 struct kvm_x86_mce mce; 670 671 kvm_reset_exception(env); 672 673 /* 674 * There must be at least one bank in use if an MCE is pending. 675 * Find it and use its values for the event injection. 676 */ 677 for (bank = 0; bank < bank_num; bank++) { 678 if (env->mce_banks[bank * 4 + 1] & MCI_STATUS_VAL) { 679 break; 680 } 681 } 682 assert(bank < bank_num); 683 684 mce.bank = bank; 685 mce.status = env->mce_banks[bank * 4 + 1]; 686 mce.mcg_status = env->mcg_status; 687 mce.addr = env->mce_banks[bank * 4 + 2]; 688 mce.misc = env->mce_banks[bank * 4 + 3]; 689 690 return kvm_vcpu_ioctl(CPU(cpu), KVM_X86_SET_MCE, &mce); 691 } 692 return 0; 693 } 694 695 static void cpu_update_state(void *opaque, int running, RunState state) 696 { 697 CPUX86State *env = opaque; 698 699 if (running) { 700 env->tsc_valid = false; 701 } 702 } 703 704 unsigned long kvm_arch_vcpu_id(CPUState *cs) 705 { 706 X86CPU *cpu = X86_CPU(cs); 707 return cpu->apic_id; 708 } 709 710 #ifndef KVM_CPUID_SIGNATURE_NEXT 711 #define KVM_CPUID_SIGNATURE_NEXT 0x40000100 712 #endif 713 714 static bool hyperv_enabled(X86CPU *cpu) 715 { 716 CPUState *cs = CPU(cpu); 717 return kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0 && 718 ((cpu->hyperv_spinlock_attempts != HYPERV_SPINLOCK_NEVER_NOTIFY) || 719 cpu->hyperv_features || cpu->hyperv_passthrough); 720 } 721 722 /* 723 * Check whether target_freq is within conservative 724 * ntp correctable bounds (250ppm) of freq 725 */ 726 static inline bool freq_within_bounds(int freq, int target_freq) 727 { 728 int max_freq = freq + (freq * 250 / 1000000); 729 int min_freq = freq - (freq * 250 / 1000000); 730 731 if (target_freq >= min_freq && target_freq <= max_freq) { 732 return true; 733 } 734 735 return false; 736 } 737 738 static int kvm_arch_set_tsc_khz(CPUState *cs) 739 { 740 X86CPU *cpu = X86_CPU(cs); 741 CPUX86State *env = &cpu->env; 742 int r, cur_freq; 743 bool set_ioctl = false; 744 745 if (!env->tsc_khz) { 746 return 0; 747 } 748 749 cur_freq = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ? 750 kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) : -ENOTSUP; 751 752 /* 753 * If TSC scaling is supported, attempt to set TSC frequency. 754 */ 755 if (kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL)) { 756 set_ioctl = true; 757 } 758 759 /* 760 * If desired TSC frequency is within bounds of NTP correction, 761 * attempt to set TSC frequency. 762 */ 763 if (cur_freq != -ENOTSUP && freq_within_bounds(cur_freq, env->tsc_khz)) { 764 set_ioctl = true; 765 } 766 767 r = set_ioctl ? 768 kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz) : 769 -ENOTSUP; 770 771 if (r < 0) { 772 /* When KVM_SET_TSC_KHZ fails, it's an error only if the current 773 * TSC frequency doesn't match the one we want. 774 */ 775 cur_freq = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ? 776 kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) : 777 -ENOTSUP; 778 if (cur_freq <= 0 || cur_freq != env->tsc_khz) { 779 warn_report("TSC frequency mismatch between " 780 "VM (%" PRId64 " kHz) and host (%d kHz), " 781 "and TSC scaling unavailable", 782 env->tsc_khz, cur_freq); 783 return r; 784 } 785 } 786 787 return 0; 788 } 789 790 static bool tsc_is_stable_and_known(CPUX86State *env) 791 { 792 if (!env->tsc_khz) { 793 return false; 794 } 795 return (env->features[FEAT_8000_0007_EDX] & CPUID_APM_INVTSC) 796 || env->user_tsc_khz; 797 } 798 799 static struct { 800 const char *desc; 801 struct { 802 uint32_t fw; 803 uint32_t bits; 804 } flags[2]; 805 uint64_t dependencies; 806 } kvm_hyperv_properties[] = { 807 [HYPERV_FEAT_RELAXED] = { 808 .desc = "relaxed timing (hv-relaxed)", 809 .flags = { 810 {.fw = FEAT_HYPERV_EAX, 811 .bits = HV_HYPERCALL_AVAILABLE}, 812 {.fw = FEAT_HV_RECOMM_EAX, 813 .bits = HV_RELAXED_TIMING_RECOMMENDED} 814 } 815 }, 816 [HYPERV_FEAT_VAPIC] = { 817 .desc = "virtual APIC (hv-vapic)", 818 .flags = { 819 {.fw = FEAT_HYPERV_EAX, 820 .bits = HV_HYPERCALL_AVAILABLE | HV_APIC_ACCESS_AVAILABLE}, 821 {.fw = FEAT_HV_RECOMM_EAX, 822 .bits = HV_APIC_ACCESS_RECOMMENDED} 823 } 824 }, 825 [HYPERV_FEAT_TIME] = { 826 .desc = "clocksources (hv-time)", 827 .flags = { 828 {.fw = FEAT_HYPERV_EAX, 829 .bits = HV_HYPERCALL_AVAILABLE | HV_TIME_REF_COUNT_AVAILABLE | 830 HV_REFERENCE_TSC_AVAILABLE} 831 } 832 }, 833 [HYPERV_FEAT_CRASH] = { 834 .desc = "crash MSRs (hv-crash)", 835 .flags = { 836 {.fw = FEAT_HYPERV_EDX, 837 .bits = HV_GUEST_CRASH_MSR_AVAILABLE} 838 } 839 }, 840 [HYPERV_FEAT_RESET] = { 841 .desc = "reset MSR (hv-reset)", 842 .flags = { 843 {.fw = FEAT_HYPERV_EAX, 844 .bits = HV_RESET_AVAILABLE} 845 } 846 }, 847 [HYPERV_FEAT_VPINDEX] = { 848 .desc = "VP_INDEX MSR (hv-vpindex)", 849 .flags = { 850 {.fw = FEAT_HYPERV_EAX, 851 .bits = HV_VP_INDEX_AVAILABLE} 852 } 853 }, 854 [HYPERV_FEAT_RUNTIME] = { 855 .desc = "VP_RUNTIME MSR (hv-runtime)", 856 .flags = { 857 {.fw = FEAT_HYPERV_EAX, 858 .bits = HV_VP_RUNTIME_AVAILABLE} 859 } 860 }, 861 [HYPERV_FEAT_SYNIC] = { 862 .desc = "synthetic interrupt controller (hv-synic)", 863 .flags = { 864 {.fw = FEAT_HYPERV_EAX, 865 .bits = HV_SYNIC_AVAILABLE} 866 } 867 }, 868 [HYPERV_FEAT_STIMER] = { 869 .desc = "synthetic timers (hv-stimer)", 870 .flags = { 871 {.fw = FEAT_HYPERV_EAX, 872 .bits = HV_SYNTIMERS_AVAILABLE} 873 }, 874 .dependencies = BIT(HYPERV_FEAT_SYNIC) | BIT(HYPERV_FEAT_TIME) 875 }, 876 [HYPERV_FEAT_FREQUENCIES] = { 877 .desc = "frequency MSRs (hv-frequencies)", 878 .flags = { 879 {.fw = FEAT_HYPERV_EAX, 880 .bits = HV_ACCESS_FREQUENCY_MSRS}, 881 {.fw = FEAT_HYPERV_EDX, 882 .bits = HV_FREQUENCY_MSRS_AVAILABLE} 883 } 884 }, 885 [HYPERV_FEAT_REENLIGHTENMENT] = { 886 .desc = "reenlightenment MSRs (hv-reenlightenment)", 887 .flags = { 888 {.fw = FEAT_HYPERV_EAX, 889 .bits = HV_ACCESS_REENLIGHTENMENTS_CONTROL} 890 } 891 }, 892 [HYPERV_FEAT_TLBFLUSH] = { 893 .desc = "paravirtualized TLB flush (hv-tlbflush)", 894 .flags = { 895 {.fw = FEAT_HV_RECOMM_EAX, 896 .bits = HV_REMOTE_TLB_FLUSH_RECOMMENDED | 897 HV_EX_PROCESSOR_MASKS_RECOMMENDED} 898 }, 899 .dependencies = BIT(HYPERV_FEAT_VPINDEX) 900 }, 901 [HYPERV_FEAT_EVMCS] = { 902 .desc = "enlightened VMCS (hv-evmcs)", 903 .flags = { 904 {.fw = FEAT_HV_RECOMM_EAX, 905 .bits = HV_ENLIGHTENED_VMCS_RECOMMENDED} 906 }, 907 .dependencies = BIT(HYPERV_FEAT_VAPIC) 908 }, 909 [HYPERV_FEAT_IPI] = { 910 .desc = "paravirtualized IPI (hv-ipi)", 911 .flags = { 912 {.fw = FEAT_HV_RECOMM_EAX, 913 .bits = HV_CLUSTER_IPI_RECOMMENDED | 914 HV_EX_PROCESSOR_MASKS_RECOMMENDED} 915 }, 916 .dependencies = BIT(HYPERV_FEAT_VPINDEX) 917 }, 918 [HYPERV_FEAT_STIMER_DIRECT] = { 919 .desc = "direct mode synthetic timers (hv-stimer-direct)", 920 .flags = { 921 {.fw = FEAT_HYPERV_EDX, 922 .bits = HV_STIMER_DIRECT_MODE_AVAILABLE} 923 }, 924 .dependencies = BIT(HYPERV_FEAT_STIMER) 925 }, 926 }; 927 928 static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max) 929 { 930 struct kvm_cpuid2 *cpuid; 931 int r, size; 932 933 size = sizeof(*cpuid) + max * sizeof(*cpuid->entries); 934 cpuid = g_malloc0(size); 935 cpuid->nent = max; 936 937 r = kvm_vcpu_ioctl(cs, KVM_GET_SUPPORTED_HV_CPUID, cpuid); 938 if (r == 0 && cpuid->nent >= max) { 939 r = -E2BIG; 940 } 941 if (r < 0) { 942 if (r == -E2BIG) { 943 g_free(cpuid); 944 return NULL; 945 } else { 946 fprintf(stderr, "KVM_GET_SUPPORTED_HV_CPUID failed: %s\n", 947 strerror(-r)); 948 exit(1); 949 } 950 } 951 return cpuid; 952 } 953 954 /* 955 * Run KVM_GET_SUPPORTED_HV_CPUID ioctl(), allocating a buffer large enough 956 * for all entries. 957 */ 958 static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs) 959 { 960 struct kvm_cpuid2 *cpuid; 961 int max = 7; /* 0x40000000..0x40000005, 0x4000000A */ 962 963 /* 964 * When the buffer is too small, KVM_GET_SUPPORTED_HV_CPUID fails with 965 * -E2BIG, however, it doesn't report back the right size. Keep increasing 966 * it and re-trying until we succeed. 967 */ 968 while ((cpuid = try_get_hv_cpuid(cs, max)) == NULL) { 969 max++; 970 } 971 return cpuid; 972 } 973 974 /* 975 * When KVM_GET_SUPPORTED_HV_CPUID is not supported we fill CPUID feature 976 * leaves from KVM_CAP_HYPERV* and present MSRs data. 977 */ 978 static struct kvm_cpuid2 *get_supported_hv_cpuid_legacy(CPUState *cs) 979 { 980 X86CPU *cpu = X86_CPU(cs); 981 struct kvm_cpuid2 *cpuid; 982 struct kvm_cpuid_entry2 *entry_feat, *entry_recomm; 983 984 /* HV_CPUID_FEATURES, HV_CPUID_ENLIGHTMENT_INFO */ 985 cpuid = g_malloc0(sizeof(*cpuid) + 2 * sizeof(*cpuid->entries)); 986 cpuid->nent = 2; 987 988 /* HV_CPUID_VENDOR_AND_MAX_FUNCTIONS */ 989 entry_feat = &cpuid->entries[0]; 990 entry_feat->function = HV_CPUID_FEATURES; 991 992 entry_recomm = &cpuid->entries[1]; 993 entry_recomm->function = HV_CPUID_ENLIGHTMENT_INFO; 994 entry_recomm->ebx = cpu->hyperv_spinlock_attempts; 995 996 if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0) { 997 entry_feat->eax |= HV_HYPERCALL_AVAILABLE; 998 entry_feat->eax |= HV_APIC_ACCESS_AVAILABLE; 999 entry_feat->edx |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE; 1000 entry_recomm->eax |= HV_RELAXED_TIMING_RECOMMENDED; 1001 entry_recomm->eax |= HV_APIC_ACCESS_RECOMMENDED; 1002 } 1003 1004 if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_TIME) > 0) { 1005 entry_feat->eax |= HV_TIME_REF_COUNT_AVAILABLE; 1006 entry_feat->eax |= HV_REFERENCE_TSC_AVAILABLE; 1007 } 1008 1009 if (has_msr_hv_frequencies) { 1010 entry_feat->eax |= HV_ACCESS_FREQUENCY_MSRS; 1011 entry_feat->edx |= HV_FREQUENCY_MSRS_AVAILABLE; 1012 } 1013 1014 if (has_msr_hv_crash) { 1015 entry_feat->edx |= HV_GUEST_CRASH_MSR_AVAILABLE; 1016 } 1017 1018 if (has_msr_hv_reenlightenment) { 1019 entry_feat->eax |= HV_ACCESS_REENLIGHTENMENTS_CONTROL; 1020 } 1021 1022 if (has_msr_hv_reset) { 1023 entry_feat->eax |= HV_RESET_AVAILABLE; 1024 } 1025 1026 if (has_msr_hv_vpindex) { 1027 entry_feat->eax |= HV_VP_INDEX_AVAILABLE; 1028 } 1029 1030 if (has_msr_hv_runtime) { 1031 entry_feat->eax |= HV_VP_RUNTIME_AVAILABLE; 1032 } 1033 1034 if (has_msr_hv_synic) { 1035 unsigned int cap = cpu->hyperv_synic_kvm_only ? 1036 KVM_CAP_HYPERV_SYNIC : KVM_CAP_HYPERV_SYNIC2; 1037 1038 if (kvm_check_extension(cs->kvm_state, cap) > 0) { 1039 entry_feat->eax |= HV_SYNIC_AVAILABLE; 1040 } 1041 } 1042 1043 if (has_msr_hv_stimer) { 1044 entry_feat->eax |= HV_SYNTIMERS_AVAILABLE; 1045 } 1046 1047 if (kvm_check_extension(cs->kvm_state, 1048 KVM_CAP_HYPERV_TLBFLUSH) > 0) { 1049 entry_recomm->eax |= HV_REMOTE_TLB_FLUSH_RECOMMENDED; 1050 entry_recomm->eax |= HV_EX_PROCESSOR_MASKS_RECOMMENDED; 1051 } 1052 1053 if (kvm_check_extension(cs->kvm_state, 1054 KVM_CAP_HYPERV_ENLIGHTENED_VMCS) > 0) { 1055 entry_recomm->eax |= HV_ENLIGHTENED_VMCS_RECOMMENDED; 1056 } 1057 1058 if (kvm_check_extension(cs->kvm_state, 1059 KVM_CAP_HYPERV_SEND_IPI) > 0) { 1060 entry_recomm->eax |= HV_CLUSTER_IPI_RECOMMENDED; 1061 entry_recomm->eax |= HV_EX_PROCESSOR_MASKS_RECOMMENDED; 1062 } 1063 1064 return cpuid; 1065 } 1066 1067 static int hv_cpuid_get_fw(struct kvm_cpuid2 *cpuid, int fw, uint32_t *r) 1068 { 1069 struct kvm_cpuid_entry2 *entry; 1070 uint32_t func; 1071 int reg; 1072 1073 switch (fw) { 1074 case FEAT_HYPERV_EAX: 1075 reg = R_EAX; 1076 func = HV_CPUID_FEATURES; 1077 break; 1078 case FEAT_HYPERV_EDX: 1079 reg = R_EDX; 1080 func = HV_CPUID_FEATURES; 1081 break; 1082 case FEAT_HV_RECOMM_EAX: 1083 reg = R_EAX; 1084 func = HV_CPUID_ENLIGHTMENT_INFO; 1085 break; 1086 default: 1087 return -EINVAL; 1088 } 1089 1090 entry = cpuid_find_entry(cpuid, func, 0); 1091 if (!entry) { 1092 return -ENOENT; 1093 } 1094 1095 switch (reg) { 1096 case R_EAX: 1097 *r = entry->eax; 1098 break; 1099 case R_EDX: 1100 *r = entry->edx; 1101 break; 1102 default: 1103 return -EINVAL; 1104 } 1105 1106 return 0; 1107 } 1108 1109 static int hv_cpuid_check_and_set(CPUState *cs, struct kvm_cpuid2 *cpuid, 1110 int feature) 1111 { 1112 X86CPU *cpu = X86_CPU(cs); 1113 CPUX86State *env = &cpu->env; 1114 uint32_t r, fw, bits; 1115 uint64_t deps; 1116 int i, dep_feat; 1117 1118 if (!hyperv_feat_enabled(cpu, feature) && !cpu->hyperv_passthrough) { 1119 return 0; 1120 } 1121 1122 deps = kvm_hyperv_properties[feature].dependencies; 1123 while (deps) { 1124 dep_feat = ctz64(deps); 1125 if (!(hyperv_feat_enabled(cpu, dep_feat))) { 1126 fprintf(stderr, 1127 "Hyper-V %s requires Hyper-V %s\n", 1128 kvm_hyperv_properties[feature].desc, 1129 kvm_hyperv_properties[dep_feat].desc); 1130 return 1; 1131 } 1132 deps &= ~(1ull << dep_feat); 1133 } 1134 1135 for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) { 1136 fw = kvm_hyperv_properties[feature].flags[i].fw; 1137 bits = kvm_hyperv_properties[feature].flags[i].bits; 1138 1139 if (!fw) { 1140 continue; 1141 } 1142 1143 if (hv_cpuid_get_fw(cpuid, fw, &r) || (r & bits) != bits) { 1144 if (hyperv_feat_enabled(cpu, feature)) { 1145 fprintf(stderr, 1146 "Hyper-V %s is not supported by kernel\n", 1147 kvm_hyperv_properties[feature].desc); 1148 return 1; 1149 } else { 1150 return 0; 1151 } 1152 } 1153 1154 env->features[fw] |= bits; 1155 } 1156 1157 if (cpu->hyperv_passthrough) { 1158 cpu->hyperv_features |= BIT(feature); 1159 } 1160 1161 return 0; 1162 } 1163 1164 /* 1165 * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent in 1166 * case of success, errno < 0 in case of failure and 0 when no Hyper-V 1167 * extentions are enabled. 1168 */ 1169 static int hyperv_handle_properties(CPUState *cs, 1170 struct kvm_cpuid_entry2 *cpuid_ent) 1171 { 1172 X86CPU *cpu = X86_CPU(cs); 1173 CPUX86State *env = &cpu->env; 1174 struct kvm_cpuid2 *cpuid; 1175 struct kvm_cpuid_entry2 *c; 1176 uint32_t cpuid_i = 0; 1177 int r; 1178 1179 if (!hyperv_enabled(cpu)) 1180 return 0; 1181 1182 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) || 1183 cpu->hyperv_passthrough) { 1184 uint16_t evmcs_version; 1185 1186 r = kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_ENLIGHTENED_VMCS, 0, 1187 (uintptr_t)&evmcs_version); 1188 1189 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) && r) { 1190 fprintf(stderr, "Hyper-V %s is not supported by kernel\n", 1191 kvm_hyperv_properties[HYPERV_FEAT_EVMCS].desc); 1192 return -ENOSYS; 1193 } 1194 1195 if (!r) { 1196 env->features[FEAT_HV_RECOMM_EAX] |= 1197 HV_ENLIGHTENED_VMCS_RECOMMENDED; 1198 env->features[FEAT_HV_NESTED_EAX] = evmcs_version; 1199 } 1200 } 1201 1202 if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_CPUID) > 0) { 1203 cpuid = get_supported_hv_cpuid(cs); 1204 } else { 1205 cpuid = get_supported_hv_cpuid_legacy(cs); 1206 } 1207 1208 if (cpu->hyperv_passthrough) { 1209 memcpy(cpuid_ent, &cpuid->entries[0], 1210 cpuid->nent * sizeof(cpuid->entries[0])); 1211 1212 c = cpuid_find_entry(cpuid, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, 0); 1213 if (c) { 1214 cpu->hyperv_vendor_id[0] = c->ebx; 1215 cpu->hyperv_vendor_id[1] = c->ecx; 1216 cpu->hyperv_vendor_id[2] = c->edx; 1217 } 1218 1219 c = cpuid_find_entry(cpuid, HV_CPUID_INTERFACE, 0); 1220 if (c) { 1221 cpu->hyperv_interface_id[0] = c->eax; 1222 cpu->hyperv_interface_id[1] = c->ebx; 1223 cpu->hyperv_interface_id[2] = c->ecx; 1224 cpu->hyperv_interface_id[3] = c->edx; 1225 } 1226 1227 c = cpuid_find_entry(cpuid, HV_CPUID_VERSION, 0); 1228 if (c) { 1229 cpu->hyperv_version_id[0] = c->eax; 1230 cpu->hyperv_version_id[1] = c->ebx; 1231 cpu->hyperv_version_id[2] = c->ecx; 1232 cpu->hyperv_version_id[3] = c->edx; 1233 } 1234 1235 c = cpuid_find_entry(cpuid, HV_CPUID_FEATURES, 0); 1236 if (c) { 1237 env->features[FEAT_HYPERV_EAX] = c->eax; 1238 env->features[FEAT_HYPERV_EBX] = c->ebx; 1239 env->features[FEAT_HYPERV_EDX] = c->edx; 1240 } 1241 1242 c = cpuid_find_entry(cpuid, HV_CPUID_IMPLEMENT_LIMITS, 0); 1243 if (c) { 1244 cpu->hv_max_vps = c->eax; 1245 cpu->hyperv_limits[0] = c->ebx; 1246 cpu->hyperv_limits[1] = c->ecx; 1247 cpu->hyperv_limits[2] = c->edx; 1248 } 1249 1250 c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0); 1251 if (c) { 1252 env->features[FEAT_HV_RECOMM_EAX] = c->eax; 1253 1254 /* hv-spinlocks may have been overriden */ 1255 if (cpu->hyperv_spinlock_attempts != HYPERV_SPINLOCK_NEVER_NOTIFY) { 1256 c->ebx = cpu->hyperv_spinlock_attempts; 1257 } 1258 } 1259 c = cpuid_find_entry(cpuid, HV_CPUID_NESTED_FEATURES, 0); 1260 if (c) { 1261 env->features[FEAT_HV_NESTED_EAX] = c->eax; 1262 } 1263 } 1264 1265 if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) { 1266 env->features[FEAT_HV_RECOMM_EAX] |= HV_NO_NONARCH_CORESHARING; 1267 } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) { 1268 c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0); 1269 if (c) { 1270 env->features[FEAT_HV_RECOMM_EAX] |= 1271 c->eax & HV_NO_NONARCH_CORESHARING; 1272 } 1273 } 1274 1275 /* Features */ 1276 r = hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RELAXED); 1277 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_VAPIC); 1278 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_TIME); 1279 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_CRASH); 1280 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RESET); 1281 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_VPINDEX); 1282 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RUNTIME); 1283 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_SYNIC); 1284 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_STIMER); 1285 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_FREQUENCIES); 1286 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_REENLIGHTENMENT); 1287 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_TLBFLUSH); 1288 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_EVMCS); 1289 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_IPI); 1290 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_STIMER_DIRECT); 1291 1292 /* Additional dependencies not covered by kvm_hyperv_properties[] */ 1293 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC) && 1294 !cpu->hyperv_synic_kvm_only && 1295 !hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX)) { 1296 fprintf(stderr, "Hyper-V %s requires Hyper-V %s\n", 1297 kvm_hyperv_properties[HYPERV_FEAT_SYNIC].desc, 1298 kvm_hyperv_properties[HYPERV_FEAT_VPINDEX].desc); 1299 r |= 1; 1300 } 1301 1302 /* Not exposed by KVM but needed to make CPU hotplug in Windows work */ 1303 env->features[FEAT_HYPERV_EDX] |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE; 1304 1305 if (r) { 1306 r = -ENOSYS; 1307 goto free; 1308 } 1309 1310 if (cpu->hyperv_passthrough) { 1311 /* We already copied all feature words from KVM as is */ 1312 r = cpuid->nent; 1313 goto free; 1314 } 1315 1316 c = &cpuid_ent[cpuid_i++]; 1317 c->function = HV_CPUID_VENDOR_AND_MAX_FUNCTIONS; 1318 c->eax = hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ? 1319 HV_CPUID_NESTED_FEATURES : HV_CPUID_IMPLEMENT_LIMITS; 1320 c->ebx = cpu->hyperv_vendor_id[0]; 1321 c->ecx = cpu->hyperv_vendor_id[1]; 1322 c->edx = cpu->hyperv_vendor_id[2]; 1323 1324 c = &cpuid_ent[cpuid_i++]; 1325 c->function = HV_CPUID_INTERFACE; 1326 c->eax = cpu->hyperv_interface_id[0]; 1327 c->ebx = cpu->hyperv_interface_id[1]; 1328 c->ecx = cpu->hyperv_interface_id[2]; 1329 c->edx = cpu->hyperv_interface_id[3]; 1330 1331 c = &cpuid_ent[cpuid_i++]; 1332 c->function = HV_CPUID_VERSION; 1333 c->eax = cpu->hyperv_version_id[0]; 1334 c->ebx = cpu->hyperv_version_id[1]; 1335 c->ecx = cpu->hyperv_version_id[2]; 1336 c->edx = cpu->hyperv_version_id[3]; 1337 1338 c = &cpuid_ent[cpuid_i++]; 1339 c->function = HV_CPUID_FEATURES; 1340 c->eax = env->features[FEAT_HYPERV_EAX]; 1341 c->ebx = env->features[FEAT_HYPERV_EBX]; 1342 c->edx = env->features[FEAT_HYPERV_EDX]; 1343 1344 c = &cpuid_ent[cpuid_i++]; 1345 c->function = HV_CPUID_ENLIGHTMENT_INFO; 1346 c->eax = env->features[FEAT_HV_RECOMM_EAX]; 1347 c->ebx = cpu->hyperv_spinlock_attempts; 1348 1349 c = &cpuid_ent[cpuid_i++]; 1350 c->function = HV_CPUID_IMPLEMENT_LIMITS; 1351 c->eax = cpu->hv_max_vps; 1352 c->ebx = cpu->hyperv_limits[0]; 1353 c->ecx = cpu->hyperv_limits[1]; 1354 c->edx = cpu->hyperv_limits[2]; 1355 1356 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS)) { 1357 __u32 function; 1358 1359 /* Create zeroed 0x40000006..0x40000009 leaves */ 1360 for (function = HV_CPUID_IMPLEMENT_LIMITS + 1; 1361 function < HV_CPUID_NESTED_FEATURES; function++) { 1362 c = &cpuid_ent[cpuid_i++]; 1363 c->function = function; 1364 } 1365 1366 c = &cpuid_ent[cpuid_i++]; 1367 c->function = HV_CPUID_NESTED_FEATURES; 1368 c->eax = env->features[FEAT_HV_NESTED_EAX]; 1369 } 1370 r = cpuid_i; 1371 1372 free: 1373 g_free(cpuid); 1374 1375 return r; 1376 } 1377 1378 static Error *hv_passthrough_mig_blocker; 1379 static Error *hv_no_nonarch_cs_mig_blocker; 1380 1381 static int hyperv_init_vcpu(X86CPU *cpu) 1382 { 1383 CPUState *cs = CPU(cpu); 1384 Error *local_err = NULL; 1385 int ret; 1386 1387 if (cpu->hyperv_passthrough && hv_passthrough_mig_blocker == NULL) { 1388 error_setg(&hv_passthrough_mig_blocker, 1389 "'hv-passthrough' CPU flag prevents migration, use explicit" 1390 " set of hv-* flags instead"); 1391 ret = migrate_add_blocker(hv_passthrough_mig_blocker, &local_err); 1392 if (local_err) { 1393 error_report_err(local_err); 1394 error_free(hv_passthrough_mig_blocker); 1395 return ret; 1396 } 1397 } 1398 1399 if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO && 1400 hv_no_nonarch_cs_mig_blocker == NULL) { 1401 error_setg(&hv_no_nonarch_cs_mig_blocker, 1402 "'hv-no-nonarch-coresharing=auto' CPU flag prevents migration" 1403 " use explicit 'hv-no-nonarch-coresharing=on' instead (but" 1404 " make sure SMT is disabled and/or that vCPUs are properly" 1405 " pinned)"); 1406 ret = migrate_add_blocker(hv_no_nonarch_cs_mig_blocker, &local_err); 1407 if (local_err) { 1408 error_report_err(local_err); 1409 error_free(hv_no_nonarch_cs_mig_blocker); 1410 return ret; 1411 } 1412 } 1413 1414 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX) && !hv_vpindex_settable) { 1415 /* 1416 * the kernel doesn't support setting vp_index; assert that its value 1417 * is in sync 1418 */ 1419 struct { 1420 struct kvm_msrs info; 1421 struct kvm_msr_entry entries[1]; 1422 } msr_data = { 1423 .info.nmsrs = 1, 1424 .entries[0].index = HV_X64_MSR_VP_INDEX, 1425 }; 1426 1427 ret = kvm_vcpu_ioctl(cs, KVM_GET_MSRS, &msr_data); 1428 if (ret < 0) { 1429 return ret; 1430 } 1431 assert(ret == 1); 1432 1433 if (msr_data.entries[0].data != hyperv_vp_index(CPU(cpu))) { 1434 error_report("kernel's vp_index != QEMU's vp_index"); 1435 return -ENXIO; 1436 } 1437 } 1438 1439 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) { 1440 uint32_t synic_cap = cpu->hyperv_synic_kvm_only ? 1441 KVM_CAP_HYPERV_SYNIC : KVM_CAP_HYPERV_SYNIC2; 1442 ret = kvm_vcpu_enable_cap(cs, synic_cap, 0); 1443 if (ret < 0) { 1444 error_report("failed to turn on HyperV SynIC in KVM: %s", 1445 strerror(-ret)); 1446 return ret; 1447 } 1448 1449 if (!cpu->hyperv_synic_kvm_only) { 1450 ret = hyperv_x86_synic_add(cpu); 1451 if (ret < 0) { 1452 error_report("failed to create HyperV SynIC: %s", 1453 strerror(-ret)); 1454 return ret; 1455 } 1456 } 1457 } 1458 1459 return 0; 1460 } 1461 1462 static Error *invtsc_mig_blocker; 1463 1464 #define KVM_MAX_CPUID_ENTRIES 100 1465 1466 int kvm_arch_init_vcpu(CPUState *cs) 1467 { 1468 struct { 1469 struct kvm_cpuid2 cpuid; 1470 struct kvm_cpuid_entry2 entries[KVM_MAX_CPUID_ENTRIES]; 1471 } cpuid_data; 1472 /* 1473 * The kernel defines these structs with padding fields so there 1474 * should be no extra padding in our cpuid_data struct. 1475 */ 1476 QEMU_BUILD_BUG_ON(sizeof(cpuid_data) != 1477 sizeof(struct kvm_cpuid2) + 1478 sizeof(struct kvm_cpuid_entry2) * KVM_MAX_CPUID_ENTRIES); 1479 1480 X86CPU *cpu = X86_CPU(cs); 1481 CPUX86State *env = &cpu->env; 1482 uint32_t limit, i, j, cpuid_i; 1483 uint32_t unused; 1484 struct kvm_cpuid_entry2 *c; 1485 uint32_t signature[3]; 1486 int kvm_base = KVM_CPUID_SIGNATURE; 1487 int max_nested_state_len; 1488 int r; 1489 Error *local_err = NULL; 1490 1491 memset(&cpuid_data, 0, sizeof(cpuid_data)); 1492 1493 cpuid_i = 0; 1494 1495 r = kvm_arch_set_tsc_khz(cs); 1496 if (r < 0) { 1497 return r; 1498 } 1499 1500 /* vcpu's TSC frequency is either specified by user, or following 1501 * the value used by KVM if the former is not present. In the 1502 * latter case, we query it from KVM and record in env->tsc_khz, 1503 * so that vcpu's TSC frequency can be migrated later via this field. 1504 */ 1505 if (!env->tsc_khz) { 1506 r = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ? 1507 kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) : 1508 -ENOTSUP; 1509 if (r > 0) { 1510 env->tsc_khz = r; 1511 } 1512 } 1513 1514 env->apic_bus_freq = KVM_APIC_BUS_FREQUENCY; 1515 1516 /* Paravirtualization CPUIDs */ 1517 r = hyperv_handle_properties(cs, cpuid_data.entries); 1518 if (r < 0) { 1519 return r; 1520 } else if (r > 0) { 1521 cpuid_i = r; 1522 kvm_base = KVM_CPUID_SIGNATURE_NEXT; 1523 has_msr_hv_hypercall = true; 1524 } 1525 1526 if (cpu->expose_kvm) { 1527 memcpy(signature, "KVMKVMKVM\0\0\0", 12); 1528 c = &cpuid_data.entries[cpuid_i++]; 1529 c->function = KVM_CPUID_SIGNATURE | kvm_base; 1530 c->eax = KVM_CPUID_FEATURES | kvm_base; 1531 c->ebx = signature[0]; 1532 c->ecx = signature[1]; 1533 c->edx = signature[2]; 1534 1535 c = &cpuid_data.entries[cpuid_i++]; 1536 c->function = KVM_CPUID_FEATURES | kvm_base; 1537 c->eax = env->features[FEAT_KVM]; 1538 c->edx = env->features[FEAT_KVM_HINTS]; 1539 } 1540 1541 cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused); 1542 1543 for (i = 0; i <= limit; i++) { 1544 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) { 1545 fprintf(stderr, "unsupported level value: 0x%x\n", limit); 1546 abort(); 1547 } 1548 c = &cpuid_data.entries[cpuid_i++]; 1549 1550 switch (i) { 1551 case 2: { 1552 /* Keep reading function 2 till all the input is received */ 1553 int times; 1554 1555 c->function = i; 1556 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC | 1557 KVM_CPUID_FLAG_STATE_READ_NEXT; 1558 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx); 1559 times = c->eax & 0xff; 1560 1561 for (j = 1; j < times; ++j) { 1562 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) { 1563 fprintf(stderr, "cpuid_data is full, no space for " 1564 "cpuid(eax:2):eax & 0xf = 0x%x\n", times); 1565 abort(); 1566 } 1567 c = &cpuid_data.entries[cpuid_i++]; 1568 c->function = i; 1569 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC; 1570 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx); 1571 } 1572 break; 1573 } 1574 case 0x1f: 1575 if (env->nr_dies < 2) { 1576 break; 1577 } 1578 /* fallthrough */ 1579 case 4: 1580 case 0xb: 1581 case 0xd: 1582 for (j = 0; ; j++) { 1583 if (i == 0xd && j == 64) { 1584 break; 1585 } 1586 1587 if (i == 0x1f && j == 64) { 1588 break; 1589 } 1590 1591 c->function = i; 1592 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 1593 c->index = j; 1594 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx); 1595 1596 if (i == 4 && c->eax == 0) { 1597 break; 1598 } 1599 if (i == 0xb && !(c->ecx & 0xff00)) { 1600 break; 1601 } 1602 if (i == 0x1f && !(c->ecx & 0xff00)) { 1603 break; 1604 } 1605 if (i == 0xd && c->eax == 0) { 1606 continue; 1607 } 1608 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) { 1609 fprintf(stderr, "cpuid_data is full, no space for " 1610 "cpuid(eax:0x%x,ecx:0x%x)\n", i, j); 1611 abort(); 1612 } 1613 c = &cpuid_data.entries[cpuid_i++]; 1614 } 1615 break; 1616 case 0x7: 1617 case 0x14: { 1618 uint32_t times; 1619 1620 c->function = i; 1621 c->index = 0; 1622 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 1623 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx); 1624 times = c->eax; 1625 1626 for (j = 1; j <= times; ++j) { 1627 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) { 1628 fprintf(stderr, "cpuid_data is full, no space for " 1629 "cpuid(eax:0x%x,ecx:0x%x)\n", i, j); 1630 abort(); 1631 } 1632 c = &cpuid_data.entries[cpuid_i++]; 1633 c->function = i; 1634 c->index = j; 1635 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 1636 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx); 1637 } 1638 break; 1639 } 1640 default: 1641 c->function = i; 1642 c->flags = 0; 1643 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx); 1644 if (!c->eax && !c->ebx && !c->ecx && !c->edx) { 1645 /* 1646 * KVM already returns all zeroes if a CPUID entry is missing, 1647 * so we can omit it and avoid hitting KVM's 80-entry limit. 1648 */ 1649 cpuid_i--; 1650 } 1651 break; 1652 } 1653 } 1654 1655 if (limit >= 0x0a) { 1656 uint32_t eax, edx; 1657 1658 cpu_x86_cpuid(env, 0x0a, 0, &eax, &unused, &unused, &edx); 1659 1660 has_architectural_pmu_version = eax & 0xff; 1661 if (has_architectural_pmu_version > 0) { 1662 num_architectural_pmu_gp_counters = (eax & 0xff00) >> 8; 1663 1664 /* Shouldn't be more than 32, since that's the number of bits 1665 * available in EBX to tell us _which_ counters are available. 1666 * Play it safe. 1667 */ 1668 if (num_architectural_pmu_gp_counters > MAX_GP_COUNTERS) { 1669 num_architectural_pmu_gp_counters = MAX_GP_COUNTERS; 1670 } 1671 1672 if (has_architectural_pmu_version > 1) { 1673 num_architectural_pmu_fixed_counters = edx & 0x1f; 1674 1675 if (num_architectural_pmu_fixed_counters > MAX_FIXED_COUNTERS) { 1676 num_architectural_pmu_fixed_counters = MAX_FIXED_COUNTERS; 1677 } 1678 } 1679 } 1680 } 1681 1682 cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused); 1683 1684 for (i = 0x80000000; i <= limit; i++) { 1685 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) { 1686 fprintf(stderr, "unsupported xlevel value: 0x%x\n", limit); 1687 abort(); 1688 } 1689 c = &cpuid_data.entries[cpuid_i++]; 1690 1691 switch (i) { 1692 case 0x8000001d: 1693 /* Query for all AMD cache information leaves */ 1694 for (j = 0; ; j++) { 1695 c->function = i; 1696 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 1697 c->index = j; 1698 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx); 1699 1700 if (c->eax == 0) { 1701 break; 1702 } 1703 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) { 1704 fprintf(stderr, "cpuid_data is full, no space for " 1705 "cpuid(eax:0x%x,ecx:0x%x)\n", i, j); 1706 abort(); 1707 } 1708 c = &cpuid_data.entries[cpuid_i++]; 1709 } 1710 break; 1711 default: 1712 c->function = i; 1713 c->flags = 0; 1714 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx); 1715 if (!c->eax && !c->ebx && !c->ecx && !c->edx) { 1716 /* 1717 * KVM already returns all zeroes if a CPUID entry is missing, 1718 * so we can omit it and avoid hitting KVM's 80-entry limit. 1719 */ 1720 cpuid_i--; 1721 } 1722 break; 1723 } 1724 } 1725 1726 /* Call Centaur's CPUID instructions they are supported. */ 1727 if (env->cpuid_xlevel2 > 0) { 1728 cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused); 1729 1730 for (i = 0xC0000000; i <= limit; i++) { 1731 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) { 1732 fprintf(stderr, "unsupported xlevel2 value: 0x%x\n", limit); 1733 abort(); 1734 } 1735 c = &cpuid_data.entries[cpuid_i++]; 1736 1737 c->function = i; 1738 c->flags = 0; 1739 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx); 1740 } 1741 } 1742 1743 cpuid_data.cpuid.nent = cpuid_i; 1744 1745 if (((env->cpuid_version >> 8)&0xF) >= 6 1746 && (env->features[FEAT_1_EDX] & (CPUID_MCE | CPUID_MCA)) == 1747 (CPUID_MCE | CPUID_MCA) 1748 && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) { 1749 uint64_t mcg_cap, unsupported_caps; 1750 int banks; 1751 int ret; 1752 1753 ret = kvm_get_mce_cap_supported(cs->kvm_state, &mcg_cap, &banks); 1754 if (ret < 0) { 1755 fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret)); 1756 return ret; 1757 } 1758 1759 if (banks < (env->mcg_cap & MCG_CAP_BANKS_MASK)) { 1760 error_report("kvm: Unsupported MCE bank count (QEMU = %d, KVM = %d)", 1761 (int)(env->mcg_cap & MCG_CAP_BANKS_MASK), banks); 1762 return -ENOTSUP; 1763 } 1764 1765 unsupported_caps = env->mcg_cap & ~(mcg_cap | MCG_CAP_BANKS_MASK); 1766 if (unsupported_caps) { 1767 if (unsupported_caps & MCG_LMCE_P) { 1768 error_report("kvm: LMCE not supported"); 1769 return -ENOTSUP; 1770 } 1771 warn_report("Unsupported MCG_CAP bits: 0x%" PRIx64, 1772 unsupported_caps); 1773 } 1774 1775 env->mcg_cap &= mcg_cap | MCG_CAP_BANKS_MASK; 1776 ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &env->mcg_cap); 1777 if (ret < 0) { 1778 fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret)); 1779 return ret; 1780 } 1781 } 1782 1783 cpu->vmsentry = qemu_add_vm_change_state_handler(cpu_update_state, env); 1784 1785 c = cpuid_find_entry(&cpuid_data.cpuid, 1, 0); 1786 if (c) { 1787 has_msr_feature_control = !!(c->ecx & CPUID_EXT_VMX) || 1788 !!(c->ecx & CPUID_EXT_SMX); 1789 } 1790 1791 if (env->mcg_cap & MCG_LMCE_P) { 1792 has_msr_mcg_ext_ctl = has_msr_feature_control = true; 1793 } 1794 1795 if (!env->user_tsc_khz) { 1796 if ((env->features[FEAT_8000_0007_EDX] & CPUID_APM_INVTSC) && 1797 invtsc_mig_blocker == NULL) { 1798 error_setg(&invtsc_mig_blocker, 1799 "State blocked by non-migratable CPU device" 1800 " (invtsc flag)"); 1801 r = migrate_add_blocker(invtsc_mig_blocker, &local_err); 1802 if (local_err) { 1803 error_report_err(local_err); 1804 error_free(invtsc_mig_blocker); 1805 return r; 1806 } 1807 } 1808 } 1809 1810 if (cpu->vmware_cpuid_freq 1811 /* Guests depend on 0x40000000 to detect this feature, so only expose 1812 * it if KVM exposes leaf 0x40000000. (Conflicts with Hyper-V) */ 1813 && cpu->expose_kvm 1814 && kvm_base == KVM_CPUID_SIGNATURE 1815 /* TSC clock must be stable and known for this feature. */ 1816 && tsc_is_stable_and_known(env)) { 1817 1818 c = &cpuid_data.entries[cpuid_i++]; 1819 c->function = KVM_CPUID_SIGNATURE | 0x10; 1820 c->eax = env->tsc_khz; 1821 c->ebx = env->apic_bus_freq / 1000; /* Hz to KHz */ 1822 c->ecx = c->edx = 0; 1823 1824 c = cpuid_find_entry(&cpuid_data.cpuid, kvm_base, 0); 1825 c->eax = MAX(c->eax, KVM_CPUID_SIGNATURE | 0x10); 1826 } 1827 1828 cpuid_data.cpuid.nent = cpuid_i; 1829 1830 cpuid_data.cpuid.padding = 0; 1831 r = kvm_vcpu_ioctl(cs, KVM_SET_CPUID2, &cpuid_data); 1832 if (r) { 1833 goto fail; 1834 } 1835 1836 if (has_xsave) { 1837 env->xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave)); 1838 memset(env->xsave_buf, 0, sizeof(struct kvm_xsave)); 1839 } 1840 1841 max_nested_state_len = kvm_max_nested_state_length(); 1842 if (max_nested_state_len > 0) { 1843 assert(max_nested_state_len >= offsetof(struct kvm_nested_state, data)); 1844 1845 if (cpu_has_vmx(env) || cpu_has_svm(env)) { 1846 struct kvm_vmx_nested_state_hdr *vmx_hdr; 1847 1848 env->nested_state = g_malloc0(max_nested_state_len); 1849 env->nested_state->size = max_nested_state_len; 1850 1851 if (cpu_has_vmx(env)) { 1852 env->nested_state->format = KVM_STATE_NESTED_FORMAT_VMX; 1853 vmx_hdr = &env->nested_state->hdr.vmx; 1854 vmx_hdr->vmxon_pa = -1ull; 1855 vmx_hdr->vmcs12_pa = -1ull; 1856 } else { 1857 env->nested_state->format = KVM_STATE_NESTED_FORMAT_SVM; 1858 } 1859 } 1860 } 1861 1862 cpu->kvm_msr_buf = g_malloc0(MSR_BUF_SIZE); 1863 1864 if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_RDTSCP)) { 1865 has_msr_tsc_aux = false; 1866 } 1867 1868 kvm_init_msrs(cpu); 1869 1870 r = hyperv_init_vcpu(cpu); 1871 if (r) { 1872 goto fail; 1873 } 1874 1875 return 0; 1876 1877 fail: 1878 migrate_del_blocker(invtsc_mig_blocker); 1879 1880 return r; 1881 } 1882 1883 int kvm_arch_destroy_vcpu(CPUState *cs) 1884 { 1885 X86CPU *cpu = X86_CPU(cs); 1886 CPUX86State *env = &cpu->env; 1887 1888 if (cpu->kvm_msr_buf) { 1889 g_free(cpu->kvm_msr_buf); 1890 cpu->kvm_msr_buf = NULL; 1891 } 1892 1893 if (env->nested_state) { 1894 g_free(env->nested_state); 1895 env->nested_state = NULL; 1896 } 1897 1898 qemu_del_vm_change_state_handler(cpu->vmsentry); 1899 1900 return 0; 1901 } 1902 1903 void kvm_arch_reset_vcpu(X86CPU *cpu) 1904 { 1905 CPUX86State *env = &cpu->env; 1906 1907 env->xcr0 = 1; 1908 if (kvm_irqchip_in_kernel()) { 1909 env->mp_state = cpu_is_bsp(cpu) ? KVM_MP_STATE_RUNNABLE : 1910 KVM_MP_STATE_UNINITIALIZED; 1911 } else { 1912 env->mp_state = KVM_MP_STATE_RUNNABLE; 1913 } 1914 1915 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) { 1916 int i; 1917 for (i = 0; i < ARRAY_SIZE(env->msr_hv_synic_sint); i++) { 1918 env->msr_hv_synic_sint[i] = HV_SINT_MASKED; 1919 } 1920 1921 hyperv_x86_synic_reset(cpu); 1922 } 1923 /* enabled by default */ 1924 env->poll_control_msr = 1; 1925 1926 sev_es_set_reset_vector(CPU(cpu)); 1927 } 1928 1929 void kvm_arch_do_init_vcpu(X86CPU *cpu) 1930 { 1931 CPUX86State *env = &cpu->env; 1932 1933 /* APs get directly into wait-for-SIPI state. */ 1934 if (env->mp_state == KVM_MP_STATE_UNINITIALIZED) { 1935 env->mp_state = KVM_MP_STATE_INIT_RECEIVED; 1936 } 1937 } 1938 1939 static int kvm_get_supported_feature_msrs(KVMState *s) 1940 { 1941 int ret = 0; 1942 1943 if (kvm_feature_msrs != NULL) { 1944 return 0; 1945 } 1946 1947 if (!kvm_check_extension(s, KVM_CAP_GET_MSR_FEATURES)) { 1948 return 0; 1949 } 1950 1951 struct kvm_msr_list msr_list; 1952 1953 msr_list.nmsrs = 0; 1954 ret = kvm_ioctl(s, KVM_GET_MSR_FEATURE_INDEX_LIST, &msr_list); 1955 if (ret < 0 && ret != -E2BIG) { 1956 error_report("Fetch KVM feature MSR list failed: %s", 1957 strerror(-ret)); 1958 return ret; 1959 } 1960 1961 assert(msr_list.nmsrs > 0); 1962 kvm_feature_msrs = (struct kvm_msr_list *) \ 1963 g_malloc0(sizeof(msr_list) + 1964 msr_list.nmsrs * sizeof(msr_list.indices[0])); 1965 1966 kvm_feature_msrs->nmsrs = msr_list.nmsrs; 1967 ret = kvm_ioctl(s, KVM_GET_MSR_FEATURE_INDEX_LIST, kvm_feature_msrs); 1968 1969 if (ret < 0) { 1970 error_report("Fetch KVM feature MSR list failed: %s", 1971 strerror(-ret)); 1972 g_free(kvm_feature_msrs); 1973 kvm_feature_msrs = NULL; 1974 return ret; 1975 } 1976 1977 return 0; 1978 } 1979 1980 static int kvm_get_supported_msrs(KVMState *s) 1981 { 1982 int ret = 0; 1983 struct kvm_msr_list msr_list, *kvm_msr_list; 1984 1985 /* 1986 * Obtain MSR list from KVM. These are the MSRs that we must 1987 * save/restore. 1988 */ 1989 msr_list.nmsrs = 0; 1990 ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list); 1991 if (ret < 0 && ret != -E2BIG) { 1992 return ret; 1993 } 1994 /* 1995 * Old kernel modules had a bug and could write beyond the provided 1996 * memory. Allocate at least a safe amount of 1K. 1997 */ 1998 kvm_msr_list = g_malloc0(MAX(1024, sizeof(msr_list) + 1999 msr_list.nmsrs * 2000 sizeof(msr_list.indices[0]))); 2001 2002 kvm_msr_list->nmsrs = msr_list.nmsrs; 2003 ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list); 2004 if (ret >= 0) { 2005 int i; 2006 2007 for (i = 0; i < kvm_msr_list->nmsrs; i++) { 2008 switch (kvm_msr_list->indices[i]) { 2009 case MSR_STAR: 2010 has_msr_star = true; 2011 break; 2012 case MSR_VM_HSAVE_PA: 2013 has_msr_hsave_pa = true; 2014 break; 2015 case MSR_TSC_AUX: 2016 has_msr_tsc_aux = true; 2017 break; 2018 case MSR_TSC_ADJUST: 2019 has_msr_tsc_adjust = true; 2020 break; 2021 case MSR_IA32_TSCDEADLINE: 2022 has_msr_tsc_deadline = true; 2023 break; 2024 case MSR_IA32_SMBASE: 2025 has_msr_smbase = true; 2026 break; 2027 case MSR_SMI_COUNT: 2028 has_msr_smi_count = true; 2029 break; 2030 case MSR_IA32_MISC_ENABLE: 2031 has_msr_misc_enable = true; 2032 break; 2033 case MSR_IA32_BNDCFGS: 2034 has_msr_bndcfgs = true; 2035 break; 2036 case MSR_IA32_XSS: 2037 has_msr_xss = true; 2038 break; 2039 case MSR_IA32_UMWAIT_CONTROL: 2040 has_msr_umwait = true; 2041 break; 2042 case HV_X64_MSR_CRASH_CTL: 2043 has_msr_hv_crash = true; 2044 break; 2045 case HV_X64_MSR_RESET: 2046 has_msr_hv_reset = true; 2047 break; 2048 case HV_X64_MSR_VP_INDEX: 2049 has_msr_hv_vpindex = true; 2050 break; 2051 case HV_X64_MSR_VP_RUNTIME: 2052 has_msr_hv_runtime = true; 2053 break; 2054 case HV_X64_MSR_SCONTROL: 2055 has_msr_hv_synic = true; 2056 break; 2057 case HV_X64_MSR_STIMER0_CONFIG: 2058 has_msr_hv_stimer = true; 2059 break; 2060 case HV_X64_MSR_TSC_FREQUENCY: 2061 has_msr_hv_frequencies = true; 2062 break; 2063 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 2064 has_msr_hv_reenlightenment = true; 2065 break; 2066 case MSR_IA32_SPEC_CTRL: 2067 has_msr_spec_ctrl = true; 2068 break; 2069 case MSR_IA32_TSX_CTRL: 2070 has_msr_tsx_ctrl = true; 2071 break; 2072 case MSR_VIRT_SSBD: 2073 has_msr_virt_ssbd = true; 2074 break; 2075 case MSR_IA32_ARCH_CAPABILITIES: 2076 has_msr_arch_capabs = true; 2077 break; 2078 case MSR_IA32_CORE_CAPABILITY: 2079 has_msr_core_capabs = true; 2080 break; 2081 case MSR_IA32_PERF_CAPABILITIES: 2082 has_msr_perf_capabs = true; 2083 break; 2084 case MSR_IA32_VMX_VMFUNC: 2085 has_msr_vmx_vmfunc = true; 2086 break; 2087 case MSR_IA32_UCODE_REV: 2088 has_msr_ucode_rev = true; 2089 break; 2090 case MSR_IA32_VMX_PROCBASED_CTLS2: 2091 has_msr_vmx_procbased_ctls2 = true; 2092 break; 2093 case MSR_IA32_PKRS: 2094 has_msr_pkrs = true; 2095 break; 2096 } 2097 } 2098 } 2099 2100 g_free(kvm_msr_list); 2101 2102 return ret; 2103 } 2104 2105 static Notifier smram_machine_done; 2106 static KVMMemoryListener smram_listener; 2107 static AddressSpace smram_address_space; 2108 static MemoryRegion smram_as_root; 2109 static MemoryRegion smram_as_mem; 2110 2111 static void register_smram_listener(Notifier *n, void *unused) 2112 { 2113 MemoryRegion *smram = 2114 (MemoryRegion *) object_resolve_path("/machine/smram", NULL); 2115 2116 /* Outer container... */ 2117 memory_region_init(&smram_as_root, OBJECT(kvm_state), "mem-container-smram", ~0ull); 2118 memory_region_set_enabled(&smram_as_root, true); 2119 2120 /* ... with two regions inside: normal system memory with low 2121 * priority, and... 2122 */ 2123 memory_region_init_alias(&smram_as_mem, OBJECT(kvm_state), "mem-smram", 2124 get_system_memory(), 0, ~0ull); 2125 memory_region_add_subregion_overlap(&smram_as_root, 0, &smram_as_mem, 0); 2126 memory_region_set_enabled(&smram_as_mem, true); 2127 2128 if (smram) { 2129 /* ... SMRAM with higher priority */ 2130 memory_region_add_subregion_overlap(&smram_as_root, 0, smram, 10); 2131 memory_region_set_enabled(smram, true); 2132 } 2133 2134 address_space_init(&smram_address_space, &smram_as_root, "KVM-SMRAM"); 2135 kvm_memory_listener_register(kvm_state, &smram_listener, 2136 &smram_address_space, 1); 2137 } 2138 2139 int kvm_arch_init(MachineState *ms, KVMState *s) 2140 { 2141 uint64_t identity_base = 0xfffbc000; 2142 uint64_t shadow_mem; 2143 int ret; 2144 struct utsname utsname; 2145 Error *local_err = NULL; 2146 2147 /* 2148 * Initialize SEV context, if required 2149 * 2150 * If no memory encryption is requested (ms->cgs == NULL) this is 2151 * a no-op. 2152 * 2153 * It's also a no-op if a non-SEV confidential guest support 2154 * mechanism is selected. SEV is the only mechanism available to 2155 * select on x86 at present, so this doesn't arise, but if new 2156 * mechanisms are supported in future (e.g. TDX), they'll need 2157 * their own initialization either here or elsewhere. 2158 */ 2159 ret = sev_kvm_init(ms->cgs, &local_err); 2160 if (ret < 0) { 2161 error_report_err(local_err); 2162 return ret; 2163 } 2164 2165 if (!kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) { 2166 error_report("kvm: KVM_CAP_IRQ_ROUTING not supported by KVM"); 2167 return -ENOTSUP; 2168 } 2169 2170 has_xsave = kvm_check_extension(s, KVM_CAP_XSAVE); 2171 has_xcrs = kvm_check_extension(s, KVM_CAP_XCRS); 2172 has_pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2); 2173 2174 hv_vpindex_settable = kvm_check_extension(s, KVM_CAP_HYPERV_VP_INDEX); 2175 2176 has_exception_payload = kvm_check_extension(s, KVM_CAP_EXCEPTION_PAYLOAD); 2177 if (has_exception_payload) { 2178 ret = kvm_vm_enable_cap(s, KVM_CAP_EXCEPTION_PAYLOAD, 0, true); 2179 if (ret < 0) { 2180 error_report("kvm: Failed to enable exception payload cap: %s", 2181 strerror(-ret)); 2182 return ret; 2183 } 2184 } 2185 2186 ret = kvm_get_supported_msrs(s); 2187 if (ret < 0) { 2188 return ret; 2189 } 2190 2191 kvm_get_supported_feature_msrs(s); 2192 2193 uname(&utsname); 2194 lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0; 2195 2196 /* 2197 * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly. 2198 * In order to use vm86 mode, an EPT identity map and a TSS are needed. 2199 * Since these must be part of guest physical memory, we need to allocate 2200 * them, both by setting their start addresses in the kernel and by 2201 * creating a corresponding e820 entry. We need 4 pages before the BIOS. 2202 * 2203 * Older KVM versions may not support setting the identity map base. In 2204 * that case we need to stick with the default, i.e. a 256K maximum BIOS 2205 * size. 2206 */ 2207 if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) { 2208 /* Allows up to 16M BIOSes. */ 2209 identity_base = 0xfeffc000; 2210 2211 ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base); 2212 if (ret < 0) { 2213 return ret; 2214 } 2215 } 2216 2217 /* Set TSS base one page after EPT identity map. */ 2218 ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000); 2219 if (ret < 0) { 2220 return ret; 2221 } 2222 2223 /* Tell fw_cfg to notify the BIOS to reserve the range. */ 2224 ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED); 2225 if (ret < 0) { 2226 fprintf(stderr, "e820_add_entry() table is full\n"); 2227 return ret; 2228 } 2229 2230 shadow_mem = object_property_get_int(OBJECT(s), "kvm-shadow-mem", &error_abort); 2231 if (shadow_mem != -1) { 2232 shadow_mem /= 4096; 2233 ret = kvm_vm_ioctl(s, KVM_SET_NR_MMU_PAGES, shadow_mem); 2234 if (ret < 0) { 2235 return ret; 2236 } 2237 } 2238 2239 if (kvm_check_extension(s, KVM_CAP_X86_SMM) && 2240 object_dynamic_cast(OBJECT(ms), TYPE_X86_MACHINE) && 2241 x86_machine_is_smm_enabled(X86_MACHINE(ms))) { 2242 smram_machine_done.notify = register_smram_listener; 2243 qemu_add_machine_init_done_notifier(&smram_machine_done); 2244 } 2245 2246 if (enable_cpu_pm) { 2247 int disable_exits = kvm_check_extension(s, KVM_CAP_X86_DISABLE_EXITS); 2248 int ret; 2249 2250 /* Work around for kernel header with a typo. TODO: fix header and drop. */ 2251 #if defined(KVM_X86_DISABLE_EXITS_HTL) && !defined(KVM_X86_DISABLE_EXITS_HLT) 2252 #define KVM_X86_DISABLE_EXITS_HLT KVM_X86_DISABLE_EXITS_HTL 2253 #endif 2254 if (disable_exits) { 2255 disable_exits &= (KVM_X86_DISABLE_EXITS_MWAIT | 2256 KVM_X86_DISABLE_EXITS_HLT | 2257 KVM_X86_DISABLE_EXITS_PAUSE | 2258 KVM_X86_DISABLE_EXITS_CSTATE); 2259 } 2260 2261 ret = kvm_vm_enable_cap(s, KVM_CAP_X86_DISABLE_EXITS, 0, 2262 disable_exits); 2263 if (ret < 0) { 2264 error_report("kvm: guest stopping CPU not supported: %s", 2265 strerror(-ret)); 2266 } 2267 } 2268 2269 return 0; 2270 } 2271 2272 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs) 2273 { 2274 lhs->selector = rhs->selector; 2275 lhs->base = rhs->base; 2276 lhs->limit = rhs->limit; 2277 lhs->type = 3; 2278 lhs->present = 1; 2279 lhs->dpl = 3; 2280 lhs->db = 0; 2281 lhs->s = 1; 2282 lhs->l = 0; 2283 lhs->g = 0; 2284 lhs->avl = 0; 2285 lhs->unusable = 0; 2286 } 2287 2288 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs) 2289 { 2290 unsigned flags = rhs->flags; 2291 lhs->selector = rhs->selector; 2292 lhs->base = rhs->base; 2293 lhs->limit = rhs->limit; 2294 lhs->type = (flags >> DESC_TYPE_SHIFT) & 15; 2295 lhs->present = (flags & DESC_P_MASK) != 0; 2296 lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3; 2297 lhs->db = (flags >> DESC_B_SHIFT) & 1; 2298 lhs->s = (flags & DESC_S_MASK) != 0; 2299 lhs->l = (flags >> DESC_L_SHIFT) & 1; 2300 lhs->g = (flags & DESC_G_MASK) != 0; 2301 lhs->avl = (flags & DESC_AVL_MASK) != 0; 2302 lhs->unusable = !lhs->present; 2303 lhs->padding = 0; 2304 } 2305 2306 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs) 2307 { 2308 lhs->selector = rhs->selector; 2309 lhs->base = rhs->base; 2310 lhs->limit = rhs->limit; 2311 lhs->flags = (rhs->type << DESC_TYPE_SHIFT) | 2312 ((rhs->present && !rhs->unusable) * DESC_P_MASK) | 2313 (rhs->dpl << DESC_DPL_SHIFT) | 2314 (rhs->db << DESC_B_SHIFT) | 2315 (rhs->s * DESC_S_MASK) | 2316 (rhs->l << DESC_L_SHIFT) | 2317 (rhs->g * DESC_G_MASK) | 2318 (rhs->avl * DESC_AVL_MASK); 2319 } 2320 2321 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set) 2322 { 2323 if (set) { 2324 *kvm_reg = *qemu_reg; 2325 } else { 2326 *qemu_reg = *kvm_reg; 2327 } 2328 } 2329 2330 static int kvm_getput_regs(X86CPU *cpu, int set) 2331 { 2332 CPUX86State *env = &cpu->env; 2333 struct kvm_regs regs; 2334 int ret = 0; 2335 2336 if (!set) { 2337 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_REGS, ®s); 2338 if (ret < 0) { 2339 return ret; 2340 } 2341 } 2342 2343 kvm_getput_reg(®s.rax, &env->regs[R_EAX], set); 2344 kvm_getput_reg(®s.rbx, &env->regs[R_EBX], set); 2345 kvm_getput_reg(®s.rcx, &env->regs[R_ECX], set); 2346 kvm_getput_reg(®s.rdx, &env->regs[R_EDX], set); 2347 kvm_getput_reg(®s.rsi, &env->regs[R_ESI], set); 2348 kvm_getput_reg(®s.rdi, &env->regs[R_EDI], set); 2349 kvm_getput_reg(®s.rsp, &env->regs[R_ESP], set); 2350 kvm_getput_reg(®s.rbp, &env->regs[R_EBP], set); 2351 #ifdef TARGET_X86_64 2352 kvm_getput_reg(®s.r8, &env->regs[8], set); 2353 kvm_getput_reg(®s.r9, &env->regs[9], set); 2354 kvm_getput_reg(®s.r10, &env->regs[10], set); 2355 kvm_getput_reg(®s.r11, &env->regs[11], set); 2356 kvm_getput_reg(®s.r12, &env->regs[12], set); 2357 kvm_getput_reg(®s.r13, &env->regs[13], set); 2358 kvm_getput_reg(®s.r14, &env->regs[14], set); 2359 kvm_getput_reg(®s.r15, &env->regs[15], set); 2360 #endif 2361 2362 kvm_getput_reg(®s.rflags, &env->eflags, set); 2363 kvm_getput_reg(®s.rip, &env->eip, set); 2364 2365 if (set) { 2366 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_REGS, ®s); 2367 } 2368 2369 return ret; 2370 } 2371 2372 static int kvm_put_fpu(X86CPU *cpu) 2373 { 2374 CPUX86State *env = &cpu->env; 2375 struct kvm_fpu fpu; 2376 int i; 2377 2378 memset(&fpu, 0, sizeof fpu); 2379 fpu.fsw = env->fpus & ~(7 << 11); 2380 fpu.fsw |= (env->fpstt & 7) << 11; 2381 fpu.fcw = env->fpuc; 2382 fpu.last_opcode = env->fpop; 2383 fpu.last_ip = env->fpip; 2384 fpu.last_dp = env->fpdp; 2385 for (i = 0; i < 8; ++i) { 2386 fpu.ftwx |= (!env->fptags[i]) << i; 2387 } 2388 memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs); 2389 for (i = 0; i < CPU_NB_REGS; i++) { 2390 stq_p(&fpu.xmm[i][0], env->xmm_regs[i].ZMM_Q(0)); 2391 stq_p(&fpu.xmm[i][8], env->xmm_regs[i].ZMM_Q(1)); 2392 } 2393 fpu.mxcsr = env->mxcsr; 2394 2395 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_FPU, &fpu); 2396 } 2397 2398 #define XSAVE_FCW_FSW 0 2399 #define XSAVE_FTW_FOP 1 2400 #define XSAVE_CWD_RIP 2 2401 #define XSAVE_CWD_RDP 4 2402 #define XSAVE_MXCSR 6 2403 #define XSAVE_ST_SPACE 8 2404 #define XSAVE_XMM_SPACE 40 2405 #define XSAVE_XSTATE_BV 128 2406 #define XSAVE_YMMH_SPACE 144 2407 #define XSAVE_BNDREGS 240 2408 #define XSAVE_BNDCSR 256 2409 #define XSAVE_OPMASK 272 2410 #define XSAVE_ZMM_Hi256 288 2411 #define XSAVE_Hi16_ZMM 416 2412 #define XSAVE_PKRU 672 2413 2414 #define XSAVE_BYTE_OFFSET(word_offset) \ 2415 ((word_offset) * sizeof_field(struct kvm_xsave, region[0])) 2416 2417 #define ASSERT_OFFSET(word_offset, field) \ 2418 QEMU_BUILD_BUG_ON(XSAVE_BYTE_OFFSET(word_offset) != \ 2419 offsetof(X86XSaveArea, field)) 2420 2421 ASSERT_OFFSET(XSAVE_FCW_FSW, legacy.fcw); 2422 ASSERT_OFFSET(XSAVE_FTW_FOP, legacy.ftw); 2423 ASSERT_OFFSET(XSAVE_CWD_RIP, legacy.fpip); 2424 ASSERT_OFFSET(XSAVE_CWD_RDP, legacy.fpdp); 2425 ASSERT_OFFSET(XSAVE_MXCSR, legacy.mxcsr); 2426 ASSERT_OFFSET(XSAVE_ST_SPACE, legacy.fpregs); 2427 ASSERT_OFFSET(XSAVE_XMM_SPACE, legacy.xmm_regs); 2428 ASSERT_OFFSET(XSAVE_XSTATE_BV, header.xstate_bv); 2429 ASSERT_OFFSET(XSAVE_YMMH_SPACE, avx_state); 2430 ASSERT_OFFSET(XSAVE_BNDREGS, bndreg_state); 2431 ASSERT_OFFSET(XSAVE_BNDCSR, bndcsr_state); 2432 ASSERT_OFFSET(XSAVE_OPMASK, opmask_state); 2433 ASSERT_OFFSET(XSAVE_ZMM_Hi256, zmm_hi256_state); 2434 ASSERT_OFFSET(XSAVE_Hi16_ZMM, hi16_zmm_state); 2435 ASSERT_OFFSET(XSAVE_PKRU, pkru_state); 2436 2437 static int kvm_put_xsave(X86CPU *cpu) 2438 { 2439 CPUX86State *env = &cpu->env; 2440 X86XSaveArea *xsave = env->xsave_buf; 2441 2442 if (!has_xsave) { 2443 return kvm_put_fpu(cpu); 2444 } 2445 x86_cpu_xsave_all_areas(cpu, xsave); 2446 2447 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave); 2448 } 2449 2450 static int kvm_put_xcrs(X86CPU *cpu) 2451 { 2452 CPUX86State *env = &cpu->env; 2453 struct kvm_xcrs xcrs = {}; 2454 2455 if (!has_xcrs) { 2456 return 0; 2457 } 2458 2459 xcrs.nr_xcrs = 1; 2460 xcrs.flags = 0; 2461 xcrs.xcrs[0].xcr = 0; 2462 xcrs.xcrs[0].value = env->xcr0; 2463 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XCRS, &xcrs); 2464 } 2465 2466 static int kvm_put_sregs(X86CPU *cpu) 2467 { 2468 CPUX86State *env = &cpu->env; 2469 struct kvm_sregs sregs; 2470 2471 memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap)); 2472 if (env->interrupt_injected >= 0) { 2473 sregs.interrupt_bitmap[env->interrupt_injected / 64] |= 2474 (uint64_t)1 << (env->interrupt_injected % 64); 2475 } 2476 2477 if ((env->eflags & VM_MASK)) { 2478 set_v8086_seg(&sregs.cs, &env->segs[R_CS]); 2479 set_v8086_seg(&sregs.ds, &env->segs[R_DS]); 2480 set_v8086_seg(&sregs.es, &env->segs[R_ES]); 2481 set_v8086_seg(&sregs.fs, &env->segs[R_FS]); 2482 set_v8086_seg(&sregs.gs, &env->segs[R_GS]); 2483 set_v8086_seg(&sregs.ss, &env->segs[R_SS]); 2484 } else { 2485 set_seg(&sregs.cs, &env->segs[R_CS]); 2486 set_seg(&sregs.ds, &env->segs[R_DS]); 2487 set_seg(&sregs.es, &env->segs[R_ES]); 2488 set_seg(&sregs.fs, &env->segs[R_FS]); 2489 set_seg(&sregs.gs, &env->segs[R_GS]); 2490 set_seg(&sregs.ss, &env->segs[R_SS]); 2491 } 2492 2493 set_seg(&sregs.tr, &env->tr); 2494 set_seg(&sregs.ldt, &env->ldt); 2495 2496 sregs.idt.limit = env->idt.limit; 2497 sregs.idt.base = env->idt.base; 2498 memset(sregs.idt.padding, 0, sizeof sregs.idt.padding); 2499 sregs.gdt.limit = env->gdt.limit; 2500 sregs.gdt.base = env->gdt.base; 2501 memset(sregs.gdt.padding, 0, sizeof sregs.gdt.padding); 2502 2503 sregs.cr0 = env->cr[0]; 2504 sregs.cr2 = env->cr[2]; 2505 sregs.cr3 = env->cr[3]; 2506 sregs.cr4 = env->cr[4]; 2507 2508 sregs.cr8 = cpu_get_apic_tpr(cpu->apic_state); 2509 sregs.apic_base = cpu_get_apic_base(cpu->apic_state); 2510 2511 sregs.efer = env->efer; 2512 2513 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs); 2514 } 2515 2516 static void kvm_msr_buf_reset(X86CPU *cpu) 2517 { 2518 memset(cpu->kvm_msr_buf, 0, MSR_BUF_SIZE); 2519 } 2520 2521 static void kvm_msr_entry_add(X86CPU *cpu, uint32_t index, uint64_t value) 2522 { 2523 struct kvm_msrs *msrs = cpu->kvm_msr_buf; 2524 void *limit = ((void *)msrs) + MSR_BUF_SIZE; 2525 struct kvm_msr_entry *entry = &msrs->entries[msrs->nmsrs]; 2526 2527 assert((void *)(entry + 1) <= limit); 2528 2529 entry->index = index; 2530 entry->reserved = 0; 2531 entry->data = value; 2532 msrs->nmsrs++; 2533 } 2534 2535 static int kvm_put_one_msr(X86CPU *cpu, int index, uint64_t value) 2536 { 2537 kvm_msr_buf_reset(cpu); 2538 kvm_msr_entry_add(cpu, index, value); 2539 2540 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf); 2541 } 2542 2543 void kvm_put_apicbase(X86CPU *cpu, uint64_t value) 2544 { 2545 int ret; 2546 2547 ret = kvm_put_one_msr(cpu, MSR_IA32_APICBASE, value); 2548 assert(ret == 1); 2549 } 2550 2551 static int kvm_put_tscdeadline_msr(X86CPU *cpu) 2552 { 2553 CPUX86State *env = &cpu->env; 2554 int ret; 2555 2556 if (!has_msr_tsc_deadline) { 2557 return 0; 2558 } 2559 2560 ret = kvm_put_one_msr(cpu, MSR_IA32_TSCDEADLINE, env->tsc_deadline); 2561 if (ret < 0) { 2562 return ret; 2563 } 2564 2565 assert(ret == 1); 2566 return 0; 2567 } 2568 2569 /* 2570 * Provide a separate write service for the feature control MSR in order to 2571 * kick the VCPU out of VMXON or even guest mode on reset. This has to be done 2572 * before writing any other state because forcibly leaving nested mode 2573 * invalidates the VCPU state. 2574 */ 2575 static int kvm_put_msr_feature_control(X86CPU *cpu) 2576 { 2577 int ret; 2578 2579 if (!has_msr_feature_control) { 2580 return 0; 2581 } 2582 2583 ret = kvm_put_one_msr(cpu, MSR_IA32_FEATURE_CONTROL, 2584 cpu->env.msr_ia32_feature_control); 2585 if (ret < 0) { 2586 return ret; 2587 } 2588 2589 assert(ret == 1); 2590 return 0; 2591 } 2592 2593 static uint64_t make_vmx_msr_value(uint32_t index, uint32_t features) 2594 { 2595 uint32_t default1, can_be_one, can_be_zero; 2596 uint32_t must_be_one; 2597 2598 switch (index) { 2599 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 2600 default1 = 0x00000016; 2601 break; 2602 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 2603 default1 = 0x0401e172; 2604 break; 2605 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 2606 default1 = 0x000011ff; 2607 break; 2608 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 2609 default1 = 0x00036dff; 2610 break; 2611 case MSR_IA32_VMX_PROCBASED_CTLS2: 2612 default1 = 0; 2613 break; 2614 default: 2615 abort(); 2616 } 2617 2618 /* If a feature bit is set, the control can be either set or clear. 2619 * Otherwise the value is limited to either 0 or 1 by default1. 2620 */ 2621 can_be_one = features | default1; 2622 can_be_zero = features | ~default1; 2623 must_be_one = ~can_be_zero; 2624 2625 /* 2626 * Bit 0:31 -> 0 if the control bit can be zero (i.e. 1 if it must be one). 2627 * Bit 32:63 -> 1 if the control bit can be one. 2628 */ 2629 return must_be_one | (((uint64_t)can_be_one) << 32); 2630 } 2631 2632 #define VMCS12_MAX_FIELD_INDEX (0x17) 2633 2634 static void kvm_msr_entry_add_vmx(X86CPU *cpu, FeatureWordArray f) 2635 { 2636 uint64_t kvm_vmx_basic = 2637 kvm_arch_get_supported_msr_feature(kvm_state, 2638 MSR_IA32_VMX_BASIC); 2639 2640 if (!kvm_vmx_basic) { 2641 /* If the kernel doesn't support VMX feature (kvm_intel.nested=0), 2642 * then kvm_vmx_basic will be 0 and KVM_SET_MSR will fail. 2643 */ 2644 return; 2645 } 2646 2647 uint64_t kvm_vmx_misc = 2648 kvm_arch_get_supported_msr_feature(kvm_state, 2649 MSR_IA32_VMX_MISC); 2650 uint64_t kvm_vmx_ept_vpid = 2651 kvm_arch_get_supported_msr_feature(kvm_state, 2652 MSR_IA32_VMX_EPT_VPID_CAP); 2653 2654 /* 2655 * If the guest is 64-bit, a value of 1 is allowed for the host address 2656 * space size vmexit control. 2657 */ 2658 uint64_t fixed_vmx_exit = f[FEAT_8000_0001_EDX] & CPUID_EXT2_LM 2659 ? (uint64_t)VMX_VM_EXIT_HOST_ADDR_SPACE_SIZE << 32 : 0; 2660 2661 /* 2662 * Bits 0-30, 32-44 and 50-53 come from the host. KVM should 2663 * not change them for backwards compatibility. 2664 */ 2665 uint64_t fixed_vmx_basic = kvm_vmx_basic & 2666 (MSR_VMX_BASIC_VMCS_REVISION_MASK | 2667 MSR_VMX_BASIC_VMXON_REGION_SIZE_MASK | 2668 MSR_VMX_BASIC_VMCS_MEM_TYPE_MASK); 2669 2670 /* 2671 * Same for bits 0-4 and 25-27. Bits 16-24 (CR3 target count) can 2672 * change in the future but are always zero for now, clear them to be 2673 * future proof. Bits 32-63 in theory could change, though KVM does 2674 * not support dual-monitor treatment and probably never will; mask 2675 * them out as well. 2676 */ 2677 uint64_t fixed_vmx_misc = kvm_vmx_misc & 2678 (MSR_VMX_MISC_PREEMPTION_TIMER_SHIFT_MASK | 2679 MSR_VMX_MISC_MAX_MSR_LIST_SIZE_MASK); 2680 2681 /* 2682 * EPT memory types should not change either, so we do not bother 2683 * adding features for them. 2684 */ 2685 uint64_t fixed_vmx_ept_mask = 2686 (f[FEAT_VMX_SECONDARY_CTLS] & VMX_SECONDARY_EXEC_ENABLE_EPT ? 2687 MSR_VMX_EPT_UC | MSR_VMX_EPT_WB : 0); 2688 uint64_t fixed_vmx_ept_vpid = kvm_vmx_ept_vpid & fixed_vmx_ept_mask; 2689 2690 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_PROCBASED_CTLS, 2691 make_vmx_msr_value(MSR_IA32_VMX_TRUE_PROCBASED_CTLS, 2692 f[FEAT_VMX_PROCBASED_CTLS])); 2693 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_PINBASED_CTLS, 2694 make_vmx_msr_value(MSR_IA32_VMX_TRUE_PINBASED_CTLS, 2695 f[FEAT_VMX_PINBASED_CTLS])); 2696 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_EXIT_CTLS, 2697 make_vmx_msr_value(MSR_IA32_VMX_TRUE_EXIT_CTLS, 2698 f[FEAT_VMX_EXIT_CTLS]) | fixed_vmx_exit); 2699 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_ENTRY_CTLS, 2700 make_vmx_msr_value(MSR_IA32_VMX_TRUE_ENTRY_CTLS, 2701 f[FEAT_VMX_ENTRY_CTLS])); 2702 kvm_msr_entry_add(cpu, MSR_IA32_VMX_PROCBASED_CTLS2, 2703 make_vmx_msr_value(MSR_IA32_VMX_PROCBASED_CTLS2, 2704 f[FEAT_VMX_SECONDARY_CTLS])); 2705 kvm_msr_entry_add(cpu, MSR_IA32_VMX_EPT_VPID_CAP, 2706 f[FEAT_VMX_EPT_VPID_CAPS] | fixed_vmx_ept_vpid); 2707 kvm_msr_entry_add(cpu, MSR_IA32_VMX_BASIC, 2708 f[FEAT_VMX_BASIC] | fixed_vmx_basic); 2709 kvm_msr_entry_add(cpu, MSR_IA32_VMX_MISC, 2710 f[FEAT_VMX_MISC] | fixed_vmx_misc); 2711 if (has_msr_vmx_vmfunc) { 2712 kvm_msr_entry_add(cpu, MSR_IA32_VMX_VMFUNC, f[FEAT_VMX_VMFUNC]); 2713 } 2714 2715 /* 2716 * Just to be safe, write these with constant values. The CRn_FIXED1 2717 * MSRs are generated by KVM based on the vCPU's CPUID. 2718 */ 2719 kvm_msr_entry_add(cpu, MSR_IA32_VMX_CR0_FIXED0, 2720 CR0_PE_MASK | CR0_PG_MASK | CR0_NE_MASK); 2721 kvm_msr_entry_add(cpu, MSR_IA32_VMX_CR4_FIXED0, 2722 CR4_VMXE_MASK); 2723 kvm_msr_entry_add(cpu, MSR_IA32_VMX_VMCS_ENUM, 2724 VMCS12_MAX_FIELD_INDEX << 1); 2725 } 2726 2727 static void kvm_msr_entry_add_perf(X86CPU *cpu, FeatureWordArray f) 2728 { 2729 uint64_t kvm_perf_cap = 2730 kvm_arch_get_supported_msr_feature(kvm_state, 2731 MSR_IA32_PERF_CAPABILITIES); 2732 2733 if (kvm_perf_cap) { 2734 kvm_msr_entry_add(cpu, MSR_IA32_PERF_CAPABILITIES, 2735 kvm_perf_cap & f[FEAT_PERF_CAPABILITIES]); 2736 } 2737 } 2738 2739 static int kvm_buf_set_msrs(X86CPU *cpu) 2740 { 2741 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf); 2742 if (ret < 0) { 2743 return ret; 2744 } 2745 2746 if (ret < cpu->kvm_msr_buf->nmsrs) { 2747 struct kvm_msr_entry *e = &cpu->kvm_msr_buf->entries[ret]; 2748 error_report("error: failed to set MSR 0x%" PRIx32 " to 0x%" PRIx64, 2749 (uint32_t)e->index, (uint64_t)e->data); 2750 } 2751 2752 assert(ret == cpu->kvm_msr_buf->nmsrs); 2753 return 0; 2754 } 2755 2756 static void kvm_init_msrs(X86CPU *cpu) 2757 { 2758 CPUX86State *env = &cpu->env; 2759 2760 kvm_msr_buf_reset(cpu); 2761 if (has_msr_arch_capabs) { 2762 kvm_msr_entry_add(cpu, MSR_IA32_ARCH_CAPABILITIES, 2763 env->features[FEAT_ARCH_CAPABILITIES]); 2764 } 2765 2766 if (has_msr_core_capabs) { 2767 kvm_msr_entry_add(cpu, MSR_IA32_CORE_CAPABILITY, 2768 env->features[FEAT_CORE_CAPABILITY]); 2769 } 2770 2771 if (has_msr_perf_capabs && cpu->enable_pmu) { 2772 kvm_msr_entry_add_perf(cpu, env->features); 2773 } 2774 2775 if (has_msr_ucode_rev) { 2776 kvm_msr_entry_add(cpu, MSR_IA32_UCODE_REV, cpu->ucode_rev); 2777 } 2778 2779 /* 2780 * Older kernels do not include VMX MSRs in KVM_GET_MSR_INDEX_LIST, but 2781 * all kernels with MSR features should have them. 2782 */ 2783 if (kvm_feature_msrs && cpu_has_vmx(env)) { 2784 kvm_msr_entry_add_vmx(cpu, env->features); 2785 } 2786 2787 assert(kvm_buf_set_msrs(cpu) == 0); 2788 } 2789 2790 static int kvm_put_msrs(X86CPU *cpu, int level) 2791 { 2792 CPUX86State *env = &cpu->env; 2793 int i; 2794 2795 kvm_msr_buf_reset(cpu); 2796 2797 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, env->sysenter_cs); 2798 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, env->sysenter_esp); 2799 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, env->sysenter_eip); 2800 kvm_msr_entry_add(cpu, MSR_PAT, env->pat); 2801 if (has_msr_star) { 2802 kvm_msr_entry_add(cpu, MSR_STAR, env->star); 2803 } 2804 if (has_msr_hsave_pa) { 2805 kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, env->vm_hsave); 2806 } 2807 if (has_msr_tsc_aux) { 2808 kvm_msr_entry_add(cpu, MSR_TSC_AUX, env->tsc_aux); 2809 } 2810 if (has_msr_tsc_adjust) { 2811 kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, env->tsc_adjust); 2812 } 2813 if (has_msr_misc_enable) { 2814 kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE, 2815 env->msr_ia32_misc_enable); 2816 } 2817 if (has_msr_smbase) { 2818 kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, env->smbase); 2819 } 2820 if (has_msr_smi_count) { 2821 kvm_msr_entry_add(cpu, MSR_SMI_COUNT, env->msr_smi_count); 2822 } 2823 if (has_msr_pkrs) { 2824 kvm_msr_entry_add(cpu, MSR_IA32_PKRS, env->pkrs); 2825 } 2826 if (has_msr_bndcfgs) { 2827 kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, env->msr_bndcfgs); 2828 } 2829 if (has_msr_xss) { 2830 kvm_msr_entry_add(cpu, MSR_IA32_XSS, env->xss); 2831 } 2832 if (has_msr_umwait) { 2833 kvm_msr_entry_add(cpu, MSR_IA32_UMWAIT_CONTROL, env->umwait); 2834 } 2835 if (has_msr_spec_ctrl) { 2836 kvm_msr_entry_add(cpu, MSR_IA32_SPEC_CTRL, env->spec_ctrl); 2837 } 2838 if (has_msr_tsx_ctrl) { 2839 kvm_msr_entry_add(cpu, MSR_IA32_TSX_CTRL, env->tsx_ctrl); 2840 } 2841 if (has_msr_virt_ssbd) { 2842 kvm_msr_entry_add(cpu, MSR_VIRT_SSBD, env->virt_ssbd); 2843 } 2844 2845 #ifdef TARGET_X86_64 2846 if (lm_capable_kernel) { 2847 kvm_msr_entry_add(cpu, MSR_CSTAR, env->cstar); 2848 kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, env->kernelgsbase); 2849 kvm_msr_entry_add(cpu, MSR_FMASK, env->fmask); 2850 kvm_msr_entry_add(cpu, MSR_LSTAR, env->lstar); 2851 } 2852 #endif 2853 2854 /* 2855 * The following MSRs have side effects on the guest or are too heavy 2856 * for normal writeback. Limit them to reset or full state updates. 2857 */ 2858 if (level >= KVM_PUT_RESET_STATE) { 2859 kvm_msr_entry_add(cpu, MSR_IA32_TSC, env->tsc); 2860 kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, env->system_time_msr); 2861 kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, env->wall_clock_msr); 2862 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF_INT)) { 2863 kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_INT, env->async_pf_int_msr); 2864 } 2865 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF)) { 2866 kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, env->async_pf_en_msr); 2867 } 2868 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_PV_EOI)) { 2869 kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, env->pv_eoi_en_msr); 2870 } 2871 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_STEAL_TIME)) { 2872 kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, env->steal_time_msr); 2873 } 2874 2875 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_POLL_CONTROL)) { 2876 kvm_msr_entry_add(cpu, MSR_KVM_POLL_CONTROL, env->poll_control_msr); 2877 } 2878 2879 if (has_architectural_pmu_version > 0) { 2880 if (has_architectural_pmu_version > 1) { 2881 /* Stop the counter. */ 2882 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0); 2883 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0); 2884 } 2885 2886 /* Set the counter values. */ 2887 for (i = 0; i < num_architectural_pmu_fixed_counters; i++) { 2888 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i, 2889 env->msr_fixed_counters[i]); 2890 } 2891 for (i = 0; i < num_architectural_pmu_gp_counters; i++) { 2892 kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i, 2893 env->msr_gp_counters[i]); 2894 kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i, 2895 env->msr_gp_evtsel[i]); 2896 } 2897 if (has_architectural_pmu_version > 1) { 2898 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS, 2899 env->msr_global_status); 2900 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL, 2901 env->msr_global_ovf_ctrl); 2902 2903 /* Now start the PMU. */ 2904 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 2905 env->msr_fixed_ctr_ctrl); 2906 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 2907 env->msr_global_ctrl); 2908 } 2909 } 2910 /* 2911 * Hyper-V partition-wide MSRs: to avoid clearing them on cpu hot-add, 2912 * only sync them to KVM on the first cpu 2913 */ 2914 if (current_cpu == first_cpu) { 2915 if (has_msr_hv_hypercall) { 2916 kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID, 2917 env->msr_hv_guest_os_id); 2918 kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL, 2919 env->msr_hv_hypercall); 2920 } 2921 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_TIME)) { 2922 kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC, 2923 env->msr_hv_tsc); 2924 } 2925 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_REENLIGHTENMENT)) { 2926 kvm_msr_entry_add(cpu, HV_X64_MSR_REENLIGHTENMENT_CONTROL, 2927 env->msr_hv_reenlightenment_control); 2928 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_CONTROL, 2929 env->msr_hv_tsc_emulation_control); 2930 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_STATUS, 2931 env->msr_hv_tsc_emulation_status); 2932 } 2933 } 2934 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VAPIC)) { 2935 kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE, 2936 env->msr_hv_vapic); 2937 } 2938 if (has_msr_hv_crash) { 2939 int j; 2940 2941 for (j = 0; j < HV_CRASH_PARAMS; j++) 2942 kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j, 2943 env->msr_hv_crash_params[j]); 2944 2945 kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_NOTIFY); 2946 } 2947 if (has_msr_hv_runtime) { 2948 kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, env->msr_hv_runtime); 2949 } 2950 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX) 2951 && hv_vpindex_settable) { 2952 kvm_msr_entry_add(cpu, HV_X64_MSR_VP_INDEX, 2953 hyperv_vp_index(CPU(cpu))); 2954 } 2955 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) { 2956 int j; 2957 2958 kvm_msr_entry_add(cpu, HV_X64_MSR_SVERSION, HV_SYNIC_VERSION); 2959 2960 kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL, 2961 env->msr_hv_synic_control); 2962 kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP, 2963 env->msr_hv_synic_evt_page); 2964 kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP, 2965 env->msr_hv_synic_msg_page); 2966 2967 for (j = 0; j < ARRAY_SIZE(env->msr_hv_synic_sint); j++) { 2968 kvm_msr_entry_add(cpu, HV_X64_MSR_SINT0 + j, 2969 env->msr_hv_synic_sint[j]); 2970 } 2971 } 2972 if (has_msr_hv_stimer) { 2973 int j; 2974 2975 for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_config); j++) { 2976 kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_CONFIG + j * 2, 2977 env->msr_hv_stimer_config[j]); 2978 } 2979 2980 for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_count); j++) { 2981 kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_COUNT + j * 2, 2982 env->msr_hv_stimer_count[j]); 2983 } 2984 } 2985 if (env->features[FEAT_1_EDX] & CPUID_MTRR) { 2986 uint64_t phys_mask = MAKE_64BIT_MASK(0, cpu->phys_bits); 2987 2988 kvm_msr_entry_add(cpu, MSR_MTRRdefType, env->mtrr_deftype); 2989 kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, env->mtrr_fixed[0]); 2990 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, env->mtrr_fixed[1]); 2991 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, env->mtrr_fixed[2]); 2992 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, env->mtrr_fixed[3]); 2993 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, env->mtrr_fixed[4]); 2994 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, env->mtrr_fixed[5]); 2995 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, env->mtrr_fixed[6]); 2996 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, env->mtrr_fixed[7]); 2997 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, env->mtrr_fixed[8]); 2998 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, env->mtrr_fixed[9]); 2999 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, env->mtrr_fixed[10]); 3000 for (i = 0; i < MSR_MTRRcap_VCNT; i++) { 3001 /* The CPU GPs if we write to a bit above the physical limit of 3002 * the host CPU (and KVM emulates that) 3003 */ 3004 uint64_t mask = env->mtrr_var[i].mask; 3005 mask &= phys_mask; 3006 3007 kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i), 3008 env->mtrr_var[i].base); 3009 kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), mask); 3010 } 3011 } 3012 if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) { 3013 int addr_num = kvm_arch_get_supported_cpuid(kvm_state, 3014 0x14, 1, R_EAX) & 0x7; 3015 3016 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CTL, 3017 env->msr_rtit_ctrl); 3018 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_STATUS, 3019 env->msr_rtit_status); 3020 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_BASE, 3021 env->msr_rtit_output_base); 3022 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_MASK, 3023 env->msr_rtit_output_mask); 3024 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CR3_MATCH, 3025 env->msr_rtit_cr3_match); 3026 for (i = 0; i < addr_num; i++) { 3027 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_ADDR0_A + i, 3028 env->msr_rtit_addrs[i]); 3029 } 3030 } 3031 3032 /* Note: MSR_IA32_FEATURE_CONTROL is written separately, see 3033 * kvm_put_msr_feature_control. */ 3034 } 3035 3036 if (env->mcg_cap) { 3037 int i; 3038 3039 kvm_msr_entry_add(cpu, MSR_MCG_STATUS, env->mcg_status); 3040 kvm_msr_entry_add(cpu, MSR_MCG_CTL, env->mcg_ctl); 3041 if (has_msr_mcg_ext_ctl) { 3042 kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, env->mcg_ext_ctl); 3043 } 3044 for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) { 3045 kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, env->mce_banks[i]); 3046 } 3047 } 3048 3049 return kvm_buf_set_msrs(cpu); 3050 } 3051 3052 3053 static int kvm_get_fpu(X86CPU *cpu) 3054 { 3055 CPUX86State *env = &cpu->env; 3056 struct kvm_fpu fpu; 3057 int i, ret; 3058 3059 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_FPU, &fpu); 3060 if (ret < 0) { 3061 return ret; 3062 } 3063 3064 env->fpstt = (fpu.fsw >> 11) & 7; 3065 env->fpus = fpu.fsw; 3066 env->fpuc = fpu.fcw; 3067 env->fpop = fpu.last_opcode; 3068 env->fpip = fpu.last_ip; 3069 env->fpdp = fpu.last_dp; 3070 for (i = 0; i < 8; ++i) { 3071 env->fptags[i] = !((fpu.ftwx >> i) & 1); 3072 } 3073 memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs); 3074 for (i = 0; i < CPU_NB_REGS; i++) { 3075 env->xmm_regs[i].ZMM_Q(0) = ldq_p(&fpu.xmm[i][0]); 3076 env->xmm_regs[i].ZMM_Q(1) = ldq_p(&fpu.xmm[i][8]); 3077 } 3078 env->mxcsr = fpu.mxcsr; 3079 3080 return 0; 3081 } 3082 3083 static int kvm_get_xsave(X86CPU *cpu) 3084 { 3085 CPUX86State *env = &cpu->env; 3086 X86XSaveArea *xsave = env->xsave_buf; 3087 int ret; 3088 3089 if (!has_xsave) { 3090 return kvm_get_fpu(cpu); 3091 } 3092 3093 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XSAVE, xsave); 3094 if (ret < 0) { 3095 return ret; 3096 } 3097 x86_cpu_xrstor_all_areas(cpu, xsave); 3098 3099 return 0; 3100 } 3101 3102 static int kvm_get_xcrs(X86CPU *cpu) 3103 { 3104 CPUX86State *env = &cpu->env; 3105 int i, ret; 3106 struct kvm_xcrs xcrs; 3107 3108 if (!has_xcrs) { 3109 return 0; 3110 } 3111 3112 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XCRS, &xcrs); 3113 if (ret < 0) { 3114 return ret; 3115 } 3116 3117 for (i = 0; i < xcrs.nr_xcrs; i++) { 3118 /* Only support xcr0 now */ 3119 if (xcrs.xcrs[i].xcr == 0) { 3120 env->xcr0 = xcrs.xcrs[i].value; 3121 break; 3122 } 3123 } 3124 return 0; 3125 } 3126 3127 static int kvm_get_sregs(X86CPU *cpu) 3128 { 3129 CPUX86State *env = &cpu->env; 3130 struct kvm_sregs sregs; 3131 int bit, i, ret; 3132 3133 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs); 3134 if (ret < 0) { 3135 return ret; 3136 } 3137 3138 /* There can only be one pending IRQ set in the bitmap at a time, so try 3139 to find it and save its number instead (-1 for none). */ 3140 env->interrupt_injected = -1; 3141 for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) { 3142 if (sregs.interrupt_bitmap[i]) { 3143 bit = ctz64(sregs.interrupt_bitmap[i]); 3144 env->interrupt_injected = i * 64 + bit; 3145 break; 3146 } 3147 } 3148 3149 get_seg(&env->segs[R_CS], &sregs.cs); 3150 get_seg(&env->segs[R_DS], &sregs.ds); 3151 get_seg(&env->segs[R_ES], &sregs.es); 3152 get_seg(&env->segs[R_FS], &sregs.fs); 3153 get_seg(&env->segs[R_GS], &sregs.gs); 3154 get_seg(&env->segs[R_SS], &sregs.ss); 3155 3156 get_seg(&env->tr, &sregs.tr); 3157 get_seg(&env->ldt, &sregs.ldt); 3158 3159 env->idt.limit = sregs.idt.limit; 3160 env->idt.base = sregs.idt.base; 3161 env->gdt.limit = sregs.gdt.limit; 3162 env->gdt.base = sregs.gdt.base; 3163 3164 env->cr[0] = sregs.cr0; 3165 env->cr[2] = sregs.cr2; 3166 env->cr[3] = sregs.cr3; 3167 env->cr[4] = sregs.cr4; 3168 3169 env->efer = sregs.efer; 3170 3171 /* changes to apic base and cr8/tpr are read back via kvm_arch_post_run */ 3172 x86_update_hflags(env); 3173 3174 return 0; 3175 } 3176 3177 static int kvm_get_msrs(X86CPU *cpu) 3178 { 3179 CPUX86State *env = &cpu->env; 3180 struct kvm_msr_entry *msrs = cpu->kvm_msr_buf->entries; 3181 int ret, i; 3182 uint64_t mtrr_top_bits; 3183 3184 kvm_msr_buf_reset(cpu); 3185 3186 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, 0); 3187 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, 0); 3188 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, 0); 3189 kvm_msr_entry_add(cpu, MSR_PAT, 0); 3190 if (has_msr_star) { 3191 kvm_msr_entry_add(cpu, MSR_STAR, 0); 3192 } 3193 if (has_msr_hsave_pa) { 3194 kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, 0); 3195 } 3196 if (has_msr_tsc_aux) { 3197 kvm_msr_entry_add(cpu, MSR_TSC_AUX, 0); 3198 } 3199 if (has_msr_tsc_adjust) { 3200 kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, 0); 3201 } 3202 if (has_msr_tsc_deadline) { 3203 kvm_msr_entry_add(cpu, MSR_IA32_TSCDEADLINE, 0); 3204 } 3205 if (has_msr_misc_enable) { 3206 kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE, 0); 3207 } 3208 if (has_msr_smbase) { 3209 kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, 0); 3210 } 3211 if (has_msr_smi_count) { 3212 kvm_msr_entry_add(cpu, MSR_SMI_COUNT, 0); 3213 } 3214 if (has_msr_feature_control) { 3215 kvm_msr_entry_add(cpu, MSR_IA32_FEATURE_CONTROL, 0); 3216 } 3217 if (has_msr_pkrs) { 3218 kvm_msr_entry_add(cpu, MSR_IA32_PKRS, 0); 3219 } 3220 if (has_msr_bndcfgs) { 3221 kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, 0); 3222 } 3223 if (has_msr_xss) { 3224 kvm_msr_entry_add(cpu, MSR_IA32_XSS, 0); 3225 } 3226 if (has_msr_umwait) { 3227 kvm_msr_entry_add(cpu, MSR_IA32_UMWAIT_CONTROL, 0); 3228 } 3229 if (has_msr_spec_ctrl) { 3230 kvm_msr_entry_add(cpu, MSR_IA32_SPEC_CTRL, 0); 3231 } 3232 if (has_msr_tsx_ctrl) { 3233 kvm_msr_entry_add(cpu, MSR_IA32_TSX_CTRL, 0); 3234 } 3235 if (has_msr_virt_ssbd) { 3236 kvm_msr_entry_add(cpu, MSR_VIRT_SSBD, 0); 3237 } 3238 if (!env->tsc_valid) { 3239 kvm_msr_entry_add(cpu, MSR_IA32_TSC, 0); 3240 env->tsc_valid = !runstate_is_running(); 3241 } 3242 3243 #ifdef TARGET_X86_64 3244 if (lm_capable_kernel) { 3245 kvm_msr_entry_add(cpu, MSR_CSTAR, 0); 3246 kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, 0); 3247 kvm_msr_entry_add(cpu, MSR_FMASK, 0); 3248 kvm_msr_entry_add(cpu, MSR_LSTAR, 0); 3249 } 3250 #endif 3251 kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, 0); 3252 kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, 0); 3253 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF_INT)) { 3254 kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_INT, 0); 3255 } 3256 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF)) { 3257 kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, 0); 3258 } 3259 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_PV_EOI)) { 3260 kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, 0); 3261 } 3262 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_STEAL_TIME)) { 3263 kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, 0); 3264 } 3265 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_POLL_CONTROL)) { 3266 kvm_msr_entry_add(cpu, MSR_KVM_POLL_CONTROL, 1); 3267 } 3268 if (has_architectural_pmu_version > 0) { 3269 if (has_architectural_pmu_version > 1) { 3270 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0); 3271 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0); 3272 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS, 0); 3273 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL, 0); 3274 } 3275 for (i = 0; i < num_architectural_pmu_fixed_counters; i++) { 3276 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i, 0); 3277 } 3278 for (i = 0; i < num_architectural_pmu_gp_counters; i++) { 3279 kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i, 0); 3280 kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i, 0); 3281 } 3282 } 3283 3284 if (env->mcg_cap) { 3285 kvm_msr_entry_add(cpu, MSR_MCG_STATUS, 0); 3286 kvm_msr_entry_add(cpu, MSR_MCG_CTL, 0); 3287 if (has_msr_mcg_ext_ctl) { 3288 kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, 0); 3289 } 3290 for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) { 3291 kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, 0); 3292 } 3293 } 3294 3295 if (has_msr_hv_hypercall) { 3296 kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL, 0); 3297 kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID, 0); 3298 } 3299 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VAPIC)) { 3300 kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE, 0); 3301 } 3302 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_TIME)) { 3303 kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC, 0); 3304 } 3305 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_REENLIGHTENMENT)) { 3306 kvm_msr_entry_add(cpu, HV_X64_MSR_REENLIGHTENMENT_CONTROL, 0); 3307 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_CONTROL, 0); 3308 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_STATUS, 0); 3309 } 3310 if (has_msr_hv_crash) { 3311 int j; 3312 3313 for (j = 0; j < HV_CRASH_PARAMS; j++) { 3314 kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j, 0); 3315 } 3316 } 3317 if (has_msr_hv_runtime) { 3318 kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, 0); 3319 } 3320 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) { 3321 uint32_t msr; 3322 3323 kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL, 0); 3324 kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP, 0); 3325 kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP, 0); 3326 for (msr = HV_X64_MSR_SINT0; msr <= HV_X64_MSR_SINT15; msr++) { 3327 kvm_msr_entry_add(cpu, msr, 0); 3328 } 3329 } 3330 if (has_msr_hv_stimer) { 3331 uint32_t msr; 3332 3333 for (msr = HV_X64_MSR_STIMER0_CONFIG; msr <= HV_X64_MSR_STIMER3_COUNT; 3334 msr++) { 3335 kvm_msr_entry_add(cpu, msr, 0); 3336 } 3337 } 3338 if (env->features[FEAT_1_EDX] & CPUID_MTRR) { 3339 kvm_msr_entry_add(cpu, MSR_MTRRdefType, 0); 3340 kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, 0); 3341 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, 0); 3342 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, 0); 3343 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, 0); 3344 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, 0); 3345 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, 0); 3346 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, 0); 3347 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, 0); 3348 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, 0); 3349 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, 0); 3350 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, 0); 3351 for (i = 0; i < MSR_MTRRcap_VCNT; i++) { 3352 kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i), 0); 3353 kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), 0); 3354 } 3355 } 3356 3357 if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) { 3358 int addr_num = 3359 kvm_arch_get_supported_cpuid(kvm_state, 0x14, 1, R_EAX) & 0x7; 3360 3361 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CTL, 0); 3362 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_STATUS, 0); 3363 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_BASE, 0); 3364 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_MASK, 0); 3365 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CR3_MATCH, 0); 3366 for (i = 0; i < addr_num; i++) { 3367 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_ADDR0_A + i, 0); 3368 } 3369 } 3370 3371 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, cpu->kvm_msr_buf); 3372 if (ret < 0) { 3373 return ret; 3374 } 3375 3376 if (ret < cpu->kvm_msr_buf->nmsrs) { 3377 struct kvm_msr_entry *e = &cpu->kvm_msr_buf->entries[ret]; 3378 error_report("error: failed to get MSR 0x%" PRIx32, 3379 (uint32_t)e->index); 3380 } 3381 3382 assert(ret == cpu->kvm_msr_buf->nmsrs); 3383 /* 3384 * MTRR masks: Each mask consists of 5 parts 3385 * a 10..0: must be zero 3386 * b 11 : valid bit 3387 * c n-1.12: actual mask bits 3388 * d 51..n: reserved must be zero 3389 * e 63.52: reserved must be zero 3390 * 3391 * 'n' is the number of physical bits supported by the CPU and is 3392 * apparently always <= 52. We know our 'n' but don't know what 3393 * the destinations 'n' is; it might be smaller, in which case 3394 * it masks (c) on loading. It might be larger, in which case 3395 * we fill 'd' so that d..c is consistent irrespetive of the 'n' 3396 * we're migrating to. 3397 */ 3398 3399 if (cpu->fill_mtrr_mask) { 3400 QEMU_BUILD_BUG_ON(TARGET_PHYS_ADDR_SPACE_BITS > 52); 3401 assert(cpu->phys_bits <= TARGET_PHYS_ADDR_SPACE_BITS); 3402 mtrr_top_bits = MAKE_64BIT_MASK(cpu->phys_bits, 52 - cpu->phys_bits); 3403 } else { 3404 mtrr_top_bits = 0; 3405 } 3406 3407 for (i = 0; i < ret; i++) { 3408 uint32_t index = msrs[i].index; 3409 switch (index) { 3410 case MSR_IA32_SYSENTER_CS: 3411 env->sysenter_cs = msrs[i].data; 3412 break; 3413 case MSR_IA32_SYSENTER_ESP: 3414 env->sysenter_esp = msrs[i].data; 3415 break; 3416 case MSR_IA32_SYSENTER_EIP: 3417 env->sysenter_eip = msrs[i].data; 3418 break; 3419 case MSR_PAT: 3420 env->pat = msrs[i].data; 3421 break; 3422 case MSR_STAR: 3423 env->star = msrs[i].data; 3424 break; 3425 #ifdef TARGET_X86_64 3426 case MSR_CSTAR: 3427 env->cstar = msrs[i].data; 3428 break; 3429 case MSR_KERNELGSBASE: 3430 env->kernelgsbase = msrs[i].data; 3431 break; 3432 case MSR_FMASK: 3433 env->fmask = msrs[i].data; 3434 break; 3435 case MSR_LSTAR: 3436 env->lstar = msrs[i].data; 3437 break; 3438 #endif 3439 case MSR_IA32_TSC: 3440 env->tsc = msrs[i].data; 3441 break; 3442 case MSR_TSC_AUX: 3443 env->tsc_aux = msrs[i].data; 3444 break; 3445 case MSR_TSC_ADJUST: 3446 env->tsc_adjust = msrs[i].data; 3447 break; 3448 case MSR_IA32_TSCDEADLINE: 3449 env->tsc_deadline = msrs[i].data; 3450 break; 3451 case MSR_VM_HSAVE_PA: 3452 env->vm_hsave = msrs[i].data; 3453 break; 3454 case MSR_KVM_SYSTEM_TIME: 3455 env->system_time_msr = msrs[i].data; 3456 break; 3457 case MSR_KVM_WALL_CLOCK: 3458 env->wall_clock_msr = msrs[i].data; 3459 break; 3460 case MSR_MCG_STATUS: 3461 env->mcg_status = msrs[i].data; 3462 break; 3463 case MSR_MCG_CTL: 3464 env->mcg_ctl = msrs[i].data; 3465 break; 3466 case MSR_MCG_EXT_CTL: 3467 env->mcg_ext_ctl = msrs[i].data; 3468 break; 3469 case MSR_IA32_MISC_ENABLE: 3470 env->msr_ia32_misc_enable = msrs[i].data; 3471 break; 3472 case MSR_IA32_SMBASE: 3473 env->smbase = msrs[i].data; 3474 break; 3475 case MSR_SMI_COUNT: 3476 env->msr_smi_count = msrs[i].data; 3477 break; 3478 case MSR_IA32_FEATURE_CONTROL: 3479 env->msr_ia32_feature_control = msrs[i].data; 3480 break; 3481 case MSR_IA32_BNDCFGS: 3482 env->msr_bndcfgs = msrs[i].data; 3483 break; 3484 case MSR_IA32_XSS: 3485 env->xss = msrs[i].data; 3486 break; 3487 case MSR_IA32_UMWAIT_CONTROL: 3488 env->umwait = msrs[i].data; 3489 break; 3490 case MSR_IA32_PKRS: 3491 env->pkrs = msrs[i].data; 3492 break; 3493 default: 3494 if (msrs[i].index >= MSR_MC0_CTL && 3495 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) { 3496 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data; 3497 } 3498 break; 3499 case MSR_KVM_ASYNC_PF_EN: 3500 env->async_pf_en_msr = msrs[i].data; 3501 break; 3502 case MSR_KVM_ASYNC_PF_INT: 3503 env->async_pf_int_msr = msrs[i].data; 3504 break; 3505 case MSR_KVM_PV_EOI_EN: 3506 env->pv_eoi_en_msr = msrs[i].data; 3507 break; 3508 case MSR_KVM_STEAL_TIME: 3509 env->steal_time_msr = msrs[i].data; 3510 break; 3511 case MSR_KVM_POLL_CONTROL: { 3512 env->poll_control_msr = msrs[i].data; 3513 break; 3514 } 3515 case MSR_CORE_PERF_FIXED_CTR_CTRL: 3516 env->msr_fixed_ctr_ctrl = msrs[i].data; 3517 break; 3518 case MSR_CORE_PERF_GLOBAL_CTRL: 3519 env->msr_global_ctrl = msrs[i].data; 3520 break; 3521 case MSR_CORE_PERF_GLOBAL_STATUS: 3522 env->msr_global_status = msrs[i].data; 3523 break; 3524 case MSR_CORE_PERF_GLOBAL_OVF_CTRL: 3525 env->msr_global_ovf_ctrl = msrs[i].data; 3526 break; 3527 case MSR_CORE_PERF_FIXED_CTR0 ... MSR_CORE_PERF_FIXED_CTR0 + MAX_FIXED_COUNTERS - 1: 3528 env->msr_fixed_counters[index - MSR_CORE_PERF_FIXED_CTR0] = msrs[i].data; 3529 break; 3530 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR0 + MAX_GP_COUNTERS - 1: 3531 env->msr_gp_counters[index - MSR_P6_PERFCTR0] = msrs[i].data; 3532 break; 3533 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL0 + MAX_GP_COUNTERS - 1: 3534 env->msr_gp_evtsel[index - MSR_P6_EVNTSEL0] = msrs[i].data; 3535 break; 3536 case HV_X64_MSR_HYPERCALL: 3537 env->msr_hv_hypercall = msrs[i].data; 3538 break; 3539 case HV_X64_MSR_GUEST_OS_ID: 3540 env->msr_hv_guest_os_id = msrs[i].data; 3541 break; 3542 case HV_X64_MSR_APIC_ASSIST_PAGE: 3543 env->msr_hv_vapic = msrs[i].data; 3544 break; 3545 case HV_X64_MSR_REFERENCE_TSC: 3546 env->msr_hv_tsc = msrs[i].data; 3547 break; 3548 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: 3549 env->msr_hv_crash_params[index - HV_X64_MSR_CRASH_P0] = msrs[i].data; 3550 break; 3551 case HV_X64_MSR_VP_RUNTIME: 3552 env->msr_hv_runtime = msrs[i].data; 3553 break; 3554 case HV_X64_MSR_SCONTROL: 3555 env->msr_hv_synic_control = msrs[i].data; 3556 break; 3557 case HV_X64_MSR_SIEFP: 3558 env->msr_hv_synic_evt_page = msrs[i].data; 3559 break; 3560 case HV_X64_MSR_SIMP: 3561 env->msr_hv_synic_msg_page = msrs[i].data; 3562 break; 3563 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: 3564 env->msr_hv_synic_sint[index - HV_X64_MSR_SINT0] = msrs[i].data; 3565 break; 3566 case HV_X64_MSR_STIMER0_CONFIG: 3567 case HV_X64_MSR_STIMER1_CONFIG: 3568 case HV_X64_MSR_STIMER2_CONFIG: 3569 case HV_X64_MSR_STIMER3_CONFIG: 3570 env->msr_hv_stimer_config[(index - HV_X64_MSR_STIMER0_CONFIG)/2] = 3571 msrs[i].data; 3572 break; 3573 case HV_X64_MSR_STIMER0_COUNT: 3574 case HV_X64_MSR_STIMER1_COUNT: 3575 case HV_X64_MSR_STIMER2_COUNT: 3576 case HV_X64_MSR_STIMER3_COUNT: 3577 env->msr_hv_stimer_count[(index - HV_X64_MSR_STIMER0_COUNT)/2] = 3578 msrs[i].data; 3579 break; 3580 case HV_X64_MSR_REENLIGHTENMENT_CONTROL: 3581 env->msr_hv_reenlightenment_control = msrs[i].data; 3582 break; 3583 case HV_X64_MSR_TSC_EMULATION_CONTROL: 3584 env->msr_hv_tsc_emulation_control = msrs[i].data; 3585 break; 3586 case HV_X64_MSR_TSC_EMULATION_STATUS: 3587 env->msr_hv_tsc_emulation_status = msrs[i].data; 3588 break; 3589 case MSR_MTRRdefType: 3590 env->mtrr_deftype = msrs[i].data; 3591 break; 3592 case MSR_MTRRfix64K_00000: 3593 env->mtrr_fixed[0] = msrs[i].data; 3594 break; 3595 case MSR_MTRRfix16K_80000: 3596 env->mtrr_fixed[1] = msrs[i].data; 3597 break; 3598 case MSR_MTRRfix16K_A0000: 3599 env->mtrr_fixed[2] = msrs[i].data; 3600 break; 3601 case MSR_MTRRfix4K_C0000: 3602 env->mtrr_fixed[3] = msrs[i].data; 3603 break; 3604 case MSR_MTRRfix4K_C8000: 3605 env->mtrr_fixed[4] = msrs[i].data; 3606 break; 3607 case MSR_MTRRfix4K_D0000: 3608 env->mtrr_fixed[5] = msrs[i].data; 3609 break; 3610 case MSR_MTRRfix4K_D8000: 3611 env->mtrr_fixed[6] = msrs[i].data; 3612 break; 3613 case MSR_MTRRfix4K_E0000: 3614 env->mtrr_fixed[7] = msrs[i].data; 3615 break; 3616 case MSR_MTRRfix4K_E8000: 3617 env->mtrr_fixed[8] = msrs[i].data; 3618 break; 3619 case MSR_MTRRfix4K_F0000: 3620 env->mtrr_fixed[9] = msrs[i].data; 3621 break; 3622 case MSR_MTRRfix4K_F8000: 3623 env->mtrr_fixed[10] = msrs[i].data; 3624 break; 3625 case MSR_MTRRphysBase(0) ... MSR_MTRRphysMask(MSR_MTRRcap_VCNT - 1): 3626 if (index & 1) { 3627 env->mtrr_var[MSR_MTRRphysIndex(index)].mask = msrs[i].data | 3628 mtrr_top_bits; 3629 } else { 3630 env->mtrr_var[MSR_MTRRphysIndex(index)].base = msrs[i].data; 3631 } 3632 break; 3633 case MSR_IA32_SPEC_CTRL: 3634 env->spec_ctrl = msrs[i].data; 3635 break; 3636 case MSR_IA32_TSX_CTRL: 3637 env->tsx_ctrl = msrs[i].data; 3638 break; 3639 case MSR_VIRT_SSBD: 3640 env->virt_ssbd = msrs[i].data; 3641 break; 3642 case MSR_IA32_RTIT_CTL: 3643 env->msr_rtit_ctrl = msrs[i].data; 3644 break; 3645 case MSR_IA32_RTIT_STATUS: 3646 env->msr_rtit_status = msrs[i].data; 3647 break; 3648 case MSR_IA32_RTIT_OUTPUT_BASE: 3649 env->msr_rtit_output_base = msrs[i].data; 3650 break; 3651 case MSR_IA32_RTIT_OUTPUT_MASK: 3652 env->msr_rtit_output_mask = msrs[i].data; 3653 break; 3654 case MSR_IA32_RTIT_CR3_MATCH: 3655 env->msr_rtit_cr3_match = msrs[i].data; 3656 break; 3657 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: 3658 env->msr_rtit_addrs[index - MSR_IA32_RTIT_ADDR0_A] = msrs[i].data; 3659 break; 3660 } 3661 } 3662 3663 return 0; 3664 } 3665 3666 static int kvm_put_mp_state(X86CPU *cpu) 3667 { 3668 struct kvm_mp_state mp_state = { .mp_state = cpu->env.mp_state }; 3669 3670 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state); 3671 } 3672 3673 static int kvm_get_mp_state(X86CPU *cpu) 3674 { 3675 CPUState *cs = CPU(cpu); 3676 CPUX86State *env = &cpu->env; 3677 struct kvm_mp_state mp_state; 3678 int ret; 3679 3680 ret = kvm_vcpu_ioctl(cs, KVM_GET_MP_STATE, &mp_state); 3681 if (ret < 0) { 3682 return ret; 3683 } 3684 env->mp_state = mp_state.mp_state; 3685 if (kvm_irqchip_in_kernel()) { 3686 cs->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED); 3687 } 3688 return 0; 3689 } 3690 3691 static int kvm_get_apic(X86CPU *cpu) 3692 { 3693 DeviceState *apic = cpu->apic_state; 3694 struct kvm_lapic_state kapic; 3695 int ret; 3696 3697 if (apic && kvm_irqchip_in_kernel()) { 3698 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_LAPIC, &kapic); 3699 if (ret < 0) { 3700 return ret; 3701 } 3702 3703 kvm_get_apic_state(apic, &kapic); 3704 } 3705 return 0; 3706 } 3707 3708 static int kvm_put_vcpu_events(X86CPU *cpu, int level) 3709 { 3710 CPUState *cs = CPU(cpu); 3711 CPUX86State *env = &cpu->env; 3712 struct kvm_vcpu_events events = {}; 3713 3714 if (!kvm_has_vcpu_events()) { 3715 return 0; 3716 } 3717 3718 events.flags = 0; 3719 3720 if (has_exception_payload) { 3721 events.flags |= KVM_VCPUEVENT_VALID_PAYLOAD; 3722 events.exception.pending = env->exception_pending; 3723 events.exception_has_payload = env->exception_has_payload; 3724 events.exception_payload = env->exception_payload; 3725 } 3726 events.exception.nr = env->exception_nr; 3727 events.exception.injected = env->exception_injected; 3728 events.exception.has_error_code = env->has_error_code; 3729 events.exception.error_code = env->error_code; 3730 3731 events.interrupt.injected = (env->interrupt_injected >= 0); 3732 events.interrupt.nr = env->interrupt_injected; 3733 events.interrupt.soft = env->soft_interrupt; 3734 3735 events.nmi.injected = env->nmi_injected; 3736 events.nmi.pending = env->nmi_pending; 3737 events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK); 3738 3739 events.sipi_vector = env->sipi_vector; 3740 3741 if (has_msr_smbase) { 3742 events.smi.smm = !!(env->hflags & HF_SMM_MASK); 3743 events.smi.smm_inside_nmi = !!(env->hflags2 & HF2_SMM_INSIDE_NMI_MASK); 3744 if (kvm_irqchip_in_kernel()) { 3745 /* As soon as these are moved to the kernel, remove them 3746 * from cs->interrupt_request. 3747 */ 3748 events.smi.pending = cs->interrupt_request & CPU_INTERRUPT_SMI; 3749 events.smi.latched_init = cs->interrupt_request & CPU_INTERRUPT_INIT; 3750 cs->interrupt_request &= ~(CPU_INTERRUPT_INIT | CPU_INTERRUPT_SMI); 3751 } else { 3752 /* Keep these in cs->interrupt_request. */ 3753 events.smi.pending = 0; 3754 events.smi.latched_init = 0; 3755 } 3756 /* Stop SMI delivery on old machine types to avoid a reboot 3757 * on an inward migration of an old VM. 3758 */ 3759 if (!cpu->kvm_no_smi_migration) { 3760 events.flags |= KVM_VCPUEVENT_VALID_SMM; 3761 } 3762 } 3763 3764 if (level >= KVM_PUT_RESET_STATE) { 3765 events.flags |= KVM_VCPUEVENT_VALID_NMI_PENDING; 3766 if (env->mp_state == KVM_MP_STATE_SIPI_RECEIVED) { 3767 events.flags |= KVM_VCPUEVENT_VALID_SIPI_VECTOR; 3768 } 3769 } 3770 3771 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events); 3772 } 3773 3774 static int kvm_get_vcpu_events(X86CPU *cpu) 3775 { 3776 CPUX86State *env = &cpu->env; 3777 struct kvm_vcpu_events events; 3778 int ret; 3779 3780 if (!kvm_has_vcpu_events()) { 3781 return 0; 3782 } 3783 3784 memset(&events, 0, sizeof(events)); 3785 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events); 3786 if (ret < 0) { 3787 return ret; 3788 } 3789 3790 if (events.flags & KVM_VCPUEVENT_VALID_PAYLOAD) { 3791 env->exception_pending = events.exception.pending; 3792 env->exception_has_payload = events.exception_has_payload; 3793 env->exception_payload = events.exception_payload; 3794 } else { 3795 env->exception_pending = 0; 3796 env->exception_has_payload = false; 3797 } 3798 env->exception_injected = events.exception.injected; 3799 env->exception_nr = 3800 (env->exception_pending || env->exception_injected) ? 3801 events.exception.nr : -1; 3802 env->has_error_code = events.exception.has_error_code; 3803 env->error_code = events.exception.error_code; 3804 3805 env->interrupt_injected = 3806 events.interrupt.injected ? events.interrupt.nr : -1; 3807 env->soft_interrupt = events.interrupt.soft; 3808 3809 env->nmi_injected = events.nmi.injected; 3810 env->nmi_pending = events.nmi.pending; 3811 if (events.nmi.masked) { 3812 env->hflags2 |= HF2_NMI_MASK; 3813 } else { 3814 env->hflags2 &= ~HF2_NMI_MASK; 3815 } 3816 3817 if (events.flags & KVM_VCPUEVENT_VALID_SMM) { 3818 if (events.smi.smm) { 3819 env->hflags |= HF_SMM_MASK; 3820 } else { 3821 env->hflags &= ~HF_SMM_MASK; 3822 } 3823 if (events.smi.pending) { 3824 cpu_interrupt(CPU(cpu), CPU_INTERRUPT_SMI); 3825 } else { 3826 cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_SMI); 3827 } 3828 if (events.smi.smm_inside_nmi) { 3829 env->hflags2 |= HF2_SMM_INSIDE_NMI_MASK; 3830 } else { 3831 env->hflags2 &= ~HF2_SMM_INSIDE_NMI_MASK; 3832 } 3833 if (events.smi.latched_init) { 3834 cpu_interrupt(CPU(cpu), CPU_INTERRUPT_INIT); 3835 } else { 3836 cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_INIT); 3837 } 3838 } 3839 3840 env->sipi_vector = events.sipi_vector; 3841 3842 return 0; 3843 } 3844 3845 static int kvm_guest_debug_workarounds(X86CPU *cpu) 3846 { 3847 CPUState *cs = CPU(cpu); 3848 CPUX86State *env = &cpu->env; 3849 int ret = 0; 3850 unsigned long reinject_trap = 0; 3851 3852 if (!kvm_has_vcpu_events()) { 3853 if (env->exception_nr == EXCP01_DB) { 3854 reinject_trap = KVM_GUESTDBG_INJECT_DB; 3855 } else if (env->exception_injected == EXCP03_INT3) { 3856 reinject_trap = KVM_GUESTDBG_INJECT_BP; 3857 } 3858 kvm_reset_exception(env); 3859 } 3860 3861 /* 3862 * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF 3863 * injected via SET_GUEST_DEBUG while updating GP regs. Work around this 3864 * by updating the debug state once again if single-stepping is on. 3865 * Another reason to call kvm_update_guest_debug here is a pending debug 3866 * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to 3867 * reinject them via SET_GUEST_DEBUG. 3868 */ 3869 if (reinject_trap || 3870 (!kvm_has_robust_singlestep() && cs->singlestep_enabled)) { 3871 ret = kvm_update_guest_debug(cs, reinject_trap); 3872 } 3873 return ret; 3874 } 3875 3876 static int kvm_put_debugregs(X86CPU *cpu) 3877 { 3878 CPUX86State *env = &cpu->env; 3879 struct kvm_debugregs dbgregs; 3880 int i; 3881 3882 if (!kvm_has_debugregs()) { 3883 return 0; 3884 } 3885 3886 memset(&dbgregs, 0, sizeof(dbgregs)); 3887 for (i = 0; i < 4; i++) { 3888 dbgregs.db[i] = env->dr[i]; 3889 } 3890 dbgregs.dr6 = env->dr[6]; 3891 dbgregs.dr7 = env->dr[7]; 3892 dbgregs.flags = 0; 3893 3894 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEBUGREGS, &dbgregs); 3895 } 3896 3897 static int kvm_get_debugregs(X86CPU *cpu) 3898 { 3899 CPUX86State *env = &cpu->env; 3900 struct kvm_debugregs dbgregs; 3901 int i, ret; 3902 3903 if (!kvm_has_debugregs()) { 3904 return 0; 3905 } 3906 3907 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_DEBUGREGS, &dbgregs); 3908 if (ret < 0) { 3909 return ret; 3910 } 3911 for (i = 0; i < 4; i++) { 3912 env->dr[i] = dbgregs.db[i]; 3913 } 3914 env->dr[4] = env->dr[6] = dbgregs.dr6; 3915 env->dr[5] = env->dr[7] = dbgregs.dr7; 3916 3917 return 0; 3918 } 3919 3920 static int kvm_put_nested_state(X86CPU *cpu) 3921 { 3922 CPUX86State *env = &cpu->env; 3923 int max_nested_state_len = kvm_max_nested_state_length(); 3924 3925 if (!env->nested_state) { 3926 return 0; 3927 } 3928 3929 /* 3930 * Copy flags that are affected by reset from env->hflags and env->hflags2. 3931 */ 3932 if (env->hflags & HF_GUEST_MASK) { 3933 env->nested_state->flags |= KVM_STATE_NESTED_GUEST_MODE; 3934 } else { 3935 env->nested_state->flags &= ~KVM_STATE_NESTED_GUEST_MODE; 3936 } 3937 3938 /* Don't set KVM_STATE_NESTED_GIF_SET on VMX as it is illegal */ 3939 if (cpu_has_svm(env) && (env->hflags2 & HF2_GIF_MASK)) { 3940 env->nested_state->flags |= KVM_STATE_NESTED_GIF_SET; 3941 } else { 3942 env->nested_state->flags &= ~KVM_STATE_NESTED_GIF_SET; 3943 } 3944 3945 assert(env->nested_state->size <= max_nested_state_len); 3946 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_NESTED_STATE, env->nested_state); 3947 } 3948 3949 static int kvm_get_nested_state(X86CPU *cpu) 3950 { 3951 CPUX86State *env = &cpu->env; 3952 int max_nested_state_len = kvm_max_nested_state_length(); 3953 int ret; 3954 3955 if (!env->nested_state) { 3956 return 0; 3957 } 3958 3959 /* 3960 * It is possible that migration restored a smaller size into 3961 * nested_state->hdr.size than what our kernel support. 3962 * We preserve migration origin nested_state->hdr.size for 3963 * call to KVM_SET_NESTED_STATE but wish that our next call 3964 * to KVM_GET_NESTED_STATE will use max size our kernel support. 3965 */ 3966 env->nested_state->size = max_nested_state_len; 3967 3968 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_NESTED_STATE, env->nested_state); 3969 if (ret < 0) { 3970 return ret; 3971 } 3972 3973 /* 3974 * Copy flags that are affected by reset to env->hflags and env->hflags2. 3975 */ 3976 if (env->nested_state->flags & KVM_STATE_NESTED_GUEST_MODE) { 3977 env->hflags |= HF_GUEST_MASK; 3978 } else { 3979 env->hflags &= ~HF_GUEST_MASK; 3980 } 3981 3982 /* Keep HF2_GIF_MASK set on !SVM as x86_cpu_pending_interrupt() needs it */ 3983 if (cpu_has_svm(env)) { 3984 if (env->nested_state->flags & KVM_STATE_NESTED_GIF_SET) { 3985 env->hflags2 |= HF2_GIF_MASK; 3986 } else { 3987 env->hflags2 &= ~HF2_GIF_MASK; 3988 } 3989 } 3990 3991 return ret; 3992 } 3993 3994 int kvm_arch_put_registers(CPUState *cpu, int level) 3995 { 3996 X86CPU *x86_cpu = X86_CPU(cpu); 3997 int ret; 3998 3999 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu)); 4000 4001 /* must be before kvm_put_nested_state so that EFER.SVME is set */ 4002 ret = kvm_put_sregs(x86_cpu); 4003 if (ret < 0) { 4004 return ret; 4005 } 4006 4007 if (level >= KVM_PUT_RESET_STATE) { 4008 ret = kvm_put_nested_state(x86_cpu); 4009 if (ret < 0) { 4010 return ret; 4011 } 4012 4013 ret = kvm_put_msr_feature_control(x86_cpu); 4014 if (ret < 0) { 4015 return ret; 4016 } 4017 } 4018 4019 if (level == KVM_PUT_FULL_STATE) { 4020 /* We don't check for kvm_arch_set_tsc_khz() errors here, 4021 * because TSC frequency mismatch shouldn't abort migration, 4022 * unless the user explicitly asked for a more strict TSC 4023 * setting (e.g. using an explicit "tsc-freq" option). 4024 */ 4025 kvm_arch_set_tsc_khz(cpu); 4026 } 4027 4028 ret = kvm_getput_regs(x86_cpu, 1); 4029 if (ret < 0) { 4030 return ret; 4031 } 4032 ret = kvm_put_xsave(x86_cpu); 4033 if (ret < 0) { 4034 return ret; 4035 } 4036 ret = kvm_put_xcrs(x86_cpu); 4037 if (ret < 0) { 4038 return ret; 4039 } 4040 /* must be before kvm_put_msrs */ 4041 ret = kvm_inject_mce_oldstyle(x86_cpu); 4042 if (ret < 0) { 4043 return ret; 4044 } 4045 ret = kvm_put_msrs(x86_cpu, level); 4046 if (ret < 0) { 4047 return ret; 4048 } 4049 ret = kvm_put_vcpu_events(x86_cpu, level); 4050 if (ret < 0) { 4051 return ret; 4052 } 4053 if (level >= KVM_PUT_RESET_STATE) { 4054 ret = kvm_put_mp_state(x86_cpu); 4055 if (ret < 0) { 4056 return ret; 4057 } 4058 } 4059 4060 ret = kvm_put_tscdeadline_msr(x86_cpu); 4061 if (ret < 0) { 4062 return ret; 4063 } 4064 ret = kvm_put_debugregs(x86_cpu); 4065 if (ret < 0) { 4066 return ret; 4067 } 4068 /* must be last */ 4069 ret = kvm_guest_debug_workarounds(x86_cpu); 4070 if (ret < 0) { 4071 return ret; 4072 } 4073 return 0; 4074 } 4075 4076 int kvm_arch_get_registers(CPUState *cs) 4077 { 4078 X86CPU *cpu = X86_CPU(cs); 4079 int ret; 4080 4081 assert(cpu_is_stopped(cs) || qemu_cpu_is_self(cs)); 4082 4083 ret = kvm_get_vcpu_events(cpu); 4084 if (ret < 0) { 4085 goto out; 4086 } 4087 /* 4088 * KVM_GET_MPSTATE can modify CS and RIP, call it before 4089 * KVM_GET_REGS and KVM_GET_SREGS. 4090 */ 4091 ret = kvm_get_mp_state(cpu); 4092 if (ret < 0) { 4093 goto out; 4094 } 4095 ret = kvm_getput_regs(cpu, 0); 4096 if (ret < 0) { 4097 goto out; 4098 } 4099 ret = kvm_get_xsave(cpu); 4100 if (ret < 0) { 4101 goto out; 4102 } 4103 ret = kvm_get_xcrs(cpu); 4104 if (ret < 0) { 4105 goto out; 4106 } 4107 ret = kvm_get_sregs(cpu); 4108 if (ret < 0) { 4109 goto out; 4110 } 4111 ret = kvm_get_msrs(cpu); 4112 if (ret < 0) { 4113 goto out; 4114 } 4115 ret = kvm_get_apic(cpu); 4116 if (ret < 0) { 4117 goto out; 4118 } 4119 ret = kvm_get_debugregs(cpu); 4120 if (ret < 0) { 4121 goto out; 4122 } 4123 ret = kvm_get_nested_state(cpu); 4124 if (ret < 0) { 4125 goto out; 4126 } 4127 ret = 0; 4128 out: 4129 cpu_sync_bndcs_hflags(&cpu->env); 4130 return ret; 4131 } 4132 4133 void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) 4134 { 4135 X86CPU *x86_cpu = X86_CPU(cpu); 4136 CPUX86State *env = &x86_cpu->env; 4137 int ret; 4138 4139 /* Inject NMI */ 4140 if (cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) { 4141 if (cpu->interrupt_request & CPU_INTERRUPT_NMI) { 4142 qemu_mutex_lock_iothread(); 4143 cpu->interrupt_request &= ~CPU_INTERRUPT_NMI; 4144 qemu_mutex_unlock_iothread(); 4145 DPRINTF("injected NMI\n"); 4146 ret = kvm_vcpu_ioctl(cpu, KVM_NMI); 4147 if (ret < 0) { 4148 fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n", 4149 strerror(-ret)); 4150 } 4151 } 4152 if (cpu->interrupt_request & CPU_INTERRUPT_SMI) { 4153 qemu_mutex_lock_iothread(); 4154 cpu->interrupt_request &= ~CPU_INTERRUPT_SMI; 4155 qemu_mutex_unlock_iothread(); 4156 DPRINTF("injected SMI\n"); 4157 ret = kvm_vcpu_ioctl(cpu, KVM_SMI); 4158 if (ret < 0) { 4159 fprintf(stderr, "KVM: injection failed, SMI lost (%s)\n", 4160 strerror(-ret)); 4161 } 4162 } 4163 } 4164 4165 if (!kvm_pic_in_kernel()) { 4166 qemu_mutex_lock_iothread(); 4167 } 4168 4169 /* Force the VCPU out of its inner loop to process any INIT requests 4170 * or (for userspace APIC, but it is cheap to combine the checks here) 4171 * pending TPR access reports. 4172 */ 4173 if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { 4174 if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) && 4175 !(env->hflags & HF_SMM_MASK)) { 4176 cpu->exit_request = 1; 4177 } 4178 if (cpu->interrupt_request & CPU_INTERRUPT_TPR) { 4179 cpu->exit_request = 1; 4180 } 4181 } 4182 4183 if (!kvm_pic_in_kernel()) { 4184 /* Try to inject an interrupt if the guest can accept it */ 4185 if (run->ready_for_interrupt_injection && 4186 (cpu->interrupt_request & CPU_INTERRUPT_HARD) && 4187 (env->eflags & IF_MASK)) { 4188 int irq; 4189 4190 cpu->interrupt_request &= ~CPU_INTERRUPT_HARD; 4191 irq = cpu_get_pic_interrupt(env); 4192 if (irq >= 0) { 4193 struct kvm_interrupt intr; 4194 4195 intr.irq = irq; 4196 DPRINTF("injected interrupt %d\n", irq); 4197 ret = kvm_vcpu_ioctl(cpu, KVM_INTERRUPT, &intr); 4198 if (ret < 0) { 4199 fprintf(stderr, 4200 "KVM: injection failed, interrupt lost (%s)\n", 4201 strerror(-ret)); 4202 } 4203 } 4204 } 4205 4206 /* If we have an interrupt but the guest is not ready to receive an 4207 * interrupt, request an interrupt window exit. This will 4208 * cause a return to userspace as soon as the guest is ready to 4209 * receive interrupts. */ 4210 if ((cpu->interrupt_request & CPU_INTERRUPT_HARD)) { 4211 run->request_interrupt_window = 1; 4212 } else { 4213 run->request_interrupt_window = 0; 4214 } 4215 4216 DPRINTF("setting tpr\n"); 4217 run->cr8 = cpu_get_apic_tpr(x86_cpu->apic_state); 4218 4219 qemu_mutex_unlock_iothread(); 4220 } 4221 } 4222 4223 MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run) 4224 { 4225 X86CPU *x86_cpu = X86_CPU(cpu); 4226 CPUX86State *env = &x86_cpu->env; 4227 4228 if (run->flags & KVM_RUN_X86_SMM) { 4229 env->hflags |= HF_SMM_MASK; 4230 } else { 4231 env->hflags &= ~HF_SMM_MASK; 4232 } 4233 if (run->if_flag) { 4234 env->eflags |= IF_MASK; 4235 } else { 4236 env->eflags &= ~IF_MASK; 4237 } 4238 4239 /* We need to protect the apic state against concurrent accesses from 4240 * different threads in case the userspace irqchip is used. */ 4241 if (!kvm_irqchip_in_kernel()) { 4242 qemu_mutex_lock_iothread(); 4243 } 4244 cpu_set_apic_tpr(x86_cpu->apic_state, run->cr8); 4245 cpu_set_apic_base(x86_cpu->apic_state, run->apic_base); 4246 if (!kvm_irqchip_in_kernel()) { 4247 qemu_mutex_unlock_iothread(); 4248 } 4249 return cpu_get_mem_attrs(env); 4250 } 4251 4252 int kvm_arch_process_async_events(CPUState *cs) 4253 { 4254 X86CPU *cpu = X86_CPU(cs); 4255 CPUX86State *env = &cpu->env; 4256 4257 if (cs->interrupt_request & CPU_INTERRUPT_MCE) { 4258 /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */ 4259 assert(env->mcg_cap); 4260 4261 cs->interrupt_request &= ~CPU_INTERRUPT_MCE; 4262 4263 kvm_cpu_synchronize_state(cs); 4264 4265 if (env->exception_nr == EXCP08_DBLE) { 4266 /* this means triple fault */ 4267 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 4268 cs->exit_request = 1; 4269 return 0; 4270 } 4271 kvm_queue_exception(env, EXCP12_MCHK, 0, 0); 4272 env->has_error_code = 0; 4273 4274 cs->halted = 0; 4275 if (kvm_irqchip_in_kernel() && env->mp_state == KVM_MP_STATE_HALTED) { 4276 env->mp_state = KVM_MP_STATE_RUNNABLE; 4277 } 4278 } 4279 4280 if ((cs->interrupt_request & CPU_INTERRUPT_INIT) && 4281 !(env->hflags & HF_SMM_MASK)) { 4282 kvm_cpu_synchronize_state(cs); 4283 do_cpu_init(cpu); 4284 } 4285 4286 if (kvm_irqchip_in_kernel()) { 4287 return 0; 4288 } 4289 4290 if (cs->interrupt_request & CPU_INTERRUPT_POLL) { 4291 cs->interrupt_request &= ~CPU_INTERRUPT_POLL; 4292 apic_poll_irq(cpu->apic_state); 4293 } 4294 if (((cs->interrupt_request & CPU_INTERRUPT_HARD) && 4295 (env->eflags & IF_MASK)) || 4296 (cs->interrupt_request & CPU_INTERRUPT_NMI)) { 4297 cs->halted = 0; 4298 } 4299 if (cs->interrupt_request & CPU_INTERRUPT_SIPI) { 4300 kvm_cpu_synchronize_state(cs); 4301 do_cpu_sipi(cpu); 4302 } 4303 if (cs->interrupt_request & CPU_INTERRUPT_TPR) { 4304 cs->interrupt_request &= ~CPU_INTERRUPT_TPR; 4305 kvm_cpu_synchronize_state(cs); 4306 apic_handle_tpr_access_report(cpu->apic_state, env->eip, 4307 env->tpr_access_type); 4308 } 4309 4310 return cs->halted; 4311 } 4312 4313 static int kvm_handle_halt(X86CPU *cpu) 4314 { 4315 CPUState *cs = CPU(cpu); 4316 CPUX86State *env = &cpu->env; 4317 4318 if (!((cs->interrupt_request & CPU_INTERRUPT_HARD) && 4319 (env->eflags & IF_MASK)) && 4320 !(cs->interrupt_request & CPU_INTERRUPT_NMI)) { 4321 cs->halted = 1; 4322 return EXCP_HLT; 4323 } 4324 4325 return 0; 4326 } 4327 4328 static int kvm_handle_tpr_access(X86CPU *cpu) 4329 { 4330 CPUState *cs = CPU(cpu); 4331 struct kvm_run *run = cs->kvm_run; 4332 4333 apic_handle_tpr_access_report(cpu->apic_state, run->tpr_access.rip, 4334 run->tpr_access.is_write ? TPR_ACCESS_WRITE 4335 : TPR_ACCESS_READ); 4336 return 1; 4337 } 4338 4339 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 4340 { 4341 static const uint8_t int3 = 0xcc; 4342 4343 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) || 4344 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&int3, 1, 1)) { 4345 return -EINVAL; 4346 } 4347 return 0; 4348 } 4349 4350 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 4351 { 4352 uint8_t int3; 4353 4354 if (cpu_memory_rw_debug(cs, bp->pc, &int3, 1, 0) || int3 != 0xcc || 4355 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) { 4356 return -EINVAL; 4357 } 4358 return 0; 4359 } 4360 4361 static struct { 4362 target_ulong addr; 4363 int len; 4364 int type; 4365 } hw_breakpoint[4]; 4366 4367 static int nb_hw_breakpoint; 4368 4369 static int find_hw_breakpoint(target_ulong addr, int len, int type) 4370 { 4371 int n; 4372 4373 for (n = 0; n < nb_hw_breakpoint; n++) { 4374 if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type && 4375 (hw_breakpoint[n].len == len || len == -1)) { 4376 return n; 4377 } 4378 } 4379 return -1; 4380 } 4381 4382 int kvm_arch_insert_hw_breakpoint(target_ulong addr, 4383 target_ulong len, int type) 4384 { 4385 switch (type) { 4386 case GDB_BREAKPOINT_HW: 4387 len = 1; 4388 break; 4389 case GDB_WATCHPOINT_WRITE: 4390 case GDB_WATCHPOINT_ACCESS: 4391 switch (len) { 4392 case 1: 4393 break; 4394 case 2: 4395 case 4: 4396 case 8: 4397 if (addr & (len - 1)) { 4398 return -EINVAL; 4399 } 4400 break; 4401 default: 4402 return -EINVAL; 4403 } 4404 break; 4405 default: 4406 return -ENOSYS; 4407 } 4408 4409 if (nb_hw_breakpoint == 4) { 4410 return -ENOBUFS; 4411 } 4412 if (find_hw_breakpoint(addr, len, type) >= 0) { 4413 return -EEXIST; 4414 } 4415 hw_breakpoint[nb_hw_breakpoint].addr = addr; 4416 hw_breakpoint[nb_hw_breakpoint].len = len; 4417 hw_breakpoint[nb_hw_breakpoint].type = type; 4418 nb_hw_breakpoint++; 4419 4420 return 0; 4421 } 4422 4423 int kvm_arch_remove_hw_breakpoint(target_ulong addr, 4424 target_ulong len, int type) 4425 { 4426 int n; 4427 4428 n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type); 4429 if (n < 0) { 4430 return -ENOENT; 4431 } 4432 nb_hw_breakpoint--; 4433 hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint]; 4434 4435 return 0; 4436 } 4437 4438 void kvm_arch_remove_all_hw_breakpoints(void) 4439 { 4440 nb_hw_breakpoint = 0; 4441 } 4442 4443 static CPUWatchpoint hw_watchpoint; 4444 4445 static int kvm_handle_debug(X86CPU *cpu, 4446 struct kvm_debug_exit_arch *arch_info) 4447 { 4448 CPUState *cs = CPU(cpu); 4449 CPUX86State *env = &cpu->env; 4450 int ret = 0; 4451 int n; 4452 4453 if (arch_info->exception == EXCP01_DB) { 4454 if (arch_info->dr6 & DR6_BS) { 4455 if (cs->singlestep_enabled) { 4456 ret = EXCP_DEBUG; 4457 } 4458 } else { 4459 for (n = 0; n < 4; n++) { 4460 if (arch_info->dr6 & (1 << n)) { 4461 switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) { 4462 case 0x0: 4463 ret = EXCP_DEBUG; 4464 break; 4465 case 0x1: 4466 ret = EXCP_DEBUG; 4467 cs->watchpoint_hit = &hw_watchpoint; 4468 hw_watchpoint.vaddr = hw_breakpoint[n].addr; 4469 hw_watchpoint.flags = BP_MEM_WRITE; 4470 break; 4471 case 0x3: 4472 ret = EXCP_DEBUG; 4473 cs->watchpoint_hit = &hw_watchpoint; 4474 hw_watchpoint.vaddr = hw_breakpoint[n].addr; 4475 hw_watchpoint.flags = BP_MEM_ACCESS; 4476 break; 4477 } 4478 } 4479 } 4480 } 4481 } else if (kvm_find_sw_breakpoint(cs, arch_info->pc)) { 4482 ret = EXCP_DEBUG; 4483 } 4484 if (ret == 0) { 4485 cpu_synchronize_state(cs); 4486 assert(env->exception_nr == -1); 4487 4488 /* pass to guest */ 4489 kvm_queue_exception(env, arch_info->exception, 4490 arch_info->exception == EXCP01_DB, 4491 arch_info->dr6); 4492 env->has_error_code = 0; 4493 } 4494 4495 return ret; 4496 } 4497 4498 void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg) 4499 { 4500 const uint8_t type_code[] = { 4501 [GDB_BREAKPOINT_HW] = 0x0, 4502 [GDB_WATCHPOINT_WRITE] = 0x1, 4503 [GDB_WATCHPOINT_ACCESS] = 0x3 4504 }; 4505 const uint8_t len_code[] = { 4506 [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2 4507 }; 4508 int n; 4509 4510 if (kvm_sw_breakpoints_active(cpu)) { 4511 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP; 4512 } 4513 if (nb_hw_breakpoint > 0) { 4514 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP; 4515 dbg->arch.debugreg[7] = 0x0600; 4516 for (n = 0; n < nb_hw_breakpoint; n++) { 4517 dbg->arch.debugreg[n] = hw_breakpoint[n].addr; 4518 dbg->arch.debugreg[7] |= (2 << (n * 2)) | 4519 (type_code[hw_breakpoint[n].type] << (16 + n*4)) | 4520 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4)); 4521 } 4522 } 4523 } 4524 4525 static bool host_supports_vmx(void) 4526 { 4527 uint32_t ecx, unused; 4528 4529 host_cpuid(1, 0, &unused, &unused, &ecx, &unused); 4530 return ecx & CPUID_EXT_VMX; 4531 } 4532 4533 #define VMX_INVALID_GUEST_STATE 0x80000021 4534 4535 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) 4536 { 4537 X86CPU *cpu = X86_CPU(cs); 4538 uint64_t code; 4539 int ret; 4540 4541 switch (run->exit_reason) { 4542 case KVM_EXIT_HLT: 4543 DPRINTF("handle_hlt\n"); 4544 qemu_mutex_lock_iothread(); 4545 ret = kvm_handle_halt(cpu); 4546 qemu_mutex_unlock_iothread(); 4547 break; 4548 case KVM_EXIT_SET_TPR: 4549 ret = 0; 4550 break; 4551 case KVM_EXIT_TPR_ACCESS: 4552 qemu_mutex_lock_iothread(); 4553 ret = kvm_handle_tpr_access(cpu); 4554 qemu_mutex_unlock_iothread(); 4555 break; 4556 case KVM_EXIT_FAIL_ENTRY: 4557 code = run->fail_entry.hardware_entry_failure_reason; 4558 fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n", 4559 code); 4560 if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) { 4561 fprintf(stderr, 4562 "\nIf you're running a guest on an Intel machine without " 4563 "unrestricted mode\n" 4564 "support, the failure can be most likely due to the guest " 4565 "entering an invalid\n" 4566 "state for Intel VT. For example, the guest maybe running " 4567 "in big real mode\n" 4568 "which is not supported on less recent Intel processors." 4569 "\n\n"); 4570 } 4571 ret = -1; 4572 break; 4573 case KVM_EXIT_EXCEPTION: 4574 fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n", 4575 run->ex.exception, run->ex.error_code); 4576 ret = -1; 4577 break; 4578 case KVM_EXIT_DEBUG: 4579 DPRINTF("kvm_exit_debug\n"); 4580 qemu_mutex_lock_iothread(); 4581 ret = kvm_handle_debug(cpu, &run->debug.arch); 4582 qemu_mutex_unlock_iothread(); 4583 break; 4584 case KVM_EXIT_HYPERV: 4585 ret = kvm_hv_handle_exit(cpu, &run->hyperv); 4586 break; 4587 case KVM_EXIT_IOAPIC_EOI: 4588 ioapic_eoi_broadcast(run->eoi.vector); 4589 ret = 0; 4590 break; 4591 default: 4592 fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); 4593 ret = -1; 4594 break; 4595 } 4596 4597 return ret; 4598 } 4599 4600 bool kvm_arch_stop_on_emulation_error(CPUState *cs) 4601 { 4602 X86CPU *cpu = X86_CPU(cs); 4603 CPUX86State *env = &cpu->env; 4604 4605 kvm_cpu_synchronize_state(cs); 4606 return !(env->cr[0] & CR0_PE_MASK) || 4607 ((env->segs[R_CS].selector & 3) != 3); 4608 } 4609 4610 void kvm_arch_init_irq_routing(KVMState *s) 4611 { 4612 /* We know at this point that we're using the in-kernel 4613 * irqchip, so we can use irqfds, and on x86 we know 4614 * we can use msi via irqfd and GSI routing. 4615 */ 4616 kvm_msi_via_irqfd_allowed = true; 4617 kvm_gsi_routing_allowed = true; 4618 4619 if (kvm_irqchip_is_split()) { 4620 int i; 4621 4622 /* If the ioapic is in QEMU and the lapics are in KVM, reserve 4623 MSI routes for signaling interrupts to the local apics. */ 4624 for (i = 0; i < IOAPIC_NUM_PINS; i++) { 4625 if (kvm_irqchip_add_msi_route(s, 0, NULL) < 0) { 4626 error_report("Could not enable split IRQ mode."); 4627 exit(1); 4628 } 4629 } 4630 } 4631 } 4632 4633 int kvm_arch_irqchip_create(KVMState *s) 4634 { 4635 int ret; 4636 if (kvm_kernel_irqchip_split()) { 4637 ret = kvm_vm_enable_cap(s, KVM_CAP_SPLIT_IRQCHIP, 0, 24); 4638 if (ret) { 4639 error_report("Could not enable split irqchip mode: %s", 4640 strerror(-ret)); 4641 exit(1); 4642 } else { 4643 DPRINTF("Enabled KVM_CAP_SPLIT_IRQCHIP\n"); 4644 kvm_split_irqchip = true; 4645 return 1; 4646 } 4647 } else { 4648 return 0; 4649 } 4650 } 4651 4652 uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address) 4653 { 4654 CPUX86State *env; 4655 uint64_t ext_id; 4656 4657 if (!first_cpu) { 4658 return address; 4659 } 4660 env = &X86_CPU(first_cpu)->env; 4661 if (!(env->features[FEAT_KVM] & (1 << KVM_FEATURE_MSI_EXT_DEST_ID))) { 4662 return address; 4663 } 4664 4665 /* 4666 * If the remappable format bit is set, or the upper bits are 4667 * already set in address_hi, or the low extended bits aren't 4668 * there anyway, do nothing. 4669 */ 4670 ext_id = address & (0xff << MSI_ADDR_DEST_IDX_SHIFT); 4671 if (!ext_id || (ext_id & (1 << MSI_ADDR_DEST_IDX_SHIFT)) || (address >> 32)) { 4672 return address; 4673 } 4674 4675 address &= ~ext_id; 4676 address |= ext_id << 35; 4677 return address; 4678 } 4679 4680 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, 4681 uint64_t address, uint32_t data, PCIDevice *dev) 4682 { 4683 X86IOMMUState *iommu = x86_iommu_get_default(); 4684 4685 if (iommu) { 4686 X86IOMMUClass *class = X86_IOMMU_DEVICE_GET_CLASS(iommu); 4687 4688 if (class->int_remap) { 4689 int ret; 4690 MSIMessage src, dst; 4691 4692 src.address = route->u.msi.address_hi; 4693 src.address <<= VTD_MSI_ADDR_HI_SHIFT; 4694 src.address |= route->u.msi.address_lo; 4695 src.data = route->u.msi.data; 4696 4697 ret = class->int_remap(iommu, &src, &dst, dev ? \ 4698 pci_requester_id(dev) : \ 4699 X86_IOMMU_SID_INVALID); 4700 if (ret) { 4701 trace_kvm_x86_fixup_msi_error(route->gsi); 4702 return 1; 4703 } 4704 4705 /* 4706 * Handled untranslated compatibilty format interrupt with 4707 * extended destination ID in the low bits 11-5. */ 4708 dst.address = kvm_swizzle_msi_ext_dest_id(dst.address); 4709 4710 route->u.msi.address_hi = dst.address >> VTD_MSI_ADDR_HI_SHIFT; 4711 route->u.msi.address_lo = dst.address & VTD_MSI_ADDR_LO_MASK; 4712 route->u.msi.data = dst.data; 4713 return 0; 4714 } 4715 } 4716 4717 address = kvm_swizzle_msi_ext_dest_id(address); 4718 route->u.msi.address_hi = address >> VTD_MSI_ADDR_HI_SHIFT; 4719 route->u.msi.address_lo = address & VTD_MSI_ADDR_LO_MASK; 4720 return 0; 4721 } 4722 4723 typedef struct MSIRouteEntry MSIRouteEntry; 4724 4725 struct MSIRouteEntry { 4726 PCIDevice *dev; /* Device pointer */ 4727 int vector; /* MSI/MSIX vector index */ 4728 int virq; /* Virtual IRQ index */ 4729 QLIST_ENTRY(MSIRouteEntry) list; 4730 }; 4731 4732 /* List of used GSI routes */ 4733 static QLIST_HEAD(, MSIRouteEntry) msi_route_list = \ 4734 QLIST_HEAD_INITIALIZER(msi_route_list); 4735 4736 static void kvm_update_msi_routes_all(void *private, bool global, 4737 uint32_t index, uint32_t mask) 4738 { 4739 int cnt = 0, vector; 4740 MSIRouteEntry *entry; 4741 MSIMessage msg; 4742 PCIDevice *dev; 4743 4744 /* TODO: explicit route update */ 4745 QLIST_FOREACH(entry, &msi_route_list, list) { 4746 cnt++; 4747 vector = entry->vector; 4748 dev = entry->dev; 4749 if (msix_enabled(dev) && !msix_is_masked(dev, vector)) { 4750 msg = msix_get_message(dev, vector); 4751 } else if (msi_enabled(dev) && !msi_is_masked(dev, vector)) { 4752 msg = msi_get_message(dev, vector); 4753 } else { 4754 /* 4755 * Either MSI/MSIX is disabled for the device, or the 4756 * specific message was masked out. Skip this one. 4757 */ 4758 continue; 4759 } 4760 kvm_irqchip_update_msi_route(kvm_state, entry->virq, msg, dev); 4761 } 4762 kvm_irqchip_commit_routes(kvm_state); 4763 trace_kvm_x86_update_msi_routes(cnt); 4764 } 4765 4766 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route, 4767 int vector, PCIDevice *dev) 4768 { 4769 static bool notify_list_inited = false; 4770 MSIRouteEntry *entry; 4771 4772 if (!dev) { 4773 /* These are (possibly) IOAPIC routes only used for split 4774 * kernel irqchip mode, while what we are housekeeping are 4775 * PCI devices only. */ 4776 return 0; 4777 } 4778 4779 entry = g_new0(MSIRouteEntry, 1); 4780 entry->dev = dev; 4781 entry->vector = vector; 4782 entry->virq = route->gsi; 4783 QLIST_INSERT_HEAD(&msi_route_list, entry, list); 4784 4785 trace_kvm_x86_add_msi_route(route->gsi); 4786 4787 if (!notify_list_inited) { 4788 /* For the first time we do add route, add ourselves into 4789 * IOMMU's IEC notify list if needed. */ 4790 X86IOMMUState *iommu = x86_iommu_get_default(); 4791 if (iommu) { 4792 x86_iommu_iec_register_notifier(iommu, 4793 kvm_update_msi_routes_all, 4794 NULL); 4795 } 4796 notify_list_inited = true; 4797 } 4798 return 0; 4799 } 4800 4801 int kvm_arch_release_virq_post(int virq) 4802 { 4803 MSIRouteEntry *entry, *next; 4804 QLIST_FOREACH_SAFE(entry, &msi_route_list, list, next) { 4805 if (entry->virq == virq) { 4806 trace_kvm_x86_remove_msi_route(virq); 4807 QLIST_REMOVE(entry, list); 4808 g_free(entry); 4809 break; 4810 } 4811 } 4812 return 0; 4813 } 4814 4815 int kvm_arch_msi_data_to_gsi(uint32_t data) 4816 { 4817 abort(); 4818 } 4819 4820 bool kvm_has_waitpkg(void) 4821 { 4822 return has_msr_umwait; 4823 } 4824