xref: /openbmc/qemu/target/i386/kvm/kvm.c (revision f14eced5)
1 /*
2  * QEMU KVM support
3  *
4  * Copyright (C) 2006-2008 Qumranet Technologies
5  * Copyright IBM, Corp. 2008
6  *
7  * Authors:
8  *  Anthony Liguori   <aliguori@us.ibm.com>
9  *
10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
11  * See the COPYING file in the top-level directory.
12  *
13  */
14 
15 #include "qemu/osdep.h"
16 #include "qapi/qapi-events-run-state.h"
17 #include "qapi/error.h"
18 #include "qapi/visitor.h"
19 #include <sys/ioctl.h>
20 #include <sys/utsname.h>
21 #include <sys/syscall.h>
22 
23 #include <linux/kvm.h>
24 #include "standard-headers/asm-x86/kvm_para.h"
25 #include "hw/xen/interface/arch-x86/cpuid.h"
26 
27 #include "cpu.h"
28 #include "host-cpu.h"
29 #include "sysemu/sysemu.h"
30 #include "sysemu/hw_accel.h"
31 #include "sysemu/kvm_int.h"
32 #include "sysemu/runstate.h"
33 #include "kvm_i386.h"
34 #include "sev.h"
35 #include "xen-emu.h"
36 #include "hyperv.h"
37 #include "hyperv-proto.h"
38 
39 #include "exec/gdbstub.h"
40 #include "qemu/host-utils.h"
41 #include "qemu/main-loop.h"
42 #include "qemu/ratelimit.h"
43 #include "qemu/config-file.h"
44 #include "qemu/error-report.h"
45 #include "qemu/memalign.h"
46 #include "hw/i386/x86.h"
47 #include "hw/i386/kvm/xen_evtchn.h"
48 #include "hw/i386/pc.h"
49 #include "hw/i386/apic.h"
50 #include "hw/i386/apic_internal.h"
51 #include "hw/i386/apic-msidef.h"
52 #include "hw/i386/intel_iommu.h"
53 #include "hw/i386/x86-iommu.h"
54 #include "hw/i386/e820_memory_layout.h"
55 
56 #include "hw/xen/xen.h"
57 
58 #include "hw/pci/pci.h"
59 #include "hw/pci/msi.h"
60 #include "hw/pci/msix.h"
61 #include "migration/blocker.h"
62 #include "exec/memattrs.h"
63 #include "trace.h"
64 
65 #include CONFIG_DEVICES
66 
67 //#define DEBUG_KVM
68 
69 #ifdef DEBUG_KVM
70 #define DPRINTF(fmt, ...) \
71     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
72 #else
73 #define DPRINTF(fmt, ...) \
74     do { } while (0)
75 #endif
76 
77 /* From arch/x86/kvm/lapic.h */
78 #define KVM_APIC_BUS_CYCLE_NS       1
79 #define KVM_APIC_BUS_FREQUENCY      (1000000000ULL / KVM_APIC_BUS_CYCLE_NS)
80 
81 #define MSR_KVM_WALL_CLOCK  0x11
82 #define MSR_KVM_SYSTEM_TIME 0x12
83 
84 /* A 4096-byte buffer can hold the 8-byte kvm_msrs header, plus
85  * 255 kvm_msr_entry structs */
86 #define MSR_BUF_SIZE 4096
87 
88 static void kvm_init_msrs(X86CPU *cpu);
89 
90 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
91     KVM_CAP_INFO(SET_TSS_ADDR),
92     KVM_CAP_INFO(EXT_CPUID),
93     KVM_CAP_INFO(MP_STATE),
94     KVM_CAP_LAST_INFO
95 };
96 
97 static bool has_msr_star;
98 static bool has_msr_hsave_pa;
99 static bool has_msr_tsc_aux;
100 static bool has_msr_tsc_adjust;
101 static bool has_msr_tsc_deadline;
102 static bool has_msr_feature_control;
103 static bool has_msr_misc_enable;
104 static bool has_msr_smbase;
105 static bool has_msr_bndcfgs;
106 static int lm_capable_kernel;
107 static bool has_msr_hv_hypercall;
108 static bool has_msr_hv_crash;
109 static bool has_msr_hv_reset;
110 static bool has_msr_hv_vpindex;
111 static bool hv_vpindex_settable;
112 static bool has_msr_hv_runtime;
113 static bool has_msr_hv_synic;
114 static bool has_msr_hv_stimer;
115 static bool has_msr_hv_frequencies;
116 static bool has_msr_hv_reenlightenment;
117 static bool has_msr_hv_syndbg_options;
118 static bool has_msr_xss;
119 static bool has_msr_umwait;
120 static bool has_msr_spec_ctrl;
121 static bool has_tsc_scale_msr;
122 static bool has_msr_tsx_ctrl;
123 static bool has_msr_virt_ssbd;
124 static bool has_msr_smi_count;
125 static bool has_msr_arch_capabs;
126 static bool has_msr_core_capabs;
127 static bool has_msr_vmx_vmfunc;
128 static bool has_msr_ucode_rev;
129 static bool has_msr_vmx_procbased_ctls2;
130 static bool has_msr_perf_capabs;
131 static bool has_msr_pkrs;
132 
133 static uint32_t has_architectural_pmu_version;
134 static uint32_t num_architectural_pmu_gp_counters;
135 static uint32_t num_architectural_pmu_fixed_counters;
136 
137 static int has_xsave;
138 static int has_xsave2;
139 static int has_xcrs;
140 static int has_pit_state2;
141 static int has_sregs2;
142 static int has_exception_payload;
143 static int has_triple_fault_event;
144 
145 static bool has_msr_mcg_ext_ctl;
146 
147 static struct kvm_cpuid2 *cpuid_cache;
148 static struct kvm_cpuid2 *hv_cpuid_cache;
149 static struct kvm_msr_list *kvm_feature_msrs;
150 
151 static KVMMSRHandlers msr_handlers[KVM_MSR_FILTER_MAX_RANGES];
152 
153 #define BUS_LOCK_SLICE_TIME 1000000000ULL /* ns */
154 static RateLimit bus_lock_ratelimit_ctrl;
155 static int kvm_get_one_msr(X86CPU *cpu, int index, uint64_t *value);
156 
157 bool kvm_has_pit_state2(void)
158 {
159     return !!has_pit_state2;
160 }
161 
162 bool kvm_has_smm(void)
163 {
164     return kvm_vm_check_extension(kvm_state, KVM_CAP_X86_SMM);
165 }
166 
167 bool kvm_has_adjust_clock_stable(void)
168 {
169     int ret = kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK);
170 
171     return (ret & KVM_CLOCK_TSC_STABLE);
172 }
173 
174 bool kvm_has_adjust_clock(void)
175 {
176     return kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK);
177 }
178 
179 bool kvm_has_exception_payload(void)
180 {
181     return has_exception_payload;
182 }
183 
184 static bool kvm_x2apic_api_set_flags(uint64_t flags)
185 {
186     KVMState *s = KVM_STATE(current_accel());
187 
188     return !kvm_vm_enable_cap(s, KVM_CAP_X2APIC_API, 0, flags);
189 }
190 
191 #define MEMORIZE(fn, _result) \
192     ({ \
193         static bool _memorized; \
194         \
195         if (_memorized) { \
196             return _result; \
197         } \
198         _memorized = true; \
199         _result = fn; \
200     })
201 
202 static bool has_x2apic_api;
203 
204 bool kvm_has_x2apic_api(void)
205 {
206     return has_x2apic_api;
207 }
208 
209 bool kvm_enable_x2apic(void)
210 {
211     return MEMORIZE(
212              kvm_x2apic_api_set_flags(KVM_X2APIC_API_USE_32BIT_IDS |
213                                       KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK),
214              has_x2apic_api);
215 }
216 
217 bool kvm_hv_vpindex_settable(void)
218 {
219     return hv_vpindex_settable;
220 }
221 
222 static int kvm_get_tsc(CPUState *cs)
223 {
224     X86CPU *cpu = X86_CPU(cs);
225     CPUX86State *env = &cpu->env;
226     uint64_t value;
227     int ret;
228 
229     if (env->tsc_valid) {
230         return 0;
231     }
232 
233     env->tsc_valid = !runstate_is_running();
234 
235     ret = kvm_get_one_msr(cpu, MSR_IA32_TSC, &value);
236     if (ret < 0) {
237         return ret;
238     }
239 
240     env->tsc = value;
241     return 0;
242 }
243 
244 static inline void do_kvm_synchronize_tsc(CPUState *cpu, run_on_cpu_data arg)
245 {
246     kvm_get_tsc(cpu);
247 }
248 
249 void kvm_synchronize_all_tsc(void)
250 {
251     CPUState *cpu;
252 
253     if (kvm_enabled()) {
254         CPU_FOREACH(cpu) {
255             run_on_cpu(cpu, do_kvm_synchronize_tsc, RUN_ON_CPU_NULL);
256         }
257     }
258 }
259 
260 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
261 {
262     struct kvm_cpuid2 *cpuid;
263     int r, size;
264 
265     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
266     cpuid = g_malloc0(size);
267     cpuid->nent = max;
268     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
269     if (r == 0 && cpuid->nent >= max) {
270         r = -E2BIG;
271     }
272     if (r < 0) {
273         if (r == -E2BIG) {
274             g_free(cpuid);
275             return NULL;
276         } else {
277             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
278                     strerror(-r));
279             exit(1);
280         }
281     }
282     return cpuid;
283 }
284 
285 /* Run KVM_GET_SUPPORTED_CPUID ioctl(), allocating a buffer large enough
286  * for all entries.
287  */
288 static struct kvm_cpuid2 *get_supported_cpuid(KVMState *s)
289 {
290     struct kvm_cpuid2 *cpuid;
291     int max = 1;
292 
293     if (cpuid_cache != NULL) {
294         return cpuid_cache;
295     }
296     while ((cpuid = try_get_cpuid(s, max)) == NULL) {
297         max *= 2;
298     }
299     cpuid_cache = cpuid;
300     return cpuid;
301 }
302 
303 static bool host_tsx_broken(void)
304 {
305     int family, model, stepping;\
306     char vendor[CPUID_VENDOR_SZ + 1];
307 
308     host_cpu_vendor_fms(vendor, &family, &model, &stepping);
309 
310     /* Check if we are running on a Haswell host known to have broken TSX */
311     return !strcmp(vendor, CPUID_VENDOR_INTEL) &&
312            (family == 6) &&
313            ((model == 63 && stepping < 4) ||
314             model == 60 || model == 69 || model == 70);
315 }
316 
317 /* Returns the value for a specific register on the cpuid entry
318  */
319 static uint32_t cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, int reg)
320 {
321     uint32_t ret = 0;
322     switch (reg) {
323     case R_EAX:
324         ret = entry->eax;
325         break;
326     case R_EBX:
327         ret = entry->ebx;
328         break;
329     case R_ECX:
330         ret = entry->ecx;
331         break;
332     case R_EDX:
333         ret = entry->edx;
334         break;
335     }
336     return ret;
337 }
338 
339 /* Find matching entry for function/index on kvm_cpuid2 struct
340  */
341 static struct kvm_cpuid_entry2 *cpuid_find_entry(struct kvm_cpuid2 *cpuid,
342                                                  uint32_t function,
343                                                  uint32_t index)
344 {
345     int i;
346     for (i = 0; i < cpuid->nent; ++i) {
347         if (cpuid->entries[i].function == function &&
348             cpuid->entries[i].index == index) {
349             return &cpuid->entries[i];
350         }
351     }
352     /* not found: */
353     return NULL;
354 }
355 
356 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function,
357                                       uint32_t index, int reg)
358 {
359     struct kvm_cpuid2 *cpuid;
360     uint32_t ret = 0;
361     uint32_t cpuid_1_edx, unused;
362     uint64_t bitmask;
363 
364     cpuid = get_supported_cpuid(s);
365 
366     struct kvm_cpuid_entry2 *entry = cpuid_find_entry(cpuid, function, index);
367     if (entry) {
368         ret = cpuid_entry_get_reg(entry, reg);
369     }
370 
371     /* Fixups for the data returned by KVM, below */
372 
373     if (function == 1 && reg == R_EDX) {
374         /* KVM before 2.6.30 misreports the following features */
375         ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
376     } else if (function == 1 && reg == R_ECX) {
377         /* We can set the hypervisor flag, even if KVM does not return it on
378          * GET_SUPPORTED_CPUID
379          */
380         ret |= CPUID_EXT_HYPERVISOR;
381         /* tsc-deadline flag is not returned by GET_SUPPORTED_CPUID, but it
382          * can be enabled if the kernel has KVM_CAP_TSC_DEADLINE_TIMER,
383          * and the irqchip is in the kernel.
384          */
385         if (kvm_irqchip_in_kernel() &&
386                 kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
387             ret |= CPUID_EXT_TSC_DEADLINE_TIMER;
388         }
389 
390         /* x2apic is reported by GET_SUPPORTED_CPUID, but it can't be enabled
391          * without the in-kernel irqchip
392          */
393         if (!kvm_irqchip_in_kernel()) {
394             ret &= ~CPUID_EXT_X2APIC;
395         }
396 
397         if (enable_cpu_pm) {
398             int disable_exits = kvm_check_extension(s,
399                                                     KVM_CAP_X86_DISABLE_EXITS);
400 
401             if (disable_exits & KVM_X86_DISABLE_EXITS_MWAIT) {
402                 ret |= CPUID_EXT_MONITOR;
403             }
404         }
405     } else if (function == 6 && reg == R_EAX) {
406         ret |= CPUID_6_EAX_ARAT; /* safe to allow because of emulated APIC */
407     } else if (function == 7 && index == 0 && reg == R_EBX) {
408         /* Not new instructions, just an optimization.  */
409         uint32_t ebx;
410         host_cpuid(7, 0, &unused, &ebx, &unused, &unused);
411         ret |= ebx & CPUID_7_0_EBX_ERMS;
412 
413         if (host_tsx_broken()) {
414             ret &= ~(CPUID_7_0_EBX_RTM | CPUID_7_0_EBX_HLE);
415         }
416     } else if (function == 7 && index == 0 && reg == R_EDX) {
417         /* Not new instructions, just an optimization.  */
418         uint32_t edx;
419         host_cpuid(7, 0, &unused, &unused, &unused, &edx);
420         ret |= edx & CPUID_7_0_EDX_FSRM;
421 
422         /*
423          * Linux v4.17-v4.20 incorrectly return ARCH_CAPABILITIES on SVM hosts.
424          * We can detect the bug by checking if MSR_IA32_ARCH_CAPABILITIES is
425          * returned by KVM_GET_MSR_INDEX_LIST.
426          */
427         if (!has_msr_arch_capabs) {
428             ret &= ~CPUID_7_0_EDX_ARCH_CAPABILITIES;
429         }
430     } else if (function == 7 && index == 1 && reg == R_EAX) {
431         /* Not new instructions, just an optimization.  */
432         uint32_t eax;
433         host_cpuid(7, 1, &eax, &unused, &unused, &unused);
434         ret |= eax & (CPUID_7_1_EAX_FZRM | CPUID_7_1_EAX_FSRS | CPUID_7_1_EAX_FSRC);
435     } else if (function == 7 && index == 2 && reg == R_EDX) {
436         uint32_t edx;
437         host_cpuid(7, 2, &unused, &unused, &unused, &edx);
438         ret |= edx & CPUID_7_2_EDX_MCDT_NO;
439     } else if (function == 0xd && index == 0 &&
440                (reg == R_EAX || reg == R_EDX)) {
441         /*
442          * The value returned by KVM_GET_SUPPORTED_CPUID does not include
443          * features that still have to be enabled with the arch_prctl
444          * system call.  QEMU needs the full value, which is retrieved
445          * with KVM_GET_DEVICE_ATTR.
446          */
447         struct kvm_device_attr attr = {
448             .group = 0,
449             .attr = KVM_X86_XCOMP_GUEST_SUPP,
450             .addr = (unsigned long) &bitmask
451         };
452 
453         bool sys_attr = kvm_check_extension(s, KVM_CAP_SYS_ATTRIBUTES);
454         if (!sys_attr) {
455             return ret;
456         }
457 
458         int rc = kvm_ioctl(s, KVM_GET_DEVICE_ATTR, &attr);
459         if (rc < 0) {
460             if (rc != -ENXIO) {
461                 warn_report("KVM_GET_DEVICE_ATTR(0, KVM_X86_XCOMP_GUEST_SUPP) "
462                             "error: %d", rc);
463             }
464             return ret;
465         }
466         ret = (reg == R_EAX) ? bitmask : bitmask >> 32;
467     } else if (function == 0x80000001 && reg == R_ECX) {
468         /*
469          * It's safe to enable TOPOEXT even if it's not returned by
470          * GET_SUPPORTED_CPUID.  Unconditionally enabling TOPOEXT here allows
471          * us to keep CPU models including TOPOEXT runnable on older kernels.
472          */
473         ret |= CPUID_EXT3_TOPOEXT;
474     } else if (function == 0x80000001 && reg == R_EDX) {
475         /* On Intel, kvm returns cpuid according to the Intel spec,
476          * so add missing bits according to the AMD spec:
477          */
478         cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
479         ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES;
480     } else if (function == KVM_CPUID_FEATURES && reg == R_EAX) {
481         /* kvm_pv_unhalt is reported by GET_SUPPORTED_CPUID, but it can't
482          * be enabled without the in-kernel irqchip
483          */
484         if (!kvm_irqchip_in_kernel()) {
485             ret &= ~(1U << KVM_FEATURE_PV_UNHALT);
486         }
487         if (kvm_irqchip_is_split()) {
488             ret |= 1U << KVM_FEATURE_MSI_EXT_DEST_ID;
489         }
490     } else if (function == KVM_CPUID_FEATURES && reg == R_EDX) {
491         ret |= 1U << KVM_HINTS_REALTIME;
492     }
493 
494     return ret;
495 }
496 
497 uint64_t kvm_arch_get_supported_msr_feature(KVMState *s, uint32_t index)
498 {
499     struct {
500         struct kvm_msrs info;
501         struct kvm_msr_entry entries[1];
502     } msr_data = {};
503     uint64_t value;
504     uint32_t ret, can_be_one, must_be_one;
505 
506     if (kvm_feature_msrs == NULL) { /* Host doesn't support feature MSRs */
507         return 0;
508     }
509 
510     /* Check if requested MSR is supported feature MSR */
511     int i;
512     for (i = 0; i < kvm_feature_msrs->nmsrs; i++)
513         if (kvm_feature_msrs->indices[i] == index) {
514             break;
515         }
516     if (i == kvm_feature_msrs->nmsrs) {
517         return 0; /* if the feature MSR is not supported, simply return 0 */
518     }
519 
520     msr_data.info.nmsrs = 1;
521     msr_data.entries[0].index = index;
522 
523     ret = kvm_ioctl(s, KVM_GET_MSRS, &msr_data);
524     if (ret != 1) {
525         error_report("KVM get MSR (index=0x%x) feature failed, %s",
526             index, strerror(-ret));
527         exit(1);
528     }
529 
530     value = msr_data.entries[0].data;
531     switch (index) {
532     case MSR_IA32_VMX_PROCBASED_CTLS2:
533         if (!has_msr_vmx_procbased_ctls2) {
534             /* KVM forgot to add these bits for some time, do this ourselves. */
535             if (kvm_arch_get_supported_cpuid(s, 0xD, 1, R_ECX) &
536                 CPUID_XSAVE_XSAVES) {
537                 value |= (uint64_t)VMX_SECONDARY_EXEC_XSAVES << 32;
538             }
539             if (kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX) &
540                 CPUID_EXT_RDRAND) {
541                 value |= (uint64_t)VMX_SECONDARY_EXEC_RDRAND_EXITING << 32;
542             }
543             if (kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX) &
544                 CPUID_7_0_EBX_INVPCID) {
545                 value |= (uint64_t)VMX_SECONDARY_EXEC_ENABLE_INVPCID << 32;
546             }
547             if (kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX) &
548                 CPUID_7_0_EBX_RDSEED) {
549                 value |= (uint64_t)VMX_SECONDARY_EXEC_RDSEED_EXITING << 32;
550             }
551             if (kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_EDX) &
552                 CPUID_EXT2_RDTSCP) {
553                 value |= (uint64_t)VMX_SECONDARY_EXEC_RDTSCP << 32;
554             }
555         }
556         /* fall through */
557     case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
558     case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
559     case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
560     case MSR_IA32_VMX_TRUE_EXIT_CTLS:
561         /*
562          * Return true for bits that can be one, but do not have to be one.
563          * The SDM tells us which bits could have a "must be one" setting,
564          * so we can do the opposite transformation in make_vmx_msr_value.
565          */
566         must_be_one = (uint32_t)value;
567         can_be_one = (uint32_t)(value >> 32);
568         return can_be_one & ~must_be_one;
569 
570     default:
571         return value;
572     }
573 }
574 
575 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
576                                      int *max_banks)
577 {
578     int r;
579 
580     r = kvm_check_extension(s, KVM_CAP_MCE);
581     if (r > 0) {
582         *max_banks = r;
583         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
584     }
585     return -ENOSYS;
586 }
587 
588 static void kvm_mce_inject(X86CPU *cpu, hwaddr paddr, int code)
589 {
590     CPUState *cs = CPU(cpu);
591     CPUX86State *env = &cpu->env;
592     uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
593                       MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
594     uint64_t mcg_status = MCG_STATUS_MCIP;
595     int flags = 0;
596 
597     if (code == BUS_MCEERR_AR) {
598         status |= MCI_STATUS_AR | 0x134;
599         mcg_status |= MCG_STATUS_RIPV | MCG_STATUS_EIPV;
600     } else {
601         status |= 0xc0;
602         mcg_status |= MCG_STATUS_RIPV;
603     }
604 
605     flags = cpu_x86_support_mca_broadcast(env) ? MCE_INJECT_BROADCAST : 0;
606     /* We need to read back the value of MSR_EXT_MCG_CTL that was set by the
607      * guest kernel back into env->mcg_ext_ctl.
608      */
609     cpu_synchronize_state(cs);
610     if (env->mcg_ext_ctl & MCG_EXT_CTL_LMCE_EN) {
611         mcg_status |= MCG_STATUS_LMCE;
612         flags = 0;
613     }
614 
615     cpu_x86_inject_mce(NULL, cpu, 9, status, mcg_status, paddr,
616                        (MCM_ADDR_PHYS << 6) | 0xc, flags);
617 }
618 
619 static void emit_hypervisor_memory_failure(MemoryFailureAction action, bool ar)
620 {
621     MemoryFailureFlags mff = {.action_required = ar, .recursive = false};
622 
623     qapi_event_send_memory_failure(MEMORY_FAILURE_RECIPIENT_HYPERVISOR, action,
624                                    &mff);
625 }
626 
627 static void hardware_memory_error(void *host_addr)
628 {
629     emit_hypervisor_memory_failure(MEMORY_FAILURE_ACTION_FATAL, true);
630     error_report("QEMU got Hardware memory error at addr %p", host_addr);
631     exit(1);
632 }
633 
634 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
635 {
636     X86CPU *cpu = X86_CPU(c);
637     CPUX86State *env = &cpu->env;
638     ram_addr_t ram_addr;
639     hwaddr paddr;
640 
641     /* If we get an action required MCE, it has been injected by KVM
642      * while the VM was running.  An action optional MCE instead should
643      * be coming from the main thread, which qemu_init_sigbus identifies
644      * as the "early kill" thread.
645      */
646     assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO);
647 
648     if ((env->mcg_cap & MCG_SER_P) && addr) {
649         ram_addr = qemu_ram_addr_from_host(addr);
650         if (ram_addr != RAM_ADDR_INVALID &&
651             kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
652             kvm_hwpoison_page_add(ram_addr);
653             kvm_mce_inject(cpu, paddr, code);
654 
655             /*
656              * Use different logging severity based on error type.
657              * If there is additional MCE reporting on the hypervisor, QEMU VA
658              * could be another source to identify the PA and MCE details.
659              */
660             if (code == BUS_MCEERR_AR) {
661                 error_report("Guest MCE Memory Error at QEMU addr %p and "
662                     "GUEST addr 0x%" HWADDR_PRIx " of type %s injected",
663                     addr, paddr, "BUS_MCEERR_AR");
664             } else {
665                  warn_report("Guest MCE Memory Error at QEMU addr %p and "
666                      "GUEST addr 0x%" HWADDR_PRIx " of type %s injected",
667                      addr, paddr, "BUS_MCEERR_AO");
668             }
669 
670             return;
671         }
672 
673         if (code == BUS_MCEERR_AO) {
674             warn_report("Hardware memory error at addr %p of type %s "
675                 "for memory used by QEMU itself instead of guest system!",
676                  addr, "BUS_MCEERR_AO");
677         }
678     }
679 
680     if (code == BUS_MCEERR_AR) {
681         hardware_memory_error(addr);
682     }
683 
684     /* Hope we are lucky for AO MCE, just notify a event */
685     emit_hypervisor_memory_failure(MEMORY_FAILURE_ACTION_IGNORE, false);
686 }
687 
688 static void kvm_reset_exception(CPUX86State *env)
689 {
690     env->exception_nr = -1;
691     env->exception_pending = 0;
692     env->exception_injected = 0;
693     env->exception_has_payload = false;
694     env->exception_payload = 0;
695 }
696 
697 static void kvm_queue_exception(CPUX86State *env,
698                                 int32_t exception_nr,
699                                 uint8_t exception_has_payload,
700                                 uint64_t exception_payload)
701 {
702     assert(env->exception_nr == -1);
703     assert(!env->exception_pending);
704     assert(!env->exception_injected);
705     assert(!env->exception_has_payload);
706 
707     env->exception_nr = exception_nr;
708 
709     if (has_exception_payload) {
710         env->exception_pending = 1;
711 
712         env->exception_has_payload = exception_has_payload;
713         env->exception_payload = exception_payload;
714     } else {
715         env->exception_injected = 1;
716 
717         if (exception_nr == EXCP01_DB) {
718             assert(exception_has_payload);
719             env->dr[6] = exception_payload;
720         } else if (exception_nr == EXCP0E_PAGE) {
721             assert(exception_has_payload);
722             env->cr[2] = exception_payload;
723         } else {
724             assert(!exception_has_payload);
725         }
726     }
727 }
728 
729 static int kvm_inject_mce_oldstyle(X86CPU *cpu)
730 {
731     CPUX86State *env = &cpu->env;
732 
733     if (!kvm_has_vcpu_events() && env->exception_nr == EXCP12_MCHK) {
734         unsigned int bank, bank_num = env->mcg_cap & 0xff;
735         struct kvm_x86_mce mce;
736 
737         kvm_reset_exception(env);
738 
739         /*
740          * There must be at least one bank in use if an MCE is pending.
741          * Find it and use its values for the event injection.
742          */
743         for (bank = 0; bank < bank_num; bank++) {
744             if (env->mce_banks[bank * 4 + 1] & MCI_STATUS_VAL) {
745                 break;
746             }
747         }
748         assert(bank < bank_num);
749 
750         mce.bank = bank;
751         mce.status = env->mce_banks[bank * 4 + 1];
752         mce.mcg_status = env->mcg_status;
753         mce.addr = env->mce_banks[bank * 4 + 2];
754         mce.misc = env->mce_banks[bank * 4 + 3];
755 
756         return kvm_vcpu_ioctl(CPU(cpu), KVM_X86_SET_MCE, &mce);
757     }
758     return 0;
759 }
760 
761 static void cpu_update_state(void *opaque, bool running, RunState state)
762 {
763     CPUX86State *env = opaque;
764 
765     if (running) {
766         env->tsc_valid = false;
767     }
768 }
769 
770 unsigned long kvm_arch_vcpu_id(CPUState *cs)
771 {
772     X86CPU *cpu = X86_CPU(cs);
773     return cpu->apic_id;
774 }
775 
776 #ifndef KVM_CPUID_SIGNATURE_NEXT
777 #define KVM_CPUID_SIGNATURE_NEXT                0x40000100
778 #endif
779 
780 static bool hyperv_enabled(X86CPU *cpu)
781 {
782     return kvm_check_extension(kvm_state, KVM_CAP_HYPERV) > 0 &&
783         ((cpu->hyperv_spinlock_attempts != HYPERV_SPINLOCK_NEVER_NOTIFY) ||
784          cpu->hyperv_features || cpu->hyperv_passthrough);
785 }
786 
787 /*
788  * Check whether target_freq is within conservative
789  * ntp correctable bounds (250ppm) of freq
790  */
791 static inline bool freq_within_bounds(int freq, int target_freq)
792 {
793         int max_freq = freq + (freq * 250 / 1000000);
794         int min_freq = freq - (freq * 250 / 1000000);
795 
796         if (target_freq >= min_freq && target_freq <= max_freq) {
797                 return true;
798         }
799 
800         return false;
801 }
802 
803 static int kvm_arch_set_tsc_khz(CPUState *cs)
804 {
805     X86CPU *cpu = X86_CPU(cs);
806     CPUX86State *env = &cpu->env;
807     int r, cur_freq;
808     bool set_ioctl = false;
809 
810     if (!env->tsc_khz) {
811         return 0;
812     }
813 
814     cur_freq = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
815                kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) : -ENOTSUP;
816 
817     /*
818      * If TSC scaling is supported, attempt to set TSC frequency.
819      */
820     if (kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL)) {
821         set_ioctl = true;
822     }
823 
824     /*
825      * If desired TSC frequency is within bounds of NTP correction,
826      * attempt to set TSC frequency.
827      */
828     if (cur_freq != -ENOTSUP && freq_within_bounds(cur_freq, env->tsc_khz)) {
829         set_ioctl = true;
830     }
831 
832     r = set_ioctl ?
833         kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz) :
834         -ENOTSUP;
835 
836     if (r < 0) {
837         /* When KVM_SET_TSC_KHZ fails, it's an error only if the current
838          * TSC frequency doesn't match the one we want.
839          */
840         cur_freq = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
841                    kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
842                    -ENOTSUP;
843         if (cur_freq <= 0 || cur_freq != env->tsc_khz) {
844             warn_report("TSC frequency mismatch between "
845                         "VM (%" PRId64 " kHz) and host (%d kHz), "
846                         "and TSC scaling unavailable",
847                         env->tsc_khz, cur_freq);
848             return r;
849         }
850     }
851 
852     return 0;
853 }
854 
855 static bool tsc_is_stable_and_known(CPUX86State *env)
856 {
857     if (!env->tsc_khz) {
858         return false;
859     }
860     return (env->features[FEAT_8000_0007_EDX] & CPUID_APM_INVTSC)
861         || env->user_tsc_khz;
862 }
863 
864 #define DEFAULT_EVMCS_VERSION ((1 << 8) | 1)
865 
866 static struct {
867     const char *desc;
868     struct {
869         uint32_t func;
870         int reg;
871         uint32_t bits;
872     } flags[2];
873     uint64_t dependencies;
874 } kvm_hyperv_properties[] = {
875     [HYPERV_FEAT_RELAXED] = {
876         .desc = "relaxed timing (hv-relaxed)",
877         .flags = {
878             {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
879              .bits = HV_RELAXED_TIMING_RECOMMENDED}
880         }
881     },
882     [HYPERV_FEAT_VAPIC] = {
883         .desc = "virtual APIC (hv-vapic)",
884         .flags = {
885             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
886              .bits = HV_APIC_ACCESS_AVAILABLE}
887         }
888     },
889     [HYPERV_FEAT_TIME] = {
890         .desc = "clocksources (hv-time)",
891         .flags = {
892             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
893              .bits = HV_TIME_REF_COUNT_AVAILABLE | HV_REFERENCE_TSC_AVAILABLE}
894         }
895     },
896     [HYPERV_FEAT_CRASH] = {
897         .desc = "crash MSRs (hv-crash)",
898         .flags = {
899             {.func = HV_CPUID_FEATURES, .reg = R_EDX,
900              .bits = HV_GUEST_CRASH_MSR_AVAILABLE}
901         }
902     },
903     [HYPERV_FEAT_RESET] = {
904         .desc = "reset MSR (hv-reset)",
905         .flags = {
906             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
907              .bits = HV_RESET_AVAILABLE}
908         }
909     },
910     [HYPERV_FEAT_VPINDEX] = {
911         .desc = "VP_INDEX MSR (hv-vpindex)",
912         .flags = {
913             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
914              .bits = HV_VP_INDEX_AVAILABLE}
915         }
916     },
917     [HYPERV_FEAT_RUNTIME] = {
918         .desc = "VP_RUNTIME MSR (hv-runtime)",
919         .flags = {
920             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
921              .bits = HV_VP_RUNTIME_AVAILABLE}
922         }
923     },
924     [HYPERV_FEAT_SYNIC] = {
925         .desc = "synthetic interrupt controller (hv-synic)",
926         .flags = {
927             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
928              .bits = HV_SYNIC_AVAILABLE}
929         }
930     },
931     [HYPERV_FEAT_STIMER] = {
932         .desc = "synthetic timers (hv-stimer)",
933         .flags = {
934             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
935              .bits = HV_SYNTIMERS_AVAILABLE}
936         },
937         .dependencies = BIT(HYPERV_FEAT_SYNIC) | BIT(HYPERV_FEAT_TIME)
938     },
939     [HYPERV_FEAT_FREQUENCIES] = {
940         .desc = "frequency MSRs (hv-frequencies)",
941         .flags = {
942             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
943              .bits = HV_ACCESS_FREQUENCY_MSRS},
944             {.func = HV_CPUID_FEATURES, .reg = R_EDX,
945              .bits = HV_FREQUENCY_MSRS_AVAILABLE}
946         }
947     },
948     [HYPERV_FEAT_REENLIGHTENMENT] = {
949         .desc = "reenlightenment MSRs (hv-reenlightenment)",
950         .flags = {
951             {.func = HV_CPUID_FEATURES, .reg = R_EAX,
952              .bits = HV_ACCESS_REENLIGHTENMENTS_CONTROL}
953         }
954     },
955     [HYPERV_FEAT_TLBFLUSH] = {
956         .desc = "paravirtualized TLB flush (hv-tlbflush)",
957         .flags = {
958             {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
959              .bits = HV_REMOTE_TLB_FLUSH_RECOMMENDED |
960              HV_EX_PROCESSOR_MASKS_RECOMMENDED}
961         },
962         .dependencies = BIT(HYPERV_FEAT_VPINDEX)
963     },
964     [HYPERV_FEAT_EVMCS] = {
965         .desc = "enlightened VMCS (hv-evmcs)",
966         .flags = {
967             {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
968              .bits = HV_ENLIGHTENED_VMCS_RECOMMENDED}
969         },
970         .dependencies = BIT(HYPERV_FEAT_VAPIC)
971     },
972     [HYPERV_FEAT_IPI] = {
973         .desc = "paravirtualized IPI (hv-ipi)",
974         .flags = {
975             {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
976              .bits = HV_CLUSTER_IPI_RECOMMENDED |
977              HV_EX_PROCESSOR_MASKS_RECOMMENDED}
978         },
979         .dependencies = BIT(HYPERV_FEAT_VPINDEX)
980     },
981     [HYPERV_FEAT_STIMER_DIRECT] = {
982         .desc = "direct mode synthetic timers (hv-stimer-direct)",
983         .flags = {
984             {.func = HV_CPUID_FEATURES, .reg = R_EDX,
985              .bits = HV_STIMER_DIRECT_MODE_AVAILABLE}
986         },
987         .dependencies = BIT(HYPERV_FEAT_STIMER)
988     },
989     [HYPERV_FEAT_AVIC] = {
990         .desc = "AVIC/APICv support (hv-avic/hv-apicv)",
991         .flags = {
992             {.func = HV_CPUID_ENLIGHTMENT_INFO, .reg = R_EAX,
993              .bits = HV_DEPRECATING_AEOI_RECOMMENDED}
994         }
995     },
996 #ifdef CONFIG_SYNDBG
997     [HYPERV_FEAT_SYNDBG] = {
998         .desc = "Enable synthetic kernel debugger channel (hv-syndbg)",
999         .flags = {
1000             {.func = HV_CPUID_FEATURES, .reg = R_EDX,
1001              .bits = HV_FEATURE_DEBUG_MSRS_AVAILABLE}
1002         },
1003         .dependencies = BIT(HYPERV_FEAT_SYNIC) | BIT(HYPERV_FEAT_RELAXED)
1004     },
1005 #endif
1006     [HYPERV_FEAT_MSR_BITMAP] = {
1007         .desc = "enlightened MSR-Bitmap (hv-emsr-bitmap)",
1008         .flags = {
1009             {.func = HV_CPUID_NESTED_FEATURES, .reg = R_EAX,
1010              .bits = HV_NESTED_MSR_BITMAP}
1011         }
1012     },
1013     [HYPERV_FEAT_XMM_INPUT] = {
1014         .desc = "XMM fast hypercall input (hv-xmm-input)",
1015         .flags = {
1016             {.func = HV_CPUID_FEATURES, .reg = R_EDX,
1017              .bits = HV_HYPERCALL_XMM_INPUT_AVAILABLE}
1018         }
1019     },
1020     [HYPERV_FEAT_TLBFLUSH_EXT] = {
1021         .desc = "Extended gva ranges for TLB flush hypercalls (hv-tlbflush-ext)",
1022         .flags = {
1023             {.func = HV_CPUID_FEATURES, .reg = R_EDX,
1024              .bits = HV_EXT_GVA_RANGES_FLUSH_AVAILABLE}
1025         },
1026         .dependencies = BIT(HYPERV_FEAT_TLBFLUSH)
1027     },
1028     [HYPERV_FEAT_TLBFLUSH_DIRECT] = {
1029         .desc = "direct TLB flush (hv-tlbflush-direct)",
1030         .flags = {
1031             {.func = HV_CPUID_NESTED_FEATURES, .reg = R_EAX,
1032              .bits = HV_NESTED_DIRECT_FLUSH}
1033         },
1034         .dependencies = BIT(HYPERV_FEAT_VAPIC)
1035     },
1036 };
1037 
1038 static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max,
1039                                            bool do_sys_ioctl)
1040 {
1041     struct kvm_cpuid2 *cpuid;
1042     int r, size;
1043 
1044     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
1045     cpuid = g_malloc0(size);
1046     cpuid->nent = max;
1047 
1048     if (do_sys_ioctl) {
1049         r = kvm_ioctl(kvm_state, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1050     } else {
1051         r = kvm_vcpu_ioctl(cs, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
1052     }
1053     if (r == 0 && cpuid->nent >= max) {
1054         r = -E2BIG;
1055     }
1056     if (r < 0) {
1057         if (r == -E2BIG) {
1058             g_free(cpuid);
1059             return NULL;
1060         } else {
1061             fprintf(stderr, "KVM_GET_SUPPORTED_HV_CPUID failed: %s\n",
1062                     strerror(-r));
1063             exit(1);
1064         }
1065     }
1066     return cpuid;
1067 }
1068 
1069 /*
1070  * Run KVM_GET_SUPPORTED_HV_CPUID ioctl(), allocating a buffer large enough
1071  * for all entries.
1072  */
1073 static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs)
1074 {
1075     struct kvm_cpuid2 *cpuid;
1076     /* 0x40000000..0x40000005, 0x4000000A, 0x40000080..0x40000082 leaves */
1077     int max = 11;
1078     int i;
1079     bool do_sys_ioctl;
1080 
1081     do_sys_ioctl =
1082         kvm_check_extension(kvm_state, KVM_CAP_SYS_HYPERV_CPUID) > 0;
1083 
1084     /*
1085      * Non-empty KVM context is needed when KVM_CAP_SYS_HYPERV_CPUID is
1086      * unsupported, kvm_hyperv_expand_features() checks for that.
1087      */
1088     assert(do_sys_ioctl || cs->kvm_state);
1089 
1090     /*
1091      * When the buffer is too small, KVM_GET_SUPPORTED_HV_CPUID fails with
1092      * -E2BIG, however, it doesn't report back the right size. Keep increasing
1093      * it and re-trying until we succeed.
1094      */
1095     while ((cpuid = try_get_hv_cpuid(cs, max, do_sys_ioctl)) == NULL) {
1096         max++;
1097     }
1098 
1099     /*
1100      * KVM_GET_SUPPORTED_HV_CPUID does not set EVMCS CPUID bit before
1101      * KVM_CAP_HYPERV_ENLIGHTENED_VMCS is enabled but we want to get the
1102      * information early, just check for the capability and set the bit
1103      * manually.
1104      */
1105     if (!do_sys_ioctl && kvm_check_extension(cs->kvm_state,
1106                             KVM_CAP_HYPERV_ENLIGHTENED_VMCS) > 0) {
1107         for (i = 0; i < cpuid->nent; i++) {
1108             if (cpuid->entries[i].function == HV_CPUID_ENLIGHTMENT_INFO) {
1109                 cpuid->entries[i].eax |= HV_ENLIGHTENED_VMCS_RECOMMENDED;
1110             }
1111         }
1112     }
1113 
1114     return cpuid;
1115 }
1116 
1117 /*
1118  * When KVM_GET_SUPPORTED_HV_CPUID is not supported we fill CPUID feature
1119  * leaves from KVM_CAP_HYPERV* and present MSRs data.
1120  */
1121 static struct kvm_cpuid2 *get_supported_hv_cpuid_legacy(CPUState *cs)
1122 {
1123     X86CPU *cpu = X86_CPU(cs);
1124     struct kvm_cpuid2 *cpuid;
1125     struct kvm_cpuid_entry2 *entry_feat, *entry_recomm;
1126 
1127     /* HV_CPUID_FEATURES, HV_CPUID_ENLIGHTMENT_INFO */
1128     cpuid = g_malloc0(sizeof(*cpuid) + 2 * sizeof(*cpuid->entries));
1129     cpuid->nent = 2;
1130 
1131     /* HV_CPUID_VENDOR_AND_MAX_FUNCTIONS */
1132     entry_feat = &cpuid->entries[0];
1133     entry_feat->function = HV_CPUID_FEATURES;
1134 
1135     entry_recomm = &cpuid->entries[1];
1136     entry_recomm->function = HV_CPUID_ENLIGHTMENT_INFO;
1137     entry_recomm->ebx = cpu->hyperv_spinlock_attempts;
1138 
1139     if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0) {
1140         entry_feat->eax |= HV_HYPERCALL_AVAILABLE;
1141         entry_feat->eax |= HV_APIC_ACCESS_AVAILABLE;
1142         entry_feat->edx |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
1143         entry_recomm->eax |= HV_RELAXED_TIMING_RECOMMENDED;
1144         entry_recomm->eax |= HV_APIC_ACCESS_RECOMMENDED;
1145     }
1146 
1147     if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_TIME) > 0) {
1148         entry_feat->eax |= HV_TIME_REF_COUNT_AVAILABLE;
1149         entry_feat->eax |= HV_REFERENCE_TSC_AVAILABLE;
1150     }
1151 
1152     if (has_msr_hv_frequencies) {
1153         entry_feat->eax |= HV_ACCESS_FREQUENCY_MSRS;
1154         entry_feat->edx |= HV_FREQUENCY_MSRS_AVAILABLE;
1155     }
1156 
1157     if (has_msr_hv_crash) {
1158         entry_feat->edx |= HV_GUEST_CRASH_MSR_AVAILABLE;
1159     }
1160 
1161     if (has_msr_hv_reenlightenment) {
1162         entry_feat->eax |= HV_ACCESS_REENLIGHTENMENTS_CONTROL;
1163     }
1164 
1165     if (has_msr_hv_reset) {
1166         entry_feat->eax |= HV_RESET_AVAILABLE;
1167     }
1168 
1169     if (has_msr_hv_vpindex) {
1170         entry_feat->eax |= HV_VP_INDEX_AVAILABLE;
1171     }
1172 
1173     if (has_msr_hv_runtime) {
1174         entry_feat->eax |= HV_VP_RUNTIME_AVAILABLE;
1175     }
1176 
1177     if (has_msr_hv_synic) {
1178         unsigned int cap = cpu->hyperv_synic_kvm_only ?
1179             KVM_CAP_HYPERV_SYNIC : KVM_CAP_HYPERV_SYNIC2;
1180 
1181         if (kvm_check_extension(cs->kvm_state, cap) > 0) {
1182             entry_feat->eax |= HV_SYNIC_AVAILABLE;
1183         }
1184     }
1185 
1186     if (has_msr_hv_stimer) {
1187         entry_feat->eax |= HV_SYNTIMERS_AVAILABLE;
1188     }
1189 
1190     if (has_msr_hv_syndbg_options) {
1191         entry_feat->edx |= HV_GUEST_DEBUGGING_AVAILABLE;
1192         entry_feat->edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE;
1193         entry_feat->ebx |= HV_PARTITION_DEBUGGING_ALLOWED;
1194     }
1195 
1196     if (kvm_check_extension(cs->kvm_state,
1197                             KVM_CAP_HYPERV_TLBFLUSH) > 0) {
1198         entry_recomm->eax |= HV_REMOTE_TLB_FLUSH_RECOMMENDED;
1199         entry_recomm->eax |= HV_EX_PROCESSOR_MASKS_RECOMMENDED;
1200     }
1201 
1202     if (kvm_check_extension(cs->kvm_state,
1203                             KVM_CAP_HYPERV_ENLIGHTENED_VMCS) > 0) {
1204         entry_recomm->eax |= HV_ENLIGHTENED_VMCS_RECOMMENDED;
1205     }
1206 
1207     if (kvm_check_extension(cs->kvm_state,
1208                             KVM_CAP_HYPERV_SEND_IPI) > 0) {
1209         entry_recomm->eax |= HV_CLUSTER_IPI_RECOMMENDED;
1210         entry_recomm->eax |= HV_EX_PROCESSOR_MASKS_RECOMMENDED;
1211     }
1212 
1213     return cpuid;
1214 }
1215 
1216 static uint32_t hv_cpuid_get_host(CPUState *cs, uint32_t func, int reg)
1217 {
1218     struct kvm_cpuid_entry2 *entry;
1219     struct kvm_cpuid2 *cpuid;
1220 
1221     if (hv_cpuid_cache) {
1222         cpuid = hv_cpuid_cache;
1223     } else {
1224         if (kvm_check_extension(kvm_state, KVM_CAP_HYPERV_CPUID) > 0) {
1225             cpuid = get_supported_hv_cpuid(cs);
1226         } else {
1227             /*
1228              * 'cs->kvm_state' may be NULL when Hyper-V features are expanded
1229              * before KVM context is created but this is only done when
1230              * KVM_CAP_SYS_HYPERV_CPUID is supported and it implies
1231              * KVM_CAP_HYPERV_CPUID.
1232              */
1233             assert(cs->kvm_state);
1234 
1235             cpuid = get_supported_hv_cpuid_legacy(cs);
1236         }
1237         hv_cpuid_cache = cpuid;
1238     }
1239 
1240     if (!cpuid) {
1241         return 0;
1242     }
1243 
1244     entry = cpuid_find_entry(cpuid, func, 0);
1245     if (!entry) {
1246         return 0;
1247     }
1248 
1249     return cpuid_entry_get_reg(entry, reg);
1250 }
1251 
1252 static bool hyperv_feature_supported(CPUState *cs, int feature)
1253 {
1254     uint32_t func, bits;
1255     int i, reg;
1256 
1257     for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
1258 
1259         func = kvm_hyperv_properties[feature].flags[i].func;
1260         reg = kvm_hyperv_properties[feature].flags[i].reg;
1261         bits = kvm_hyperv_properties[feature].flags[i].bits;
1262 
1263         if (!func) {
1264             continue;
1265         }
1266 
1267         if ((hv_cpuid_get_host(cs, func, reg) & bits) != bits) {
1268             return false;
1269         }
1270     }
1271 
1272     return true;
1273 }
1274 
1275 /* Checks that all feature dependencies are enabled */
1276 static bool hv_feature_check_deps(X86CPU *cpu, int feature, Error **errp)
1277 {
1278     uint64_t deps;
1279     int dep_feat;
1280 
1281     deps = kvm_hyperv_properties[feature].dependencies;
1282     while (deps) {
1283         dep_feat = ctz64(deps);
1284         if (!(hyperv_feat_enabled(cpu, dep_feat))) {
1285             error_setg(errp, "Hyper-V %s requires Hyper-V %s",
1286                        kvm_hyperv_properties[feature].desc,
1287                        kvm_hyperv_properties[dep_feat].desc);
1288             return false;
1289         }
1290         deps &= ~(1ull << dep_feat);
1291     }
1292 
1293     return true;
1294 }
1295 
1296 static uint32_t hv_build_cpuid_leaf(CPUState *cs, uint32_t func, int reg)
1297 {
1298     X86CPU *cpu = X86_CPU(cs);
1299     uint32_t r = 0;
1300     int i, j;
1301 
1302     for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties); i++) {
1303         if (!hyperv_feat_enabled(cpu, i)) {
1304             continue;
1305         }
1306 
1307         for (j = 0; j < ARRAY_SIZE(kvm_hyperv_properties[i].flags); j++) {
1308             if (kvm_hyperv_properties[i].flags[j].func != func) {
1309                 continue;
1310             }
1311             if (kvm_hyperv_properties[i].flags[j].reg != reg) {
1312                 continue;
1313             }
1314 
1315             r |= kvm_hyperv_properties[i].flags[j].bits;
1316         }
1317     }
1318 
1319     /* HV_CPUID_NESTED_FEATURES.EAX also encodes the supported eVMCS range */
1320     if (func == HV_CPUID_NESTED_FEATURES && reg == R_EAX) {
1321         if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS)) {
1322             r |= DEFAULT_EVMCS_VERSION;
1323         }
1324     }
1325 
1326     return r;
1327 }
1328 
1329 /*
1330  * Expand Hyper-V CPU features. In partucular, check that all the requested
1331  * features are supported by the host and the sanity of the configuration
1332  * (that all the required dependencies are included). Also, this takes care
1333  * of 'hv_passthrough' mode and fills the environment with all supported
1334  * Hyper-V features.
1335  */
1336 bool kvm_hyperv_expand_features(X86CPU *cpu, Error **errp)
1337 {
1338     CPUState *cs = CPU(cpu);
1339     Error *local_err = NULL;
1340     int feat;
1341 
1342     if (!hyperv_enabled(cpu))
1343         return true;
1344 
1345     /*
1346      * When kvm_hyperv_expand_features is called at CPU feature expansion
1347      * time per-CPU kvm_state is not available yet so we can only proceed
1348      * when KVM_CAP_SYS_HYPERV_CPUID is supported.
1349      */
1350     if (!cs->kvm_state &&
1351         !kvm_check_extension(kvm_state, KVM_CAP_SYS_HYPERV_CPUID))
1352         return true;
1353 
1354     if (cpu->hyperv_passthrough) {
1355         cpu->hyperv_vendor_id[0] =
1356             hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_EBX);
1357         cpu->hyperv_vendor_id[1] =
1358             hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_ECX);
1359         cpu->hyperv_vendor_id[2] =
1360             hv_cpuid_get_host(cs, HV_CPUID_VENDOR_AND_MAX_FUNCTIONS, R_EDX);
1361         cpu->hyperv_vendor = g_realloc(cpu->hyperv_vendor,
1362                                        sizeof(cpu->hyperv_vendor_id) + 1);
1363         memcpy(cpu->hyperv_vendor, cpu->hyperv_vendor_id,
1364                sizeof(cpu->hyperv_vendor_id));
1365         cpu->hyperv_vendor[sizeof(cpu->hyperv_vendor_id)] = 0;
1366 
1367         cpu->hyperv_interface_id[0] =
1368             hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_EAX);
1369         cpu->hyperv_interface_id[1] =
1370             hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_EBX);
1371         cpu->hyperv_interface_id[2] =
1372             hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_ECX);
1373         cpu->hyperv_interface_id[3] =
1374             hv_cpuid_get_host(cs, HV_CPUID_INTERFACE, R_EDX);
1375 
1376         cpu->hyperv_ver_id_build =
1377             hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EAX);
1378         cpu->hyperv_ver_id_major =
1379             hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EBX) >> 16;
1380         cpu->hyperv_ver_id_minor =
1381             hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EBX) & 0xffff;
1382         cpu->hyperv_ver_id_sp =
1383             hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_ECX);
1384         cpu->hyperv_ver_id_sb =
1385             hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EDX) >> 24;
1386         cpu->hyperv_ver_id_sn =
1387             hv_cpuid_get_host(cs, HV_CPUID_VERSION, R_EDX) & 0xffffff;
1388 
1389         cpu->hv_max_vps = hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS,
1390                                             R_EAX);
1391         cpu->hyperv_limits[0] =
1392             hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_EBX);
1393         cpu->hyperv_limits[1] =
1394             hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_ECX);
1395         cpu->hyperv_limits[2] =
1396             hv_cpuid_get_host(cs, HV_CPUID_IMPLEMENT_LIMITS, R_EDX);
1397 
1398         cpu->hyperv_spinlock_attempts =
1399             hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EBX);
1400 
1401         /*
1402          * Mark feature as enabled in 'cpu->hyperv_features' as
1403          * hv_build_cpuid_leaf() uses this info to build guest CPUIDs.
1404          */
1405         for (feat = 0; feat < ARRAY_SIZE(kvm_hyperv_properties); feat++) {
1406             if (hyperv_feature_supported(cs, feat)) {
1407                 cpu->hyperv_features |= BIT(feat);
1408             }
1409         }
1410     } else {
1411         /* Check features availability and dependencies */
1412         for (feat = 0; feat < ARRAY_SIZE(kvm_hyperv_properties); feat++) {
1413             /* If the feature was not requested skip it. */
1414             if (!hyperv_feat_enabled(cpu, feat)) {
1415                 continue;
1416             }
1417 
1418             /* Check if the feature is supported by KVM */
1419             if (!hyperv_feature_supported(cs, feat)) {
1420                 error_setg(errp, "Hyper-V %s is not supported by kernel",
1421                            kvm_hyperv_properties[feat].desc);
1422                 return false;
1423             }
1424 
1425             /* Check dependencies */
1426             if (!hv_feature_check_deps(cpu, feat, &local_err)) {
1427                 error_propagate(errp, local_err);
1428                 return false;
1429             }
1430         }
1431     }
1432 
1433     /* Additional dependencies not covered by kvm_hyperv_properties[] */
1434     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC) &&
1435         !cpu->hyperv_synic_kvm_only &&
1436         !hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX)) {
1437         error_setg(errp, "Hyper-V %s requires Hyper-V %s",
1438                    kvm_hyperv_properties[HYPERV_FEAT_SYNIC].desc,
1439                    kvm_hyperv_properties[HYPERV_FEAT_VPINDEX].desc);
1440         return false;
1441     }
1442 
1443     return true;
1444 }
1445 
1446 /*
1447  * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent.
1448  */
1449 static int hyperv_fill_cpuids(CPUState *cs,
1450                               struct kvm_cpuid_entry2 *cpuid_ent)
1451 {
1452     X86CPU *cpu = X86_CPU(cs);
1453     struct kvm_cpuid_entry2 *c;
1454     uint32_t signature[3];
1455     uint32_t cpuid_i = 0, max_cpuid_leaf = 0;
1456     uint32_t nested_eax =
1457         hv_build_cpuid_leaf(cs, HV_CPUID_NESTED_FEATURES, R_EAX);
1458 
1459     max_cpuid_leaf = nested_eax ? HV_CPUID_NESTED_FEATURES :
1460         HV_CPUID_IMPLEMENT_LIMITS;
1461 
1462     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNDBG)) {
1463         max_cpuid_leaf =
1464             MAX(max_cpuid_leaf, HV_CPUID_SYNDBG_PLATFORM_CAPABILITIES);
1465     }
1466 
1467     c = &cpuid_ent[cpuid_i++];
1468     c->function = HV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
1469     c->eax = max_cpuid_leaf;
1470     c->ebx = cpu->hyperv_vendor_id[0];
1471     c->ecx = cpu->hyperv_vendor_id[1];
1472     c->edx = cpu->hyperv_vendor_id[2];
1473 
1474     c = &cpuid_ent[cpuid_i++];
1475     c->function = HV_CPUID_INTERFACE;
1476     c->eax = cpu->hyperv_interface_id[0];
1477     c->ebx = cpu->hyperv_interface_id[1];
1478     c->ecx = cpu->hyperv_interface_id[2];
1479     c->edx = cpu->hyperv_interface_id[3];
1480 
1481     c = &cpuid_ent[cpuid_i++];
1482     c->function = HV_CPUID_VERSION;
1483     c->eax = cpu->hyperv_ver_id_build;
1484     c->ebx = (uint32_t)cpu->hyperv_ver_id_major << 16 |
1485         cpu->hyperv_ver_id_minor;
1486     c->ecx = cpu->hyperv_ver_id_sp;
1487     c->edx = (uint32_t)cpu->hyperv_ver_id_sb << 24 |
1488         (cpu->hyperv_ver_id_sn & 0xffffff);
1489 
1490     c = &cpuid_ent[cpuid_i++];
1491     c->function = HV_CPUID_FEATURES;
1492     c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EAX);
1493     c->ebx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EBX);
1494     c->edx = hv_build_cpuid_leaf(cs, HV_CPUID_FEATURES, R_EDX);
1495 
1496     /* Unconditionally required with any Hyper-V enlightenment */
1497     c->eax |= HV_HYPERCALL_AVAILABLE;
1498 
1499     /* SynIC and Vmbus devices require messages/signals hypercalls */
1500     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC) &&
1501         !cpu->hyperv_synic_kvm_only) {
1502         c->ebx |= HV_POST_MESSAGES | HV_SIGNAL_EVENTS;
1503     }
1504 
1505 
1506     /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
1507     c->edx |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
1508 
1509     c = &cpuid_ent[cpuid_i++];
1510     c->function = HV_CPUID_ENLIGHTMENT_INFO;
1511     c->eax = hv_build_cpuid_leaf(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX);
1512     c->ebx = cpu->hyperv_spinlock_attempts;
1513 
1514     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VAPIC) &&
1515         !hyperv_feat_enabled(cpu, HYPERV_FEAT_AVIC)) {
1516         c->eax |= HV_APIC_ACCESS_RECOMMENDED;
1517     }
1518 
1519     if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
1520         c->eax |= HV_NO_NONARCH_CORESHARING;
1521     } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
1522         c->eax |= hv_cpuid_get_host(cs, HV_CPUID_ENLIGHTMENT_INFO, R_EAX) &
1523             HV_NO_NONARCH_CORESHARING;
1524     }
1525 
1526     c = &cpuid_ent[cpuid_i++];
1527     c->function = HV_CPUID_IMPLEMENT_LIMITS;
1528     c->eax = cpu->hv_max_vps;
1529     c->ebx = cpu->hyperv_limits[0];
1530     c->ecx = cpu->hyperv_limits[1];
1531     c->edx = cpu->hyperv_limits[2];
1532 
1533     if (nested_eax) {
1534         uint32_t function;
1535 
1536         /* Create zeroed 0x40000006..0x40000009 leaves */
1537         for (function = HV_CPUID_IMPLEMENT_LIMITS + 1;
1538              function < HV_CPUID_NESTED_FEATURES; function++) {
1539             c = &cpuid_ent[cpuid_i++];
1540             c->function = function;
1541         }
1542 
1543         c = &cpuid_ent[cpuid_i++];
1544         c->function = HV_CPUID_NESTED_FEATURES;
1545         c->eax = nested_eax;
1546     }
1547 
1548     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNDBG)) {
1549         c = &cpuid_ent[cpuid_i++];
1550         c->function = HV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS;
1551         c->eax = hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ?
1552             HV_CPUID_NESTED_FEATURES : HV_CPUID_IMPLEMENT_LIMITS;
1553         memcpy(signature, "Microsoft VS", 12);
1554         c->eax = 0;
1555         c->ebx = signature[0];
1556         c->ecx = signature[1];
1557         c->edx = signature[2];
1558 
1559         c = &cpuid_ent[cpuid_i++];
1560         c->function = HV_CPUID_SYNDBG_INTERFACE;
1561         memcpy(signature, "VS#1\0\0\0\0\0\0\0\0", 12);
1562         c->eax = signature[0];
1563         c->ebx = 0;
1564         c->ecx = 0;
1565         c->edx = 0;
1566 
1567         c = &cpuid_ent[cpuid_i++];
1568         c->function = HV_CPUID_SYNDBG_PLATFORM_CAPABILITIES;
1569         c->eax = HV_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING;
1570         c->ebx = 0;
1571         c->ecx = 0;
1572         c->edx = 0;
1573     }
1574 
1575     return cpuid_i;
1576 }
1577 
1578 static Error *hv_passthrough_mig_blocker;
1579 static Error *hv_no_nonarch_cs_mig_blocker;
1580 
1581 /* Checks that the exposed eVMCS version range is supported by KVM */
1582 static bool evmcs_version_supported(uint16_t evmcs_version,
1583                                     uint16_t supported_evmcs_version)
1584 {
1585     uint8_t min_version = evmcs_version & 0xff;
1586     uint8_t max_version = evmcs_version >> 8;
1587     uint8_t min_supported_version = supported_evmcs_version & 0xff;
1588     uint8_t max_supported_version = supported_evmcs_version >> 8;
1589 
1590     return (min_version >= min_supported_version) &&
1591         (max_version <= max_supported_version);
1592 }
1593 
1594 static int hyperv_init_vcpu(X86CPU *cpu)
1595 {
1596     CPUState *cs = CPU(cpu);
1597     Error *local_err = NULL;
1598     int ret;
1599 
1600     if (cpu->hyperv_passthrough && hv_passthrough_mig_blocker == NULL) {
1601         error_setg(&hv_passthrough_mig_blocker,
1602                    "'hv-passthrough' CPU flag prevents migration, use explicit"
1603                    " set of hv-* flags instead");
1604         ret = migrate_add_blocker(hv_passthrough_mig_blocker, &local_err);
1605         if (ret < 0) {
1606             error_report_err(local_err);
1607             return ret;
1608         }
1609     }
1610 
1611     if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO &&
1612         hv_no_nonarch_cs_mig_blocker == NULL) {
1613         error_setg(&hv_no_nonarch_cs_mig_blocker,
1614                    "'hv-no-nonarch-coresharing=auto' CPU flag prevents migration"
1615                    " use explicit 'hv-no-nonarch-coresharing=on' instead (but"
1616                    " make sure SMT is disabled and/or that vCPUs are properly"
1617                    " pinned)");
1618         ret = migrate_add_blocker(hv_no_nonarch_cs_mig_blocker, &local_err);
1619         if (ret < 0) {
1620             error_report_err(local_err);
1621             return ret;
1622         }
1623     }
1624 
1625     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX) && !hv_vpindex_settable) {
1626         /*
1627          * the kernel doesn't support setting vp_index; assert that its value
1628          * is in sync
1629          */
1630         uint64_t value;
1631 
1632         ret = kvm_get_one_msr(cpu, HV_X64_MSR_VP_INDEX, &value);
1633         if (ret < 0) {
1634             return ret;
1635         }
1636 
1637         if (value != hyperv_vp_index(CPU(cpu))) {
1638             error_report("kernel's vp_index != QEMU's vp_index");
1639             return -ENXIO;
1640         }
1641     }
1642 
1643     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
1644         uint32_t synic_cap = cpu->hyperv_synic_kvm_only ?
1645             KVM_CAP_HYPERV_SYNIC : KVM_CAP_HYPERV_SYNIC2;
1646         ret = kvm_vcpu_enable_cap(cs, synic_cap, 0);
1647         if (ret < 0) {
1648             error_report("failed to turn on HyperV SynIC in KVM: %s",
1649                          strerror(-ret));
1650             return ret;
1651         }
1652 
1653         if (!cpu->hyperv_synic_kvm_only) {
1654             ret = hyperv_x86_synic_add(cpu);
1655             if (ret < 0) {
1656                 error_report("failed to create HyperV SynIC: %s",
1657                              strerror(-ret));
1658                 return ret;
1659             }
1660         }
1661     }
1662 
1663     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS)) {
1664         uint16_t evmcs_version = DEFAULT_EVMCS_VERSION;
1665         uint16_t supported_evmcs_version;
1666 
1667         ret = kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_ENLIGHTENED_VMCS, 0,
1668                                   (uintptr_t)&supported_evmcs_version);
1669 
1670         /*
1671          * KVM is required to support EVMCS ver.1. as that's what 'hv-evmcs'
1672          * option sets. Note: we hardcode the maximum supported eVMCS version
1673          * to '1' as well so 'hv-evmcs' feature is migratable even when (and if)
1674          * ver.2 is implemented. A new option (e.g. 'hv-evmcs=2') will then have
1675          * to be added.
1676          */
1677         if (ret < 0) {
1678             error_report("Hyper-V %s is not supported by kernel",
1679                          kvm_hyperv_properties[HYPERV_FEAT_EVMCS].desc);
1680             return ret;
1681         }
1682 
1683         if (!evmcs_version_supported(evmcs_version, supported_evmcs_version)) {
1684             error_report("eVMCS version range [%d..%d] is not supported by "
1685                          "kernel (supported: [%d..%d])", evmcs_version & 0xff,
1686                          evmcs_version >> 8, supported_evmcs_version & 0xff,
1687                          supported_evmcs_version >> 8);
1688             return -ENOTSUP;
1689         }
1690     }
1691 
1692     if (cpu->hyperv_enforce_cpuid) {
1693         ret = kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_ENFORCE_CPUID, 0, 1);
1694         if (ret < 0) {
1695             error_report("failed to enable KVM_CAP_HYPERV_ENFORCE_CPUID: %s",
1696                          strerror(-ret));
1697             return ret;
1698         }
1699     }
1700 
1701     return 0;
1702 }
1703 
1704 static Error *invtsc_mig_blocker;
1705 
1706 #define KVM_MAX_CPUID_ENTRIES  100
1707 
1708 static void kvm_init_xsave(CPUX86State *env)
1709 {
1710     if (has_xsave2) {
1711         env->xsave_buf_len = QEMU_ALIGN_UP(has_xsave2, 4096);
1712     } else if (has_xsave) {
1713         env->xsave_buf_len = sizeof(struct kvm_xsave);
1714     } else {
1715         return;
1716     }
1717 
1718     env->xsave_buf = qemu_memalign(4096, env->xsave_buf_len);
1719     memset(env->xsave_buf, 0, env->xsave_buf_len);
1720     /*
1721      * The allocated storage must be large enough for all of the
1722      * possible XSAVE state components.
1723      */
1724     assert(kvm_arch_get_supported_cpuid(kvm_state, 0xd, 0, R_ECX) <=
1725            env->xsave_buf_len);
1726 }
1727 
1728 static void kvm_init_nested_state(CPUX86State *env)
1729 {
1730     struct kvm_vmx_nested_state_hdr *vmx_hdr;
1731     uint32_t size;
1732 
1733     if (!env->nested_state) {
1734         return;
1735     }
1736 
1737     size = env->nested_state->size;
1738 
1739     memset(env->nested_state, 0, size);
1740     env->nested_state->size = size;
1741 
1742     if (cpu_has_vmx(env)) {
1743         env->nested_state->format = KVM_STATE_NESTED_FORMAT_VMX;
1744         vmx_hdr = &env->nested_state->hdr.vmx;
1745         vmx_hdr->vmxon_pa = -1ull;
1746         vmx_hdr->vmcs12_pa = -1ull;
1747     } else if (cpu_has_svm(env)) {
1748         env->nested_state->format = KVM_STATE_NESTED_FORMAT_SVM;
1749     }
1750 }
1751 
1752 int kvm_arch_init_vcpu(CPUState *cs)
1753 {
1754     struct {
1755         struct kvm_cpuid2 cpuid;
1756         struct kvm_cpuid_entry2 entries[KVM_MAX_CPUID_ENTRIES];
1757     } cpuid_data;
1758     /*
1759      * The kernel defines these structs with padding fields so there
1760      * should be no extra padding in our cpuid_data struct.
1761      */
1762     QEMU_BUILD_BUG_ON(sizeof(cpuid_data) !=
1763                       sizeof(struct kvm_cpuid2) +
1764                       sizeof(struct kvm_cpuid_entry2) * KVM_MAX_CPUID_ENTRIES);
1765 
1766     X86CPU *cpu = X86_CPU(cs);
1767     CPUX86State *env = &cpu->env;
1768     uint32_t limit, i, j, cpuid_i;
1769     uint32_t unused;
1770     struct kvm_cpuid_entry2 *c;
1771     uint32_t signature[3];
1772     int kvm_base = KVM_CPUID_SIGNATURE;
1773     int max_nested_state_len;
1774     int r;
1775     Error *local_err = NULL;
1776 
1777     memset(&cpuid_data, 0, sizeof(cpuid_data));
1778 
1779     cpuid_i = 0;
1780 
1781     has_xsave2 = kvm_check_extension(cs->kvm_state, KVM_CAP_XSAVE2);
1782 
1783     r = kvm_arch_set_tsc_khz(cs);
1784     if (r < 0) {
1785         return r;
1786     }
1787 
1788     /* vcpu's TSC frequency is either specified by user, or following
1789      * the value used by KVM if the former is not present. In the
1790      * latter case, we query it from KVM and record in env->tsc_khz,
1791      * so that vcpu's TSC frequency can be migrated later via this field.
1792      */
1793     if (!env->tsc_khz) {
1794         r = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
1795             kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
1796             -ENOTSUP;
1797         if (r > 0) {
1798             env->tsc_khz = r;
1799         }
1800     }
1801 
1802     env->apic_bus_freq = KVM_APIC_BUS_FREQUENCY;
1803 
1804     /*
1805      * kvm_hyperv_expand_features() is called here for the second time in case
1806      * KVM_CAP_SYS_HYPERV_CPUID is not supported. While we can't possibly handle
1807      * 'query-cpu-model-expansion' in this case as we don't have a KVM vCPU to
1808      * check which Hyper-V enlightenments are supported and which are not, we
1809      * can still proceed and check/expand Hyper-V enlightenments here so legacy
1810      * behavior is preserved.
1811      */
1812     if (!kvm_hyperv_expand_features(cpu, &local_err)) {
1813         error_report_err(local_err);
1814         return -ENOSYS;
1815     }
1816 
1817     if (hyperv_enabled(cpu)) {
1818         r = hyperv_init_vcpu(cpu);
1819         if (r) {
1820             return r;
1821         }
1822 
1823         cpuid_i = hyperv_fill_cpuids(cs, cpuid_data.entries);
1824         kvm_base = KVM_CPUID_SIGNATURE_NEXT;
1825         has_msr_hv_hypercall = true;
1826     }
1827 
1828     if (cs->kvm_state->xen_version) {
1829 #ifdef CONFIG_XEN_EMU
1830         struct kvm_cpuid_entry2 *xen_max_leaf;
1831 
1832         memcpy(signature, "XenVMMXenVMM", 12);
1833 
1834         xen_max_leaf = c = &cpuid_data.entries[cpuid_i++];
1835         c->function = kvm_base + XEN_CPUID_SIGNATURE;
1836         c->eax = kvm_base + XEN_CPUID_TIME;
1837         c->ebx = signature[0];
1838         c->ecx = signature[1];
1839         c->edx = signature[2];
1840 
1841         c = &cpuid_data.entries[cpuid_i++];
1842         c->function = kvm_base + XEN_CPUID_VENDOR;
1843         c->eax = cs->kvm_state->xen_version;
1844         c->ebx = 0;
1845         c->ecx = 0;
1846         c->edx = 0;
1847 
1848         c = &cpuid_data.entries[cpuid_i++];
1849         c->function = kvm_base + XEN_CPUID_HVM_MSR;
1850         /* Number of hypercall-transfer pages */
1851         c->eax = 1;
1852         /* Hypercall MSR base address */
1853         if (hyperv_enabled(cpu)) {
1854             c->ebx = XEN_HYPERCALL_MSR_HYPERV;
1855             kvm_xen_init(cs->kvm_state, c->ebx);
1856         } else {
1857             c->ebx = XEN_HYPERCALL_MSR;
1858         }
1859         c->ecx = 0;
1860         c->edx = 0;
1861 
1862         c = &cpuid_data.entries[cpuid_i++];
1863         c->function = kvm_base + XEN_CPUID_TIME;
1864         c->eax = ((!!tsc_is_stable_and_known(env) << 1) |
1865             (!!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_RDTSCP) << 2));
1866         /* default=0 (emulate if necessary) */
1867         c->ebx = 0;
1868         /* guest tsc frequency */
1869         c->ecx = env->user_tsc_khz;
1870         /* guest tsc incarnation (migration count) */
1871         c->edx = 0;
1872 
1873         c = &cpuid_data.entries[cpuid_i++];
1874         c->function = kvm_base + XEN_CPUID_HVM;
1875         xen_max_leaf->eax = kvm_base + XEN_CPUID_HVM;
1876         if (cs->kvm_state->xen_version >= XEN_VERSION(4, 5)) {
1877             c->function = kvm_base + XEN_CPUID_HVM;
1878 
1879             if (cpu->xen_vapic) {
1880                 c->eax |= XEN_HVM_CPUID_APIC_ACCESS_VIRT;
1881                 c->eax |= XEN_HVM_CPUID_X2APIC_VIRT;
1882             }
1883 
1884             c->eax |= XEN_HVM_CPUID_IOMMU_MAPPINGS;
1885 
1886             if (cs->kvm_state->xen_version >= XEN_VERSION(4, 6)) {
1887                 c->eax |= XEN_HVM_CPUID_VCPU_ID_PRESENT;
1888                 c->ebx = cs->cpu_index;
1889             }
1890         }
1891 
1892         r = kvm_xen_init_vcpu(cs);
1893         if (r) {
1894             return r;
1895         }
1896 
1897         kvm_base += 0x100;
1898 #else /* CONFIG_XEN_EMU */
1899         /* This should never happen as kvm_arch_init() would have died first. */
1900         fprintf(stderr, "Cannot enable Xen CPUID without Xen support\n");
1901         abort();
1902 #endif
1903     } else if (cpu->expose_kvm) {
1904         memcpy(signature, "KVMKVMKVM\0\0\0", 12);
1905         c = &cpuid_data.entries[cpuid_i++];
1906         c->function = KVM_CPUID_SIGNATURE | kvm_base;
1907         c->eax = KVM_CPUID_FEATURES | kvm_base;
1908         c->ebx = signature[0];
1909         c->ecx = signature[1];
1910         c->edx = signature[2];
1911 
1912         c = &cpuid_data.entries[cpuid_i++];
1913         c->function = KVM_CPUID_FEATURES | kvm_base;
1914         c->eax = env->features[FEAT_KVM];
1915         c->edx = env->features[FEAT_KVM_HINTS];
1916     }
1917 
1918     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
1919 
1920     if (cpu->kvm_pv_enforce_cpuid) {
1921         r = kvm_vcpu_enable_cap(cs, KVM_CAP_ENFORCE_PV_FEATURE_CPUID, 0, 1);
1922         if (r < 0) {
1923             fprintf(stderr,
1924                     "failed to enable KVM_CAP_ENFORCE_PV_FEATURE_CPUID: %s",
1925                     strerror(-r));
1926             abort();
1927         }
1928     }
1929 
1930     for (i = 0; i <= limit; i++) {
1931         if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1932             fprintf(stderr, "unsupported level value: 0x%x\n", limit);
1933             abort();
1934         }
1935         c = &cpuid_data.entries[cpuid_i++];
1936 
1937         switch (i) {
1938         case 2: {
1939             /* Keep reading function 2 till all the input is received */
1940             int times;
1941 
1942             c->function = i;
1943             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
1944                        KVM_CPUID_FLAG_STATE_READ_NEXT;
1945             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1946             times = c->eax & 0xff;
1947 
1948             for (j = 1; j < times; ++j) {
1949                 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1950                     fprintf(stderr, "cpuid_data is full, no space for "
1951                             "cpuid(eax:2):eax & 0xf = 0x%x\n", times);
1952                     abort();
1953                 }
1954                 c = &cpuid_data.entries[cpuid_i++];
1955                 c->function = i;
1956                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
1957                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1958             }
1959             break;
1960         }
1961         case 0x1f:
1962             if (env->nr_dies < 2) {
1963                 break;
1964             }
1965             /* fallthrough */
1966         case 4:
1967         case 0xb:
1968         case 0xd:
1969             for (j = 0; ; j++) {
1970                 if (i == 0xd && j == 64) {
1971                     break;
1972                 }
1973 
1974                 if (i == 0x1f && j == 64) {
1975                     break;
1976                 }
1977 
1978                 c->function = i;
1979                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1980                 c->index = j;
1981                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
1982 
1983                 if (i == 4 && c->eax == 0) {
1984                     break;
1985                 }
1986                 if (i == 0xb && !(c->ecx & 0xff00)) {
1987                     break;
1988                 }
1989                 if (i == 0x1f && !(c->ecx & 0xff00)) {
1990                     break;
1991                 }
1992                 if (i == 0xd && c->eax == 0) {
1993                     continue;
1994                 }
1995                 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1996                     fprintf(stderr, "cpuid_data is full, no space for "
1997                             "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
1998                     abort();
1999                 }
2000                 c = &cpuid_data.entries[cpuid_i++];
2001             }
2002             break;
2003         case 0x7:
2004         case 0x12:
2005             for (j = 0; ; j++) {
2006                 c->function = i;
2007                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
2008                 c->index = j;
2009                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
2010 
2011                 if (j > 1 && (c->eax & 0xf) != 1) {
2012                     break;
2013                 }
2014 
2015                 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
2016                     fprintf(stderr, "cpuid_data is full, no space for "
2017                                 "cpuid(eax:0x12,ecx:0x%x)\n", j);
2018                     abort();
2019                 }
2020                 c = &cpuid_data.entries[cpuid_i++];
2021             }
2022             break;
2023         case 0x14:
2024         case 0x1d:
2025         case 0x1e: {
2026             uint32_t times;
2027 
2028             c->function = i;
2029             c->index = 0;
2030             c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
2031             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
2032             times = c->eax;
2033 
2034             for (j = 1; j <= times; ++j) {
2035                 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
2036                     fprintf(stderr, "cpuid_data is full, no space for "
2037                                 "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
2038                     abort();
2039                 }
2040                 c = &cpuid_data.entries[cpuid_i++];
2041                 c->function = i;
2042                 c->index = j;
2043                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
2044                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
2045             }
2046             break;
2047         }
2048         default:
2049             c->function = i;
2050             c->flags = 0;
2051             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
2052             if (!c->eax && !c->ebx && !c->ecx && !c->edx) {
2053                 /*
2054                  * KVM already returns all zeroes if a CPUID entry is missing,
2055                  * so we can omit it and avoid hitting KVM's 80-entry limit.
2056                  */
2057                 cpuid_i--;
2058             }
2059             break;
2060         }
2061     }
2062 
2063     if (limit >= 0x0a) {
2064         uint32_t eax, edx;
2065 
2066         cpu_x86_cpuid(env, 0x0a, 0, &eax, &unused, &unused, &edx);
2067 
2068         has_architectural_pmu_version = eax & 0xff;
2069         if (has_architectural_pmu_version > 0) {
2070             num_architectural_pmu_gp_counters = (eax & 0xff00) >> 8;
2071 
2072             /* Shouldn't be more than 32, since that's the number of bits
2073              * available in EBX to tell us _which_ counters are available.
2074              * Play it safe.
2075              */
2076             if (num_architectural_pmu_gp_counters > MAX_GP_COUNTERS) {
2077                 num_architectural_pmu_gp_counters = MAX_GP_COUNTERS;
2078             }
2079 
2080             if (has_architectural_pmu_version > 1) {
2081                 num_architectural_pmu_fixed_counters = edx & 0x1f;
2082 
2083                 if (num_architectural_pmu_fixed_counters > MAX_FIXED_COUNTERS) {
2084                     num_architectural_pmu_fixed_counters = MAX_FIXED_COUNTERS;
2085                 }
2086             }
2087         }
2088     }
2089 
2090     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
2091 
2092     for (i = 0x80000000; i <= limit; i++) {
2093         if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
2094             fprintf(stderr, "unsupported xlevel value: 0x%x\n", limit);
2095             abort();
2096         }
2097         c = &cpuid_data.entries[cpuid_i++];
2098 
2099         switch (i) {
2100         case 0x8000001d:
2101             /* Query for all AMD cache information leaves */
2102             for (j = 0; ; j++) {
2103                 c->function = i;
2104                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
2105                 c->index = j;
2106                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
2107 
2108                 if (c->eax == 0) {
2109                     break;
2110                 }
2111                 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
2112                     fprintf(stderr, "cpuid_data is full, no space for "
2113                             "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
2114                     abort();
2115                 }
2116                 c = &cpuid_data.entries[cpuid_i++];
2117             }
2118             break;
2119         default:
2120             c->function = i;
2121             c->flags = 0;
2122             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
2123             if (!c->eax && !c->ebx && !c->ecx && !c->edx) {
2124                 /*
2125                  * KVM already returns all zeroes if a CPUID entry is missing,
2126                  * so we can omit it and avoid hitting KVM's 80-entry limit.
2127                  */
2128                 cpuid_i--;
2129             }
2130             break;
2131         }
2132     }
2133 
2134     /* Call Centaur's CPUID instructions they are supported. */
2135     if (env->cpuid_xlevel2 > 0) {
2136         cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
2137 
2138         for (i = 0xC0000000; i <= limit; i++) {
2139             if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
2140                 fprintf(stderr, "unsupported xlevel2 value: 0x%x\n", limit);
2141                 abort();
2142             }
2143             c = &cpuid_data.entries[cpuid_i++];
2144 
2145             c->function = i;
2146             c->flags = 0;
2147             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
2148         }
2149     }
2150 
2151     cpuid_data.cpuid.nent = cpuid_i;
2152 
2153     if (((env->cpuid_version >> 8)&0xF) >= 6
2154         && (env->features[FEAT_1_EDX] & (CPUID_MCE | CPUID_MCA)) ==
2155            (CPUID_MCE | CPUID_MCA)
2156         && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) {
2157         uint64_t mcg_cap, unsupported_caps;
2158         int banks;
2159         int ret;
2160 
2161         ret = kvm_get_mce_cap_supported(cs->kvm_state, &mcg_cap, &banks);
2162         if (ret < 0) {
2163             fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret));
2164             return ret;
2165         }
2166 
2167         if (banks < (env->mcg_cap & MCG_CAP_BANKS_MASK)) {
2168             error_report("kvm: Unsupported MCE bank count (QEMU = %d, KVM = %d)",
2169                          (int)(env->mcg_cap & MCG_CAP_BANKS_MASK), banks);
2170             return -ENOTSUP;
2171         }
2172 
2173         unsupported_caps = env->mcg_cap & ~(mcg_cap | MCG_CAP_BANKS_MASK);
2174         if (unsupported_caps) {
2175             if (unsupported_caps & MCG_LMCE_P) {
2176                 error_report("kvm: LMCE not supported");
2177                 return -ENOTSUP;
2178             }
2179             warn_report("Unsupported MCG_CAP bits: 0x%" PRIx64,
2180                         unsupported_caps);
2181         }
2182 
2183         env->mcg_cap &= mcg_cap | MCG_CAP_BANKS_MASK;
2184         ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &env->mcg_cap);
2185         if (ret < 0) {
2186             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
2187             return ret;
2188         }
2189     }
2190 
2191     cpu->vmsentry = qemu_add_vm_change_state_handler(cpu_update_state, env);
2192 
2193     c = cpuid_find_entry(&cpuid_data.cpuid, 1, 0);
2194     if (c) {
2195         has_msr_feature_control = !!(c->ecx & CPUID_EXT_VMX) ||
2196                                   !!(c->ecx & CPUID_EXT_SMX);
2197     }
2198 
2199     c = cpuid_find_entry(&cpuid_data.cpuid, 7, 0);
2200     if (c && (c->ebx & CPUID_7_0_EBX_SGX)) {
2201         has_msr_feature_control = true;
2202     }
2203 
2204     if (env->mcg_cap & MCG_LMCE_P) {
2205         has_msr_mcg_ext_ctl = has_msr_feature_control = true;
2206     }
2207 
2208     if (!env->user_tsc_khz) {
2209         if ((env->features[FEAT_8000_0007_EDX] & CPUID_APM_INVTSC) &&
2210             invtsc_mig_blocker == NULL) {
2211             error_setg(&invtsc_mig_blocker,
2212                        "State blocked by non-migratable CPU device"
2213                        " (invtsc flag)");
2214             r = migrate_add_blocker(invtsc_mig_blocker, &local_err);
2215             if (r < 0) {
2216                 error_report_err(local_err);
2217                 return r;
2218             }
2219         }
2220     }
2221 
2222     if (cpu->vmware_cpuid_freq
2223         /* Guests depend on 0x40000000 to detect this feature, so only expose
2224          * it if KVM exposes leaf 0x40000000. (Conflicts with Hyper-V) */
2225         && cpu->expose_kvm
2226         && kvm_base == KVM_CPUID_SIGNATURE
2227         /* TSC clock must be stable and known for this feature. */
2228         && tsc_is_stable_and_known(env)) {
2229 
2230         c = &cpuid_data.entries[cpuid_i++];
2231         c->function = KVM_CPUID_SIGNATURE | 0x10;
2232         c->eax = env->tsc_khz;
2233         c->ebx = env->apic_bus_freq / 1000; /* Hz to KHz */
2234         c->ecx = c->edx = 0;
2235 
2236         c = cpuid_find_entry(&cpuid_data.cpuid, kvm_base, 0);
2237         c->eax = MAX(c->eax, KVM_CPUID_SIGNATURE | 0x10);
2238     }
2239 
2240     cpuid_data.cpuid.nent = cpuid_i;
2241 
2242     cpuid_data.cpuid.padding = 0;
2243     r = kvm_vcpu_ioctl(cs, KVM_SET_CPUID2, &cpuid_data);
2244     if (r) {
2245         goto fail;
2246     }
2247     kvm_init_xsave(env);
2248 
2249     max_nested_state_len = kvm_max_nested_state_length();
2250     if (max_nested_state_len > 0) {
2251         assert(max_nested_state_len >= offsetof(struct kvm_nested_state, data));
2252 
2253         if (cpu_has_vmx(env) || cpu_has_svm(env)) {
2254             env->nested_state = g_malloc0(max_nested_state_len);
2255             env->nested_state->size = max_nested_state_len;
2256 
2257             kvm_init_nested_state(env);
2258         }
2259     }
2260 
2261     cpu->kvm_msr_buf = g_malloc0(MSR_BUF_SIZE);
2262 
2263     if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_RDTSCP)) {
2264         has_msr_tsc_aux = false;
2265     }
2266 
2267     kvm_init_msrs(cpu);
2268 
2269     return 0;
2270 
2271  fail:
2272     migrate_del_blocker(invtsc_mig_blocker);
2273 
2274     return r;
2275 }
2276 
2277 int kvm_arch_destroy_vcpu(CPUState *cs)
2278 {
2279     X86CPU *cpu = X86_CPU(cs);
2280     CPUX86State *env = &cpu->env;
2281 
2282     g_free(env->xsave_buf);
2283 
2284     g_free(cpu->kvm_msr_buf);
2285     cpu->kvm_msr_buf = NULL;
2286 
2287     g_free(env->nested_state);
2288     env->nested_state = NULL;
2289 
2290     qemu_del_vm_change_state_handler(cpu->vmsentry);
2291 
2292     return 0;
2293 }
2294 
2295 void kvm_arch_reset_vcpu(X86CPU *cpu)
2296 {
2297     CPUX86State *env = &cpu->env;
2298 
2299     env->xcr0 = 1;
2300     if (kvm_irqchip_in_kernel()) {
2301         env->mp_state = cpu_is_bsp(cpu) ? KVM_MP_STATE_RUNNABLE :
2302                                           KVM_MP_STATE_UNINITIALIZED;
2303     } else {
2304         env->mp_state = KVM_MP_STATE_RUNNABLE;
2305     }
2306 
2307     /* enabled by default */
2308     env->poll_control_msr = 1;
2309 
2310     kvm_init_nested_state(env);
2311 
2312     sev_es_set_reset_vector(CPU(cpu));
2313 }
2314 
2315 void kvm_arch_after_reset_vcpu(X86CPU *cpu)
2316 {
2317     CPUX86State *env = &cpu->env;
2318     int i;
2319 
2320     /*
2321      * Reset SynIC after all other devices have been reset to let them remove
2322      * their SINT routes first.
2323      */
2324     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
2325         for (i = 0; i < ARRAY_SIZE(env->msr_hv_synic_sint); i++) {
2326             env->msr_hv_synic_sint[i] = HV_SINT_MASKED;
2327         }
2328 
2329         hyperv_x86_synic_reset(cpu);
2330     }
2331 }
2332 
2333 void kvm_arch_do_init_vcpu(X86CPU *cpu)
2334 {
2335     CPUX86State *env = &cpu->env;
2336 
2337     /* APs get directly into wait-for-SIPI state.  */
2338     if (env->mp_state == KVM_MP_STATE_UNINITIALIZED) {
2339         env->mp_state = KVM_MP_STATE_INIT_RECEIVED;
2340     }
2341 }
2342 
2343 static int kvm_get_supported_feature_msrs(KVMState *s)
2344 {
2345     int ret = 0;
2346 
2347     if (kvm_feature_msrs != NULL) {
2348         return 0;
2349     }
2350 
2351     if (!kvm_check_extension(s, KVM_CAP_GET_MSR_FEATURES)) {
2352         return 0;
2353     }
2354 
2355     struct kvm_msr_list msr_list;
2356 
2357     msr_list.nmsrs = 0;
2358     ret = kvm_ioctl(s, KVM_GET_MSR_FEATURE_INDEX_LIST, &msr_list);
2359     if (ret < 0 && ret != -E2BIG) {
2360         error_report("Fetch KVM feature MSR list failed: %s",
2361             strerror(-ret));
2362         return ret;
2363     }
2364 
2365     assert(msr_list.nmsrs > 0);
2366     kvm_feature_msrs = g_malloc0(sizeof(msr_list) +
2367                  msr_list.nmsrs * sizeof(msr_list.indices[0]));
2368 
2369     kvm_feature_msrs->nmsrs = msr_list.nmsrs;
2370     ret = kvm_ioctl(s, KVM_GET_MSR_FEATURE_INDEX_LIST, kvm_feature_msrs);
2371 
2372     if (ret < 0) {
2373         error_report("Fetch KVM feature MSR list failed: %s",
2374             strerror(-ret));
2375         g_free(kvm_feature_msrs);
2376         kvm_feature_msrs = NULL;
2377         return ret;
2378     }
2379 
2380     return 0;
2381 }
2382 
2383 static int kvm_get_supported_msrs(KVMState *s)
2384 {
2385     int ret = 0;
2386     struct kvm_msr_list msr_list, *kvm_msr_list;
2387 
2388     /*
2389      *  Obtain MSR list from KVM.  These are the MSRs that we must
2390      *  save/restore.
2391      */
2392     msr_list.nmsrs = 0;
2393     ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
2394     if (ret < 0 && ret != -E2BIG) {
2395         return ret;
2396     }
2397     /*
2398      * Old kernel modules had a bug and could write beyond the provided
2399      * memory. Allocate at least a safe amount of 1K.
2400      */
2401     kvm_msr_list = g_malloc0(MAX(1024, sizeof(msr_list) +
2402                                           msr_list.nmsrs *
2403                                           sizeof(msr_list.indices[0])));
2404 
2405     kvm_msr_list->nmsrs = msr_list.nmsrs;
2406     ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
2407     if (ret >= 0) {
2408         int i;
2409 
2410         for (i = 0; i < kvm_msr_list->nmsrs; i++) {
2411             switch (kvm_msr_list->indices[i]) {
2412             case MSR_STAR:
2413                 has_msr_star = true;
2414                 break;
2415             case MSR_VM_HSAVE_PA:
2416                 has_msr_hsave_pa = true;
2417                 break;
2418             case MSR_TSC_AUX:
2419                 has_msr_tsc_aux = true;
2420                 break;
2421             case MSR_TSC_ADJUST:
2422                 has_msr_tsc_adjust = true;
2423                 break;
2424             case MSR_IA32_TSCDEADLINE:
2425                 has_msr_tsc_deadline = true;
2426                 break;
2427             case MSR_IA32_SMBASE:
2428                 has_msr_smbase = true;
2429                 break;
2430             case MSR_SMI_COUNT:
2431                 has_msr_smi_count = true;
2432                 break;
2433             case MSR_IA32_MISC_ENABLE:
2434                 has_msr_misc_enable = true;
2435                 break;
2436             case MSR_IA32_BNDCFGS:
2437                 has_msr_bndcfgs = true;
2438                 break;
2439             case MSR_IA32_XSS:
2440                 has_msr_xss = true;
2441                 break;
2442             case MSR_IA32_UMWAIT_CONTROL:
2443                 has_msr_umwait = true;
2444                 break;
2445             case HV_X64_MSR_CRASH_CTL:
2446                 has_msr_hv_crash = true;
2447                 break;
2448             case HV_X64_MSR_RESET:
2449                 has_msr_hv_reset = true;
2450                 break;
2451             case HV_X64_MSR_VP_INDEX:
2452                 has_msr_hv_vpindex = true;
2453                 break;
2454             case HV_X64_MSR_VP_RUNTIME:
2455                 has_msr_hv_runtime = true;
2456                 break;
2457             case HV_X64_MSR_SCONTROL:
2458                 has_msr_hv_synic = true;
2459                 break;
2460             case HV_X64_MSR_STIMER0_CONFIG:
2461                 has_msr_hv_stimer = true;
2462                 break;
2463             case HV_X64_MSR_TSC_FREQUENCY:
2464                 has_msr_hv_frequencies = true;
2465                 break;
2466             case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
2467                 has_msr_hv_reenlightenment = true;
2468                 break;
2469             case HV_X64_MSR_SYNDBG_OPTIONS:
2470                 has_msr_hv_syndbg_options = true;
2471                 break;
2472             case MSR_IA32_SPEC_CTRL:
2473                 has_msr_spec_ctrl = true;
2474                 break;
2475             case MSR_AMD64_TSC_RATIO:
2476                 has_tsc_scale_msr = true;
2477                 break;
2478             case MSR_IA32_TSX_CTRL:
2479                 has_msr_tsx_ctrl = true;
2480                 break;
2481             case MSR_VIRT_SSBD:
2482                 has_msr_virt_ssbd = true;
2483                 break;
2484             case MSR_IA32_ARCH_CAPABILITIES:
2485                 has_msr_arch_capabs = true;
2486                 break;
2487             case MSR_IA32_CORE_CAPABILITY:
2488                 has_msr_core_capabs = true;
2489                 break;
2490             case MSR_IA32_PERF_CAPABILITIES:
2491                 has_msr_perf_capabs = true;
2492                 break;
2493             case MSR_IA32_VMX_VMFUNC:
2494                 has_msr_vmx_vmfunc = true;
2495                 break;
2496             case MSR_IA32_UCODE_REV:
2497                 has_msr_ucode_rev = true;
2498                 break;
2499             case MSR_IA32_VMX_PROCBASED_CTLS2:
2500                 has_msr_vmx_procbased_ctls2 = true;
2501                 break;
2502             case MSR_IA32_PKRS:
2503                 has_msr_pkrs = true;
2504                 break;
2505             }
2506         }
2507     }
2508 
2509     g_free(kvm_msr_list);
2510 
2511     return ret;
2512 }
2513 
2514 static bool kvm_rdmsr_core_thread_count(X86CPU *cpu, uint32_t msr,
2515                                         uint64_t *val)
2516 {
2517     CPUState *cs = CPU(cpu);
2518 
2519     *val = cs->nr_threads * cs->nr_cores; /* thread count, bits 15..0 */
2520     *val |= ((uint32_t)cs->nr_cores << 16); /* core count, bits 31..16 */
2521 
2522     return true;
2523 }
2524 
2525 static Notifier smram_machine_done;
2526 static KVMMemoryListener smram_listener;
2527 static AddressSpace smram_address_space;
2528 static MemoryRegion smram_as_root;
2529 static MemoryRegion smram_as_mem;
2530 
2531 static void register_smram_listener(Notifier *n, void *unused)
2532 {
2533     MemoryRegion *smram =
2534         (MemoryRegion *) object_resolve_path("/machine/smram", NULL);
2535 
2536     /* Outer container... */
2537     memory_region_init(&smram_as_root, OBJECT(kvm_state), "mem-container-smram", ~0ull);
2538     memory_region_set_enabled(&smram_as_root, true);
2539 
2540     /* ... with two regions inside: normal system memory with low
2541      * priority, and...
2542      */
2543     memory_region_init_alias(&smram_as_mem, OBJECT(kvm_state), "mem-smram",
2544                              get_system_memory(), 0, ~0ull);
2545     memory_region_add_subregion_overlap(&smram_as_root, 0, &smram_as_mem, 0);
2546     memory_region_set_enabled(&smram_as_mem, true);
2547 
2548     if (smram) {
2549         /* ... SMRAM with higher priority */
2550         memory_region_add_subregion_overlap(&smram_as_root, 0, smram, 10);
2551         memory_region_set_enabled(smram, true);
2552     }
2553 
2554     address_space_init(&smram_address_space, &smram_as_root, "KVM-SMRAM");
2555     kvm_memory_listener_register(kvm_state, &smram_listener,
2556                                  &smram_address_space, 1, "kvm-smram");
2557 }
2558 
2559 int kvm_arch_get_default_type(MachineState *ms)
2560 {
2561     return 0;
2562 }
2563 
2564 int kvm_arch_init(MachineState *ms, KVMState *s)
2565 {
2566     uint64_t identity_base = 0xfffbc000;
2567     uint64_t shadow_mem;
2568     int ret;
2569     struct utsname utsname;
2570     Error *local_err = NULL;
2571 
2572     /*
2573      * Initialize SEV context, if required
2574      *
2575      * If no memory encryption is requested (ms->cgs == NULL) this is
2576      * a no-op.
2577      *
2578      * It's also a no-op if a non-SEV confidential guest support
2579      * mechanism is selected.  SEV is the only mechanism available to
2580      * select on x86 at present, so this doesn't arise, but if new
2581      * mechanisms are supported in future (e.g. TDX), they'll need
2582      * their own initialization either here or elsewhere.
2583      */
2584     ret = sev_kvm_init(ms->cgs, &local_err);
2585     if (ret < 0) {
2586         error_report_err(local_err);
2587         return ret;
2588     }
2589 
2590     if (!kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
2591         error_report("kvm: KVM_CAP_IRQ_ROUTING not supported by KVM");
2592         return -ENOTSUP;
2593     }
2594 
2595     has_xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
2596     has_xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
2597     has_pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
2598     has_sregs2 = kvm_check_extension(s, KVM_CAP_SREGS2) > 0;
2599 
2600     hv_vpindex_settable = kvm_check_extension(s, KVM_CAP_HYPERV_VP_INDEX);
2601 
2602     has_exception_payload = kvm_check_extension(s, KVM_CAP_EXCEPTION_PAYLOAD);
2603     if (has_exception_payload) {
2604         ret = kvm_vm_enable_cap(s, KVM_CAP_EXCEPTION_PAYLOAD, 0, true);
2605         if (ret < 0) {
2606             error_report("kvm: Failed to enable exception payload cap: %s",
2607                          strerror(-ret));
2608             return ret;
2609         }
2610     }
2611 
2612     has_triple_fault_event = kvm_check_extension(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT);
2613     if (has_triple_fault_event) {
2614         ret = kvm_vm_enable_cap(s, KVM_CAP_X86_TRIPLE_FAULT_EVENT, 0, true);
2615         if (ret < 0) {
2616             error_report("kvm: Failed to enable triple fault event cap: %s",
2617                          strerror(-ret));
2618             return ret;
2619         }
2620     }
2621 
2622     if (s->xen_version) {
2623 #ifdef CONFIG_XEN_EMU
2624         if (!object_dynamic_cast(OBJECT(ms), TYPE_PC_MACHINE)) {
2625             error_report("kvm: Xen support only available in PC machine");
2626             return -ENOTSUP;
2627         }
2628         /* hyperv_enabled() doesn't work yet. */
2629         uint32_t msr = XEN_HYPERCALL_MSR;
2630         ret = kvm_xen_init(s, msr);
2631         if (ret < 0) {
2632             return ret;
2633         }
2634 #else
2635         error_report("kvm: Xen support not enabled in qemu");
2636         return -ENOTSUP;
2637 #endif
2638     }
2639 
2640     ret = kvm_get_supported_msrs(s);
2641     if (ret < 0) {
2642         return ret;
2643     }
2644 
2645     kvm_get_supported_feature_msrs(s);
2646 
2647     uname(&utsname);
2648     lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
2649 
2650     /*
2651      * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
2652      * In order to use vm86 mode, an EPT identity map and a TSS  are needed.
2653      * Since these must be part of guest physical memory, we need to allocate
2654      * them, both by setting their start addresses in the kernel and by
2655      * creating a corresponding e820 entry. We need 4 pages before the BIOS.
2656      *
2657      * Older KVM versions may not support setting the identity map base. In
2658      * that case we need to stick with the default, i.e. a 256K maximum BIOS
2659      * size.
2660      */
2661     if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
2662         /* Allows up to 16M BIOSes. */
2663         identity_base = 0xfeffc000;
2664 
2665         ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
2666         if (ret < 0) {
2667             return ret;
2668         }
2669     }
2670 
2671     /* Set TSS base one page after EPT identity map. */
2672     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
2673     if (ret < 0) {
2674         return ret;
2675     }
2676 
2677     /* Tell fw_cfg to notify the BIOS to reserve the range. */
2678     ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
2679     if (ret < 0) {
2680         fprintf(stderr, "e820_add_entry() table is full\n");
2681         return ret;
2682     }
2683 
2684     shadow_mem = object_property_get_int(OBJECT(s), "kvm-shadow-mem", &error_abort);
2685     if (shadow_mem != -1) {
2686         shadow_mem /= 4096;
2687         ret = kvm_vm_ioctl(s, KVM_SET_NR_MMU_PAGES, shadow_mem);
2688         if (ret < 0) {
2689             return ret;
2690         }
2691     }
2692 
2693     if (kvm_check_extension(s, KVM_CAP_X86_SMM) &&
2694         object_dynamic_cast(OBJECT(ms), TYPE_X86_MACHINE) &&
2695         x86_machine_is_smm_enabled(X86_MACHINE(ms))) {
2696         smram_machine_done.notify = register_smram_listener;
2697         qemu_add_machine_init_done_notifier(&smram_machine_done);
2698     }
2699 
2700     if (enable_cpu_pm) {
2701         int disable_exits = kvm_check_extension(s, KVM_CAP_X86_DISABLE_EXITS);
2702 /* Work around for kernel header with a typo. TODO: fix header and drop. */
2703 #if defined(KVM_X86_DISABLE_EXITS_HTL) && !defined(KVM_X86_DISABLE_EXITS_HLT)
2704 #define KVM_X86_DISABLE_EXITS_HLT KVM_X86_DISABLE_EXITS_HTL
2705 #endif
2706         if (disable_exits) {
2707             disable_exits &= (KVM_X86_DISABLE_EXITS_MWAIT |
2708                               KVM_X86_DISABLE_EXITS_HLT |
2709                               KVM_X86_DISABLE_EXITS_PAUSE |
2710                               KVM_X86_DISABLE_EXITS_CSTATE);
2711         }
2712 
2713         ret = kvm_vm_enable_cap(s, KVM_CAP_X86_DISABLE_EXITS, 0,
2714                                 disable_exits);
2715         if (ret < 0) {
2716             error_report("kvm: guest stopping CPU not supported: %s",
2717                          strerror(-ret));
2718         }
2719     }
2720 
2721     if (object_dynamic_cast(OBJECT(ms), TYPE_X86_MACHINE)) {
2722         X86MachineState *x86ms = X86_MACHINE(ms);
2723 
2724         if (x86ms->bus_lock_ratelimit > 0) {
2725             ret = kvm_check_extension(s, KVM_CAP_X86_BUS_LOCK_EXIT);
2726             if (!(ret & KVM_BUS_LOCK_DETECTION_EXIT)) {
2727                 error_report("kvm: bus lock detection unsupported");
2728                 return -ENOTSUP;
2729             }
2730             ret = kvm_vm_enable_cap(s, KVM_CAP_X86_BUS_LOCK_EXIT, 0,
2731                                     KVM_BUS_LOCK_DETECTION_EXIT);
2732             if (ret < 0) {
2733                 error_report("kvm: Failed to enable bus lock detection cap: %s",
2734                              strerror(-ret));
2735                 return ret;
2736             }
2737             ratelimit_init(&bus_lock_ratelimit_ctrl);
2738             ratelimit_set_speed(&bus_lock_ratelimit_ctrl,
2739                                 x86ms->bus_lock_ratelimit, BUS_LOCK_SLICE_TIME);
2740         }
2741     }
2742 
2743     if (s->notify_vmexit != NOTIFY_VMEXIT_OPTION_DISABLE &&
2744         kvm_check_extension(s, KVM_CAP_X86_NOTIFY_VMEXIT)) {
2745             uint64_t notify_window_flags =
2746                 ((uint64_t)s->notify_window << 32) |
2747                 KVM_X86_NOTIFY_VMEXIT_ENABLED |
2748                 KVM_X86_NOTIFY_VMEXIT_USER;
2749             ret = kvm_vm_enable_cap(s, KVM_CAP_X86_NOTIFY_VMEXIT, 0,
2750                                     notify_window_flags);
2751             if (ret < 0) {
2752                 error_report("kvm: Failed to enable notify vmexit cap: %s",
2753                              strerror(-ret));
2754                 return ret;
2755             }
2756     }
2757     if (kvm_vm_check_extension(s, KVM_CAP_X86_USER_SPACE_MSR)) {
2758         bool r;
2759 
2760         ret = kvm_vm_enable_cap(s, KVM_CAP_X86_USER_SPACE_MSR, 0,
2761                                 KVM_MSR_EXIT_REASON_FILTER);
2762         if (ret) {
2763             error_report("Could not enable user space MSRs: %s",
2764                          strerror(-ret));
2765             exit(1);
2766         }
2767 
2768         r = kvm_filter_msr(s, MSR_CORE_THREAD_COUNT,
2769                            kvm_rdmsr_core_thread_count, NULL);
2770         if (!r) {
2771             error_report("Could not install MSR_CORE_THREAD_COUNT handler: %s",
2772                          strerror(-ret));
2773             exit(1);
2774         }
2775     }
2776 
2777     return 0;
2778 }
2779 
2780 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
2781 {
2782     lhs->selector = rhs->selector;
2783     lhs->base = rhs->base;
2784     lhs->limit = rhs->limit;
2785     lhs->type = 3;
2786     lhs->present = 1;
2787     lhs->dpl = 3;
2788     lhs->db = 0;
2789     lhs->s = 1;
2790     lhs->l = 0;
2791     lhs->g = 0;
2792     lhs->avl = 0;
2793     lhs->unusable = 0;
2794 }
2795 
2796 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
2797 {
2798     unsigned flags = rhs->flags;
2799     lhs->selector = rhs->selector;
2800     lhs->base = rhs->base;
2801     lhs->limit = rhs->limit;
2802     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
2803     lhs->present = (flags & DESC_P_MASK) != 0;
2804     lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
2805     lhs->db = (flags >> DESC_B_SHIFT) & 1;
2806     lhs->s = (flags & DESC_S_MASK) != 0;
2807     lhs->l = (flags >> DESC_L_SHIFT) & 1;
2808     lhs->g = (flags & DESC_G_MASK) != 0;
2809     lhs->avl = (flags & DESC_AVL_MASK) != 0;
2810     lhs->unusable = !lhs->present;
2811     lhs->padding = 0;
2812 }
2813 
2814 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
2815 {
2816     lhs->selector = rhs->selector;
2817     lhs->base = rhs->base;
2818     lhs->limit = rhs->limit;
2819     lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
2820                  ((rhs->present && !rhs->unusable) * DESC_P_MASK) |
2821                  (rhs->dpl << DESC_DPL_SHIFT) |
2822                  (rhs->db << DESC_B_SHIFT) |
2823                  (rhs->s * DESC_S_MASK) |
2824                  (rhs->l << DESC_L_SHIFT) |
2825                  (rhs->g * DESC_G_MASK) |
2826                  (rhs->avl * DESC_AVL_MASK);
2827 }
2828 
2829 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
2830 {
2831     if (set) {
2832         *kvm_reg = *qemu_reg;
2833     } else {
2834         *qemu_reg = *kvm_reg;
2835     }
2836 }
2837 
2838 static int kvm_getput_regs(X86CPU *cpu, int set)
2839 {
2840     CPUX86State *env = &cpu->env;
2841     struct kvm_regs regs;
2842     int ret = 0;
2843 
2844     if (!set) {
2845         ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_REGS, &regs);
2846         if (ret < 0) {
2847             return ret;
2848         }
2849     }
2850 
2851     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
2852     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
2853     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
2854     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
2855     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
2856     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
2857     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
2858     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
2859 #ifdef TARGET_X86_64
2860     kvm_getput_reg(&regs.r8, &env->regs[8], set);
2861     kvm_getput_reg(&regs.r9, &env->regs[9], set);
2862     kvm_getput_reg(&regs.r10, &env->regs[10], set);
2863     kvm_getput_reg(&regs.r11, &env->regs[11], set);
2864     kvm_getput_reg(&regs.r12, &env->regs[12], set);
2865     kvm_getput_reg(&regs.r13, &env->regs[13], set);
2866     kvm_getput_reg(&regs.r14, &env->regs[14], set);
2867     kvm_getput_reg(&regs.r15, &env->regs[15], set);
2868 #endif
2869 
2870     kvm_getput_reg(&regs.rflags, &env->eflags, set);
2871     kvm_getput_reg(&regs.rip, &env->eip, set);
2872 
2873     if (set) {
2874         ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_REGS, &regs);
2875     }
2876 
2877     return ret;
2878 }
2879 
2880 static int kvm_put_fpu(X86CPU *cpu)
2881 {
2882     CPUX86State *env = &cpu->env;
2883     struct kvm_fpu fpu;
2884     int i;
2885 
2886     memset(&fpu, 0, sizeof fpu);
2887     fpu.fsw = env->fpus & ~(7 << 11);
2888     fpu.fsw |= (env->fpstt & 7) << 11;
2889     fpu.fcw = env->fpuc;
2890     fpu.last_opcode = env->fpop;
2891     fpu.last_ip = env->fpip;
2892     fpu.last_dp = env->fpdp;
2893     for (i = 0; i < 8; ++i) {
2894         fpu.ftwx |= (!env->fptags[i]) << i;
2895     }
2896     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
2897     for (i = 0; i < CPU_NB_REGS; i++) {
2898         stq_p(&fpu.xmm[i][0], env->xmm_regs[i].ZMM_Q(0));
2899         stq_p(&fpu.xmm[i][8], env->xmm_regs[i].ZMM_Q(1));
2900     }
2901     fpu.mxcsr = env->mxcsr;
2902 
2903     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_FPU, &fpu);
2904 }
2905 
2906 static int kvm_put_xsave(X86CPU *cpu)
2907 {
2908     CPUX86State *env = &cpu->env;
2909     void *xsave = env->xsave_buf;
2910 
2911     if (!has_xsave) {
2912         return kvm_put_fpu(cpu);
2913     }
2914     x86_cpu_xsave_all_areas(cpu, xsave, env->xsave_buf_len);
2915 
2916     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
2917 }
2918 
2919 static int kvm_put_xcrs(X86CPU *cpu)
2920 {
2921     CPUX86State *env = &cpu->env;
2922     struct kvm_xcrs xcrs = {};
2923 
2924     if (!has_xcrs) {
2925         return 0;
2926     }
2927 
2928     xcrs.nr_xcrs = 1;
2929     xcrs.flags = 0;
2930     xcrs.xcrs[0].xcr = 0;
2931     xcrs.xcrs[0].value = env->xcr0;
2932     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XCRS, &xcrs);
2933 }
2934 
2935 static int kvm_put_sregs(X86CPU *cpu)
2936 {
2937     CPUX86State *env = &cpu->env;
2938     struct kvm_sregs sregs;
2939 
2940     /*
2941      * The interrupt_bitmap is ignored because KVM_SET_SREGS is
2942      * always followed by KVM_SET_VCPU_EVENTS.
2943      */
2944     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
2945 
2946     if ((env->eflags & VM_MASK)) {
2947         set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
2948         set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
2949         set_v8086_seg(&sregs.es, &env->segs[R_ES]);
2950         set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
2951         set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
2952         set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
2953     } else {
2954         set_seg(&sregs.cs, &env->segs[R_CS]);
2955         set_seg(&sregs.ds, &env->segs[R_DS]);
2956         set_seg(&sregs.es, &env->segs[R_ES]);
2957         set_seg(&sregs.fs, &env->segs[R_FS]);
2958         set_seg(&sregs.gs, &env->segs[R_GS]);
2959         set_seg(&sregs.ss, &env->segs[R_SS]);
2960     }
2961 
2962     set_seg(&sregs.tr, &env->tr);
2963     set_seg(&sregs.ldt, &env->ldt);
2964 
2965     sregs.idt.limit = env->idt.limit;
2966     sregs.idt.base = env->idt.base;
2967     memset(sregs.idt.padding, 0, sizeof sregs.idt.padding);
2968     sregs.gdt.limit = env->gdt.limit;
2969     sregs.gdt.base = env->gdt.base;
2970     memset(sregs.gdt.padding, 0, sizeof sregs.gdt.padding);
2971 
2972     sregs.cr0 = env->cr[0];
2973     sregs.cr2 = env->cr[2];
2974     sregs.cr3 = env->cr[3];
2975     sregs.cr4 = env->cr[4];
2976 
2977     sregs.cr8 = cpu_get_apic_tpr(cpu->apic_state);
2978     sregs.apic_base = cpu_get_apic_base(cpu->apic_state);
2979 
2980     sregs.efer = env->efer;
2981 
2982     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs);
2983 }
2984 
2985 static int kvm_put_sregs2(X86CPU *cpu)
2986 {
2987     CPUX86State *env = &cpu->env;
2988     struct kvm_sregs2 sregs;
2989     int i;
2990 
2991     sregs.flags = 0;
2992 
2993     if ((env->eflags & VM_MASK)) {
2994         set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
2995         set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
2996         set_v8086_seg(&sregs.es, &env->segs[R_ES]);
2997         set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
2998         set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
2999         set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
3000     } else {
3001         set_seg(&sregs.cs, &env->segs[R_CS]);
3002         set_seg(&sregs.ds, &env->segs[R_DS]);
3003         set_seg(&sregs.es, &env->segs[R_ES]);
3004         set_seg(&sregs.fs, &env->segs[R_FS]);
3005         set_seg(&sregs.gs, &env->segs[R_GS]);
3006         set_seg(&sregs.ss, &env->segs[R_SS]);
3007     }
3008 
3009     set_seg(&sregs.tr, &env->tr);
3010     set_seg(&sregs.ldt, &env->ldt);
3011 
3012     sregs.idt.limit = env->idt.limit;
3013     sregs.idt.base = env->idt.base;
3014     memset(sregs.idt.padding, 0, sizeof sregs.idt.padding);
3015     sregs.gdt.limit = env->gdt.limit;
3016     sregs.gdt.base = env->gdt.base;
3017     memset(sregs.gdt.padding, 0, sizeof sregs.gdt.padding);
3018 
3019     sregs.cr0 = env->cr[0];
3020     sregs.cr2 = env->cr[2];
3021     sregs.cr3 = env->cr[3];
3022     sregs.cr4 = env->cr[4];
3023 
3024     sregs.cr8 = cpu_get_apic_tpr(cpu->apic_state);
3025     sregs.apic_base = cpu_get_apic_base(cpu->apic_state);
3026 
3027     sregs.efer = env->efer;
3028 
3029     if (env->pdptrs_valid) {
3030         for (i = 0; i < 4; i++) {
3031             sregs.pdptrs[i] = env->pdptrs[i];
3032         }
3033         sregs.flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
3034     }
3035 
3036     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS2, &sregs);
3037 }
3038 
3039 
3040 static void kvm_msr_buf_reset(X86CPU *cpu)
3041 {
3042     memset(cpu->kvm_msr_buf, 0, MSR_BUF_SIZE);
3043 }
3044 
3045 static void kvm_msr_entry_add(X86CPU *cpu, uint32_t index, uint64_t value)
3046 {
3047     struct kvm_msrs *msrs = cpu->kvm_msr_buf;
3048     void *limit = ((void *)msrs) + MSR_BUF_SIZE;
3049     struct kvm_msr_entry *entry = &msrs->entries[msrs->nmsrs];
3050 
3051     assert((void *)(entry + 1) <= limit);
3052 
3053     entry->index = index;
3054     entry->reserved = 0;
3055     entry->data = value;
3056     msrs->nmsrs++;
3057 }
3058 
3059 static int kvm_put_one_msr(X86CPU *cpu, int index, uint64_t value)
3060 {
3061     kvm_msr_buf_reset(cpu);
3062     kvm_msr_entry_add(cpu, index, value);
3063 
3064     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
3065 }
3066 
3067 static int kvm_get_one_msr(X86CPU *cpu, int index, uint64_t *value)
3068 {
3069     int ret;
3070     struct {
3071         struct kvm_msrs info;
3072         struct kvm_msr_entry entries[1];
3073     } msr_data = {
3074         .info.nmsrs = 1,
3075         .entries[0].index = index,
3076     };
3077 
3078     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
3079     if (ret < 0) {
3080         return ret;
3081     }
3082     assert(ret == 1);
3083     *value = msr_data.entries[0].data;
3084     return ret;
3085 }
3086 void kvm_put_apicbase(X86CPU *cpu, uint64_t value)
3087 {
3088     int ret;
3089 
3090     ret = kvm_put_one_msr(cpu, MSR_IA32_APICBASE, value);
3091     assert(ret == 1);
3092 }
3093 
3094 static int kvm_put_tscdeadline_msr(X86CPU *cpu)
3095 {
3096     CPUX86State *env = &cpu->env;
3097     int ret;
3098 
3099     if (!has_msr_tsc_deadline) {
3100         return 0;
3101     }
3102 
3103     ret = kvm_put_one_msr(cpu, MSR_IA32_TSCDEADLINE, env->tsc_deadline);
3104     if (ret < 0) {
3105         return ret;
3106     }
3107 
3108     assert(ret == 1);
3109     return 0;
3110 }
3111 
3112 /*
3113  * Provide a separate write service for the feature control MSR in order to
3114  * kick the VCPU out of VMXON or even guest mode on reset. This has to be done
3115  * before writing any other state because forcibly leaving nested mode
3116  * invalidates the VCPU state.
3117  */
3118 static int kvm_put_msr_feature_control(X86CPU *cpu)
3119 {
3120     int ret;
3121 
3122     if (!has_msr_feature_control) {
3123         return 0;
3124     }
3125 
3126     ret = kvm_put_one_msr(cpu, MSR_IA32_FEATURE_CONTROL,
3127                           cpu->env.msr_ia32_feature_control);
3128     if (ret < 0) {
3129         return ret;
3130     }
3131 
3132     assert(ret == 1);
3133     return 0;
3134 }
3135 
3136 static uint64_t make_vmx_msr_value(uint32_t index, uint32_t features)
3137 {
3138     uint32_t default1, can_be_one, can_be_zero;
3139     uint32_t must_be_one;
3140 
3141     switch (index) {
3142     case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3143         default1 = 0x00000016;
3144         break;
3145     case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3146         default1 = 0x0401e172;
3147         break;
3148     case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3149         default1 = 0x000011ff;
3150         break;
3151     case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3152         default1 = 0x00036dff;
3153         break;
3154     case MSR_IA32_VMX_PROCBASED_CTLS2:
3155         default1 = 0;
3156         break;
3157     default:
3158         abort();
3159     }
3160 
3161     /* If a feature bit is set, the control can be either set or clear.
3162      * Otherwise the value is limited to either 0 or 1 by default1.
3163      */
3164     can_be_one = features | default1;
3165     can_be_zero = features | ~default1;
3166     must_be_one = ~can_be_zero;
3167 
3168     /*
3169      * Bit 0:31 -> 0 if the control bit can be zero (i.e. 1 if it must be one).
3170      * Bit 32:63 -> 1 if the control bit can be one.
3171      */
3172     return must_be_one | (((uint64_t)can_be_one) << 32);
3173 }
3174 
3175 static void kvm_msr_entry_add_vmx(X86CPU *cpu, FeatureWordArray f)
3176 {
3177     uint64_t kvm_vmx_basic =
3178         kvm_arch_get_supported_msr_feature(kvm_state,
3179                                            MSR_IA32_VMX_BASIC);
3180 
3181     if (!kvm_vmx_basic) {
3182         /* If the kernel doesn't support VMX feature (kvm_intel.nested=0),
3183          * then kvm_vmx_basic will be 0 and KVM_SET_MSR will fail.
3184          */
3185         return;
3186     }
3187 
3188     uint64_t kvm_vmx_misc =
3189         kvm_arch_get_supported_msr_feature(kvm_state,
3190                                            MSR_IA32_VMX_MISC);
3191     uint64_t kvm_vmx_ept_vpid =
3192         kvm_arch_get_supported_msr_feature(kvm_state,
3193                                            MSR_IA32_VMX_EPT_VPID_CAP);
3194 
3195     /*
3196      * If the guest is 64-bit, a value of 1 is allowed for the host address
3197      * space size vmexit control.
3198      */
3199     uint64_t fixed_vmx_exit = f[FEAT_8000_0001_EDX] & CPUID_EXT2_LM
3200         ? (uint64_t)VMX_VM_EXIT_HOST_ADDR_SPACE_SIZE << 32 : 0;
3201 
3202     /*
3203      * Bits 0-30, 32-44 and 50-53 come from the host.  KVM should
3204      * not change them for backwards compatibility.
3205      */
3206     uint64_t fixed_vmx_basic = kvm_vmx_basic &
3207         (MSR_VMX_BASIC_VMCS_REVISION_MASK |
3208          MSR_VMX_BASIC_VMXON_REGION_SIZE_MASK |
3209          MSR_VMX_BASIC_VMCS_MEM_TYPE_MASK);
3210 
3211     /*
3212      * Same for bits 0-4 and 25-27.  Bits 16-24 (CR3 target count) can
3213      * change in the future but are always zero for now, clear them to be
3214      * future proof.  Bits 32-63 in theory could change, though KVM does
3215      * not support dual-monitor treatment and probably never will; mask
3216      * them out as well.
3217      */
3218     uint64_t fixed_vmx_misc = kvm_vmx_misc &
3219         (MSR_VMX_MISC_PREEMPTION_TIMER_SHIFT_MASK |
3220          MSR_VMX_MISC_MAX_MSR_LIST_SIZE_MASK);
3221 
3222     /*
3223      * EPT memory types should not change either, so we do not bother
3224      * adding features for them.
3225      */
3226     uint64_t fixed_vmx_ept_mask =
3227             (f[FEAT_VMX_SECONDARY_CTLS] & VMX_SECONDARY_EXEC_ENABLE_EPT ?
3228              MSR_VMX_EPT_UC | MSR_VMX_EPT_WB : 0);
3229     uint64_t fixed_vmx_ept_vpid = kvm_vmx_ept_vpid & fixed_vmx_ept_mask;
3230 
3231     kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
3232                       make_vmx_msr_value(MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
3233                                          f[FEAT_VMX_PROCBASED_CTLS]));
3234     kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_PINBASED_CTLS,
3235                       make_vmx_msr_value(MSR_IA32_VMX_TRUE_PINBASED_CTLS,
3236                                          f[FEAT_VMX_PINBASED_CTLS]));
3237     kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_EXIT_CTLS,
3238                       make_vmx_msr_value(MSR_IA32_VMX_TRUE_EXIT_CTLS,
3239                                          f[FEAT_VMX_EXIT_CTLS]) | fixed_vmx_exit);
3240     kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_ENTRY_CTLS,
3241                       make_vmx_msr_value(MSR_IA32_VMX_TRUE_ENTRY_CTLS,
3242                                          f[FEAT_VMX_ENTRY_CTLS]));
3243     kvm_msr_entry_add(cpu, MSR_IA32_VMX_PROCBASED_CTLS2,
3244                       make_vmx_msr_value(MSR_IA32_VMX_PROCBASED_CTLS2,
3245                                          f[FEAT_VMX_SECONDARY_CTLS]));
3246     kvm_msr_entry_add(cpu, MSR_IA32_VMX_EPT_VPID_CAP,
3247                       f[FEAT_VMX_EPT_VPID_CAPS] | fixed_vmx_ept_vpid);
3248     kvm_msr_entry_add(cpu, MSR_IA32_VMX_BASIC,
3249                       f[FEAT_VMX_BASIC] | fixed_vmx_basic);
3250     kvm_msr_entry_add(cpu, MSR_IA32_VMX_MISC,
3251                       f[FEAT_VMX_MISC] | fixed_vmx_misc);
3252     if (has_msr_vmx_vmfunc) {
3253         kvm_msr_entry_add(cpu, MSR_IA32_VMX_VMFUNC, f[FEAT_VMX_VMFUNC]);
3254     }
3255 
3256     /*
3257      * Just to be safe, write these with constant values.  The CRn_FIXED1
3258      * MSRs are generated by KVM based on the vCPU's CPUID.
3259      */
3260     kvm_msr_entry_add(cpu, MSR_IA32_VMX_CR0_FIXED0,
3261                       CR0_PE_MASK | CR0_PG_MASK | CR0_NE_MASK);
3262     kvm_msr_entry_add(cpu, MSR_IA32_VMX_CR4_FIXED0,
3263                       CR4_VMXE_MASK);
3264 
3265     if (f[FEAT_VMX_SECONDARY_CTLS] & VMX_SECONDARY_EXEC_TSC_SCALING) {
3266         /* TSC multiplier (0x2032).  */
3267         kvm_msr_entry_add(cpu, MSR_IA32_VMX_VMCS_ENUM, 0x32);
3268     } else {
3269         /* Preemption timer (0x482E).  */
3270         kvm_msr_entry_add(cpu, MSR_IA32_VMX_VMCS_ENUM, 0x2E);
3271     }
3272 }
3273 
3274 static void kvm_msr_entry_add_perf(X86CPU *cpu, FeatureWordArray f)
3275 {
3276     uint64_t kvm_perf_cap =
3277         kvm_arch_get_supported_msr_feature(kvm_state,
3278                                            MSR_IA32_PERF_CAPABILITIES);
3279 
3280     if (kvm_perf_cap) {
3281         kvm_msr_entry_add(cpu, MSR_IA32_PERF_CAPABILITIES,
3282                         kvm_perf_cap & f[FEAT_PERF_CAPABILITIES]);
3283     }
3284 }
3285 
3286 static int kvm_buf_set_msrs(X86CPU *cpu)
3287 {
3288     int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
3289     if (ret < 0) {
3290         return ret;
3291     }
3292 
3293     if (ret < cpu->kvm_msr_buf->nmsrs) {
3294         struct kvm_msr_entry *e = &cpu->kvm_msr_buf->entries[ret];
3295         error_report("error: failed to set MSR 0x%" PRIx32 " to 0x%" PRIx64,
3296                      (uint32_t)e->index, (uint64_t)e->data);
3297     }
3298 
3299     assert(ret == cpu->kvm_msr_buf->nmsrs);
3300     return 0;
3301 }
3302 
3303 static void kvm_init_msrs(X86CPU *cpu)
3304 {
3305     CPUX86State *env = &cpu->env;
3306 
3307     kvm_msr_buf_reset(cpu);
3308     if (has_msr_arch_capabs) {
3309         kvm_msr_entry_add(cpu, MSR_IA32_ARCH_CAPABILITIES,
3310                           env->features[FEAT_ARCH_CAPABILITIES]);
3311     }
3312 
3313     if (has_msr_core_capabs) {
3314         kvm_msr_entry_add(cpu, MSR_IA32_CORE_CAPABILITY,
3315                           env->features[FEAT_CORE_CAPABILITY]);
3316     }
3317 
3318     if (has_msr_perf_capabs && cpu->enable_pmu) {
3319         kvm_msr_entry_add_perf(cpu, env->features);
3320     }
3321 
3322     if (has_msr_ucode_rev) {
3323         kvm_msr_entry_add(cpu, MSR_IA32_UCODE_REV, cpu->ucode_rev);
3324     }
3325 
3326     /*
3327      * Older kernels do not include VMX MSRs in KVM_GET_MSR_INDEX_LIST, but
3328      * all kernels with MSR features should have them.
3329      */
3330     if (kvm_feature_msrs && cpu_has_vmx(env)) {
3331         kvm_msr_entry_add_vmx(cpu, env->features);
3332     }
3333 
3334     assert(kvm_buf_set_msrs(cpu) == 0);
3335 }
3336 
3337 static int kvm_put_msrs(X86CPU *cpu, int level)
3338 {
3339     CPUX86State *env = &cpu->env;
3340     int i;
3341 
3342     kvm_msr_buf_reset(cpu);
3343 
3344     kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, env->sysenter_cs);
3345     kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
3346     kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
3347     kvm_msr_entry_add(cpu, MSR_PAT, env->pat);
3348     if (has_msr_star) {
3349         kvm_msr_entry_add(cpu, MSR_STAR, env->star);
3350     }
3351     if (has_msr_hsave_pa) {
3352         kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, env->vm_hsave);
3353     }
3354     if (has_msr_tsc_aux) {
3355         kvm_msr_entry_add(cpu, MSR_TSC_AUX, env->tsc_aux);
3356     }
3357     if (has_msr_tsc_adjust) {
3358         kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, env->tsc_adjust);
3359     }
3360     if (has_msr_misc_enable) {
3361         kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE,
3362                           env->msr_ia32_misc_enable);
3363     }
3364     if (has_msr_smbase) {
3365         kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, env->smbase);
3366     }
3367     if (has_msr_smi_count) {
3368         kvm_msr_entry_add(cpu, MSR_SMI_COUNT, env->msr_smi_count);
3369     }
3370     if (has_msr_pkrs) {
3371         kvm_msr_entry_add(cpu, MSR_IA32_PKRS, env->pkrs);
3372     }
3373     if (has_msr_bndcfgs) {
3374         kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, env->msr_bndcfgs);
3375     }
3376     if (has_msr_xss) {
3377         kvm_msr_entry_add(cpu, MSR_IA32_XSS, env->xss);
3378     }
3379     if (has_msr_umwait) {
3380         kvm_msr_entry_add(cpu, MSR_IA32_UMWAIT_CONTROL, env->umwait);
3381     }
3382     if (has_msr_spec_ctrl) {
3383         kvm_msr_entry_add(cpu, MSR_IA32_SPEC_CTRL, env->spec_ctrl);
3384     }
3385     if (has_tsc_scale_msr) {
3386         kvm_msr_entry_add(cpu, MSR_AMD64_TSC_RATIO, env->amd_tsc_scale_msr);
3387     }
3388 
3389     if (has_msr_tsx_ctrl) {
3390         kvm_msr_entry_add(cpu, MSR_IA32_TSX_CTRL, env->tsx_ctrl);
3391     }
3392     if (has_msr_virt_ssbd) {
3393         kvm_msr_entry_add(cpu, MSR_VIRT_SSBD, env->virt_ssbd);
3394     }
3395 
3396 #ifdef TARGET_X86_64
3397     if (lm_capable_kernel) {
3398         kvm_msr_entry_add(cpu, MSR_CSTAR, env->cstar);
3399         kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, env->kernelgsbase);
3400         kvm_msr_entry_add(cpu, MSR_FMASK, env->fmask);
3401         kvm_msr_entry_add(cpu, MSR_LSTAR, env->lstar);
3402     }
3403 #endif
3404 
3405     /*
3406      * The following MSRs have side effects on the guest or are too heavy
3407      * for normal writeback. Limit them to reset or full state updates.
3408      */
3409     if (level >= KVM_PUT_RESET_STATE) {
3410         kvm_msr_entry_add(cpu, MSR_IA32_TSC, env->tsc);
3411         kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, env->system_time_msr);
3412         kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
3413         if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF_INT)) {
3414             kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_INT, env->async_pf_int_msr);
3415         }
3416         if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF)) {
3417             kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, env->async_pf_en_msr);
3418         }
3419         if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_PV_EOI)) {
3420             kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, env->pv_eoi_en_msr);
3421         }
3422         if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_STEAL_TIME)) {
3423             kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, env->steal_time_msr);
3424         }
3425 
3426         if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_POLL_CONTROL)) {
3427             kvm_msr_entry_add(cpu, MSR_KVM_POLL_CONTROL, env->poll_control_msr);
3428         }
3429 
3430         if (has_architectural_pmu_version > 0) {
3431             if (has_architectural_pmu_version > 1) {
3432                 /* Stop the counter.  */
3433                 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0);
3434                 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0);
3435             }
3436 
3437             /* Set the counter values.  */
3438             for (i = 0; i < num_architectural_pmu_fixed_counters; i++) {
3439                 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i,
3440                                   env->msr_fixed_counters[i]);
3441             }
3442             for (i = 0; i < num_architectural_pmu_gp_counters; i++) {
3443                 kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i,
3444                                   env->msr_gp_counters[i]);
3445                 kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i,
3446                                   env->msr_gp_evtsel[i]);
3447             }
3448             if (has_architectural_pmu_version > 1) {
3449                 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS,
3450                                   env->msr_global_status);
3451                 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
3452                                   env->msr_global_ovf_ctrl);
3453 
3454                 /* Now start the PMU.  */
3455                 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL,
3456                                   env->msr_fixed_ctr_ctrl);
3457                 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL,
3458                                   env->msr_global_ctrl);
3459             }
3460         }
3461         /*
3462          * Hyper-V partition-wide MSRs: to avoid clearing them on cpu hot-add,
3463          * only sync them to KVM on the first cpu
3464          */
3465         if (current_cpu == first_cpu) {
3466             if (has_msr_hv_hypercall) {
3467                 kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID,
3468                                   env->msr_hv_guest_os_id);
3469                 kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL,
3470                                   env->msr_hv_hypercall);
3471             }
3472             if (hyperv_feat_enabled(cpu, HYPERV_FEAT_TIME)) {
3473                 kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC,
3474                                   env->msr_hv_tsc);
3475             }
3476             if (hyperv_feat_enabled(cpu, HYPERV_FEAT_REENLIGHTENMENT)) {
3477                 kvm_msr_entry_add(cpu, HV_X64_MSR_REENLIGHTENMENT_CONTROL,
3478                                   env->msr_hv_reenlightenment_control);
3479                 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_CONTROL,
3480                                   env->msr_hv_tsc_emulation_control);
3481                 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_STATUS,
3482                                   env->msr_hv_tsc_emulation_status);
3483             }
3484 #ifdef CONFIG_SYNDBG
3485             if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNDBG) &&
3486                 has_msr_hv_syndbg_options) {
3487                 kvm_msr_entry_add(cpu, HV_X64_MSR_SYNDBG_OPTIONS,
3488                                   hyperv_syndbg_query_options());
3489             }
3490 #endif
3491         }
3492         if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VAPIC)) {
3493             kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE,
3494                               env->msr_hv_vapic);
3495         }
3496         if (has_msr_hv_crash) {
3497             int j;
3498 
3499             for (j = 0; j < HV_CRASH_PARAMS; j++)
3500                 kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j,
3501                                   env->msr_hv_crash_params[j]);
3502 
3503             kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_NOTIFY);
3504         }
3505         if (has_msr_hv_runtime) {
3506             kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, env->msr_hv_runtime);
3507         }
3508         if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX)
3509             && hv_vpindex_settable) {
3510             kvm_msr_entry_add(cpu, HV_X64_MSR_VP_INDEX,
3511                               hyperv_vp_index(CPU(cpu)));
3512         }
3513         if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
3514             int j;
3515 
3516             kvm_msr_entry_add(cpu, HV_X64_MSR_SVERSION, HV_SYNIC_VERSION);
3517 
3518             kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL,
3519                               env->msr_hv_synic_control);
3520             kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP,
3521                               env->msr_hv_synic_evt_page);
3522             kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP,
3523                               env->msr_hv_synic_msg_page);
3524 
3525             for (j = 0; j < ARRAY_SIZE(env->msr_hv_synic_sint); j++) {
3526                 kvm_msr_entry_add(cpu, HV_X64_MSR_SINT0 + j,
3527                                   env->msr_hv_synic_sint[j]);
3528             }
3529         }
3530         if (has_msr_hv_stimer) {
3531             int j;
3532 
3533             for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_config); j++) {
3534                 kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_CONFIG + j * 2,
3535                                 env->msr_hv_stimer_config[j]);
3536             }
3537 
3538             for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_count); j++) {
3539                 kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_COUNT + j * 2,
3540                                 env->msr_hv_stimer_count[j]);
3541             }
3542         }
3543         if (env->features[FEAT_1_EDX] & CPUID_MTRR) {
3544             uint64_t phys_mask = MAKE_64BIT_MASK(0, cpu->phys_bits);
3545 
3546             kvm_msr_entry_add(cpu, MSR_MTRRdefType, env->mtrr_deftype);
3547             kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, env->mtrr_fixed[0]);
3548             kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, env->mtrr_fixed[1]);
3549             kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, env->mtrr_fixed[2]);
3550             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, env->mtrr_fixed[3]);
3551             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, env->mtrr_fixed[4]);
3552             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, env->mtrr_fixed[5]);
3553             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, env->mtrr_fixed[6]);
3554             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, env->mtrr_fixed[7]);
3555             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, env->mtrr_fixed[8]);
3556             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, env->mtrr_fixed[9]);
3557             kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, env->mtrr_fixed[10]);
3558             for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
3559                 /* The CPU GPs if we write to a bit above the physical limit of
3560                  * the host CPU (and KVM emulates that)
3561                  */
3562                 uint64_t mask = env->mtrr_var[i].mask;
3563                 mask &= phys_mask;
3564 
3565                 kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i),
3566                                   env->mtrr_var[i].base);
3567                 kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), mask);
3568             }
3569         }
3570         if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) {
3571             int addr_num = kvm_arch_get_supported_cpuid(kvm_state,
3572                                                     0x14, 1, R_EAX) & 0x7;
3573 
3574             kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CTL,
3575                             env->msr_rtit_ctrl);
3576             kvm_msr_entry_add(cpu, MSR_IA32_RTIT_STATUS,
3577                             env->msr_rtit_status);
3578             kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_BASE,
3579                             env->msr_rtit_output_base);
3580             kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_MASK,
3581                             env->msr_rtit_output_mask);
3582             kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CR3_MATCH,
3583                             env->msr_rtit_cr3_match);
3584             for (i = 0; i < addr_num; i++) {
3585                 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_ADDR0_A + i,
3586                             env->msr_rtit_addrs[i]);
3587             }
3588         }
3589 
3590         if (env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_SGX_LC) {
3591             kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH0,
3592                               env->msr_ia32_sgxlepubkeyhash[0]);
3593             kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH1,
3594                               env->msr_ia32_sgxlepubkeyhash[1]);
3595             kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH2,
3596                               env->msr_ia32_sgxlepubkeyhash[2]);
3597             kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH3,
3598                               env->msr_ia32_sgxlepubkeyhash[3]);
3599         }
3600 
3601         if (env->features[FEAT_XSAVE] & CPUID_D_1_EAX_XFD) {
3602             kvm_msr_entry_add(cpu, MSR_IA32_XFD,
3603                               env->msr_xfd);
3604             kvm_msr_entry_add(cpu, MSR_IA32_XFD_ERR,
3605                               env->msr_xfd_err);
3606         }
3607 
3608         if (kvm_enabled() && cpu->enable_pmu &&
3609             (env->features[FEAT_7_0_EDX] & CPUID_7_0_EDX_ARCH_LBR)) {
3610             uint64_t depth;
3611             int ret;
3612 
3613             /*
3614              * Only migrate Arch LBR states when the host Arch LBR depth
3615              * equals that of source guest's, this is to avoid mismatch
3616              * of guest/host config for the msr hence avoid unexpected
3617              * misbehavior.
3618              */
3619             ret = kvm_get_one_msr(cpu, MSR_ARCH_LBR_DEPTH, &depth);
3620 
3621             if (ret == 1 && !!depth && depth == env->msr_lbr_depth) {
3622                 kvm_msr_entry_add(cpu, MSR_ARCH_LBR_CTL, env->msr_lbr_ctl);
3623                 kvm_msr_entry_add(cpu, MSR_ARCH_LBR_DEPTH, env->msr_lbr_depth);
3624 
3625                 for (i = 0; i < ARCH_LBR_NR_ENTRIES; i++) {
3626                     if (!env->lbr_records[i].from) {
3627                         continue;
3628                     }
3629                     kvm_msr_entry_add(cpu, MSR_ARCH_LBR_FROM_0 + i,
3630                                       env->lbr_records[i].from);
3631                     kvm_msr_entry_add(cpu, MSR_ARCH_LBR_TO_0 + i,
3632                                       env->lbr_records[i].to);
3633                     kvm_msr_entry_add(cpu, MSR_ARCH_LBR_INFO_0 + i,
3634                                       env->lbr_records[i].info);
3635                 }
3636             }
3637         }
3638 
3639         /* Note: MSR_IA32_FEATURE_CONTROL is written separately, see
3640          *       kvm_put_msr_feature_control. */
3641     }
3642 
3643     if (env->mcg_cap) {
3644         kvm_msr_entry_add(cpu, MSR_MCG_STATUS, env->mcg_status);
3645         kvm_msr_entry_add(cpu, MSR_MCG_CTL, env->mcg_ctl);
3646         if (has_msr_mcg_ext_ctl) {
3647             kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, env->mcg_ext_ctl);
3648         }
3649         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
3650             kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, env->mce_banks[i]);
3651         }
3652     }
3653 
3654     return kvm_buf_set_msrs(cpu);
3655 }
3656 
3657 
3658 static int kvm_get_fpu(X86CPU *cpu)
3659 {
3660     CPUX86State *env = &cpu->env;
3661     struct kvm_fpu fpu;
3662     int i, ret;
3663 
3664     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_FPU, &fpu);
3665     if (ret < 0) {
3666         return ret;
3667     }
3668 
3669     env->fpstt = (fpu.fsw >> 11) & 7;
3670     env->fpus = fpu.fsw;
3671     env->fpuc = fpu.fcw;
3672     env->fpop = fpu.last_opcode;
3673     env->fpip = fpu.last_ip;
3674     env->fpdp = fpu.last_dp;
3675     for (i = 0; i < 8; ++i) {
3676         env->fptags[i] = !((fpu.ftwx >> i) & 1);
3677     }
3678     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
3679     for (i = 0; i < CPU_NB_REGS; i++) {
3680         env->xmm_regs[i].ZMM_Q(0) = ldq_p(&fpu.xmm[i][0]);
3681         env->xmm_regs[i].ZMM_Q(1) = ldq_p(&fpu.xmm[i][8]);
3682     }
3683     env->mxcsr = fpu.mxcsr;
3684 
3685     return 0;
3686 }
3687 
3688 static int kvm_get_xsave(X86CPU *cpu)
3689 {
3690     CPUX86State *env = &cpu->env;
3691     void *xsave = env->xsave_buf;
3692     int type, ret;
3693 
3694     if (!has_xsave) {
3695         return kvm_get_fpu(cpu);
3696     }
3697 
3698     type = has_xsave2 ? KVM_GET_XSAVE2 : KVM_GET_XSAVE;
3699     ret = kvm_vcpu_ioctl(CPU(cpu), type, xsave);
3700     if (ret < 0) {
3701         return ret;
3702     }
3703     x86_cpu_xrstor_all_areas(cpu, xsave, env->xsave_buf_len);
3704 
3705     return 0;
3706 }
3707 
3708 static int kvm_get_xcrs(X86CPU *cpu)
3709 {
3710     CPUX86State *env = &cpu->env;
3711     int i, ret;
3712     struct kvm_xcrs xcrs;
3713 
3714     if (!has_xcrs) {
3715         return 0;
3716     }
3717 
3718     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XCRS, &xcrs);
3719     if (ret < 0) {
3720         return ret;
3721     }
3722 
3723     for (i = 0; i < xcrs.nr_xcrs; i++) {
3724         /* Only support xcr0 now */
3725         if (xcrs.xcrs[i].xcr == 0) {
3726             env->xcr0 = xcrs.xcrs[i].value;
3727             break;
3728         }
3729     }
3730     return 0;
3731 }
3732 
3733 static int kvm_get_sregs(X86CPU *cpu)
3734 {
3735     CPUX86State *env = &cpu->env;
3736     struct kvm_sregs sregs;
3737     int ret;
3738 
3739     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
3740     if (ret < 0) {
3741         return ret;
3742     }
3743 
3744     /*
3745      * The interrupt_bitmap is ignored because KVM_GET_SREGS is
3746      * always preceded by KVM_GET_VCPU_EVENTS.
3747      */
3748 
3749     get_seg(&env->segs[R_CS], &sregs.cs);
3750     get_seg(&env->segs[R_DS], &sregs.ds);
3751     get_seg(&env->segs[R_ES], &sregs.es);
3752     get_seg(&env->segs[R_FS], &sregs.fs);
3753     get_seg(&env->segs[R_GS], &sregs.gs);
3754     get_seg(&env->segs[R_SS], &sregs.ss);
3755 
3756     get_seg(&env->tr, &sregs.tr);
3757     get_seg(&env->ldt, &sregs.ldt);
3758 
3759     env->idt.limit = sregs.idt.limit;
3760     env->idt.base = sregs.idt.base;
3761     env->gdt.limit = sregs.gdt.limit;
3762     env->gdt.base = sregs.gdt.base;
3763 
3764     env->cr[0] = sregs.cr0;
3765     env->cr[2] = sregs.cr2;
3766     env->cr[3] = sregs.cr3;
3767     env->cr[4] = sregs.cr4;
3768 
3769     env->efer = sregs.efer;
3770 
3771     /* changes to apic base and cr8/tpr are read back via kvm_arch_post_run */
3772     x86_update_hflags(env);
3773 
3774     return 0;
3775 }
3776 
3777 static int kvm_get_sregs2(X86CPU *cpu)
3778 {
3779     CPUX86State *env = &cpu->env;
3780     struct kvm_sregs2 sregs;
3781     int i, ret;
3782 
3783     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS2, &sregs);
3784     if (ret < 0) {
3785         return ret;
3786     }
3787 
3788     get_seg(&env->segs[R_CS], &sregs.cs);
3789     get_seg(&env->segs[R_DS], &sregs.ds);
3790     get_seg(&env->segs[R_ES], &sregs.es);
3791     get_seg(&env->segs[R_FS], &sregs.fs);
3792     get_seg(&env->segs[R_GS], &sregs.gs);
3793     get_seg(&env->segs[R_SS], &sregs.ss);
3794 
3795     get_seg(&env->tr, &sregs.tr);
3796     get_seg(&env->ldt, &sregs.ldt);
3797 
3798     env->idt.limit = sregs.idt.limit;
3799     env->idt.base = sregs.idt.base;
3800     env->gdt.limit = sregs.gdt.limit;
3801     env->gdt.base = sregs.gdt.base;
3802 
3803     env->cr[0] = sregs.cr0;
3804     env->cr[2] = sregs.cr2;
3805     env->cr[3] = sregs.cr3;
3806     env->cr[4] = sregs.cr4;
3807 
3808     env->efer = sregs.efer;
3809 
3810     env->pdptrs_valid = sregs.flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;
3811 
3812     if (env->pdptrs_valid) {
3813         for (i = 0; i < 4; i++) {
3814             env->pdptrs[i] = sregs.pdptrs[i];
3815         }
3816     }
3817 
3818     /* changes to apic base and cr8/tpr are read back via kvm_arch_post_run */
3819     x86_update_hflags(env);
3820 
3821     return 0;
3822 }
3823 
3824 static int kvm_get_msrs(X86CPU *cpu)
3825 {
3826     CPUX86State *env = &cpu->env;
3827     struct kvm_msr_entry *msrs = cpu->kvm_msr_buf->entries;
3828     int ret, i;
3829     uint64_t mtrr_top_bits;
3830 
3831     kvm_msr_buf_reset(cpu);
3832 
3833     kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, 0);
3834     kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, 0);
3835     kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, 0);
3836     kvm_msr_entry_add(cpu, MSR_PAT, 0);
3837     if (has_msr_star) {
3838         kvm_msr_entry_add(cpu, MSR_STAR, 0);
3839     }
3840     if (has_msr_hsave_pa) {
3841         kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, 0);
3842     }
3843     if (has_msr_tsc_aux) {
3844         kvm_msr_entry_add(cpu, MSR_TSC_AUX, 0);
3845     }
3846     if (has_msr_tsc_adjust) {
3847         kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, 0);
3848     }
3849     if (has_msr_tsc_deadline) {
3850         kvm_msr_entry_add(cpu, MSR_IA32_TSCDEADLINE, 0);
3851     }
3852     if (has_msr_misc_enable) {
3853         kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE, 0);
3854     }
3855     if (has_msr_smbase) {
3856         kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, 0);
3857     }
3858     if (has_msr_smi_count) {
3859         kvm_msr_entry_add(cpu, MSR_SMI_COUNT, 0);
3860     }
3861     if (has_msr_feature_control) {
3862         kvm_msr_entry_add(cpu, MSR_IA32_FEATURE_CONTROL, 0);
3863     }
3864     if (has_msr_pkrs) {
3865         kvm_msr_entry_add(cpu, MSR_IA32_PKRS, 0);
3866     }
3867     if (has_msr_bndcfgs) {
3868         kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, 0);
3869     }
3870     if (has_msr_xss) {
3871         kvm_msr_entry_add(cpu, MSR_IA32_XSS, 0);
3872     }
3873     if (has_msr_umwait) {
3874         kvm_msr_entry_add(cpu, MSR_IA32_UMWAIT_CONTROL, 0);
3875     }
3876     if (has_msr_spec_ctrl) {
3877         kvm_msr_entry_add(cpu, MSR_IA32_SPEC_CTRL, 0);
3878     }
3879     if (has_tsc_scale_msr) {
3880         kvm_msr_entry_add(cpu, MSR_AMD64_TSC_RATIO, 0);
3881     }
3882 
3883     if (has_msr_tsx_ctrl) {
3884         kvm_msr_entry_add(cpu, MSR_IA32_TSX_CTRL, 0);
3885     }
3886     if (has_msr_virt_ssbd) {
3887         kvm_msr_entry_add(cpu, MSR_VIRT_SSBD, 0);
3888     }
3889     if (!env->tsc_valid) {
3890         kvm_msr_entry_add(cpu, MSR_IA32_TSC, 0);
3891         env->tsc_valid = !runstate_is_running();
3892     }
3893 
3894 #ifdef TARGET_X86_64
3895     if (lm_capable_kernel) {
3896         kvm_msr_entry_add(cpu, MSR_CSTAR, 0);
3897         kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, 0);
3898         kvm_msr_entry_add(cpu, MSR_FMASK, 0);
3899         kvm_msr_entry_add(cpu, MSR_LSTAR, 0);
3900     }
3901 #endif
3902     kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, 0);
3903     kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, 0);
3904     if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF_INT)) {
3905         kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_INT, 0);
3906     }
3907     if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF)) {
3908         kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, 0);
3909     }
3910     if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_PV_EOI)) {
3911         kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, 0);
3912     }
3913     if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_STEAL_TIME)) {
3914         kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, 0);
3915     }
3916     if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_POLL_CONTROL)) {
3917         kvm_msr_entry_add(cpu, MSR_KVM_POLL_CONTROL, 1);
3918     }
3919     if (has_architectural_pmu_version > 0) {
3920         if (has_architectural_pmu_version > 1) {
3921             kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0);
3922             kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0);
3923             kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS, 0);
3924             kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL, 0);
3925         }
3926         for (i = 0; i < num_architectural_pmu_fixed_counters; i++) {
3927             kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i, 0);
3928         }
3929         for (i = 0; i < num_architectural_pmu_gp_counters; i++) {
3930             kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i, 0);
3931             kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i, 0);
3932         }
3933     }
3934 
3935     if (env->mcg_cap) {
3936         kvm_msr_entry_add(cpu, MSR_MCG_STATUS, 0);
3937         kvm_msr_entry_add(cpu, MSR_MCG_CTL, 0);
3938         if (has_msr_mcg_ext_ctl) {
3939             kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, 0);
3940         }
3941         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
3942             kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, 0);
3943         }
3944     }
3945 
3946     if (has_msr_hv_hypercall) {
3947         kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL, 0);
3948         kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID, 0);
3949     }
3950     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VAPIC)) {
3951         kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE, 0);
3952     }
3953     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_TIME)) {
3954         kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC, 0);
3955     }
3956     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_REENLIGHTENMENT)) {
3957         kvm_msr_entry_add(cpu, HV_X64_MSR_REENLIGHTENMENT_CONTROL, 0);
3958         kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_CONTROL, 0);
3959         kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_STATUS, 0);
3960     }
3961     if (has_msr_hv_syndbg_options) {
3962         kvm_msr_entry_add(cpu, HV_X64_MSR_SYNDBG_OPTIONS, 0);
3963     }
3964     if (has_msr_hv_crash) {
3965         int j;
3966 
3967         for (j = 0; j < HV_CRASH_PARAMS; j++) {
3968             kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j, 0);
3969         }
3970     }
3971     if (has_msr_hv_runtime) {
3972         kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, 0);
3973     }
3974     if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
3975         uint32_t msr;
3976 
3977         kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL, 0);
3978         kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP, 0);
3979         kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP, 0);
3980         for (msr = HV_X64_MSR_SINT0; msr <= HV_X64_MSR_SINT15; msr++) {
3981             kvm_msr_entry_add(cpu, msr, 0);
3982         }
3983     }
3984     if (has_msr_hv_stimer) {
3985         uint32_t msr;
3986 
3987         for (msr = HV_X64_MSR_STIMER0_CONFIG; msr <= HV_X64_MSR_STIMER3_COUNT;
3988              msr++) {
3989             kvm_msr_entry_add(cpu, msr, 0);
3990         }
3991     }
3992     if (env->features[FEAT_1_EDX] & CPUID_MTRR) {
3993         kvm_msr_entry_add(cpu, MSR_MTRRdefType, 0);
3994         kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, 0);
3995         kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, 0);
3996         kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, 0);
3997         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, 0);
3998         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, 0);
3999         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, 0);
4000         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, 0);
4001         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, 0);
4002         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, 0);
4003         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, 0);
4004         kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, 0);
4005         for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
4006             kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i), 0);
4007             kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), 0);
4008         }
4009     }
4010 
4011     if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) {
4012         int addr_num =
4013             kvm_arch_get_supported_cpuid(kvm_state, 0x14, 1, R_EAX) & 0x7;
4014 
4015         kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CTL, 0);
4016         kvm_msr_entry_add(cpu, MSR_IA32_RTIT_STATUS, 0);
4017         kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_BASE, 0);
4018         kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_MASK, 0);
4019         kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CR3_MATCH, 0);
4020         for (i = 0; i < addr_num; i++) {
4021             kvm_msr_entry_add(cpu, MSR_IA32_RTIT_ADDR0_A + i, 0);
4022         }
4023     }
4024 
4025     if (env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_SGX_LC) {
4026         kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH0, 0);
4027         kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH1, 0);
4028         kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH2, 0);
4029         kvm_msr_entry_add(cpu, MSR_IA32_SGXLEPUBKEYHASH3, 0);
4030     }
4031 
4032     if (env->features[FEAT_XSAVE] & CPUID_D_1_EAX_XFD) {
4033         kvm_msr_entry_add(cpu, MSR_IA32_XFD, 0);
4034         kvm_msr_entry_add(cpu, MSR_IA32_XFD_ERR, 0);
4035     }
4036 
4037     if (kvm_enabled() && cpu->enable_pmu &&
4038         (env->features[FEAT_7_0_EDX] & CPUID_7_0_EDX_ARCH_LBR)) {
4039         uint64_t depth;
4040 
4041         ret = kvm_get_one_msr(cpu, MSR_ARCH_LBR_DEPTH, &depth);
4042         if (ret == 1 && depth == ARCH_LBR_NR_ENTRIES) {
4043             kvm_msr_entry_add(cpu, MSR_ARCH_LBR_CTL, 0);
4044             kvm_msr_entry_add(cpu, MSR_ARCH_LBR_DEPTH, 0);
4045 
4046             for (i = 0; i < ARCH_LBR_NR_ENTRIES; i++) {
4047                 kvm_msr_entry_add(cpu, MSR_ARCH_LBR_FROM_0 + i, 0);
4048                 kvm_msr_entry_add(cpu, MSR_ARCH_LBR_TO_0 + i, 0);
4049                 kvm_msr_entry_add(cpu, MSR_ARCH_LBR_INFO_0 + i, 0);
4050             }
4051         }
4052     }
4053 
4054     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, cpu->kvm_msr_buf);
4055     if (ret < 0) {
4056         return ret;
4057     }
4058 
4059     if (ret < cpu->kvm_msr_buf->nmsrs) {
4060         struct kvm_msr_entry *e = &cpu->kvm_msr_buf->entries[ret];
4061         error_report("error: failed to get MSR 0x%" PRIx32,
4062                      (uint32_t)e->index);
4063     }
4064 
4065     assert(ret == cpu->kvm_msr_buf->nmsrs);
4066     /*
4067      * MTRR masks: Each mask consists of 5 parts
4068      * a  10..0: must be zero
4069      * b  11   : valid bit
4070      * c n-1.12: actual mask bits
4071      * d  51..n: reserved must be zero
4072      * e  63.52: reserved must be zero
4073      *
4074      * 'n' is the number of physical bits supported by the CPU and is
4075      * apparently always <= 52.   We know our 'n' but don't know what
4076      * the destinations 'n' is; it might be smaller, in which case
4077      * it masks (c) on loading. It might be larger, in which case
4078      * we fill 'd' so that d..c is consistent irrespetive of the 'n'
4079      * we're migrating to.
4080      */
4081 
4082     if (cpu->fill_mtrr_mask) {
4083         QEMU_BUILD_BUG_ON(TARGET_PHYS_ADDR_SPACE_BITS > 52);
4084         assert(cpu->phys_bits <= TARGET_PHYS_ADDR_SPACE_BITS);
4085         mtrr_top_bits = MAKE_64BIT_MASK(cpu->phys_bits, 52 - cpu->phys_bits);
4086     } else {
4087         mtrr_top_bits = 0;
4088     }
4089 
4090     for (i = 0; i < ret; i++) {
4091         uint32_t index = msrs[i].index;
4092         switch (index) {
4093         case MSR_IA32_SYSENTER_CS:
4094             env->sysenter_cs = msrs[i].data;
4095             break;
4096         case MSR_IA32_SYSENTER_ESP:
4097             env->sysenter_esp = msrs[i].data;
4098             break;
4099         case MSR_IA32_SYSENTER_EIP:
4100             env->sysenter_eip = msrs[i].data;
4101             break;
4102         case MSR_PAT:
4103             env->pat = msrs[i].data;
4104             break;
4105         case MSR_STAR:
4106             env->star = msrs[i].data;
4107             break;
4108 #ifdef TARGET_X86_64
4109         case MSR_CSTAR:
4110             env->cstar = msrs[i].data;
4111             break;
4112         case MSR_KERNELGSBASE:
4113             env->kernelgsbase = msrs[i].data;
4114             break;
4115         case MSR_FMASK:
4116             env->fmask = msrs[i].data;
4117             break;
4118         case MSR_LSTAR:
4119             env->lstar = msrs[i].data;
4120             break;
4121 #endif
4122         case MSR_IA32_TSC:
4123             env->tsc = msrs[i].data;
4124             break;
4125         case MSR_TSC_AUX:
4126             env->tsc_aux = msrs[i].data;
4127             break;
4128         case MSR_TSC_ADJUST:
4129             env->tsc_adjust = msrs[i].data;
4130             break;
4131         case MSR_IA32_TSCDEADLINE:
4132             env->tsc_deadline = msrs[i].data;
4133             break;
4134         case MSR_VM_HSAVE_PA:
4135             env->vm_hsave = msrs[i].data;
4136             break;
4137         case MSR_KVM_SYSTEM_TIME:
4138             env->system_time_msr = msrs[i].data;
4139             break;
4140         case MSR_KVM_WALL_CLOCK:
4141             env->wall_clock_msr = msrs[i].data;
4142             break;
4143         case MSR_MCG_STATUS:
4144             env->mcg_status = msrs[i].data;
4145             break;
4146         case MSR_MCG_CTL:
4147             env->mcg_ctl = msrs[i].data;
4148             break;
4149         case MSR_MCG_EXT_CTL:
4150             env->mcg_ext_ctl = msrs[i].data;
4151             break;
4152         case MSR_IA32_MISC_ENABLE:
4153             env->msr_ia32_misc_enable = msrs[i].data;
4154             break;
4155         case MSR_IA32_SMBASE:
4156             env->smbase = msrs[i].data;
4157             break;
4158         case MSR_SMI_COUNT:
4159             env->msr_smi_count = msrs[i].data;
4160             break;
4161         case MSR_IA32_FEATURE_CONTROL:
4162             env->msr_ia32_feature_control = msrs[i].data;
4163             break;
4164         case MSR_IA32_BNDCFGS:
4165             env->msr_bndcfgs = msrs[i].data;
4166             break;
4167         case MSR_IA32_XSS:
4168             env->xss = msrs[i].data;
4169             break;
4170         case MSR_IA32_UMWAIT_CONTROL:
4171             env->umwait = msrs[i].data;
4172             break;
4173         case MSR_IA32_PKRS:
4174             env->pkrs = msrs[i].data;
4175             break;
4176         default:
4177             if (msrs[i].index >= MSR_MC0_CTL &&
4178                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
4179                 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
4180             }
4181             break;
4182         case MSR_KVM_ASYNC_PF_EN:
4183             env->async_pf_en_msr = msrs[i].data;
4184             break;
4185         case MSR_KVM_ASYNC_PF_INT:
4186             env->async_pf_int_msr = msrs[i].data;
4187             break;
4188         case MSR_KVM_PV_EOI_EN:
4189             env->pv_eoi_en_msr = msrs[i].data;
4190             break;
4191         case MSR_KVM_STEAL_TIME:
4192             env->steal_time_msr = msrs[i].data;
4193             break;
4194         case MSR_KVM_POLL_CONTROL: {
4195             env->poll_control_msr = msrs[i].data;
4196             break;
4197         }
4198         case MSR_CORE_PERF_FIXED_CTR_CTRL:
4199             env->msr_fixed_ctr_ctrl = msrs[i].data;
4200             break;
4201         case MSR_CORE_PERF_GLOBAL_CTRL:
4202             env->msr_global_ctrl = msrs[i].data;
4203             break;
4204         case MSR_CORE_PERF_GLOBAL_STATUS:
4205             env->msr_global_status = msrs[i].data;
4206             break;
4207         case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
4208             env->msr_global_ovf_ctrl = msrs[i].data;
4209             break;
4210         case MSR_CORE_PERF_FIXED_CTR0 ... MSR_CORE_PERF_FIXED_CTR0 + MAX_FIXED_COUNTERS - 1:
4211             env->msr_fixed_counters[index - MSR_CORE_PERF_FIXED_CTR0] = msrs[i].data;
4212             break;
4213         case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR0 + MAX_GP_COUNTERS - 1:
4214             env->msr_gp_counters[index - MSR_P6_PERFCTR0] = msrs[i].data;
4215             break;
4216         case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL0 + MAX_GP_COUNTERS - 1:
4217             env->msr_gp_evtsel[index - MSR_P6_EVNTSEL0] = msrs[i].data;
4218             break;
4219         case HV_X64_MSR_HYPERCALL:
4220             env->msr_hv_hypercall = msrs[i].data;
4221             break;
4222         case HV_X64_MSR_GUEST_OS_ID:
4223             env->msr_hv_guest_os_id = msrs[i].data;
4224             break;
4225         case HV_X64_MSR_APIC_ASSIST_PAGE:
4226             env->msr_hv_vapic = msrs[i].data;
4227             break;
4228         case HV_X64_MSR_REFERENCE_TSC:
4229             env->msr_hv_tsc = msrs[i].data;
4230             break;
4231         case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
4232             env->msr_hv_crash_params[index - HV_X64_MSR_CRASH_P0] = msrs[i].data;
4233             break;
4234         case HV_X64_MSR_VP_RUNTIME:
4235             env->msr_hv_runtime = msrs[i].data;
4236             break;
4237         case HV_X64_MSR_SCONTROL:
4238             env->msr_hv_synic_control = msrs[i].data;
4239             break;
4240         case HV_X64_MSR_SIEFP:
4241             env->msr_hv_synic_evt_page = msrs[i].data;
4242             break;
4243         case HV_X64_MSR_SIMP:
4244             env->msr_hv_synic_msg_page = msrs[i].data;
4245             break;
4246         case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
4247             env->msr_hv_synic_sint[index - HV_X64_MSR_SINT0] = msrs[i].data;
4248             break;
4249         case HV_X64_MSR_STIMER0_CONFIG:
4250         case HV_X64_MSR_STIMER1_CONFIG:
4251         case HV_X64_MSR_STIMER2_CONFIG:
4252         case HV_X64_MSR_STIMER3_CONFIG:
4253             env->msr_hv_stimer_config[(index - HV_X64_MSR_STIMER0_CONFIG)/2] =
4254                                 msrs[i].data;
4255             break;
4256         case HV_X64_MSR_STIMER0_COUNT:
4257         case HV_X64_MSR_STIMER1_COUNT:
4258         case HV_X64_MSR_STIMER2_COUNT:
4259         case HV_X64_MSR_STIMER3_COUNT:
4260             env->msr_hv_stimer_count[(index - HV_X64_MSR_STIMER0_COUNT)/2] =
4261                                 msrs[i].data;
4262             break;
4263         case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
4264             env->msr_hv_reenlightenment_control = msrs[i].data;
4265             break;
4266         case HV_X64_MSR_TSC_EMULATION_CONTROL:
4267             env->msr_hv_tsc_emulation_control = msrs[i].data;
4268             break;
4269         case HV_X64_MSR_TSC_EMULATION_STATUS:
4270             env->msr_hv_tsc_emulation_status = msrs[i].data;
4271             break;
4272         case HV_X64_MSR_SYNDBG_OPTIONS:
4273             env->msr_hv_syndbg_options = msrs[i].data;
4274             break;
4275         case MSR_MTRRdefType:
4276             env->mtrr_deftype = msrs[i].data;
4277             break;
4278         case MSR_MTRRfix64K_00000:
4279             env->mtrr_fixed[0] = msrs[i].data;
4280             break;
4281         case MSR_MTRRfix16K_80000:
4282             env->mtrr_fixed[1] = msrs[i].data;
4283             break;
4284         case MSR_MTRRfix16K_A0000:
4285             env->mtrr_fixed[2] = msrs[i].data;
4286             break;
4287         case MSR_MTRRfix4K_C0000:
4288             env->mtrr_fixed[3] = msrs[i].data;
4289             break;
4290         case MSR_MTRRfix4K_C8000:
4291             env->mtrr_fixed[4] = msrs[i].data;
4292             break;
4293         case MSR_MTRRfix4K_D0000:
4294             env->mtrr_fixed[5] = msrs[i].data;
4295             break;
4296         case MSR_MTRRfix4K_D8000:
4297             env->mtrr_fixed[6] = msrs[i].data;
4298             break;
4299         case MSR_MTRRfix4K_E0000:
4300             env->mtrr_fixed[7] = msrs[i].data;
4301             break;
4302         case MSR_MTRRfix4K_E8000:
4303             env->mtrr_fixed[8] = msrs[i].data;
4304             break;
4305         case MSR_MTRRfix4K_F0000:
4306             env->mtrr_fixed[9] = msrs[i].data;
4307             break;
4308         case MSR_MTRRfix4K_F8000:
4309             env->mtrr_fixed[10] = msrs[i].data;
4310             break;
4311         case MSR_MTRRphysBase(0) ... MSR_MTRRphysMask(MSR_MTRRcap_VCNT - 1):
4312             if (index & 1) {
4313                 env->mtrr_var[MSR_MTRRphysIndex(index)].mask = msrs[i].data |
4314                                                                mtrr_top_bits;
4315             } else {
4316                 env->mtrr_var[MSR_MTRRphysIndex(index)].base = msrs[i].data;
4317             }
4318             break;
4319         case MSR_IA32_SPEC_CTRL:
4320             env->spec_ctrl = msrs[i].data;
4321             break;
4322         case MSR_AMD64_TSC_RATIO:
4323             env->amd_tsc_scale_msr = msrs[i].data;
4324             break;
4325         case MSR_IA32_TSX_CTRL:
4326             env->tsx_ctrl = msrs[i].data;
4327             break;
4328         case MSR_VIRT_SSBD:
4329             env->virt_ssbd = msrs[i].data;
4330             break;
4331         case MSR_IA32_RTIT_CTL:
4332             env->msr_rtit_ctrl = msrs[i].data;
4333             break;
4334         case MSR_IA32_RTIT_STATUS:
4335             env->msr_rtit_status = msrs[i].data;
4336             break;
4337         case MSR_IA32_RTIT_OUTPUT_BASE:
4338             env->msr_rtit_output_base = msrs[i].data;
4339             break;
4340         case MSR_IA32_RTIT_OUTPUT_MASK:
4341             env->msr_rtit_output_mask = msrs[i].data;
4342             break;
4343         case MSR_IA32_RTIT_CR3_MATCH:
4344             env->msr_rtit_cr3_match = msrs[i].data;
4345             break;
4346         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
4347             env->msr_rtit_addrs[index - MSR_IA32_RTIT_ADDR0_A] = msrs[i].data;
4348             break;
4349         case MSR_IA32_SGXLEPUBKEYHASH0 ... MSR_IA32_SGXLEPUBKEYHASH3:
4350             env->msr_ia32_sgxlepubkeyhash[index - MSR_IA32_SGXLEPUBKEYHASH0] =
4351                            msrs[i].data;
4352             break;
4353         case MSR_IA32_XFD:
4354             env->msr_xfd = msrs[i].data;
4355             break;
4356         case MSR_IA32_XFD_ERR:
4357             env->msr_xfd_err = msrs[i].data;
4358             break;
4359         case MSR_ARCH_LBR_CTL:
4360             env->msr_lbr_ctl = msrs[i].data;
4361             break;
4362         case MSR_ARCH_LBR_DEPTH:
4363             env->msr_lbr_depth = msrs[i].data;
4364             break;
4365         case MSR_ARCH_LBR_FROM_0 ... MSR_ARCH_LBR_FROM_0 + 31:
4366             env->lbr_records[index - MSR_ARCH_LBR_FROM_0].from = msrs[i].data;
4367             break;
4368         case MSR_ARCH_LBR_TO_0 ... MSR_ARCH_LBR_TO_0 + 31:
4369             env->lbr_records[index - MSR_ARCH_LBR_TO_0].to = msrs[i].data;
4370             break;
4371         case MSR_ARCH_LBR_INFO_0 ... MSR_ARCH_LBR_INFO_0 + 31:
4372             env->lbr_records[index - MSR_ARCH_LBR_INFO_0].info = msrs[i].data;
4373             break;
4374         }
4375     }
4376 
4377     return 0;
4378 }
4379 
4380 static int kvm_put_mp_state(X86CPU *cpu)
4381 {
4382     struct kvm_mp_state mp_state = { .mp_state = cpu->env.mp_state };
4383 
4384     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
4385 }
4386 
4387 static int kvm_get_mp_state(X86CPU *cpu)
4388 {
4389     CPUState *cs = CPU(cpu);
4390     CPUX86State *env = &cpu->env;
4391     struct kvm_mp_state mp_state;
4392     int ret;
4393 
4394     ret = kvm_vcpu_ioctl(cs, KVM_GET_MP_STATE, &mp_state);
4395     if (ret < 0) {
4396         return ret;
4397     }
4398     env->mp_state = mp_state.mp_state;
4399     if (kvm_irqchip_in_kernel()) {
4400         cs->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
4401     }
4402     return 0;
4403 }
4404 
4405 static int kvm_get_apic(X86CPU *cpu)
4406 {
4407     DeviceState *apic = cpu->apic_state;
4408     struct kvm_lapic_state kapic;
4409     int ret;
4410 
4411     if (apic && kvm_irqchip_in_kernel()) {
4412         ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_LAPIC, &kapic);
4413         if (ret < 0) {
4414             return ret;
4415         }
4416 
4417         kvm_get_apic_state(apic, &kapic);
4418     }
4419     return 0;
4420 }
4421 
4422 static int kvm_put_vcpu_events(X86CPU *cpu, int level)
4423 {
4424     CPUState *cs = CPU(cpu);
4425     CPUX86State *env = &cpu->env;
4426     struct kvm_vcpu_events events = {};
4427 
4428     if (!kvm_has_vcpu_events()) {
4429         return 0;
4430     }
4431 
4432     events.flags = 0;
4433 
4434     if (has_exception_payload) {
4435         events.flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
4436         events.exception.pending = env->exception_pending;
4437         events.exception_has_payload = env->exception_has_payload;
4438         events.exception_payload = env->exception_payload;
4439     }
4440     events.exception.nr = env->exception_nr;
4441     events.exception.injected = env->exception_injected;
4442     events.exception.has_error_code = env->has_error_code;
4443     events.exception.error_code = env->error_code;
4444 
4445     events.interrupt.injected = (env->interrupt_injected >= 0);
4446     events.interrupt.nr = env->interrupt_injected;
4447     events.interrupt.soft = env->soft_interrupt;
4448 
4449     events.nmi.injected = env->nmi_injected;
4450     events.nmi.pending = env->nmi_pending;
4451     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
4452 
4453     events.sipi_vector = env->sipi_vector;
4454 
4455     if (has_msr_smbase) {
4456         events.smi.smm = !!(env->hflags & HF_SMM_MASK);
4457         events.smi.smm_inside_nmi = !!(env->hflags2 & HF2_SMM_INSIDE_NMI_MASK);
4458         if (kvm_irqchip_in_kernel()) {
4459             /* As soon as these are moved to the kernel, remove them
4460              * from cs->interrupt_request.
4461              */
4462             events.smi.pending = cs->interrupt_request & CPU_INTERRUPT_SMI;
4463             events.smi.latched_init = cs->interrupt_request & CPU_INTERRUPT_INIT;
4464             cs->interrupt_request &= ~(CPU_INTERRUPT_INIT | CPU_INTERRUPT_SMI);
4465         } else {
4466             /* Keep these in cs->interrupt_request.  */
4467             events.smi.pending = 0;
4468             events.smi.latched_init = 0;
4469         }
4470         /* Stop SMI delivery on old machine types to avoid a reboot
4471          * on an inward migration of an old VM.
4472          */
4473         if (!cpu->kvm_no_smi_migration) {
4474             events.flags |= KVM_VCPUEVENT_VALID_SMM;
4475         }
4476     }
4477 
4478     if (level >= KVM_PUT_RESET_STATE) {
4479         events.flags |= KVM_VCPUEVENT_VALID_NMI_PENDING;
4480         if (env->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
4481             events.flags |= KVM_VCPUEVENT_VALID_SIPI_VECTOR;
4482         }
4483     }
4484 
4485     if (has_triple_fault_event) {
4486         events.flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT;
4487         events.triple_fault.pending = env->triple_fault_pending;
4488     }
4489 
4490     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
4491 }
4492 
4493 static int kvm_get_vcpu_events(X86CPU *cpu)
4494 {
4495     CPUX86State *env = &cpu->env;
4496     struct kvm_vcpu_events events;
4497     int ret;
4498 
4499     if (!kvm_has_vcpu_events()) {
4500         return 0;
4501     }
4502 
4503     memset(&events, 0, sizeof(events));
4504     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
4505     if (ret < 0) {
4506        return ret;
4507     }
4508 
4509     if (events.flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
4510         env->exception_pending = events.exception.pending;
4511         env->exception_has_payload = events.exception_has_payload;
4512         env->exception_payload = events.exception_payload;
4513     } else {
4514         env->exception_pending = 0;
4515         env->exception_has_payload = false;
4516     }
4517     env->exception_injected = events.exception.injected;
4518     env->exception_nr =
4519         (env->exception_pending || env->exception_injected) ?
4520         events.exception.nr : -1;
4521     env->has_error_code = events.exception.has_error_code;
4522     env->error_code = events.exception.error_code;
4523 
4524     env->interrupt_injected =
4525         events.interrupt.injected ? events.interrupt.nr : -1;
4526     env->soft_interrupt = events.interrupt.soft;
4527 
4528     env->nmi_injected = events.nmi.injected;
4529     env->nmi_pending = events.nmi.pending;
4530     if (events.nmi.masked) {
4531         env->hflags2 |= HF2_NMI_MASK;
4532     } else {
4533         env->hflags2 &= ~HF2_NMI_MASK;
4534     }
4535 
4536     if (events.flags & KVM_VCPUEVENT_VALID_SMM) {
4537         if (events.smi.smm) {
4538             env->hflags |= HF_SMM_MASK;
4539         } else {
4540             env->hflags &= ~HF_SMM_MASK;
4541         }
4542         if (events.smi.pending) {
4543             cpu_interrupt(CPU(cpu), CPU_INTERRUPT_SMI);
4544         } else {
4545             cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_SMI);
4546         }
4547         if (events.smi.smm_inside_nmi) {
4548             env->hflags2 |= HF2_SMM_INSIDE_NMI_MASK;
4549         } else {
4550             env->hflags2 &= ~HF2_SMM_INSIDE_NMI_MASK;
4551         }
4552         if (events.smi.latched_init) {
4553             cpu_interrupt(CPU(cpu), CPU_INTERRUPT_INIT);
4554         } else {
4555             cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_INIT);
4556         }
4557     }
4558 
4559     if (events.flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) {
4560         env->triple_fault_pending = events.triple_fault.pending;
4561     }
4562 
4563     env->sipi_vector = events.sipi_vector;
4564 
4565     return 0;
4566 }
4567 
4568 static int kvm_guest_debug_workarounds(X86CPU *cpu)
4569 {
4570     CPUState *cs = CPU(cpu);
4571     CPUX86State *env = &cpu->env;
4572     int ret = 0;
4573     unsigned long reinject_trap = 0;
4574 
4575     if (!kvm_has_vcpu_events()) {
4576         if (env->exception_nr == EXCP01_DB) {
4577             reinject_trap = KVM_GUESTDBG_INJECT_DB;
4578         } else if (env->exception_injected == EXCP03_INT3) {
4579             reinject_trap = KVM_GUESTDBG_INJECT_BP;
4580         }
4581         kvm_reset_exception(env);
4582     }
4583 
4584     /*
4585      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
4586      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
4587      * by updating the debug state once again if single-stepping is on.
4588      * Another reason to call kvm_update_guest_debug here is a pending debug
4589      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
4590      * reinject them via SET_GUEST_DEBUG.
4591      */
4592     if (reinject_trap ||
4593         (!kvm_has_robust_singlestep() && cs->singlestep_enabled)) {
4594         ret = kvm_update_guest_debug(cs, reinject_trap);
4595     }
4596     return ret;
4597 }
4598 
4599 static int kvm_put_debugregs(X86CPU *cpu)
4600 {
4601     CPUX86State *env = &cpu->env;
4602     struct kvm_debugregs dbgregs;
4603     int i;
4604 
4605     if (!kvm_has_debugregs()) {
4606         return 0;
4607     }
4608 
4609     memset(&dbgregs, 0, sizeof(dbgregs));
4610     for (i = 0; i < 4; i++) {
4611         dbgregs.db[i] = env->dr[i];
4612     }
4613     dbgregs.dr6 = env->dr[6];
4614     dbgregs.dr7 = env->dr[7];
4615     dbgregs.flags = 0;
4616 
4617     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEBUGREGS, &dbgregs);
4618 }
4619 
4620 static int kvm_get_debugregs(X86CPU *cpu)
4621 {
4622     CPUX86State *env = &cpu->env;
4623     struct kvm_debugregs dbgregs;
4624     int i, ret;
4625 
4626     if (!kvm_has_debugregs()) {
4627         return 0;
4628     }
4629 
4630     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_DEBUGREGS, &dbgregs);
4631     if (ret < 0) {
4632         return ret;
4633     }
4634     for (i = 0; i < 4; i++) {
4635         env->dr[i] = dbgregs.db[i];
4636     }
4637     env->dr[4] = env->dr[6] = dbgregs.dr6;
4638     env->dr[5] = env->dr[7] = dbgregs.dr7;
4639 
4640     return 0;
4641 }
4642 
4643 static int kvm_put_nested_state(X86CPU *cpu)
4644 {
4645     CPUX86State *env = &cpu->env;
4646     int max_nested_state_len = kvm_max_nested_state_length();
4647 
4648     if (!env->nested_state) {
4649         return 0;
4650     }
4651 
4652     /*
4653      * Copy flags that are affected by reset from env->hflags and env->hflags2.
4654      */
4655     if (env->hflags & HF_GUEST_MASK) {
4656         env->nested_state->flags |= KVM_STATE_NESTED_GUEST_MODE;
4657     } else {
4658         env->nested_state->flags &= ~KVM_STATE_NESTED_GUEST_MODE;
4659     }
4660 
4661     /* Don't set KVM_STATE_NESTED_GIF_SET on VMX as it is illegal */
4662     if (cpu_has_svm(env) && (env->hflags2 & HF2_GIF_MASK)) {
4663         env->nested_state->flags |= KVM_STATE_NESTED_GIF_SET;
4664     } else {
4665         env->nested_state->flags &= ~KVM_STATE_NESTED_GIF_SET;
4666     }
4667 
4668     assert(env->nested_state->size <= max_nested_state_len);
4669     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_NESTED_STATE, env->nested_state);
4670 }
4671 
4672 static int kvm_get_nested_state(X86CPU *cpu)
4673 {
4674     CPUX86State *env = &cpu->env;
4675     int max_nested_state_len = kvm_max_nested_state_length();
4676     int ret;
4677 
4678     if (!env->nested_state) {
4679         return 0;
4680     }
4681 
4682     /*
4683      * It is possible that migration restored a smaller size into
4684      * nested_state->hdr.size than what our kernel support.
4685      * We preserve migration origin nested_state->hdr.size for
4686      * call to KVM_SET_NESTED_STATE but wish that our next call
4687      * to KVM_GET_NESTED_STATE will use max size our kernel support.
4688      */
4689     env->nested_state->size = max_nested_state_len;
4690 
4691     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_NESTED_STATE, env->nested_state);
4692     if (ret < 0) {
4693         return ret;
4694     }
4695 
4696     /*
4697      * Copy flags that are affected by reset to env->hflags and env->hflags2.
4698      */
4699     if (env->nested_state->flags & KVM_STATE_NESTED_GUEST_MODE) {
4700         env->hflags |= HF_GUEST_MASK;
4701     } else {
4702         env->hflags &= ~HF_GUEST_MASK;
4703     }
4704 
4705     /* Keep HF2_GIF_MASK set on !SVM as x86_cpu_pending_interrupt() needs it */
4706     if (cpu_has_svm(env)) {
4707         if (env->nested_state->flags & KVM_STATE_NESTED_GIF_SET) {
4708             env->hflags2 |= HF2_GIF_MASK;
4709         } else {
4710             env->hflags2 &= ~HF2_GIF_MASK;
4711         }
4712     }
4713 
4714     return ret;
4715 }
4716 
4717 int kvm_arch_put_registers(CPUState *cpu, int level)
4718 {
4719     X86CPU *x86_cpu = X86_CPU(cpu);
4720     int ret;
4721 
4722     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
4723 
4724     /*
4725      * Put MSR_IA32_FEATURE_CONTROL first, this ensures the VM gets out of VMX
4726      * root operation upon vCPU reset. kvm_put_msr_feature_control() should also
4727      * precede kvm_put_nested_state() when 'real' nested state is set.
4728      */
4729     if (level >= KVM_PUT_RESET_STATE) {
4730         ret = kvm_put_msr_feature_control(x86_cpu);
4731         if (ret < 0) {
4732             return ret;
4733         }
4734     }
4735 
4736     /* must be before kvm_put_nested_state so that EFER.SVME is set */
4737     ret = has_sregs2 ? kvm_put_sregs2(x86_cpu) : kvm_put_sregs(x86_cpu);
4738     if (ret < 0) {
4739         return ret;
4740     }
4741 
4742     if (level >= KVM_PUT_RESET_STATE) {
4743         ret = kvm_put_nested_state(x86_cpu);
4744         if (ret < 0) {
4745             return ret;
4746         }
4747     }
4748 
4749     if (level == KVM_PUT_FULL_STATE) {
4750         /* We don't check for kvm_arch_set_tsc_khz() errors here,
4751          * because TSC frequency mismatch shouldn't abort migration,
4752          * unless the user explicitly asked for a more strict TSC
4753          * setting (e.g. using an explicit "tsc-freq" option).
4754          */
4755         kvm_arch_set_tsc_khz(cpu);
4756     }
4757 
4758 #ifdef CONFIG_XEN_EMU
4759     if (xen_mode == XEN_EMULATE && level == KVM_PUT_FULL_STATE) {
4760         ret = kvm_put_xen_state(cpu);
4761         if (ret < 0) {
4762             return ret;
4763         }
4764     }
4765 #endif
4766 
4767     ret = kvm_getput_regs(x86_cpu, 1);
4768     if (ret < 0) {
4769         return ret;
4770     }
4771     ret = kvm_put_xsave(x86_cpu);
4772     if (ret < 0) {
4773         return ret;
4774     }
4775     ret = kvm_put_xcrs(x86_cpu);
4776     if (ret < 0) {
4777         return ret;
4778     }
4779     /* must be before kvm_put_msrs */
4780     ret = kvm_inject_mce_oldstyle(x86_cpu);
4781     if (ret < 0) {
4782         return ret;
4783     }
4784     ret = kvm_put_msrs(x86_cpu, level);
4785     if (ret < 0) {
4786         return ret;
4787     }
4788     ret = kvm_put_vcpu_events(x86_cpu, level);
4789     if (ret < 0) {
4790         return ret;
4791     }
4792     if (level >= KVM_PUT_RESET_STATE) {
4793         ret = kvm_put_mp_state(x86_cpu);
4794         if (ret < 0) {
4795             return ret;
4796         }
4797     }
4798 
4799     ret = kvm_put_tscdeadline_msr(x86_cpu);
4800     if (ret < 0) {
4801         return ret;
4802     }
4803     ret = kvm_put_debugregs(x86_cpu);
4804     if (ret < 0) {
4805         return ret;
4806     }
4807     /* must be last */
4808     ret = kvm_guest_debug_workarounds(x86_cpu);
4809     if (ret < 0) {
4810         return ret;
4811     }
4812     return 0;
4813 }
4814 
4815 int kvm_arch_get_registers(CPUState *cs)
4816 {
4817     X86CPU *cpu = X86_CPU(cs);
4818     int ret;
4819 
4820     assert(cpu_is_stopped(cs) || qemu_cpu_is_self(cs));
4821 
4822     ret = kvm_get_vcpu_events(cpu);
4823     if (ret < 0) {
4824         goto out;
4825     }
4826     /*
4827      * KVM_GET_MPSTATE can modify CS and RIP, call it before
4828      * KVM_GET_REGS and KVM_GET_SREGS.
4829      */
4830     ret = kvm_get_mp_state(cpu);
4831     if (ret < 0) {
4832         goto out;
4833     }
4834     ret = kvm_getput_regs(cpu, 0);
4835     if (ret < 0) {
4836         goto out;
4837     }
4838     ret = kvm_get_xsave(cpu);
4839     if (ret < 0) {
4840         goto out;
4841     }
4842     ret = kvm_get_xcrs(cpu);
4843     if (ret < 0) {
4844         goto out;
4845     }
4846     ret = has_sregs2 ? kvm_get_sregs2(cpu) : kvm_get_sregs(cpu);
4847     if (ret < 0) {
4848         goto out;
4849     }
4850     ret = kvm_get_msrs(cpu);
4851     if (ret < 0) {
4852         goto out;
4853     }
4854     ret = kvm_get_apic(cpu);
4855     if (ret < 0) {
4856         goto out;
4857     }
4858     ret = kvm_get_debugregs(cpu);
4859     if (ret < 0) {
4860         goto out;
4861     }
4862     ret = kvm_get_nested_state(cpu);
4863     if (ret < 0) {
4864         goto out;
4865     }
4866 #ifdef CONFIG_XEN_EMU
4867     if (xen_mode == XEN_EMULATE) {
4868         ret = kvm_get_xen_state(cs);
4869         if (ret < 0) {
4870             goto out;
4871         }
4872     }
4873 #endif
4874     ret = 0;
4875  out:
4876     cpu_sync_bndcs_hflags(&cpu->env);
4877     return ret;
4878 }
4879 
4880 void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
4881 {
4882     X86CPU *x86_cpu = X86_CPU(cpu);
4883     CPUX86State *env = &x86_cpu->env;
4884     int ret;
4885 
4886     /* Inject NMI */
4887     if (cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
4888         if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
4889             qemu_mutex_lock_iothread();
4890             cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
4891             qemu_mutex_unlock_iothread();
4892             DPRINTF("injected NMI\n");
4893             ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
4894             if (ret < 0) {
4895                 fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
4896                         strerror(-ret));
4897             }
4898         }
4899         if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
4900             qemu_mutex_lock_iothread();
4901             cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
4902             qemu_mutex_unlock_iothread();
4903             DPRINTF("injected SMI\n");
4904             ret = kvm_vcpu_ioctl(cpu, KVM_SMI);
4905             if (ret < 0) {
4906                 fprintf(stderr, "KVM: injection failed, SMI lost (%s)\n",
4907                         strerror(-ret));
4908             }
4909         }
4910     }
4911 
4912     if (!kvm_pic_in_kernel()) {
4913         qemu_mutex_lock_iothread();
4914     }
4915 
4916     /* Force the VCPU out of its inner loop to process any INIT requests
4917      * or (for userspace APIC, but it is cheap to combine the checks here)
4918      * pending TPR access reports.
4919      */
4920     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
4921         if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
4922             !(env->hflags & HF_SMM_MASK)) {
4923             cpu->exit_request = 1;
4924         }
4925         if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
4926             cpu->exit_request = 1;
4927         }
4928     }
4929 
4930     if (!kvm_pic_in_kernel()) {
4931         /* Try to inject an interrupt if the guest can accept it */
4932         if (run->ready_for_interrupt_injection &&
4933             (cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
4934             (env->eflags & IF_MASK)) {
4935             int irq;
4936 
4937             cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
4938             irq = cpu_get_pic_interrupt(env);
4939             if (irq >= 0) {
4940                 struct kvm_interrupt intr;
4941 
4942                 intr.irq = irq;
4943                 DPRINTF("injected interrupt %d\n", irq);
4944                 ret = kvm_vcpu_ioctl(cpu, KVM_INTERRUPT, &intr);
4945                 if (ret < 0) {
4946                     fprintf(stderr,
4947                             "KVM: injection failed, interrupt lost (%s)\n",
4948                             strerror(-ret));
4949                 }
4950             }
4951         }
4952 
4953         /* If we have an interrupt but the guest is not ready to receive an
4954          * interrupt, request an interrupt window exit.  This will
4955          * cause a return to userspace as soon as the guest is ready to
4956          * receive interrupts. */
4957         if ((cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
4958             run->request_interrupt_window = 1;
4959         } else {
4960             run->request_interrupt_window = 0;
4961         }
4962 
4963         DPRINTF("setting tpr\n");
4964         run->cr8 = cpu_get_apic_tpr(x86_cpu->apic_state);
4965 
4966         qemu_mutex_unlock_iothread();
4967     }
4968 }
4969 
4970 static void kvm_rate_limit_on_bus_lock(void)
4971 {
4972     uint64_t delay_ns = ratelimit_calculate_delay(&bus_lock_ratelimit_ctrl, 1);
4973 
4974     if (delay_ns) {
4975         g_usleep(delay_ns / SCALE_US);
4976     }
4977 }
4978 
4979 MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
4980 {
4981     X86CPU *x86_cpu = X86_CPU(cpu);
4982     CPUX86State *env = &x86_cpu->env;
4983 
4984     if (run->flags & KVM_RUN_X86_SMM) {
4985         env->hflags |= HF_SMM_MASK;
4986     } else {
4987         env->hflags &= ~HF_SMM_MASK;
4988     }
4989     if (run->if_flag) {
4990         env->eflags |= IF_MASK;
4991     } else {
4992         env->eflags &= ~IF_MASK;
4993     }
4994     if (run->flags & KVM_RUN_X86_BUS_LOCK) {
4995         kvm_rate_limit_on_bus_lock();
4996     }
4997 
4998 #ifdef CONFIG_XEN_EMU
4999     /*
5000      * If the callback is asserted as a GSI (or PCI INTx) then check if
5001      * vcpu_info->evtchn_upcall_pending has been cleared, and deassert
5002      * the callback IRQ if so. Ideally we could hook into the PIC/IOAPIC
5003      * EOI and only resample then, exactly how the VFIO eventfd pairs
5004      * are designed to work for level triggered interrupts.
5005      */
5006     if (x86_cpu->env.xen_callback_asserted) {
5007         kvm_xen_maybe_deassert_callback(cpu);
5008     }
5009 #endif
5010 
5011     /* We need to protect the apic state against concurrent accesses from
5012      * different threads in case the userspace irqchip is used. */
5013     if (!kvm_irqchip_in_kernel()) {
5014         qemu_mutex_lock_iothread();
5015     }
5016     cpu_set_apic_tpr(x86_cpu->apic_state, run->cr8);
5017     cpu_set_apic_base(x86_cpu->apic_state, run->apic_base);
5018     if (!kvm_irqchip_in_kernel()) {
5019         qemu_mutex_unlock_iothread();
5020     }
5021     return cpu_get_mem_attrs(env);
5022 }
5023 
5024 int kvm_arch_process_async_events(CPUState *cs)
5025 {
5026     X86CPU *cpu = X86_CPU(cs);
5027     CPUX86State *env = &cpu->env;
5028 
5029     if (cs->interrupt_request & CPU_INTERRUPT_MCE) {
5030         /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
5031         assert(env->mcg_cap);
5032 
5033         cs->interrupt_request &= ~CPU_INTERRUPT_MCE;
5034 
5035         kvm_cpu_synchronize_state(cs);
5036 
5037         if (env->exception_nr == EXCP08_DBLE) {
5038             /* this means triple fault */
5039             qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
5040             cs->exit_request = 1;
5041             return 0;
5042         }
5043         kvm_queue_exception(env, EXCP12_MCHK, 0, 0);
5044         env->has_error_code = 0;
5045 
5046         cs->halted = 0;
5047         if (kvm_irqchip_in_kernel() && env->mp_state == KVM_MP_STATE_HALTED) {
5048             env->mp_state = KVM_MP_STATE_RUNNABLE;
5049         }
5050     }
5051 
5052     if ((cs->interrupt_request & CPU_INTERRUPT_INIT) &&
5053         !(env->hflags & HF_SMM_MASK)) {
5054         kvm_cpu_synchronize_state(cs);
5055         do_cpu_init(cpu);
5056     }
5057 
5058     if (kvm_irqchip_in_kernel()) {
5059         return 0;
5060     }
5061 
5062     if (cs->interrupt_request & CPU_INTERRUPT_POLL) {
5063         cs->interrupt_request &= ~CPU_INTERRUPT_POLL;
5064         apic_poll_irq(cpu->apic_state);
5065     }
5066     if (((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
5067          (env->eflags & IF_MASK)) ||
5068         (cs->interrupt_request & CPU_INTERRUPT_NMI)) {
5069         cs->halted = 0;
5070     }
5071     if (cs->interrupt_request & CPU_INTERRUPT_SIPI) {
5072         kvm_cpu_synchronize_state(cs);
5073         do_cpu_sipi(cpu);
5074     }
5075     if (cs->interrupt_request & CPU_INTERRUPT_TPR) {
5076         cs->interrupt_request &= ~CPU_INTERRUPT_TPR;
5077         kvm_cpu_synchronize_state(cs);
5078         apic_handle_tpr_access_report(cpu->apic_state, env->eip,
5079                                       env->tpr_access_type);
5080     }
5081 
5082     return cs->halted;
5083 }
5084 
5085 static int kvm_handle_halt(X86CPU *cpu)
5086 {
5087     CPUState *cs = CPU(cpu);
5088     CPUX86State *env = &cpu->env;
5089 
5090     if (!((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
5091           (env->eflags & IF_MASK)) &&
5092         !(cs->interrupt_request & CPU_INTERRUPT_NMI)) {
5093         cs->halted = 1;
5094         return EXCP_HLT;
5095     }
5096 
5097     return 0;
5098 }
5099 
5100 static int kvm_handle_tpr_access(X86CPU *cpu)
5101 {
5102     CPUState *cs = CPU(cpu);
5103     struct kvm_run *run = cs->kvm_run;
5104 
5105     apic_handle_tpr_access_report(cpu->apic_state, run->tpr_access.rip,
5106                                   run->tpr_access.is_write ? TPR_ACCESS_WRITE
5107                                                            : TPR_ACCESS_READ);
5108     return 1;
5109 }
5110 
5111 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
5112 {
5113     static const uint8_t int3 = 0xcc;
5114 
5115     if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
5116         cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&int3, 1, 1)) {
5117         return -EINVAL;
5118     }
5119     return 0;
5120 }
5121 
5122 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
5123 {
5124     uint8_t int3;
5125 
5126     if (cpu_memory_rw_debug(cs, bp->pc, &int3, 1, 0)) {
5127         return -EINVAL;
5128     }
5129     if (int3 != 0xcc) {
5130         return 0;
5131     }
5132     if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
5133         return -EINVAL;
5134     }
5135     return 0;
5136 }
5137 
5138 static struct {
5139     target_ulong addr;
5140     int len;
5141     int type;
5142 } hw_breakpoint[4];
5143 
5144 static int nb_hw_breakpoint;
5145 
5146 static int find_hw_breakpoint(target_ulong addr, int len, int type)
5147 {
5148     int n;
5149 
5150     for (n = 0; n < nb_hw_breakpoint; n++) {
5151         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
5152             (hw_breakpoint[n].len == len || len == -1)) {
5153             return n;
5154         }
5155     }
5156     return -1;
5157 }
5158 
5159 int kvm_arch_insert_hw_breakpoint(vaddr addr, vaddr len, int type)
5160 {
5161     switch (type) {
5162     case GDB_BREAKPOINT_HW:
5163         len = 1;
5164         break;
5165     case GDB_WATCHPOINT_WRITE:
5166     case GDB_WATCHPOINT_ACCESS:
5167         switch (len) {
5168         case 1:
5169             break;
5170         case 2:
5171         case 4:
5172         case 8:
5173             if (addr & (len - 1)) {
5174                 return -EINVAL;
5175             }
5176             break;
5177         default:
5178             return -EINVAL;
5179         }
5180         break;
5181     default:
5182         return -ENOSYS;
5183     }
5184 
5185     if (nb_hw_breakpoint == 4) {
5186         return -ENOBUFS;
5187     }
5188     if (find_hw_breakpoint(addr, len, type) >= 0) {
5189         return -EEXIST;
5190     }
5191     hw_breakpoint[nb_hw_breakpoint].addr = addr;
5192     hw_breakpoint[nb_hw_breakpoint].len = len;
5193     hw_breakpoint[nb_hw_breakpoint].type = type;
5194     nb_hw_breakpoint++;
5195 
5196     return 0;
5197 }
5198 
5199 int kvm_arch_remove_hw_breakpoint(vaddr addr, vaddr len, int type)
5200 {
5201     int n;
5202 
5203     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
5204     if (n < 0) {
5205         return -ENOENT;
5206     }
5207     nb_hw_breakpoint--;
5208     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
5209 
5210     return 0;
5211 }
5212 
5213 void kvm_arch_remove_all_hw_breakpoints(void)
5214 {
5215     nb_hw_breakpoint = 0;
5216 }
5217 
5218 static CPUWatchpoint hw_watchpoint;
5219 
5220 static int kvm_handle_debug(X86CPU *cpu,
5221                             struct kvm_debug_exit_arch *arch_info)
5222 {
5223     CPUState *cs = CPU(cpu);
5224     CPUX86State *env = &cpu->env;
5225     int ret = 0;
5226     int n;
5227 
5228     if (arch_info->exception == EXCP01_DB) {
5229         if (arch_info->dr6 & DR6_BS) {
5230             if (cs->singlestep_enabled) {
5231                 ret = EXCP_DEBUG;
5232             }
5233         } else {
5234             for (n = 0; n < 4; n++) {
5235                 if (arch_info->dr6 & (1 << n)) {
5236                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
5237                     case 0x0:
5238                         ret = EXCP_DEBUG;
5239                         break;
5240                     case 0x1:
5241                         ret = EXCP_DEBUG;
5242                         cs->watchpoint_hit = &hw_watchpoint;
5243                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
5244                         hw_watchpoint.flags = BP_MEM_WRITE;
5245                         break;
5246                     case 0x3:
5247                         ret = EXCP_DEBUG;
5248                         cs->watchpoint_hit = &hw_watchpoint;
5249                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
5250                         hw_watchpoint.flags = BP_MEM_ACCESS;
5251                         break;
5252                     }
5253                 }
5254             }
5255         }
5256     } else if (kvm_find_sw_breakpoint(cs, arch_info->pc)) {
5257         ret = EXCP_DEBUG;
5258     }
5259     if (ret == 0) {
5260         cpu_synchronize_state(cs);
5261         assert(env->exception_nr == -1);
5262 
5263         /* pass to guest */
5264         kvm_queue_exception(env, arch_info->exception,
5265                             arch_info->exception == EXCP01_DB,
5266                             arch_info->dr6);
5267         env->has_error_code = 0;
5268     }
5269 
5270     return ret;
5271 }
5272 
5273 void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg)
5274 {
5275     const uint8_t type_code[] = {
5276         [GDB_BREAKPOINT_HW] = 0x0,
5277         [GDB_WATCHPOINT_WRITE] = 0x1,
5278         [GDB_WATCHPOINT_ACCESS] = 0x3
5279     };
5280     const uint8_t len_code[] = {
5281         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
5282     };
5283     int n;
5284 
5285     if (kvm_sw_breakpoints_active(cpu)) {
5286         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
5287     }
5288     if (nb_hw_breakpoint > 0) {
5289         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
5290         dbg->arch.debugreg[7] = 0x0600;
5291         for (n = 0; n < nb_hw_breakpoint; n++) {
5292             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
5293             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
5294                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
5295                 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
5296         }
5297     }
5298 }
5299 
5300 static bool kvm_install_msr_filters(KVMState *s)
5301 {
5302     uint64_t zero = 0;
5303     struct kvm_msr_filter filter = {
5304         .flags = KVM_MSR_FILTER_DEFAULT_ALLOW,
5305     };
5306     int r, i, j = 0;
5307 
5308     for (i = 0; i < KVM_MSR_FILTER_MAX_RANGES; i++) {
5309         KVMMSRHandlers *handler = &msr_handlers[i];
5310         if (handler->msr) {
5311             struct kvm_msr_filter_range *range = &filter.ranges[j++];
5312 
5313             *range = (struct kvm_msr_filter_range) {
5314                 .flags = 0,
5315                 .nmsrs = 1,
5316                 .base = handler->msr,
5317                 .bitmap = (__u8 *)&zero,
5318             };
5319 
5320             if (handler->rdmsr) {
5321                 range->flags |= KVM_MSR_FILTER_READ;
5322             }
5323 
5324             if (handler->wrmsr) {
5325                 range->flags |= KVM_MSR_FILTER_WRITE;
5326             }
5327         }
5328     }
5329 
5330     r = kvm_vm_ioctl(s, KVM_X86_SET_MSR_FILTER, &filter);
5331     if (r) {
5332         return false;
5333     }
5334 
5335     return true;
5336 }
5337 
5338 bool kvm_filter_msr(KVMState *s, uint32_t msr, QEMURDMSRHandler *rdmsr,
5339                     QEMUWRMSRHandler *wrmsr)
5340 {
5341     int i;
5342 
5343     for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) {
5344         if (!msr_handlers[i].msr) {
5345             msr_handlers[i] = (KVMMSRHandlers) {
5346                 .msr = msr,
5347                 .rdmsr = rdmsr,
5348                 .wrmsr = wrmsr,
5349             };
5350 
5351             if (!kvm_install_msr_filters(s)) {
5352                 msr_handlers[i] = (KVMMSRHandlers) { };
5353                 return false;
5354             }
5355 
5356             return true;
5357         }
5358     }
5359 
5360     return false;
5361 }
5362 
5363 static int kvm_handle_rdmsr(X86CPU *cpu, struct kvm_run *run)
5364 {
5365     int i;
5366     bool r;
5367 
5368     for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) {
5369         KVMMSRHandlers *handler = &msr_handlers[i];
5370         if (run->msr.index == handler->msr) {
5371             if (handler->rdmsr) {
5372                 r = handler->rdmsr(cpu, handler->msr,
5373                                    (uint64_t *)&run->msr.data);
5374                 run->msr.error = r ? 0 : 1;
5375                 return 0;
5376             }
5377         }
5378     }
5379 
5380     assert(false);
5381 }
5382 
5383 static int kvm_handle_wrmsr(X86CPU *cpu, struct kvm_run *run)
5384 {
5385     int i;
5386     bool r;
5387 
5388     for (i = 0; i < ARRAY_SIZE(msr_handlers); i++) {
5389         KVMMSRHandlers *handler = &msr_handlers[i];
5390         if (run->msr.index == handler->msr) {
5391             if (handler->wrmsr) {
5392                 r = handler->wrmsr(cpu, handler->msr, run->msr.data);
5393                 run->msr.error = r ? 0 : 1;
5394                 return 0;
5395             }
5396         }
5397     }
5398 
5399     assert(false);
5400 }
5401 
5402 static bool has_sgx_provisioning;
5403 
5404 static bool __kvm_enable_sgx_provisioning(KVMState *s)
5405 {
5406     int fd, ret;
5407 
5408     if (!kvm_vm_check_extension(s, KVM_CAP_SGX_ATTRIBUTE)) {
5409         return false;
5410     }
5411 
5412     fd = qemu_open_old("/dev/sgx_provision", O_RDONLY);
5413     if (fd < 0) {
5414         return false;
5415     }
5416 
5417     ret = kvm_vm_enable_cap(s, KVM_CAP_SGX_ATTRIBUTE, 0, fd);
5418     if (ret) {
5419         error_report("Could not enable SGX PROVISIONKEY: %s", strerror(-ret));
5420         exit(1);
5421     }
5422     close(fd);
5423     return true;
5424 }
5425 
5426 bool kvm_enable_sgx_provisioning(KVMState *s)
5427 {
5428     return MEMORIZE(__kvm_enable_sgx_provisioning(s), has_sgx_provisioning);
5429 }
5430 
5431 static bool host_supports_vmx(void)
5432 {
5433     uint32_t ecx, unused;
5434 
5435     host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
5436     return ecx & CPUID_EXT_VMX;
5437 }
5438 
5439 #define VMX_INVALID_GUEST_STATE 0x80000021
5440 
5441 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
5442 {
5443     X86CPU *cpu = X86_CPU(cs);
5444     uint64_t code;
5445     int ret;
5446     bool ctx_invalid;
5447     char str[256];
5448     KVMState *state;
5449 
5450     switch (run->exit_reason) {
5451     case KVM_EXIT_HLT:
5452         DPRINTF("handle_hlt\n");
5453         qemu_mutex_lock_iothread();
5454         ret = kvm_handle_halt(cpu);
5455         qemu_mutex_unlock_iothread();
5456         break;
5457     case KVM_EXIT_SET_TPR:
5458         ret = 0;
5459         break;
5460     case KVM_EXIT_TPR_ACCESS:
5461         qemu_mutex_lock_iothread();
5462         ret = kvm_handle_tpr_access(cpu);
5463         qemu_mutex_unlock_iothread();
5464         break;
5465     case KVM_EXIT_FAIL_ENTRY:
5466         code = run->fail_entry.hardware_entry_failure_reason;
5467         fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
5468                 code);
5469         if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
5470             fprintf(stderr,
5471                     "\nIf you're running a guest on an Intel machine without "
5472                         "unrestricted mode\n"
5473                     "support, the failure can be most likely due to the guest "
5474                         "entering an invalid\n"
5475                     "state for Intel VT. For example, the guest maybe running "
5476                         "in big real mode\n"
5477                     "which is not supported on less recent Intel processors."
5478                         "\n\n");
5479         }
5480         ret = -1;
5481         break;
5482     case KVM_EXIT_EXCEPTION:
5483         fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
5484                 run->ex.exception, run->ex.error_code);
5485         ret = -1;
5486         break;
5487     case KVM_EXIT_DEBUG:
5488         DPRINTF("kvm_exit_debug\n");
5489         qemu_mutex_lock_iothread();
5490         ret = kvm_handle_debug(cpu, &run->debug.arch);
5491         qemu_mutex_unlock_iothread();
5492         break;
5493     case KVM_EXIT_HYPERV:
5494         ret = kvm_hv_handle_exit(cpu, &run->hyperv);
5495         break;
5496     case KVM_EXIT_IOAPIC_EOI:
5497         ioapic_eoi_broadcast(run->eoi.vector);
5498         ret = 0;
5499         break;
5500     case KVM_EXIT_X86_BUS_LOCK:
5501         /* already handled in kvm_arch_post_run */
5502         ret = 0;
5503         break;
5504     case KVM_EXIT_NOTIFY:
5505         ctx_invalid = !!(run->notify.flags & KVM_NOTIFY_CONTEXT_INVALID);
5506         state = KVM_STATE(current_accel());
5507         sprintf(str, "Encounter a notify exit with %svalid context in"
5508                      " guest. There can be possible misbehaves in guest."
5509                      " Please have a look.", ctx_invalid ? "in" : "");
5510         if (ctx_invalid ||
5511             state->notify_vmexit == NOTIFY_VMEXIT_OPTION_INTERNAL_ERROR) {
5512             warn_report("KVM internal error: %s", str);
5513             ret = -1;
5514         } else {
5515             warn_report_once("KVM: %s", str);
5516             ret = 0;
5517         }
5518         break;
5519     case KVM_EXIT_X86_RDMSR:
5520         /* We only enable MSR filtering, any other exit is bogus */
5521         assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
5522         ret = kvm_handle_rdmsr(cpu, run);
5523         break;
5524     case KVM_EXIT_X86_WRMSR:
5525         /* We only enable MSR filtering, any other exit is bogus */
5526         assert(run->msr.reason == KVM_MSR_EXIT_REASON_FILTER);
5527         ret = kvm_handle_wrmsr(cpu, run);
5528         break;
5529 #ifdef CONFIG_XEN_EMU
5530     case KVM_EXIT_XEN:
5531         ret = kvm_xen_handle_exit(cpu, &run->xen);
5532         break;
5533 #endif
5534     default:
5535         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
5536         ret = -1;
5537         break;
5538     }
5539 
5540     return ret;
5541 }
5542 
5543 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
5544 {
5545     X86CPU *cpu = X86_CPU(cs);
5546     CPUX86State *env = &cpu->env;
5547 
5548     kvm_cpu_synchronize_state(cs);
5549     return !(env->cr[0] & CR0_PE_MASK) ||
5550            ((env->segs[R_CS].selector  & 3) != 3);
5551 }
5552 
5553 void kvm_arch_init_irq_routing(KVMState *s)
5554 {
5555     /* We know at this point that we're using the in-kernel
5556      * irqchip, so we can use irqfds, and on x86 we know
5557      * we can use msi via irqfd and GSI routing.
5558      */
5559     kvm_msi_via_irqfd_allowed = true;
5560     kvm_gsi_routing_allowed = true;
5561 
5562     if (kvm_irqchip_is_split()) {
5563         KVMRouteChange c = kvm_irqchip_begin_route_changes(s);
5564         int i;
5565 
5566         /* If the ioapic is in QEMU and the lapics are in KVM, reserve
5567            MSI routes for signaling interrupts to the local apics. */
5568         for (i = 0; i < IOAPIC_NUM_PINS; i++) {
5569             if (kvm_irqchip_add_msi_route(&c, 0, NULL) < 0) {
5570                 error_report("Could not enable split IRQ mode.");
5571                 exit(1);
5572             }
5573         }
5574         kvm_irqchip_commit_route_changes(&c);
5575     }
5576 }
5577 
5578 int kvm_arch_irqchip_create(KVMState *s)
5579 {
5580     int ret;
5581     if (kvm_kernel_irqchip_split()) {
5582         ret = kvm_vm_enable_cap(s, KVM_CAP_SPLIT_IRQCHIP, 0, 24);
5583         if (ret) {
5584             error_report("Could not enable split irqchip mode: %s",
5585                          strerror(-ret));
5586             exit(1);
5587         } else {
5588             DPRINTF("Enabled KVM_CAP_SPLIT_IRQCHIP\n");
5589             kvm_split_irqchip = true;
5590             return 1;
5591         }
5592     } else {
5593         return 0;
5594     }
5595 }
5596 
5597 uint64_t kvm_swizzle_msi_ext_dest_id(uint64_t address)
5598 {
5599     CPUX86State *env;
5600     uint64_t ext_id;
5601 
5602     if (!first_cpu) {
5603         return address;
5604     }
5605     env = &X86_CPU(first_cpu)->env;
5606     if (!(env->features[FEAT_KVM] & (1 << KVM_FEATURE_MSI_EXT_DEST_ID))) {
5607         return address;
5608     }
5609 
5610     /*
5611      * If the remappable format bit is set, or the upper bits are
5612      * already set in address_hi, or the low extended bits aren't
5613      * there anyway, do nothing.
5614      */
5615     ext_id = address & (0xff << MSI_ADDR_DEST_IDX_SHIFT);
5616     if (!ext_id || (ext_id & (1 << MSI_ADDR_DEST_IDX_SHIFT)) || (address >> 32)) {
5617         return address;
5618     }
5619 
5620     address &= ~ext_id;
5621     address |= ext_id << 35;
5622     return address;
5623 }
5624 
5625 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
5626                              uint64_t address, uint32_t data, PCIDevice *dev)
5627 {
5628     X86IOMMUState *iommu = x86_iommu_get_default();
5629 
5630     if (iommu) {
5631         X86IOMMUClass *class = X86_IOMMU_DEVICE_GET_CLASS(iommu);
5632 
5633         if (class->int_remap) {
5634             int ret;
5635             MSIMessage src, dst;
5636 
5637             src.address = route->u.msi.address_hi;
5638             src.address <<= VTD_MSI_ADDR_HI_SHIFT;
5639             src.address |= route->u.msi.address_lo;
5640             src.data = route->u.msi.data;
5641 
5642             ret = class->int_remap(iommu, &src, &dst, dev ?     \
5643                                    pci_requester_id(dev) :      \
5644                                    X86_IOMMU_SID_INVALID);
5645             if (ret) {
5646                 trace_kvm_x86_fixup_msi_error(route->gsi);
5647                 return 1;
5648             }
5649 
5650             /*
5651              * Handled untranslated compatibility format interrupt with
5652              * extended destination ID in the low bits 11-5. */
5653             dst.address = kvm_swizzle_msi_ext_dest_id(dst.address);
5654 
5655             route->u.msi.address_hi = dst.address >> VTD_MSI_ADDR_HI_SHIFT;
5656             route->u.msi.address_lo = dst.address & VTD_MSI_ADDR_LO_MASK;
5657             route->u.msi.data = dst.data;
5658             return 0;
5659         }
5660     }
5661 
5662 #ifdef CONFIG_XEN_EMU
5663     if (xen_mode == XEN_EMULATE) {
5664         int handled = xen_evtchn_translate_pirq_msi(route, address, data);
5665 
5666         /*
5667          * If it was a PIRQ and successfully routed (handled == 0) or it was
5668          * an error (handled < 0), return. If it wasn't a PIRQ, keep going.
5669          */
5670         if (handled <= 0) {
5671             return handled;
5672         }
5673     }
5674 #endif
5675 
5676     address = kvm_swizzle_msi_ext_dest_id(address);
5677     route->u.msi.address_hi = address >> VTD_MSI_ADDR_HI_SHIFT;
5678     route->u.msi.address_lo = address & VTD_MSI_ADDR_LO_MASK;
5679     return 0;
5680 }
5681 
5682 typedef struct MSIRouteEntry MSIRouteEntry;
5683 
5684 struct MSIRouteEntry {
5685     PCIDevice *dev;             /* Device pointer */
5686     int vector;                 /* MSI/MSIX vector index */
5687     int virq;                   /* Virtual IRQ index */
5688     QLIST_ENTRY(MSIRouteEntry) list;
5689 };
5690 
5691 /* List of used GSI routes */
5692 static QLIST_HEAD(, MSIRouteEntry) msi_route_list = \
5693     QLIST_HEAD_INITIALIZER(msi_route_list);
5694 
5695 void kvm_update_msi_routes_all(void *private, bool global,
5696                                uint32_t index, uint32_t mask)
5697 {
5698     int cnt = 0, vector;
5699     MSIRouteEntry *entry;
5700     MSIMessage msg;
5701     PCIDevice *dev;
5702 
5703     /* TODO: explicit route update */
5704     QLIST_FOREACH(entry, &msi_route_list, list) {
5705         cnt++;
5706         vector = entry->vector;
5707         dev = entry->dev;
5708         if (msix_enabled(dev) && !msix_is_masked(dev, vector)) {
5709             msg = msix_get_message(dev, vector);
5710         } else if (msi_enabled(dev) && !msi_is_masked(dev, vector)) {
5711             msg = msi_get_message(dev, vector);
5712         } else {
5713             /*
5714              * Either MSI/MSIX is disabled for the device, or the
5715              * specific message was masked out.  Skip this one.
5716              */
5717             continue;
5718         }
5719         kvm_irqchip_update_msi_route(kvm_state, entry->virq, msg, dev);
5720     }
5721     kvm_irqchip_commit_routes(kvm_state);
5722     trace_kvm_x86_update_msi_routes(cnt);
5723 }
5724 
5725 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
5726                                 int vector, PCIDevice *dev)
5727 {
5728     static bool notify_list_inited = false;
5729     MSIRouteEntry *entry;
5730 
5731     if (!dev) {
5732         /* These are (possibly) IOAPIC routes only used for split
5733          * kernel irqchip mode, while what we are housekeeping are
5734          * PCI devices only. */
5735         return 0;
5736     }
5737 
5738     entry = g_new0(MSIRouteEntry, 1);
5739     entry->dev = dev;
5740     entry->vector = vector;
5741     entry->virq = route->gsi;
5742     QLIST_INSERT_HEAD(&msi_route_list, entry, list);
5743 
5744     trace_kvm_x86_add_msi_route(route->gsi);
5745 
5746     if (!notify_list_inited) {
5747         /* For the first time we do add route, add ourselves into
5748          * IOMMU's IEC notify list if needed. */
5749         X86IOMMUState *iommu = x86_iommu_get_default();
5750         if (iommu) {
5751             x86_iommu_iec_register_notifier(iommu,
5752                                             kvm_update_msi_routes_all,
5753                                             NULL);
5754         }
5755         notify_list_inited = true;
5756     }
5757     return 0;
5758 }
5759 
5760 int kvm_arch_release_virq_post(int virq)
5761 {
5762     MSIRouteEntry *entry, *next;
5763     QLIST_FOREACH_SAFE(entry, &msi_route_list, list, next) {
5764         if (entry->virq == virq) {
5765             trace_kvm_x86_remove_msi_route(virq);
5766             QLIST_REMOVE(entry, list);
5767             g_free(entry);
5768             break;
5769         }
5770     }
5771     return 0;
5772 }
5773 
5774 int kvm_arch_msi_data_to_gsi(uint32_t data)
5775 {
5776     abort();
5777 }
5778 
5779 bool kvm_has_waitpkg(void)
5780 {
5781     return has_msr_umwait;
5782 }
5783 
5784 bool kvm_arch_cpu_check_are_resettable(void)
5785 {
5786     return !sev_es_enabled();
5787 }
5788 
5789 #define ARCH_REQ_XCOMP_GUEST_PERM       0x1025
5790 
5791 void kvm_request_xsave_components(X86CPU *cpu, uint64_t mask)
5792 {
5793     KVMState *s = kvm_state;
5794     uint64_t supported;
5795 
5796     mask &= XSTATE_DYNAMIC_MASK;
5797     if (!mask) {
5798         return;
5799     }
5800     /*
5801      * Just ignore bits that are not in CPUID[EAX=0xD,ECX=0].
5802      * ARCH_REQ_XCOMP_GUEST_PERM would fail, and QEMU has warned
5803      * about them already because they are not supported features.
5804      */
5805     supported = kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EAX);
5806     supported |= (uint64_t)kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EDX) << 32;
5807     mask &= supported;
5808 
5809     while (mask) {
5810         int bit = ctz64(mask);
5811         int rc = syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_GUEST_PERM, bit);
5812         if (rc) {
5813             /*
5814              * Older kernel version (<5.17) do not support
5815              * ARCH_REQ_XCOMP_GUEST_PERM, but also do not return
5816              * any dynamic feature from kvm_arch_get_supported_cpuid.
5817              */
5818             warn_report("prctl(ARCH_REQ_XCOMP_GUEST_PERM) failure "
5819                         "for feature bit %d", bit);
5820         }
5821         mask &= ~BIT_ULL(bit);
5822     }
5823 }
5824 
5825 static int kvm_arch_get_notify_vmexit(Object *obj, Error **errp)
5826 {
5827     KVMState *s = KVM_STATE(obj);
5828     return s->notify_vmexit;
5829 }
5830 
5831 static void kvm_arch_set_notify_vmexit(Object *obj, int value, Error **errp)
5832 {
5833     KVMState *s = KVM_STATE(obj);
5834 
5835     if (s->fd != -1) {
5836         error_setg(errp, "Cannot set properties after the accelerator has been initialized");
5837         return;
5838     }
5839 
5840     s->notify_vmexit = value;
5841 }
5842 
5843 static void kvm_arch_get_notify_window(Object *obj, Visitor *v,
5844                                        const char *name, void *opaque,
5845                                        Error **errp)
5846 {
5847     KVMState *s = KVM_STATE(obj);
5848     uint32_t value = s->notify_window;
5849 
5850     visit_type_uint32(v, name, &value, errp);
5851 }
5852 
5853 static void kvm_arch_set_notify_window(Object *obj, Visitor *v,
5854                                        const char *name, void *opaque,
5855                                        Error **errp)
5856 {
5857     KVMState *s = KVM_STATE(obj);
5858     uint32_t value;
5859 
5860     if (s->fd != -1) {
5861         error_setg(errp, "Cannot set properties after the accelerator has been initialized");
5862         return;
5863     }
5864 
5865     if (!visit_type_uint32(v, name, &value, errp)) {
5866         return;
5867     }
5868 
5869     s->notify_window = value;
5870 }
5871 
5872 static void kvm_arch_get_xen_version(Object *obj, Visitor *v,
5873                                      const char *name, void *opaque,
5874                                      Error **errp)
5875 {
5876     KVMState *s = KVM_STATE(obj);
5877     uint32_t value = s->xen_version;
5878 
5879     visit_type_uint32(v, name, &value, errp);
5880 }
5881 
5882 static void kvm_arch_set_xen_version(Object *obj, Visitor *v,
5883                                      const char *name, void *opaque,
5884                                      Error **errp)
5885 {
5886     KVMState *s = KVM_STATE(obj);
5887     Error *error = NULL;
5888     uint32_t value;
5889 
5890     visit_type_uint32(v, name, &value, &error);
5891     if (error) {
5892         error_propagate(errp, error);
5893         return;
5894     }
5895 
5896     s->xen_version = value;
5897     if (value && xen_mode == XEN_DISABLED) {
5898         xen_mode = XEN_EMULATE;
5899     }
5900 }
5901 
5902 static void kvm_arch_get_xen_gnttab_max_frames(Object *obj, Visitor *v,
5903                                                const char *name, void *opaque,
5904                                                Error **errp)
5905 {
5906     KVMState *s = KVM_STATE(obj);
5907     uint16_t value = s->xen_gnttab_max_frames;
5908 
5909     visit_type_uint16(v, name, &value, errp);
5910 }
5911 
5912 static void kvm_arch_set_xen_gnttab_max_frames(Object *obj, Visitor *v,
5913                                                const char *name, void *opaque,
5914                                                Error **errp)
5915 {
5916     KVMState *s = KVM_STATE(obj);
5917     Error *error = NULL;
5918     uint16_t value;
5919 
5920     visit_type_uint16(v, name, &value, &error);
5921     if (error) {
5922         error_propagate(errp, error);
5923         return;
5924     }
5925 
5926     s->xen_gnttab_max_frames = value;
5927 }
5928 
5929 static void kvm_arch_get_xen_evtchn_max_pirq(Object *obj, Visitor *v,
5930                                              const char *name, void *opaque,
5931                                              Error **errp)
5932 {
5933     KVMState *s = KVM_STATE(obj);
5934     uint16_t value = s->xen_evtchn_max_pirq;
5935 
5936     visit_type_uint16(v, name, &value, errp);
5937 }
5938 
5939 static void kvm_arch_set_xen_evtchn_max_pirq(Object *obj, Visitor *v,
5940                                              const char *name, void *opaque,
5941                                              Error **errp)
5942 {
5943     KVMState *s = KVM_STATE(obj);
5944     Error *error = NULL;
5945     uint16_t value;
5946 
5947     visit_type_uint16(v, name, &value, &error);
5948     if (error) {
5949         error_propagate(errp, error);
5950         return;
5951     }
5952 
5953     s->xen_evtchn_max_pirq = value;
5954 }
5955 
5956 void kvm_arch_accel_class_init(ObjectClass *oc)
5957 {
5958     object_class_property_add_enum(oc, "notify-vmexit", "NotifyVMexitOption",
5959                                    &NotifyVmexitOption_lookup,
5960                                    kvm_arch_get_notify_vmexit,
5961                                    kvm_arch_set_notify_vmexit);
5962     object_class_property_set_description(oc, "notify-vmexit",
5963                                           "Enable notify VM exit");
5964 
5965     object_class_property_add(oc, "notify-window", "uint32",
5966                               kvm_arch_get_notify_window,
5967                               kvm_arch_set_notify_window,
5968                               NULL, NULL);
5969     object_class_property_set_description(oc, "notify-window",
5970                                           "Clock cycles without an event window "
5971                                           "after which a notification VM exit occurs");
5972 
5973     object_class_property_add(oc, "xen-version", "uint32",
5974                               kvm_arch_get_xen_version,
5975                               kvm_arch_set_xen_version,
5976                               NULL, NULL);
5977     object_class_property_set_description(oc, "xen-version",
5978                                           "Xen version to be emulated "
5979                                           "(in XENVER_version form "
5980                                           "e.g. 0x4000a for 4.10)");
5981 
5982     object_class_property_add(oc, "xen-gnttab-max-frames", "uint16",
5983                               kvm_arch_get_xen_gnttab_max_frames,
5984                               kvm_arch_set_xen_gnttab_max_frames,
5985                               NULL, NULL);
5986     object_class_property_set_description(oc, "xen-gnttab-max-frames",
5987                                           "Maximum number of grant table frames");
5988 
5989     object_class_property_add(oc, "xen-evtchn-max-pirq", "uint16",
5990                               kvm_arch_get_xen_evtchn_max_pirq,
5991                               kvm_arch_set_xen_evtchn_max_pirq,
5992                               NULL, NULL);
5993     object_class_property_set_description(oc, "xen-evtchn-max-pirq",
5994                                           "Maximum number of Xen PIRQs");
5995 }
5996 
5997 void kvm_set_max_apic_id(uint32_t max_apic_id)
5998 {
5999     kvm_vm_enable_cap(kvm_state, KVM_CAP_MAX_VCPU_ID, 0, max_apic_id);
6000 }
6001