xref: /openbmc/qemu/target/i386/whpx/whpx-all.c (revision 21063bce)
1 /*
2  * QEMU Windows Hypervisor Platform accelerator (WHPX)
3  *
4  * Copyright Microsoft Corp. 2017
5  *
6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
7  * See the COPYING file in the top-level directory.
8  *
9  */
10 
11 #include "qemu/osdep.h"
12 #include "cpu.h"
13 #include "exec/address-spaces.h"
14 #include "exec/ioport.h"
15 #include "exec/gdbstub.h"
16 #include "qemu/accel.h"
17 #include "sysemu/whpx.h"
18 #include "sysemu/cpus.h"
19 #include "sysemu/runstate.h"
20 #include "qemu/main-loop.h"
21 #include "hw/boards.h"
22 #include "hw/intc/ioapic.h"
23 #include "hw/i386/apic_internal.h"
24 #include "qemu/error-report.h"
25 #include "qapi/error.h"
26 #include "qapi/qapi-types-common.h"
27 #include "qapi/qapi-visit-common.h"
28 #include "migration/blocker.h"
29 #include <winerror.h>
30 
31 #include "whpx-internal.h"
32 #include "whpx-accel-ops.h"
33 
34 #include <WinHvPlatform.h>
35 #include <WinHvEmulation.h>
36 
37 #define HYPERV_APIC_BUS_FREQUENCY      (200000000ULL)
38 
39 static const WHV_REGISTER_NAME whpx_register_names[] = {
40 
41     /* X64 General purpose registers */
42     WHvX64RegisterRax,
43     WHvX64RegisterRcx,
44     WHvX64RegisterRdx,
45     WHvX64RegisterRbx,
46     WHvX64RegisterRsp,
47     WHvX64RegisterRbp,
48     WHvX64RegisterRsi,
49     WHvX64RegisterRdi,
50     WHvX64RegisterR8,
51     WHvX64RegisterR9,
52     WHvX64RegisterR10,
53     WHvX64RegisterR11,
54     WHvX64RegisterR12,
55     WHvX64RegisterR13,
56     WHvX64RegisterR14,
57     WHvX64RegisterR15,
58     WHvX64RegisterRip,
59     WHvX64RegisterRflags,
60 
61     /* X64 Segment registers */
62     WHvX64RegisterEs,
63     WHvX64RegisterCs,
64     WHvX64RegisterSs,
65     WHvX64RegisterDs,
66     WHvX64RegisterFs,
67     WHvX64RegisterGs,
68     WHvX64RegisterLdtr,
69     WHvX64RegisterTr,
70 
71     /* X64 Table registers */
72     WHvX64RegisterIdtr,
73     WHvX64RegisterGdtr,
74 
75     /* X64 Control Registers */
76     WHvX64RegisterCr0,
77     WHvX64RegisterCr2,
78     WHvX64RegisterCr3,
79     WHvX64RegisterCr4,
80     WHvX64RegisterCr8,
81 
82     /* X64 Debug Registers */
83     /*
84      * WHvX64RegisterDr0,
85      * WHvX64RegisterDr1,
86      * WHvX64RegisterDr2,
87      * WHvX64RegisterDr3,
88      * WHvX64RegisterDr6,
89      * WHvX64RegisterDr7,
90      */
91 
92     /* X64 Floating Point and Vector Registers */
93     WHvX64RegisterXmm0,
94     WHvX64RegisterXmm1,
95     WHvX64RegisterXmm2,
96     WHvX64RegisterXmm3,
97     WHvX64RegisterXmm4,
98     WHvX64RegisterXmm5,
99     WHvX64RegisterXmm6,
100     WHvX64RegisterXmm7,
101     WHvX64RegisterXmm8,
102     WHvX64RegisterXmm9,
103     WHvX64RegisterXmm10,
104     WHvX64RegisterXmm11,
105     WHvX64RegisterXmm12,
106     WHvX64RegisterXmm13,
107     WHvX64RegisterXmm14,
108     WHvX64RegisterXmm15,
109     WHvX64RegisterFpMmx0,
110     WHvX64RegisterFpMmx1,
111     WHvX64RegisterFpMmx2,
112     WHvX64RegisterFpMmx3,
113     WHvX64RegisterFpMmx4,
114     WHvX64RegisterFpMmx5,
115     WHvX64RegisterFpMmx6,
116     WHvX64RegisterFpMmx7,
117     WHvX64RegisterFpControlStatus,
118     WHvX64RegisterXmmControlStatus,
119 
120     /* X64 MSRs */
121     WHvX64RegisterEfer,
122 #ifdef TARGET_X86_64
123     WHvX64RegisterKernelGsBase,
124 #endif
125     WHvX64RegisterApicBase,
126     /* WHvX64RegisterPat, */
127     WHvX64RegisterSysenterCs,
128     WHvX64RegisterSysenterEip,
129     WHvX64RegisterSysenterEsp,
130     WHvX64RegisterStar,
131 #ifdef TARGET_X86_64
132     WHvX64RegisterLstar,
133     WHvX64RegisterCstar,
134     WHvX64RegisterSfmask,
135 #endif
136 
137     /* Interrupt / Event Registers */
138     /*
139      * WHvRegisterPendingInterruption,
140      * WHvRegisterInterruptState,
141      * WHvRegisterPendingEvent0,
142      * WHvRegisterPendingEvent1
143      * WHvX64RegisterDeliverabilityNotifications,
144      */
145 };
146 
147 struct whpx_register_set {
148     WHV_REGISTER_VALUE values[RTL_NUMBER_OF(whpx_register_names)];
149 };
150 
151 /*
152  * The current implementation of instruction stepping sets the TF flag
153  * in RFLAGS, causing the CPU to raise an INT1 after each instruction.
154  * This corresponds to the WHvX64ExceptionTypeDebugTrapOrFault exception.
155  *
156  * This approach has a few limitations:
157  *     1. Stepping over a PUSHF/SAHF instruction will save the TF flag
158  *        along with the other flags, possibly restoring it later. It would
159  *        result in another INT1 when the flags are restored, triggering
160  *        a stop in gdb that could be cleared by doing another step.
161  *
162  *        Stepping over a POPF/LAHF instruction will let it overwrite the
163  *        TF flags, ending the stepping mode.
164  *
165  *     2. Stepping over an instruction raising an exception (e.g. INT, DIV,
166  *        or anything that could result in a page fault) will save the flags
167  *        to the stack, clear the TF flag, and let the guest execute the
168  *        handler. Normally, the guest will restore the original flags,
169  *        that will continue single-stepping.
170  *
171  *     3. Debuggers running on the guest may wish to set TF to do instruction
172  *        stepping. INT1 events generated by it would be intercepted by us,
173  *        as long as the gdb is connected to QEMU.
174  *
175  * In practice this means that:
176  *     1. Stepping through flags-modifying instructions may cause gdb to
177  *        continue or stop in unexpected places. This will be fully recoverable
178  *        and will not crash the target.
179  *
180  *     2. Stepping over an instruction that triggers an exception will step
181  *        over the exception handler, not into it.
182  *
183  *     3. Debugging the guest via gdb, while running debugger on the guest
184  *        at the same time may lead to unexpected effects. Removing all
185  *        breakpoints set via QEMU will prevent any further interference
186  *        with the guest-level debuggers.
187  *
188  * The limitations can be addressed as shown below:
189  *     1. PUSHF/SAHF/POPF/LAHF/IRET instructions can be emulated instead of
190  *        stepping through them. The exact semantics of the instructions is
191  *        defined in the "Combined Volume Set of Intel 64 and IA-32
192  *        Architectures Software Developer's Manuals", however it involves a
193  *        fair amount of corner cases due to compatibility with real mode,
194  *        virtual 8086 mode, and differences between 64-bit and 32-bit modes.
195  *
196  *     2. We could step into the guest's exception handlers using the following
197  *        sequence:
198  *          a. Temporarily enable catching of all exception types via
199  *             whpx_set_exception_exit_bitmap().
200  *          b. Once an exception is intercepted, read the IDT/GDT and locate
201  *             the original handler.
202  *          c. Patch the original handler, injecting an INT3 at the beginning.
203  *          d. Update the exception exit bitmap to only catch the
204  *             WHvX64ExceptionTypeBreakpointTrap exception.
205  *          e. Let the affected CPU run in the exclusive mode.
206  *          f. Restore the original handler and the exception exit bitmap.
207  *        Note that handling all corner cases related to IDT/GDT is harder
208  *        than it may seem. See x86_cpu_get_phys_page_attrs_debug() for a
209  *        rough idea.
210  *
211  *     3. In order to properly support guest-level debugging in parallel with
212  *        the QEMU-level debugging, we would need to be able to pass some INT1
213  *        events to the guest. This could be done via the following methods:
214  *          a. Using the WHvRegisterPendingEvent register. As of Windows 21H1,
215  *             it seems to only work for interrupts and not software
216  *             exceptions.
217  *          b. Locating and patching the original handler by parsing IDT/GDT.
218  *             This involves relatively complex logic outlined in the previous
219  *             paragraph.
220  *          c. Emulating the exception invocation (i.e. manually updating RIP,
221  *             RFLAGS, and pushing the old values to stack). This is even more
222  *             complicated than the previous option, since it involves checking
223  *             CPL, gate attributes, and doing various adjustments depending
224  *             on the current CPU mode, whether the CPL is changing, etc.
225  */
226 typedef enum WhpxStepMode {
227     WHPX_STEP_NONE = 0,
228     /* Halt other VCPUs */
229     WHPX_STEP_EXCLUSIVE,
230 } WhpxStepMode;
231 
232 struct whpx_vcpu {
233     WHV_EMULATOR_HANDLE emulator;
234     bool window_registered;
235     bool interruptable;
236     bool ready_for_pic_interrupt;
237     uint64_t tpr;
238     uint64_t apic_base;
239     bool interruption_pending;
240 
241     /* Must be the last field as it may have a tail */
242     WHV_RUN_VP_EXIT_CONTEXT exit_ctx;
243 };
244 
245 static bool whpx_allowed;
246 static bool whp_dispatch_initialized;
247 static HMODULE hWinHvPlatform, hWinHvEmulation;
248 static uint32_t max_vcpu_index;
249 static WHV_PROCESSOR_XSAVE_FEATURES whpx_xsave_cap;
250 
251 struct whpx_state whpx_global;
252 struct WHPDispatch whp_dispatch;
253 
254 static bool whpx_has_xsave(void)
255 {
256     return whpx_xsave_cap.XsaveSupport;
257 }
258 
259 /*
260  * VP support
261  */
262 
263 static struct whpx_vcpu *get_whpx_vcpu(CPUState *cpu)
264 {
265     return (struct whpx_vcpu *)cpu->hax_vcpu;
266 }
267 
268 static WHV_X64_SEGMENT_REGISTER whpx_seg_q2h(const SegmentCache *qs, int v86,
269                                              int r86)
270 {
271     WHV_X64_SEGMENT_REGISTER hs;
272     unsigned flags = qs->flags;
273 
274     hs.Base = qs->base;
275     hs.Limit = qs->limit;
276     hs.Selector = qs->selector;
277 
278     if (v86) {
279         hs.Attributes = 0;
280         hs.SegmentType = 3;
281         hs.Present = 1;
282         hs.DescriptorPrivilegeLevel = 3;
283         hs.NonSystemSegment = 1;
284 
285     } else {
286         hs.Attributes = (flags >> DESC_TYPE_SHIFT);
287 
288         if (r86) {
289             /* hs.Base &= 0xfffff; */
290         }
291     }
292 
293     return hs;
294 }
295 
296 static SegmentCache whpx_seg_h2q(const WHV_X64_SEGMENT_REGISTER *hs)
297 {
298     SegmentCache qs;
299 
300     qs.base = hs->Base;
301     qs.limit = hs->Limit;
302     qs.selector = hs->Selector;
303 
304     qs.flags = ((uint32_t)hs->Attributes) << DESC_TYPE_SHIFT;
305 
306     return qs;
307 }
308 
309 /* X64 Extended Control Registers */
310 static void whpx_set_xcrs(CPUState *cpu)
311 {
312     CPUX86State *env = cpu->env_ptr;
313     HRESULT hr;
314     struct whpx_state *whpx = &whpx_global;
315     WHV_REGISTER_VALUE xcr0;
316     WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
317 
318     if (!whpx_has_xsave()) {
319         return;
320     }
321 
322     /* Only xcr0 is supported by the hypervisor currently */
323     xcr0.Reg64 = env->xcr0;
324     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
325         whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
326     if (FAILED(hr)) {
327         error_report("WHPX: Failed to set register xcr0, hr=%08lx", hr);
328     }
329 }
330 
331 static int whpx_set_tsc(CPUState *cpu)
332 {
333     CPUX86State *env = cpu->env_ptr;
334     WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
335     WHV_REGISTER_VALUE tsc_val;
336     HRESULT hr;
337     struct whpx_state *whpx = &whpx_global;
338 
339     /*
340      * Suspend the partition prior to setting the TSC to reduce the variance
341      * in TSC across vCPUs. When the first vCPU runs post suspend, the
342      * partition is automatically resumed.
343      */
344     if (whp_dispatch.WHvSuspendPartitionTime) {
345 
346         /*
347          * Unable to suspend partition while setting TSC is not a fatal
348          * error. It just increases the likelihood of TSC variance between
349          * vCPUs and some guest OS are able to handle that just fine.
350          */
351         hr = whp_dispatch.WHvSuspendPartitionTime(whpx->partition);
352         if (FAILED(hr)) {
353             warn_report("WHPX: Failed to suspend partition, hr=%08lx", hr);
354         }
355     }
356 
357     tsc_val.Reg64 = env->tsc;
358     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
359         whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
360     if (FAILED(hr)) {
361         error_report("WHPX: Failed to set TSC, hr=%08lx", hr);
362         return -1;
363     }
364 
365     return 0;
366 }
367 
368 /*
369  * The CR8 register in the CPU is mapped to the TPR register of the APIC,
370  * however, they use a slightly different encoding. Specifically:
371  *
372  *     APIC.TPR[bits 7:4] = CR8[bits 3:0]
373  *
374  * This mechanism is described in section 10.8.6.1 of Volume 3 of Intel 64
375  * and IA-32 Architectures Software Developer's Manual.
376  *
377  * The functions below translate the value of CR8 to TPR and vice versa.
378  */
379 
380 static uint64_t whpx_apic_tpr_to_cr8(uint64_t tpr)
381 {
382     return tpr >> 4;
383 }
384 
385 static uint64_t whpx_cr8_to_apic_tpr(uint64_t cr8)
386 {
387     return cr8 << 4;
388 }
389 
390 static void whpx_set_registers(CPUState *cpu, int level)
391 {
392     struct whpx_state *whpx = &whpx_global;
393     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
394     CPUX86State *env = cpu->env_ptr;
395     X86CPU *x86_cpu = X86_CPU(cpu);
396     struct whpx_register_set vcxt;
397     HRESULT hr;
398     int idx;
399     int idx_next;
400     int i;
401     int v86, r86;
402 
403     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
404 
405     /*
406      * Following MSRs have side effects on the guest or are too heavy for
407      * runtime. Limit them to full state update.
408      */
409     if (level >= WHPX_SET_RESET_STATE) {
410         whpx_set_tsc(cpu);
411     }
412 
413     memset(&vcxt, 0, sizeof(struct whpx_register_set));
414 
415     v86 = (env->eflags & VM_MASK);
416     r86 = !(env->cr[0] & CR0_PE_MASK);
417 
418     vcpu->tpr = whpx_apic_tpr_to_cr8(cpu_get_apic_tpr(x86_cpu->apic_state));
419     vcpu->apic_base = cpu_get_apic_base(x86_cpu->apic_state);
420 
421     idx = 0;
422 
423     /* Indexes for first 16 registers match between HV and QEMU definitions */
424     idx_next = 16;
425     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
426         vcxt.values[idx].Reg64 = (uint64_t)env->regs[idx];
427     }
428     idx = idx_next;
429 
430     /* Same goes for RIP and RFLAGS */
431     assert(whpx_register_names[idx] == WHvX64RegisterRip);
432     vcxt.values[idx++].Reg64 = env->eip;
433 
434     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
435     vcxt.values[idx++].Reg64 = env->eflags;
436 
437     /* Translate 6+4 segment registers. HV and QEMU order matches  */
438     assert(idx == WHvX64RegisterEs);
439     for (i = 0; i < 6; i += 1, idx += 1) {
440         vcxt.values[idx].Segment = whpx_seg_q2h(&env->segs[i], v86, r86);
441     }
442 
443     assert(idx == WHvX64RegisterLdtr);
444     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->ldt, 0, 0);
445 
446     assert(idx == WHvX64RegisterTr);
447     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->tr, 0, 0);
448 
449     assert(idx == WHvX64RegisterIdtr);
450     vcxt.values[idx].Table.Base = env->idt.base;
451     vcxt.values[idx].Table.Limit = env->idt.limit;
452     idx += 1;
453 
454     assert(idx == WHvX64RegisterGdtr);
455     vcxt.values[idx].Table.Base = env->gdt.base;
456     vcxt.values[idx].Table.Limit = env->gdt.limit;
457     idx += 1;
458 
459     /* CR0, 2, 3, 4, 8 */
460     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
461     vcxt.values[idx++].Reg64 = env->cr[0];
462     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
463     vcxt.values[idx++].Reg64 = env->cr[2];
464     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
465     vcxt.values[idx++].Reg64 = env->cr[3];
466     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
467     vcxt.values[idx++].Reg64 = env->cr[4];
468     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
469     vcxt.values[idx++].Reg64 = vcpu->tpr;
470 
471     /* 8 Debug Registers - Skipped */
472 
473     /*
474      * Extended control registers needs to be handled separately depending
475      * on whether xsave is supported/enabled or not.
476      */
477     whpx_set_xcrs(cpu);
478 
479     /* 16 XMM registers */
480     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
481     idx_next = idx + 16;
482     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
483         vcxt.values[idx].Reg128.Low64 = env->xmm_regs[i].ZMM_Q(0);
484         vcxt.values[idx].Reg128.High64 = env->xmm_regs[i].ZMM_Q(1);
485     }
486     idx = idx_next;
487 
488     /* 8 FP registers */
489     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
490     for (i = 0; i < 8; i += 1, idx += 1) {
491         vcxt.values[idx].Fp.AsUINT128.Low64 = env->fpregs[i].mmx.MMX_Q(0);
492         /* vcxt.values[idx].Fp.AsUINT128.High64 =
493                env->fpregs[i].mmx.MMX_Q(1);
494         */
495     }
496 
497     /* FP control status register */
498     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
499     vcxt.values[idx].FpControlStatus.FpControl = env->fpuc;
500     vcxt.values[idx].FpControlStatus.FpStatus =
501         (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
502     vcxt.values[idx].FpControlStatus.FpTag = 0;
503     for (i = 0; i < 8; ++i) {
504         vcxt.values[idx].FpControlStatus.FpTag |= (!env->fptags[i]) << i;
505     }
506     vcxt.values[idx].FpControlStatus.Reserved = 0;
507     vcxt.values[idx].FpControlStatus.LastFpOp = env->fpop;
508     vcxt.values[idx].FpControlStatus.LastFpRip = env->fpip;
509     idx += 1;
510 
511     /* XMM control status register */
512     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
513     vcxt.values[idx].XmmControlStatus.LastFpRdp = 0;
514     vcxt.values[idx].XmmControlStatus.XmmStatusControl = env->mxcsr;
515     vcxt.values[idx].XmmControlStatus.XmmStatusControlMask = 0x0000ffff;
516     idx += 1;
517 
518     /* MSRs */
519     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
520     vcxt.values[idx++].Reg64 = env->efer;
521 #ifdef TARGET_X86_64
522     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
523     vcxt.values[idx++].Reg64 = env->kernelgsbase;
524 #endif
525 
526     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
527     vcxt.values[idx++].Reg64 = vcpu->apic_base;
528 
529     /* WHvX64RegisterPat - Skipped */
530 
531     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
532     vcxt.values[idx++].Reg64 = env->sysenter_cs;
533     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
534     vcxt.values[idx++].Reg64 = env->sysenter_eip;
535     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
536     vcxt.values[idx++].Reg64 = env->sysenter_esp;
537     assert(whpx_register_names[idx] == WHvX64RegisterStar);
538     vcxt.values[idx++].Reg64 = env->star;
539 #ifdef TARGET_X86_64
540     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
541     vcxt.values[idx++].Reg64 = env->lstar;
542     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
543     vcxt.values[idx++].Reg64 = env->cstar;
544     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
545     vcxt.values[idx++].Reg64 = env->fmask;
546 #endif
547 
548     /* Interrupt / Event Registers - Skipped */
549 
550     assert(idx == RTL_NUMBER_OF(whpx_register_names));
551 
552     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
553         whpx->partition, cpu->cpu_index,
554         whpx_register_names,
555         RTL_NUMBER_OF(whpx_register_names),
556         &vcxt.values[0]);
557 
558     if (FAILED(hr)) {
559         error_report("WHPX: Failed to set virtual processor context, hr=%08lx",
560                      hr);
561     }
562 
563     return;
564 }
565 
566 static int whpx_get_tsc(CPUState *cpu)
567 {
568     CPUX86State *env = cpu->env_ptr;
569     WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
570     WHV_REGISTER_VALUE tsc_val;
571     HRESULT hr;
572     struct whpx_state *whpx = &whpx_global;
573 
574     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
575         whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
576     if (FAILED(hr)) {
577         error_report("WHPX: Failed to get TSC, hr=%08lx", hr);
578         return -1;
579     }
580 
581     env->tsc = tsc_val.Reg64;
582     return 0;
583 }
584 
585 /* X64 Extended Control Registers */
586 static void whpx_get_xcrs(CPUState *cpu)
587 {
588     CPUX86State *env = cpu->env_ptr;
589     HRESULT hr;
590     struct whpx_state *whpx = &whpx_global;
591     WHV_REGISTER_VALUE xcr0;
592     WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
593 
594     if (!whpx_has_xsave()) {
595         return;
596     }
597 
598     /* Only xcr0 is supported by the hypervisor currently */
599     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
600         whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
601     if (FAILED(hr)) {
602         error_report("WHPX: Failed to get register xcr0, hr=%08lx", hr);
603         return;
604     }
605 
606     env->xcr0 = xcr0.Reg64;
607 }
608 
609 static void whpx_get_registers(CPUState *cpu)
610 {
611     struct whpx_state *whpx = &whpx_global;
612     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
613     CPUX86State *env = cpu->env_ptr;
614     X86CPU *x86_cpu = X86_CPU(cpu);
615     struct whpx_register_set vcxt;
616     uint64_t tpr, apic_base;
617     HRESULT hr;
618     int idx;
619     int idx_next;
620     int i;
621 
622     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
623 
624     if (!env->tsc_valid) {
625         whpx_get_tsc(cpu);
626         env->tsc_valid = !runstate_is_running();
627     }
628 
629     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
630         whpx->partition, cpu->cpu_index,
631         whpx_register_names,
632         RTL_NUMBER_OF(whpx_register_names),
633         &vcxt.values[0]);
634     if (FAILED(hr)) {
635         error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
636                      hr);
637     }
638 
639     if (whpx_apic_in_platform()) {
640         /*
641          * Fetch the TPR value from the emulated APIC. It may get overwritten
642          * below with the value from CR8 returned by
643          * WHvGetVirtualProcessorRegisters().
644          */
645         whpx_apic_get(x86_cpu->apic_state);
646         vcpu->tpr = whpx_apic_tpr_to_cr8(
647             cpu_get_apic_tpr(x86_cpu->apic_state));
648     }
649 
650     idx = 0;
651 
652     /* Indexes for first 16 registers match between HV and QEMU definitions */
653     idx_next = 16;
654     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
655         env->regs[idx] = vcxt.values[idx].Reg64;
656     }
657     idx = idx_next;
658 
659     /* Same goes for RIP and RFLAGS */
660     assert(whpx_register_names[idx] == WHvX64RegisterRip);
661     env->eip = vcxt.values[idx++].Reg64;
662     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
663     env->eflags = vcxt.values[idx++].Reg64;
664 
665     /* Translate 6+4 segment registers. HV and QEMU order matches  */
666     assert(idx == WHvX64RegisterEs);
667     for (i = 0; i < 6; i += 1, idx += 1) {
668         env->segs[i] = whpx_seg_h2q(&vcxt.values[idx].Segment);
669     }
670 
671     assert(idx == WHvX64RegisterLdtr);
672     env->ldt = whpx_seg_h2q(&vcxt.values[idx++].Segment);
673     assert(idx == WHvX64RegisterTr);
674     env->tr = whpx_seg_h2q(&vcxt.values[idx++].Segment);
675     assert(idx == WHvX64RegisterIdtr);
676     env->idt.base = vcxt.values[idx].Table.Base;
677     env->idt.limit = vcxt.values[idx].Table.Limit;
678     idx += 1;
679     assert(idx == WHvX64RegisterGdtr);
680     env->gdt.base = vcxt.values[idx].Table.Base;
681     env->gdt.limit = vcxt.values[idx].Table.Limit;
682     idx += 1;
683 
684     /* CR0, 2, 3, 4, 8 */
685     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
686     env->cr[0] = vcxt.values[idx++].Reg64;
687     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
688     env->cr[2] = vcxt.values[idx++].Reg64;
689     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
690     env->cr[3] = vcxt.values[idx++].Reg64;
691     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
692     env->cr[4] = vcxt.values[idx++].Reg64;
693     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
694     tpr = vcxt.values[idx++].Reg64;
695     if (tpr != vcpu->tpr) {
696         vcpu->tpr = tpr;
697         cpu_set_apic_tpr(x86_cpu->apic_state, whpx_cr8_to_apic_tpr(tpr));
698     }
699 
700     /* 8 Debug Registers - Skipped */
701 
702     /*
703      * Extended control registers needs to be handled separately depending
704      * on whether xsave is supported/enabled or not.
705      */
706     whpx_get_xcrs(cpu);
707 
708     /* 16 XMM registers */
709     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
710     idx_next = idx + 16;
711     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
712         env->xmm_regs[i].ZMM_Q(0) = vcxt.values[idx].Reg128.Low64;
713         env->xmm_regs[i].ZMM_Q(1) = vcxt.values[idx].Reg128.High64;
714     }
715     idx = idx_next;
716 
717     /* 8 FP registers */
718     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
719     for (i = 0; i < 8; i += 1, idx += 1) {
720         env->fpregs[i].mmx.MMX_Q(0) = vcxt.values[idx].Fp.AsUINT128.Low64;
721         /* env->fpregs[i].mmx.MMX_Q(1) =
722                vcxt.values[idx].Fp.AsUINT128.High64;
723         */
724     }
725 
726     /* FP control status register */
727     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
728     env->fpuc = vcxt.values[idx].FpControlStatus.FpControl;
729     env->fpstt = (vcxt.values[idx].FpControlStatus.FpStatus >> 11) & 0x7;
730     env->fpus = vcxt.values[idx].FpControlStatus.FpStatus & ~0x3800;
731     for (i = 0; i < 8; ++i) {
732         env->fptags[i] = !((vcxt.values[idx].FpControlStatus.FpTag >> i) & 1);
733     }
734     env->fpop = vcxt.values[idx].FpControlStatus.LastFpOp;
735     env->fpip = vcxt.values[idx].FpControlStatus.LastFpRip;
736     idx += 1;
737 
738     /* XMM control status register */
739     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
740     env->mxcsr = vcxt.values[idx].XmmControlStatus.XmmStatusControl;
741     idx += 1;
742 
743     /* MSRs */
744     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
745     env->efer = vcxt.values[idx++].Reg64;
746 #ifdef TARGET_X86_64
747     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
748     env->kernelgsbase = vcxt.values[idx++].Reg64;
749 #endif
750 
751     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
752     apic_base = vcxt.values[idx++].Reg64;
753     if (apic_base != vcpu->apic_base) {
754         vcpu->apic_base = apic_base;
755         cpu_set_apic_base(x86_cpu->apic_state, vcpu->apic_base);
756     }
757 
758     /* WHvX64RegisterPat - Skipped */
759 
760     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
761     env->sysenter_cs = vcxt.values[idx++].Reg64;
762     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
763     env->sysenter_eip = vcxt.values[idx++].Reg64;
764     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
765     env->sysenter_esp = vcxt.values[idx++].Reg64;
766     assert(whpx_register_names[idx] == WHvX64RegisterStar);
767     env->star = vcxt.values[idx++].Reg64;
768 #ifdef TARGET_X86_64
769     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
770     env->lstar = vcxt.values[idx++].Reg64;
771     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
772     env->cstar = vcxt.values[idx++].Reg64;
773     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
774     env->fmask = vcxt.values[idx++].Reg64;
775 #endif
776 
777     /* Interrupt / Event Registers - Skipped */
778 
779     assert(idx == RTL_NUMBER_OF(whpx_register_names));
780 
781     if (whpx_apic_in_platform()) {
782         whpx_apic_get(x86_cpu->apic_state);
783     }
784 
785     x86_update_hflags(env);
786 
787     return;
788 }
789 
790 static HRESULT CALLBACK whpx_emu_ioport_callback(
791     void *ctx,
792     WHV_EMULATOR_IO_ACCESS_INFO *IoAccess)
793 {
794     MemTxAttrs attrs = { 0 };
795     address_space_rw(&address_space_io, IoAccess->Port, attrs,
796                      &IoAccess->Data, IoAccess->AccessSize,
797                      IoAccess->Direction);
798     return S_OK;
799 }
800 
801 static HRESULT CALLBACK whpx_emu_mmio_callback(
802     void *ctx,
803     WHV_EMULATOR_MEMORY_ACCESS_INFO *ma)
804 {
805     cpu_physical_memory_rw(ma->GpaAddress, ma->Data, ma->AccessSize,
806                            ma->Direction);
807     return S_OK;
808 }
809 
810 static HRESULT CALLBACK whpx_emu_getreg_callback(
811     void *ctx,
812     const WHV_REGISTER_NAME *RegisterNames,
813     UINT32 RegisterCount,
814     WHV_REGISTER_VALUE *RegisterValues)
815 {
816     HRESULT hr;
817     struct whpx_state *whpx = &whpx_global;
818     CPUState *cpu = (CPUState *)ctx;
819 
820     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
821         whpx->partition, cpu->cpu_index,
822         RegisterNames, RegisterCount,
823         RegisterValues);
824     if (FAILED(hr)) {
825         error_report("WHPX: Failed to get virtual processor registers,"
826                      " hr=%08lx", hr);
827     }
828 
829     return hr;
830 }
831 
832 static HRESULT CALLBACK whpx_emu_setreg_callback(
833     void *ctx,
834     const WHV_REGISTER_NAME *RegisterNames,
835     UINT32 RegisterCount,
836     const WHV_REGISTER_VALUE *RegisterValues)
837 {
838     HRESULT hr;
839     struct whpx_state *whpx = &whpx_global;
840     CPUState *cpu = (CPUState *)ctx;
841 
842     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
843         whpx->partition, cpu->cpu_index,
844         RegisterNames, RegisterCount,
845         RegisterValues);
846     if (FAILED(hr)) {
847         error_report("WHPX: Failed to set virtual processor registers,"
848                      " hr=%08lx", hr);
849     }
850 
851     /*
852      * The emulator just successfully wrote the register state. We clear the
853      * dirty state so we avoid the double write on resume of the VP.
854      */
855     cpu->vcpu_dirty = false;
856 
857     return hr;
858 }
859 
860 static HRESULT CALLBACK whpx_emu_translate_callback(
861     void *ctx,
862     WHV_GUEST_VIRTUAL_ADDRESS Gva,
863     WHV_TRANSLATE_GVA_FLAGS TranslateFlags,
864     WHV_TRANSLATE_GVA_RESULT_CODE *TranslationResult,
865     WHV_GUEST_PHYSICAL_ADDRESS *Gpa)
866 {
867     HRESULT hr;
868     struct whpx_state *whpx = &whpx_global;
869     CPUState *cpu = (CPUState *)ctx;
870     WHV_TRANSLATE_GVA_RESULT res;
871 
872     hr = whp_dispatch.WHvTranslateGva(whpx->partition, cpu->cpu_index,
873                                       Gva, TranslateFlags, &res, Gpa);
874     if (FAILED(hr)) {
875         error_report("WHPX: Failed to translate GVA, hr=%08lx", hr);
876     } else {
877         *TranslationResult = res.ResultCode;
878     }
879 
880     return hr;
881 }
882 
883 static const WHV_EMULATOR_CALLBACKS whpx_emu_callbacks = {
884     .Size = sizeof(WHV_EMULATOR_CALLBACKS),
885     .WHvEmulatorIoPortCallback = whpx_emu_ioport_callback,
886     .WHvEmulatorMemoryCallback = whpx_emu_mmio_callback,
887     .WHvEmulatorGetVirtualProcessorRegisters = whpx_emu_getreg_callback,
888     .WHvEmulatorSetVirtualProcessorRegisters = whpx_emu_setreg_callback,
889     .WHvEmulatorTranslateGvaPage = whpx_emu_translate_callback,
890 };
891 
892 static int whpx_handle_mmio(CPUState *cpu, WHV_MEMORY_ACCESS_CONTEXT *ctx)
893 {
894     HRESULT hr;
895     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
896     WHV_EMULATOR_STATUS emu_status;
897 
898     hr = whp_dispatch.WHvEmulatorTryMmioEmulation(
899         vcpu->emulator, cpu,
900         &vcpu->exit_ctx.VpContext, ctx,
901         &emu_status);
902     if (FAILED(hr)) {
903         error_report("WHPX: Failed to parse MMIO access, hr=%08lx", hr);
904         return -1;
905     }
906 
907     if (!emu_status.EmulationSuccessful) {
908         error_report("WHPX: Failed to emulate MMIO access with"
909                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
910         return -1;
911     }
912 
913     return 0;
914 }
915 
916 static int whpx_handle_portio(CPUState *cpu,
917                               WHV_X64_IO_PORT_ACCESS_CONTEXT *ctx)
918 {
919     HRESULT hr;
920     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
921     WHV_EMULATOR_STATUS emu_status;
922 
923     hr = whp_dispatch.WHvEmulatorTryIoEmulation(
924         vcpu->emulator, cpu,
925         &vcpu->exit_ctx.VpContext, ctx,
926         &emu_status);
927     if (FAILED(hr)) {
928         error_report("WHPX: Failed to parse PortIO access, hr=%08lx", hr);
929         return -1;
930     }
931 
932     if (!emu_status.EmulationSuccessful) {
933         error_report("WHPX: Failed to emulate PortIO access with"
934                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
935         return -1;
936     }
937 
938     return 0;
939 }
940 
941 /*
942  * Controls whether we should intercept various exceptions on the guest,
943  * namely breakpoint/single-step events.
944  *
945  * The 'exceptions' argument accepts a bitmask, e.g:
946  * (1 << WHvX64ExceptionTypeDebugTrapOrFault) | (...)
947  */
948 static HRESULT whpx_set_exception_exit_bitmap(UINT64 exceptions)
949 {
950     struct whpx_state *whpx = &whpx_global;
951     WHV_PARTITION_PROPERTY prop = { 0, };
952     HRESULT hr;
953 
954     if (exceptions == whpx->exception_exit_bitmap) {
955         return S_OK;
956     }
957 
958     prop.ExceptionExitBitmap = exceptions;
959 
960     hr = whp_dispatch.WHvSetPartitionProperty(
961         whpx->partition,
962         WHvPartitionPropertyCodeExceptionExitBitmap,
963         &prop,
964         sizeof(WHV_PARTITION_PROPERTY));
965 
966     if (SUCCEEDED(hr)) {
967         whpx->exception_exit_bitmap = exceptions;
968     }
969 
970     return hr;
971 }
972 
973 
974 /*
975  * This function is called before/after stepping over a single instruction.
976  * It will update the CPU registers to arm/disarm the instruction stepping
977  * accordingly.
978  */
979 static HRESULT whpx_vcpu_configure_single_stepping(CPUState *cpu,
980     bool set,
981     uint64_t *exit_context_rflags)
982 {
983     WHV_REGISTER_NAME reg_name;
984     WHV_REGISTER_VALUE reg_value;
985     HRESULT hr;
986     struct whpx_state *whpx = &whpx_global;
987 
988     /*
989      * If we are trying to step over a single instruction, we need to set the
990      * TF bit in rflags. Otherwise, clear it.
991      */
992     reg_name = WHvX64RegisterRflags;
993     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
994         whpx->partition,
995         cpu->cpu_index,
996         &reg_name,
997         1,
998         &reg_value);
999 
1000     if (FAILED(hr)) {
1001         error_report("WHPX: Failed to get rflags, hr=%08lx", hr);
1002         return hr;
1003     }
1004 
1005     if (exit_context_rflags) {
1006         assert(*exit_context_rflags == reg_value.Reg64);
1007     }
1008 
1009     if (set) {
1010         /* Raise WHvX64ExceptionTypeDebugTrapOrFault after each instruction */
1011         reg_value.Reg64 |= TF_MASK;
1012     } else {
1013         reg_value.Reg64 &= ~TF_MASK;
1014     }
1015 
1016     if (exit_context_rflags) {
1017         *exit_context_rflags = reg_value.Reg64;
1018     }
1019 
1020     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1021         whpx->partition,
1022         cpu->cpu_index,
1023         &reg_name,
1024         1,
1025         &reg_value);
1026 
1027     if (FAILED(hr)) {
1028         error_report("WHPX: Failed to set rflags,"
1029             " hr=%08lx",
1030             hr);
1031         return hr;
1032     }
1033 
1034     reg_name = WHvRegisterInterruptState;
1035     reg_value.Reg64 = 0;
1036 
1037     /* Suspend delivery of hardware interrupts during single-stepping. */
1038     reg_value.InterruptState.InterruptShadow = set != 0;
1039 
1040     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1041     whpx->partition,
1042         cpu->cpu_index,
1043         &reg_name,
1044         1,
1045         &reg_value);
1046 
1047     if (FAILED(hr)) {
1048         error_report("WHPX: Failed to set InterruptState,"
1049             " hr=%08lx",
1050             hr);
1051         return hr;
1052     }
1053 
1054     if (!set) {
1055         /*
1056          * We have just finished stepping over a single instruction,
1057          * and intercepted the INT1 generated by it.
1058          * We need to now hide the INT1 from the guest,
1059          * as it would not be expecting it.
1060          */
1061 
1062         reg_name = WHvX64RegisterPendingDebugException;
1063         hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
1064         whpx->partition,
1065             cpu->cpu_index,
1066             &reg_name,
1067             1,
1068             &reg_value);
1069 
1070         if (FAILED(hr)) {
1071             error_report("WHPX: Failed to get pending debug exceptions,"
1072                          "hr=%08lx", hr);
1073             return hr;
1074         }
1075 
1076         if (reg_value.PendingDebugException.SingleStep) {
1077             reg_value.PendingDebugException.SingleStep = 0;
1078 
1079             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1080                 whpx->partition,
1081                 cpu->cpu_index,
1082                 &reg_name,
1083                 1,
1084                 &reg_value);
1085 
1086             if (FAILED(hr)) {
1087                 error_report("WHPX: Failed to clear pending debug exceptions,"
1088                              "hr=%08lx", hr);
1089              return hr;
1090             }
1091         }
1092 
1093     }
1094 
1095     return S_OK;
1096 }
1097 
1098 /* Tries to find a breakpoint at the specified address. */
1099 static struct whpx_breakpoint *whpx_lookup_breakpoint_by_addr(uint64_t address)
1100 {
1101     struct whpx_state *whpx = &whpx_global;
1102     int i;
1103 
1104     if (whpx->breakpoints.breakpoints) {
1105         for (i = 0; i < whpx->breakpoints.breakpoints->used; i++) {
1106             if (address == whpx->breakpoints.breakpoints->data[i].address) {
1107                 return &whpx->breakpoints.breakpoints->data[i];
1108             }
1109         }
1110     }
1111 
1112     return NULL;
1113 }
1114 
1115 /*
1116  * Linux uses int3 (0xCC) during startup (see int3_selftest()) and for
1117  * debugging user-mode applications. Since the WHPX API does not offer
1118  * an easy way to pass the intercepted exception back to the guest, we
1119  * resort to using INT1 instead, and let the guest always handle INT3.
1120  */
1121 static const uint8_t whpx_breakpoint_instruction = 0xF1;
1122 
1123 /*
1124  * The WHPX QEMU backend implements breakpoints by writing the INT1
1125  * instruction into memory (ignoring the DRx registers). This raises a few
1126  * issues that need to be carefully handled:
1127  *
1128  * 1. Although unlikely, other parts of QEMU may set multiple breakpoints
1129  *    at the same location, and later remove them in arbitrary order.
1130  *    This should not cause memory corruption, and should only remove the
1131  *    physical breakpoint instruction when the last QEMU breakpoint is gone.
1132  *
1133  * 2. Writing arbitrary virtual memory may fail if it's not mapped to a valid
1134  *    physical location. Hence, physically adding/removing a breakpoint can
1135  *    theoretically fail at any time. We need to keep track of it.
1136  *
1137  * The function below rebuilds a list of low-level breakpoints (one per
1138  * address, tracking the original instruction and any errors) from the list of
1139  * high-level breakpoints (set via cpu_breakpoint_insert()).
1140  *
1141  * In order to optimize performance, this function stores the list of
1142  * high-level breakpoints (a.k.a. CPU breakpoints) used to compute the
1143  * low-level ones, so that it won't be re-invoked until these breakpoints
1144  * change.
1145  *
1146  * Note that this function decides which breakpoints should be inserted into,
1147  * memory, but doesn't actually do it. The memory accessing is done in
1148  * whpx_apply_breakpoints().
1149  */
1150 static void whpx_translate_cpu_breakpoints(
1151     struct whpx_breakpoints *breakpoints,
1152     CPUState *cpu,
1153     int cpu_breakpoint_count)
1154 {
1155     CPUBreakpoint *bp;
1156     int cpu_bp_index = 0;
1157 
1158     breakpoints->original_addresses =
1159         g_renew(vaddr, breakpoints->original_addresses, cpu_breakpoint_count);
1160 
1161     breakpoints->original_address_count = cpu_breakpoint_count;
1162 
1163     int max_breakpoints = cpu_breakpoint_count +
1164         (breakpoints->breakpoints ? breakpoints->breakpoints->used : 0);
1165 
1166     struct whpx_breakpoint_collection *new_breakpoints =
1167         g_malloc0(sizeof(struct whpx_breakpoint_collection)
1168                   + max_breakpoints * sizeof(struct whpx_breakpoint));
1169 
1170     new_breakpoints->allocated = max_breakpoints;
1171     new_breakpoints->used = 0;
1172 
1173     /*
1174      * 1. Preserve all old breakpoints that could not be automatically
1175      * cleared when the CPU got stopped.
1176      */
1177     if (breakpoints->breakpoints) {
1178         int i;
1179         for (i = 0; i < breakpoints->breakpoints->used; i++) {
1180             if (breakpoints->breakpoints->data[i].state != WHPX_BP_CLEARED) {
1181                 new_breakpoints->data[new_breakpoints->used++] =
1182                     breakpoints->breakpoints->data[i];
1183             }
1184         }
1185     }
1186 
1187     /* 2. Map all CPU breakpoints to WHPX breakpoints */
1188     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
1189         int i;
1190         bool found = false;
1191 
1192         /* This will be used to detect changed CPU breakpoints later. */
1193         breakpoints->original_addresses[cpu_bp_index++] = bp->pc;
1194 
1195         for (i = 0; i < new_breakpoints->used; i++) {
1196             /*
1197              * WARNING: This loop has O(N^2) complexity, where N is the
1198              * number of breakpoints. It should not be a bottleneck in
1199              * real-world scenarios, since it only needs to run once after
1200              * the breakpoints have been modified.
1201              * If this ever becomes a concern, it can be optimized by storing
1202              * high-level breakpoint objects in a tree or hash map.
1203              */
1204 
1205             if (new_breakpoints->data[i].address == bp->pc) {
1206                 /* There was already a breakpoint at this address. */
1207                 if (new_breakpoints->data[i].state == WHPX_BP_CLEAR_PENDING) {
1208                     new_breakpoints->data[i].state = WHPX_BP_SET;
1209                 } else if (new_breakpoints->data[i].state == WHPX_BP_SET) {
1210                     new_breakpoints->data[i].state = WHPX_BP_SET_PENDING;
1211                 }
1212 
1213                 found = true;
1214                 break;
1215             }
1216         }
1217 
1218         if (!found && new_breakpoints->used < new_breakpoints->allocated) {
1219             /* No WHPX breakpoint at this address. Create one. */
1220             new_breakpoints->data[new_breakpoints->used].address = bp->pc;
1221             new_breakpoints->data[new_breakpoints->used].state =
1222                 WHPX_BP_SET_PENDING;
1223             new_breakpoints->used++;
1224         }
1225     }
1226 
1227     /*
1228      * Free the previous breakpoint list. This can be optimized by keeping
1229      * it as shadow buffer for the next computation instead of freeing
1230      * it immediately.
1231      */
1232     g_free(breakpoints->breakpoints);
1233 
1234     breakpoints->breakpoints = new_breakpoints;
1235 }
1236 
1237 /*
1238  * Physically inserts/removes the breakpoints by reading and writing the
1239  * physical memory, keeping a track of the failed attempts.
1240  *
1241  * Passing resuming=true  will try to set all previously unset breakpoints.
1242  * Passing resuming=false will remove all inserted ones.
1243  */
1244 static void whpx_apply_breakpoints(
1245     struct whpx_breakpoint_collection *breakpoints,
1246     CPUState *cpu,
1247     bool resuming)
1248 {
1249     int i, rc;
1250     if (!breakpoints) {
1251         return;
1252     }
1253 
1254     for (i = 0; i < breakpoints->used; i++) {
1255         /* Decide what to do right now based on the last known state. */
1256         WhpxBreakpointState state = breakpoints->data[i].state;
1257         switch (state) {
1258         case WHPX_BP_CLEARED:
1259             if (resuming) {
1260                 state = WHPX_BP_SET_PENDING;
1261             }
1262             break;
1263         case WHPX_BP_SET_PENDING:
1264             if (!resuming) {
1265                 state = WHPX_BP_CLEARED;
1266             }
1267             break;
1268         case WHPX_BP_SET:
1269             if (!resuming) {
1270                 state = WHPX_BP_CLEAR_PENDING;
1271             }
1272             break;
1273         case WHPX_BP_CLEAR_PENDING:
1274             if (resuming) {
1275                 state = WHPX_BP_SET;
1276             }
1277             break;
1278         }
1279 
1280         if (state == WHPX_BP_SET_PENDING) {
1281             /* Remember the original instruction. */
1282             rc = cpu_memory_rw_debug(cpu,
1283                 breakpoints->data[i].address,
1284                 &breakpoints->data[i].original_instruction,
1285                 1,
1286                 false);
1287 
1288             if (!rc) {
1289                 /* Write the breakpoint instruction. */
1290                 rc = cpu_memory_rw_debug(cpu,
1291                     breakpoints->data[i].address,
1292                     (void *)&whpx_breakpoint_instruction,
1293                     1,
1294                     true);
1295             }
1296 
1297             if (!rc) {
1298                 state = WHPX_BP_SET;
1299             }
1300 
1301         }
1302 
1303         if (state == WHPX_BP_CLEAR_PENDING) {
1304             /* Restore the original instruction. */
1305             rc = cpu_memory_rw_debug(cpu,
1306                 breakpoints->data[i].address,
1307                 &breakpoints->data[i].original_instruction,
1308                 1,
1309                 true);
1310 
1311             if (!rc) {
1312                 state = WHPX_BP_CLEARED;
1313             }
1314         }
1315 
1316         breakpoints->data[i].state = state;
1317     }
1318 }
1319 
1320 /*
1321  * This function is called when the a VCPU is about to start and no other
1322  * VCPUs have been started so far. Since the VCPU start order could be
1323  * arbitrary, it doesn't have to be VCPU#0.
1324  *
1325  * It is used to commit the breakpoints into memory, and configure WHPX
1326  * to intercept debug exceptions.
1327  *
1328  * Note that whpx_set_exception_exit_bitmap() cannot be called if one or
1329  * more VCPUs are already running, so this is the best place to do it.
1330  */
1331 static int whpx_first_vcpu_starting(CPUState *cpu)
1332 {
1333     struct whpx_state *whpx = &whpx_global;
1334     HRESULT hr;
1335 
1336     g_assert(qemu_mutex_iothread_locked());
1337 
1338     if (!QTAILQ_EMPTY(&cpu->breakpoints) ||
1339             (whpx->breakpoints.breakpoints &&
1340              whpx->breakpoints.breakpoints->used)) {
1341         CPUBreakpoint *bp;
1342         int i = 0;
1343         bool update_pending = false;
1344 
1345         QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
1346             if (i >= whpx->breakpoints.original_address_count ||
1347                 bp->pc != whpx->breakpoints.original_addresses[i]) {
1348                 update_pending = true;
1349             }
1350 
1351             i++;
1352         }
1353 
1354         if (i != whpx->breakpoints.original_address_count) {
1355             update_pending = true;
1356         }
1357 
1358         if (update_pending) {
1359             /*
1360              * The CPU breakpoints have changed since the last call to
1361              * whpx_translate_cpu_breakpoints(). WHPX breakpoints must
1362              * now be recomputed.
1363              */
1364             whpx_translate_cpu_breakpoints(&whpx->breakpoints, cpu, i);
1365         }
1366 
1367         /* Actually insert the breakpoints into the memory. */
1368         whpx_apply_breakpoints(whpx->breakpoints.breakpoints, cpu, true);
1369     }
1370 
1371     uint64_t exception_mask;
1372     if (whpx->step_pending ||
1373         (whpx->breakpoints.breakpoints &&
1374          whpx->breakpoints.breakpoints->used)) {
1375         /*
1376          * We are either attempting to single-step one or more CPUs, or
1377          * have one or more breakpoints enabled. Both require intercepting
1378          * the WHvX64ExceptionTypeBreakpointTrap exception.
1379          */
1380 
1381         exception_mask = 1UL << WHvX64ExceptionTypeDebugTrapOrFault;
1382     } else {
1383         /* Let the guest handle all exceptions. */
1384         exception_mask = 0;
1385     }
1386 
1387     hr = whpx_set_exception_exit_bitmap(exception_mask);
1388     if (!SUCCEEDED(hr)) {
1389         error_report("WHPX: Failed to update exception exit mask,"
1390                      "hr=%08lx.", hr);
1391         return 1;
1392     }
1393 
1394     return 0;
1395 }
1396 
1397 /*
1398  * This function is called when the last VCPU has finished running.
1399  * It is used to remove any previously set breakpoints from memory.
1400  */
1401 static int whpx_last_vcpu_stopping(CPUState *cpu)
1402 {
1403     whpx_apply_breakpoints(whpx_global.breakpoints.breakpoints, cpu, false);
1404     return 0;
1405 }
1406 
1407 /* Returns the address of the next instruction that is about to be executed. */
1408 static vaddr whpx_vcpu_get_pc(CPUState *cpu, bool exit_context_valid)
1409 {
1410     if (cpu->vcpu_dirty) {
1411         /* The CPU registers have been modified by other parts of QEMU. */
1412         CPUArchState *env = (CPUArchState *)(cpu->env_ptr);
1413         return env->eip;
1414     } else if (exit_context_valid) {
1415         /*
1416          * The CPU registers have not been modified by neither other parts
1417          * of QEMU, nor this port by calling WHvSetVirtualProcessorRegisters().
1418          * This is the most common case.
1419          */
1420         struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
1421         return vcpu->exit_ctx.VpContext.Rip;
1422     } else {
1423         /*
1424          * The CPU registers have been modified by a call to
1425          * WHvSetVirtualProcessorRegisters() and must be re-queried from
1426          * the target.
1427          */
1428         WHV_REGISTER_VALUE reg_value;
1429         WHV_REGISTER_NAME reg_name = WHvX64RegisterRip;
1430         HRESULT hr;
1431         struct whpx_state *whpx = &whpx_global;
1432 
1433         hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
1434             whpx->partition,
1435             cpu->cpu_index,
1436             &reg_name,
1437             1,
1438             &reg_value);
1439 
1440         if (FAILED(hr)) {
1441             error_report("WHPX: Failed to get PC, hr=%08lx", hr);
1442             return 0;
1443         }
1444 
1445         return reg_value.Reg64;
1446     }
1447 }
1448 
1449 static int whpx_handle_halt(CPUState *cpu)
1450 {
1451     CPUX86State *env = cpu->env_ptr;
1452     int ret = 0;
1453 
1454     qemu_mutex_lock_iothread();
1455     if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
1456           (env->eflags & IF_MASK)) &&
1457         !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
1458         cpu->exception_index = EXCP_HLT;
1459         cpu->halted = true;
1460         ret = 1;
1461     }
1462     qemu_mutex_unlock_iothread();
1463 
1464     return ret;
1465 }
1466 
1467 static void whpx_vcpu_pre_run(CPUState *cpu)
1468 {
1469     HRESULT hr;
1470     struct whpx_state *whpx = &whpx_global;
1471     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
1472     CPUX86State *env = cpu->env_ptr;
1473     X86CPU *x86_cpu = X86_CPU(cpu);
1474     int irq;
1475     uint8_t tpr;
1476     WHV_X64_PENDING_INTERRUPTION_REGISTER new_int;
1477     UINT32 reg_count = 0;
1478     WHV_REGISTER_VALUE reg_values[3];
1479     WHV_REGISTER_NAME reg_names[3];
1480 
1481     memset(&new_int, 0, sizeof(new_int));
1482     memset(reg_values, 0, sizeof(reg_values));
1483 
1484     qemu_mutex_lock_iothread();
1485 
1486     /* Inject NMI */
1487     if (!vcpu->interruption_pending &&
1488         cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
1489         if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
1490             cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
1491             vcpu->interruptable = false;
1492             new_int.InterruptionType = WHvX64PendingNmi;
1493             new_int.InterruptionPending = 1;
1494             new_int.InterruptionVector = 2;
1495         }
1496         if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
1497             cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
1498         }
1499     }
1500 
1501     /*
1502      * Force the VCPU out of its inner loop to process any INIT requests or
1503      * commit pending TPR access.
1504      */
1505     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
1506         if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
1507             !(env->hflags & HF_SMM_MASK)) {
1508             cpu->exit_request = 1;
1509         }
1510         if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
1511             cpu->exit_request = 1;
1512         }
1513     }
1514 
1515     /* Get pending hard interruption or replay one that was overwritten */
1516     if (!whpx_apic_in_platform()) {
1517         if (!vcpu->interruption_pending &&
1518             vcpu->interruptable && (env->eflags & IF_MASK)) {
1519             assert(!new_int.InterruptionPending);
1520             if (cpu->interrupt_request & CPU_INTERRUPT_HARD) {
1521                 cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
1522                 irq = cpu_get_pic_interrupt(env);
1523                 if (irq >= 0) {
1524                     new_int.InterruptionType = WHvX64PendingInterrupt;
1525                     new_int.InterruptionPending = 1;
1526                     new_int.InterruptionVector = irq;
1527                 }
1528             }
1529         }
1530 
1531         /* Setup interrupt state if new one was prepared */
1532         if (new_int.InterruptionPending) {
1533             reg_values[reg_count].PendingInterruption = new_int;
1534             reg_names[reg_count] = WHvRegisterPendingInterruption;
1535             reg_count += 1;
1536         }
1537     } else if (vcpu->ready_for_pic_interrupt &&
1538                (cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
1539         cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
1540         irq = cpu_get_pic_interrupt(env);
1541         if (irq >= 0) {
1542             reg_names[reg_count] = WHvRegisterPendingEvent;
1543             reg_values[reg_count].ExtIntEvent = (WHV_X64_PENDING_EXT_INT_EVENT)
1544             {
1545                 .EventPending = 1,
1546                 .EventType = WHvX64PendingEventExtInt,
1547                 .Vector = irq,
1548             };
1549             reg_count += 1;
1550         }
1551      }
1552 
1553     /* Sync the TPR to the CR8 if was modified during the intercept */
1554     tpr = whpx_apic_tpr_to_cr8(cpu_get_apic_tpr(x86_cpu->apic_state));
1555     if (tpr != vcpu->tpr) {
1556         vcpu->tpr = tpr;
1557         reg_values[reg_count].Reg64 = tpr;
1558         cpu->exit_request = 1;
1559         reg_names[reg_count] = WHvX64RegisterCr8;
1560         reg_count += 1;
1561     }
1562 
1563     /* Update the state of the interrupt delivery notification */
1564     if (!vcpu->window_registered &&
1565         cpu->interrupt_request & CPU_INTERRUPT_HARD) {
1566         reg_values[reg_count].DeliverabilityNotifications =
1567             (WHV_X64_DELIVERABILITY_NOTIFICATIONS_REGISTER) {
1568                 .InterruptNotification = 1
1569             };
1570         vcpu->window_registered = 1;
1571         reg_names[reg_count] = WHvX64RegisterDeliverabilityNotifications;
1572         reg_count += 1;
1573     }
1574 
1575     qemu_mutex_unlock_iothread();
1576     vcpu->ready_for_pic_interrupt = false;
1577 
1578     if (reg_count) {
1579         hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1580             whpx->partition, cpu->cpu_index,
1581             reg_names, reg_count, reg_values);
1582         if (FAILED(hr)) {
1583             error_report("WHPX: Failed to set interrupt state registers,"
1584                          " hr=%08lx", hr);
1585         }
1586     }
1587 
1588     return;
1589 }
1590 
1591 static void whpx_vcpu_post_run(CPUState *cpu)
1592 {
1593     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
1594     CPUX86State *env = cpu->env_ptr;
1595     X86CPU *x86_cpu = X86_CPU(cpu);
1596 
1597     env->eflags = vcpu->exit_ctx.VpContext.Rflags;
1598 
1599     uint64_t tpr = vcpu->exit_ctx.VpContext.Cr8;
1600     if (vcpu->tpr != tpr) {
1601         vcpu->tpr = tpr;
1602         qemu_mutex_lock_iothread();
1603         cpu_set_apic_tpr(x86_cpu->apic_state, whpx_cr8_to_apic_tpr(vcpu->tpr));
1604         qemu_mutex_unlock_iothread();
1605     }
1606 
1607     vcpu->interruption_pending =
1608         vcpu->exit_ctx.VpContext.ExecutionState.InterruptionPending;
1609 
1610     vcpu->interruptable =
1611         !vcpu->exit_ctx.VpContext.ExecutionState.InterruptShadow;
1612 
1613     return;
1614 }
1615 
1616 static void whpx_vcpu_process_async_events(CPUState *cpu)
1617 {
1618     CPUX86State *env = cpu->env_ptr;
1619     X86CPU *x86_cpu = X86_CPU(cpu);
1620     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
1621 
1622     if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
1623         !(env->hflags & HF_SMM_MASK)) {
1624         whpx_cpu_synchronize_state(cpu);
1625         do_cpu_init(x86_cpu);
1626         vcpu->interruptable = true;
1627     }
1628 
1629     if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
1630         cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
1631         apic_poll_irq(x86_cpu->apic_state);
1632     }
1633 
1634     if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
1635          (env->eflags & IF_MASK)) ||
1636         (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
1637         cpu->halted = false;
1638     }
1639 
1640     if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
1641         whpx_cpu_synchronize_state(cpu);
1642         do_cpu_sipi(x86_cpu);
1643     }
1644 
1645     if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
1646         cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
1647         whpx_cpu_synchronize_state(cpu);
1648         apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
1649                                       env->tpr_access_type);
1650     }
1651 
1652     return;
1653 }
1654 
1655 static int whpx_vcpu_run(CPUState *cpu)
1656 {
1657     HRESULT hr;
1658     struct whpx_state *whpx = &whpx_global;
1659     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
1660     struct whpx_breakpoint *stepped_over_bp = NULL;
1661     WhpxStepMode exclusive_step_mode = WHPX_STEP_NONE;
1662     int ret;
1663 
1664     g_assert(qemu_mutex_iothread_locked());
1665 
1666     if (whpx->running_cpus++ == 0) {
1667         /* Insert breakpoints into memory, update exception exit bitmap. */
1668         ret = whpx_first_vcpu_starting(cpu);
1669         if (ret != 0) {
1670             return ret;
1671         }
1672     }
1673 
1674     if (whpx->breakpoints.breakpoints &&
1675         whpx->breakpoints.breakpoints->used > 0)
1676     {
1677         uint64_t pc = whpx_vcpu_get_pc(cpu, true);
1678         stepped_over_bp = whpx_lookup_breakpoint_by_addr(pc);
1679         if (stepped_over_bp && stepped_over_bp->state != WHPX_BP_SET) {
1680             stepped_over_bp = NULL;
1681         }
1682 
1683         if (stepped_over_bp) {
1684             /*
1685              * We are trying to run the instruction overwritten by an active
1686              * breakpoint. We will temporarily disable the breakpoint, suspend
1687              * other CPUs, and step over the instruction.
1688              */
1689             exclusive_step_mode = WHPX_STEP_EXCLUSIVE;
1690         }
1691     }
1692 
1693     if (exclusive_step_mode == WHPX_STEP_NONE) {
1694         whpx_vcpu_process_async_events(cpu);
1695         if (cpu->halted && !whpx_apic_in_platform()) {
1696             cpu->exception_index = EXCP_HLT;
1697             qatomic_set(&cpu->exit_request, false);
1698             return 0;
1699         }
1700     }
1701 
1702     qemu_mutex_unlock_iothread();
1703 
1704     if (exclusive_step_mode != WHPX_STEP_NONE) {
1705         start_exclusive();
1706         g_assert(cpu == current_cpu);
1707         g_assert(!cpu->running);
1708         cpu->running = true;
1709 
1710         hr = whpx_set_exception_exit_bitmap(
1711             1UL << WHvX64ExceptionTypeDebugTrapOrFault);
1712         if (!SUCCEEDED(hr)) {
1713             error_report("WHPX: Failed to update exception exit mask, "
1714                          "hr=%08lx.", hr);
1715             return 1;
1716         }
1717 
1718         if (stepped_over_bp) {
1719             /* Temporarily disable the triggered breakpoint. */
1720             cpu_memory_rw_debug(cpu,
1721                 stepped_over_bp->address,
1722                 &stepped_over_bp->original_instruction,
1723                 1,
1724                 true);
1725         }
1726     } else {
1727         cpu_exec_start(cpu);
1728     }
1729 
1730     do {
1731         if (cpu->vcpu_dirty) {
1732             whpx_set_registers(cpu, WHPX_SET_RUNTIME_STATE);
1733             cpu->vcpu_dirty = false;
1734         }
1735 
1736         if (exclusive_step_mode == WHPX_STEP_NONE) {
1737             whpx_vcpu_pre_run(cpu);
1738 
1739             if (qatomic_read(&cpu->exit_request)) {
1740                 whpx_vcpu_kick(cpu);
1741             }
1742         }
1743 
1744         if (exclusive_step_mode != WHPX_STEP_NONE || cpu->singlestep_enabled) {
1745             whpx_vcpu_configure_single_stepping(cpu, true, NULL);
1746         }
1747 
1748         hr = whp_dispatch.WHvRunVirtualProcessor(
1749             whpx->partition, cpu->cpu_index,
1750             &vcpu->exit_ctx, sizeof(vcpu->exit_ctx));
1751 
1752         if (FAILED(hr)) {
1753             error_report("WHPX: Failed to exec a virtual processor,"
1754                          " hr=%08lx", hr);
1755             ret = -1;
1756             break;
1757         }
1758 
1759         if (exclusive_step_mode != WHPX_STEP_NONE || cpu->singlestep_enabled) {
1760             whpx_vcpu_configure_single_stepping(cpu,
1761                 false,
1762                 &vcpu->exit_ctx.VpContext.Rflags);
1763         }
1764 
1765         whpx_vcpu_post_run(cpu);
1766 
1767         switch (vcpu->exit_ctx.ExitReason) {
1768         case WHvRunVpExitReasonMemoryAccess:
1769             ret = whpx_handle_mmio(cpu, &vcpu->exit_ctx.MemoryAccess);
1770             break;
1771 
1772         case WHvRunVpExitReasonX64IoPortAccess:
1773             ret = whpx_handle_portio(cpu, &vcpu->exit_ctx.IoPortAccess);
1774             break;
1775 
1776         case WHvRunVpExitReasonX64InterruptWindow:
1777             vcpu->ready_for_pic_interrupt = 1;
1778             vcpu->window_registered = 0;
1779             ret = 0;
1780             break;
1781 
1782         case WHvRunVpExitReasonX64ApicEoi:
1783             assert(whpx_apic_in_platform());
1784             ioapic_eoi_broadcast(vcpu->exit_ctx.ApicEoi.InterruptVector);
1785             break;
1786 
1787         case WHvRunVpExitReasonX64Halt:
1788             /*
1789              * WARNING: as of build 19043.1526 (21H1), this exit reason is no
1790              * longer used.
1791              */
1792             ret = whpx_handle_halt(cpu);
1793             break;
1794 
1795         case WHvRunVpExitReasonX64ApicInitSipiTrap: {
1796             WHV_INTERRUPT_CONTROL ipi = {0};
1797             uint64_t icr = vcpu->exit_ctx.ApicInitSipi.ApicIcr;
1798             uint32_t delivery_mode =
1799                 (icr & APIC_ICR_DELIV_MOD) >> APIC_ICR_DELIV_MOD_SHIFT;
1800             int dest_shorthand =
1801                 (icr & APIC_ICR_DEST_SHORT) >> APIC_ICR_DEST_SHORT_SHIFT;
1802             bool broadcast = false;
1803             bool include_self = false;
1804             uint32_t i;
1805 
1806             /* We only registered for INIT and SIPI exits. */
1807             if ((delivery_mode != APIC_DM_INIT) &&
1808                 (delivery_mode != APIC_DM_SIPI)) {
1809                 error_report(
1810                     "WHPX: Unexpected APIC exit that is not a INIT or SIPI");
1811                 break;
1812             }
1813 
1814             if (delivery_mode == APIC_DM_INIT) {
1815                 ipi.Type = WHvX64InterruptTypeInit;
1816             } else {
1817                 ipi.Type = WHvX64InterruptTypeSipi;
1818             }
1819 
1820             ipi.DestinationMode =
1821                 ((icr & APIC_ICR_DEST_MOD) >> APIC_ICR_DEST_MOD_SHIFT) ?
1822                     WHvX64InterruptDestinationModeLogical :
1823                     WHvX64InterruptDestinationModePhysical;
1824 
1825             ipi.TriggerMode =
1826                 ((icr & APIC_ICR_TRIGGER_MOD) >> APIC_ICR_TRIGGER_MOD_SHIFT) ?
1827                     WHvX64InterruptTriggerModeLevel :
1828                     WHvX64InterruptTriggerModeEdge;
1829 
1830             ipi.Vector = icr & APIC_VECTOR_MASK;
1831             switch (dest_shorthand) {
1832             /* no shorthand. Bits 56-63 contain the destination. */
1833             case 0:
1834                 ipi.Destination = (icr >> 56) & APIC_VECTOR_MASK;
1835                 hr = whp_dispatch.WHvRequestInterrupt(whpx->partition,
1836                         &ipi, sizeof(ipi));
1837                 if (FAILED(hr)) {
1838                     error_report("WHPX: Failed to request interrupt  hr=%08lx",
1839                         hr);
1840                 }
1841 
1842                 break;
1843 
1844             /* self */
1845             case 1:
1846                 include_self = true;
1847                 break;
1848 
1849             /* broadcast, including self */
1850             case 2:
1851                 broadcast = true;
1852                 include_self = true;
1853                 break;
1854 
1855             /* broadcast, excluding self */
1856             case 3:
1857                 broadcast = true;
1858                 break;
1859             }
1860 
1861             if (!broadcast && !include_self) {
1862                 break;
1863             }
1864 
1865             for (i = 0; i <= max_vcpu_index; i++) {
1866                 if (i == cpu->cpu_index && !include_self) {
1867                     continue;
1868                 }
1869 
1870                 /*
1871                  * Assuming that APIC Ids are identity mapped since
1872                  * WHvX64RegisterApicId & WHvX64RegisterInitialApicId registers
1873                  * are not handled yet and the hypervisor doesn't allow the
1874                  * guest to modify the APIC ID.
1875                  */
1876                 ipi.Destination = i;
1877                 hr = whp_dispatch.WHvRequestInterrupt(whpx->partition,
1878                         &ipi, sizeof(ipi));
1879                 if (FAILED(hr)) {
1880                     error_report(
1881                         "WHPX: Failed to request SIPI for %d,  hr=%08lx",
1882                         i, hr);
1883                 }
1884             }
1885 
1886             break;
1887         }
1888 
1889         case WHvRunVpExitReasonCanceled:
1890             if (exclusive_step_mode != WHPX_STEP_NONE) {
1891                 /*
1892                  * We are trying to step over a single instruction, and
1893                  * likely got a request to stop from another thread.
1894                  * Delay it until we are done stepping
1895                  * over.
1896                  */
1897                 ret = 0;
1898             } else {
1899                 cpu->exception_index = EXCP_INTERRUPT;
1900                 ret = 1;
1901             }
1902             break;
1903         case WHvRunVpExitReasonX64MsrAccess: {
1904             WHV_REGISTER_VALUE reg_values[3] = {0};
1905             WHV_REGISTER_NAME reg_names[3];
1906             UINT32 reg_count;
1907 
1908             reg_names[0] = WHvX64RegisterRip;
1909             reg_names[1] = WHvX64RegisterRax;
1910             reg_names[2] = WHvX64RegisterRdx;
1911 
1912             reg_values[0].Reg64 =
1913                 vcpu->exit_ctx.VpContext.Rip +
1914                 vcpu->exit_ctx.VpContext.InstructionLength;
1915 
1916             /*
1917              * For all unsupported MSR access we:
1918              *     ignore writes
1919              *     return 0 on read.
1920              */
1921             reg_count = vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite ?
1922                         1 : 3;
1923 
1924             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1925                 whpx->partition,
1926                 cpu->cpu_index,
1927                 reg_names, reg_count,
1928                 reg_values);
1929 
1930             if (FAILED(hr)) {
1931                 error_report("WHPX: Failed to set MsrAccess state "
1932                              " registers, hr=%08lx", hr);
1933             }
1934             ret = 0;
1935             break;
1936         }
1937         case WHvRunVpExitReasonX64Cpuid: {
1938             WHV_REGISTER_VALUE reg_values[5];
1939             WHV_REGISTER_NAME reg_names[5];
1940             UINT32 reg_count = 5;
1941             UINT64 cpuid_fn, rip = 0, rax = 0, rcx = 0, rdx = 0, rbx = 0;
1942             X86CPU *x86_cpu = X86_CPU(cpu);
1943             CPUX86State *env = &x86_cpu->env;
1944 
1945             memset(reg_values, 0, sizeof(reg_values));
1946 
1947             rip = vcpu->exit_ctx.VpContext.Rip +
1948                   vcpu->exit_ctx.VpContext.InstructionLength;
1949             cpuid_fn = vcpu->exit_ctx.CpuidAccess.Rax;
1950 
1951             /*
1952              * Ideally, these should be supplied to the hypervisor during VCPU
1953              * initialization and it should be able to satisfy this request.
1954              * But, currently, WHPX doesn't support setting CPUID values in the
1955              * hypervisor once the partition has been setup, which is too late
1956              * since VCPUs are realized later. For now, use the values from
1957              * QEMU to satisfy these requests, until WHPX adds support for
1958              * being able to set these values in the hypervisor at runtime.
1959              */
1960             cpu_x86_cpuid(env, cpuid_fn, 0, (UINT32 *)&rax, (UINT32 *)&rbx,
1961                 (UINT32 *)&rcx, (UINT32 *)&rdx);
1962             switch (cpuid_fn) {
1963             case 0x40000000:
1964                 /* Expose the vmware cpu frequency cpuid leaf */
1965                 rax = 0x40000010;
1966                 rbx = rcx = rdx = 0;
1967                 break;
1968 
1969             case 0x40000010:
1970                 rax = env->tsc_khz;
1971                 rbx = env->apic_bus_freq / 1000; /* Hz to KHz */
1972                 rcx = rdx = 0;
1973                 break;
1974 
1975             case 0x80000001:
1976                 /* Remove any support of OSVW */
1977                 rcx &= ~CPUID_EXT3_OSVW;
1978                 break;
1979             }
1980 
1981             reg_names[0] = WHvX64RegisterRip;
1982             reg_names[1] = WHvX64RegisterRax;
1983             reg_names[2] = WHvX64RegisterRcx;
1984             reg_names[3] = WHvX64RegisterRdx;
1985             reg_names[4] = WHvX64RegisterRbx;
1986 
1987             reg_values[0].Reg64 = rip;
1988             reg_values[1].Reg64 = rax;
1989             reg_values[2].Reg64 = rcx;
1990             reg_values[3].Reg64 = rdx;
1991             reg_values[4].Reg64 = rbx;
1992 
1993             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1994                 whpx->partition, cpu->cpu_index,
1995                 reg_names,
1996                 reg_count,
1997                 reg_values);
1998 
1999             if (FAILED(hr)) {
2000                 error_report("WHPX: Failed to set CpuidAccess state registers,"
2001                              " hr=%08lx", hr);
2002             }
2003             ret = 0;
2004             break;
2005         }
2006         case WHvRunVpExitReasonException:
2007             whpx_get_registers(cpu);
2008 
2009             if ((vcpu->exit_ctx.VpException.ExceptionType ==
2010                  WHvX64ExceptionTypeDebugTrapOrFault) &&
2011                 (vcpu->exit_ctx.VpException.InstructionByteCount >= 1) &&
2012                 (vcpu->exit_ctx.VpException.InstructionBytes[0] ==
2013                  whpx_breakpoint_instruction)) {
2014                 /* Stopped at a software breakpoint. */
2015                 cpu->exception_index = EXCP_DEBUG;
2016             } else if ((vcpu->exit_ctx.VpException.ExceptionType ==
2017                         WHvX64ExceptionTypeDebugTrapOrFault) &&
2018                        !cpu->singlestep_enabled) {
2019                 /*
2020                  * Just finished stepping over a breakpoint, but the
2021                  * gdb does not expect us to do single-stepping.
2022                  * Don't do anything special.
2023                  */
2024                 cpu->exception_index = EXCP_INTERRUPT;
2025             } else {
2026                 /* Another exception or debug event. Report it to GDB. */
2027                 cpu->exception_index = EXCP_DEBUG;
2028             }
2029 
2030             ret = 1;
2031             break;
2032         case WHvRunVpExitReasonNone:
2033         case WHvRunVpExitReasonUnrecoverableException:
2034         case WHvRunVpExitReasonInvalidVpRegisterValue:
2035         case WHvRunVpExitReasonUnsupportedFeature:
2036         default:
2037             error_report("WHPX: Unexpected VP exit code %d",
2038                          vcpu->exit_ctx.ExitReason);
2039             whpx_get_registers(cpu);
2040             qemu_mutex_lock_iothread();
2041             qemu_system_guest_panicked(cpu_get_crash_info(cpu));
2042             qemu_mutex_unlock_iothread();
2043             break;
2044         }
2045 
2046     } while (!ret);
2047 
2048     if (stepped_over_bp) {
2049         /* Restore the breakpoint we stepped over */
2050         cpu_memory_rw_debug(cpu,
2051             stepped_over_bp->address,
2052             (void *)&whpx_breakpoint_instruction,
2053             1,
2054             true);
2055     }
2056 
2057     if (exclusive_step_mode != WHPX_STEP_NONE) {
2058         g_assert(cpu_in_exclusive_context(cpu));
2059         cpu->running = false;
2060         end_exclusive();
2061 
2062         exclusive_step_mode = WHPX_STEP_NONE;
2063     } else {
2064         cpu_exec_end(cpu);
2065     }
2066 
2067     qemu_mutex_lock_iothread();
2068     current_cpu = cpu;
2069 
2070     if (--whpx->running_cpus == 0) {
2071         whpx_last_vcpu_stopping(cpu);
2072     }
2073 
2074     qatomic_set(&cpu->exit_request, false);
2075 
2076     return ret < 0;
2077 }
2078 
2079 static void do_whpx_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
2080 {
2081     if (!cpu->vcpu_dirty) {
2082         whpx_get_registers(cpu);
2083         cpu->vcpu_dirty = true;
2084     }
2085 }
2086 
2087 static void do_whpx_cpu_synchronize_post_reset(CPUState *cpu,
2088                                                run_on_cpu_data arg)
2089 {
2090     whpx_set_registers(cpu, WHPX_SET_RESET_STATE);
2091     cpu->vcpu_dirty = false;
2092 }
2093 
2094 static void do_whpx_cpu_synchronize_post_init(CPUState *cpu,
2095                                               run_on_cpu_data arg)
2096 {
2097     whpx_set_registers(cpu, WHPX_SET_FULL_STATE);
2098     cpu->vcpu_dirty = false;
2099 }
2100 
2101 static void do_whpx_cpu_synchronize_pre_loadvm(CPUState *cpu,
2102                                                run_on_cpu_data arg)
2103 {
2104     cpu->vcpu_dirty = true;
2105 }
2106 
2107 /*
2108  * CPU support.
2109  */
2110 
2111 void whpx_cpu_synchronize_state(CPUState *cpu)
2112 {
2113     if (!cpu->vcpu_dirty) {
2114         run_on_cpu(cpu, do_whpx_cpu_synchronize_state, RUN_ON_CPU_NULL);
2115     }
2116 }
2117 
2118 void whpx_cpu_synchronize_post_reset(CPUState *cpu)
2119 {
2120     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
2121 }
2122 
2123 void whpx_cpu_synchronize_post_init(CPUState *cpu)
2124 {
2125     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
2126 }
2127 
2128 void whpx_cpu_synchronize_pre_loadvm(CPUState *cpu)
2129 {
2130     run_on_cpu(cpu, do_whpx_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
2131 }
2132 
2133 void whpx_cpu_synchronize_pre_resume(bool step_pending)
2134 {
2135     whpx_global.step_pending = step_pending;
2136 }
2137 
2138 /*
2139  * Vcpu support.
2140  */
2141 
2142 static Error *whpx_migration_blocker;
2143 
2144 static void whpx_cpu_update_state(void *opaque, bool running, RunState state)
2145 {
2146     CPUX86State *env = opaque;
2147 
2148     if (running) {
2149         env->tsc_valid = false;
2150     }
2151 }
2152 
2153 int whpx_init_vcpu(CPUState *cpu)
2154 {
2155     HRESULT hr;
2156     struct whpx_state *whpx = &whpx_global;
2157     struct whpx_vcpu *vcpu = NULL;
2158     Error *local_error = NULL;
2159     CPUX86State *env = cpu->env_ptr;
2160     X86CPU *x86_cpu = X86_CPU(cpu);
2161     UINT64 freq = 0;
2162     int ret;
2163 
2164     /* Add migration blockers for all unsupported features of the
2165      * Windows Hypervisor Platform
2166      */
2167     if (whpx_migration_blocker == NULL) {
2168         error_setg(&whpx_migration_blocker,
2169                "State blocked due to non-migratable CPUID feature support,"
2170                "dirty memory tracking support, and XSAVE/XRSTOR support");
2171 
2172         if (migrate_add_blocker(whpx_migration_blocker, &local_error) < 0) {
2173             error_report_err(local_error);
2174             error_free(whpx_migration_blocker);
2175             ret = -EINVAL;
2176             goto error;
2177         }
2178     }
2179 
2180     vcpu = g_new0(struct whpx_vcpu, 1);
2181 
2182     if (!vcpu) {
2183         error_report("WHPX: Failed to allocte VCPU context.");
2184         ret = -ENOMEM;
2185         goto error;
2186     }
2187 
2188     hr = whp_dispatch.WHvEmulatorCreateEmulator(
2189         &whpx_emu_callbacks,
2190         &vcpu->emulator);
2191     if (FAILED(hr)) {
2192         error_report("WHPX: Failed to setup instruction completion support,"
2193                      " hr=%08lx", hr);
2194         ret = -EINVAL;
2195         goto error;
2196     }
2197 
2198     hr = whp_dispatch.WHvCreateVirtualProcessor(
2199         whpx->partition, cpu->cpu_index, 0);
2200     if (FAILED(hr)) {
2201         error_report("WHPX: Failed to create a virtual processor,"
2202                      " hr=%08lx", hr);
2203         whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
2204         ret = -EINVAL;
2205         goto error;
2206     }
2207 
2208     /*
2209      * vcpu's TSC frequency is either specified by user, or use the value
2210      * provided by Hyper-V if the former is not present. In the latter case, we
2211      * query it from Hyper-V and record in env->tsc_khz, so that vcpu's TSC
2212      * frequency can be migrated later via this field.
2213      */
2214     if (!env->tsc_khz) {
2215         hr = whp_dispatch.WHvGetCapability(
2216             WHvCapabilityCodeProcessorClockFrequency, &freq, sizeof(freq),
2217                 NULL);
2218         if (hr != WHV_E_UNKNOWN_CAPABILITY) {
2219             if (FAILED(hr)) {
2220                 printf("WHPX: Failed to query tsc frequency, hr=0x%08lx\n", hr);
2221             } else {
2222                 env->tsc_khz = freq / 1000; /* Hz to KHz */
2223             }
2224         }
2225     }
2226 
2227     env->apic_bus_freq = HYPERV_APIC_BUS_FREQUENCY;
2228     hr = whp_dispatch.WHvGetCapability(
2229         WHvCapabilityCodeInterruptClockFrequency, &freq, sizeof(freq), NULL);
2230     if (hr != WHV_E_UNKNOWN_CAPABILITY) {
2231         if (FAILED(hr)) {
2232             printf("WHPX: Failed to query apic bus frequency hr=0x%08lx\n", hr);
2233         } else {
2234             env->apic_bus_freq = freq;
2235         }
2236     }
2237 
2238     /*
2239      * If the vmware cpuid frequency leaf option is set, and we have a valid
2240      * tsc value, trap the corresponding cpuid's.
2241      */
2242     if (x86_cpu->vmware_cpuid_freq && env->tsc_khz) {
2243         UINT32 cpuidExitList[] = {1, 0x80000001, 0x40000000, 0x40000010};
2244 
2245         hr = whp_dispatch.WHvSetPartitionProperty(
2246                 whpx->partition,
2247                 WHvPartitionPropertyCodeCpuidExitList,
2248                 cpuidExitList,
2249                 RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
2250 
2251         if (FAILED(hr)) {
2252             error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
2253                         hr);
2254             ret = -EINVAL;
2255             goto error;
2256         }
2257     }
2258 
2259     vcpu->interruptable = true;
2260     cpu->vcpu_dirty = true;
2261     cpu->hax_vcpu = (struct hax_vcpu_state *)vcpu;
2262     max_vcpu_index = max(max_vcpu_index, cpu->cpu_index);
2263     qemu_add_vm_change_state_handler(whpx_cpu_update_state, cpu->env_ptr);
2264 
2265     return 0;
2266 
2267 error:
2268     g_free(vcpu);
2269 
2270     return ret;
2271 }
2272 
2273 int whpx_vcpu_exec(CPUState *cpu)
2274 {
2275     int ret;
2276     int fatal;
2277 
2278     for (;;) {
2279         if (cpu->exception_index >= EXCP_INTERRUPT) {
2280             ret = cpu->exception_index;
2281             cpu->exception_index = -1;
2282             break;
2283         }
2284 
2285         fatal = whpx_vcpu_run(cpu);
2286 
2287         if (fatal) {
2288             error_report("WHPX: Failed to exec a virtual processor");
2289             abort();
2290         }
2291     }
2292 
2293     return ret;
2294 }
2295 
2296 void whpx_destroy_vcpu(CPUState *cpu)
2297 {
2298     struct whpx_state *whpx = &whpx_global;
2299     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
2300 
2301     whp_dispatch.WHvDeleteVirtualProcessor(whpx->partition, cpu->cpu_index);
2302     whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
2303     g_free(cpu->hax_vcpu);
2304     return;
2305 }
2306 
2307 void whpx_vcpu_kick(CPUState *cpu)
2308 {
2309     struct whpx_state *whpx = &whpx_global;
2310     whp_dispatch.WHvCancelRunVirtualProcessor(
2311         whpx->partition, cpu->cpu_index, 0);
2312 }
2313 
2314 /*
2315  * Memory support.
2316  */
2317 
2318 static void whpx_update_mapping(hwaddr start_pa, ram_addr_t size,
2319                                 void *host_va, int add, int rom,
2320                                 const char *name)
2321 {
2322     struct whpx_state *whpx = &whpx_global;
2323     HRESULT hr;
2324 
2325     /*
2326     if (add) {
2327         printf("WHPX: ADD PA:%p Size:%p, Host:%p, %s, '%s'\n",
2328                (void*)start_pa, (void*)size, host_va,
2329                (rom ? "ROM" : "RAM"), name);
2330     } else {
2331         printf("WHPX: DEL PA:%p Size:%p, Host:%p,      '%s'\n",
2332                (void*)start_pa, (void*)size, host_va, name);
2333     }
2334     */
2335 
2336     if (add) {
2337         hr = whp_dispatch.WHvMapGpaRange(whpx->partition,
2338                                          host_va,
2339                                          start_pa,
2340                                          size,
2341                                          (WHvMapGpaRangeFlagRead |
2342                                           WHvMapGpaRangeFlagExecute |
2343                                           (rom ? 0 : WHvMapGpaRangeFlagWrite)));
2344     } else {
2345         hr = whp_dispatch.WHvUnmapGpaRange(whpx->partition,
2346                                            start_pa,
2347                                            size);
2348     }
2349 
2350     if (FAILED(hr)) {
2351         error_report("WHPX: Failed to %s GPA range '%s' PA:%p, Size:%p bytes,"
2352                      " Host:%p, hr=%08lx",
2353                      (add ? "MAP" : "UNMAP"), name,
2354                      (void *)(uintptr_t)start_pa, (void *)size, host_va, hr);
2355     }
2356 }
2357 
2358 static void whpx_process_section(MemoryRegionSection *section, int add)
2359 {
2360     MemoryRegion *mr = section->mr;
2361     hwaddr start_pa = section->offset_within_address_space;
2362     ram_addr_t size = int128_get64(section->size);
2363     unsigned int delta;
2364     uint64_t host_va;
2365 
2366     if (!memory_region_is_ram(mr)) {
2367         return;
2368     }
2369 
2370     delta = qemu_real_host_page_size() - (start_pa & ~qemu_real_host_page_mask());
2371     delta &= ~qemu_real_host_page_mask();
2372     if (delta > size) {
2373         return;
2374     }
2375     start_pa += delta;
2376     size -= delta;
2377     size &= qemu_real_host_page_mask();
2378     if (!size || (start_pa & ~qemu_real_host_page_mask())) {
2379         return;
2380     }
2381 
2382     host_va = (uintptr_t)memory_region_get_ram_ptr(mr)
2383             + section->offset_within_region + delta;
2384 
2385     whpx_update_mapping(start_pa, size, (void *)(uintptr_t)host_va, add,
2386                         memory_region_is_rom(mr), mr->name);
2387 }
2388 
2389 static void whpx_region_add(MemoryListener *listener,
2390                            MemoryRegionSection *section)
2391 {
2392     memory_region_ref(section->mr);
2393     whpx_process_section(section, 1);
2394 }
2395 
2396 static void whpx_region_del(MemoryListener *listener,
2397                            MemoryRegionSection *section)
2398 {
2399     whpx_process_section(section, 0);
2400     memory_region_unref(section->mr);
2401 }
2402 
2403 static void whpx_transaction_begin(MemoryListener *listener)
2404 {
2405 }
2406 
2407 static void whpx_transaction_commit(MemoryListener *listener)
2408 {
2409 }
2410 
2411 static void whpx_log_sync(MemoryListener *listener,
2412                          MemoryRegionSection *section)
2413 {
2414     MemoryRegion *mr = section->mr;
2415 
2416     if (!memory_region_is_ram(mr)) {
2417         return;
2418     }
2419 
2420     memory_region_set_dirty(mr, 0, int128_get64(section->size));
2421 }
2422 
2423 static MemoryListener whpx_memory_listener = {
2424     .name = "whpx",
2425     .begin = whpx_transaction_begin,
2426     .commit = whpx_transaction_commit,
2427     .region_add = whpx_region_add,
2428     .region_del = whpx_region_del,
2429     .log_sync = whpx_log_sync,
2430     .priority = 10,
2431 };
2432 
2433 static void whpx_memory_init(void)
2434 {
2435     memory_listener_register(&whpx_memory_listener, &address_space_memory);
2436 }
2437 
2438 /*
2439  * Load the functions from the given library, using the given handle. If a
2440  * handle is provided, it is used, otherwise the library is opened. The
2441  * handle will be updated on return with the opened one.
2442  */
2443 static bool load_whp_dispatch_fns(HMODULE *handle,
2444     WHPFunctionList function_list)
2445 {
2446     HMODULE hLib = *handle;
2447 
2448     #define WINHV_PLATFORM_DLL "WinHvPlatform.dll"
2449     #define WINHV_EMULATION_DLL "WinHvEmulation.dll"
2450     #define WHP_LOAD_FIELD_OPTIONAL(return_type, function_name, signature) \
2451         whp_dispatch.function_name = \
2452             (function_name ## _t)GetProcAddress(hLib, #function_name); \
2453 
2454     #define WHP_LOAD_FIELD(return_type, function_name, signature) \
2455         whp_dispatch.function_name = \
2456             (function_name ## _t)GetProcAddress(hLib, #function_name); \
2457         if (!whp_dispatch.function_name) { \
2458             error_report("Could not load function %s", #function_name); \
2459             goto error; \
2460         } \
2461 
2462     #define WHP_LOAD_LIB(lib_name, handle_lib) \
2463     if (!handle_lib) { \
2464         handle_lib = LoadLibrary(lib_name); \
2465         if (!handle_lib) { \
2466             error_report("Could not load library %s.", lib_name); \
2467             goto error; \
2468         } \
2469     } \
2470 
2471     switch (function_list) {
2472     case WINHV_PLATFORM_FNS_DEFAULT:
2473         WHP_LOAD_LIB(WINHV_PLATFORM_DLL, hLib)
2474         LIST_WINHVPLATFORM_FUNCTIONS(WHP_LOAD_FIELD)
2475         break;
2476 
2477     case WINHV_EMULATION_FNS_DEFAULT:
2478         WHP_LOAD_LIB(WINHV_EMULATION_DLL, hLib)
2479         LIST_WINHVEMULATION_FUNCTIONS(WHP_LOAD_FIELD)
2480         break;
2481 
2482     case WINHV_PLATFORM_FNS_SUPPLEMENTAL:
2483         WHP_LOAD_LIB(WINHV_PLATFORM_DLL, hLib)
2484         LIST_WINHVPLATFORM_FUNCTIONS_SUPPLEMENTAL(WHP_LOAD_FIELD_OPTIONAL)
2485         break;
2486     }
2487 
2488     *handle = hLib;
2489     return true;
2490 
2491 error:
2492     if (hLib) {
2493         FreeLibrary(hLib);
2494     }
2495 
2496     return false;
2497 }
2498 
2499 static void whpx_set_kernel_irqchip(Object *obj, Visitor *v,
2500                                    const char *name, void *opaque,
2501                                    Error **errp)
2502 {
2503     struct whpx_state *whpx = &whpx_global;
2504     OnOffSplit mode;
2505 
2506     if (!visit_type_OnOffSplit(v, name, &mode, errp)) {
2507         return;
2508     }
2509 
2510     switch (mode) {
2511     case ON_OFF_SPLIT_ON:
2512         whpx->kernel_irqchip_allowed = true;
2513         whpx->kernel_irqchip_required = true;
2514         break;
2515 
2516     case ON_OFF_SPLIT_OFF:
2517         whpx->kernel_irqchip_allowed = false;
2518         whpx->kernel_irqchip_required = false;
2519         break;
2520 
2521     case ON_OFF_SPLIT_SPLIT:
2522         error_setg(errp, "WHPX: split irqchip currently not supported");
2523         error_append_hint(errp,
2524             "Try without kernel-irqchip or with kernel-irqchip=on|off");
2525         break;
2526 
2527     default:
2528         /*
2529          * The value was checked in visit_type_OnOffSplit() above. If
2530          * we get here, then something is wrong in QEMU.
2531          */
2532         abort();
2533     }
2534 }
2535 
2536 /*
2537  * Partition support
2538  */
2539 
2540 static int whpx_accel_init(MachineState *ms)
2541 {
2542     struct whpx_state *whpx;
2543     int ret;
2544     HRESULT hr;
2545     WHV_CAPABILITY whpx_cap;
2546     UINT32 whpx_cap_size;
2547     WHV_PARTITION_PROPERTY prop;
2548     UINT32 cpuidExitList[] = {1, 0x80000001};
2549     WHV_CAPABILITY_FEATURES features = {0};
2550 
2551     whpx = &whpx_global;
2552 
2553     if (!init_whp_dispatch()) {
2554         ret = -ENOSYS;
2555         goto error;
2556     }
2557 
2558     whpx->mem_quota = ms->ram_size;
2559 
2560     hr = whp_dispatch.WHvGetCapability(
2561         WHvCapabilityCodeHypervisorPresent, &whpx_cap,
2562         sizeof(whpx_cap), &whpx_cap_size);
2563     if (FAILED(hr) || !whpx_cap.HypervisorPresent) {
2564         error_report("WHPX: No accelerator found, hr=%08lx", hr);
2565         ret = -ENOSPC;
2566         goto error;
2567     }
2568 
2569     hr = whp_dispatch.WHvGetCapability(
2570         WHvCapabilityCodeFeatures, &features, sizeof(features), NULL);
2571     if (FAILED(hr)) {
2572         error_report("WHPX: Failed to query capabilities, hr=%08lx", hr);
2573         ret = -EINVAL;
2574         goto error;
2575     }
2576 
2577     hr = whp_dispatch.WHvCreatePartition(&whpx->partition);
2578     if (FAILED(hr)) {
2579         error_report("WHPX: Failed to create partition, hr=%08lx", hr);
2580         ret = -EINVAL;
2581         goto error;
2582     }
2583 
2584     /*
2585      * Query the XSAVE capability of the partition. Any error here is not
2586      * considered fatal.
2587      */
2588     hr = whp_dispatch.WHvGetPartitionProperty(
2589         whpx->partition,
2590         WHvPartitionPropertyCodeProcessorXsaveFeatures,
2591         &whpx_xsave_cap,
2592         sizeof(whpx_xsave_cap),
2593         &whpx_cap_size);
2594 
2595     /*
2596      * Windows version which don't support this property will return with the
2597      * specific error code.
2598      */
2599     if (FAILED(hr) && hr != WHV_E_UNKNOWN_PROPERTY) {
2600         error_report("WHPX: Failed to query XSAVE capability, hr=%08lx", hr);
2601     }
2602 
2603     if (!whpx_has_xsave()) {
2604         printf("WHPX: Partition is not XSAVE capable\n");
2605     }
2606 
2607     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
2608     prop.ProcessorCount = ms->smp.cpus;
2609     hr = whp_dispatch.WHvSetPartitionProperty(
2610         whpx->partition,
2611         WHvPartitionPropertyCodeProcessorCount,
2612         &prop,
2613         sizeof(WHV_PARTITION_PROPERTY));
2614 
2615     if (FAILED(hr)) {
2616         error_report("WHPX: Failed to set partition core count to %d,"
2617                      " hr=%08lx", ms->smp.cores, hr);
2618         ret = -EINVAL;
2619         goto error;
2620     }
2621 
2622     /*
2623      * Error out if WHP doesn't support apic emulation and user is requiring
2624      * it.
2625      */
2626     if (whpx->kernel_irqchip_required && (!features.LocalApicEmulation ||
2627             !whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2)) {
2628         error_report("WHPX: kernel irqchip requested, but unavailable. "
2629             "Try without kernel-irqchip or with kernel-irqchip=off");
2630         ret = -EINVAL;
2631         goto error;
2632     }
2633 
2634     if (whpx->kernel_irqchip_allowed && features.LocalApicEmulation &&
2635         whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2) {
2636         WHV_X64_LOCAL_APIC_EMULATION_MODE mode =
2637             WHvX64LocalApicEmulationModeXApic;
2638         printf("WHPX: setting APIC emulation mode in the hypervisor\n");
2639         hr = whp_dispatch.WHvSetPartitionProperty(
2640             whpx->partition,
2641             WHvPartitionPropertyCodeLocalApicEmulationMode,
2642             &mode,
2643             sizeof(mode));
2644         if (FAILED(hr)) {
2645             error_report("WHPX: Failed to enable kernel irqchip hr=%08lx", hr);
2646             if (whpx->kernel_irqchip_required) {
2647                 error_report("WHPX: kernel irqchip requested, but unavailable");
2648                 ret = -EINVAL;
2649                 goto error;
2650             }
2651         } else {
2652             whpx->apic_in_platform = true;
2653         }
2654     }
2655 
2656     /* Register for MSR and CPUID exits */
2657     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
2658     prop.ExtendedVmExits.X64MsrExit = 1;
2659     prop.ExtendedVmExits.X64CpuidExit = 1;
2660     prop.ExtendedVmExits.ExceptionExit = 1;
2661     if (whpx_apic_in_platform()) {
2662         prop.ExtendedVmExits.X64ApicInitSipiExitTrap = 1;
2663     }
2664 
2665     hr = whp_dispatch.WHvSetPartitionProperty(
2666             whpx->partition,
2667             WHvPartitionPropertyCodeExtendedVmExits,
2668             &prop,
2669             sizeof(WHV_PARTITION_PROPERTY));
2670     if (FAILED(hr)) {
2671         error_report("WHPX: Failed to enable MSR & CPUIDexit, hr=%08lx", hr);
2672         ret = -EINVAL;
2673         goto error;
2674     }
2675 
2676     hr = whp_dispatch.WHvSetPartitionProperty(
2677         whpx->partition,
2678         WHvPartitionPropertyCodeCpuidExitList,
2679         cpuidExitList,
2680         RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
2681 
2682     if (FAILED(hr)) {
2683         error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
2684                      hr);
2685         ret = -EINVAL;
2686         goto error;
2687     }
2688 
2689     /*
2690      * We do not want to intercept any exceptions from the guest,
2691      * until we actually start debugging with gdb.
2692      */
2693     whpx->exception_exit_bitmap = -1;
2694     hr = whpx_set_exception_exit_bitmap(0);
2695 
2696     if (FAILED(hr)) {
2697         error_report("WHPX: Failed to set exception exit bitmap, hr=%08lx", hr);
2698         ret = -EINVAL;
2699         goto error;
2700     }
2701 
2702     hr = whp_dispatch.WHvSetupPartition(whpx->partition);
2703     if (FAILED(hr)) {
2704         error_report("WHPX: Failed to setup partition, hr=%08lx", hr);
2705         ret = -EINVAL;
2706         goto error;
2707     }
2708 
2709     whpx_memory_init();
2710 
2711     printf("Windows Hypervisor Platform accelerator is operational\n");
2712     return 0;
2713 
2714 error:
2715 
2716     if (NULL != whpx->partition) {
2717         whp_dispatch.WHvDeletePartition(whpx->partition);
2718         whpx->partition = NULL;
2719     }
2720 
2721     return ret;
2722 }
2723 
2724 int whpx_enabled(void)
2725 {
2726     return whpx_allowed;
2727 }
2728 
2729 bool whpx_apic_in_platform(void) {
2730     return whpx_global.apic_in_platform;
2731 }
2732 
2733 static void whpx_accel_class_init(ObjectClass *oc, void *data)
2734 {
2735     AccelClass *ac = ACCEL_CLASS(oc);
2736     ac->name = "WHPX";
2737     ac->init_machine = whpx_accel_init;
2738     ac->allowed = &whpx_allowed;
2739 
2740     object_class_property_add(oc, "kernel-irqchip", "on|off|split",
2741         NULL, whpx_set_kernel_irqchip,
2742         NULL, NULL);
2743     object_class_property_set_description(oc, "kernel-irqchip",
2744         "Configure WHPX in-kernel irqchip");
2745 }
2746 
2747 static void whpx_accel_instance_init(Object *obj)
2748 {
2749     struct whpx_state *whpx = &whpx_global;
2750 
2751     memset(whpx, 0, sizeof(struct whpx_state));
2752     /* Turn on kernel-irqchip, by default */
2753     whpx->kernel_irqchip_allowed = true;
2754 }
2755 
2756 static const TypeInfo whpx_accel_type = {
2757     .name = ACCEL_CLASS_NAME("whpx"),
2758     .parent = TYPE_ACCEL,
2759     .instance_init = whpx_accel_instance_init,
2760     .class_init = whpx_accel_class_init,
2761 };
2762 
2763 static void whpx_type_init(void)
2764 {
2765     type_register_static(&whpx_accel_type);
2766 }
2767 
2768 bool init_whp_dispatch(void)
2769 {
2770     if (whp_dispatch_initialized) {
2771         return true;
2772     }
2773 
2774     if (!load_whp_dispatch_fns(&hWinHvPlatform, WINHV_PLATFORM_FNS_DEFAULT)) {
2775         goto error;
2776     }
2777 
2778     if (!load_whp_dispatch_fns(&hWinHvEmulation, WINHV_EMULATION_FNS_DEFAULT)) {
2779         goto error;
2780     }
2781 
2782     assert(load_whp_dispatch_fns(&hWinHvPlatform,
2783         WINHV_PLATFORM_FNS_SUPPLEMENTAL));
2784     whp_dispatch_initialized = true;
2785 
2786     return true;
2787 error:
2788     if (hWinHvPlatform) {
2789         FreeLibrary(hWinHvPlatform);
2790     }
2791 
2792     if (hWinHvEmulation) {
2793         FreeLibrary(hWinHvEmulation);
2794     }
2795 
2796     return false;
2797 }
2798 
2799 type_init(whpx_type_init);
2800