1 /*
2 * Copyright (c) 2018-2019 Maxime Villard, All rights reserved.
3 *
4 * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU.
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
8 */
9
10 #include "qemu/osdep.h"
11 #include "cpu.h"
12 #include "exec/address-spaces.h"
13 #include "exec/ioport.h"
14 #include "qemu/accel.h"
15 #include "sysemu/nvmm.h"
16 #include "sysemu/cpus.h"
17 #include "sysemu/runstate.h"
18 #include "qemu/main-loop.h"
19 #include "qemu/error-report.h"
20 #include "qapi/error.h"
21 #include "qemu/queue.h"
22 #include "migration/blocker.h"
23 #include "strings.h"
24
25 #include "nvmm-accel-ops.h"
26
27 #include <nvmm.h>
28
29 struct AccelCPUState {
30 struct nvmm_vcpu vcpu;
31 uint8_t tpr;
32 bool stop;
33
34 /* Window-exiting for INTs/NMIs. */
35 bool int_window_exit;
36 bool nmi_window_exit;
37
38 /* The guest is in an interrupt shadow (POP SS, etc). */
39 bool int_shadow;
40 };
41
42 struct qemu_machine {
43 struct nvmm_capability cap;
44 struct nvmm_machine mach;
45 };
46
47 /* -------------------------------------------------------------------------- */
48
49 static bool nvmm_allowed;
50 static struct qemu_machine qemu_mach;
51
52 static struct nvmm_machine *
get_nvmm_mach(void)53 get_nvmm_mach(void)
54 {
55 return &qemu_mach.mach;
56 }
57
58 /* -------------------------------------------------------------------------- */
59
60 static void
nvmm_set_segment(struct nvmm_x64_state_seg * nseg,const SegmentCache * qseg)61 nvmm_set_segment(struct nvmm_x64_state_seg *nseg, const SegmentCache *qseg)
62 {
63 uint32_t attrib = qseg->flags;
64
65 nseg->selector = qseg->selector;
66 nseg->limit = qseg->limit;
67 nseg->base = qseg->base;
68 nseg->attrib.type = __SHIFTOUT(attrib, DESC_TYPE_MASK);
69 nseg->attrib.s = __SHIFTOUT(attrib, DESC_S_MASK);
70 nseg->attrib.dpl = __SHIFTOUT(attrib, DESC_DPL_MASK);
71 nseg->attrib.p = __SHIFTOUT(attrib, DESC_P_MASK);
72 nseg->attrib.avl = __SHIFTOUT(attrib, DESC_AVL_MASK);
73 nseg->attrib.l = __SHIFTOUT(attrib, DESC_L_MASK);
74 nseg->attrib.def = __SHIFTOUT(attrib, DESC_B_MASK);
75 nseg->attrib.g = __SHIFTOUT(attrib, DESC_G_MASK);
76 }
77
78 static void
nvmm_set_registers(CPUState * cpu)79 nvmm_set_registers(CPUState *cpu)
80 {
81 CPUX86State *env = cpu_env(cpu);
82 struct nvmm_machine *mach = get_nvmm_mach();
83 AccelCPUState *qcpu = cpu->accel;
84 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
85 struct nvmm_x64_state *state = vcpu->state;
86 uint64_t bitmap;
87 size_t i;
88 int ret;
89
90 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
91
92 /* GPRs. */
93 state->gprs[NVMM_X64_GPR_RAX] = env->regs[R_EAX];
94 state->gprs[NVMM_X64_GPR_RCX] = env->regs[R_ECX];
95 state->gprs[NVMM_X64_GPR_RDX] = env->regs[R_EDX];
96 state->gprs[NVMM_X64_GPR_RBX] = env->regs[R_EBX];
97 state->gprs[NVMM_X64_GPR_RSP] = env->regs[R_ESP];
98 state->gprs[NVMM_X64_GPR_RBP] = env->regs[R_EBP];
99 state->gprs[NVMM_X64_GPR_RSI] = env->regs[R_ESI];
100 state->gprs[NVMM_X64_GPR_RDI] = env->regs[R_EDI];
101 #ifdef TARGET_X86_64
102 state->gprs[NVMM_X64_GPR_R8] = env->regs[R_R8];
103 state->gprs[NVMM_X64_GPR_R9] = env->regs[R_R9];
104 state->gprs[NVMM_X64_GPR_R10] = env->regs[R_R10];
105 state->gprs[NVMM_X64_GPR_R11] = env->regs[R_R11];
106 state->gprs[NVMM_X64_GPR_R12] = env->regs[R_R12];
107 state->gprs[NVMM_X64_GPR_R13] = env->regs[R_R13];
108 state->gprs[NVMM_X64_GPR_R14] = env->regs[R_R14];
109 state->gprs[NVMM_X64_GPR_R15] = env->regs[R_R15];
110 #endif
111
112 /* RIP and RFLAGS. */
113 state->gprs[NVMM_X64_GPR_RIP] = env->eip;
114 state->gprs[NVMM_X64_GPR_RFLAGS] = env->eflags;
115
116 /* Segments. */
117 nvmm_set_segment(&state->segs[NVMM_X64_SEG_CS], &env->segs[R_CS]);
118 nvmm_set_segment(&state->segs[NVMM_X64_SEG_DS], &env->segs[R_DS]);
119 nvmm_set_segment(&state->segs[NVMM_X64_SEG_ES], &env->segs[R_ES]);
120 nvmm_set_segment(&state->segs[NVMM_X64_SEG_FS], &env->segs[R_FS]);
121 nvmm_set_segment(&state->segs[NVMM_X64_SEG_GS], &env->segs[R_GS]);
122 nvmm_set_segment(&state->segs[NVMM_X64_SEG_SS], &env->segs[R_SS]);
123
124 /* Special segments. */
125 nvmm_set_segment(&state->segs[NVMM_X64_SEG_GDT], &env->gdt);
126 nvmm_set_segment(&state->segs[NVMM_X64_SEG_LDT], &env->ldt);
127 nvmm_set_segment(&state->segs[NVMM_X64_SEG_TR], &env->tr);
128 nvmm_set_segment(&state->segs[NVMM_X64_SEG_IDT], &env->idt);
129
130 /* Control registers. */
131 state->crs[NVMM_X64_CR_CR0] = env->cr[0];
132 state->crs[NVMM_X64_CR_CR2] = env->cr[2];
133 state->crs[NVMM_X64_CR_CR3] = env->cr[3];
134 state->crs[NVMM_X64_CR_CR4] = env->cr[4];
135 state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
136 state->crs[NVMM_X64_CR_XCR0] = env->xcr0;
137
138 /* Debug registers. */
139 state->drs[NVMM_X64_DR_DR0] = env->dr[0];
140 state->drs[NVMM_X64_DR_DR1] = env->dr[1];
141 state->drs[NVMM_X64_DR_DR2] = env->dr[2];
142 state->drs[NVMM_X64_DR_DR3] = env->dr[3];
143 state->drs[NVMM_X64_DR_DR6] = env->dr[6];
144 state->drs[NVMM_X64_DR_DR7] = env->dr[7];
145
146 /* FPU. */
147 state->fpu.fx_cw = env->fpuc;
148 state->fpu.fx_sw = (env->fpus & ~0x3800) | ((env->fpstt & 0x7) << 11);
149 state->fpu.fx_tw = 0;
150 for (i = 0; i < 8; i++) {
151 state->fpu.fx_tw |= (!env->fptags[i]) << i;
152 }
153 state->fpu.fx_opcode = env->fpop;
154 state->fpu.fx_ip.fa_64 = env->fpip;
155 state->fpu.fx_dp.fa_64 = env->fpdp;
156 state->fpu.fx_mxcsr = env->mxcsr;
157 state->fpu.fx_mxcsr_mask = 0x0000FFFF;
158 assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
159 memcpy(state->fpu.fx_87_ac, env->fpregs, sizeof(env->fpregs));
160 for (i = 0; i < CPU_NB_REGS; i++) {
161 memcpy(&state->fpu.fx_xmm[i].xmm_bytes[0],
162 &env->xmm_regs[i].ZMM_Q(0), 8);
163 memcpy(&state->fpu.fx_xmm[i].xmm_bytes[8],
164 &env->xmm_regs[i].ZMM_Q(1), 8);
165 }
166
167 /* MSRs. */
168 state->msrs[NVMM_X64_MSR_EFER] = env->efer;
169 state->msrs[NVMM_X64_MSR_STAR] = env->star;
170 #ifdef TARGET_X86_64
171 state->msrs[NVMM_X64_MSR_LSTAR] = env->lstar;
172 state->msrs[NVMM_X64_MSR_CSTAR] = env->cstar;
173 state->msrs[NVMM_X64_MSR_SFMASK] = env->fmask;
174 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = env->kernelgsbase;
175 #endif
176 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = env->sysenter_cs;
177 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = env->sysenter_esp;
178 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = env->sysenter_eip;
179 state->msrs[NVMM_X64_MSR_PAT] = env->pat;
180 state->msrs[NVMM_X64_MSR_TSC] = env->tsc;
181
182 bitmap =
183 NVMM_X64_STATE_SEGS |
184 NVMM_X64_STATE_GPRS |
185 NVMM_X64_STATE_CRS |
186 NVMM_X64_STATE_DRS |
187 NVMM_X64_STATE_MSRS |
188 NVMM_X64_STATE_FPU;
189
190 ret = nvmm_vcpu_setstate(mach, vcpu, bitmap);
191 if (ret == -1) {
192 error_report("NVMM: Failed to set virtual processor context,"
193 " error=%d", errno);
194 }
195 }
196
197 static void
nvmm_get_segment(SegmentCache * qseg,const struct nvmm_x64_state_seg * nseg)198 nvmm_get_segment(SegmentCache *qseg, const struct nvmm_x64_state_seg *nseg)
199 {
200 qseg->selector = nseg->selector;
201 qseg->limit = nseg->limit;
202 qseg->base = nseg->base;
203
204 qseg->flags =
205 __SHIFTIN((uint32_t)nseg->attrib.type, DESC_TYPE_MASK) |
206 __SHIFTIN((uint32_t)nseg->attrib.s, DESC_S_MASK) |
207 __SHIFTIN((uint32_t)nseg->attrib.dpl, DESC_DPL_MASK) |
208 __SHIFTIN((uint32_t)nseg->attrib.p, DESC_P_MASK) |
209 __SHIFTIN((uint32_t)nseg->attrib.avl, DESC_AVL_MASK) |
210 __SHIFTIN((uint32_t)nseg->attrib.l, DESC_L_MASK) |
211 __SHIFTIN((uint32_t)nseg->attrib.def, DESC_B_MASK) |
212 __SHIFTIN((uint32_t)nseg->attrib.g, DESC_G_MASK);
213 }
214
215 static void
nvmm_get_registers(CPUState * cpu)216 nvmm_get_registers(CPUState *cpu)
217 {
218 CPUX86State *env = cpu_env(cpu);
219 struct nvmm_machine *mach = get_nvmm_mach();
220 AccelCPUState *qcpu = cpu->accel;
221 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
222 X86CPU *x86_cpu = X86_CPU(cpu);
223 struct nvmm_x64_state *state = vcpu->state;
224 uint64_t bitmap, tpr;
225 size_t i;
226 int ret;
227
228 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
229
230 bitmap =
231 NVMM_X64_STATE_SEGS |
232 NVMM_X64_STATE_GPRS |
233 NVMM_X64_STATE_CRS |
234 NVMM_X64_STATE_DRS |
235 NVMM_X64_STATE_MSRS |
236 NVMM_X64_STATE_FPU;
237
238 ret = nvmm_vcpu_getstate(mach, vcpu, bitmap);
239 if (ret == -1) {
240 error_report("NVMM: Failed to get virtual processor context,"
241 " error=%d", errno);
242 }
243
244 /* GPRs. */
245 env->regs[R_EAX] = state->gprs[NVMM_X64_GPR_RAX];
246 env->regs[R_ECX] = state->gprs[NVMM_X64_GPR_RCX];
247 env->regs[R_EDX] = state->gprs[NVMM_X64_GPR_RDX];
248 env->regs[R_EBX] = state->gprs[NVMM_X64_GPR_RBX];
249 env->regs[R_ESP] = state->gprs[NVMM_X64_GPR_RSP];
250 env->regs[R_EBP] = state->gprs[NVMM_X64_GPR_RBP];
251 env->regs[R_ESI] = state->gprs[NVMM_X64_GPR_RSI];
252 env->regs[R_EDI] = state->gprs[NVMM_X64_GPR_RDI];
253 #ifdef TARGET_X86_64
254 env->regs[R_R8] = state->gprs[NVMM_X64_GPR_R8];
255 env->regs[R_R9] = state->gprs[NVMM_X64_GPR_R9];
256 env->regs[R_R10] = state->gprs[NVMM_X64_GPR_R10];
257 env->regs[R_R11] = state->gprs[NVMM_X64_GPR_R11];
258 env->regs[R_R12] = state->gprs[NVMM_X64_GPR_R12];
259 env->regs[R_R13] = state->gprs[NVMM_X64_GPR_R13];
260 env->regs[R_R14] = state->gprs[NVMM_X64_GPR_R14];
261 env->regs[R_R15] = state->gprs[NVMM_X64_GPR_R15];
262 #endif
263
264 /* RIP and RFLAGS. */
265 env->eip = state->gprs[NVMM_X64_GPR_RIP];
266 env->eflags = state->gprs[NVMM_X64_GPR_RFLAGS];
267
268 /* Segments. */
269 nvmm_get_segment(&env->segs[R_ES], &state->segs[NVMM_X64_SEG_ES]);
270 nvmm_get_segment(&env->segs[R_CS], &state->segs[NVMM_X64_SEG_CS]);
271 nvmm_get_segment(&env->segs[R_SS], &state->segs[NVMM_X64_SEG_SS]);
272 nvmm_get_segment(&env->segs[R_DS], &state->segs[NVMM_X64_SEG_DS]);
273 nvmm_get_segment(&env->segs[R_FS], &state->segs[NVMM_X64_SEG_FS]);
274 nvmm_get_segment(&env->segs[R_GS], &state->segs[NVMM_X64_SEG_GS]);
275
276 /* Special segments. */
277 nvmm_get_segment(&env->gdt, &state->segs[NVMM_X64_SEG_GDT]);
278 nvmm_get_segment(&env->ldt, &state->segs[NVMM_X64_SEG_LDT]);
279 nvmm_get_segment(&env->tr, &state->segs[NVMM_X64_SEG_TR]);
280 nvmm_get_segment(&env->idt, &state->segs[NVMM_X64_SEG_IDT]);
281
282 /* Control registers. */
283 env->cr[0] = state->crs[NVMM_X64_CR_CR0];
284 env->cr[2] = state->crs[NVMM_X64_CR_CR2];
285 env->cr[3] = state->crs[NVMM_X64_CR_CR3];
286 env->cr[4] = state->crs[NVMM_X64_CR_CR4];
287 tpr = state->crs[NVMM_X64_CR_CR8];
288 if (tpr != qcpu->tpr) {
289 qcpu->tpr = tpr;
290 cpu_set_apic_tpr(x86_cpu->apic_state, tpr);
291 }
292 env->xcr0 = state->crs[NVMM_X64_CR_XCR0];
293
294 /* Debug registers. */
295 env->dr[0] = state->drs[NVMM_X64_DR_DR0];
296 env->dr[1] = state->drs[NVMM_X64_DR_DR1];
297 env->dr[2] = state->drs[NVMM_X64_DR_DR2];
298 env->dr[3] = state->drs[NVMM_X64_DR_DR3];
299 env->dr[6] = state->drs[NVMM_X64_DR_DR6];
300 env->dr[7] = state->drs[NVMM_X64_DR_DR7];
301
302 /* FPU. */
303 env->fpuc = state->fpu.fx_cw;
304 env->fpstt = (state->fpu.fx_sw >> 11) & 0x7;
305 env->fpus = state->fpu.fx_sw & ~0x3800;
306 for (i = 0; i < 8; i++) {
307 env->fptags[i] = !((state->fpu.fx_tw >> i) & 1);
308 }
309 env->fpop = state->fpu.fx_opcode;
310 env->fpip = state->fpu.fx_ip.fa_64;
311 env->fpdp = state->fpu.fx_dp.fa_64;
312 env->mxcsr = state->fpu.fx_mxcsr;
313 assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs));
314 memcpy(env->fpregs, state->fpu.fx_87_ac, sizeof(env->fpregs));
315 for (i = 0; i < CPU_NB_REGS; i++) {
316 memcpy(&env->xmm_regs[i].ZMM_Q(0),
317 &state->fpu.fx_xmm[i].xmm_bytes[0], 8);
318 memcpy(&env->xmm_regs[i].ZMM_Q(1),
319 &state->fpu.fx_xmm[i].xmm_bytes[8], 8);
320 }
321
322 /* MSRs. */
323 env->efer = state->msrs[NVMM_X64_MSR_EFER];
324 env->star = state->msrs[NVMM_X64_MSR_STAR];
325 #ifdef TARGET_X86_64
326 env->lstar = state->msrs[NVMM_X64_MSR_LSTAR];
327 env->cstar = state->msrs[NVMM_X64_MSR_CSTAR];
328 env->fmask = state->msrs[NVMM_X64_MSR_SFMASK];
329 env->kernelgsbase = state->msrs[NVMM_X64_MSR_KERNELGSBASE];
330 #endif
331 env->sysenter_cs = state->msrs[NVMM_X64_MSR_SYSENTER_CS];
332 env->sysenter_esp = state->msrs[NVMM_X64_MSR_SYSENTER_ESP];
333 env->sysenter_eip = state->msrs[NVMM_X64_MSR_SYSENTER_EIP];
334 env->pat = state->msrs[NVMM_X64_MSR_PAT];
335 env->tsc = state->msrs[NVMM_X64_MSR_TSC];
336
337 x86_update_hflags(env);
338 }
339
340 static bool
nvmm_can_take_int(CPUState * cpu)341 nvmm_can_take_int(CPUState *cpu)
342 {
343 CPUX86State *env = cpu_env(cpu);
344 AccelCPUState *qcpu = cpu->accel;
345 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
346 struct nvmm_machine *mach = get_nvmm_mach();
347
348 if (qcpu->int_window_exit) {
349 return false;
350 }
351
352 if (qcpu->int_shadow || !(env->eflags & IF_MASK)) {
353 struct nvmm_x64_state *state = vcpu->state;
354
355 /* Exit on interrupt window. */
356 nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_INTR);
357 state->intr.int_window_exiting = 1;
358 nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_INTR);
359
360 return false;
361 }
362
363 return true;
364 }
365
366 static bool
nvmm_can_take_nmi(CPUState * cpu)367 nvmm_can_take_nmi(CPUState *cpu)
368 {
369 AccelCPUState *qcpu = cpu->accel;
370
371 /*
372 * Contrary to INTs, NMIs always schedule an exit when they are
373 * completed. Therefore, if window-exiting is enabled, it means
374 * NMIs are blocked.
375 */
376 if (qcpu->nmi_window_exit) {
377 return false;
378 }
379
380 return true;
381 }
382
383 /*
384 * Called before the VCPU is run. We inject events generated by the I/O
385 * thread, and synchronize the guest TPR.
386 */
387 static void
nvmm_vcpu_pre_run(CPUState * cpu)388 nvmm_vcpu_pre_run(CPUState *cpu)
389 {
390 CPUX86State *env = cpu_env(cpu);
391 struct nvmm_machine *mach = get_nvmm_mach();
392 AccelCPUState *qcpu = cpu->accel;
393 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
394 X86CPU *x86_cpu = X86_CPU(cpu);
395 struct nvmm_x64_state *state = vcpu->state;
396 struct nvmm_vcpu_event *event = vcpu->event;
397 bool has_event = false;
398 bool sync_tpr = false;
399 uint8_t tpr;
400 int ret;
401
402 qemu_mutex_lock_iothread();
403
404 tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
405 if (tpr != qcpu->tpr) {
406 qcpu->tpr = tpr;
407 sync_tpr = true;
408 }
409
410 /*
411 * Force the VCPU out of its inner loop to process any INIT requests
412 * or commit pending TPR access.
413 */
414 if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
415 cpu->exit_request = 1;
416 }
417
418 if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
419 if (nvmm_can_take_nmi(cpu)) {
420 cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
421 event->type = NVMM_VCPU_EVENT_INTR;
422 event->vector = 2;
423 has_event = true;
424 }
425 }
426
427 if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
428 if (nvmm_can_take_int(cpu)) {
429 cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
430 event->type = NVMM_VCPU_EVENT_INTR;
431 event->vector = cpu_get_pic_interrupt(env);
432 has_event = true;
433 }
434 }
435
436 /* Don't want SMIs. */
437 if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
438 cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
439 }
440
441 if (sync_tpr) {
442 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_CRS);
443 if (ret == -1) {
444 error_report("NVMM: Failed to get CPU state,"
445 " error=%d", errno);
446 }
447
448 state->crs[NVMM_X64_CR_CR8] = qcpu->tpr;
449
450 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_CRS);
451 if (ret == -1) {
452 error_report("NVMM: Failed to set CPU state,"
453 " error=%d", errno);
454 }
455 }
456
457 if (has_event) {
458 ret = nvmm_vcpu_inject(mach, vcpu);
459 if (ret == -1) {
460 error_report("NVMM: Failed to inject event,"
461 " error=%d", errno);
462 }
463 }
464
465 qemu_mutex_unlock_iothread();
466 }
467
468 /*
469 * Called after the VCPU ran. We synchronize the host view of the TPR and
470 * RFLAGS.
471 */
472 static void
nvmm_vcpu_post_run(CPUState * cpu,struct nvmm_vcpu_exit * exit)473 nvmm_vcpu_post_run(CPUState *cpu, struct nvmm_vcpu_exit *exit)
474 {
475 AccelCPUState *qcpu = cpu->accel;
476 X86CPU *x86_cpu = X86_CPU(cpu);
477 CPUX86State *env = &x86_cpu->env;
478 uint64_t tpr;
479
480 env->eflags = exit->exitstate.rflags;
481 qcpu->int_shadow = exit->exitstate.int_shadow;
482 qcpu->int_window_exit = exit->exitstate.int_window_exiting;
483 qcpu->nmi_window_exit = exit->exitstate.nmi_window_exiting;
484
485 tpr = exit->exitstate.cr8;
486 if (qcpu->tpr != tpr) {
487 qcpu->tpr = tpr;
488 qemu_mutex_lock_iothread();
489 cpu_set_apic_tpr(x86_cpu->apic_state, qcpu->tpr);
490 qemu_mutex_unlock_iothread();
491 }
492 }
493
494 /* -------------------------------------------------------------------------- */
495
496 static void
nvmm_io_callback(struct nvmm_io * io)497 nvmm_io_callback(struct nvmm_io *io)
498 {
499 MemTxAttrs attrs = { 0 };
500 int ret;
501
502 ret = address_space_rw(&address_space_io, io->port, attrs, io->data,
503 io->size, !io->in);
504 if (ret != MEMTX_OK) {
505 error_report("NVMM: I/O Transaction Failed "
506 "[%s, port=%u, size=%zu]", (io->in ? "in" : "out"),
507 io->port, io->size);
508 }
509
510 /* Needed, otherwise infinite loop. */
511 current_cpu->vcpu_dirty = false;
512 }
513
514 static void
nvmm_mem_callback(struct nvmm_mem * mem)515 nvmm_mem_callback(struct nvmm_mem *mem)
516 {
517 cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write);
518
519 /* Needed, otherwise infinite loop. */
520 current_cpu->vcpu_dirty = false;
521 }
522
523 static struct nvmm_assist_callbacks nvmm_callbacks = {
524 .io = nvmm_io_callback,
525 .mem = nvmm_mem_callback
526 };
527
528 /* -------------------------------------------------------------------------- */
529
530 static int
nvmm_handle_mem(struct nvmm_machine * mach,struct nvmm_vcpu * vcpu)531 nvmm_handle_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
532 {
533 int ret;
534
535 ret = nvmm_assist_mem(mach, vcpu);
536 if (ret == -1) {
537 error_report("NVMM: Mem Assist Failed [gpa=%p]",
538 (void *)vcpu->exit->u.mem.gpa);
539 }
540
541 return ret;
542 }
543
544 static int
nvmm_handle_io(struct nvmm_machine * mach,struct nvmm_vcpu * vcpu)545 nvmm_handle_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
546 {
547 int ret;
548
549 ret = nvmm_assist_io(mach, vcpu);
550 if (ret == -1) {
551 error_report("NVMM: I/O Assist Failed [port=%d]",
552 (int)vcpu->exit->u.io.port);
553 }
554
555 return ret;
556 }
557
558 static int
nvmm_handle_rdmsr(struct nvmm_machine * mach,CPUState * cpu,struct nvmm_vcpu_exit * exit)559 nvmm_handle_rdmsr(struct nvmm_machine *mach, CPUState *cpu,
560 struct nvmm_vcpu_exit *exit)
561 {
562 AccelCPUState *qcpu = cpu->accel;
563 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
564 X86CPU *x86_cpu = X86_CPU(cpu);
565 struct nvmm_x64_state *state = vcpu->state;
566 uint64_t val;
567 int ret;
568
569 switch (exit->u.rdmsr.msr) {
570 case MSR_IA32_APICBASE:
571 val = cpu_get_apic_base(x86_cpu->apic_state);
572 break;
573 case MSR_MTRRcap:
574 case MSR_MTRRdefType:
575 case MSR_MCG_CAP:
576 case MSR_MCG_STATUS:
577 val = 0;
578 break;
579 default: /* More MSRs to add? */
580 val = 0;
581 error_report("NVMM: Unexpected RDMSR 0x%x, ignored",
582 exit->u.rdmsr.msr);
583 break;
584 }
585
586 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
587 if (ret == -1) {
588 return -1;
589 }
590
591 state->gprs[NVMM_X64_GPR_RAX] = (val & 0xFFFFFFFF);
592 state->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
593 state->gprs[NVMM_X64_GPR_RIP] = exit->u.rdmsr.npc;
594
595 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
596 if (ret == -1) {
597 return -1;
598 }
599
600 return 0;
601 }
602
603 static int
nvmm_handle_wrmsr(struct nvmm_machine * mach,CPUState * cpu,struct nvmm_vcpu_exit * exit)604 nvmm_handle_wrmsr(struct nvmm_machine *mach, CPUState *cpu,
605 struct nvmm_vcpu_exit *exit)
606 {
607 AccelCPUState *qcpu = cpu->accel;
608 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
609 X86CPU *x86_cpu = X86_CPU(cpu);
610 struct nvmm_x64_state *state = vcpu->state;
611 uint64_t val;
612 int ret;
613
614 val = exit->u.wrmsr.val;
615
616 switch (exit->u.wrmsr.msr) {
617 case MSR_IA32_APICBASE:
618 cpu_set_apic_base(x86_cpu->apic_state, val);
619 break;
620 case MSR_MTRRdefType:
621 case MSR_MCG_STATUS:
622 break;
623 default: /* More MSRs to add? */
624 error_report("NVMM: Unexpected WRMSR 0x%x [val=0x%lx], ignored",
625 exit->u.wrmsr.msr, val);
626 break;
627 }
628
629 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS);
630 if (ret == -1) {
631 return -1;
632 }
633
634 state->gprs[NVMM_X64_GPR_RIP] = exit->u.wrmsr.npc;
635
636 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS);
637 if (ret == -1) {
638 return -1;
639 }
640
641 return 0;
642 }
643
644 static int
nvmm_handle_halted(struct nvmm_machine * mach,CPUState * cpu,struct nvmm_vcpu_exit * exit)645 nvmm_handle_halted(struct nvmm_machine *mach, CPUState *cpu,
646 struct nvmm_vcpu_exit *exit)
647 {
648 CPUX86State *env = cpu_env(cpu);
649 int ret = 0;
650
651 qemu_mutex_lock_iothread();
652
653 if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
654 (env->eflags & IF_MASK)) &&
655 !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
656 cpu->exception_index = EXCP_HLT;
657 cpu->halted = true;
658 ret = 1;
659 }
660
661 qemu_mutex_unlock_iothread();
662
663 return ret;
664 }
665
666 static int
nvmm_inject_ud(struct nvmm_machine * mach,struct nvmm_vcpu * vcpu)667 nvmm_inject_ud(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu)
668 {
669 struct nvmm_vcpu_event *event = vcpu->event;
670
671 event->type = NVMM_VCPU_EVENT_EXCP;
672 event->vector = 6;
673 event->u.excp.error = 0;
674
675 return nvmm_vcpu_inject(mach, vcpu);
676 }
677
678 static int
nvmm_vcpu_loop(CPUState * cpu)679 nvmm_vcpu_loop(CPUState *cpu)
680 {
681 struct nvmm_machine *mach = get_nvmm_mach();
682 AccelCPUState *qcpu = cpu->accel;
683 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
684 X86CPU *x86_cpu = X86_CPU(cpu);
685 CPUX86State *env = &x86_cpu->env;
686 struct nvmm_vcpu_exit *exit = vcpu->exit;
687 int ret;
688
689 /*
690 * Some asynchronous events must be handled outside of the inner
691 * VCPU loop. They are handled here.
692 */
693 if (cpu->interrupt_request & CPU_INTERRUPT_INIT) {
694 nvmm_cpu_synchronize_state(cpu);
695 do_cpu_init(x86_cpu);
696 /* set int/nmi windows back to the reset state */
697 }
698 if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
699 cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
700 apic_poll_irq(x86_cpu->apic_state);
701 }
702 if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
703 (env->eflags & IF_MASK)) ||
704 (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
705 cpu->halted = false;
706 }
707 if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
708 nvmm_cpu_synchronize_state(cpu);
709 do_cpu_sipi(x86_cpu);
710 }
711 if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
712 cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
713 nvmm_cpu_synchronize_state(cpu);
714 apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
715 env->tpr_access_type);
716 }
717
718 if (cpu->halted) {
719 cpu->exception_index = EXCP_HLT;
720 qatomic_set(&cpu->exit_request, false);
721 return 0;
722 }
723
724 qemu_mutex_unlock_iothread();
725 cpu_exec_start(cpu);
726
727 /*
728 * Inner VCPU loop.
729 */
730 do {
731 if (cpu->vcpu_dirty) {
732 nvmm_set_registers(cpu);
733 cpu->vcpu_dirty = false;
734 }
735
736 if (qcpu->stop) {
737 cpu->exception_index = EXCP_INTERRUPT;
738 qcpu->stop = false;
739 ret = 1;
740 break;
741 }
742
743 nvmm_vcpu_pre_run(cpu);
744
745 if (qatomic_read(&cpu->exit_request)) {
746 #if NVMM_USER_VERSION >= 2
747 nvmm_vcpu_stop(vcpu);
748 #else
749 qemu_cpu_kick_self();
750 #endif
751 }
752
753 /* Read exit_request before the kernel reads the immediate exit flag */
754 smp_rmb();
755 ret = nvmm_vcpu_run(mach, vcpu);
756 if (ret == -1) {
757 error_report("NVMM: Failed to exec a virtual processor,"
758 " error=%d", errno);
759 break;
760 }
761
762 nvmm_vcpu_post_run(cpu, exit);
763
764 switch (exit->reason) {
765 case NVMM_VCPU_EXIT_NONE:
766 break;
767 #if NVMM_USER_VERSION >= 2
768 case NVMM_VCPU_EXIT_STOPPED:
769 /*
770 * The kernel cleared the immediate exit flag; cpu->exit_request
771 * must be cleared after
772 */
773 smp_wmb();
774 qcpu->stop = true;
775 break;
776 #endif
777 case NVMM_VCPU_EXIT_MEMORY:
778 ret = nvmm_handle_mem(mach, vcpu);
779 break;
780 case NVMM_VCPU_EXIT_IO:
781 ret = nvmm_handle_io(mach, vcpu);
782 break;
783 case NVMM_VCPU_EXIT_INT_READY:
784 case NVMM_VCPU_EXIT_NMI_READY:
785 case NVMM_VCPU_EXIT_TPR_CHANGED:
786 break;
787 case NVMM_VCPU_EXIT_HALTED:
788 ret = nvmm_handle_halted(mach, cpu, exit);
789 break;
790 case NVMM_VCPU_EXIT_SHUTDOWN:
791 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
792 cpu->exception_index = EXCP_INTERRUPT;
793 ret = 1;
794 break;
795 case NVMM_VCPU_EXIT_RDMSR:
796 ret = nvmm_handle_rdmsr(mach, cpu, exit);
797 break;
798 case NVMM_VCPU_EXIT_WRMSR:
799 ret = nvmm_handle_wrmsr(mach, cpu, exit);
800 break;
801 case NVMM_VCPU_EXIT_MONITOR:
802 case NVMM_VCPU_EXIT_MWAIT:
803 ret = nvmm_inject_ud(mach, vcpu);
804 break;
805 default:
806 error_report("NVMM: Unexpected VM exit code 0x%lx [hw=0x%lx]",
807 exit->reason, exit->u.inv.hwcode);
808 nvmm_get_registers(cpu);
809 qemu_mutex_lock_iothread();
810 qemu_system_guest_panicked(cpu_get_crash_info(cpu));
811 qemu_mutex_unlock_iothread();
812 ret = -1;
813 break;
814 }
815 } while (ret == 0);
816
817 cpu_exec_end(cpu);
818 qemu_mutex_lock_iothread();
819
820 qatomic_set(&cpu->exit_request, false);
821
822 return ret < 0;
823 }
824
825 /* -------------------------------------------------------------------------- */
826
827 static void
do_nvmm_cpu_synchronize_state(CPUState * cpu,run_on_cpu_data arg)828 do_nvmm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
829 {
830 nvmm_get_registers(cpu);
831 cpu->vcpu_dirty = true;
832 }
833
834 static void
do_nvmm_cpu_synchronize_post_reset(CPUState * cpu,run_on_cpu_data arg)835 do_nvmm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
836 {
837 nvmm_set_registers(cpu);
838 cpu->vcpu_dirty = false;
839 }
840
841 static void
do_nvmm_cpu_synchronize_post_init(CPUState * cpu,run_on_cpu_data arg)842 do_nvmm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
843 {
844 nvmm_set_registers(cpu);
845 cpu->vcpu_dirty = false;
846 }
847
848 static void
do_nvmm_cpu_synchronize_pre_loadvm(CPUState * cpu,run_on_cpu_data arg)849 do_nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
850 {
851 cpu->vcpu_dirty = true;
852 }
853
nvmm_cpu_synchronize_state(CPUState * cpu)854 void nvmm_cpu_synchronize_state(CPUState *cpu)
855 {
856 if (!cpu->vcpu_dirty) {
857 run_on_cpu(cpu, do_nvmm_cpu_synchronize_state, RUN_ON_CPU_NULL);
858 }
859 }
860
nvmm_cpu_synchronize_post_reset(CPUState * cpu)861 void nvmm_cpu_synchronize_post_reset(CPUState *cpu)
862 {
863 run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
864 }
865
nvmm_cpu_synchronize_post_init(CPUState * cpu)866 void nvmm_cpu_synchronize_post_init(CPUState *cpu)
867 {
868 run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
869 }
870
nvmm_cpu_synchronize_pre_loadvm(CPUState * cpu)871 void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu)
872 {
873 run_on_cpu(cpu, do_nvmm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
874 }
875
876 /* -------------------------------------------------------------------------- */
877
878 static Error *nvmm_migration_blocker;
879
880 /*
881 * The nvmm_vcpu_stop() mechanism breaks races between entering the VMM
882 * and another thread signaling the vCPU thread to exit.
883 */
884
885 static void
nvmm_ipi_signal(int sigcpu)886 nvmm_ipi_signal(int sigcpu)
887 {
888 if (current_cpu) {
889 AccelCPUState *qcpu = current_cpu->accel;
890 #if NVMM_USER_VERSION >= 2
891 struct nvmm_vcpu *vcpu = &qcpu->vcpu;
892 nvmm_vcpu_stop(vcpu);
893 #else
894 qcpu->stop = true;
895 #endif
896 }
897 }
898
899 static void
nvmm_init_cpu_signals(void)900 nvmm_init_cpu_signals(void)
901 {
902 struct sigaction sigact;
903 sigset_t set;
904
905 /* Install the IPI handler. */
906 memset(&sigact, 0, sizeof(sigact));
907 sigact.sa_handler = nvmm_ipi_signal;
908 sigaction(SIG_IPI, &sigact, NULL);
909
910 /* Allow IPIs on the current thread. */
911 sigprocmask(SIG_BLOCK, NULL, &set);
912 sigdelset(&set, SIG_IPI);
913 pthread_sigmask(SIG_SETMASK, &set, NULL);
914 }
915
916 int
nvmm_init_vcpu(CPUState * cpu)917 nvmm_init_vcpu(CPUState *cpu)
918 {
919 struct nvmm_machine *mach = get_nvmm_mach();
920 struct nvmm_vcpu_conf_cpuid cpuid;
921 struct nvmm_vcpu_conf_tpr tpr;
922 Error *local_error = NULL;
923 AccelCPUState *qcpu;
924 int ret, err;
925
926 nvmm_init_cpu_signals();
927
928 if (nvmm_migration_blocker == NULL) {
929 error_setg(&nvmm_migration_blocker,
930 "NVMM: Migration not supported");
931
932 if (migrate_add_blocker(&nvmm_migration_blocker, &local_error) < 0) {
933 error_report_err(local_error);
934 return -EINVAL;
935 }
936 }
937
938 qcpu = g_new0(AccelCPUState, 1);
939
940 ret = nvmm_vcpu_create(mach, cpu->cpu_index, &qcpu->vcpu);
941 if (ret == -1) {
942 err = errno;
943 error_report("NVMM: Failed to create a virtual processor,"
944 " error=%d", err);
945 g_free(qcpu);
946 return -err;
947 }
948
949 memset(&cpuid, 0, sizeof(cpuid));
950 cpuid.mask = 1;
951 cpuid.leaf = 0x00000001;
952 cpuid.u.mask.set.edx = CPUID_MCE | CPUID_MCA | CPUID_MTRR;
953 ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CPUID,
954 &cpuid);
955 if (ret == -1) {
956 err = errno;
957 error_report("NVMM: Failed to configure a virtual processor,"
958 " error=%d", err);
959 g_free(qcpu);
960 return -err;
961 }
962
963 ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CALLBACKS,
964 &nvmm_callbacks);
965 if (ret == -1) {
966 err = errno;
967 error_report("NVMM: Failed to configure a virtual processor,"
968 " error=%d", err);
969 g_free(qcpu);
970 return -err;
971 }
972
973 if (qemu_mach.cap.arch.vcpu_conf_support & NVMM_CAP_ARCH_VCPU_CONF_TPR) {
974 memset(&tpr, 0, sizeof(tpr));
975 tpr.exit_changed = 1;
976 ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_TPR, &tpr);
977 if (ret == -1) {
978 err = errno;
979 error_report("NVMM: Failed to configure a virtual processor,"
980 " error=%d", err);
981 g_free(qcpu);
982 return -err;
983 }
984 }
985
986 cpu->vcpu_dirty = true;
987 cpu->accel = qcpu;
988
989 return 0;
990 }
991
992 int
nvmm_vcpu_exec(CPUState * cpu)993 nvmm_vcpu_exec(CPUState *cpu)
994 {
995 int ret, fatal;
996
997 while (1) {
998 if (cpu->exception_index >= EXCP_INTERRUPT) {
999 ret = cpu->exception_index;
1000 cpu->exception_index = -1;
1001 break;
1002 }
1003
1004 fatal = nvmm_vcpu_loop(cpu);
1005
1006 if (fatal) {
1007 error_report("NVMM: Failed to execute a VCPU.");
1008 abort();
1009 }
1010 }
1011
1012 return ret;
1013 }
1014
1015 void
nvmm_destroy_vcpu(CPUState * cpu)1016 nvmm_destroy_vcpu(CPUState *cpu)
1017 {
1018 struct nvmm_machine *mach = get_nvmm_mach();
1019 AccelCPUState *qcpu = cpu->accel;
1020
1021 nvmm_vcpu_destroy(mach, &qcpu->vcpu);
1022 g_free(cpu->accel);
1023 }
1024
1025 /* -------------------------------------------------------------------------- */
1026
1027 static void
nvmm_update_mapping(hwaddr start_pa,ram_addr_t size,uintptr_t hva,bool add,bool rom,const char * name)1028 nvmm_update_mapping(hwaddr start_pa, ram_addr_t size, uintptr_t hva,
1029 bool add, bool rom, const char *name)
1030 {
1031 struct nvmm_machine *mach = get_nvmm_mach();
1032 int ret, prot;
1033
1034 if (add) {
1035 prot = PROT_READ | PROT_EXEC;
1036 if (!rom) {
1037 prot |= PROT_WRITE;
1038 }
1039 ret = nvmm_gpa_map(mach, hva, start_pa, size, prot);
1040 } else {
1041 ret = nvmm_gpa_unmap(mach, hva, start_pa, size);
1042 }
1043
1044 if (ret == -1) {
1045 error_report("NVMM: Failed to %s GPA range '%s' PA:%p, "
1046 "Size:%p bytes, HostVA:%p, error=%d",
1047 (add ? "map" : "unmap"), name, (void *)(uintptr_t)start_pa,
1048 (void *)size, (void *)hva, errno);
1049 }
1050 }
1051
1052 static void
nvmm_process_section(MemoryRegionSection * section,int add)1053 nvmm_process_section(MemoryRegionSection *section, int add)
1054 {
1055 MemoryRegion *mr = section->mr;
1056 hwaddr start_pa = section->offset_within_address_space;
1057 ram_addr_t size = int128_get64(section->size);
1058 unsigned int delta;
1059 uintptr_t hva;
1060
1061 if (!memory_region_is_ram(mr)) {
1062 return;
1063 }
1064
1065 /* Adjust start_pa and size so that they are page-aligned. */
1066 delta = qemu_real_host_page_size() - (start_pa & ~qemu_real_host_page_mask());
1067 delta &= ~qemu_real_host_page_mask();
1068 if (delta > size) {
1069 return;
1070 }
1071 start_pa += delta;
1072 size -= delta;
1073 size &= qemu_real_host_page_mask();
1074 if (!size || (start_pa & ~qemu_real_host_page_mask())) {
1075 return;
1076 }
1077
1078 hva = (uintptr_t)memory_region_get_ram_ptr(mr) +
1079 section->offset_within_region + delta;
1080
1081 nvmm_update_mapping(start_pa, size, hva, add,
1082 memory_region_is_rom(mr), mr->name);
1083 }
1084
1085 static void
nvmm_region_add(MemoryListener * listener,MemoryRegionSection * section)1086 nvmm_region_add(MemoryListener *listener, MemoryRegionSection *section)
1087 {
1088 memory_region_ref(section->mr);
1089 nvmm_process_section(section, 1);
1090 }
1091
1092 static void
nvmm_region_del(MemoryListener * listener,MemoryRegionSection * section)1093 nvmm_region_del(MemoryListener *listener, MemoryRegionSection *section)
1094 {
1095 nvmm_process_section(section, 0);
1096 memory_region_unref(section->mr);
1097 }
1098
1099 static void
nvmm_transaction_begin(MemoryListener * listener)1100 nvmm_transaction_begin(MemoryListener *listener)
1101 {
1102 /* nothing */
1103 }
1104
1105 static void
nvmm_transaction_commit(MemoryListener * listener)1106 nvmm_transaction_commit(MemoryListener *listener)
1107 {
1108 /* nothing */
1109 }
1110
1111 static void
nvmm_log_sync(MemoryListener * listener,MemoryRegionSection * section)1112 nvmm_log_sync(MemoryListener *listener, MemoryRegionSection *section)
1113 {
1114 MemoryRegion *mr = section->mr;
1115
1116 if (!memory_region_is_ram(mr)) {
1117 return;
1118 }
1119
1120 memory_region_set_dirty(mr, 0, int128_get64(section->size));
1121 }
1122
1123 static MemoryListener nvmm_memory_listener = {
1124 .name = "nvmm",
1125 .begin = nvmm_transaction_begin,
1126 .commit = nvmm_transaction_commit,
1127 .region_add = nvmm_region_add,
1128 .region_del = nvmm_region_del,
1129 .log_sync = nvmm_log_sync,
1130 .priority = MEMORY_LISTENER_PRIORITY_ACCEL,
1131 };
1132
1133 static void
nvmm_ram_block_added(RAMBlockNotifier * n,void * host,size_t size,size_t max_size)1134 nvmm_ram_block_added(RAMBlockNotifier *n, void *host, size_t size,
1135 size_t max_size)
1136 {
1137 struct nvmm_machine *mach = get_nvmm_mach();
1138 uintptr_t hva = (uintptr_t)host;
1139 int ret;
1140
1141 ret = nvmm_hva_map(mach, hva, max_size);
1142
1143 if (ret == -1) {
1144 error_report("NVMM: Failed to map HVA, HostVA:%p "
1145 "Size:%p bytes, error=%d",
1146 (void *)hva, (void *)size, errno);
1147 }
1148 }
1149
1150 static struct RAMBlockNotifier nvmm_ram_notifier = {
1151 .ram_block_added = nvmm_ram_block_added
1152 };
1153
1154 /* -------------------------------------------------------------------------- */
1155
1156 static int
nvmm_accel_init(MachineState * ms)1157 nvmm_accel_init(MachineState *ms)
1158 {
1159 int ret, err;
1160
1161 ret = nvmm_init();
1162 if (ret == -1) {
1163 err = errno;
1164 error_report("NVMM: Initialization failed, error=%d", errno);
1165 return -err;
1166 }
1167
1168 ret = nvmm_capability(&qemu_mach.cap);
1169 if (ret == -1) {
1170 err = errno;
1171 error_report("NVMM: Unable to fetch capability, error=%d", errno);
1172 return -err;
1173 }
1174 if (qemu_mach.cap.version < NVMM_KERN_VERSION) {
1175 error_report("NVMM: Unsupported version %u", qemu_mach.cap.version);
1176 return -EPROGMISMATCH;
1177 }
1178 if (qemu_mach.cap.state_size != sizeof(struct nvmm_x64_state)) {
1179 error_report("NVMM: Wrong state size %u", qemu_mach.cap.state_size);
1180 return -EPROGMISMATCH;
1181 }
1182
1183 ret = nvmm_machine_create(&qemu_mach.mach);
1184 if (ret == -1) {
1185 err = errno;
1186 error_report("NVMM: Machine creation failed, error=%d", errno);
1187 return -err;
1188 }
1189
1190 memory_listener_register(&nvmm_memory_listener, &address_space_memory);
1191 ram_block_notifier_add(&nvmm_ram_notifier);
1192
1193 printf("NetBSD Virtual Machine Monitor accelerator is operational\n");
1194 return 0;
1195 }
1196
1197 int
nvmm_enabled(void)1198 nvmm_enabled(void)
1199 {
1200 return nvmm_allowed;
1201 }
1202
1203 static void
nvmm_accel_class_init(ObjectClass * oc,void * data)1204 nvmm_accel_class_init(ObjectClass *oc, void *data)
1205 {
1206 AccelClass *ac = ACCEL_CLASS(oc);
1207 ac->name = "NVMM";
1208 ac->init_machine = nvmm_accel_init;
1209 ac->allowed = &nvmm_allowed;
1210 }
1211
1212 static const TypeInfo nvmm_accel_type = {
1213 .name = ACCEL_CLASS_NAME("nvmm"),
1214 .parent = TYPE_ACCEL,
1215 .class_init = nvmm_accel_class_init,
1216 };
1217
1218 static void
nvmm_type_init(void)1219 nvmm_type_init(void)
1220 {
1221 type_register_static(&nvmm_accel_type);
1222 }
1223
1224 type_init(nvmm_type_init);
1225