1 /* 2 * Copyright (c) 2018-2019 Maxime Villard, All rights reserved. 3 * 4 * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU. 5 * 6 * This work is licensed under the terms of the GNU GPL, version 2 or later. 7 * See the COPYING file in the top-level directory. 8 */ 9 10 #include "qemu/osdep.h" 11 #include "cpu.h" 12 #include "exec/address-spaces.h" 13 #include "exec/ioport.h" 14 #include "qemu/accel.h" 15 #include "sysemu/nvmm.h" 16 #include "sysemu/cpus.h" 17 #include "sysemu/runstate.h" 18 #include "qemu/main-loop.h" 19 #include "qemu/error-report.h" 20 #include "qapi/error.h" 21 #include "qemu/queue.h" 22 #include "migration/blocker.h" 23 #include "strings.h" 24 25 #include "nvmm-accel-ops.h" 26 27 #include <nvmm.h> 28 29 struct AccelCPUState { 30 struct nvmm_vcpu vcpu; 31 uint8_t tpr; 32 bool stop; 33 34 /* Window-exiting for INTs/NMIs. */ 35 bool int_window_exit; 36 bool nmi_window_exit; 37 38 /* The guest is in an interrupt shadow (POP SS, etc). */ 39 bool int_shadow; 40 }; 41 42 struct qemu_machine { 43 struct nvmm_capability cap; 44 struct nvmm_machine mach; 45 }; 46 47 /* -------------------------------------------------------------------------- */ 48 49 static bool nvmm_allowed; 50 static struct qemu_machine qemu_mach; 51 52 static struct nvmm_machine * 53 get_nvmm_mach(void) 54 { 55 return &qemu_mach.mach; 56 } 57 58 /* -------------------------------------------------------------------------- */ 59 60 static void 61 nvmm_set_segment(struct nvmm_x64_state_seg *nseg, const SegmentCache *qseg) 62 { 63 uint32_t attrib = qseg->flags; 64 65 nseg->selector = qseg->selector; 66 nseg->limit = qseg->limit; 67 nseg->base = qseg->base; 68 nseg->attrib.type = __SHIFTOUT(attrib, DESC_TYPE_MASK); 69 nseg->attrib.s = __SHIFTOUT(attrib, DESC_S_MASK); 70 nseg->attrib.dpl = __SHIFTOUT(attrib, DESC_DPL_MASK); 71 nseg->attrib.p = __SHIFTOUT(attrib, DESC_P_MASK); 72 nseg->attrib.avl = __SHIFTOUT(attrib, DESC_AVL_MASK); 73 nseg->attrib.l = __SHIFTOUT(attrib, DESC_L_MASK); 74 nseg->attrib.def = __SHIFTOUT(attrib, DESC_B_MASK); 75 nseg->attrib.g = __SHIFTOUT(attrib, DESC_G_MASK); 76 } 77 78 static void 79 nvmm_set_registers(CPUState *cpu) 80 { 81 CPUX86State *env = cpu->env_ptr; 82 struct nvmm_machine *mach = get_nvmm_mach(); 83 AccelCPUState *qcpu = cpu->accel; 84 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 85 struct nvmm_x64_state *state = vcpu->state; 86 uint64_t bitmap; 87 size_t i; 88 int ret; 89 90 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu)); 91 92 /* GPRs. */ 93 state->gprs[NVMM_X64_GPR_RAX] = env->regs[R_EAX]; 94 state->gprs[NVMM_X64_GPR_RCX] = env->regs[R_ECX]; 95 state->gprs[NVMM_X64_GPR_RDX] = env->regs[R_EDX]; 96 state->gprs[NVMM_X64_GPR_RBX] = env->regs[R_EBX]; 97 state->gprs[NVMM_X64_GPR_RSP] = env->regs[R_ESP]; 98 state->gprs[NVMM_X64_GPR_RBP] = env->regs[R_EBP]; 99 state->gprs[NVMM_X64_GPR_RSI] = env->regs[R_ESI]; 100 state->gprs[NVMM_X64_GPR_RDI] = env->regs[R_EDI]; 101 #ifdef TARGET_X86_64 102 state->gprs[NVMM_X64_GPR_R8] = env->regs[R_R8]; 103 state->gprs[NVMM_X64_GPR_R9] = env->regs[R_R9]; 104 state->gprs[NVMM_X64_GPR_R10] = env->regs[R_R10]; 105 state->gprs[NVMM_X64_GPR_R11] = env->regs[R_R11]; 106 state->gprs[NVMM_X64_GPR_R12] = env->regs[R_R12]; 107 state->gprs[NVMM_X64_GPR_R13] = env->regs[R_R13]; 108 state->gprs[NVMM_X64_GPR_R14] = env->regs[R_R14]; 109 state->gprs[NVMM_X64_GPR_R15] = env->regs[R_R15]; 110 #endif 111 112 /* RIP and RFLAGS. */ 113 state->gprs[NVMM_X64_GPR_RIP] = env->eip; 114 state->gprs[NVMM_X64_GPR_RFLAGS] = env->eflags; 115 116 /* Segments. */ 117 nvmm_set_segment(&state->segs[NVMM_X64_SEG_CS], &env->segs[R_CS]); 118 nvmm_set_segment(&state->segs[NVMM_X64_SEG_DS], &env->segs[R_DS]); 119 nvmm_set_segment(&state->segs[NVMM_X64_SEG_ES], &env->segs[R_ES]); 120 nvmm_set_segment(&state->segs[NVMM_X64_SEG_FS], &env->segs[R_FS]); 121 nvmm_set_segment(&state->segs[NVMM_X64_SEG_GS], &env->segs[R_GS]); 122 nvmm_set_segment(&state->segs[NVMM_X64_SEG_SS], &env->segs[R_SS]); 123 124 /* Special segments. */ 125 nvmm_set_segment(&state->segs[NVMM_X64_SEG_GDT], &env->gdt); 126 nvmm_set_segment(&state->segs[NVMM_X64_SEG_LDT], &env->ldt); 127 nvmm_set_segment(&state->segs[NVMM_X64_SEG_TR], &env->tr); 128 nvmm_set_segment(&state->segs[NVMM_X64_SEG_IDT], &env->idt); 129 130 /* Control registers. */ 131 state->crs[NVMM_X64_CR_CR0] = env->cr[0]; 132 state->crs[NVMM_X64_CR_CR2] = env->cr[2]; 133 state->crs[NVMM_X64_CR_CR3] = env->cr[3]; 134 state->crs[NVMM_X64_CR_CR4] = env->cr[4]; 135 state->crs[NVMM_X64_CR_CR8] = qcpu->tpr; 136 state->crs[NVMM_X64_CR_XCR0] = env->xcr0; 137 138 /* Debug registers. */ 139 state->drs[NVMM_X64_DR_DR0] = env->dr[0]; 140 state->drs[NVMM_X64_DR_DR1] = env->dr[1]; 141 state->drs[NVMM_X64_DR_DR2] = env->dr[2]; 142 state->drs[NVMM_X64_DR_DR3] = env->dr[3]; 143 state->drs[NVMM_X64_DR_DR6] = env->dr[6]; 144 state->drs[NVMM_X64_DR_DR7] = env->dr[7]; 145 146 /* FPU. */ 147 state->fpu.fx_cw = env->fpuc; 148 state->fpu.fx_sw = (env->fpus & ~0x3800) | ((env->fpstt & 0x7) << 11); 149 state->fpu.fx_tw = 0; 150 for (i = 0; i < 8; i++) { 151 state->fpu.fx_tw |= (!env->fptags[i]) << i; 152 } 153 state->fpu.fx_opcode = env->fpop; 154 state->fpu.fx_ip.fa_64 = env->fpip; 155 state->fpu.fx_dp.fa_64 = env->fpdp; 156 state->fpu.fx_mxcsr = env->mxcsr; 157 state->fpu.fx_mxcsr_mask = 0x0000FFFF; 158 assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs)); 159 memcpy(state->fpu.fx_87_ac, env->fpregs, sizeof(env->fpregs)); 160 for (i = 0; i < CPU_NB_REGS; i++) { 161 memcpy(&state->fpu.fx_xmm[i].xmm_bytes[0], 162 &env->xmm_regs[i].ZMM_Q(0), 8); 163 memcpy(&state->fpu.fx_xmm[i].xmm_bytes[8], 164 &env->xmm_regs[i].ZMM_Q(1), 8); 165 } 166 167 /* MSRs. */ 168 state->msrs[NVMM_X64_MSR_EFER] = env->efer; 169 state->msrs[NVMM_X64_MSR_STAR] = env->star; 170 #ifdef TARGET_X86_64 171 state->msrs[NVMM_X64_MSR_LSTAR] = env->lstar; 172 state->msrs[NVMM_X64_MSR_CSTAR] = env->cstar; 173 state->msrs[NVMM_X64_MSR_SFMASK] = env->fmask; 174 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = env->kernelgsbase; 175 #endif 176 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = env->sysenter_cs; 177 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = env->sysenter_esp; 178 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = env->sysenter_eip; 179 state->msrs[NVMM_X64_MSR_PAT] = env->pat; 180 state->msrs[NVMM_X64_MSR_TSC] = env->tsc; 181 182 bitmap = 183 NVMM_X64_STATE_SEGS | 184 NVMM_X64_STATE_GPRS | 185 NVMM_X64_STATE_CRS | 186 NVMM_X64_STATE_DRS | 187 NVMM_X64_STATE_MSRS | 188 NVMM_X64_STATE_FPU; 189 190 ret = nvmm_vcpu_setstate(mach, vcpu, bitmap); 191 if (ret == -1) { 192 error_report("NVMM: Failed to set virtual processor context," 193 " error=%d", errno); 194 } 195 } 196 197 static void 198 nvmm_get_segment(SegmentCache *qseg, const struct nvmm_x64_state_seg *nseg) 199 { 200 qseg->selector = nseg->selector; 201 qseg->limit = nseg->limit; 202 qseg->base = nseg->base; 203 204 qseg->flags = 205 __SHIFTIN((uint32_t)nseg->attrib.type, DESC_TYPE_MASK) | 206 __SHIFTIN((uint32_t)nseg->attrib.s, DESC_S_MASK) | 207 __SHIFTIN((uint32_t)nseg->attrib.dpl, DESC_DPL_MASK) | 208 __SHIFTIN((uint32_t)nseg->attrib.p, DESC_P_MASK) | 209 __SHIFTIN((uint32_t)nseg->attrib.avl, DESC_AVL_MASK) | 210 __SHIFTIN((uint32_t)nseg->attrib.l, DESC_L_MASK) | 211 __SHIFTIN((uint32_t)nseg->attrib.def, DESC_B_MASK) | 212 __SHIFTIN((uint32_t)nseg->attrib.g, DESC_G_MASK); 213 } 214 215 static void 216 nvmm_get_registers(CPUState *cpu) 217 { 218 CPUX86State *env = cpu->env_ptr; 219 struct nvmm_machine *mach = get_nvmm_mach(); 220 AccelCPUState *qcpu = cpu->accel; 221 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 222 X86CPU *x86_cpu = X86_CPU(cpu); 223 struct nvmm_x64_state *state = vcpu->state; 224 uint64_t bitmap, tpr; 225 size_t i; 226 int ret; 227 228 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu)); 229 230 bitmap = 231 NVMM_X64_STATE_SEGS | 232 NVMM_X64_STATE_GPRS | 233 NVMM_X64_STATE_CRS | 234 NVMM_X64_STATE_DRS | 235 NVMM_X64_STATE_MSRS | 236 NVMM_X64_STATE_FPU; 237 238 ret = nvmm_vcpu_getstate(mach, vcpu, bitmap); 239 if (ret == -1) { 240 error_report("NVMM: Failed to get virtual processor context," 241 " error=%d", errno); 242 } 243 244 /* GPRs. */ 245 env->regs[R_EAX] = state->gprs[NVMM_X64_GPR_RAX]; 246 env->regs[R_ECX] = state->gprs[NVMM_X64_GPR_RCX]; 247 env->regs[R_EDX] = state->gprs[NVMM_X64_GPR_RDX]; 248 env->regs[R_EBX] = state->gprs[NVMM_X64_GPR_RBX]; 249 env->regs[R_ESP] = state->gprs[NVMM_X64_GPR_RSP]; 250 env->regs[R_EBP] = state->gprs[NVMM_X64_GPR_RBP]; 251 env->regs[R_ESI] = state->gprs[NVMM_X64_GPR_RSI]; 252 env->regs[R_EDI] = state->gprs[NVMM_X64_GPR_RDI]; 253 #ifdef TARGET_X86_64 254 env->regs[R_R8] = state->gprs[NVMM_X64_GPR_R8]; 255 env->regs[R_R9] = state->gprs[NVMM_X64_GPR_R9]; 256 env->regs[R_R10] = state->gprs[NVMM_X64_GPR_R10]; 257 env->regs[R_R11] = state->gprs[NVMM_X64_GPR_R11]; 258 env->regs[R_R12] = state->gprs[NVMM_X64_GPR_R12]; 259 env->regs[R_R13] = state->gprs[NVMM_X64_GPR_R13]; 260 env->regs[R_R14] = state->gprs[NVMM_X64_GPR_R14]; 261 env->regs[R_R15] = state->gprs[NVMM_X64_GPR_R15]; 262 #endif 263 264 /* RIP and RFLAGS. */ 265 env->eip = state->gprs[NVMM_X64_GPR_RIP]; 266 env->eflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 267 268 /* Segments. */ 269 nvmm_get_segment(&env->segs[R_ES], &state->segs[NVMM_X64_SEG_ES]); 270 nvmm_get_segment(&env->segs[R_CS], &state->segs[NVMM_X64_SEG_CS]); 271 nvmm_get_segment(&env->segs[R_SS], &state->segs[NVMM_X64_SEG_SS]); 272 nvmm_get_segment(&env->segs[R_DS], &state->segs[NVMM_X64_SEG_DS]); 273 nvmm_get_segment(&env->segs[R_FS], &state->segs[NVMM_X64_SEG_FS]); 274 nvmm_get_segment(&env->segs[R_GS], &state->segs[NVMM_X64_SEG_GS]); 275 276 /* Special segments. */ 277 nvmm_get_segment(&env->gdt, &state->segs[NVMM_X64_SEG_GDT]); 278 nvmm_get_segment(&env->ldt, &state->segs[NVMM_X64_SEG_LDT]); 279 nvmm_get_segment(&env->tr, &state->segs[NVMM_X64_SEG_TR]); 280 nvmm_get_segment(&env->idt, &state->segs[NVMM_X64_SEG_IDT]); 281 282 /* Control registers. */ 283 env->cr[0] = state->crs[NVMM_X64_CR_CR0]; 284 env->cr[2] = state->crs[NVMM_X64_CR_CR2]; 285 env->cr[3] = state->crs[NVMM_X64_CR_CR3]; 286 env->cr[4] = state->crs[NVMM_X64_CR_CR4]; 287 tpr = state->crs[NVMM_X64_CR_CR8]; 288 if (tpr != qcpu->tpr) { 289 qcpu->tpr = tpr; 290 cpu_set_apic_tpr(x86_cpu->apic_state, tpr); 291 } 292 env->xcr0 = state->crs[NVMM_X64_CR_XCR0]; 293 294 /* Debug registers. */ 295 env->dr[0] = state->drs[NVMM_X64_DR_DR0]; 296 env->dr[1] = state->drs[NVMM_X64_DR_DR1]; 297 env->dr[2] = state->drs[NVMM_X64_DR_DR2]; 298 env->dr[3] = state->drs[NVMM_X64_DR_DR3]; 299 env->dr[6] = state->drs[NVMM_X64_DR_DR6]; 300 env->dr[7] = state->drs[NVMM_X64_DR_DR7]; 301 302 /* FPU. */ 303 env->fpuc = state->fpu.fx_cw; 304 env->fpstt = (state->fpu.fx_sw >> 11) & 0x7; 305 env->fpus = state->fpu.fx_sw & ~0x3800; 306 for (i = 0; i < 8; i++) { 307 env->fptags[i] = !((state->fpu.fx_tw >> i) & 1); 308 } 309 env->fpop = state->fpu.fx_opcode; 310 env->fpip = state->fpu.fx_ip.fa_64; 311 env->fpdp = state->fpu.fx_dp.fa_64; 312 env->mxcsr = state->fpu.fx_mxcsr; 313 assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs)); 314 memcpy(env->fpregs, state->fpu.fx_87_ac, sizeof(env->fpregs)); 315 for (i = 0; i < CPU_NB_REGS; i++) { 316 memcpy(&env->xmm_regs[i].ZMM_Q(0), 317 &state->fpu.fx_xmm[i].xmm_bytes[0], 8); 318 memcpy(&env->xmm_regs[i].ZMM_Q(1), 319 &state->fpu.fx_xmm[i].xmm_bytes[8], 8); 320 } 321 322 /* MSRs. */ 323 env->efer = state->msrs[NVMM_X64_MSR_EFER]; 324 env->star = state->msrs[NVMM_X64_MSR_STAR]; 325 #ifdef TARGET_X86_64 326 env->lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 327 env->cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 328 env->fmask = state->msrs[NVMM_X64_MSR_SFMASK]; 329 env->kernelgsbase = state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 330 #endif 331 env->sysenter_cs = state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 332 env->sysenter_esp = state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 333 env->sysenter_eip = state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 334 env->pat = state->msrs[NVMM_X64_MSR_PAT]; 335 env->tsc = state->msrs[NVMM_X64_MSR_TSC]; 336 337 x86_update_hflags(env); 338 } 339 340 static bool 341 nvmm_can_take_int(CPUState *cpu) 342 { 343 CPUX86State *env = cpu->env_ptr; 344 AccelCPUState *qcpu = cpu->accel; 345 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 346 struct nvmm_machine *mach = get_nvmm_mach(); 347 348 if (qcpu->int_window_exit) { 349 return false; 350 } 351 352 if (qcpu->int_shadow || !(env->eflags & IF_MASK)) { 353 struct nvmm_x64_state *state = vcpu->state; 354 355 /* Exit on interrupt window. */ 356 nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_INTR); 357 state->intr.int_window_exiting = 1; 358 nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_INTR); 359 360 return false; 361 } 362 363 return true; 364 } 365 366 static bool 367 nvmm_can_take_nmi(CPUState *cpu) 368 { 369 AccelCPUState *qcpu = cpu->accel; 370 371 /* 372 * Contrary to INTs, NMIs always schedule an exit when they are 373 * completed. Therefore, if window-exiting is enabled, it means 374 * NMIs are blocked. 375 */ 376 if (qcpu->nmi_window_exit) { 377 return false; 378 } 379 380 return true; 381 } 382 383 /* 384 * Called before the VCPU is run. We inject events generated by the I/O 385 * thread, and synchronize the guest TPR. 386 */ 387 static void 388 nvmm_vcpu_pre_run(CPUState *cpu) 389 { 390 CPUX86State *env = cpu->env_ptr; 391 struct nvmm_machine *mach = get_nvmm_mach(); 392 AccelCPUState *qcpu = cpu->accel; 393 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 394 X86CPU *x86_cpu = X86_CPU(cpu); 395 struct nvmm_x64_state *state = vcpu->state; 396 struct nvmm_vcpu_event *event = vcpu->event; 397 bool has_event = false; 398 bool sync_tpr = false; 399 uint8_t tpr; 400 int ret; 401 402 qemu_mutex_lock_iothread(); 403 404 tpr = cpu_get_apic_tpr(x86_cpu->apic_state); 405 if (tpr != qcpu->tpr) { 406 qcpu->tpr = tpr; 407 sync_tpr = true; 408 } 409 410 /* 411 * Force the VCPU out of its inner loop to process any INIT requests 412 * or commit pending TPR access. 413 */ 414 if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { 415 cpu->exit_request = 1; 416 } 417 418 if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_NMI)) { 419 if (nvmm_can_take_nmi(cpu)) { 420 cpu->interrupt_request &= ~CPU_INTERRUPT_NMI; 421 event->type = NVMM_VCPU_EVENT_INTR; 422 event->vector = 2; 423 has_event = true; 424 } 425 } 426 427 if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_HARD)) { 428 if (nvmm_can_take_int(cpu)) { 429 cpu->interrupt_request &= ~CPU_INTERRUPT_HARD; 430 event->type = NVMM_VCPU_EVENT_INTR; 431 event->vector = cpu_get_pic_interrupt(env); 432 has_event = true; 433 } 434 } 435 436 /* Don't want SMIs. */ 437 if (cpu->interrupt_request & CPU_INTERRUPT_SMI) { 438 cpu->interrupt_request &= ~CPU_INTERRUPT_SMI; 439 } 440 441 if (sync_tpr) { 442 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_CRS); 443 if (ret == -1) { 444 error_report("NVMM: Failed to get CPU state," 445 " error=%d", errno); 446 } 447 448 state->crs[NVMM_X64_CR_CR8] = qcpu->tpr; 449 450 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_CRS); 451 if (ret == -1) { 452 error_report("NVMM: Failed to set CPU state," 453 " error=%d", errno); 454 } 455 } 456 457 if (has_event) { 458 ret = nvmm_vcpu_inject(mach, vcpu); 459 if (ret == -1) { 460 error_report("NVMM: Failed to inject event," 461 " error=%d", errno); 462 } 463 } 464 465 qemu_mutex_unlock_iothread(); 466 } 467 468 /* 469 * Called after the VCPU ran. We synchronize the host view of the TPR and 470 * RFLAGS. 471 */ 472 static void 473 nvmm_vcpu_post_run(CPUState *cpu, struct nvmm_vcpu_exit *exit) 474 { 475 AccelCPUState *qcpu = cpu->accel; 476 CPUX86State *env = cpu->env_ptr; 477 X86CPU *x86_cpu = X86_CPU(cpu); 478 uint64_t tpr; 479 480 env->eflags = exit->exitstate.rflags; 481 qcpu->int_shadow = exit->exitstate.int_shadow; 482 qcpu->int_window_exit = exit->exitstate.int_window_exiting; 483 qcpu->nmi_window_exit = exit->exitstate.nmi_window_exiting; 484 485 tpr = exit->exitstate.cr8; 486 if (qcpu->tpr != tpr) { 487 qcpu->tpr = tpr; 488 qemu_mutex_lock_iothread(); 489 cpu_set_apic_tpr(x86_cpu->apic_state, qcpu->tpr); 490 qemu_mutex_unlock_iothread(); 491 } 492 } 493 494 /* -------------------------------------------------------------------------- */ 495 496 static void 497 nvmm_io_callback(struct nvmm_io *io) 498 { 499 MemTxAttrs attrs = { 0 }; 500 int ret; 501 502 ret = address_space_rw(&address_space_io, io->port, attrs, io->data, 503 io->size, !io->in); 504 if (ret != MEMTX_OK) { 505 error_report("NVMM: I/O Transaction Failed " 506 "[%s, port=%u, size=%zu]", (io->in ? "in" : "out"), 507 io->port, io->size); 508 } 509 510 /* Needed, otherwise infinite loop. */ 511 current_cpu->vcpu_dirty = false; 512 } 513 514 static void 515 nvmm_mem_callback(struct nvmm_mem *mem) 516 { 517 cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write); 518 519 /* Needed, otherwise infinite loop. */ 520 current_cpu->vcpu_dirty = false; 521 } 522 523 static struct nvmm_assist_callbacks nvmm_callbacks = { 524 .io = nvmm_io_callback, 525 .mem = nvmm_mem_callback 526 }; 527 528 /* -------------------------------------------------------------------------- */ 529 530 static int 531 nvmm_handle_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 532 { 533 int ret; 534 535 ret = nvmm_assist_mem(mach, vcpu); 536 if (ret == -1) { 537 error_report("NVMM: Mem Assist Failed [gpa=%p]", 538 (void *)vcpu->exit->u.mem.gpa); 539 } 540 541 return ret; 542 } 543 544 static int 545 nvmm_handle_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 546 { 547 int ret; 548 549 ret = nvmm_assist_io(mach, vcpu); 550 if (ret == -1) { 551 error_report("NVMM: I/O Assist Failed [port=%d]", 552 (int)vcpu->exit->u.io.port); 553 } 554 555 return ret; 556 } 557 558 static int 559 nvmm_handle_rdmsr(struct nvmm_machine *mach, CPUState *cpu, 560 struct nvmm_vcpu_exit *exit) 561 { 562 AccelCPUState *qcpu = cpu->accel; 563 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 564 X86CPU *x86_cpu = X86_CPU(cpu); 565 struct nvmm_x64_state *state = vcpu->state; 566 uint64_t val; 567 int ret; 568 569 switch (exit->u.rdmsr.msr) { 570 case MSR_IA32_APICBASE: 571 val = cpu_get_apic_base(x86_cpu->apic_state); 572 break; 573 case MSR_MTRRcap: 574 case MSR_MTRRdefType: 575 case MSR_MCG_CAP: 576 case MSR_MCG_STATUS: 577 val = 0; 578 break; 579 default: /* More MSRs to add? */ 580 val = 0; 581 error_report("NVMM: Unexpected RDMSR 0x%x, ignored", 582 exit->u.rdmsr.msr); 583 break; 584 } 585 586 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS); 587 if (ret == -1) { 588 return -1; 589 } 590 591 state->gprs[NVMM_X64_GPR_RAX] = (val & 0xFFFFFFFF); 592 state->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 593 state->gprs[NVMM_X64_GPR_RIP] = exit->u.rdmsr.npc; 594 595 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 596 if (ret == -1) { 597 return -1; 598 } 599 600 return 0; 601 } 602 603 static int 604 nvmm_handle_wrmsr(struct nvmm_machine *mach, CPUState *cpu, 605 struct nvmm_vcpu_exit *exit) 606 { 607 AccelCPUState *qcpu = cpu->accel; 608 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 609 X86CPU *x86_cpu = X86_CPU(cpu); 610 struct nvmm_x64_state *state = vcpu->state; 611 uint64_t val; 612 int ret; 613 614 val = exit->u.wrmsr.val; 615 616 switch (exit->u.wrmsr.msr) { 617 case MSR_IA32_APICBASE: 618 cpu_set_apic_base(x86_cpu->apic_state, val); 619 break; 620 case MSR_MTRRdefType: 621 case MSR_MCG_STATUS: 622 break; 623 default: /* More MSRs to add? */ 624 error_report("NVMM: Unexpected WRMSR 0x%x [val=0x%lx], ignored", 625 exit->u.wrmsr.msr, val); 626 break; 627 } 628 629 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS); 630 if (ret == -1) { 631 return -1; 632 } 633 634 state->gprs[NVMM_X64_GPR_RIP] = exit->u.wrmsr.npc; 635 636 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 637 if (ret == -1) { 638 return -1; 639 } 640 641 return 0; 642 } 643 644 static int 645 nvmm_handle_halted(struct nvmm_machine *mach, CPUState *cpu, 646 struct nvmm_vcpu_exit *exit) 647 { 648 CPUX86State *env = cpu->env_ptr; 649 int ret = 0; 650 651 qemu_mutex_lock_iothread(); 652 653 if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) && 654 (env->eflags & IF_MASK)) && 655 !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) { 656 cpu->exception_index = EXCP_HLT; 657 cpu->halted = true; 658 ret = 1; 659 } 660 661 qemu_mutex_unlock_iothread(); 662 663 return ret; 664 } 665 666 static int 667 nvmm_inject_ud(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 668 { 669 struct nvmm_vcpu_event *event = vcpu->event; 670 671 event->type = NVMM_VCPU_EVENT_EXCP; 672 event->vector = 6; 673 event->u.excp.error = 0; 674 675 return nvmm_vcpu_inject(mach, vcpu); 676 } 677 678 static int 679 nvmm_vcpu_loop(CPUState *cpu) 680 { 681 CPUX86State *env = cpu->env_ptr; 682 struct nvmm_machine *mach = get_nvmm_mach(); 683 AccelCPUState *qcpu = cpu->accel; 684 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 685 X86CPU *x86_cpu = X86_CPU(cpu); 686 struct nvmm_vcpu_exit *exit = vcpu->exit; 687 int ret; 688 689 /* 690 * Some asynchronous events must be handled outside of the inner 691 * VCPU loop. They are handled here. 692 */ 693 if (cpu->interrupt_request & CPU_INTERRUPT_INIT) { 694 nvmm_cpu_synchronize_state(cpu); 695 do_cpu_init(x86_cpu); 696 /* set int/nmi windows back to the reset state */ 697 } 698 if (cpu->interrupt_request & CPU_INTERRUPT_POLL) { 699 cpu->interrupt_request &= ~CPU_INTERRUPT_POLL; 700 apic_poll_irq(x86_cpu->apic_state); 701 } 702 if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) && 703 (env->eflags & IF_MASK)) || 704 (cpu->interrupt_request & CPU_INTERRUPT_NMI)) { 705 cpu->halted = false; 706 } 707 if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) { 708 nvmm_cpu_synchronize_state(cpu); 709 do_cpu_sipi(x86_cpu); 710 } 711 if (cpu->interrupt_request & CPU_INTERRUPT_TPR) { 712 cpu->interrupt_request &= ~CPU_INTERRUPT_TPR; 713 nvmm_cpu_synchronize_state(cpu); 714 apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip, 715 env->tpr_access_type); 716 } 717 718 if (cpu->halted) { 719 cpu->exception_index = EXCP_HLT; 720 qatomic_set(&cpu->exit_request, false); 721 return 0; 722 } 723 724 qemu_mutex_unlock_iothread(); 725 cpu_exec_start(cpu); 726 727 /* 728 * Inner VCPU loop. 729 */ 730 do { 731 if (cpu->vcpu_dirty) { 732 nvmm_set_registers(cpu); 733 cpu->vcpu_dirty = false; 734 } 735 736 if (qcpu->stop) { 737 cpu->exception_index = EXCP_INTERRUPT; 738 qcpu->stop = false; 739 ret = 1; 740 break; 741 } 742 743 nvmm_vcpu_pre_run(cpu); 744 745 if (qatomic_read(&cpu->exit_request)) { 746 #if NVMM_USER_VERSION >= 2 747 nvmm_vcpu_stop(vcpu); 748 #else 749 qemu_cpu_kick_self(); 750 #endif 751 } 752 753 /* Read exit_request before the kernel reads the immediate exit flag */ 754 smp_rmb(); 755 ret = nvmm_vcpu_run(mach, vcpu); 756 if (ret == -1) { 757 error_report("NVMM: Failed to exec a virtual processor," 758 " error=%d", errno); 759 break; 760 } 761 762 nvmm_vcpu_post_run(cpu, exit); 763 764 switch (exit->reason) { 765 case NVMM_VCPU_EXIT_NONE: 766 break; 767 #if NVMM_USER_VERSION >= 2 768 case NVMM_VCPU_EXIT_STOPPED: 769 /* 770 * The kernel cleared the immediate exit flag; cpu->exit_request 771 * must be cleared after 772 */ 773 smp_wmb(); 774 qcpu->stop = true; 775 break; 776 #endif 777 case NVMM_VCPU_EXIT_MEMORY: 778 ret = nvmm_handle_mem(mach, vcpu); 779 break; 780 case NVMM_VCPU_EXIT_IO: 781 ret = nvmm_handle_io(mach, vcpu); 782 break; 783 case NVMM_VCPU_EXIT_INT_READY: 784 case NVMM_VCPU_EXIT_NMI_READY: 785 case NVMM_VCPU_EXIT_TPR_CHANGED: 786 break; 787 case NVMM_VCPU_EXIT_HALTED: 788 ret = nvmm_handle_halted(mach, cpu, exit); 789 break; 790 case NVMM_VCPU_EXIT_SHUTDOWN: 791 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 792 cpu->exception_index = EXCP_INTERRUPT; 793 ret = 1; 794 break; 795 case NVMM_VCPU_EXIT_RDMSR: 796 ret = nvmm_handle_rdmsr(mach, cpu, exit); 797 break; 798 case NVMM_VCPU_EXIT_WRMSR: 799 ret = nvmm_handle_wrmsr(mach, cpu, exit); 800 break; 801 case NVMM_VCPU_EXIT_MONITOR: 802 case NVMM_VCPU_EXIT_MWAIT: 803 ret = nvmm_inject_ud(mach, vcpu); 804 break; 805 default: 806 error_report("NVMM: Unexpected VM exit code 0x%lx [hw=0x%lx]", 807 exit->reason, exit->u.inv.hwcode); 808 nvmm_get_registers(cpu); 809 qemu_mutex_lock_iothread(); 810 qemu_system_guest_panicked(cpu_get_crash_info(cpu)); 811 qemu_mutex_unlock_iothread(); 812 ret = -1; 813 break; 814 } 815 } while (ret == 0); 816 817 cpu_exec_end(cpu); 818 qemu_mutex_lock_iothread(); 819 820 qatomic_set(&cpu->exit_request, false); 821 822 return ret < 0; 823 } 824 825 /* -------------------------------------------------------------------------- */ 826 827 static void 828 do_nvmm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg) 829 { 830 nvmm_get_registers(cpu); 831 cpu->vcpu_dirty = true; 832 } 833 834 static void 835 do_nvmm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg) 836 { 837 nvmm_set_registers(cpu); 838 cpu->vcpu_dirty = false; 839 } 840 841 static void 842 do_nvmm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) 843 { 844 nvmm_set_registers(cpu); 845 cpu->vcpu_dirty = false; 846 } 847 848 static void 849 do_nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg) 850 { 851 cpu->vcpu_dirty = true; 852 } 853 854 void nvmm_cpu_synchronize_state(CPUState *cpu) 855 { 856 if (!cpu->vcpu_dirty) { 857 run_on_cpu(cpu, do_nvmm_cpu_synchronize_state, RUN_ON_CPU_NULL); 858 } 859 } 860 861 void nvmm_cpu_synchronize_post_reset(CPUState *cpu) 862 { 863 run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); 864 } 865 866 void nvmm_cpu_synchronize_post_init(CPUState *cpu) 867 { 868 run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_init, RUN_ON_CPU_NULL); 869 } 870 871 void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu) 872 { 873 run_on_cpu(cpu, do_nvmm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL); 874 } 875 876 /* -------------------------------------------------------------------------- */ 877 878 static Error *nvmm_migration_blocker; 879 880 /* 881 * The nvmm_vcpu_stop() mechanism breaks races between entering the VMM 882 * and another thread signaling the vCPU thread to exit. 883 */ 884 885 static void 886 nvmm_ipi_signal(int sigcpu) 887 { 888 if (current_cpu) { 889 AccelCPUState *qcpu = current_cpu->accel; 890 #if NVMM_USER_VERSION >= 2 891 struct nvmm_vcpu *vcpu = &qcpu->vcpu; 892 nvmm_vcpu_stop(vcpu); 893 #else 894 qcpu->stop = true; 895 #endif 896 } 897 } 898 899 static void 900 nvmm_init_cpu_signals(void) 901 { 902 struct sigaction sigact; 903 sigset_t set; 904 905 /* Install the IPI handler. */ 906 memset(&sigact, 0, sizeof(sigact)); 907 sigact.sa_handler = nvmm_ipi_signal; 908 sigaction(SIG_IPI, &sigact, NULL); 909 910 /* Allow IPIs on the current thread. */ 911 sigprocmask(SIG_BLOCK, NULL, &set); 912 sigdelset(&set, SIG_IPI); 913 pthread_sigmask(SIG_SETMASK, &set, NULL); 914 } 915 916 int 917 nvmm_init_vcpu(CPUState *cpu) 918 { 919 struct nvmm_machine *mach = get_nvmm_mach(); 920 struct nvmm_vcpu_conf_cpuid cpuid; 921 struct nvmm_vcpu_conf_tpr tpr; 922 Error *local_error = NULL; 923 AccelCPUState *qcpu; 924 int ret, err; 925 926 nvmm_init_cpu_signals(); 927 928 if (nvmm_migration_blocker == NULL) { 929 error_setg(&nvmm_migration_blocker, 930 "NVMM: Migration not supported"); 931 932 if (migrate_add_blocker(nvmm_migration_blocker, &local_error) < 0) { 933 error_report_err(local_error); 934 error_free(nvmm_migration_blocker); 935 return -EINVAL; 936 } 937 } 938 939 qcpu = g_new0(AccelCPUState, 1); 940 941 ret = nvmm_vcpu_create(mach, cpu->cpu_index, &qcpu->vcpu); 942 if (ret == -1) { 943 err = errno; 944 error_report("NVMM: Failed to create a virtual processor," 945 " error=%d", err); 946 g_free(qcpu); 947 return -err; 948 } 949 950 memset(&cpuid, 0, sizeof(cpuid)); 951 cpuid.mask = 1; 952 cpuid.leaf = 0x00000001; 953 cpuid.u.mask.set.edx = CPUID_MCE | CPUID_MCA | CPUID_MTRR; 954 ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CPUID, 955 &cpuid); 956 if (ret == -1) { 957 err = errno; 958 error_report("NVMM: Failed to configure a virtual processor," 959 " error=%d", err); 960 g_free(qcpu); 961 return -err; 962 } 963 964 ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CALLBACKS, 965 &nvmm_callbacks); 966 if (ret == -1) { 967 err = errno; 968 error_report("NVMM: Failed to configure a virtual processor," 969 " error=%d", err); 970 g_free(qcpu); 971 return -err; 972 } 973 974 if (qemu_mach.cap.arch.vcpu_conf_support & NVMM_CAP_ARCH_VCPU_CONF_TPR) { 975 memset(&tpr, 0, sizeof(tpr)); 976 tpr.exit_changed = 1; 977 ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_TPR, &tpr); 978 if (ret == -1) { 979 err = errno; 980 error_report("NVMM: Failed to configure a virtual processor," 981 " error=%d", err); 982 g_free(qcpu); 983 return -err; 984 } 985 } 986 987 cpu->vcpu_dirty = true; 988 cpu->accel = qcpu; 989 990 return 0; 991 } 992 993 int 994 nvmm_vcpu_exec(CPUState *cpu) 995 { 996 int ret, fatal; 997 998 while (1) { 999 if (cpu->exception_index >= EXCP_INTERRUPT) { 1000 ret = cpu->exception_index; 1001 cpu->exception_index = -1; 1002 break; 1003 } 1004 1005 fatal = nvmm_vcpu_loop(cpu); 1006 1007 if (fatal) { 1008 error_report("NVMM: Failed to execute a VCPU."); 1009 abort(); 1010 } 1011 } 1012 1013 return ret; 1014 } 1015 1016 void 1017 nvmm_destroy_vcpu(CPUState *cpu) 1018 { 1019 struct nvmm_machine *mach = get_nvmm_mach(); 1020 AccelCPUState *qcpu = cpu->accel; 1021 1022 nvmm_vcpu_destroy(mach, &qcpu->vcpu); 1023 g_free(cpu->accel); 1024 } 1025 1026 /* -------------------------------------------------------------------------- */ 1027 1028 static void 1029 nvmm_update_mapping(hwaddr start_pa, ram_addr_t size, uintptr_t hva, 1030 bool add, bool rom, const char *name) 1031 { 1032 struct nvmm_machine *mach = get_nvmm_mach(); 1033 int ret, prot; 1034 1035 if (add) { 1036 prot = PROT_READ | PROT_EXEC; 1037 if (!rom) { 1038 prot |= PROT_WRITE; 1039 } 1040 ret = nvmm_gpa_map(mach, hva, start_pa, size, prot); 1041 } else { 1042 ret = nvmm_gpa_unmap(mach, hva, start_pa, size); 1043 } 1044 1045 if (ret == -1) { 1046 error_report("NVMM: Failed to %s GPA range '%s' PA:%p, " 1047 "Size:%p bytes, HostVA:%p, error=%d", 1048 (add ? "map" : "unmap"), name, (void *)(uintptr_t)start_pa, 1049 (void *)size, (void *)hva, errno); 1050 } 1051 } 1052 1053 static void 1054 nvmm_process_section(MemoryRegionSection *section, int add) 1055 { 1056 MemoryRegion *mr = section->mr; 1057 hwaddr start_pa = section->offset_within_address_space; 1058 ram_addr_t size = int128_get64(section->size); 1059 unsigned int delta; 1060 uintptr_t hva; 1061 1062 if (!memory_region_is_ram(mr)) { 1063 return; 1064 } 1065 1066 /* Adjust start_pa and size so that they are page-aligned. */ 1067 delta = qemu_real_host_page_size() - (start_pa & ~qemu_real_host_page_mask()); 1068 delta &= ~qemu_real_host_page_mask(); 1069 if (delta > size) { 1070 return; 1071 } 1072 start_pa += delta; 1073 size -= delta; 1074 size &= qemu_real_host_page_mask(); 1075 if (!size || (start_pa & ~qemu_real_host_page_mask())) { 1076 return; 1077 } 1078 1079 hva = (uintptr_t)memory_region_get_ram_ptr(mr) + 1080 section->offset_within_region + delta; 1081 1082 nvmm_update_mapping(start_pa, size, hva, add, 1083 memory_region_is_rom(mr), mr->name); 1084 } 1085 1086 static void 1087 nvmm_region_add(MemoryListener *listener, MemoryRegionSection *section) 1088 { 1089 memory_region_ref(section->mr); 1090 nvmm_process_section(section, 1); 1091 } 1092 1093 static void 1094 nvmm_region_del(MemoryListener *listener, MemoryRegionSection *section) 1095 { 1096 nvmm_process_section(section, 0); 1097 memory_region_unref(section->mr); 1098 } 1099 1100 static void 1101 nvmm_transaction_begin(MemoryListener *listener) 1102 { 1103 /* nothing */ 1104 } 1105 1106 static void 1107 nvmm_transaction_commit(MemoryListener *listener) 1108 { 1109 /* nothing */ 1110 } 1111 1112 static void 1113 nvmm_log_sync(MemoryListener *listener, MemoryRegionSection *section) 1114 { 1115 MemoryRegion *mr = section->mr; 1116 1117 if (!memory_region_is_ram(mr)) { 1118 return; 1119 } 1120 1121 memory_region_set_dirty(mr, 0, int128_get64(section->size)); 1122 } 1123 1124 static MemoryListener nvmm_memory_listener = { 1125 .name = "nvmm", 1126 .begin = nvmm_transaction_begin, 1127 .commit = nvmm_transaction_commit, 1128 .region_add = nvmm_region_add, 1129 .region_del = nvmm_region_del, 1130 .log_sync = nvmm_log_sync, 1131 .priority = MEMORY_LISTENER_PRIORITY_ACCEL, 1132 }; 1133 1134 static void 1135 nvmm_ram_block_added(RAMBlockNotifier *n, void *host, size_t size, 1136 size_t max_size) 1137 { 1138 struct nvmm_machine *mach = get_nvmm_mach(); 1139 uintptr_t hva = (uintptr_t)host; 1140 int ret; 1141 1142 ret = nvmm_hva_map(mach, hva, max_size); 1143 1144 if (ret == -1) { 1145 error_report("NVMM: Failed to map HVA, HostVA:%p " 1146 "Size:%p bytes, error=%d", 1147 (void *)hva, (void *)size, errno); 1148 } 1149 } 1150 1151 static struct RAMBlockNotifier nvmm_ram_notifier = { 1152 .ram_block_added = nvmm_ram_block_added 1153 }; 1154 1155 /* -------------------------------------------------------------------------- */ 1156 1157 static int 1158 nvmm_accel_init(MachineState *ms) 1159 { 1160 int ret, err; 1161 1162 ret = nvmm_init(); 1163 if (ret == -1) { 1164 err = errno; 1165 error_report("NVMM: Initialization failed, error=%d", errno); 1166 return -err; 1167 } 1168 1169 ret = nvmm_capability(&qemu_mach.cap); 1170 if (ret == -1) { 1171 err = errno; 1172 error_report("NVMM: Unable to fetch capability, error=%d", errno); 1173 return -err; 1174 } 1175 if (qemu_mach.cap.version < NVMM_KERN_VERSION) { 1176 error_report("NVMM: Unsupported version %u", qemu_mach.cap.version); 1177 return -EPROGMISMATCH; 1178 } 1179 if (qemu_mach.cap.state_size != sizeof(struct nvmm_x64_state)) { 1180 error_report("NVMM: Wrong state size %u", qemu_mach.cap.state_size); 1181 return -EPROGMISMATCH; 1182 } 1183 1184 ret = nvmm_machine_create(&qemu_mach.mach); 1185 if (ret == -1) { 1186 err = errno; 1187 error_report("NVMM: Machine creation failed, error=%d", errno); 1188 return -err; 1189 } 1190 1191 memory_listener_register(&nvmm_memory_listener, &address_space_memory); 1192 ram_block_notifier_add(&nvmm_ram_notifier); 1193 1194 printf("NetBSD Virtual Machine Monitor accelerator is operational\n"); 1195 return 0; 1196 } 1197 1198 int 1199 nvmm_enabled(void) 1200 { 1201 return nvmm_allowed; 1202 } 1203 1204 static void 1205 nvmm_accel_class_init(ObjectClass *oc, void *data) 1206 { 1207 AccelClass *ac = ACCEL_CLASS(oc); 1208 ac->name = "NVMM"; 1209 ac->init_machine = nvmm_accel_init; 1210 ac->allowed = &nvmm_allowed; 1211 } 1212 1213 static const TypeInfo nvmm_accel_type = { 1214 .name = ACCEL_CLASS_NAME("nvmm"), 1215 .parent = TYPE_ACCEL, 1216 .class_init = nvmm_accel_class_init, 1217 }; 1218 1219 static void 1220 nvmm_type_init(void) 1221 { 1222 type_register_static(&nvmm_accel_type); 1223 } 1224 1225 type_init(nvmm_type_init); 1226