/* * Copyright (c) 2018-2019 Maxime Villard, All rights reserved. * * NetBSD Virtual Machine Monitor (NVMM) accelerator for QEMU. * * This work is licensed under the terms of the GNU GPL, version 2 or later. * See the COPYING file in the top-level directory. */ #include "qemu/osdep.h" #include "cpu.h" #include "exec/address-spaces.h" #include "exec/ioport.h" #include "qemu/accel.h" #include "sysemu/nvmm.h" #include "sysemu/cpus.h" #include "sysemu/runstate.h" #include "qemu/main-loop.h" #include "qemu/error-report.h" #include "qapi/error.h" #include "qemu/queue.h" #include "migration/blocker.h" #include "strings.h" #include "nvmm-accel-ops.h" #include struct AccelCPUState { struct nvmm_vcpu vcpu; uint8_t tpr; bool stop; bool dirty; /* Window-exiting for INTs/NMIs. */ bool int_window_exit; bool nmi_window_exit; /* The guest is in an interrupt shadow (POP SS, etc). */ bool int_shadow; }; struct qemu_machine { struct nvmm_capability cap; struct nvmm_machine mach; }; /* -------------------------------------------------------------------------- */ static bool nvmm_allowed; static struct qemu_machine qemu_mach; static struct nvmm_machine * get_nvmm_mach(void) { return &qemu_mach.mach; } /* -------------------------------------------------------------------------- */ static void nvmm_set_segment(struct nvmm_x64_state_seg *nseg, const SegmentCache *qseg) { uint32_t attrib = qseg->flags; nseg->selector = qseg->selector; nseg->limit = qseg->limit; nseg->base = qseg->base; nseg->attrib.type = __SHIFTOUT(attrib, DESC_TYPE_MASK); nseg->attrib.s = __SHIFTOUT(attrib, DESC_S_MASK); nseg->attrib.dpl = __SHIFTOUT(attrib, DESC_DPL_MASK); nseg->attrib.p = __SHIFTOUT(attrib, DESC_P_MASK); nseg->attrib.avl = __SHIFTOUT(attrib, DESC_AVL_MASK); nseg->attrib.l = __SHIFTOUT(attrib, DESC_L_MASK); nseg->attrib.def = __SHIFTOUT(attrib, DESC_B_MASK); nseg->attrib.g = __SHIFTOUT(attrib, DESC_G_MASK); } static void nvmm_set_registers(CPUState *cpu) { CPUX86State *env = cpu_env(cpu); struct nvmm_machine *mach = get_nvmm_mach(); AccelCPUState *qcpu = cpu->accel; struct nvmm_vcpu *vcpu = &qcpu->vcpu; struct nvmm_x64_state *state = vcpu->state; uint64_t bitmap; size_t i; int ret; assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu)); /* GPRs. */ state->gprs[NVMM_X64_GPR_RAX] = env->regs[R_EAX]; state->gprs[NVMM_X64_GPR_RCX] = env->regs[R_ECX]; state->gprs[NVMM_X64_GPR_RDX] = env->regs[R_EDX]; state->gprs[NVMM_X64_GPR_RBX] = env->regs[R_EBX]; state->gprs[NVMM_X64_GPR_RSP] = env->regs[R_ESP]; state->gprs[NVMM_X64_GPR_RBP] = env->regs[R_EBP]; state->gprs[NVMM_X64_GPR_RSI] = env->regs[R_ESI]; state->gprs[NVMM_X64_GPR_RDI] = env->regs[R_EDI]; #ifdef TARGET_X86_64 state->gprs[NVMM_X64_GPR_R8] = env->regs[R_R8]; state->gprs[NVMM_X64_GPR_R9] = env->regs[R_R9]; state->gprs[NVMM_X64_GPR_R10] = env->regs[R_R10]; state->gprs[NVMM_X64_GPR_R11] = env->regs[R_R11]; state->gprs[NVMM_X64_GPR_R12] = env->regs[R_R12]; state->gprs[NVMM_X64_GPR_R13] = env->regs[R_R13]; state->gprs[NVMM_X64_GPR_R14] = env->regs[R_R14]; state->gprs[NVMM_X64_GPR_R15] = env->regs[R_R15]; #endif /* RIP and RFLAGS. */ state->gprs[NVMM_X64_GPR_RIP] = env->eip; state->gprs[NVMM_X64_GPR_RFLAGS] = env->eflags; /* Segments. */ nvmm_set_segment(&state->segs[NVMM_X64_SEG_CS], &env->segs[R_CS]); nvmm_set_segment(&state->segs[NVMM_X64_SEG_DS], &env->segs[R_DS]); nvmm_set_segment(&state->segs[NVMM_X64_SEG_ES], &env->segs[R_ES]); nvmm_set_segment(&state->segs[NVMM_X64_SEG_FS], &env->segs[R_FS]); nvmm_set_segment(&state->segs[NVMM_X64_SEG_GS], &env->segs[R_GS]); nvmm_set_segment(&state->segs[NVMM_X64_SEG_SS], &env->segs[R_SS]); /* Special segments. */ nvmm_set_segment(&state->segs[NVMM_X64_SEG_GDT], &env->gdt); nvmm_set_segment(&state->segs[NVMM_X64_SEG_LDT], &env->ldt); nvmm_set_segment(&state->segs[NVMM_X64_SEG_TR], &env->tr); nvmm_set_segment(&state->segs[NVMM_X64_SEG_IDT], &env->idt); /* Control registers. */ state->crs[NVMM_X64_CR_CR0] = env->cr[0]; state->crs[NVMM_X64_CR_CR2] = env->cr[2]; state->crs[NVMM_X64_CR_CR3] = env->cr[3]; state->crs[NVMM_X64_CR_CR4] = env->cr[4]; state->crs[NVMM_X64_CR_CR8] = qcpu->tpr; state->crs[NVMM_X64_CR_XCR0] = env->xcr0; /* Debug registers. */ state->drs[NVMM_X64_DR_DR0] = env->dr[0]; state->drs[NVMM_X64_DR_DR1] = env->dr[1]; state->drs[NVMM_X64_DR_DR2] = env->dr[2]; state->drs[NVMM_X64_DR_DR3] = env->dr[3]; state->drs[NVMM_X64_DR_DR6] = env->dr[6]; state->drs[NVMM_X64_DR_DR7] = env->dr[7]; /* FPU. */ state->fpu.fx_cw = env->fpuc; state->fpu.fx_sw = (env->fpus & ~0x3800) | ((env->fpstt & 0x7) << 11); state->fpu.fx_tw = 0; for (i = 0; i < 8; i++) { state->fpu.fx_tw |= (!env->fptags[i]) << i; } state->fpu.fx_opcode = env->fpop; state->fpu.fx_ip.fa_64 = env->fpip; state->fpu.fx_dp.fa_64 = env->fpdp; state->fpu.fx_mxcsr = env->mxcsr; state->fpu.fx_mxcsr_mask = 0x0000FFFF; assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs)); memcpy(state->fpu.fx_87_ac, env->fpregs, sizeof(env->fpregs)); for (i = 0; i < CPU_NB_REGS; i++) { memcpy(&state->fpu.fx_xmm[i].xmm_bytes[0], &env->xmm_regs[i].ZMM_Q(0), 8); memcpy(&state->fpu.fx_xmm[i].xmm_bytes[8], &env->xmm_regs[i].ZMM_Q(1), 8); } /* MSRs. */ state->msrs[NVMM_X64_MSR_EFER] = env->efer; state->msrs[NVMM_X64_MSR_STAR] = env->star; #ifdef TARGET_X86_64 state->msrs[NVMM_X64_MSR_LSTAR] = env->lstar; state->msrs[NVMM_X64_MSR_CSTAR] = env->cstar; state->msrs[NVMM_X64_MSR_SFMASK] = env->fmask; state->msrs[NVMM_X64_MSR_KERNELGSBASE] = env->kernelgsbase; #endif state->msrs[NVMM_X64_MSR_SYSENTER_CS] = env->sysenter_cs; state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = env->sysenter_esp; state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = env->sysenter_eip; state->msrs[NVMM_X64_MSR_PAT] = env->pat; state->msrs[NVMM_X64_MSR_TSC] = env->tsc; bitmap = NVMM_X64_STATE_SEGS | NVMM_X64_STATE_GPRS | NVMM_X64_STATE_CRS | NVMM_X64_STATE_DRS | NVMM_X64_STATE_MSRS | NVMM_X64_STATE_FPU; ret = nvmm_vcpu_setstate(mach, vcpu, bitmap); if (ret == -1) { error_report("NVMM: Failed to set virtual processor context," " error=%d", errno); } } static void nvmm_get_segment(SegmentCache *qseg, const struct nvmm_x64_state_seg *nseg) { qseg->selector = nseg->selector; qseg->limit = nseg->limit; qseg->base = nseg->base; qseg->flags = __SHIFTIN((uint32_t)nseg->attrib.type, DESC_TYPE_MASK) | __SHIFTIN((uint32_t)nseg->attrib.s, DESC_S_MASK) | __SHIFTIN((uint32_t)nseg->attrib.dpl, DESC_DPL_MASK) | __SHIFTIN((uint32_t)nseg->attrib.p, DESC_P_MASK) | __SHIFTIN((uint32_t)nseg->attrib.avl, DESC_AVL_MASK) | __SHIFTIN((uint32_t)nseg->attrib.l, DESC_L_MASK) | __SHIFTIN((uint32_t)nseg->attrib.def, DESC_B_MASK) | __SHIFTIN((uint32_t)nseg->attrib.g, DESC_G_MASK); } static void nvmm_get_registers(CPUState *cpu) { CPUX86State *env = cpu_env(cpu); struct nvmm_machine *mach = get_nvmm_mach(); AccelCPUState *qcpu = cpu->accel; struct nvmm_vcpu *vcpu = &qcpu->vcpu; X86CPU *x86_cpu = X86_CPU(cpu); struct nvmm_x64_state *state = vcpu->state; uint64_t bitmap, tpr; size_t i; int ret; assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu)); bitmap = NVMM_X64_STATE_SEGS | NVMM_X64_STATE_GPRS | NVMM_X64_STATE_CRS | NVMM_X64_STATE_DRS | NVMM_X64_STATE_MSRS | NVMM_X64_STATE_FPU; ret = nvmm_vcpu_getstate(mach, vcpu, bitmap); if (ret == -1) { error_report("NVMM: Failed to get virtual processor context," " error=%d", errno); } /* GPRs. */ env->regs[R_EAX] = state->gprs[NVMM_X64_GPR_RAX]; env->regs[R_ECX] = state->gprs[NVMM_X64_GPR_RCX]; env->regs[R_EDX] = state->gprs[NVMM_X64_GPR_RDX]; env->regs[R_EBX] = state->gprs[NVMM_X64_GPR_RBX]; env->regs[R_ESP] = state->gprs[NVMM_X64_GPR_RSP]; env->regs[R_EBP] = state->gprs[NVMM_X64_GPR_RBP]; env->regs[R_ESI] = state->gprs[NVMM_X64_GPR_RSI]; env->regs[R_EDI] = state->gprs[NVMM_X64_GPR_RDI]; #ifdef TARGET_X86_64 env->regs[R_R8] = state->gprs[NVMM_X64_GPR_R8]; env->regs[R_R9] = state->gprs[NVMM_X64_GPR_R9]; env->regs[R_R10] = state->gprs[NVMM_X64_GPR_R10]; env->regs[R_R11] = state->gprs[NVMM_X64_GPR_R11]; env->regs[R_R12] = state->gprs[NVMM_X64_GPR_R12]; env->regs[R_R13] = state->gprs[NVMM_X64_GPR_R13]; env->regs[R_R14] = state->gprs[NVMM_X64_GPR_R14]; env->regs[R_R15] = state->gprs[NVMM_X64_GPR_R15]; #endif /* RIP and RFLAGS. */ env->eip = state->gprs[NVMM_X64_GPR_RIP]; env->eflags = state->gprs[NVMM_X64_GPR_RFLAGS]; /* Segments. */ nvmm_get_segment(&env->segs[R_ES], &state->segs[NVMM_X64_SEG_ES]); nvmm_get_segment(&env->segs[R_CS], &state->segs[NVMM_X64_SEG_CS]); nvmm_get_segment(&env->segs[R_SS], &state->segs[NVMM_X64_SEG_SS]); nvmm_get_segment(&env->segs[R_DS], &state->segs[NVMM_X64_SEG_DS]); nvmm_get_segment(&env->segs[R_FS], &state->segs[NVMM_X64_SEG_FS]); nvmm_get_segment(&env->segs[R_GS], &state->segs[NVMM_X64_SEG_GS]); /* Special segments. */ nvmm_get_segment(&env->gdt, &state->segs[NVMM_X64_SEG_GDT]); nvmm_get_segment(&env->ldt, &state->segs[NVMM_X64_SEG_LDT]); nvmm_get_segment(&env->tr, &state->segs[NVMM_X64_SEG_TR]); nvmm_get_segment(&env->idt, &state->segs[NVMM_X64_SEG_IDT]); /* Control registers. */ env->cr[0] = state->crs[NVMM_X64_CR_CR0]; env->cr[2] = state->crs[NVMM_X64_CR_CR2]; env->cr[3] = state->crs[NVMM_X64_CR_CR3]; env->cr[4] = state->crs[NVMM_X64_CR_CR4]; tpr = state->crs[NVMM_X64_CR_CR8]; if (tpr != qcpu->tpr) { qcpu->tpr = tpr; cpu_set_apic_tpr(x86_cpu->apic_state, tpr); } env->xcr0 = state->crs[NVMM_X64_CR_XCR0]; /* Debug registers. */ env->dr[0] = state->drs[NVMM_X64_DR_DR0]; env->dr[1] = state->drs[NVMM_X64_DR_DR1]; env->dr[2] = state->drs[NVMM_X64_DR_DR2]; env->dr[3] = state->drs[NVMM_X64_DR_DR3]; env->dr[6] = state->drs[NVMM_X64_DR_DR6]; env->dr[7] = state->drs[NVMM_X64_DR_DR7]; /* FPU. */ env->fpuc = state->fpu.fx_cw; env->fpstt = (state->fpu.fx_sw >> 11) & 0x7; env->fpus = state->fpu.fx_sw & ~0x3800; for (i = 0; i < 8; i++) { env->fptags[i] = !((state->fpu.fx_tw >> i) & 1); } env->fpop = state->fpu.fx_opcode; env->fpip = state->fpu.fx_ip.fa_64; env->fpdp = state->fpu.fx_dp.fa_64; env->mxcsr = state->fpu.fx_mxcsr; assert(sizeof(state->fpu.fx_87_ac) == sizeof(env->fpregs)); memcpy(env->fpregs, state->fpu.fx_87_ac, sizeof(env->fpregs)); for (i = 0; i < CPU_NB_REGS; i++) { memcpy(&env->xmm_regs[i].ZMM_Q(0), &state->fpu.fx_xmm[i].xmm_bytes[0], 8); memcpy(&env->xmm_regs[i].ZMM_Q(1), &state->fpu.fx_xmm[i].xmm_bytes[8], 8); } /* MSRs. */ env->efer = state->msrs[NVMM_X64_MSR_EFER]; env->star = state->msrs[NVMM_X64_MSR_STAR]; #ifdef TARGET_X86_64 env->lstar = state->msrs[NVMM_X64_MSR_LSTAR]; env->cstar = state->msrs[NVMM_X64_MSR_CSTAR]; env->fmask = state->msrs[NVMM_X64_MSR_SFMASK]; env->kernelgsbase = state->msrs[NVMM_X64_MSR_KERNELGSBASE]; #endif env->sysenter_cs = state->msrs[NVMM_X64_MSR_SYSENTER_CS]; env->sysenter_esp = state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; env->sysenter_eip = state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; env->pat = state->msrs[NVMM_X64_MSR_PAT]; env->tsc = state->msrs[NVMM_X64_MSR_TSC]; x86_update_hflags(env); } static bool nvmm_can_take_int(CPUState *cpu) { AccelCPUState *qcpu = cpu->accel; struct nvmm_vcpu *vcpu = &qcpu->vcpu; struct nvmm_machine *mach = get_nvmm_mach(); if (qcpu->int_window_exit) { return false; } if (qcpu->int_shadow || !(cpu_env(cpu)->eflags & IF_MASK)) { struct nvmm_x64_state *state = vcpu->state; /* Exit on interrupt window. */ nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_INTR); state->intr.int_window_exiting = 1; nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_INTR); return false; } return true; } static bool nvmm_can_take_nmi(CPUState *cpu) { AccelCPUState *qcpu = cpu->accel; /* * Contrary to INTs, NMIs always schedule an exit when they are * completed. Therefore, if window-exiting is enabled, it means * NMIs are blocked. */ if (qcpu->nmi_window_exit) { return false; } return true; } /* * Called before the VCPU is run. We inject events generated by the I/O * thread, and synchronize the guest TPR. */ static void nvmm_vcpu_pre_run(CPUState *cpu) { CPUX86State *env = cpu_env(cpu); struct nvmm_machine *mach = get_nvmm_mach(); AccelCPUState *qcpu = cpu->accel; struct nvmm_vcpu *vcpu = &qcpu->vcpu; X86CPU *x86_cpu = X86_CPU(cpu); struct nvmm_x64_state *state = vcpu->state; struct nvmm_vcpu_event *event = vcpu->event; bool has_event = false; bool sync_tpr = false; uint8_t tpr; int ret; bql_lock(); tpr = cpu_get_apic_tpr(x86_cpu->apic_state); if (tpr != qcpu->tpr) { qcpu->tpr = tpr; sync_tpr = true; } /* * Force the VCPU out of its inner loop to process any INIT requests * or commit pending TPR access. */ if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { cpu->exit_request = 1; } if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_NMI)) { if (nvmm_can_take_nmi(cpu)) { cpu->interrupt_request &= ~CPU_INTERRUPT_NMI; event->type = NVMM_VCPU_EVENT_INTR; event->vector = 2; has_event = true; } } if (!has_event && (cpu->interrupt_request & CPU_INTERRUPT_HARD)) { if (nvmm_can_take_int(cpu)) { cpu->interrupt_request &= ~CPU_INTERRUPT_HARD; event->type = NVMM_VCPU_EVENT_INTR; event->vector = cpu_get_pic_interrupt(env); has_event = true; } } /* Don't want SMIs. */ if (cpu->interrupt_request & CPU_INTERRUPT_SMI) { cpu->interrupt_request &= ~CPU_INTERRUPT_SMI; } if (sync_tpr) { ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_CRS); if (ret == -1) { error_report("NVMM: Failed to get CPU state," " error=%d", errno); } state->crs[NVMM_X64_CR_CR8] = qcpu->tpr; ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_CRS); if (ret == -1) { error_report("NVMM: Failed to set CPU state," " error=%d", errno); } } if (has_event) { ret = nvmm_vcpu_inject(mach, vcpu); if (ret == -1) { error_report("NVMM: Failed to inject event," " error=%d", errno); } } bql_unlock(); } /* * Called after the VCPU ran. We synchronize the host view of the TPR and * RFLAGS. */ static void nvmm_vcpu_post_run(CPUState *cpu, struct nvmm_vcpu_exit *exit) { AccelCPUState *qcpu = cpu->accel; X86CPU *x86_cpu = X86_CPU(cpu); CPUX86State *env = &x86_cpu->env; uint64_t tpr; env->eflags = exit->exitstate.rflags; qcpu->int_shadow = exit->exitstate.int_shadow; qcpu->int_window_exit = exit->exitstate.int_window_exiting; qcpu->nmi_window_exit = exit->exitstate.nmi_window_exiting; tpr = exit->exitstate.cr8; if (qcpu->tpr != tpr) { qcpu->tpr = tpr; bql_lock(); cpu_set_apic_tpr(x86_cpu->apic_state, qcpu->tpr); bql_unlock(); } } /* -------------------------------------------------------------------------- */ static void nvmm_io_callback(struct nvmm_io *io) { MemTxAttrs attrs = { 0 }; int ret; ret = address_space_rw(&address_space_io, io->port, attrs, io->data, io->size, !io->in); if (ret != MEMTX_OK) { error_report("NVMM: I/O Transaction Failed " "[%s, port=%u, size=%zu]", (io->in ? "in" : "out"), io->port, io->size); } /* Needed, otherwise infinite loop. */ current_cpu->accel->dirty = false; } static void nvmm_mem_callback(struct nvmm_mem *mem) { cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write); /* Needed, otherwise infinite loop. */ current_cpu->accel->dirty = false; } static struct nvmm_assist_callbacks nvmm_callbacks = { .io = nvmm_io_callback, .mem = nvmm_mem_callback }; /* -------------------------------------------------------------------------- */ static int nvmm_handle_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) { int ret; ret = nvmm_assist_mem(mach, vcpu); if (ret == -1) { error_report("NVMM: Mem Assist Failed [gpa=%p]", (void *)vcpu->exit->u.mem.gpa); } return ret; } static int nvmm_handle_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) { int ret; ret = nvmm_assist_io(mach, vcpu); if (ret == -1) { error_report("NVMM: I/O Assist Failed [port=%d]", (int)vcpu->exit->u.io.port); } return ret; } static int nvmm_handle_rdmsr(struct nvmm_machine *mach, CPUState *cpu, struct nvmm_vcpu_exit *exit) { AccelCPUState *qcpu = cpu->accel; struct nvmm_vcpu *vcpu = &qcpu->vcpu; X86CPU *x86_cpu = X86_CPU(cpu); struct nvmm_x64_state *state = vcpu->state; uint64_t val; int ret; switch (exit->u.rdmsr.msr) { case MSR_IA32_APICBASE: val = cpu_get_apic_base(x86_cpu->apic_state); break; case MSR_MTRRcap: case MSR_MTRRdefType: case MSR_MCG_CAP: case MSR_MCG_STATUS: val = 0; break; default: /* More MSRs to add? */ val = 0; error_report("NVMM: Unexpected RDMSR 0x%x, ignored", exit->u.rdmsr.msr); break; } ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS); if (ret == -1) { return -1; } state->gprs[NVMM_X64_GPR_RAX] = (val & 0xFFFFFFFF); state->gprs[NVMM_X64_GPR_RDX] = (val >> 32); state->gprs[NVMM_X64_GPR_RIP] = exit->u.rdmsr.npc; ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); if (ret == -1) { return -1; } return 0; } static int nvmm_handle_wrmsr(struct nvmm_machine *mach, CPUState *cpu, struct nvmm_vcpu_exit *exit) { AccelCPUState *qcpu = cpu->accel; struct nvmm_vcpu *vcpu = &qcpu->vcpu; X86CPU *x86_cpu = X86_CPU(cpu); struct nvmm_x64_state *state = vcpu->state; uint64_t val; int ret; val = exit->u.wrmsr.val; switch (exit->u.wrmsr.msr) { case MSR_IA32_APICBASE: cpu_set_apic_base(x86_cpu->apic_state, val); break; case MSR_MTRRdefType: case MSR_MCG_STATUS: break; default: /* More MSRs to add? */ error_report("NVMM: Unexpected WRMSR 0x%x [val=0x%lx], ignored", exit->u.wrmsr.msr, val); break; } ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_GPRS); if (ret == -1) { return -1; } state->gprs[NVMM_X64_GPR_RIP] = exit->u.wrmsr.npc; ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); if (ret == -1) { return -1; } return 0; } static int nvmm_handle_halted(struct nvmm_machine *mach, CPUState *cpu, struct nvmm_vcpu_exit *exit) { int ret = 0; bql_lock(); if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) && (cpu_env(cpu)->eflags & IF_MASK)) && !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) { cpu->exception_index = EXCP_HLT; cpu->halted = true; ret = 1; } bql_unlock(); return ret; } static int nvmm_inject_ud(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) { struct nvmm_vcpu_event *event = vcpu->event; event->type = NVMM_VCPU_EVENT_EXCP; event->vector = 6; event->u.excp.error = 0; return nvmm_vcpu_inject(mach, vcpu); } static int nvmm_vcpu_loop(CPUState *cpu) { struct nvmm_machine *mach = get_nvmm_mach(); AccelCPUState *qcpu = cpu->accel; struct nvmm_vcpu *vcpu = &qcpu->vcpu; X86CPU *x86_cpu = X86_CPU(cpu); CPUX86State *env = &x86_cpu->env; struct nvmm_vcpu_exit *exit = vcpu->exit; int ret; /* * Some asynchronous events must be handled outside of the inner * VCPU loop. They are handled here. */ if (cpu->interrupt_request & CPU_INTERRUPT_INIT) { nvmm_cpu_synchronize_state(cpu); do_cpu_init(x86_cpu); /* set int/nmi windows back to the reset state */ } if (cpu->interrupt_request & CPU_INTERRUPT_POLL) { cpu->interrupt_request &= ~CPU_INTERRUPT_POLL; apic_poll_irq(x86_cpu->apic_state); } if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) && (env->eflags & IF_MASK)) || (cpu->interrupt_request & CPU_INTERRUPT_NMI)) { cpu->halted = false; } if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) { nvmm_cpu_synchronize_state(cpu); do_cpu_sipi(x86_cpu); } if (cpu->interrupt_request & CPU_INTERRUPT_TPR) { cpu->interrupt_request &= ~CPU_INTERRUPT_TPR; nvmm_cpu_synchronize_state(cpu); apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip, env->tpr_access_type); } if (cpu->halted) { cpu->exception_index = EXCP_HLT; qatomic_set(&cpu->exit_request, false); return 0; } bql_unlock(); cpu_exec_start(cpu); /* * Inner VCPU loop. */ do { if (cpu->accel->dirty) { nvmm_set_registers(cpu); cpu->accel->dirty = false; } if (qcpu->stop) { cpu->exception_index = EXCP_INTERRUPT; qcpu->stop = false; ret = 1; break; } nvmm_vcpu_pre_run(cpu); if (qatomic_read(&cpu->exit_request)) { #if NVMM_USER_VERSION >= 2 nvmm_vcpu_stop(vcpu); #else qemu_cpu_kick_self(); #endif } /* Read exit_request before the kernel reads the immediate exit flag */ smp_rmb(); ret = nvmm_vcpu_run(mach, vcpu); if (ret == -1) { error_report("NVMM: Failed to exec a virtual processor," " error=%d", errno); break; } nvmm_vcpu_post_run(cpu, exit); switch (exit->reason) { case NVMM_VCPU_EXIT_NONE: break; #if NVMM_USER_VERSION >= 2 case NVMM_VCPU_EXIT_STOPPED: /* * The kernel cleared the immediate exit flag; cpu->exit_request * must be cleared after */ smp_wmb(); qcpu->stop = true; break; #endif case NVMM_VCPU_EXIT_MEMORY: ret = nvmm_handle_mem(mach, vcpu); break; case NVMM_VCPU_EXIT_IO: ret = nvmm_handle_io(mach, vcpu); break; case NVMM_VCPU_EXIT_INT_READY: case NVMM_VCPU_EXIT_NMI_READY: case NVMM_VCPU_EXIT_TPR_CHANGED: break; case NVMM_VCPU_EXIT_HALTED: ret = nvmm_handle_halted(mach, cpu, exit); break; case NVMM_VCPU_EXIT_SHUTDOWN: qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); cpu->exception_index = EXCP_INTERRUPT; ret = 1; break; case NVMM_VCPU_EXIT_RDMSR: ret = nvmm_handle_rdmsr(mach, cpu, exit); break; case NVMM_VCPU_EXIT_WRMSR: ret = nvmm_handle_wrmsr(mach, cpu, exit); break; case NVMM_VCPU_EXIT_MONITOR: case NVMM_VCPU_EXIT_MWAIT: ret = nvmm_inject_ud(mach, vcpu); break; default: error_report("NVMM: Unexpected VM exit code 0x%lx [hw=0x%lx]", exit->reason, exit->u.inv.hwcode); nvmm_get_registers(cpu); bql_lock(); qemu_system_guest_panicked(cpu_get_crash_info(cpu)); bql_unlock(); ret = -1; break; } } while (ret == 0); cpu_exec_end(cpu); bql_lock(); qatomic_set(&cpu->exit_request, false); return ret < 0; } /* -------------------------------------------------------------------------- */ static void do_nvmm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg) { nvmm_get_registers(cpu); cpu->accel->dirty = true; } static void do_nvmm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg) { nvmm_set_registers(cpu); cpu->accel->dirty = false; } static void do_nvmm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) { nvmm_set_registers(cpu); cpu->accel->dirty = false; } static void do_nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg) { cpu->accel->dirty = true; } void nvmm_cpu_synchronize_state(CPUState *cpu) { if (!cpu->accel->dirty) { run_on_cpu(cpu, do_nvmm_cpu_synchronize_state, RUN_ON_CPU_NULL); } } void nvmm_cpu_synchronize_post_reset(CPUState *cpu) { run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); } void nvmm_cpu_synchronize_post_init(CPUState *cpu) { run_on_cpu(cpu, do_nvmm_cpu_synchronize_post_init, RUN_ON_CPU_NULL); } void nvmm_cpu_synchronize_pre_loadvm(CPUState *cpu) { run_on_cpu(cpu, do_nvmm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL); } /* -------------------------------------------------------------------------- */ static Error *nvmm_migration_blocker; /* * The nvmm_vcpu_stop() mechanism breaks races between entering the VMM * and another thread signaling the vCPU thread to exit. */ static void nvmm_ipi_signal(int sigcpu) { if (current_cpu) { AccelCPUState *qcpu = current_cpu->accel; #if NVMM_USER_VERSION >= 2 struct nvmm_vcpu *vcpu = &qcpu->vcpu; nvmm_vcpu_stop(vcpu); #else qcpu->stop = true; #endif } } static void nvmm_init_cpu_signals(void) { struct sigaction sigact; sigset_t set; /* Install the IPI handler. */ memset(&sigact, 0, sizeof(sigact)); sigact.sa_handler = nvmm_ipi_signal; sigaction(SIG_IPI, &sigact, NULL); /* Allow IPIs on the current thread. */ sigprocmask(SIG_BLOCK, NULL, &set); sigdelset(&set, SIG_IPI); pthread_sigmask(SIG_SETMASK, &set, NULL); } int nvmm_init_vcpu(CPUState *cpu) { struct nvmm_machine *mach = get_nvmm_mach(); struct nvmm_vcpu_conf_cpuid cpuid; struct nvmm_vcpu_conf_tpr tpr; Error *local_error = NULL; AccelCPUState *qcpu; int ret, err; nvmm_init_cpu_signals(); if (nvmm_migration_blocker == NULL) { error_setg(&nvmm_migration_blocker, "NVMM: Migration not supported"); if (migrate_add_blocker(&nvmm_migration_blocker, &local_error) < 0) { error_report_err(local_error); return -EINVAL; } } qcpu = g_new0(AccelCPUState, 1); ret = nvmm_vcpu_create(mach, cpu->cpu_index, &qcpu->vcpu); if (ret == -1) { err = errno; error_report("NVMM: Failed to create a virtual processor," " error=%d", err); g_free(qcpu); return -err; } memset(&cpuid, 0, sizeof(cpuid)); cpuid.mask = 1; cpuid.leaf = 0x00000001; cpuid.u.mask.set.edx = CPUID_MCE | CPUID_MCA | CPUID_MTRR; ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CPUID, &cpuid); if (ret == -1) { err = errno; error_report("NVMM: Failed to configure a virtual processor," " error=%d", err); g_free(qcpu); return -err; } ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_CALLBACKS, &nvmm_callbacks); if (ret == -1) { err = errno; error_report("NVMM: Failed to configure a virtual processor," " error=%d", err); g_free(qcpu); return -err; } if (qemu_mach.cap.arch.vcpu_conf_support & NVMM_CAP_ARCH_VCPU_CONF_TPR) { memset(&tpr, 0, sizeof(tpr)); tpr.exit_changed = 1; ret = nvmm_vcpu_configure(mach, &qcpu->vcpu, NVMM_VCPU_CONF_TPR, &tpr); if (ret == -1) { err = errno; error_report("NVMM: Failed to configure a virtual processor," " error=%d", err); g_free(qcpu); return -err; } } cpu->accel->dirty = true; cpu->accel = qcpu; return 0; } int nvmm_vcpu_exec(CPUState *cpu) { int ret, fatal; while (1) { if (cpu->exception_index >= EXCP_INTERRUPT) { ret = cpu->exception_index; cpu->exception_index = -1; break; } fatal = nvmm_vcpu_loop(cpu); if (fatal) { error_report("NVMM: Failed to execute a VCPU."); abort(); } } return ret; } void nvmm_destroy_vcpu(CPUState *cpu) { struct nvmm_machine *mach = get_nvmm_mach(); AccelCPUState *qcpu = cpu->accel; nvmm_vcpu_destroy(mach, &qcpu->vcpu); g_free(cpu->accel); } /* -------------------------------------------------------------------------- */ static void nvmm_update_mapping(hwaddr start_pa, ram_addr_t size, uintptr_t hva, bool add, bool rom, const char *name) { struct nvmm_machine *mach = get_nvmm_mach(); int ret, prot; if (add) { prot = PROT_READ | PROT_EXEC; if (!rom) { prot |= PROT_WRITE; } ret = nvmm_gpa_map(mach, hva, start_pa, size, prot); } else { ret = nvmm_gpa_unmap(mach, hva, start_pa, size); } if (ret == -1) { error_report("NVMM: Failed to %s GPA range '%s' PA:%p, " "Size:%p bytes, HostVA:%p, error=%d", (add ? "map" : "unmap"), name, (void *)(uintptr_t)start_pa, (void *)size, (void *)hva, errno); } } static void nvmm_process_section(MemoryRegionSection *section, int add) { MemoryRegion *mr = section->mr; hwaddr start_pa = section->offset_within_address_space; ram_addr_t size = int128_get64(section->size); unsigned int delta; uintptr_t hva; if (!memory_region_is_ram(mr)) { return; } /* Adjust start_pa and size so that they are page-aligned. */ delta = qemu_real_host_page_size() - (start_pa & ~qemu_real_host_page_mask()); delta &= ~qemu_real_host_page_mask(); if (delta > size) { return; } start_pa += delta; size -= delta; size &= qemu_real_host_page_mask(); if (!size || (start_pa & ~qemu_real_host_page_mask())) { return; } hva = (uintptr_t)memory_region_get_ram_ptr(mr) + section->offset_within_region + delta; nvmm_update_mapping(start_pa, size, hva, add, memory_region_is_rom(mr), mr->name); } static void nvmm_region_add(MemoryListener *listener, MemoryRegionSection *section) { memory_region_ref(section->mr); nvmm_process_section(section, 1); } static void nvmm_region_del(MemoryListener *listener, MemoryRegionSection *section) { nvmm_process_section(section, 0); memory_region_unref(section->mr); } static void nvmm_transaction_begin(MemoryListener *listener) { /* nothing */ } static void nvmm_transaction_commit(MemoryListener *listener) { /* nothing */ } static void nvmm_log_sync(MemoryListener *listener, MemoryRegionSection *section) { MemoryRegion *mr = section->mr; if (!memory_region_is_ram(mr)) { return; } memory_region_set_dirty(mr, 0, int128_get64(section->size)); } static MemoryListener nvmm_memory_listener = { .name = "nvmm", .begin = nvmm_transaction_begin, .commit = nvmm_transaction_commit, .region_add = nvmm_region_add, .region_del = nvmm_region_del, .log_sync = nvmm_log_sync, .priority = MEMORY_LISTENER_PRIORITY_ACCEL, }; static void nvmm_ram_block_added(RAMBlockNotifier *n, void *host, size_t size, size_t max_size) { struct nvmm_machine *mach = get_nvmm_mach(); uintptr_t hva = (uintptr_t)host; int ret; ret = nvmm_hva_map(mach, hva, max_size); if (ret == -1) { error_report("NVMM: Failed to map HVA, HostVA:%p " "Size:%p bytes, error=%d", (void *)hva, (void *)size, errno); } } static struct RAMBlockNotifier nvmm_ram_notifier = { .ram_block_added = nvmm_ram_block_added }; /* -------------------------------------------------------------------------- */ static int nvmm_accel_init(MachineState *ms) { int ret, err; ret = nvmm_init(); if (ret == -1) { err = errno; error_report("NVMM: Initialization failed, error=%d", errno); return -err; } ret = nvmm_capability(&qemu_mach.cap); if (ret == -1) { err = errno; error_report("NVMM: Unable to fetch capability, error=%d", errno); return -err; } if (qemu_mach.cap.version < NVMM_KERN_VERSION) { error_report("NVMM: Unsupported version %u", qemu_mach.cap.version); return -EPROGMISMATCH; } if (qemu_mach.cap.state_size != sizeof(struct nvmm_x64_state)) { error_report("NVMM: Wrong state size %u", qemu_mach.cap.state_size); return -EPROGMISMATCH; } ret = nvmm_machine_create(&qemu_mach.mach); if (ret == -1) { err = errno; error_report("NVMM: Machine creation failed, error=%d", errno); return -err; } memory_listener_register(&nvmm_memory_listener, &address_space_memory); ram_block_notifier_add(&nvmm_ram_notifier); printf("NetBSD Virtual Machine Monitor accelerator is operational\n"); return 0; } int nvmm_enabled(void) { return nvmm_allowed; } static void nvmm_accel_class_init(ObjectClass *oc, void *data) { AccelClass *ac = ACCEL_CLASS(oc); ac->name = "NVMM"; ac->init_machine = nvmm_accel_init; ac->allowed = &nvmm_allowed; } static const TypeInfo nvmm_accel_type = { .name = ACCEL_CLASS_NAME("nvmm"), .parent = TYPE_ACCEL, .class_init = nvmm_accel_class_init, }; static void nvmm_type_init(void) { type_register_static(&nvmm_accel_type); } type_init(nvmm_type_init);