Lines Matching full:vcpu
26 STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
27 STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
28 STATS_DESC_COUNTER(VCPU, mmio_exit_user),
29 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
30 STATS_DESC_COUNTER(VCPU, csr_exit_user),
31 STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
32 STATS_DESC_COUNTER(VCPU, signal_exits),
33 STATS_DESC_COUNTER(VCPU, exits)
45 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) in kvm_riscv_reset_vcpu() argument
47 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; in kvm_riscv_reset_vcpu()
48 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; in kvm_riscv_reset_vcpu()
49 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; in kvm_riscv_reset_vcpu()
50 struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context; in kvm_riscv_reset_vcpu()
59 loaded = (vcpu->cpu != -1); in kvm_riscv_reset_vcpu()
61 kvm_arch_vcpu_put(vcpu); in kvm_riscv_reset_vcpu()
63 vcpu->arch.last_exit_cpu = -1; in kvm_riscv_reset_vcpu()
69 kvm_riscv_vcpu_fp_reset(vcpu); in kvm_riscv_reset_vcpu()
71 kvm_riscv_vcpu_vector_reset(vcpu); in kvm_riscv_reset_vcpu()
73 kvm_riscv_vcpu_timer_reset(vcpu); in kvm_riscv_reset_vcpu()
75 kvm_riscv_vcpu_aia_reset(vcpu); in kvm_riscv_reset_vcpu()
77 bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); in kvm_riscv_reset_vcpu()
78 bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); in kvm_riscv_reset_vcpu()
80 kvm_riscv_vcpu_pmu_reset(vcpu); in kvm_riscv_reset_vcpu()
82 vcpu->arch.hfence_head = 0; in kvm_riscv_reset_vcpu()
83 vcpu->arch.hfence_tail = 0; in kvm_riscv_reset_vcpu()
84 memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); in kvm_riscv_reset_vcpu()
88 kvm_arch_vcpu_load(vcpu, smp_processor_id()); in kvm_riscv_reset_vcpu()
97 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_create() argument
101 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; in kvm_arch_vcpu_create()
103 spin_lock_init(&vcpu->arch.mp_state_lock); in kvm_arch_vcpu_create()
105 /* Mark this VCPU never ran */ in kvm_arch_vcpu_create()
106 vcpu->arch.ran_atleast_once = false; in kvm_arch_vcpu_create()
107 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; in kvm_arch_vcpu_create()
108 bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); in kvm_arch_vcpu_create()
110 /* Setup ISA features available to VCPU */ in kvm_arch_vcpu_create()
111 kvm_riscv_vcpu_setup_isa(vcpu); in kvm_arch_vcpu_create()
114 vcpu->arch.mvendorid = sbi_get_mvendorid(); in kvm_arch_vcpu_create()
115 vcpu->arch.marchid = sbi_get_marchid(); in kvm_arch_vcpu_create()
116 vcpu->arch.mimpid = sbi_get_mimpid(); in kvm_arch_vcpu_create()
118 /* Setup VCPU hfence queue */ in kvm_arch_vcpu_create()
119 spin_lock_init(&vcpu->arch.hfence_lock); in kvm_arch_vcpu_create()
122 cntx = &vcpu->arch.guest_reset_context; in kvm_arch_vcpu_create()
129 if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) in kvm_arch_vcpu_create()
135 /* Setup VCPU timer */ in kvm_arch_vcpu_create()
136 kvm_riscv_vcpu_timer_init(vcpu); in kvm_arch_vcpu_create()
139 kvm_riscv_vcpu_pmu_init(vcpu); in kvm_arch_vcpu_create()
141 /* Setup VCPU AIA */ in kvm_arch_vcpu_create()
142 rc = kvm_riscv_vcpu_aia_init(vcpu); in kvm_arch_vcpu_create()
146 /* Reset VCPU */ in kvm_arch_vcpu_create()
147 kvm_riscv_reset_vcpu(vcpu); in kvm_arch_vcpu_create()
152 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_postcreate() argument
155 * vcpu with id 0 is the designated boot cpu. in kvm_arch_vcpu_postcreate()
159 if (vcpu->vcpu_idx != 0) in kvm_arch_vcpu_postcreate()
160 kvm_riscv_vcpu_power_off(vcpu); in kvm_arch_vcpu_postcreate()
163 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_destroy() argument
165 /* Cleanup VCPU AIA context */ in kvm_arch_vcpu_destroy()
166 kvm_riscv_vcpu_aia_deinit(vcpu); in kvm_arch_vcpu_destroy()
168 /* Cleanup VCPU timer */ in kvm_arch_vcpu_destroy()
169 kvm_riscv_vcpu_timer_deinit(vcpu); in kvm_arch_vcpu_destroy()
171 kvm_riscv_vcpu_pmu_deinit(vcpu); in kvm_arch_vcpu_destroy()
174 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); in kvm_arch_vcpu_destroy()
177 kvm_riscv_vcpu_free_vector_context(vcpu); in kvm_arch_vcpu_destroy()
180 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) in kvm_cpu_has_pending_timer() argument
182 return kvm_riscv_vcpu_timer_pending(vcpu); in kvm_cpu_has_pending_timer()
185 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_blocking() argument
187 kvm_riscv_aia_wakeon_hgei(vcpu, true); in kvm_arch_vcpu_blocking()
190 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_unblocking() argument
192 kvm_riscv_aia_wakeon_hgei(vcpu, false); in kvm_arch_vcpu_unblocking()
195 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_runnable() argument
197 return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) && in kvm_arch_vcpu_runnable()
198 !kvm_riscv_vcpu_stopped(vcpu) && !vcpu->arch.pause); in kvm_arch_vcpu_runnable()
201 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_should_kick() argument
203 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; in kvm_arch_vcpu_should_kick()
206 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_in_kernel() argument
208 return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false; in kvm_arch_vcpu_in_kernel()
211 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) in kvm_arch_vcpu_fault() argument
219 struct kvm_vcpu *vcpu = filp->private_data; in kvm_arch_vcpu_async_ioctl() local
229 return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); in kvm_arch_vcpu_async_ioctl()
231 return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); in kvm_arch_vcpu_async_ioctl()
240 struct kvm_vcpu *vcpu = filp->private_data; in kvm_arch_vcpu_ioctl() local
254 r = kvm_riscv_vcpu_set_reg(vcpu, ®); in kvm_arch_vcpu_ioctl()
256 r = kvm_riscv_vcpu_get_reg(vcpu, ®); in kvm_arch_vcpu_ioctl()
268 reg_list.n = kvm_riscv_vcpu_num_regs(vcpu); in kvm_arch_vcpu_ioctl()
274 r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg); in kvm_arch_vcpu_ioctl()
284 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_get_sregs() argument
290 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_set_sregs() argument
296 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) in kvm_arch_vcpu_ioctl_get_fpu() argument
301 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) in kvm_arch_vcpu_ioctl_set_fpu() argument
306 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_translate() argument
312 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) in kvm_arch_vcpu_ioctl_get_regs() argument
317 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) in kvm_arch_vcpu_ioctl_set_regs() argument
322 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) in kvm_riscv_vcpu_flush_interrupts() argument
324 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; in kvm_riscv_vcpu_flush_interrupts()
327 if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { in kvm_riscv_vcpu_flush_interrupts()
328 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); in kvm_riscv_vcpu_flush_interrupts()
329 val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; in kvm_riscv_vcpu_flush_interrupts()
336 kvm_riscv_vcpu_aia_flush_interrupts(vcpu); in kvm_riscv_vcpu_flush_interrupts()
339 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) in kvm_riscv_vcpu_sync_interrupts() argument
342 struct kvm_vcpu_arch *v = &vcpu->arch; in kvm_riscv_vcpu_sync_interrupts()
343 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; in kvm_riscv_vcpu_sync_interrupts()
363 kvm_riscv_vcpu_aia_sync_interrupts(vcpu); in kvm_riscv_vcpu_sync_interrupts()
366 kvm_riscv_vcpu_timer_sync(vcpu); in kvm_riscv_vcpu_sync_interrupts()
369 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) in kvm_riscv_vcpu_set_interrupt() argument
382 set_bit(irq, vcpu->arch.irqs_pending); in kvm_riscv_vcpu_set_interrupt()
384 set_bit(irq, vcpu->arch.irqs_pending_mask); in kvm_riscv_vcpu_set_interrupt()
386 kvm_vcpu_kick(vcpu); in kvm_riscv_vcpu_set_interrupt()
391 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) in kvm_riscv_vcpu_unset_interrupt() argument
404 clear_bit(irq, vcpu->arch.irqs_pending); in kvm_riscv_vcpu_unset_interrupt()
406 set_bit(irq, vcpu->arch.irqs_pending_mask); in kvm_riscv_vcpu_unset_interrupt()
411 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) in kvm_riscv_vcpu_has_interrupts() argument
415 ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) in kvm_riscv_vcpu_has_interrupts()
417 ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & in kvm_riscv_vcpu_has_interrupts()
419 if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) in kvm_riscv_vcpu_has_interrupts()
423 return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); in kvm_riscv_vcpu_has_interrupts()
426 void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) in __kvm_riscv_vcpu_power_off() argument
428 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); in __kvm_riscv_vcpu_power_off()
429 kvm_make_request(KVM_REQ_SLEEP, vcpu); in __kvm_riscv_vcpu_power_off()
430 kvm_vcpu_kick(vcpu); in __kvm_riscv_vcpu_power_off()
433 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) in kvm_riscv_vcpu_power_off() argument
435 spin_lock(&vcpu->arch.mp_state_lock); in kvm_riscv_vcpu_power_off()
436 __kvm_riscv_vcpu_power_off(vcpu); in kvm_riscv_vcpu_power_off()
437 spin_unlock(&vcpu->arch.mp_state_lock); in kvm_riscv_vcpu_power_off()
440 void __kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) in __kvm_riscv_vcpu_power_on() argument
442 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); in __kvm_riscv_vcpu_power_on()
443 kvm_vcpu_wake_up(vcpu); in __kvm_riscv_vcpu_power_on()
446 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) in kvm_riscv_vcpu_power_on() argument
448 spin_lock(&vcpu->arch.mp_state_lock); in kvm_riscv_vcpu_power_on()
449 __kvm_riscv_vcpu_power_on(vcpu); in kvm_riscv_vcpu_power_on()
450 spin_unlock(&vcpu->arch.mp_state_lock); in kvm_riscv_vcpu_power_on()
453 bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu) in kvm_riscv_vcpu_stopped() argument
455 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; in kvm_riscv_vcpu_stopped()
458 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_get_mpstate() argument
461 *mp_state = READ_ONCE(vcpu->arch.mp_state); in kvm_arch_vcpu_ioctl_get_mpstate()
466 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_set_mpstate() argument
471 spin_lock(&vcpu->arch.mp_state_lock); in kvm_arch_vcpu_ioctl_set_mpstate()
475 WRITE_ONCE(vcpu->arch.mp_state, *mp_state); in kvm_arch_vcpu_ioctl_set_mpstate()
478 __kvm_riscv_vcpu_power_off(vcpu); in kvm_arch_vcpu_ioctl_set_mpstate()
484 spin_unlock(&vcpu->arch.mp_state_lock); in kvm_arch_vcpu_ioctl_set_mpstate()
489 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, in kvm_arch_vcpu_ioctl_set_guest_debug() argument
518 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) in kvm_arch_vcpu_load() argument
520 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; in kvm_arch_vcpu_load()
532 kvm_riscv_vcpu_update_config(vcpu->arch.isa); in kvm_arch_vcpu_load()
534 kvm_riscv_gstage_update_hgatp(vcpu); in kvm_arch_vcpu_load()
536 kvm_riscv_vcpu_timer_restore(vcpu); in kvm_arch_vcpu_load()
538 kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); in kvm_arch_vcpu_load()
539 kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, in kvm_arch_vcpu_load()
540 vcpu->arch.isa); in kvm_arch_vcpu_load()
541 kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); in kvm_arch_vcpu_load()
542 kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, in kvm_arch_vcpu_load()
543 vcpu->arch.isa); in kvm_arch_vcpu_load()
545 kvm_riscv_vcpu_aia_load(vcpu, cpu); in kvm_arch_vcpu_load()
547 vcpu->cpu = cpu; in kvm_arch_vcpu_load()
550 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_put() argument
552 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; in kvm_arch_vcpu_put()
554 vcpu->cpu = -1; in kvm_arch_vcpu_put()
556 kvm_riscv_vcpu_aia_put(vcpu); in kvm_arch_vcpu_put()
558 kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context, in kvm_arch_vcpu_put()
559 vcpu->arch.isa); in kvm_arch_vcpu_put()
560 kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); in kvm_arch_vcpu_put()
562 kvm_riscv_vcpu_timer_save(vcpu); in kvm_arch_vcpu_put()
563 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, in kvm_arch_vcpu_put()
564 vcpu->arch.isa); in kvm_arch_vcpu_put()
565 kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); in kvm_arch_vcpu_put()
578 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) in kvm_riscv_check_vcpu_requests() argument
580 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); in kvm_riscv_check_vcpu_requests()
582 if (kvm_request_pending(vcpu)) { in kvm_riscv_check_vcpu_requests()
583 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) { in kvm_riscv_check_vcpu_requests()
584 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_riscv_check_vcpu_requests()
586 (!kvm_riscv_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), in kvm_riscv_check_vcpu_requests()
588 kvm_vcpu_srcu_read_lock(vcpu); in kvm_riscv_check_vcpu_requests()
590 if (kvm_riscv_vcpu_stopped(vcpu) || vcpu->arch.pause) { in kvm_riscv_check_vcpu_requests()
595 kvm_make_request(KVM_REQ_SLEEP, vcpu); in kvm_riscv_check_vcpu_requests()
599 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) in kvm_riscv_check_vcpu_requests()
600 kvm_riscv_reset_vcpu(vcpu); in kvm_riscv_check_vcpu_requests()
602 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) in kvm_riscv_check_vcpu_requests()
603 kvm_riscv_gstage_update_hgatp(vcpu); in kvm_riscv_check_vcpu_requests()
605 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) in kvm_riscv_check_vcpu_requests()
606 kvm_riscv_fence_i_process(vcpu); in kvm_riscv_check_vcpu_requests()
612 if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) in kvm_riscv_check_vcpu_requests()
613 kvm_riscv_hfence_gvma_vmid_all_process(vcpu); in kvm_riscv_check_vcpu_requests()
615 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) in kvm_riscv_check_vcpu_requests()
616 kvm_riscv_hfence_vvma_all_process(vcpu); in kvm_riscv_check_vcpu_requests()
618 if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) in kvm_riscv_check_vcpu_requests()
619 kvm_riscv_hfence_process(vcpu); in kvm_riscv_check_vcpu_requests()
623 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) in kvm_riscv_update_hvip() argument
625 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; in kvm_riscv_update_hvip()
628 kvm_riscv_vcpu_aia_update_hvip(vcpu); in kvm_riscv_update_hvip()
632 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
633 * the vCPU is running.
638 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu) in kvm_riscv_vcpu_enter_exit() argument
641 __kvm_riscv_switch_to(&vcpu->arch); in kvm_riscv_vcpu_enter_exit()
642 vcpu->arch.last_exit_cpu = vcpu->cpu; in kvm_riscv_vcpu_enter_exit()
646 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) in kvm_arch_vcpu_ioctl_run() argument
650 struct kvm_run *run = vcpu->run; in kvm_arch_vcpu_ioctl_run()
652 /* Mark this VCPU ran at least once */ in kvm_arch_vcpu_ioctl_run()
653 vcpu->arch.ran_atleast_once = true; in kvm_arch_vcpu_ioctl_run()
655 kvm_vcpu_srcu_read_lock(vcpu); in kvm_arch_vcpu_ioctl_run()
660 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run); in kvm_arch_vcpu_ioctl_run()
664 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run); in kvm_arch_vcpu_ioctl_run()
668 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run); in kvm_arch_vcpu_ioctl_run()
675 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_arch_vcpu_ioctl_run()
680 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_arch_vcpu_ioctl_run()
684 vcpu_load(vcpu); in kvm_arch_vcpu_ioctl_run()
686 kvm_sigset_activate(vcpu); in kvm_arch_vcpu_ioctl_run()
692 ret = xfer_to_guest_mode_handle_work(vcpu); in kvm_arch_vcpu_ioctl_run()
697 kvm_riscv_gstage_vmid_update(vcpu); in kvm_arch_vcpu_ioctl_run()
699 kvm_riscv_check_vcpu_requests(vcpu); in kvm_arch_vcpu_ioctl_run()
704 ret = kvm_riscv_vcpu_aia_update(vcpu); in kvm_arch_vcpu_ioctl_run()
714 * interrupts and before the final VCPU requests check. in kvm_arch_vcpu_ioctl_run()
716 * Documentation/virt/kvm/vcpu-requests.rst in kvm_arch_vcpu_ioctl_run()
718 vcpu->mode = IN_GUEST_MODE; in kvm_arch_vcpu_ioctl_run()
720 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_arch_vcpu_ioctl_run()
724 * We might have got VCPU interrupts updated asynchronously in kvm_arch_vcpu_ioctl_run()
727 kvm_riscv_vcpu_flush_interrupts(vcpu); in kvm_arch_vcpu_ioctl_run()
730 kvm_riscv_update_hvip(vcpu); in kvm_arch_vcpu_ioctl_run()
733 kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || in kvm_arch_vcpu_ioctl_run()
734 kvm_request_pending(vcpu) || in kvm_arch_vcpu_ioctl_run()
736 vcpu->mode = OUTSIDE_GUEST_MODE; in kvm_arch_vcpu_ioctl_run()
739 kvm_vcpu_srcu_read_lock(vcpu); in kvm_arch_vcpu_ioctl_run()
749 kvm_riscv_local_tlb_sanitize(vcpu); in kvm_arch_vcpu_ioctl_run()
753 kvm_riscv_vcpu_enter_exit(vcpu); in kvm_arch_vcpu_ioctl_run()
755 vcpu->mode = OUTSIDE_GUEST_MODE; in kvm_arch_vcpu_ioctl_run()
756 vcpu->stat.exits++; in kvm_arch_vcpu_ioctl_run()
763 trap.sepc = vcpu->arch.guest_context.sepc; in kvm_arch_vcpu_ioctl_run()
770 kvm_riscv_vcpu_sync_interrupts(vcpu); in kvm_arch_vcpu_ioctl_run()
791 kvm_vcpu_srcu_read_lock(vcpu); in kvm_arch_vcpu_ioctl_run()
793 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap); in kvm_arch_vcpu_ioctl_run()
796 kvm_sigset_deactivate(vcpu); in kvm_arch_vcpu_ioctl_run()
798 vcpu_put(vcpu); in kvm_arch_vcpu_ioctl_run()
800 kvm_vcpu_srcu_read_unlock(vcpu); in kvm_arch_vcpu_ioctl_run()