45 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)  in kvm_riscv_reset_vcpu()  argument
47 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;  in kvm_riscv_reset_vcpu()
48 	struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;  in kvm_riscv_reset_vcpu()
49 	struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;  in kvm_riscv_reset_vcpu()
50 	struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context;  in kvm_riscv_reset_vcpu()
59 	loaded = (vcpu->cpu != -1);  in kvm_riscv_reset_vcpu()
61 		kvm_arch_vcpu_put(vcpu);  in kvm_riscv_reset_vcpu()
63 	vcpu->arch.last_exit_cpu = -1;  in kvm_riscv_reset_vcpu()
69 	kvm_riscv_vcpu_fp_reset(vcpu);  in kvm_riscv_reset_vcpu()
71 	kvm_riscv_vcpu_vector_reset(vcpu);  in kvm_riscv_reset_vcpu()
73 	kvm_riscv_vcpu_timer_reset(vcpu);  in kvm_riscv_reset_vcpu()
75 	kvm_riscv_vcpu_aia_reset(vcpu);  in kvm_riscv_reset_vcpu()
77 	bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS);  in kvm_riscv_reset_vcpu()
78 	bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS);  in kvm_riscv_reset_vcpu()
80 	kvm_riscv_vcpu_pmu_reset(vcpu);  in kvm_riscv_reset_vcpu()
82 	vcpu->arch.hfence_head = 0;  in kvm_riscv_reset_vcpu()
83 	vcpu->arch.hfence_tail = 0;  in kvm_riscv_reset_vcpu()
84 	memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));  in kvm_riscv_reset_vcpu()
88 		kvm_arch_vcpu_load(vcpu, smp_processor_id());  in kvm_riscv_reset_vcpu()
97 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_create()  argument
101 	struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;  in kvm_arch_vcpu_create()
104 	vcpu->arch.ran_atleast_once = false;  in kvm_arch_vcpu_create()
105 	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;  in kvm_arch_vcpu_create()
106 	bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX);  in kvm_arch_vcpu_create()
109 	kvm_riscv_vcpu_setup_isa(vcpu);  in kvm_arch_vcpu_create()
112 	vcpu->arch.mvendorid = sbi_get_mvendorid();  in kvm_arch_vcpu_create()
113 	vcpu->arch.marchid = sbi_get_marchid();  in kvm_arch_vcpu_create()
114 	vcpu->arch.mimpid = sbi_get_mimpid();  in kvm_arch_vcpu_create()
117 	spin_lock_init(&vcpu->arch.hfence_lock);  in kvm_arch_vcpu_create()
120 	cntx = &vcpu->arch.guest_reset_context;  in kvm_arch_vcpu_create()
127 	if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx))  in kvm_arch_vcpu_create()
134 	kvm_riscv_vcpu_timer_init(vcpu);  in kvm_arch_vcpu_create()
137 	kvm_riscv_vcpu_pmu_init(vcpu);  in kvm_arch_vcpu_create()
140 	rc = kvm_riscv_vcpu_aia_init(vcpu);  in kvm_arch_vcpu_create()
145 	kvm_riscv_reset_vcpu(vcpu);  in kvm_arch_vcpu_create()
150 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_postcreate()  argument
157 	if (vcpu->vcpu_idx != 0)  in kvm_arch_vcpu_postcreate()
158 		kvm_riscv_vcpu_power_off(vcpu);  in kvm_arch_vcpu_postcreate()
161 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_destroy()  argument
164 	kvm_riscv_vcpu_aia_deinit(vcpu);  in kvm_arch_vcpu_destroy()
167 	kvm_riscv_vcpu_timer_deinit(vcpu);  in kvm_arch_vcpu_destroy()
169 	kvm_riscv_vcpu_pmu_deinit(vcpu);  in kvm_arch_vcpu_destroy()
172 	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);  in kvm_arch_vcpu_destroy()
175 	kvm_riscv_vcpu_free_vector_context(vcpu);  in kvm_arch_vcpu_destroy()
178 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)  in kvm_cpu_has_pending_timer()  argument
180 	return kvm_riscv_vcpu_timer_pending(vcpu);  in kvm_cpu_has_pending_timer()
183 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_blocking()  argument
185 	kvm_riscv_aia_wakeon_hgei(vcpu, true);  in kvm_arch_vcpu_blocking()
188 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_unblocking()  argument
190 	kvm_riscv_aia_wakeon_hgei(vcpu, false);  in kvm_arch_vcpu_unblocking()
193 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_runnable()  argument
195 	return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) &&  in kvm_arch_vcpu_runnable()
196 		!vcpu->arch.power_off && !vcpu->arch.pause);  in kvm_arch_vcpu_runnable()
199 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_should_kick()  argument
201 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;  in kvm_arch_vcpu_should_kick()
204 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_in_kernel()  argument
206 	return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false;  in kvm_arch_vcpu_in_kernel()
209 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)  in kvm_arch_vcpu_fault()  argument
217 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_async_ioctl()  local
227 			return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);  in kvm_arch_vcpu_async_ioctl()
229 			return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);  in kvm_arch_vcpu_async_ioctl()
238 	struct kvm_vcpu *vcpu = filp->private_data;  in kvm_arch_vcpu_ioctl()  local
252 			r = kvm_riscv_vcpu_set_reg(vcpu, &reg);  in kvm_arch_vcpu_ioctl()
254 			r = kvm_riscv_vcpu_get_reg(vcpu, &reg);  in kvm_arch_vcpu_ioctl()
266 		reg_list.n = kvm_riscv_vcpu_num_regs(vcpu);  in kvm_arch_vcpu_ioctl()
272 		r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg);  in kvm_arch_vcpu_ioctl()
282 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_sregs()  argument
288 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_sregs()  argument
294 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_get_fpu()  argument
299 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)  in kvm_arch_vcpu_ioctl_set_fpu()  argument
304 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_translate()  argument
310 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_get_regs()  argument
315 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)  in kvm_arch_vcpu_ioctl_set_regs()  argument
320 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)  in kvm_riscv_vcpu_flush_interrupts()  argument
322 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;  in kvm_riscv_vcpu_flush_interrupts()
325 	if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) {  in kvm_riscv_vcpu_flush_interrupts()
326 		mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0);  in kvm_riscv_vcpu_flush_interrupts()
327 		val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask;  in kvm_riscv_vcpu_flush_interrupts()
334 	kvm_riscv_vcpu_aia_flush_interrupts(vcpu);  in kvm_riscv_vcpu_flush_interrupts()
337 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)  in kvm_riscv_vcpu_sync_interrupts()  argument
340 	struct kvm_vcpu_arch *v = &vcpu->arch;  in kvm_riscv_vcpu_sync_interrupts()
341 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;  in kvm_riscv_vcpu_sync_interrupts()
361 	kvm_riscv_vcpu_aia_sync_interrupts(vcpu);  in kvm_riscv_vcpu_sync_interrupts()
364 	kvm_riscv_vcpu_timer_sync(vcpu);  in kvm_riscv_vcpu_sync_interrupts()
367 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)  in kvm_riscv_vcpu_set_interrupt()  argument
380 	set_bit(irq, vcpu->arch.irqs_pending);  in kvm_riscv_vcpu_set_interrupt()
382 	set_bit(irq, vcpu->arch.irqs_pending_mask);  in kvm_riscv_vcpu_set_interrupt()
384 	kvm_vcpu_kick(vcpu);  in kvm_riscv_vcpu_set_interrupt()
389 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)  in kvm_riscv_vcpu_unset_interrupt()  argument
402 	clear_bit(irq, vcpu->arch.irqs_pending);  in kvm_riscv_vcpu_unset_interrupt()
404 	set_bit(irq, vcpu->arch.irqs_pending_mask);  in kvm_riscv_vcpu_unset_interrupt()
409 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)  in kvm_riscv_vcpu_has_interrupts()  argument
413 	ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)  in kvm_riscv_vcpu_has_interrupts()
415 	ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK &  in kvm_riscv_vcpu_has_interrupts()
417 	if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie)  in kvm_riscv_vcpu_has_interrupts()
421 	return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask);  in kvm_riscv_vcpu_has_interrupts()
424 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)  in kvm_riscv_vcpu_power_off()  argument
426 	vcpu->arch.power_off = true;  in kvm_riscv_vcpu_power_off()
427 	kvm_make_request(KVM_REQ_SLEEP, vcpu);  in kvm_riscv_vcpu_power_off()
428 	kvm_vcpu_kick(vcpu);  in kvm_riscv_vcpu_power_off()
431 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)  in kvm_riscv_vcpu_power_on()  argument
433 	vcpu->arch.power_off = false;  in kvm_riscv_vcpu_power_on()
434 	kvm_vcpu_wake_up(vcpu);  in kvm_riscv_vcpu_power_on()
437 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_get_mpstate()  argument
440 	if (vcpu->arch.power_off)  in kvm_arch_vcpu_ioctl_get_mpstate()
448 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_mpstate()  argument
455 		vcpu->arch.power_off = false;  in kvm_arch_vcpu_ioctl_set_mpstate()
458 		kvm_riscv_vcpu_power_off(vcpu);  in kvm_arch_vcpu_ioctl_set_mpstate()
467 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,  in kvm_arch_vcpu_ioctl_set_guest_debug()  argument
496 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)  in kvm_arch_vcpu_load()  argument
498 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;  in kvm_arch_vcpu_load()
510 	kvm_riscv_vcpu_update_config(vcpu->arch.isa);  in kvm_arch_vcpu_load()
512 	kvm_riscv_gstage_update_hgatp(vcpu);  in kvm_arch_vcpu_load()
514 	kvm_riscv_vcpu_timer_restore(vcpu);  in kvm_arch_vcpu_load()
516 	kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context);  in kvm_arch_vcpu_load()
517 	kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,  in kvm_arch_vcpu_load()
518 					vcpu->arch.isa);  in kvm_arch_vcpu_load()
519 	kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context);  in kvm_arch_vcpu_load()
520 	kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context,  in kvm_arch_vcpu_load()
521 					    vcpu->arch.isa);  in kvm_arch_vcpu_load()
523 	kvm_riscv_vcpu_aia_load(vcpu, cpu);  in kvm_arch_vcpu_load()
525 	vcpu->cpu = cpu;  in kvm_arch_vcpu_load()
528 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_put()  argument
530 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;  in kvm_arch_vcpu_put()
532 	vcpu->cpu = -1;  in kvm_arch_vcpu_put()
534 	kvm_riscv_vcpu_aia_put(vcpu);  in kvm_arch_vcpu_put()
536 	kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context,  in kvm_arch_vcpu_put()
537 				     vcpu->arch.isa);  in kvm_arch_vcpu_put()
538 	kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context);  in kvm_arch_vcpu_put()
540 	kvm_riscv_vcpu_timer_save(vcpu);  in kvm_arch_vcpu_put()
541 	kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context,  in kvm_arch_vcpu_put()
542 					 vcpu->arch.isa);  in kvm_arch_vcpu_put()
543 	kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context);  in kvm_arch_vcpu_put()
556 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)  in kvm_riscv_check_vcpu_requests()  argument
558 	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);  in kvm_riscv_check_vcpu_requests()
560 	if (kvm_request_pending(vcpu)) {  in kvm_riscv_check_vcpu_requests()
561 		if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) {  in kvm_riscv_check_vcpu_requests()
562 			kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_riscv_check_vcpu_requests()
564 				(!vcpu->arch.power_off) && (!vcpu->arch.pause),  in kvm_riscv_check_vcpu_requests()
566 			kvm_vcpu_srcu_read_lock(vcpu);  in kvm_riscv_check_vcpu_requests()
568 			if (vcpu->arch.power_off || vcpu->arch.pause) {  in kvm_riscv_check_vcpu_requests()
573 				kvm_make_request(KVM_REQ_SLEEP, vcpu);  in kvm_riscv_check_vcpu_requests()
577 		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))  in kvm_riscv_check_vcpu_requests()
578 			kvm_riscv_reset_vcpu(vcpu);  in kvm_riscv_check_vcpu_requests()
580 		if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))  in kvm_riscv_check_vcpu_requests()
581 			kvm_riscv_gstage_update_hgatp(vcpu);  in kvm_riscv_check_vcpu_requests()
583 		if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))  in kvm_riscv_check_vcpu_requests()
584 			kvm_riscv_fence_i_process(vcpu);  in kvm_riscv_check_vcpu_requests()
590 		if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu))  in kvm_riscv_check_vcpu_requests()
591 			kvm_riscv_hfence_gvma_vmid_all_process(vcpu);  in kvm_riscv_check_vcpu_requests()
593 		if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))  in kvm_riscv_check_vcpu_requests()
594 			kvm_riscv_hfence_vvma_all_process(vcpu);  in kvm_riscv_check_vcpu_requests()
596 		if (kvm_check_request(KVM_REQ_HFENCE, vcpu))  in kvm_riscv_check_vcpu_requests()
597 			kvm_riscv_hfence_process(vcpu);  in kvm_riscv_check_vcpu_requests()
601 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)  in kvm_riscv_update_hvip()  argument
603 	struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;  in kvm_riscv_update_hvip()
606 	kvm_riscv_vcpu_aia_update_hvip(vcpu);  in kvm_riscv_update_hvip()
616 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu)  in kvm_riscv_vcpu_enter_exit()  argument
619 	__kvm_riscv_switch_to(&vcpu->arch);  in kvm_riscv_vcpu_enter_exit()
620 	vcpu->arch.last_exit_cpu = vcpu->cpu;  in kvm_riscv_vcpu_enter_exit()
624 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)  in kvm_arch_vcpu_ioctl_run()  argument
628 	struct kvm_run *run = vcpu->run;  in kvm_arch_vcpu_ioctl_run()
631 	vcpu->arch.ran_atleast_once = true;  in kvm_arch_vcpu_ioctl_run()
633 	kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
638 		ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);  in kvm_arch_vcpu_ioctl_run()
642 		ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);  in kvm_arch_vcpu_ioctl_run()
646 		ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run);  in kvm_arch_vcpu_ioctl_run()
653 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
658 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
662 	vcpu_load(vcpu);  in kvm_arch_vcpu_ioctl_run()
664 	kvm_sigset_activate(vcpu);  in kvm_arch_vcpu_ioctl_run()
670 		ret = xfer_to_guest_mode_handle_work(vcpu);  in kvm_arch_vcpu_ioctl_run()
675 		kvm_riscv_gstage_vmid_update(vcpu);  in kvm_arch_vcpu_ioctl_run()
677 		kvm_riscv_check_vcpu_requests(vcpu);  in kvm_arch_vcpu_ioctl_run()
682 		ret = kvm_riscv_vcpu_aia_update(vcpu);  in kvm_arch_vcpu_ioctl_run()
696 		vcpu->mode = IN_GUEST_MODE;  in kvm_arch_vcpu_ioctl_run()
698 		kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()
705 		kvm_riscv_vcpu_flush_interrupts(vcpu);  in kvm_arch_vcpu_ioctl_run()
708 		kvm_riscv_update_hvip(vcpu);  in kvm_arch_vcpu_ioctl_run()
711 		    kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||  in kvm_arch_vcpu_ioctl_run()
712 		    kvm_request_pending(vcpu) ||  in kvm_arch_vcpu_ioctl_run()
714 			vcpu->mode = OUTSIDE_GUEST_MODE;  in kvm_arch_vcpu_ioctl_run()
717 			kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
727 		kvm_riscv_local_tlb_sanitize(vcpu);  in kvm_arch_vcpu_ioctl_run()
731 		kvm_riscv_vcpu_enter_exit(vcpu);  in kvm_arch_vcpu_ioctl_run()
733 		vcpu->mode = OUTSIDE_GUEST_MODE;  in kvm_arch_vcpu_ioctl_run()
734 		vcpu->stat.exits++;  in kvm_arch_vcpu_ioctl_run()
741 		trap.sepc = vcpu->arch.guest_context.sepc;  in kvm_arch_vcpu_ioctl_run()
748 		kvm_riscv_vcpu_sync_interrupts(vcpu);  in kvm_arch_vcpu_ioctl_run()
769 		kvm_vcpu_srcu_read_lock(vcpu);  in kvm_arch_vcpu_ioctl_run()
771 		ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);  in kvm_arch_vcpu_ioctl_run()
774 	kvm_sigset_deactivate(vcpu);  in kvm_arch_vcpu_ioctl_run()
776 	vcpu_put(vcpu);  in kvm_arch_vcpu_ioctl_run()
778 	kvm_vcpu_srcu_read_unlock(vcpu);  in kvm_arch_vcpu_ioctl_run()