1 // SPDX-License-Identifier: GPL-2.0
16 #include <asm/insn-def.h>
21 					  gpa_t gpa, gpa_t gpsz,  in kvm_riscv_local_hfence_gvma_vmid_gpa()  argument
33 		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))  in kvm_riscv_local_hfence_gvma_vmid_gpa()
34 			asm volatile (HINVAL_GVMA(%0, %1)  in kvm_riscv_local_hfence_gvma_vmid_gpa()
38 		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))  in kvm_riscv_local_hfence_gvma_vmid_gpa()
39 			asm volatile (HFENCE_GVMA(%0, %1)  in kvm_riscv_local_hfence_gvma_vmid_gpa()
46 	asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory");  in kvm_riscv_local_hfence_gvma_vmid_all()
49 void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz,  in kvm_riscv_local_hfence_gvma_gpa()  argument
61 		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))  in kvm_riscv_local_hfence_gvma_gpa()
62 			asm volatile(HINVAL_GVMA(%0, zero)  in kvm_riscv_local_hfence_gvma_gpa()
66 		for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order))  in kvm_riscv_local_hfence_gvma_gpa()
67 			asm volatile(HFENCE_GVMA(%0, zero)  in kvm_riscv_local_hfence_gvma_gpa()
95 			asm volatile(HINVAL_VVMA(%0, %1)  in kvm_riscv_local_hfence_vvma_asid_gva()
100 			asm volatile(HFENCE_VVMA(%0, %1)  in kvm_riscv_local_hfence_vvma_asid_gva()
114 	asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory");  in kvm_riscv_local_hfence_vvma_asid_all()
135 			asm volatile(HINVAL_VVMA(%0, zero)  in kvm_riscv_local_hfence_vvma_gva()
140 			asm volatile(HFENCE_VVMA(%0, zero)  in kvm_riscv_local_hfence_vvma_gva()
163 	    vcpu->arch.last_exit_cpu == vcpu->cpu)  in kvm_riscv_local_tlb_sanitize()
167 	 * On RISC-V platforms with hardware VMID support, we share same  in kvm_riscv_local_tlb_sanitize()
169 	 * have stale G-stage TLB entries on the current Host CPU due to  in kvm_riscv_local_tlb_sanitize()
173 	 * To cleanup stale TLB entries, we simply flush all G-stage TLB  in kvm_riscv_local_tlb_sanitize()
177 	vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);  in kvm_riscv_local_tlb_sanitize()
191 	vmid = &vcpu->kvm->arch.vmid;  in kvm_riscv_hfence_gvma_vmid_all_process()
192 	kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid));  in kvm_riscv_hfence_gvma_vmid_all_process()
199 	vmid = &vcpu->kvm->arch.vmid;  in kvm_riscv_hfence_vvma_all_process()
200 	kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid));  in kvm_riscv_hfence_vvma_all_process()
207 	struct kvm_vcpu_arch *varch = &vcpu->arch;  in vcpu_hfence_dequeue()
209 	spin_lock(&varch->hfence_lock);  in vcpu_hfence_dequeue()
211 	if (varch->hfence_queue[varch->hfence_head].type) {  in vcpu_hfence_dequeue()
212 		memcpy(out_data, &varch->hfence_queue[varch->hfence_head],  in vcpu_hfence_dequeue()
214 		varch->hfence_queue[varch->hfence_head].type = 0;  in vcpu_hfence_dequeue()
216 		varch->hfence_head++;  in vcpu_hfence_dequeue()
217 		if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE)  in vcpu_hfence_dequeue()
218 			varch->hfence_head = 0;  in vcpu_hfence_dequeue()
223 	spin_unlock(&varch->hfence_lock);  in vcpu_hfence_dequeue()
232 	struct kvm_vcpu_arch *varch = &vcpu->arch;  in vcpu_hfence_enqueue()
234 	spin_lock(&varch->hfence_lock);  in vcpu_hfence_enqueue()
236 	if (!varch->hfence_queue[varch->hfence_tail].type) {  in vcpu_hfence_enqueue()
237 		memcpy(&varch->hfence_queue[varch->hfence_tail],  in vcpu_hfence_enqueue()
240 		varch->hfence_tail++;  in vcpu_hfence_enqueue()
241 		if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE)  in vcpu_hfence_enqueue()
242 			varch->hfence_tail = 0;  in vcpu_hfence_enqueue()
247 	spin_unlock(&varch->hfence_lock);  in vcpu_hfence_enqueue()
254 	struct kvm_riscv_hfence d = { 0 };  in kvm_riscv_hfence_process()
255 	struct kvm_vmid *v = &vcpu->kvm->arch.vmid;  in kvm_riscv_hfence_process()
263 						READ_ONCE(v->vmid),  in kvm_riscv_hfence_process()
269 						READ_ONCE(v->vmid), d.asid,  in kvm_riscv_hfence_process()
275 						READ_ONCE(v->vmid), d.asid);  in kvm_riscv_hfence_process()
280 						READ_ONCE(v->vmid),  in kvm_riscv_hfence_process()
301 		if (hbase != -1UL) {  in make_xfence_request()
302 			if (vcpu->vcpu_id < hbase)  in make_xfence_request()
304 			if (!(hmask & (1UL << (vcpu->vcpu_id - hbase))))  in make_xfence_request()
310 		if (!data || !data->type)  in make_xfence_request()
334 				    gpa_t gpa, gpa_t gpsz,  in kvm_riscv_hfence_gvma_vmid_gpa()  argument
340 	data.asid = 0;  in kvm_riscv_hfence_gvma_vmid_gpa()
341 	data.addr = gpa;  in kvm_riscv_hfence_gvma_vmid_gpa()
379 	data.addr = data.size = data.order = 0;  in kvm_riscv_hfence_vvma_asid_all()
392 	data.asid = 0;  in kvm_riscv_hfence_vvma_gva()