1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/cpumask.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/module.h> 14 #include <linux/kvm_host.h> 15 #include <asm/csr.h> 16 #include <asm/sbi.h> 17 18 static unsigned long vmid_version = 1; 19 static unsigned long vmid_next; 20 static unsigned long vmid_bits; 21 static DEFINE_SPINLOCK(vmid_lock); 22 23 void kvm_riscv_stage2_vmid_detect(void) 24 { 25 unsigned long old; 26 27 /* Figure-out number of VMID bits in HW */ 28 old = csr_read(CSR_HGATP); 29 csr_write(CSR_HGATP, old | HGATP_VMID_MASK); 30 vmid_bits = csr_read(CSR_HGATP); 31 vmid_bits = (vmid_bits & HGATP_VMID_MASK) >> HGATP_VMID_SHIFT; 32 vmid_bits = fls_long(vmid_bits); 33 csr_write(CSR_HGATP, old); 34 35 /* We polluted local TLB so flush all guest TLB */ 36 __kvm_riscv_hfence_gvma_all(); 37 38 /* We don't use VMID bits if they are not sufficient */ 39 if ((1UL << vmid_bits) < num_possible_cpus()) 40 vmid_bits = 0; 41 } 42 43 unsigned long kvm_riscv_stage2_vmid_bits(void) 44 { 45 return vmid_bits; 46 } 47 48 int kvm_riscv_stage2_vmid_init(struct kvm *kvm) 49 { 50 /* Mark the initial VMID and VMID version invalid */ 51 kvm->arch.vmid.vmid_version = 0; 52 kvm->arch.vmid.vmid = 0; 53 54 return 0; 55 } 56 57 bool kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid *vmid) 58 { 59 if (!vmid_bits) 60 return false; 61 62 return unlikely(READ_ONCE(vmid->vmid_version) != 63 READ_ONCE(vmid_version)); 64 } 65 66 void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu) 67 { 68 int i; 69 struct kvm_vcpu *v; 70 struct cpumask hmask; 71 struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid; 72 73 if (!kvm_riscv_stage2_vmid_ver_changed(vmid)) 74 return; 75 76 spin_lock(&vmid_lock); 77 78 /* 79 * We need to re-check the vmid_version here to ensure that if 80 * another vcpu already allocated a valid vmid for this vm. 81 */ 82 if (!kvm_riscv_stage2_vmid_ver_changed(vmid)) { 83 spin_unlock(&vmid_lock); 84 return; 85 } 86 87 /* First user of a new VMID version? */ 88 if (unlikely(vmid_next == 0)) { 89 WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1); 90 vmid_next = 1; 91 92 /* 93 * We ran out of VMIDs so we increment vmid_version and 94 * start assigning VMIDs from 1. 95 * 96 * This also means existing VMIDs assignement to all Guest 97 * instances is invalid and we have force VMID re-assignement 98 * for all Guest instances. The Guest instances that were not 99 * running will automatically pick-up new VMIDs because will 100 * call kvm_riscv_stage2_vmid_update() whenever they enter 101 * in-kernel run loop. For Guest instances that are already 102 * running, we force VM exits on all host CPUs using IPI and 103 * flush all Guest TLBs. 104 */ 105 riscv_cpuid_to_hartid_mask(cpu_online_mask, &hmask); 106 sbi_remote_hfence_gvma(cpumask_bits(&hmask), 0, 0); 107 } 108 109 vmid->vmid = vmid_next; 110 vmid_next++; 111 vmid_next &= (1 << vmid_bits) - 1; 112 113 WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version)); 114 115 spin_unlock(&vmid_lock); 116 117 /* Request stage2 page table update for all VCPUs */ 118 kvm_for_each_vcpu(i, v, vcpu->kvm) 119 kvm_make_request(KVM_REQ_UPDATE_HGATP, v); 120 } 121