1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022 Ventana Micro Systems Inc. 4 */ 5 6 #include <linux/bitmap.h> 7 #include <linux/cpumask.h> 8 #include <linux/errno.h> 9 #include <linux/err.h> 10 #include <linux/module.h> 11 #include <linux/smp.h> 12 #include <linux/kvm_host.h> 13 #include <asm/cacheflush.h> 14 #include <asm/csr.h> 15 16 /* 17 * Instruction encoding of hfence.gvma is: 18 * HFENCE.GVMA rs1, rs2 19 * HFENCE.GVMA zero, rs2 20 * HFENCE.GVMA rs1 21 * HFENCE.GVMA 22 * 23 * rs1!=zero and rs2!=zero ==> HFENCE.GVMA rs1, rs2 24 * rs1==zero and rs2!=zero ==> HFENCE.GVMA zero, rs2 25 * rs1!=zero and rs2==zero ==> HFENCE.GVMA rs1 26 * rs1==zero and rs2==zero ==> HFENCE.GVMA 27 * 28 * Instruction encoding of HFENCE.GVMA is: 29 * 0110001 rs2(5) rs1(5) 000 00000 1110011 30 */ 31 32 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, 33 gpa_t gpa, gpa_t gpsz, 34 unsigned long order) 35 { 36 gpa_t pos; 37 38 if (PTRS_PER_PTE < (gpsz >> order)) { 39 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 40 return; 41 } 42 43 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) { 44 /* 45 * rs1 = a0 (GPA >> 2) 46 * rs2 = a1 (VMID) 47 * HFENCE.GVMA a0, a1 48 * 0110001 01011 01010 000 00000 1110011 49 */ 50 asm volatile ("srli a0, %0, 2\n" 51 "add a1, %1, zero\n" 52 ".word 0x62b50073\n" 53 :: "r" (pos), "r" (vmid) 54 : "a0", "a1", "memory"); 55 } 56 } 57 58 void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) 59 { 60 /* 61 * rs1 = zero 62 * rs2 = a0 (VMID) 63 * HFENCE.GVMA zero, a0 64 * 0110001 01010 00000 000 00000 1110011 65 */ 66 asm volatile ("add a0, %0, zero\n" 67 ".word 0x62a00073\n" 68 :: "r" (vmid) : "a0", "memory"); 69 } 70 71 void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, 72 unsigned long order) 73 { 74 gpa_t pos; 75 76 if (PTRS_PER_PTE < (gpsz >> order)) { 77 kvm_riscv_local_hfence_gvma_all(); 78 return; 79 } 80 81 for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) { 82 /* 83 * rs1 = a0 (GPA >> 2) 84 * rs2 = zero 85 * HFENCE.GVMA a0 86 * 0110001 00000 01010 000 00000 1110011 87 */ 88 asm volatile ("srli a0, %0, 2\n" 89 ".word 0x62050073\n" 90 :: "r" (pos) : "a0", "memory"); 91 } 92 } 93 94 void kvm_riscv_local_hfence_gvma_all(void) 95 { 96 /* 97 * rs1 = zero 98 * rs2 = zero 99 * HFENCE.GVMA 100 * 0110001 00000 00000 000 00000 1110011 101 */ 102 asm volatile (".word 0x62000073" ::: "memory"); 103 } 104 105 /* 106 * Instruction encoding of hfence.gvma is: 107 * HFENCE.VVMA rs1, rs2 108 * HFENCE.VVMA zero, rs2 109 * HFENCE.VVMA rs1 110 * HFENCE.VVMA 111 * 112 * rs1!=zero and rs2!=zero ==> HFENCE.VVMA rs1, rs2 113 * rs1==zero and rs2!=zero ==> HFENCE.VVMA zero, rs2 114 * rs1!=zero and rs2==zero ==> HFENCE.VVMA rs1 115 * rs1==zero and rs2==zero ==> HFENCE.VVMA 116 * 117 * Instruction encoding of HFENCE.VVMA is: 118 * 0010001 rs2(5) rs1(5) 000 00000 1110011 119 */ 120 121 void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, 122 unsigned long asid, 123 unsigned long gva, 124 unsigned long gvsz, 125 unsigned long order) 126 { 127 unsigned long pos, hgatp; 128 129 if (PTRS_PER_PTE < (gvsz >> order)) { 130 kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); 131 return; 132 } 133 134 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 135 136 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) { 137 /* 138 * rs1 = a0 (GVA) 139 * rs2 = a1 (ASID) 140 * HFENCE.VVMA a0, a1 141 * 0010001 01011 01010 000 00000 1110011 142 */ 143 asm volatile ("add a0, %0, zero\n" 144 "add a1, %1, zero\n" 145 ".word 0x22b50073\n" 146 :: "r" (pos), "r" (asid) 147 : "a0", "a1", "memory"); 148 } 149 150 csr_write(CSR_HGATP, hgatp); 151 } 152 153 void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, 154 unsigned long asid) 155 { 156 unsigned long hgatp; 157 158 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 159 160 /* 161 * rs1 = zero 162 * rs2 = a0 (ASID) 163 * HFENCE.VVMA zero, a0 164 * 0010001 01010 00000 000 00000 1110011 165 */ 166 asm volatile ("add a0, %0, zero\n" 167 ".word 0x22a00073\n" 168 :: "r" (asid) : "a0", "memory"); 169 170 csr_write(CSR_HGATP, hgatp); 171 } 172 173 void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, 174 unsigned long gva, unsigned long gvsz, 175 unsigned long order) 176 { 177 unsigned long pos, hgatp; 178 179 if (PTRS_PER_PTE < (gvsz >> order)) { 180 kvm_riscv_local_hfence_vvma_all(vmid); 181 return; 182 } 183 184 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 185 186 for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) { 187 /* 188 * rs1 = a0 (GVA) 189 * rs2 = zero 190 * HFENCE.VVMA a0 191 * 0010001 00000 01010 000 00000 1110011 192 */ 193 asm volatile ("add a0, %0, zero\n" 194 ".word 0x22050073\n" 195 :: "r" (pos) : "a0", "memory"); 196 } 197 198 csr_write(CSR_HGATP, hgatp); 199 } 200 201 void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) 202 { 203 unsigned long hgatp; 204 205 hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); 206 207 /* 208 * rs1 = zero 209 * rs2 = zero 210 * HFENCE.VVMA 211 * 0010001 00000 00000 000 00000 1110011 212 */ 213 asm volatile (".word 0x22000073" ::: "memory"); 214 215 csr_write(CSR_HGATP, hgatp); 216 } 217 218 void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) 219 { 220 unsigned long vmid; 221 222 if (!kvm_riscv_gstage_vmid_bits() || 223 vcpu->arch.last_exit_cpu == vcpu->cpu) 224 return; 225 226 /* 227 * On RISC-V platforms with hardware VMID support, we share same 228 * VMID for all VCPUs of a particular Guest/VM. This means we might 229 * have stale G-stage TLB entries on the current Host CPU due to 230 * some other VCPU of the same Guest which ran previously on the 231 * current Host CPU. 232 * 233 * To cleanup stale TLB entries, we simply flush all G-stage TLB 234 * entries by VMID whenever underlying Host CPU changes for a VCPU. 235 */ 236 237 vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); 238 kvm_riscv_local_hfence_gvma_vmid_all(vmid); 239 } 240 241 void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) 242 { 243 local_flush_icache_all(); 244 } 245 246 void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu) 247 { 248 struct kvm_vmid *vmid; 249 250 vmid = &vcpu->kvm->arch.vmid; 251 kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid)); 252 } 253 254 void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu) 255 { 256 struct kvm_vmid *vmid; 257 258 vmid = &vcpu->kvm->arch.vmid; 259 kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid)); 260 } 261 262 static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu, 263 struct kvm_riscv_hfence *out_data) 264 { 265 bool ret = false; 266 struct kvm_vcpu_arch *varch = &vcpu->arch; 267 268 spin_lock(&varch->hfence_lock); 269 270 if (varch->hfence_queue[varch->hfence_head].type) { 271 memcpy(out_data, &varch->hfence_queue[varch->hfence_head], 272 sizeof(*out_data)); 273 varch->hfence_queue[varch->hfence_head].type = 0; 274 275 varch->hfence_head++; 276 if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE) 277 varch->hfence_head = 0; 278 279 ret = true; 280 } 281 282 spin_unlock(&varch->hfence_lock); 283 284 return ret; 285 } 286 287 static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu, 288 const struct kvm_riscv_hfence *data) 289 { 290 bool ret = false; 291 struct kvm_vcpu_arch *varch = &vcpu->arch; 292 293 spin_lock(&varch->hfence_lock); 294 295 if (!varch->hfence_queue[varch->hfence_tail].type) { 296 memcpy(&varch->hfence_queue[varch->hfence_tail], 297 data, sizeof(*data)); 298 299 varch->hfence_tail++; 300 if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE) 301 varch->hfence_tail = 0; 302 303 ret = true; 304 } 305 306 spin_unlock(&varch->hfence_lock); 307 308 return ret; 309 } 310 311 void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu) 312 { 313 struct kvm_riscv_hfence d = { 0 }; 314 struct kvm_vmid *v = &vcpu->kvm->arch.vmid; 315 316 while (vcpu_hfence_dequeue(vcpu, &d)) { 317 switch (d.type) { 318 case KVM_RISCV_HFENCE_UNKNOWN: 319 break; 320 case KVM_RISCV_HFENCE_GVMA_VMID_GPA: 321 kvm_riscv_local_hfence_gvma_vmid_gpa( 322 READ_ONCE(v->vmid), 323 d.addr, d.size, d.order); 324 break; 325 case KVM_RISCV_HFENCE_VVMA_ASID_GVA: 326 kvm_riscv_local_hfence_vvma_asid_gva( 327 READ_ONCE(v->vmid), d.asid, 328 d.addr, d.size, d.order); 329 break; 330 case KVM_RISCV_HFENCE_VVMA_ASID_ALL: 331 kvm_riscv_local_hfence_vvma_asid_all( 332 READ_ONCE(v->vmid), d.asid); 333 break; 334 case KVM_RISCV_HFENCE_VVMA_GVA: 335 kvm_riscv_local_hfence_vvma_gva( 336 READ_ONCE(v->vmid), 337 d.addr, d.size, d.order); 338 break; 339 default: 340 break; 341 } 342 } 343 } 344 345 static void make_xfence_request(struct kvm *kvm, 346 unsigned long hbase, unsigned long hmask, 347 unsigned int req, unsigned int fallback_req, 348 const struct kvm_riscv_hfence *data) 349 { 350 unsigned long i; 351 struct kvm_vcpu *vcpu; 352 unsigned int actual_req = req; 353 DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); 354 355 bitmap_clear(vcpu_mask, 0, KVM_MAX_VCPUS); 356 kvm_for_each_vcpu(i, vcpu, kvm) { 357 if (hbase != -1UL) { 358 if (vcpu->vcpu_id < hbase) 359 continue; 360 if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) 361 continue; 362 } 363 364 bitmap_set(vcpu_mask, i, 1); 365 366 if (!data || !data->type) 367 continue; 368 369 /* 370 * Enqueue hfence data to VCPU hfence queue. If we don't 371 * have space in the VCPU hfence queue then fallback to 372 * a more conservative hfence request. 373 */ 374 if (!vcpu_hfence_enqueue(vcpu, data)) 375 actual_req = fallback_req; 376 } 377 378 kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask); 379 } 380 381 void kvm_riscv_fence_i(struct kvm *kvm, 382 unsigned long hbase, unsigned long hmask) 383 { 384 make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I, 385 KVM_REQ_FENCE_I, NULL); 386 } 387 388 void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, 389 unsigned long hbase, unsigned long hmask, 390 gpa_t gpa, gpa_t gpsz, 391 unsigned long order) 392 { 393 struct kvm_riscv_hfence data; 394 395 data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA; 396 data.asid = 0; 397 data.addr = gpa; 398 data.size = gpsz; 399 data.order = order; 400 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 401 KVM_REQ_HFENCE_GVMA_VMID_ALL, &data); 402 } 403 404 void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, 405 unsigned long hbase, unsigned long hmask) 406 { 407 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL, 408 KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL); 409 } 410 411 void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, 412 unsigned long hbase, unsigned long hmask, 413 unsigned long gva, unsigned long gvsz, 414 unsigned long order, unsigned long asid) 415 { 416 struct kvm_riscv_hfence data; 417 418 data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA; 419 data.asid = asid; 420 data.addr = gva; 421 data.size = gvsz; 422 data.order = order; 423 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 424 KVM_REQ_HFENCE_VVMA_ALL, &data); 425 } 426 427 void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, 428 unsigned long hbase, unsigned long hmask, 429 unsigned long asid) 430 { 431 struct kvm_riscv_hfence data; 432 433 data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL; 434 data.asid = asid; 435 data.addr = data.size = data.order = 0; 436 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 437 KVM_REQ_HFENCE_VVMA_ALL, &data); 438 } 439 440 void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, 441 unsigned long hbase, unsigned long hmask, 442 unsigned long gva, unsigned long gvsz, 443 unsigned long order) 444 { 445 struct kvm_riscv_hfence data; 446 447 data.type = KVM_RISCV_HFENCE_VVMA_GVA; 448 data.asid = 0; 449 data.addr = gva; 450 data.size = gvsz; 451 data.order = order; 452 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, 453 KVM_REQ_HFENCE_VVMA_ALL, &data); 454 } 455 456 void kvm_riscv_hfence_vvma_all(struct kvm *kvm, 457 unsigned long hbase, unsigned long hmask) 458 { 459 make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL, 460 KVM_REQ_HFENCE_VVMA_ALL, NULL); 461 } 462