1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/entry-kvm.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/kdebug.h> 14 #include <linux/module.h> 15 #include <linux/percpu.h> 16 #include <linux/uaccess.h> 17 #include <linux/vmalloc.h> 18 #include <linux/sched/signal.h> 19 #include <linux/fs.h> 20 #include <linux/kvm_host.h> 21 #include <asm/csr.h> 22 #include <asm/cacheflush.h> 23 #include <asm/hwcap.h> 24 #include <asm/sbi.h> 25 #include <asm/vector.h> 26 #include <asm/kvm_vcpu_vector.h> 27 28 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 29 KVM_GENERIC_VCPU_STATS(), 30 STATS_DESC_COUNTER(VCPU, ecall_exit_stat), 31 STATS_DESC_COUNTER(VCPU, wfi_exit_stat), 32 STATS_DESC_COUNTER(VCPU, mmio_exit_user), 33 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), 34 STATS_DESC_COUNTER(VCPU, csr_exit_user), 35 STATS_DESC_COUNTER(VCPU, csr_exit_kernel), 36 STATS_DESC_COUNTER(VCPU, signal_exits), 37 STATS_DESC_COUNTER(VCPU, exits) 38 }; 39 40 const struct kvm_stats_header kvm_vcpu_stats_header = { 41 .name_size = KVM_STATS_NAME_SIZE, 42 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 43 .id_offset = sizeof(struct kvm_stats_header), 44 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 45 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 46 sizeof(kvm_vcpu_stats_desc), 47 }; 48 49 #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) 50 51 #define KVM_ISA_EXT_ARR(ext) [KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext 52 53 /* Mapping between KVM ISA Extension ID & Host ISA extension ID */ 54 static const unsigned long kvm_isa_ext_arr[] = { 55 [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, 56 [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, 57 [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, 58 [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, 59 [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, 60 [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, 61 [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, 62 [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, 63 64 KVM_ISA_EXT_ARR(SSAIA), 65 KVM_ISA_EXT_ARR(SSTC), 66 KVM_ISA_EXT_ARR(SVINVAL), 67 KVM_ISA_EXT_ARR(SVNAPOT), 68 KVM_ISA_EXT_ARR(SVPBMT), 69 KVM_ISA_EXT_ARR(ZBB), 70 KVM_ISA_EXT_ARR(ZIHINTPAUSE), 71 KVM_ISA_EXT_ARR(ZICBOM), 72 KVM_ISA_EXT_ARR(ZICBOZ), 73 }; 74 75 static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext) 76 { 77 unsigned long i; 78 79 for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 80 if (kvm_isa_ext_arr[i] == base_ext) 81 return i; 82 } 83 84 return KVM_RISCV_ISA_EXT_MAX; 85 } 86 87 static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext) 88 { 89 switch (ext) { 90 case KVM_RISCV_ISA_EXT_H: 91 return false; 92 case KVM_RISCV_ISA_EXT_V: 93 return riscv_v_vstate_ctrl_user_allowed(); 94 default: 95 break; 96 } 97 98 return true; 99 } 100 101 static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) 102 { 103 switch (ext) { 104 case KVM_RISCV_ISA_EXT_A: 105 case KVM_RISCV_ISA_EXT_C: 106 case KVM_RISCV_ISA_EXT_I: 107 case KVM_RISCV_ISA_EXT_M: 108 case KVM_RISCV_ISA_EXT_SSAIA: 109 case KVM_RISCV_ISA_EXT_SSTC: 110 case KVM_RISCV_ISA_EXT_SVINVAL: 111 case KVM_RISCV_ISA_EXT_SVNAPOT: 112 case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: 113 case KVM_RISCV_ISA_EXT_ZBB: 114 return false; 115 default: 116 break; 117 } 118 119 return true; 120 } 121 122 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) 123 { 124 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 125 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 126 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 127 struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context; 128 bool loaded; 129 130 /** 131 * The preemption should be disabled here because it races with 132 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which 133 * also calls vcpu_load/put. 134 */ 135 get_cpu(); 136 loaded = (vcpu->cpu != -1); 137 if (loaded) 138 kvm_arch_vcpu_put(vcpu); 139 140 vcpu->arch.last_exit_cpu = -1; 141 142 memcpy(csr, reset_csr, sizeof(*csr)); 143 144 memcpy(cntx, reset_cntx, sizeof(*cntx)); 145 146 kvm_riscv_vcpu_fp_reset(vcpu); 147 148 kvm_riscv_vcpu_vector_reset(vcpu); 149 150 kvm_riscv_vcpu_timer_reset(vcpu); 151 152 kvm_riscv_vcpu_aia_reset(vcpu); 153 154 bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); 155 bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); 156 157 kvm_riscv_vcpu_pmu_reset(vcpu); 158 159 vcpu->arch.hfence_head = 0; 160 vcpu->arch.hfence_tail = 0; 161 memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); 162 163 /* Reset the guest CSRs for hotplug usecase */ 164 if (loaded) 165 kvm_arch_vcpu_load(vcpu, smp_processor_id()); 166 put_cpu(); 167 } 168 169 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 170 { 171 return 0; 172 } 173 174 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 175 { 176 int rc; 177 struct kvm_cpu_context *cntx; 178 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 179 unsigned long host_isa, i; 180 181 /* Mark this VCPU never ran */ 182 vcpu->arch.ran_atleast_once = false; 183 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 184 bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); 185 186 /* Setup ISA features available to VCPU */ 187 for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) { 188 host_isa = kvm_isa_ext_arr[i]; 189 if (__riscv_isa_extension_available(NULL, host_isa) && 190 kvm_riscv_vcpu_isa_enable_allowed(i)) 191 set_bit(host_isa, vcpu->arch.isa); 192 } 193 194 /* Setup vendor, arch, and implementation details */ 195 vcpu->arch.mvendorid = sbi_get_mvendorid(); 196 vcpu->arch.marchid = sbi_get_marchid(); 197 vcpu->arch.mimpid = sbi_get_mimpid(); 198 199 /* Setup VCPU hfence queue */ 200 spin_lock_init(&vcpu->arch.hfence_lock); 201 202 /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ 203 cntx = &vcpu->arch.guest_reset_context; 204 cntx->sstatus = SR_SPP | SR_SPIE; 205 cntx->hstatus = 0; 206 cntx->hstatus |= HSTATUS_VTW; 207 cntx->hstatus |= HSTATUS_SPVP; 208 cntx->hstatus |= HSTATUS_SPV; 209 210 if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) 211 return -ENOMEM; 212 213 /* By default, make CY, TM, and IR counters accessible in VU mode */ 214 reset_csr->scounteren = 0x7; 215 216 /* Setup VCPU timer */ 217 kvm_riscv_vcpu_timer_init(vcpu); 218 219 /* setup performance monitoring */ 220 kvm_riscv_vcpu_pmu_init(vcpu); 221 222 /* Setup VCPU AIA */ 223 rc = kvm_riscv_vcpu_aia_init(vcpu); 224 if (rc) 225 return rc; 226 227 /* Reset VCPU */ 228 kvm_riscv_reset_vcpu(vcpu); 229 230 return 0; 231 } 232 233 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 234 { 235 /** 236 * vcpu with id 0 is the designated boot cpu. 237 * Keep all vcpus with non-zero id in power-off state so that 238 * they can be brought up using SBI HSM extension. 239 */ 240 if (vcpu->vcpu_idx != 0) 241 kvm_riscv_vcpu_power_off(vcpu); 242 } 243 244 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 245 { 246 /* Cleanup VCPU AIA context */ 247 kvm_riscv_vcpu_aia_deinit(vcpu); 248 249 /* Cleanup VCPU timer */ 250 kvm_riscv_vcpu_timer_deinit(vcpu); 251 252 kvm_riscv_vcpu_pmu_deinit(vcpu); 253 254 /* Free unused pages pre-allocated for G-stage page table mappings */ 255 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 256 257 /* Free vector context space for host and guest kernel */ 258 kvm_riscv_vcpu_free_vector_context(vcpu); 259 } 260 261 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 262 { 263 return kvm_riscv_vcpu_timer_pending(vcpu); 264 } 265 266 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 267 { 268 kvm_riscv_aia_wakeon_hgei(vcpu, true); 269 } 270 271 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 272 { 273 kvm_riscv_aia_wakeon_hgei(vcpu, false); 274 } 275 276 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 277 { 278 return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) && 279 !vcpu->arch.power_off && !vcpu->arch.pause); 280 } 281 282 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 283 { 284 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 285 } 286 287 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 288 { 289 return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false; 290 } 291 292 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 293 { 294 return VM_FAULT_SIGBUS; 295 } 296 297 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, 298 const struct kvm_one_reg *reg) 299 { 300 unsigned long __user *uaddr = 301 (unsigned long __user *)(unsigned long)reg->addr; 302 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 303 KVM_REG_SIZE_MASK | 304 KVM_REG_RISCV_CONFIG); 305 unsigned long reg_val; 306 307 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 308 return -EINVAL; 309 310 switch (reg_num) { 311 case KVM_REG_RISCV_CONFIG_REG(isa): 312 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; 313 break; 314 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 315 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 316 return -EINVAL; 317 reg_val = riscv_cbom_block_size; 318 break; 319 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 320 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 321 return -EINVAL; 322 reg_val = riscv_cboz_block_size; 323 break; 324 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 325 reg_val = vcpu->arch.mvendorid; 326 break; 327 case KVM_REG_RISCV_CONFIG_REG(marchid): 328 reg_val = vcpu->arch.marchid; 329 break; 330 case KVM_REG_RISCV_CONFIG_REG(mimpid): 331 reg_val = vcpu->arch.mimpid; 332 break; 333 default: 334 return -EINVAL; 335 } 336 337 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 338 return -EFAULT; 339 340 return 0; 341 } 342 343 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, 344 const struct kvm_one_reg *reg) 345 { 346 unsigned long __user *uaddr = 347 (unsigned long __user *)(unsigned long)reg->addr; 348 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 349 KVM_REG_SIZE_MASK | 350 KVM_REG_RISCV_CONFIG); 351 unsigned long i, isa_ext, reg_val; 352 353 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 354 return -EINVAL; 355 356 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 357 return -EFAULT; 358 359 switch (reg_num) { 360 case KVM_REG_RISCV_CONFIG_REG(isa): 361 /* 362 * This ONE REG interface is only defined for 363 * single letter extensions. 364 */ 365 if (fls(reg_val) >= RISCV_ISA_EXT_BASE) 366 return -EINVAL; 367 368 if (!vcpu->arch.ran_atleast_once) { 369 /* Ignore the enable/disable request for certain extensions */ 370 for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { 371 isa_ext = kvm_riscv_vcpu_base2isa_ext(i); 372 if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { 373 reg_val &= ~BIT(i); 374 continue; 375 } 376 if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext)) 377 if (reg_val & BIT(i)) 378 reg_val &= ~BIT(i); 379 if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext)) 380 if (!(reg_val & BIT(i))) 381 reg_val |= BIT(i); 382 } 383 reg_val &= riscv_isa_extension_base(NULL); 384 /* Do not modify anything beyond single letter extensions */ 385 reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) | 386 (reg_val & KVM_RISCV_BASE_ISA_MASK); 387 vcpu->arch.isa[0] = reg_val; 388 kvm_riscv_vcpu_fp_reset(vcpu); 389 } else { 390 return -EOPNOTSUPP; 391 } 392 break; 393 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 394 return -EOPNOTSUPP; 395 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 396 return -EOPNOTSUPP; 397 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 398 if (!vcpu->arch.ran_atleast_once) 399 vcpu->arch.mvendorid = reg_val; 400 else 401 return -EBUSY; 402 break; 403 case KVM_REG_RISCV_CONFIG_REG(marchid): 404 if (!vcpu->arch.ran_atleast_once) 405 vcpu->arch.marchid = reg_val; 406 else 407 return -EBUSY; 408 break; 409 case KVM_REG_RISCV_CONFIG_REG(mimpid): 410 if (!vcpu->arch.ran_atleast_once) 411 vcpu->arch.mimpid = reg_val; 412 else 413 return -EBUSY; 414 break; 415 default: 416 return -EINVAL; 417 } 418 419 return 0; 420 } 421 422 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, 423 const struct kvm_one_reg *reg) 424 { 425 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 426 unsigned long __user *uaddr = 427 (unsigned long __user *)(unsigned long)reg->addr; 428 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 429 KVM_REG_SIZE_MASK | 430 KVM_REG_RISCV_CORE); 431 unsigned long reg_val; 432 433 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 434 return -EINVAL; 435 if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) 436 return -EINVAL; 437 438 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 439 reg_val = cntx->sepc; 440 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 441 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 442 reg_val = ((unsigned long *)cntx)[reg_num]; 443 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) 444 reg_val = (cntx->sstatus & SR_SPP) ? 445 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U; 446 else 447 return -EINVAL; 448 449 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 450 return -EFAULT; 451 452 return 0; 453 } 454 455 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, 456 const struct kvm_one_reg *reg) 457 { 458 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 459 unsigned long __user *uaddr = 460 (unsigned long __user *)(unsigned long)reg->addr; 461 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 462 KVM_REG_SIZE_MASK | 463 KVM_REG_RISCV_CORE); 464 unsigned long reg_val; 465 466 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 467 return -EINVAL; 468 if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) 469 return -EINVAL; 470 471 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 472 return -EFAULT; 473 474 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 475 cntx->sepc = reg_val; 476 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 477 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 478 ((unsigned long *)cntx)[reg_num] = reg_val; 479 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) { 480 if (reg_val == KVM_RISCV_MODE_S) 481 cntx->sstatus |= SR_SPP; 482 else 483 cntx->sstatus &= ~SR_SPP; 484 } else 485 return -EINVAL; 486 487 return 0; 488 } 489 490 static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, 491 unsigned long reg_num, 492 unsigned long *out_val) 493 { 494 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 495 496 if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) 497 return -EINVAL; 498 499 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 500 kvm_riscv_vcpu_flush_interrupts(vcpu); 501 *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; 502 *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; 503 } else 504 *out_val = ((unsigned long *)csr)[reg_num]; 505 506 return 0; 507 } 508 509 static inline int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, 510 unsigned long reg_num, 511 unsigned long reg_val) 512 { 513 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 514 515 if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) 516 return -EINVAL; 517 518 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 519 reg_val &= VSIP_VALID_MASK; 520 reg_val <<= VSIP_TO_HVIP_SHIFT; 521 } 522 523 ((unsigned long *)csr)[reg_num] = reg_val; 524 525 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) 526 WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); 527 528 return 0; 529 } 530 531 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, 532 const struct kvm_one_reg *reg) 533 { 534 int rc; 535 unsigned long __user *uaddr = 536 (unsigned long __user *)(unsigned long)reg->addr; 537 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 538 KVM_REG_SIZE_MASK | 539 KVM_REG_RISCV_CSR); 540 unsigned long reg_val, reg_subtype; 541 542 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 543 return -EINVAL; 544 545 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 546 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 547 switch (reg_subtype) { 548 case KVM_REG_RISCV_CSR_GENERAL: 549 rc = kvm_riscv_vcpu_general_get_csr(vcpu, reg_num, ®_val); 550 break; 551 case KVM_REG_RISCV_CSR_AIA: 552 rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); 553 break; 554 default: 555 rc = -EINVAL; 556 break; 557 } 558 if (rc) 559 return rc; 560 561 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 562 return -EFAULT; 563 564 return 0; 565 } 566 567 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, 568 const struct kvm_one_reg *reg) 569 { 570 int rc; 571 unsigned long __user *uaddr = 572 (unsigned long __user *)(unsigned long)reg->addr; 573 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 574 KVM_REG_SIZE_MASK | 575 KVM_REG_RISCV_CSR); 576 unsigned long reg_val, reg_subtype; 577 578 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 579 return -EINVAL; 580 581 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 582 return -EFAULT; 583 584 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 585 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 586 switch (reg_subtype) { 587 case KVM_REG_RISCV_CSR_GENERAL: 588 rc = kvm_riscv_vcpu_general_set_csr(vcpu, reg_num, reg_val); 589 break; 590 case KVM_REG_RISCV_CSR_AIA: 591 rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); 592 break; 593 default: 594 rc = -EINVAL; 595 break; 596 } 597 if (rc) 598 return rc; 599 600 return 0; 601 } 602 603 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu, 604 const struct kvm_one_reg *reg) 605 { 606 unsigned long __user *uaddr = 607 (unsigned long __user *)(unsigned long)reg->addr; 608 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 609 KVM_REG_SIZE_MASK | 610 KVM_REG_RISCV_ISA_EXT); 611 unsigned long reg_val = 0; 612 unsigned long host_isa_ext; 613 614 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 615 return -EINVAL; 616 617 if (reg_num >= KVM_RISCV_ISA_EXT_MAX || 618 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) 619 return -EINVAL; 620 621 host_isa_ext = kvm_isa_ext_arr[reg_num]; 622 if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext)) 623 reg_val = 1; /* Mark the given extension as available */ 624 625 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 626 return -EFAULT; 627 628 return 0; 629 } 630 631 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu, 632 const struct kvm_one_reg *reg) 633 { 634 unsigned long __user *uaddr = 635 (unsigned long __user *)(unsigned long)reg->addr; 636 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 637 KVM_REG_SIZE_MASK | 638 KVM_REG_RISCV_ISA_EXT); 639 unsigned long reg_val; 640 unsigned long host_isa_ext; 641 642 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 643 return -EINVAL; 644 645 if (reg_num >= KVM_RISCV_ISA_EXT_MAX || 646 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) 647 return -EINVAL; 648 649 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 650 return -EFAULT; 651 652 host_isa_ext = kvm_isa_ext_arr[reg_num]; 653 if (!__riscv_isa_extension_available(NULL, host_isa_ext)) 654 return -EOPNOTSUPP; 655 656 if (!vcpu->arch.ran_atleast_once) { 657 /* 658 * All multi-letter extension and a few single letter 659 * extension can be disabled 660 */ 661 if (reg_val == 1 && 662 kvm_riscv_vcpu_isa_enable_allowed(reg_num)) 663 set_bit(host_isa_ext, vcpu->arch.isa); 664 else if (!reg_val && 665 kvm_riscv_vcpu_isa_disable_allowed(reg_num)) 666 clear_bit(host_isa_ext, vcpu->arch.isa); 667 else 668 return -EINVAL; 669 kvm_riscv_vcpu_fp_reset(vcpu); 670 } else { 671 return -EOPNOTSUPP; 672 } 673 674 return 0; 675 } 676 677 static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, 678 const struct kvm_one_reg *reg) 679 { 680 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 681 case KVM_REG_RISCV_CONFIG: 682 return kvm_riscv_vcpu_set_reg_config(vcpu, reg); 683 case KVM_REG_RISCV_CORE: 684 return kvm_riscv_vcpu_set_reg_core(vcpu, reg); 685 case KVM_REG_RISCV_CSR: 686 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg); 687 case KVM_REG_RISCV_TIMER: 688 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg); 689 case KVM_REG_RISCV_FP_F: 690 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 691 KVM_REG_RISCV_FP_F); 692 case KVM_REG_RISCV_FP_D: 693 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 694 KVM_REG_RISCV_FP_D); 695 case KVM_REG_RISCV_ISA_EXT: 696 return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); 697 case KVM_REG_RISCV_SBI_EXT: 698 return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); 699 case KVM_REG_RISCV_VECTOR: 700 return kvm_riscv_vcpu_set_reg_vector(vcpu, reg, 701 KVM_REG_RISCV_VECTOR); 702 default: 703 break; 704 } 705 706 return -EINVAL; 707 } 708 709 static int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, 710 const struct kvm_one_reg *reg) 711 { 712 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 713 case KVM_REG_RISCV_CONFIG: 714 return kvm_riscv_vcpu_get_reg_config(vcpu, reg); 715 case KVM_REG_RISCV_CORE: 716 return kvm_riscv_vcpu_get_reg_core(vcpu, reg); 717 case KVM_REG_RISCV_CSR: 718 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg); 719 case KVM_REG_RISCV_TIMER: 720 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg); 721 case KVM_REG_RISCV_FP_F: 722 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 723 KVM_REG_RISCV_FP_F); 724 case KVM_REG_RISCV_FP_D: 725 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 726 KVM_REG_RISCV_FP_D); 727 case KVM_REG_RISCV_ISA_EXT: 728 return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); 729 case KVM_REG_RISCV_SBI_EXT: 730 return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); 731 case KVM_REG_RISCV_VECTOR: 732 return kvm_riscv_vcpu_get_reg_vector(vcpu, reg, 733 KVM_REG_RISCV_VECTOR); 734 default: 735 break; 736 } 737 738 return -EINVAL; 739 } 740 741 long kvm_arch_vcpu_async_ioctl(struct file *filp, 742 unsigned int ioctl, unsigned long arg) 743 { 744 struct kvm_vcpu *vcpu = filp->private_data; 745 void __user *argp = (void __user *)arg; 746 747 if (ioctl == KVM_INTERRUPT) { 748 struct kvm_interrupt irq; 749 750 if (copy_from_user(&irq, argp, sizeof(irq))) 751 return -EFAULT; 752 753 if (irq.irq == KVM_INTERRUPT_SET) 754 return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); 755 else 756 return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 757 } 758 759 return -ENOIOCTLCMD; 760 } 761 762 long kvm_arch_vcpu_ioctl(struct file *filp, 763 unsigned int ioctl, unsigned long arg) 764 { 765 struct kvm_vcpu *vcpu = filp->private_data; 766 void __user *argp = (void __user *)arg; 767 long r = -EINVAL; 768 769 switch (ioctl) { 770 case KVM_SET_ONE_REG: 771 case KVM_GET_ONE_REG: { 772 struct kvm_one_reg reg; 773 774 r = -EFAULT; 775 if (copy_from_user(®, argp, sizeof(reg))) 776 break; 777 778 if (ioctl == KVM_SET_ONE_REG) 779 r = kvm_riscv_vcpu_set_reg(vcpu, ®); 780 else 781 r = kvm_riscv_vcpu_get_reg(vcpu, ®); 782 break; 783 } 784 default: 785 break; 786 } 787 788 return r; 789 } 790 791 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 792 struct kvm_sregs *sregs) 793 { 794 return -EINVAL; 795 } 796 797 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 798 struct kvm_sregs *sregs) 799 { 800 return -EINVAL; 801 } 802 803 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 804 { 805 return -EINVAL; 806 } 807 808 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 809 { 810 return -EINVAL; 811 } 812 813 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 814 struct kvm_translation *tr) 815 { 816 return -EINVAL; 817 } 818 819 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 820 { 821 return -EINVAL; 822 } 823 824 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 825 { 826 return -EINVAL; 827 } 828 829 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) 830 { 831 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 832 unsigned long mask, val; 833 834 if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { 835 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); 836 val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; 837 838 csr->hvip &= ~mask; 839 csr->hvip |= val; 840 } 841 842 /* Flush AIA high interrupts */ 843 kvm_riscv_vcpu_aia_flush_interrupts(vcpu); 844 } 845 846 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) 847 { 848 unsigned long hvip; 849 struct kvm_vcpu_arch *v = &vcpu->arch; 850 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 851 852 /* Read current HVIP and VSIE CSRs */ 853 csr->vsie = csr_read(CSR_VSIE); 854 855 /* Sync-up HVIP.VSSIP bit changes does by Guest */ 856 hvip = csr_read(CSR_HVIP); 857 if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { 858 if (hvip & (1UL << IRQ_VS_SOFT)) { 859 if (!test_and_set_bit(IRQ_VS_SOFT, 860 v->irqs_pending_mask)) 861 set_bit(IRQ_VS_SOFT, v->irqs_pending); 862 } else { 863 if (!test_and_set_bit(IRQ_VS_SOFT, 864 v->irqs_pending_mask)) 865 clear_bit(IRQ_VS_SOFT, v->irqs_pending); 866 } 867 } 868 869 /* Sync-up AIA high interrupts */ 870 kvm_riscv_vcpu_aia_sync_interrupts(vcpu); 871 872 /* Sync-up timer CSRs */ 873 kvm_riscv_vcpu_timer_sync(vcpu); 874 } 875 876 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 877 { 878 /* 879 * We only allow VS-mode software, timer, and external 880 * interrupts when irq is one of the local interrupts 881 * defined by RISC-V privilege specification. 882 */ 883 if (irq < IRQ_LOCAL_MAX && 884 irq != IRQ_VS_SOFT && 885 irq != IRQ_VS_TIMER && 886 irq != IRQ_VS_EXT) 887 return -EINVAL; 888 889 set_bit(irq, vcpu->arch.irqs_pending); 890 smp_mb__before_atomic(); 891 set_bit(irq, vcpu->arch.irqs_pending_mask); 892 893 kvm_vcpu_kick(vcpu); 894 895 return 0; 896 } 897 898 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 899 { 900 /* 901 * We only allow VS-mode software, timer, and external 902 * interrupts when irq is one of the local interrupts 903 * defined by RISC-V privilege specification. 904 */ 905 if (irq < IRQ_LOCAL_MAX && 906 irq != IRQ_VS_SOFT && 907 irq != IRQ_VS_TIMER && 908 irq != IRQ_VS_EXT) 909 return -EINVAL; 910 911 clear_bit(irq, vcpu->arch.irqs_pending); 912 smp_mb__before_atomic(); 913 set_bit(irq, vcpu->arch.irqs_pending_mask); 914 915 return 0; 916 } 917 918 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 919 { 920 unsigned long ie; 921 922 ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) 923 << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; 924 ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & 925 (unsigned long)mask; 926 if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) 927 return true; 928 929 /* Check AIA high interrupts */ 930 return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); 931 } 932 933 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) 934 { 935 vcpu->arch.power_off = true; 936 kvm_make_request(KVM_REQ_SLEEP, vcpu); 937 kvm_vcpu_kick(vcpu); 938 } 939 940 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) 941 { 942 vcpu->arch.power_off = false; 943 kvm_vcpu_wake_up(vcpu); 944 } 945 946 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 947 struct kvm_mp_state *mp_state) 948 { 949 if (vcpu->arch.power_off) 950 mp_state->mp_state = KVM_MP_STATE_STOPPED; 951 else 952 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 953 954 return 0; 955 } 956 957 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 958 struct kvm_mp_state *mp_state) 959 { 960 int ret = 0; 961 962 switch (mp_state->mp_state) { 963 case KVM_MP_STATE_RUNNABLE: 964 vcpu->arch.power_off = false; 965 break; 966 case KVM_MP_STATE_STOPPED: 967 kvm_riscv_vcpu_power_off(vcpu); 968 break; 969 default: 970 ret = -EINVAL; 971 } 972 973 return ret; 974 } 975 976 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 977 struct kvm_guest_debug *dbg) 978 { 979 /* TODO; To be implemented later. */ 980 return -EINVAL; 981 } 982 983 static void kvm_riscv_vcpu_update_config(const unsigned long *isa) 984 { 985 u64 henvcfg = 0; 986 987 if (riscv_isa_extension_available(isa, SVPBMT)) 988 henvcfg |= ENVCFG_PBMTE; 989 990 if (riscv_isa_extension_available(isa, SSTC)) 991 henvcfg |= ENVCFG_STCE; 992 993 if (riscv_isa_extension_available(isa, ZICBOM)) 994 henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); 995 996 if (riscv_isa_extension_available(isa, ZICBOZ)) 997 henvcfg |= ENVCFG_CBZE; 998 999 csr_write(CSR_HENVCFG, henvcfg); 1000 #ifdef CONFIG_32BIT 1001 csr_write(CSR_HENVCFGH, henvcfg >> 32); 1002 #endif 1003 } 1004 1005 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1006 { 1007 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 1008 1009 csr_write(CSR_VSSTATUS, csr->vsstatus); 1010 csr_write(CSR_VSIE, csr->vsie); 1011 csr_write(CSR_VSTVEC, csr->vstvec); 1012 csr_write(CSR_VSSCRATCH, csr->vsscratch); 1013 csr_write(CSR_VSEPC, csr->vsepc); 1014 csr_write(CSR_VSCAUSE, csr->vscause); 1015 csr_write(CSR_VSTVAL, csr->vstval); 1016 csr_write(CSR_HVIP, csr->hvip); 1017 csr_write(CSR_VSATP, csr->vsatp); 1018 1019 kvm_riscv_vcpu_update_config(vcpu->arch.isa); 1020 1021 kvm_riscv_gstage_update_hgatp(vcpu); 1022 1023 kvm_riscv_vcpu_timer_restore(vcpu); 1024 1025 kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); 1026 kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, 1027 vcpu->arch.isa); 1028 kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); 1029 kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, 1030 vcpu->arch.isa); 1031 1032 kvm_riscv_vcpu_aia_load(vcpu, cpu); 1033 1034 vcpu->cpu = cpu; 1035 } 1036 1037 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 1038 { 1039 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 1040 1041 vcpu->cpu = -1; 1042 1043 kvm_riscv_vcpu_aia_put(vcpu); 1044 1045 kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context, 1046 vcpu->arch.isa); 1047 kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); 1048 1049 kvm_riscv_vcpu_timer_save(vcpu); 1050 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, 1051 vcpu->arch.isa); 1052 kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); 1053 1054 csr->vsstatus = csr_read(CSR_VSSTATUS); 1055 csr->vsie = csr_read(CSR_VSIE); 1056 csr->vstvec = csr_read(CSR_VSTVEC); 1057 csr->vsscratch = csr_read(CSR_VSSCRATCH); 1058 csr->vsepc = csr_read(CSR_VSEPC); 1059 csr->vscause = csr_read(CSR_VSCAUSE); 1060 csr->vstval = csr_read(CSR_VSTVAL); 1061 csr->hvip = csr_read(CSR_HVIP); 1062 csr->vsatp = csr_read(CSR_VSATP); 1063 } 1064 1065 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) 1066 { 1067 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 1068 1069 if (kvm_request_pending(vcpu)) { 1070 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) { 1071 kvm_vcpu_srcu_read_unlock(vcpu); 1072 rcuwait_wait_event(wait, 1073 (!vcpu->arch.power_off) && (!vcpu->arch.pause), 1074 TASK_INTERRUPTIBLE); 1075 kvm_vcpu_srcu_read_lock(vcpu); 1076 1077 if (vcpu->arch.power_off || vcpu->arch.pause) { 1078 /* 1079 * Awaken to handle a signal, request to 1080 * sleep again later. 1081 */ 1082 kvm_make_request(KVM_REQ_SLEEP, vcpu); 1083 } 1084 } 1085 1086 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 1087 kvm_riscv_reset_vcpu(vcpu); 1088 1089 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) 1090 kvm_riscv_gstage_update_hgatp(vcpu); 1091 1092 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) 1093 kvm_riscv_fence_i_process(vcpu); 1094 1095 /* 1096 * The generic KVM_REQ_TLB_FLUSH is same as 1097 * KVM_REQ_HFENCE_GVMA_VMID_ALL 1098 */ 1099 if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) 1100 kvm_riscv_hfence_gvma_vmid_all_process(vcpu); 1101 1102 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) 1103 kvm_riscv_hfence_vvma_all_process(vcpu); 1104 1105 if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) 1106 kvm_riscv_hfence_process(vcpu); 1107 } 1108 } 1109 1110 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) 1111 { 1112 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 1113 1114 csr_write(CSR_HVIP, csr->hvip); 1115 kvm_riscv_vcpu_aia_update_hvip(vcpu); 1116 } 1117 1118 /* 1119 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while 1120 * the vCPU is running. 1121 * 1122 * This must be noinstr as instrumentation may make use of RCU, and this is not 1123 * safe during the EQS. 1124 */ 1125 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu) 1126 { 1127 guest_state_enter_irqoff(); 1128 __kvm_riscv_switch_to(&vcpu->arch); 1129 vcpu->arch.last_exit_cpu = vcpu->cpu; 1130 guest_state_exit_irqoff(); 1131 } 1132 1133 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1134 { 1135 int ret; 1136 struct kvm_cpu_trap trap; 1137 struct kvm_run *run = vcpu->run; 1138 1139 /* Mark this VCPU ran at least once */ 1140 vcpu->arch.ran_atleast_once = true; 1141 1142 kvm_vcpu_srcu_read_lock(vcpu); 1143 1144 switch (run->exit_reason) { 1145 case KVM_EXIT_MMIO: 1146 /* Process MMIO value returned from user-space */ 1147 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run); 1148 break; 1149 case KVM_EXIT_RISCV_SBI: 1150 /* Process SBI value returned from user-space */ 1151 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run); 1152 break; 1153 case KVM_EXIT_RISCV_CSR: 1154 /* Process CSR value returned from user-space */ 1155 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run); 1156 break; 1157 default: 1158 ret = 0; 1159 break; 1160 } 1161 if (ret) { 1162 kvm_vcpu_srcu_read_unlock(vcpu); 1163 return ret; 1164 } 1165 1166 if (run->immediate_exit) { 1167 kvm_vcpu_srcu_read_unlock(vcpu); 1168 return -EINTR; 1169 } 1170 1171 vcpu_load(vcpu); 1172 1173 kvm_sigset_activate(vcpu); 1174 1175 ret = 1; 1176 run->exit_reason = KVM_EXIT_UNKNOWN; 1177 while (ret > 0) { 1178 /* Check conditions before entering the guest */ 1179 ret = xfer_to_guest_mode_handle_work(vcpu); 1180 if (ret) 1181 continue; 1182 ret = 1; 1183 1184 kvm_riscv_gstage_vmid_update(vcpu); 1185 1186 kvm_riscv_check_vcpu_requests(vcpu); 1187 1188 preempt_disable(); 1189 1190 /* Update AIA HW state before entering guest */ 1191 ret = kvm_riscv_vcpu_aia_update(vcpu); 1192 if (ret <= 0) { 1193 preempt_enable(); 1194 continue; 1195 } 1196 1197 local_irq_disable(); 1198 1199 /* 1200 * Ensure we set mode to IN_GUEST_MODE after we disable 1201 * interrupts and before the final VCPU requests check. 1202 * See the comment in kvm_vcpu_exiting_guest_mode() and 1203 * Documentation/virt/kvm/vcpu-requests.rst 1204 */ 1205 vcpu->mode = IN_GUEST_MODE; 1206 1207 kvm_vcpu_srcu_read_unlock(vcpu); 1208 smp_mb__after_srcu_read_unlock(); 1209 1210 /* 1211 * We might have got VCPU interrupts updated asynchronously 1212 * so update it in HW. 1213 */ 1214 kvm_riscv_vcpu_flush_interrupts(vcpu); 1215 1216 /* Update HVIP CSR for current CPU */ 1217 kvm_riscv_update_hvip(vcpu); 1218 1219 if (ret <= 0 || 1220 kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || 1221 kvm_request_pending(vcpu) || 1222 xfer_to_guest_mode_work_pending()) { 1223 vcpu->mode = OUTSIDE_GUEST_MODE; 1224 local_irq_enable(); 1225 preempt_enable(); 1226 kvm_vcpu_srcu_read_lock(vcpu); 1227 continue; 1228 } 1229 1230 /* 1231 * Cleanup stale TLB enteries 1232 * 1233 * Note: This should be done after G-stage VMID has been 1234 * updated using kvm_riscv_gstage_vmid_ver_changed() 1235 */ 1236 kvm_riscv_local_tlb_sanitize(vcpu); 1237 1238 guest_timing_enter_irqoff(); 1239 1240 kvm_riscv_vcpu_enter_exit(vcpu); 1241 1242 vcpu->mode = OUTSIDE_GUEST_MODE; 1243 vcpu->stat.exits++; 1244 1245 /* 1246 * Save SCAUSE, STVAL, HTVAL, and HTINST because we might 1247 * get an interrupt between __kvm_riscv_switch_to() and 1248 * local_irq_enable() which can potentially change CSRs. 1249 */ 1250 trap.sepc = vcpu->arch.guest_context.sepc; 1251 trap.scause = csr_read(CSR_SCAUSE); 1252 trap.stval = csr_read(CSR_STVAL); 1253 trap.htval = csr_read(CSR_HTVAL); 1254 trap.htinst = csr_read(CSR_HTINST); 1255 1256 /* Syncup interrupts state with HW */ 1257 kvm_riscv_vcpu_sync_interrupts(vcpu); 1258 1259 /* 1260 * We must ensure that any pending interrupts are taken before 1261 * we exit guest timing so that timer ticks are accounted as 1262 * guest time. Transiently unmask interrupts so that any 1263 * pending interrupts are taken. 1264 * 1265 * There's no barrier which ensures that pending interrupts are 1266 * recognised, so we just hope that the CPU takes any pending 1267 * interrupts between the enable and disable. 1268 */ 1269 local_irq_enable(); 1270 local_irq_disable(); 1271 1272 guest_timing_exit_irqoff(); 1273 1274 local_irq_enable(); 1275 1276 preempt_enable(); 1277 1278 kvm_vcpu_srcu_read_lock(vcpu); 1279 1280 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap); 1281 } 1282 1283 kvm_sigset_deactivate(vcpu); 1284 1285 vcpu_put(vcpu); 1286 1287 kvm_vcpu_srcu_read_unlock(vcpu); 1288 1289 return ret; 1290 } 1291