1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2023 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/uaccess.h> 14 #include <linux/kvm_host.h> 15 #include <asm/cacheflush.h> 16 #include <asm/hwcap.h> 17 #include <asm/kvm_vcpu_vector.h> 18 #include <asm/vector.h> 19 20 #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) 21 22 #define KVM_ISA_EXT_ARR(ext) \ 23 [KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext 24 25 /* Mapping between KVM ISA Extension ID & Host ISA extension ID */ 26 static const unsigned long kvm_isa_ext_arr[] = { 27 /* Single letter extensions (alphabetically sorted) */ 28 [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, 29 [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, 30 [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, 31 [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, 32 [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, 33 [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, 34 [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, 35 [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, 36 /* Multi letter extensions (alphabetically sorted) */ 37 KVM_ISA_EXT_ARR(SSAIA), 38 KVM_ISA_EXT_ARR(SSTC), 39 KVM_ISA_EXT_ARR(SVINVAL), 40 KVM_ISA_EXT_ARR(SVNAPOT), 41 KVM_ISA_EXT_ARR(SVPBMT), 42 KVM_ISA_EXT_ARR(ZBA), 43 KVM_ISA_EXT_ARR(ZBB), 44 KVM_ISA_EXT_ARR(ZBS), 45 KVM_ISA_EXT_ARR(ZICBOM), 46 KVM_ISA_EXT_ARR(ZICBOZ), 47 KVM_ISA_EXT_ARR(ZICNTR), 48 KVM_ISA_EXT_ARR(ZICSR), 49 KVM_ISA_EXT_ARR(ZIFENCEI), 50 KVM_ISA_EXT_ARR(ZIHINTPAUSE), 51 KVM_ISA_EXT_ARR(ZIHPM), 52 }; 53 54 static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext) 55 { 56 unsigned long i; 57 58 for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 59 if (kvm_isa_ext_arr[i] == base_ext) 60 return i; 61 } 62 63 return KVM_RISCV_ISA_EXT_MAX; 64 } 65 66 static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext) 67 { 68 switch (ext) { 69 case KVM_RISCV_ISA_EXT_H: 70 return false; 71 case KVM_RISCV_ISA_EXT_V: 72 return riscv_v_vstate_ctrl_user_allowed(); 73 default: 74 break; 75 } 76 77 return true; 78 } 79 80 static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) 81 { 82 switch (ext) { 83 case KVM_RISCV_ISA_EXT_A: 84 case KVM_RISCV_ISA_EXT_C: 85 case KVM_RISCV_ISA_EXT_I: 86 case KVM_RISCV_ISA_EXT_M: 87 case KVM_RISCV_ISA_EXT_SSAIA: 88 case KVM_RISCV_ISA_EXT_SSTC: 89 case KVM_RISCV_ISA_EXT_SVINVAL: 90 case KVM_RISCV_ISA_EXT_SVNAPOT: 91 case KVM_RISCV_ISA_EXT_ZBA: 92 case KVM_RISCV_ISA_EXT_ZBB: 93 case KVM_RISCV_ISA_EXT_ZBS: 94 case KVM_RISCV_ISA_EXT_ZICNTR: 95 case KVM_RISCV_ISA_EXT_ZICSR: 96 case KVM_RISCV_ISA_EXT_ZIFENCEI: 97 case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: 98 case KVM_RISCV_ISA_EXT_ZIHPM: 99 return false; 100 default: 101 break; 102 } 103 104 return true; 105 } 106 107 void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu) 108 { 109 unsigned long host_isa, i; 110 111 for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) { 112 host_isa = kvm_isa_ext_arr[i]; 113 if (__riscv_isa_extension_available(NULL, host_isa) && 114 kvm_riscv_vcpu_isa_enable_allowed(i)) 115 set_bit(host_isa, vcpu->arch.isa); 116 } 117 } 118 119 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, 120 const struct kvm_one_reg *reg) 121 { 122 unsigned long __user *uaddr = 123 (unsigned long __user *)(unsigned long)reg->addr; 124 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 125 KVM_REG_SIZE_MASK | 126 KVM_REG_RISCV_CONFIG); 127 unsigned long reg_val; 128 129 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 130 return -EINVAL; 131 132 switch (reg_num) { 133 case KVM_REG_RISCV_CONFIG_REG(isa): 134 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; 135 break; 136 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 137 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 138 return -ENOENT; 139 reg_val = riscv_cbom_block_size; 140 break; 141 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 142 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 143 return -ENOENT; 144 reg_val = riscv_cboz_block_size; 145 break; 146 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 147 reg_val = vcpu->arch.mvendorid; 148 break; 149 case KVM_REG_RISCV_CONFIG_REG(marchid): 150 reg_val = vcpu->arch.marchid; 151 break; 152 case KVM_REG_RISCV_CONFIG_REG(mimpid): 153 reg_val = vcpu->arch.mimpid; 154 break; 155 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 156 reg_val = satp_mode >> SATP_MODE_SHIFT; 157 break; 158 default: 159 return -ENOENT; 160 } 161 162 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 163 return -EFAULT; 164 165 return 0; 166 } 167 168 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, 169 const struct kvm_one_reg *reg) 170 { 171 unsigned long __user *uaddr = 172 (unsigned long __user *)(unsigned long)reg->addr; 173 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 174 KVM_REG_SIZE_MASK | 175 KVM_REG_RISCV_CONFIG); 176 unsigned long i, isa_ext, reg_val; 177 178 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 179 return -EINVAL; 180 181 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 182 return -EFAULT; 183 184 switch (reg_num) { 185 case KVM_REG_RISCV_CONFIG_REG(isa): 186 /* 187 * This ONE REG interface is only defined for 188 * single letter extensions. 189 */ 190 if (fls(reg_val) >= RISCV_ISA_EXT_BASE) 191 return -EINVAL; 192 193 /* 194 * Return early (i.e. do nothing) if reg_val is the same 195 * value retrievable via kvm_riscv_vcpu_get_reg_config(). 196 */ 197 if (reg_val == (vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK)) 198 break; 199 200 if (!vcpu->arch.ran_atleast_once) { 201 /* Ignore the enable/disable request for certain extensions */ 202 for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { 203 isa_ext = kvm_riscv_vcpu_base2isa_ext(i); 204 if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { 205 reg_val &= ~BIT(i); 206 continue; 207 } 208 if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext)) 209 if (reg_val & BIT(i)) 210 reg_val &= ~BIT(i); 211 if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext)) 212 if (!(reg_val & BIT(i))) 213 reg_val |= BIT(i); 214 } 215 reg_val &= riscv_isa_extension_base(NULL); 216 /* Do not modify anything beyond single letter extensions */ 217 reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) | 218 (reg_val & KVM_RISCV_BASE_ISA_MASK); 219 vcpu->arch.isa[0] = reg_val; 220 kvm_riscv_vcpu_fp_reset(vcpu); 221 } else { 222 return -EBUSY; 223 } 224 break; 225 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 226 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 227 return -ENOENT; 228 if (reg_val != riscv_cbom_block_size) 229 return -EINVAL; 230 break; 231 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 232 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 233 return -ENOENT; 234 if (reg_val != riscv_cboz_block_size) 235 return -EINVAL; 236 break; 237 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 238 if (reg_val == vcpu->arch.mvendorid) 239 break; 240 if (!vcpu->arch.ran_atleast_once) 241 vcpu->arch.mvendorid = reg_val; 242 else 243 return -EBUSY; 244 break; 245 case KVM_REG_RISCV_CONFIG_REG(marchid): 246 if (reg_val == vcpu->arch.marchid) 247 break; 248 if (!vcpu->arch.ran_atleast_once) 249 vcpu->arch.marchid = reg_val; 250 else 251 return -EBUSY; 252 break; 253 case KVM_REG_RISCV_CONFIG_REG(mimpid): 254 if (reg_val == vcpu->arch.mimpid) 255 break; 256 if (!vcpu->arch.ran_atleast_once) 257 vcpu->arch.mimpid = reg_val; 258 else 259 return -EBUSY; 260 break; 261 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 262 if (reg_val != (satp_mode >> SATP_MODE_SHIFT)) 263 return -EINVAL; 264 break; 265 default: 266 return -ENOENT; 267 } 268 269 return 0; 270 } 271 272 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, 273 const struct kvm_one_reg *reg) 274 { 275 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 276 unsigned long __user *uaddr = 277 (unsigned long __user *)(unsigned long)reg->addr; 278 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 279 KVM_REG_SIZE_MASK | 280 KVM_REG_RISCV_CORE); 281 unsigned long reg_val; 282 283 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 284 return -EINVAL; 285 if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) 286 return -ENOENT; 287 288 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 289 reg_val = cntx->sepc; 290 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 291 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 292 reg_val = ((unsigned long *)cntx)[reg_num]; 293 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) 294 reg_val = (cntx->sstatus & SR_SPP) ? 295 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U; 296 else 297 return -ENOENT; 298 299 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 300 return -EFAULT; 301 302 return 0; 303 } 304 305 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, 306 const struct kvm_one_reg *reg) 307 { 308 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 309 unsigned long __user *uaddr = 310 (unsigned long __user *)(unsigned long)reg->addr; 311 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 312 KVM_REG_SIZE_MASK | 313 KVM_REG_RISCV_CORE); 314 unsigned long reg_val; 315 316 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 317 return -EINVAL; 318 if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) 319 return -ENOENT; 320 321 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 322 return -EFAULT; 323 324 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 325 cntx->sepc = reg_val; 326 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 327 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 328 ((unsigned long *)cntx)[reg_num] = reg_val; 329 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) { 330 if (reg_val == KVM_RISCV_MODE_S) 331 cntx->sstatus |= SR_SPP; 332 else 333 cntx->sstatus &= ~SR_SPP; 334 } else 335 return -ENOENT; 336 337 return 0; 338 } 339 340 static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, 341 unsigned long reg_num, 342 unsigned long *out_val) 343 { 344 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 345 346 if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) 347 return -ENOENT; 348 349 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 350 kvm_riscv_vcpu_flush_interrupts(vcpu); 351 *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; 352 *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; 353 } else 354 *out_val = ((unsigned long *)csr)[reg_num]; 355 356 return 0; 357 } 358 359 static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, 360 unsigned long reg_num, 361 unsigned long reg_val) 362 { 363 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 364 365 if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) 366 return -ENOENT; 367 368 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 369 reg_val &= VSIP_VALID_MASK; 370 reg_val <<= VSIP_TO_HVIP_SHIFT; 371 } 372 373 ((unsigned long *)csr)[reg_num] = reg_val; 374 375 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) 376 WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); 377 378 return 0; 379 } 380 381 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, 382 const struct kvm_one_reg *reg) 383 { 384 int rc; 385 unsigned long __user *uaddr = 386 (unsigned long __user *)(unsigned long)reg->addr; 387 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 388 KVM_REG_SIZE_MASK | 389 KVM_REG_RISCV_CSR); 390 unsigned long reg_val, reg_subtype; 391 392 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 393 return -EINVAL; 394 395 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 396 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 397 switch (reg_subtype) { 398 case KVM_REG_RISCV_CSR_GENERAL: 399 rc = kvm_riscv_vcpu_general_get_csr(vcpu, reg_num, ®_val); 400 break; 401 case KVM_REG_RISCV_CSR_AIA: 402 rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); 403 break; 404 default: 405 rc = -ENOENT; 406 break; 407 } 408 if (rc) 409 return rc; 410 411 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 412 return -EFAULT; 413 414 return 0; 415 } 416 417 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, 418 const struct kvm_one_reg *reg) 419 { 420 int rc; 421 unsigned long __user *uaddr = 422 (unsigned long __user *)(unsigned long)reg->addr; 423 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 424 KVM_REG_SIZE_MASK | 425 KVM_REG_RISCV_CSR); 426 unsigned long reg_val, reg_subtype; 427 428 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 429 return -EINVAL; 430 431 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 432 return -EFAULT; 433 434 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 435 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 436 switch (reg_subtype) { 437 case KVM_REG_RISCV_CSR_GENERAL: 438 rc = kvm_riscv_vcpu_general_set_csr(vcpu, reg_num, reg_val); 439 break; 440 case KVM_REG_RISCV_CSR_AIA: 441 rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); 442 break; 443 default: 444 rc = -ENOENT; 445 break; 446 } 447 if (rc) 448 return rc; 449 450 return 0; 451 } 452 453 static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, 454 unsigned long reg_num, 455 unsigned long *reg_val) 456 { 457 unsigned long host_isa_ext; 458 459 if (reg_num >= KVM_RISCV_ISA_EXT_MAX || 460 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) 461 return -ENOENT; 462 463 host_isa_ext = kvm_isa_ext_arr[reg_num]; 464 if (!__riscv_isa_extension_available(NULL, host_isa_ext)) 465 return -ENOENT; 466 467 *reg_val = 0; 468 if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext)) 469 *reg_val = 1; /* Mark the given extension as available */ 470 471 return 0; 472 } 473 474 static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, 475 unsigned long reg_num, 476 unsigned long reg_val) 477 { 478 unsigned long host_isa_ext; 479 480 if (reg_num >= KVM_RISCV_ISA_EXT_MAX || 481 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) 482 return -ENOENT; 483 484 host_isa_ext = kvm_isa_ext_arr[reg_num]; 485 if (!__riscv_isa_extension_available(NULL, host_isa_ext)) 486 return -ENOENT; 487 488 if (reg_val == test_bit(host_isa_ext, vcpu->arch.isa)) 489 return 0; 490 491 if (!vcpu->arch.ran_atleast_once) { 492 /* 493 * All multi-letter extension and a few single letter 494 * extension can be disabled 495 */ 496 if (reg_val == 1 && 497 kvm_riscv_vcpu_isa_enable_allowed(reg_num)) 498 set_bit(host_isa_ext, vcpu->arch.isa); 499 else if (!reg_val && 500 kvm_riscv_vcpu_isa_disable_allowed(reg_num)) 501 clear_bit(host_isa_ext, vcpu->arch.isa); 502 else 503 return -EINVAL; 504 kvm_riscv_vcpu_fp_reset(vcpu); 505 } else { 506 return -EBUSY; 507 } 508 509 return 0; 510 } 511 512 static int riscv_vcpu_get_isa_ext_multi(struct kvm_vcpu *vcpu, 513 unsigned long reg_num, 514 unsigned long *reg_val) 515 { 516 unsigned long i, ext_id, ext_val; 517 518 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 519 return -ENOENT; 520 521 for (i = 0; i < BITS_PER_LONG; i++) { 522 ext_id = i + reg_num * BITS_PER_LONG; 523 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 524 break; 525 526 ext_val = 0; 527 riscv_vcpu_get_isa_ext_single(vcpu, ext_id, &ext_val); 528 if (ext_val) 529 *reg_val |= KVM_REG_RISCV_ISA_MULTI_MASK(ext_id); 530 } 531 532 return 0; 533 } 534 535 static int riscv_vcpu_set_isa_ext_multi(struct kvm_vcpu *vcpu, 536 unsigned long reg_num, 537 unsigned long reg_val, bool enable) 538 { 539 unsigned long i, ext_id; 540 541 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 542 return -ENOENT; 543 544 for_each_set_bit(i, ®_val, BITS_PER_LONG) { 545 ext_id = i + reg_num * BITS_PER_LONG; 546 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 547 break; 548 549 riscv_vcpu_set_isa_ext_single(vcpu, ext_id, enable); 550 } 551 552 return 0; 553 } 554 555 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu, 556 const struct kvm_one_reg *reg) 557 { 558 int rc; 559 unsigned long __user *uaddr = 560 (unsigned long __user *)(unsigned long)reg->addr; 561 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 562 KVM_REG_SIZE_MASK | 563 KVM_REG_RISCV_ISA_EXT); 564 unsigned long reg_val, reg_subtype; 565 566 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 567 return -EINVAL; 568 569 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 570 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 571 572 reg_val = 0; 573 switch (reg_subtype) { 574 case KVM_REG_RISCV_ISA_SINGLE: 575 rc = riscv_vcpu_get_isa_ext_single(vcpu, reg_num, ®_val); 576 break; 577 case KVM_REG_RISCV_ISA_MULTI_EN: 578 case KVM_REG_RISCV_ISA_MULTI_DIS: 579 rc = riscv_vcpu_get_isa_ext_multi(vcpu, reg_num, ®_val); 580 if (!rc && reg_subtype == KVM_REG_RISCV_ISA_MULTI_DIS) 581 reg_val = ~reg_val; 582 break; 583 default: 584 rc = -ENOENT; 585 } 586 if (rc) 587 return rc; 588 589 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 590 return -EFAULT; 591 592 return 0; 593 } 594 595 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu, 596 const struct kvm_one_reg *reg) 597 { 598 unsigned long __user *uaddr = 599 (unsigned long __user *)(unsigned long)reg->addr; 600 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 601 KVM_REG_SIZE_MASK | 602 KVM_REG_RISCV_ISA_EXT); 603 unsigned long reg_val, reg_subtype; 604 605 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 606 return -EINVAL; 607 608 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 609 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 610 611 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 612 return -EFAULT; 613 614 switch (reg_subtype) { 615 case KVM_REG_RISCV_ISA_SINGLE: 616 return riscv_vcpu_set_isa_ext_single(vcpu, reg_num, reg_val); 617 case KVM_REG_RISCV_ISA_MULTI_EN: 618 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, true); 619 case KVM_REG_RISCV_ISA_MULTI_DIS: 620 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, false); 621 default: 622 return -ENOENT; 623 } 624 625 return 0; 626 } 627 628 static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, 629 u64 __user *uindices) 630 { 631 int n = 0; 632 633 for (int i = 0; i < sizeof(struct kvm_riscv_config)/sizeof(unsigned long); 634 i++) { 635 u64 size; 636 u64 reg; 637 638 /* 639 * Avoid reporting config reg if the corresponding extension 640 * was not available. 641 */ 642 if (i == KVM_REG_RISCV_CONFIG_REG(zicbom_block_size) && 643 !riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 644 continue; 645 else if (i == KVM_REG_RISCV_CONFIG_REG(zicboz_block_size) && 646 !riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 647 continue; 648 649 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 650 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; 651 652 if (uindices) { 653 if (put_user(reg, uindices)) 654 return -EFAULT; 655 uindices++; 656 } 657 658 n++; 659 } 660 661 return n; 662 } 663 664 static unsigned long num_config_regs(const struct kvm_vcpu *vcpu) 665 { 666 return copy_config_reg_indices(vcpu, NULL); 667 } 668 669 static inline unsigned long num_core_regs(void) 670 { 671 return sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 672 } 673 674 static int copy_core_reg_indices(u64 __user *uindices) 675 { 676 int n = num_core_regs(); 677 678 for (int i = 0; i < n; i++) { 679 u64 size = IS_ENABLED(CONFIG_32BIT) ? 680 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 681 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CORE | i; 682 683 if (uindices) { 684 if (put_user(reg, uindices)) 685 return -EFAULT; 686 uindices++; 687 } 688 } 689 690 return n; 691 } 692 693 static inline unsigned long num_csr_regs(const struct kvm_vcpu *vcpu) 694 { 695 unsigned long n = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 696 697 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 698 n += sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 699 700 return n; 701 } 702 703 static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu, 704 u64 __user *uindices) 705 { 706 int n1 = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 707 int n2 = 0; 708 709 /* copy general csr regs */ 710 for (int i = 0; i < n1; i++) { 711 u64 size = IS_ENABLED(CONFIG_32BIT) ? 712 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 713 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 714 KVM_REG_RISCV_CSR_GENERAL | i; 715 716 if (uindices) { 717 if (put_user(reg, uindices)) 718 return -EFAULT; 719 uindices++; 720 } 721 } 722 723 /* copy AIA csr regs */ 724 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) { 725 n2 = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 726 727 for (int i = 0; i < n2; i++) { 728 u64 size = IS_ENABLED(CONFIG_32BIT) ? 729 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 730 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 731 KVM_REG_RISCV_CSR_AIA | i; 732 733 if (uindices) { 734 if (put_user(reg, uindices)) 735 return -EFAULT; 736 uindices++; 737 } 738 } 739 } 740 741 return n1 + n2; 742 } 743 744 static inline unsigned long num_timer_regs(void) 745 { 746 return sizeof(struct kvm_riscv_timer) / sizeof(u64); 747 } 748 749 static int copy_timer_reg_indices(u64 __user *uindices) 750 { 751 int n = num_timer_regs(); 752 753 for (int i = 0; i < n; i++) { 754 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 755 KVM_REG_RISCV_TIMER | i; 756 757 if (uindices) { 758 if (put_user(reg, uindices)) 759 return -EFAULT; 760 uindices++; 761 } 762 } 763 764 return n; 765 } 766 767 static inline unsigned long num_fp_f_regs(const struct kvm_vcpu *vcpu) 768 { 769 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 770 771 if (riscv_isa_extension_available(vcpu->arch.isa, f)) 772 return sizeof(cntx->fp.f) / sizeof(u32); 773 else 774 return 0; 775 } 776 777 static int copy_fp_f_reg_indices(const struct kvm_vcpu *vcpu, 778 u64 __user *uindices) 779 { 780 int n = num_fp_f_regs(vcpu); 781 782 for (int i = 0; i < n; i++) { 783 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | 784 KVM_REG_RISCV_FP_F | i; 785 786 if (uindices) { 787 if (put_user(reg, uindices)) 788 return -EFAULT; 789 uindices++; 790 } 791 } 792 793 return n; 794 } 795 796 static inline unsigned long num_fp_d_regs(const struct kvm_vcpu *vcpu) 797 { 798 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 799 800 if (riscv_isa_extension_available(vcpu->arch.isa, d)) 801 return sizeof(cntx->fp.d.f) / sizeof(u64) + 1; 802 else 803 return 0; 804 } 805 806 static int copy_fp_d_reg_indices(const struct kvm_vcpu *vcpu, 807 u64 __user *uindices) 808 { 809 int i; 810 int n = num_fp_d_regs(vcpu); 811 u64 reg; 812 813 /* copy fp.d.f indices */ 814 for (i = 0; i < n-1; i++) { 815 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 816 KVM_REG_RISCV_FP_D | i; 817 818 if (uindices) { 819 if (put_user(reg, uindices)) 820 return -EFAULT; 821 uindices++; 822 } 823 } 824 825 /* copy fp.d.fcsr indices */ 826 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_D | i; 827 if (uindices) { 828 if (put_user(reg, uindices)) 829 return -EFAULT; 830 uindices++; 831 } 832 833 return n; 834 } 835 836 static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, 837 u64 __user *uindices) 838 { 839 unsigned int n = 0; 840 unsigned long isa_ext; 841 842 for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 843 u64 size = IS_ENABLED(CONFIG_32BIT) ? 844 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 845 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i; 846 847 isa_ext = kvm_isa_ext_arr[i]; 848 if (!__riscv_isa_extension_available(NULL, isa_ext)) 849 continue; 850 851 if (uindices) { 852 if (put_user(reg, uindices)) 853 return -EFAULT; 854 uindices++; 855 } 856 857 n++; 858 } 859 860 return n; 861 } 862 863 static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu) 864 { 865 return copy_isa_ext_reg_indices(vcpu, NULL);; 866 } 867 868 static inline unsigned long num_sbi_ext_regs(void) 869 { 870 /* 871 * number of KVM_REG_RISCV_SBI_SINGLE + 872 * 2 x (number of KVM_REG_RISCV_SBI_MULTI) 873 */ 874 return KVM_RISCV_SBI_EXT_MAX + 2*(KVM_REG_RISCV_SBI_MULTI_REG_LAST+1); 875 } 876 877 static int copy_sbi_ext_reg_indices(u64 __user *uindices) 878 { 879 int n; 880 881 /* copy KVM_REG_RISCV_SBI_SINGLE */ 882 n = KVM_RISCV_SBI_EXT_MAX; 883 for (int i = 0; i < n; i++) { 884 u64 size = IS_ENABLED(CONFIG_32BIT) ? 885 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 886 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | 887 KVM_REG_RISCV_SBI_SINGLE | i; 888 889 if (uindices) { 890 if (put_user(reg, uindices)) 891 return -EFAULT; 892 uindices++; 893 } 894 } 895 896 /* copy KVM_REG_RISCV_SBI_MULTI */ 897 n = KVM_REG_RISCV_SBI_MULTI_REG_LAST + 1; 898 for (int i = 0; i < n; i++) { 899 u64 size = IS_ENABLED(CONFIG_32BIT) ? 900 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 901 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | 902 KVM_REG_RISCV_SBI_MULTI_EN | i; 903 904 if (uindices) { 905 if (put_user(reg, uindices)) 906 return -EFAULT; 907 uindices++; 908 } 909 910 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_SBI_EXT | 911 KVM_REG_RISCV_SBI_MULTI_DIS | i; 912 913 if (uindices) { 914 if (put_user(reg, uindices)) 915 return -EFAULT; 916 uindices++; 917 } 918 } 919 920 return num_sbi_ext_regs(); 921 } 922 923 /* 924 * kvm_riscv_vcpu_num_regs - how many registers do we present via KVM_GET/SET_ONE_REG 925 * 926 * This is for all registers. 927 */ 928 unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu) 929 { 930 unsigned long res = 0; 931 932 res += num_config_regs(vcpu); 933 res += num_core_regs(); 934 res += num_csr_regs(vcpu); 935 res += num_timer_regs(); 936 res += num_fp_f_regs(vcpu); 937 res += num_fp_d_regs(vcpu); 938 res += num_isa_ext_regs(vcpu); 939 res += num_sbi_ext_regs(); 940 941 return res; 942 } 943 944 /* 945 * kvm_riscv_vcpu_copy_reg_indices - get indices of all registers. 946 */ 947 int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, 948 u64 __user *uindices) 949 { 950 int ret; 951 952 ret = copy_config_reg_indices(vcpu, uindices); 953 if (ret < 0) 954 return ret; 955 uindices += ret; 956 957 ret = copy_core_reg_indices(uindices); 958 if (ret < 0) 959 return ret; 960 uindices += ret; 961 962 ret = copy_csr_reg_indices(vcpu, uindices); 963 if (ret < 0) 964 return ret; 965 uindices += ret; 966 967 ret = copy_timer_reg_indices(uindices); 968 if (ret < 0) 969 return ret; 970 uindices += ret; 971 972 ret = copy_fp_f_reg_indices(vcpu, uindices); 973 if (ret < 0) 974 return ret; 975 uindices += ret; 976 977 ret = copy_fp_d_reg_indices(vcpu, uindices); 978 if (ret < 0) 979 return ret; 980 uindices += ret; 981 982 ret = copy_isa_ext_reg_indices(vcpu, uindices); 983 if (ret < 0) 984 return ret; 985 uindices += ret; 986 987 ret = copy_sbi_ext_reg_indices(uindices); 988 if (ret < 0) 989 return ret; 990 991 return 0; 992 } 993 994 int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, 995 const struct kvm_one_reg *reg) 996 { 997 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 998 case KVM_REG_RISCV_CONFIG: 999 return kvm_riscv_vcpu_set_reg_config(vcpu, reg); 1000 case KVM_REG_RISCV_CORE: 1001 return kvm_riscv_vcpu_set_reg_core(vcpu, reg); 1002 case KVM_REG_RISCV_CSR: 1003 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg); 1004 case KVM_REG_RISCV_TIMER: 1005 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg); 1006 case KVM_REG_RISCV_FP_F: 1007 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1008 KVM_REG_RISCV_FP_F); 1009 case KVM_REG_RISCV_FP_D: 1010 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1011 KVM_REG_RISCV_FP_D); 1012 case KVM_REG_RISCV_ISA_EXT: 1013 return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); 1014 case KVM_REG_RISCV_SBI_EXT: 1015 return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); 1016 case KVM_REG_RISCV_VECTOR: 1017 return kvm_riscv_vcpu_set_reg_vector(vcpu, reg); 1018 default: 1019 break; 1020 } 1021 1022 return -ENOENT; 1023 } 1024 1025 int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, 1026 const struct kvm_one_reg *reg) 1027 { 1028 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 1029 case KVM_REG_RISCV_CONFIG: 1030 return kvm_riscv_vcpu_get_reg_config(vcpu, reg); 1031 case KVM_REG_RISCV_CORE: 1032 return kvm_riscv_vcpu_get_reg_core(vcpu, reg); 1033 case KVM_REG_RISCV_CSR: 1034 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg); 1035 case KVM_REG_RISCV_TIMER: 1036 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg); 1037 case KVM_REG_RISCV_FP_F: 1038 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1039 KVM_REG_RISCV_FP_F); 1040 case KVM_REG_RISCV_FP_D: 1041 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1042 KVM_REG_RISCV_FP_D); 1043 case KVM_REG_RISCV_ISA_EXT: 1044 return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); 1045 case KVM_REG_RISCV_SBI_EXT: 1046 return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); 1047 case KVM_REG_RISCV_VECTOR: 1048 return kvm_riscv_vcpu_get_reg_vector(vcpu, reg); 1049 default: 1050 break; 1051 } 1052 1053 return -ENOENT; 1054 } 1055