1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 * 6 * Derived from arch/arm/kvm/guest.c: 7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 8 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 9 */ 10 11 #include <linux/bits.h> 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/nospec.h> 15 #include <linux/kvm_host.h> 16 #include <linux/module.h> 17 #include <linux/stddef.h> 18 #include <linux/string.h> 19 #include <linux/vmalloc.h> 20 #include <linux/fs.h> 21 #include <kvm/arm_psci.h> 22 #include <asm/cputype.h> 23 #include <linux/uaccess.h> 24 #include <asm/fpsimd.h> 25 #include <asm/kvm.h> 26 #include <asm/kvm_emulate.h> 27 #include <asm/kvm_coproc.h> 28 #include <asm/sigcontext.h> 29 30 #include "trace.h" 31 32 struct kvm_stats_debugfs_item debugfs_entries[] = { 33 VCPU_STAT("halt_successful_poll", halt_successful_poll), 34 VCPU_STAT("halt_attempted_poll", halt_attempted_poll), 35 VCPU_STAT("halt_poll_invalid", halt_poll_invalid), 36 VCPU_STAT("halt_wakeup", halt_wakeup), 37 VCPU_STAT("hvc_exit_stat", hvc_exit_stat), 38 VCPU_STAT("wfe_exit_stat", wfe_exit_stat), 39 VCPU_STAT("wfi_exit_stat", wfi_exit_stat), 40 VCPU_STAT("mmio_exit_user", mmio_exit_user), 41 VCPU_STAT("mmio_exit_kernel", mmio_exit_kernel), 42 VCPU_STAT("exits", exits), 43 VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), 44 VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), 45 { NULL } 46 }; 47 48 static bool core_reg_offset_is_vreg(u64 off) 49 { 50 return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && 51 off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr); 52 } 53 54 static u64 core_reg_offset_from_id(u64 id) 55 { 56 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE); 57 } 58 59 static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off) 60 { 61 int size; 62 63 switch (off) { 64 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 65 KVM_REG_ARM_CORE_REG(regs.regs[30]): 66 case KVM_REG_ARM_CORE_REG(regs.sp): 67 case KVM_REG_ARM_CORE_REG(regs.pc): 68 case KVM_REG_ARM_CORE_REG(regs.pstate): 69 case KVM_REG_ARM_CORE_REG(sp_el1): 70 case KVM_REG_ARM_CORE_REG(elr_el1): 71 case KVM_REG_ARM_CORE_REG(spsr[0]) ... 72 KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]): 73 size = sizeof(__u64); 74 break; 75 76 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 77 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 78 size = sizeof(__uint128_t); 79 break; 80 81 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 82 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 83 size = sizeof(__u32); 84 break; 85 86 default: 87 return -EINVAL; 88 } 89 90 if (!IS_ALIGNED(off, size / sizeof(__u32))) 91 return -EINVAL; 92 93 /* 94 * The KVM_REG_ARM64_SVE regs must be used instead of 95 * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on 96 * SVE-enabled vcpus: 97 */ 98 if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off)) 99 return -EINVAL; 100 101 return size; 102 } 103 104 static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 105 { 106 u64 off = core_reg_offset_from_id(reg->id); 107 int size = core_reg_size_from_offset(vcpu, off); 108 109 if (size < 0) 110 return NULL; 111 112 if (KVM_REG_SIZE(reg->id) != size) 113 return NULL; 114 115 switch (off) { 116 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 117 KVM_REG_ARM_CORE_REG(regs.regs[30]): 118 off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); 119 off /= 2; 120 return &vcpu->arch.ctxt.regs.regs[off]; 121 122 case KVM_REG_ARM_CORE_REG(regs.sp): 123 return &vcpu->arch.ctxt.regs.sp; 124 125 case KVM_REG_ARM_CORE_REG(regs.pc): 126 return &vcpu->arch.ctxt.regs.pc; 127 128 case KVM_REG_ARM_CORE_REG(regs.pstate): 129 return &vcpu->arch.ctxt.regs.pstate; 130 131 case KVM_REG_ARM_CORE_REG(sp_el1): 132 return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); 133 134 case KVM_REG_ARM_CORE_REG(elr_el1): 135 return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); 136 137 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]): 138 return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1); 139 140 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]): 141 return &vcpu->arch.ctxt.spsr_abt; 142 143 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]): 144 return &vcpu->arch.ctxt.spsr_und; 145 146 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]): 147 return &vcpu->arch.ctxt.spsr_irq; 148 149 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]): 150 return &vcpu->arch.ctxt.spsr_fiq; 151 152 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 153 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 154 off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); 155 off /= 4; 156 return &vcpu->arch.ctxt.fp_regs.vregs[off]; 157 158 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 159 return &vcpu->arch.ctxt.fp_regs.fpsr; 160 161 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 162 return &vcpu->arch.ctxt.fp_regs.fpcr; 163 164 default: 165 return NULL; 166 } 167 } 168 169 static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 170 { 171 /* 172 * Because the kvm_regs structure is a mix of 32, 64 and 173 * 128bit fields, we index it as if it was a 32bit 174 * array. Hence below, nr_regs is the number of entries, and 175 * off the index in the "array". 176 */ 177 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 178 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 179 void *addr; 180 u32 off; 181 182 /* Our ID is an index into the kvm_regs struct. */ 183 off = core_reg_offset_from_id(reg->id); 184 if (off >= nr_regs || 185 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 186 return -ENOENT; 187 188 addr = core_reg_addr(vcpu, reg); 189 if (!addr) 190 return -EINVAL; 191 192 if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id))) 193 return -EFAULT; 194 195 return 0; 196 } 197 198 static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 199 { 200 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 201 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 202 __uint128_t tmp; 203 void *valp = &tmp, *addr; 204 u64 off; 205 int err = 0; 206 207 /* Our ID is an index into the kvm_regs struct. */ 208 off = core_reg_offset_from_id(reg->id); 209 if (off >= nr_regs || 210 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 211 return -ENOENT; 212 213 addr = core_reg_addr(vcpu, reg); 214 if (!addr) 215 return -EINVAL; 216 217 if (KVM_REG_SIZE(reg->id) > sizeof(tmp)) 218 return -EINVAL; 219 220 if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) { 221 err = -EFAULT; 222 goto out; 223 } 224 225 if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) { 226 u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK; 227 switch (mode) { 228 case PSR_AA32_MODE_USR: 229 if (!system_supports_32bit_el0()) 230 return -EINVAL; 231 break; 232 case PSR_AA32_MODE_FIQ: 233 case PSR_AA32_MODE_IRQ: 234 case PSR_AA32_MODE_SVC: 235 case PSR_AA32_MODE_ABT: 236 case PSR_AA32_MODE_UND: 237 if (!vcpu_el1_is_32bit(vcpu)) 238 return -EINVAL; 239 break; 240 case PSR_MODE_EL0t: 241 case PSR_MODE_EL1t: 242 case PSR_MODE_EL1h: 243 if (vcpu_el1_is_32bit(vcpu)) 244 return -EINVAL; 245 break; 246 default: 247 err = -EINVAL; 248 goto out; 249 } 250 } 251 252 memcpy(addr, valp, KVM_REG_SIZE(reg->id)); 253 254 if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) { 255 int i; 256 257 for (i = 0; i < 16; i++) 258 *vcpu_reg32(vcpu, i) = (u32)*vcpu_reg32(vcpu, i); 259 } 260 out: 261 return err; 262 } 263 264 #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) 265 #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) 266 #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) 267 268 static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 269 { 270 unsigned int max_vq, vq; 271 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 272 273 if (!vcpu_has_sve(vcpu)) 274 return -ENOENT; 275 276 if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl))) 277 return -EINVAL; 278 279 memset(vqs, 0, sizeof(vqs)); 280 281 max_vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); 282 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 283 if (sve_vq_available(vq)) 284 vqs[vq_word(vq)] |= vq_mask(vq); 285 286 if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs))) 287 return -EFAULT; 288 289 return 0; 290 } 291 292 static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 293 { 294 unsigned int max_vq, vq; 295 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 296 297 if (!vcpu_has_sve(vcpu)) 298 return -ENOENT; 299 300 if (kvm_arm_vcpu_sve_finalized(vcpu)) 301 return -EPERM; /* too late! */ 302 303 if (WARN_ON(vcpu->arch.sve_state)) 304 return -EINVAL; 305 306 if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs))) 307 return -EFAULT; 308 309 max_vq = 0; 310 for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq) 311 if (vq_present(vqs, vq)) 312 max_vq = vq; 313 314 if (max_vq > sve_vq_from_vl(kvm_sve_max_vl)) 315 return -EINVAL; 316 317 /* 318 * Vector lengths supported by the host can't currently be 319 * hidden from the guest individually: instead we can only set a 320 * maximum via ZCR_EL2.LEN. So, make sure the available vector 321 * lengths match the set requested exactly up to the requested 322 * maximum: 323 */ 324 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 325 if (vq_present(vqs, vq) != sve_vq_available(vq)) 326 return -EINVAL; 327 328 /* Can't run with no vector lengths at all: */ 329 if (max_vq < SVE_VQ_MIN) 330 return -EINVAL; 331 332 /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */ 333 vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq); 334 335 return 0; 336 } 337 338 #define SVE_REG_SLICE_SHIFT 0 339 #define SVE_REG_SLICE_BITS 5 340 #define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS) 341 #define SVE_REG_ID_BITS 5 342 343 #define SVE_REG_SLICE_MASK \ 344 GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \ 345 SVE_REG_SLICE_SHIFT) 346 #define SVE_REG_ID_MASK \ 347 GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT) 348 349 #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS) 350 351 #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0)) 352 #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0)) 353 354 /* 355 * Number of register slices required to cover each whole SVE register. 356 * NOTE: Only the first slice every exists, for now. 357 * If you are tempted to modify this, you must also rework sve_reg_to_region() 358 * to match: 359 */ 360 #define vcpu_sve_slices(vcpu) 1 361 362 /* Bounds of a single SVE register slice within vcpu->arch.sve_state */ 363 struct sve_state_reg_region { 364 unsigned int koffset; /* offset into sve_state in kernel memory */ 365 unsigned int klen; /* length in kernel memory */ 366 unsigned int upad; /* extra trailing padding in user memory */ 367 }; 368 369 /* 370 * Validate SVE register ID and get sanitised bounds for user/kernel SVE 371 * register copy 372 */ 373 static int sve_reg_to_region(struct sve_state_reg_region *region, 374 struct kvm_vcpu *vcpu, 375 const struct kvm_one_reg *reg) 376 { 377 /* reg ID ranges for Z- registers */ 378 const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0); 379 const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1, 380 SVE_NUM_SLICES - 1); 381 382 /* reg ID ranges for P- registers and FFR (which are contiguous) */ 383 const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0); 384 const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1); 385 386 unsigned int vq; 387 unsigned int reg_num; 388 389 unsigned int reqoffset, reqlen; /* User-requested offset and length */ 390 unsigned int maxlen; /* Maximum permitted length */ 391 392 size_t sve_state_size; 393 394 const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1, 395 SVE_NUM_SLICES - 1); 396 397 /* Verify that the P-regs and FFR really do have contiguous IDs: */ 398 BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1); 399 400 /* Verify that we match the UAPI header: */ 401 BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES); 402 403 reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT; 404 405 if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) { 406 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 407 return -ENOENT; 408 409 vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); 410 411 reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) - 412 SVE_SIG_REGS_OFFSET; 413 reqlen = KVM_SVE_ZREG_SIZE; 414 maxlen = SVE_SIG_ZREG_SIZE(vq); 415 } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) { 416 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 417 return -ENOENT; 418 419 vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); 420 421 reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) - 422 SVE_SIG_REGS_OFFSET; 423 reqlen = KVM_SVE_PREG_SIZE; 424 maxlen = SVE_SIG_PREG_SIZE(vq); 425 } else { 426 return -EINVAL; 427 } 428 429 sve_state_size = vcpu_sve_state_size(vcpu); 430 if (WARN_ON(!sve_state_size)) 431 return -EINVAL; 432 433 region->koffset = array_index_nospec(reqoffset, sve_state_size); 434 region->klen = min(maxlen, reqlen); 435 region->upad = reqlen - region->klen; 436 437 return 0; 438 } 439 440 static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 441 { 442 int ret; 443 struct sve_state_reg_region region; 444 char __user *uptr = (char __user *)reg->addr; 445 446 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 447 if (reg->id == KVM_REG_ARM64_SVE_VLS) 448 return get_sve_vls(vcpu, reg); 449 450 /* Try to interpret reg ID as an architectural SVE register... */ 451 ret = sve_reg_to_region(®ion, vcpu, reg); 452 if (ret) 453 return ret; 454 455 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 456 return -EPERM; 457 458 if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, 459 region.klen) || 460 clear_user(uptr + region.klen, region.upad)) 461 return -EFAULT; 462 463 return 0; 464 } 465 466 static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 467 { 468 int ret; 469 struct sve_state_reg_region region; 470 const char __user *uptr = (const char __user *)reg->addr; 471 472 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 473 if (reg->id == KVM_REG_ARM64_SVE_VLS) 474 return set_sve_vls(vcpu, reg); 475 476 /* Try to interpret reg ID as an architectural SVE register... */ 477 ret = sve_reg_to_region(®ion, vcpu, reg); 478 if (ret) 479 return ret; 480 481 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 482 return -EPERM; 483 484 if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, 485 region.klen)) 486 return -EFAULT; 487 488 return 0; 489 } 490 491 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 492 { 493 return -EINVAL; 494 } 495 496 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 497 { 498 return -EINVAL; 499 } 500 501 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, 502 u64 __user *uindices) 503 { 504 unsigned int i; 505 int n = 0; 506 507 for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) { 508 u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i; 509 int size = core_reg_size_from_offset(vcpu, i); 510 511 if (size < 0) 512 continue; 513 514 switch (size) { 515 case sizeof(__u32): 516 reg |= KVM_REG_SIZE_U32; 517 break; 518 519 case sizeof(__u64): 520 reg |= KVM_REG_SIZE_U64; 521 break; 522 523 case sizeof(__uint128_t): 524 reg |= KVM_REG_SIZE_U128; 525 break; 526 527 default: 528 WARN_ON(1); 529 continue; 530 } 531 532 if (uindices) { 533 if (put_user(reg, uindices)) 534 return -EFAULT; 535 uindices++; 536 } 537 538 n++; 539 } 540 541 return n; 542 } 543 544 static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) 545 { 546 return copy_core_reg_indices(vcpu, NULL); 547 } 548 549 /** 550 * ARM64 versions of the TIMER registers, always available on arm64 551 */ 552 553 #define NUM_TIMER_REGS 3 554 555 static bool is_timer_reg(u64 index) 556 { 557 switch (index) { 558 case KVM_REG_ARM_TIMER_CTL: 559 case KVM_REG_ARM_TIMER_CNT: 560 case KVM_REG_ARM_TIMER_CVAL: 561 return true; 562 } 563 return false; 564 } 565 566 static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 567 { 568 if (put_user(KVM_REG_ARM_TIMER_CTL, uindices)) 569 return -EFAULT; 570 uindices++; 571 if (put_user(KVM_REG_ARM_TIMER_CNT, uindices)) 572 return -EFAULT; 573 uindices++; 574 if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices)) 575 return -EFAULT; 576 577 return 0; 578 } 579 580 static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 581 { 582 void __user *uaddr = (void __user *)(long)reg->addr; 583 u64 val; 584 int ret; 585 586 ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id)); 587 if (ret != 0) 588 return -EFAULT; 589 590 return kvm_arm_timer_set_reg(vcpu, reg->id, val); 591 } 592 593 static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 594 { 595 void __user *uaddr = (void __user *)(long)reg->addr; 596 u64 val; 597 598 val = kvm_arm_timer_get_reg(vcpu, reg->id); 599 return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0; 600 } 601 602 static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) 603 { 604 const unsigned int slices = vcpu_sve_slices(vcpu); 605 606 if (!vcpu_has_sve(vcpu)) 607 return 0; 608 609 /* Policed by KVM_GET_REG_LIST: */ 610 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 611 612 return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */) 613 + 1; /* KVM_REG_ARM64_SVE_VLS */ 614 } 615 616 static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu, 617 u64 __user *uindices) 618 { 619 const unsigned int slices = vcpu_sve_slices(vcpu); 620 u64 reg; 621 unsigned int i, n; 622 int num_regs = 0; 623 624 if (!vcpu_has_sve(vcpu)) 625 return 0; 626 627 /* Policed by KVM_GET_REG_LIST: */ 628 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 629 630 /* 631 * Enumerate this first, so that userspace can save/restore in 632 * the order reported by KVM_GET_REG_LIST: 633 */ 634 reg = KVM_REG_ARM64_SVE_VLS; 635 if (put_user(reg, uindices++)) 636 return -EFAULT; 637 ++num_regs; 638 639 for (i = 0; i < slices; i++) { 640 for (n = 0; n < SVE_NUM_ZREGS; n++) { 641 reg = KVM_REG_ARM64_SVE_ZREG(n, i); 642 if (put_user(reg, uindices++)) 643 return -EFAULT; 644 num_regs++; 645 } 646 647 for (n = 0; n < SVE_NUM_PREGS; n++) { 648 reg = KVM_REG_ARM64_SVE_PREG(n, i); 649 if (put_user(reg, uindices++)) 650 return -EFAULT; 651 num_regs++; 652 } 653 654 reg = KVM_REG_ARM64_SVE_FFR(i); 655 if (put_user(reg, uindices++)) 656 return -EFAULT; 657 num_regs++; 658 } 659 660 return num_regs; 661 } 662 663 /** 664 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG 665 * 666 * This is for all registers. 667 */ 668 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) 669 { 670 unsigned long res = 0; 671 672 res += num_core_regs(vcpu); 673 res += num_sve_regs(vcpu); 674 res += kvm_arm_num_sys_reg_descs(vcpu); 675 res += kvm_arm_get_fw_num_regs(vcpu); 676 res += NUM_TIMER_REGS; 677 678 return res; 679 } 680 681 /** 682 * kvm_arm_copy_reg_indices - get indices of all registers. 683 * 684 * We do core registers right here, then we append system regs. 685 */ 686 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 687 { 688 int ret; 689 690 ret = copy_core_reg_indices(vcpu, uindices); 691 if (ret < 0) 692 return ret; 693 uindices += ret; 694 695 ret = copy_sve_reg_indices(vcpu, uindices); 696 if (ret < 0) 697 return ret; 698 uindices += ret; 699 700 ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices); 701 if (ret < 0) 702 return ret; 703 uindices += kvm_arm_get_fw_num_regs(vcpu); 704 705 ret = copy_timer_indices(vcpu, uindices); 706 if (ret < 0) 707 return ret; 708 uindices += NUM_TIMER_REGS; 709 710 return kvm_arm_copy_sys_reg_indices(vcpu, uindices); 711 } 712 713 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 714 { 715 /* We currently use nothing arch-specific in upper 32 bits */ 716 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 717 return -EINVAL; 718 719 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 720 case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg); 721 case KVM_REG_ARM_FW: return kvm_arm_get_fw_reg(vcpu, reg); 722 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg); 723 } 724 725 if (is_timer_reg(reg->id)) 726 return get_timer_reg(vcpu, reg); 727 728 return kvm_arm_sys_reg_get_reg(vcpu, reg); 729 } 730 731 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 732 { 733 /* We currently use nothing arch-specific in upper 32 bits */ 734 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 735 return -EINVAL; 736 737 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 738 case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg); 739 case KVM_REG_ARM_FW: return kvm_arm_set_fw_reg(vcpu, reg); 740 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg); 741 } 742 743 if (is_timer_reg(reg->id)) 744 return set_timer_reg(vcpu, reg); 745 746 return kvm_arm_sys_reg_set_reg(vcpu, reg); 747 } 748 749 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 750 struct kvm_sregs *sregs) 751 { 752 return -EINVAL; 753 } 754 755 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 756 struct kvm_sregs *sregs) 757 { 758 return -EINVAL; 759 } 760 761 int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 762 struct kvm_vcpu_events *events) 763 { 764 events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE); 765 events->exception.serror_has_esr = cpus_have_const_cap(ARM64_HAS_RAS_EXTN); 766 767 if (events->exception.serror_pending && events->exception.serror_has_esr) 768 events->exception.serror_esr = vcpu_get_vsesr(vcpu); 769 770 /* 771 * We never return a pending ext_dabt here because we deliver it to 772 * the virtual CPU directly when setting the event and it's no longer 773 * 'pending' at this point. 774 */ 775 776 return 0; 777 } 778 779 int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 780 struct kvm_vcpu_events *events) 781 { 782 bool serror_pending = events->exception.serror_pending; 783 bool has_esr = events->exception.serror_has_esr; 784 bool ext_dabt_pending = events->exception.ext_dabt_pending; 785 786 if (serror_pending && has_esr) { 787 if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) 788 return -EINVAL; 789 790 if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK)) 791 kvm_set_sei_esr(vcpu, events->exception.serror_esr); 792 else 793 return -EINVAL; 794 } else if (serror_pending) { 795 kvm_inject_vabt(vcpu); 796 } 797 798 if (ext_dabt_pending) 799 kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); 800 801 return 0; 802 } 803 804 int __attribute_const__ kvm_target_cpu(void) 805 { 806 unsigned long implementor = read_cpuid_implementor(); 807 unsigned long part_number = read_cpuid_part_number(); 808 809 switch (implementor) { 810 case ARM_CPU_IMP_ARM: 811 switch (part_number) { 812 case ARM_CPU_PART_AEM_V8: 813 return KVM_ARM_TARGET_AEM_V8; 814 case ARM_CPU_PART_FOUNDATION: 815 return KVM_ARM_TARGET_FOUNDATION_V8; 816 case ARM_CPU_PART_CORTEX_A53: 817 return KVM_ARM_TARGET_CORTEX_A53; 818 case ARM_CPU_PART_CORTEX_A57: 819 return KVM_ARM_TARGET_CORTEX_A57; 820 } 821 break; 822 case ARM_CPU_IMP_APM: 823 switch (part_number) { 824 case APM_CPU_PART_POTENZA: 825 return KVM_ARM_TARGET_XGENE_POTENZA; 826 } 827 break; 828 } 829 830 /* Return a default generic target */ 831 return KVM_ARM_TARGET_GENERIC_V8; 832 } 833 834 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) 835 { 836 int target = kvm_target_cpu(); 837 838 if (target < 0) 839 return -ENODEV; 840 841 memset(init, 0, sizeof(*init)); 842 843 /* 844 * For now, we don't return any features. 845 * In future, we might use features to return target 846 * specific features available for the preferred 847 * target type. 848 */ 849 init->target = (__u32)target; 850 851 return 0; 852 } 853 854 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 855 { 856 return -EINVAL; 857 } 858 859 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 860 { 861 return -EINVAL; 862 } 863 864 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 865 struct kvm_translation *tr) 866 { 867 return -EINVAL; 868 } 869 870 #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ 871 KVM_GUESTDBG_USE_SW_BP | \ 872 KVM_GUESTDBG_USE_HW | \ 873 KVM_GUESTDBG_SINGLESTEP) 874 875 /** 876 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging 877 * @kvm: pointer to the KVM struct 878 * @kvm_guest_debug: the ioctl data buffer 879 * 880 * This sets up and enables the VM for guest debugging. Userspace 881 * passes in a control flag to enable different debug types and 882 * potentially other architecture specific information in the rest of 883 * the structure. 884 */ 885 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 886 struct kvm_guest_debug *dbg) 887 { 888 int ret = 0; 889 890 trace_kvm_set_guest_debug(vcpu, dbg->control); 891 892 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) { 893 ret = -EINVAL; 894 goto out; 895 } 896 897 if (dbg->control & KVM_GUESTDBG_ENABLE) { 898 vcpu->guest_debug = dbg->control; 899 900 /* Hardware assisted Break and Watch points */ 901 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) { 902 vcpu->arch.external_debug_state = dbg->arch; 903 } 904 905 } else { 906 /* If not enabled clear all flags */ 907 vcpu->guest_debug = 0; 908 } 909 910 out: 911 return ret; 912 } 913 914 int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, 915 struct kvm_device_attr *attr) 916 { 917 int ret; 918 919 switch (attr->group) { 920 case KVM_ARM_VCPU_PMU_V3_CTRL: 921 ret = kvm_arm_pmu_v3_set_attr(vcpu, attr); 922 break; 923 case KVM_ARM_VCPU_TIMER_CTRL: 924 ret = kvm_arm_timer_set_attr(vcpu, attr); 925 break; 926 case KVM_ARM_VCPU_PVTIME_CTRL: 927 ret = kvm_arm_pvtime_set_attr(vcpu, attr); 928 break; 929 default: 930 ret = -ENXIO; 931 break; 932 } 933 934 return ret; 935 } 936 937 int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, 938 struct kvm_device_attr *attr) 939 { 940 int ret; 941 942 switch (attr->group) { 943 case KVM_ARM_VCPU_PMU_V3_CTRL: 944 ret = kvm_arm_pmu_v3_get_attr(vcpu, attr); 945 break; 946 case KVM_ARM_VCPU_TIMER_CTRL: 947 ret = kvm_arm_timer_get_attr(vcpu, attr); 948 break; 949 case KVM_ARM_VCPU_PVTIME_CTRL: 950 ret = kvm_arm_pvtime_get_attr(vcpu, attr); 951 break; 952 default: 953 ret = -ENXIO; 954 break; 955 } 956 957 return ret; 958 } 959 960 int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, 961 struct kvm_device_attr *attr) 962 { 963 int ret; 964 965 switch (attr->group) { 966 case KVM_ARM_VCPU_PMU_V3_CTRL: 967 ret = kvm_arm_pmu_v3_has_attr(vcpu, attr); 968 break; 969 case KVM_ARM_VCPU_TIMER_CTRL: 970 ret = kvm_arm_timer_has_attr(vcpu, attr); 971 break; 972 case KVM_ARM_VCPU_PVTIME_CTRL: 973 ret = kvm_arm_pvtime_has_attr(vcpu, attr); 974 break; 975 default: 976 ret = -ENXIO; 977 break; 978 } 979 980 return ret; 981 } 982