1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 * 6 * Derived from arch/arm/kvm/guest.c: 7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 8 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 9 */ 10 11 #include <linux/bits.h> 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/nospec.h> 15 #include <linux/kvm_host.h> 16 #include <linux/module.h> 17 #include <linux/stddef.h> 18 #include <linux/string.h> 19 #include <linux/vmalloc.h> 20 #include <linux/fs.h> 21 #include <kvm/arm_psci.h> 22 #include <asm/cputype.h> 23 #include <linux/uaccess.h> 24 #include <asm/fpsimd.h> 25 #include <asm/kvm.h> 26 #include <asm/kvm_emulate.h> 27 #include <asm/sigcontext.h> 28 29 #include "trace.h" 30 31 struct kvm_stats_debugfs_item debugfs_entries[] = { 32 VCPU_STAT("halt_successful_poll", halt_successful_poll), 33 VCPU_STAT("halt_attempted_poll", halt_attempted_poll), 34 VCPU_STAT("halt_poll_invalid", halt_poll_invalid), 35 VCPU_STAT("halt_wakeup", halt_wakeup), 36 VCPU_STAT("hvc_exit_stat", hvc_exit_stat), 37 VCPU_STAT("wfe_exit_stat", wfe_exit_stat), 38 VCPU_STAT("wfi_exit_stat", wfi_exit_stat), 39 VCPU_STAT("mmio_exit_user", mmio_exit_user), 40 VCPU_STAT("mmio_exit_kernel", mmio_exit_kernel), 41 VCPU_STAT("exits", exits), 42 VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), 43 VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), 44 { NULL } 45 }; 46 47 static bool core_reg_offset_is_vreg(u64 off) 48 { 49 return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && 50 off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr); 51 } 52 53 static u64 core_reg_offset_from_id(u64 id) 54 { 55 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE); 56 } 57 58 static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off) 59 { 60 int size; 61 62 switch (off) { 63 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 64 KVM_REG_ARM_CORE_REG(regs.regs[30]): 65 case KVM_REG_ARM_CORE_REG(regs.sp): 66 case KVM_REG_ARM_CORE_REG(regs.pc): 67 case KVM_REG_ARM_CORE_REG(regs.pstate): 68 case KVM_REG_ARM_CORE_REG(sp_el1): 69 case KVM_REG_ARM_CORE_REG(elr_el1): 70 case KVM_REG_ARM_CORE_REG(spsr[0]) ... 71 KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]): 72 size = sizeof(__u64); 73 break; 74 75 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 76 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 77 size = sizeof(__uint128_t); 78 break; 79 80 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 81 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 82 size = sizeof(__u32); 83 break; 84 85 default: 86 return -EINVAL; 87 } 88 89 if (!IS_ALIGNED(off, size / sizeof(__u32))) 90 return -EINVAL; 91 92 /* 93 * The KVM_REG_ARM64_SVE regs must be used instead of 94 * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on 95 * SVE-enabled vcpus: 96 */ 97 if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off)) 98 return -EINVAL; 99 100 return size; 101 } 102 103 static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 104 { 105 u64 off = core_reg_offset_from_id(reg->id); 106 int size = core_reg_size_from_offset(vcpu, off); 107 108 if (size < 0) 109 return NULL; 110 111 if (KVM_REG_SIZE(reg->id) != size) 112 return NULL; 113 114 switch (off) { 115 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 116 KVM_REG_ARM_CORE_REG(regs.regs[30]): 117 off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); 118 off /= 2; 119 return &vcpu->arch.ctxt.regs.regs[off]; 120 121 case KVM_REG_ARM_CORE_REG(regs.sp): 122 return &vcpu->arch.ctxt.regs.sp; 123 124 case KVM_REG_ARM_CORE_REG(regs.pc): 125 return &vcpu->arch.ctxt.regs.pc; 126 127 case KVM_REG_ARM_CORE_REG(regs.pstate): 128 return &vcpu->arch.ctxt.regs.pstate; 129 130 case KVM_REG_ARM_CORE_REG(sp_el1): 131 return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); 132 133 case KVM_REG_ARM_CORE_REG(elr_el1): 134 return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); 135 136 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]): 137 return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1); 138 139 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]): 140 return &vcpu->arch.ctxt.spsr_abt; 141 142 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]): 143 return &vcpu->arch.ctxt.spsr_und; 144 145 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]): 146 return &vcpu->arch.ctxt.spsr_irq; 147 148 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]): 149 return &vcpu->arch.ctxt.spsr_fiq; 150 151 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 152 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 153 off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); 154 off /= 4; 155 return &vcpu->arch.ctxt.fp_regs.vregs[off]; 156 157 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 158 return &vcpu->arch.ctxt.fp_regs.fpsr; 159 160 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 161 return &vcpu->arch.ctxt.fp_regs.fpcr; 162 163 default: 164 return NULL; 165 } 166 } 167 168 static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 169 { 170 /* 171 * Because the kvm_regs structure is a mix of 32, 64 and 172 * 128bit fields, we index it as if it was a 32bit 173 * array. Hence below, nr_regs is the number of entries, and 174 * off the index in the "array". 175 */ 176 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 177 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 178 void *addr; 179 u32 off; 180 181 /* Our ID is an index into the kvm_regs struct. */ 182 off = core_reg_offset_from_id(reg->id); 183 if (off >= nr_regs || 184 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 185 return -ENOENT; 186 187 addr = core_reg_addr(vcpu, reg); 188 if (!addr) 189 return -EINVAL; 190 191 if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id))) 192 return -EFAULT; 193 194 return 0; 195 } 196 197 static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 198 { 199 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 200 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 201 __uint128_t tmp; 202 void *valp = &tmp, *addr; 203 u64 off; 204 int err = 0; 205 206 /* Our ID is an index into the kvm_regs struct. */ 207 off = core_reg_offset_from_id(reg->id); 208 if (off >= nr_regs || 209 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 210 return -ENOENT; 211 212 addr = core_reg_addr(vcpu, reg); 213 if (!addr) 214 return -EINVAL; 215 216 if (KVM_REG_SIZE(reg->id) > sizeof(tmp)) 217 return -EINVAL; 218 219 if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) { 220 err = -EFAULT; 221 goto out; 222 } 223 224 if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) { 225 u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK; 226 switch (mode) { 227 case PSR_AA32_MODE_USR: 228 if (!system_supports_32bit_el0()) 229 return -EINVAL; 230 break; 231 case PSR_AA32_MODE_FIQ: 232 case PSR_AA32_MODE_IRQ: 233 case PSR_AA32_MODE_SVC: 234 case PSR_AA32_MODE_ABT: 235 case PSR_AA32_MODE_UND: 236 if (!vcpu_el1_is_32bit(vcpu)) 237 return -EINVAL; 238 break; 239 case PSR_MODE_EL0t: 240 case PSR_MODE_EL1t: 241 case PSR_MODE_EL1h: 242 if (vcpu_el1_is_32bit(vcpu)) 243 return -EINVAL; 244 break; 245 default: 246 err = -EINVAL; 247 goto out; 248 } 249 } 250 251 memcpy(addr, valp, KVM_REG_SIZE(reg->id)); 252 253 if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) { 254 int i, nr_reg; 255 256 switch (*vcpu_cpsr(vcpu)) { 257 /* 258 * Either we are dealing with user mode, and only the 259 * first 15 registers (+ PC) must be narrowed to 32bit. 260 * AArch32 r0-r14 conveniently map to AArch64 x0-x14. 261 */ 262 case PSR_AA32_MODE_USR: 263 case PSR_AA32_MODE_SYS: 264 nr_reg = 15; 265 break; 266 267 /* 268 * Otherwide, this is a priviledged mode, and *all* the 269 * registers must be narrowed to 32bit. 270 */ 271 default: 272 nr_reg = 31; 273 break; 274 } 275 276 for (i = 0; i < nr_reg; i++) 277 vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i)); 278 279 *vcpu_pc(vcpu) = (u32)*vcpu_pc(vcpu); 280 } 281 out: 282 return err; 283 } 284 285 #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) 286 #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) 287 #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) 288 289 static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 290 { 291 unsigned int max_vq, vq; 292 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 293 294 if (!vcpu_has_sve(vcpu)) 295 return -ENOENT; 296 297 if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl))) 298 return -EINVAL; 299 300 memset(vqs, 0, sizeof(vqs)); 301 302 max_vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); 303 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 304 if (sve_vq_available(vq)) 305 vqs[vq_word(vq)] |= vq_mask(vq); 306 307 if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs))) 308 return -EFAULT; 309 310 return 0; 311 } 312 313 static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 314 { 315 unsigned int max_vq, vq; 316 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 317 318 if (!vcpu_has_sve(vcpu)) 319 return -ENOENT; 320 321 if (kvm_arm_vcpu_sve_finalized(vcpu)) 322 return -EPERM; /* too late! */ 323 324 if (WARN_ON(vcpu->arch.sve_state)) 325 return -EINVAL; 326 327 if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs))) 328 return -EFAULT; 329 330 max_vq = 0; 331 for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq) 332 if (vq_present(vqs, vq)) 333 max_vq = vq; 334 335 if (max_vq > sve_vq_from_vl(kvm_sve_max_vl)) 336 return -EINVAL; 337 338 /* 339 * Vector lengths supported by the host can't currently be 340 * hidden from the guest individually: instead we can only set a 341 * maximum via ZCR_EL2.LEN. So, make sure the available vector 342 * lengths match the set requested exactly up to the requested 343 * maximum: 344 */ 345 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 346 if (vq_present(vqs, vq) != sve_vq_available(vq)) 347 return -EINVAL; 348 349 /* Can't run with no vector lengths at all: */ 350 if (max_vq < SVE_VQ_MIN) 351 return -EINVAL; 352 353 /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */ 354 vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq); 355 356 return 0; 357 } 358 359 #define SVE_REG_SLICE_SHIFT 0 360 #define SVE_REG_SLICE_BITS 5 361 #define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS) 362 #define SVE_REG_ID_BITS 5 363 364 #define SVE_REG_SLICE_MASK \ 365 GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \ 366 SVE_REG_SLICE_SHIFT) 367 #define SVE_REG_ID_MASK \ 368 GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT) 369 370 #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS) 371 372 #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0)) 373 #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0)) 374 375 /* 376 * Number of register slices required to cover each whole SVE register. 377 * NOTE: Only the first slice every exists, for now. 378 * If you are tempted to modify this, you must also rework sve_reg_to_region() 379 * to match: 380 */ 381 #define vcpu_sve_slices(vcpu) 1 382 383 /* Bounds of a single SVE register slice within vcpu->arch.sve_state */ 384 struct sve_state_reg_region { 385 unsigned int koffset; /* offset into sve_state in kernel memory */ 386 unsigned int klen; /* length in kernel memory */ 387 unsigned int upad; /* extra trailing padding in user memory */ 388 }; 389 390 /* 391 * Validate SVE register ID and get sanitised bounds for user/kernel SVE 392 * register copy 393 */ 394 static int sve_reg_to_region(struct sve_state_reg_region *region, 395 struct kvm_vcpu *vcpu, 396 const struct kvm_one_reg *reg) 397 { 398 /* reg ID ranges for Z- registers */ 399 const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0); 400 const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1, 401 SVE_NUM_SLICES - 1); 402 403 /* reg ID ranges for P- registers and FFR (which are contiguous) */ 404 const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0); 405 const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1); 406 407 unsigned int vq; 408 unsigned int reg_num; 409 410 unsigned int reqoffset, reqlen; /* User-requested offset and length */ 411 unsigned int maxlen; /* Maximum permitted length */ 412 413 size_t sve_state_size; 414 415 const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1, 416 SVE_NUM_SLICES - 1); 417 418 /* Verify that the P-regs and FFR really do have contiguous IDs: */ 419 BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1); 420 421 /* Verify that we match the UAPI header: */ 422 BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES); 423 424 reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT; 425 426 if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) { 427 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 428 return -ENOENT; 429 430 vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); 431 432 reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) - 433 SVE_SIG_REGS_OFFSET; 434 reqlen = KVM_SVE_ZREG_SIZE; 435 maxlen = SVE_SIG_ZREG_SIZE(vq); 436 } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) { 437 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 438 return -ENOENT; 439 440 vq = sve_vq_from_vl(vcpu->arch.sve_max_vl); 441 442 reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) - 443 SVE_SIG_REGS_OFFSET; 444 reqlen = KVM_SVE_PREG_SIZE; 445 maxlen = SVE_SIG_PREG_SIZE(vq); 446 } else { 447 return -EINVAL; 448 } 449 450 sve_state_size = vcpu_sve_state_size(vcpu); 451 if (WARN_ON(!sve_state_size)) 452 return -EINVAL; 453 454 region->koffset = array_index_nospec(reqoffset, sve_state_size); 455 region->klen = min(maxlen, reqlen); 456 region->upad = reqlen - region->klen; 457 458 return 0; 459 } 460 461 static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 462 { 463 int ret; 464 struct sve_state_reg_region region; 465 char __user *uptr = (char __user *)reg->addr; 466 467 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 468 if (reg->id == KVM_REG_ARM64_SVE_VLS) 469 return get_sve_vls(vcpu, reg); 470 471 /* Try to interpret reg ID as an architectural SVE register... */ 472 ret = sve_reg_to_region(®ion, vcpu, reg); 473 if (ret) 474 return ret; 475 476 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 477 return -EPERM; 478 479 if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, 480 region.klen) || 481 clear_user(uptr + region.klen, region.upad)) 482 return -EFAULT; 483 484 return 0; 485 } 486 487 static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 488 { 489 int ret; 490 struct sve_state_reg_region region; 491 const char __user *uptr = (const char __user *)reg->addr; 492 493 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 494 if (reg->id == KVM_REG_ARM64_SVE_VLS) 495 return set_sve_vls(vcpu, reg); 496 497 /* Try to interpret reg ID as an architectural SVE register... */ 498 ret = sve_reg_to_region(®ion, vcpu, reg); 499 if (ret) 500 return ret; 501 502 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 503 return -EPERM; 504 505 if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, 506 region.klen)) 507 return -EFAULT; 508 509 return 0; 510 } 511 512 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 513 { 514 return -EINVAL; 515 } 516 517 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 518 { 519 return -EINVAL; 520 } 521 522 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, 523 u64 __user *uindices) 524 { 525 unsigned int i; 526 int n = 0; 527 528 for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) { 529 u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i; 530 int size = core_reg_size_from_offset(vcpu, i); 531 532 if (size < 0) 533 continue; 534 535 switch (size) { 536 case sizeof(__u32): 537 reg |= KVM_REG_SIZE_U32; 538 break; 539 540 case sizeof(__u64): 541 reg |= KVM_REG_SIZE_U64; 542 break; 543 544 case sizeof(__uint128_t): 545 reg |= KVM_REG_SIZE_U128; 546 break; 547 548 default: 549 WARN_ON(1); 550 continue; 551 } 552 553 if (uindices) { 554 if (put_user(reg, uindices)) 555 return -EFAULT; 556 uindices++; 557 } 558 559 n++; 560 } 561 562 return n; 563 } 564 565 static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) 566 { 567 return copy_core_reg_indices(vcpu, NULL); 568 } 569 570 /** 571 * ARM64 versions of the TIMER registers, always available on arm64 572 */ 573 574 #define NUM_TIMER_REGS 3 575 576 static bool is_timer_reg(u64 index) 577 { 578 switch (index) { 579 case KVM_REG_ARM_TIMER_CTL: 580 case KVM_REG_ARM_TIMER_CNT: 581 case KVM_REG_ARM_TIMER_CVAL: 582 return true; 583 } 584 return false; 585 } 586 587 static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 588 { 589 if (put_user(KVM_REG_ARM_TIMER_CTL, uindices)) 590 return -EFAULT; 591 uindices++; 592 if (put_user(KVM_REG_ARM_TIMER_CNT, uindices)) 593 return -EFAULT; 594 uindices++; 595 if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices)) 596 return -EFAULT; 597 598 return 0; 599 } 600 601 static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 602 { 603 void __user *uaddr = (void __user *)(long)reg->addr; 604 u64 val; 605 int ret; 606 607 ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id)); 608 if (ret != 0) 609 return -EFAULT; 610 611 return kvm_arm_timer_set_reg(vcpu, reg->id, val); 612 } 613 614 static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 615 { 616 void __user *uaddr = (void __user *)(long)reg->addr; 617 u64 val; 618 619 val = kvm_arm_timer_get_reg(vcpu, reg->id); 620 return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0; 621 } 622 623 static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) 624 { 625 const unsigned int slices = vcpu_sve_slices(vcpu); 626 627 if (!vcpu_has_sve(vcpu)) 628 return 0; 629 630 /* Policed by KVM_GET_REG_LIST: */ 631 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 632 633 return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */) 634 + 1; /* KVM_REG_ARM64_SVE_VLS */ 635 } 636 637 static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu, 638 u64 __user *uindices) 639 { 640 const unsigned int slices = vcpu_sve_slices(vcpu); 641 u64 reg; 642 unsigned int i, n; 643 int num_regs = 0; 644 645 if (!vcpu_has_sve(vcpu)) 646 return 0; 647 648 /* Policed by KVM_GET_REG_LIST: */ 649 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 650 651 /* 652 * Enumerate this first, so that userspace can save/restore in 653 * the order reported by KVM_GET_REG_LIST: 654 */ 655 reg = KVM_REG_ARM64_SVE_VLS; 656 if (put_user(reg, uindices++)) 657 return -EFAULT; 658 ++num_regs; 659 660 for (i = 0; i < slices; i++) { 661 for (n = 0; n < SVE_NUM_ZREGS; n++) { 662 reg = KVM_REG_ARM64_SVE_ZREG(n, i); 663 if (put_user(reg, uindices++)) 664 return -EFAULT; 665 num_regs++; 666 } 667 668 for (n = 0; n < SVE_NUM_PREGS; n++) { 669 reg = KVM_REG_ARM64_SVE_PREG(n, i); 670 if (put_user(reg, uindices++)) 671 return -EFAULT; 672 num_regs++; 673 } 674 675 reg = KVM_REG_ARM64_SVE_FFR(i); 676 if (put_user(reg, uindices++)) 677 return -EFAULT; 678 num_regs++; 679 } 680 681 return num_regs; 682 } 683 684 /** 685 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG 686 * 687 * This is for all registers. 688 */ 689 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) 690 { 691 unsigned long res = 0; 692 693 res += num_core_regs(vcpu); 694 res += num_sve_regs(vcpu); 695 res += kvm_arm_num_sys_reg_descs(vcpu); 696 res += kvm_arm_get_fw_num_regs(vcpu); 697 res += NUM_TIMER_REGS; 698 699 return res; 700 } 701 702 /** 703 * kvm_arm_copy_reg_indices - get indices of all registers. 704 * 705 * We do core registers right here, then we append system regs. 706 */ 707 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 708 { 709 int ret; 710 711 ret = copy_core_reg_indices(vcpu, uindices); 712 if (ret < 0) 713 return ret; 714 uindices += ret; 715 716 ret = copy_sve_reg_indices(vcpu, uindices); 717 if (ret < 0) 718 return ret; 719 uindices += ret; 720 721 ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices); 722 if (ret < 0) 723 return ret; 724 uindices += kvm_arm_get_fw_num_regs(vcpu); 725 726 ret = copy_timer_indices(vcpu, uindices); 727 if (ret < 0) 728 return ret; 729 uindices += NUM_TIMER_REGS; 730 731 return kvm_arm_copy_sys_reg_indices(vcpu, uindices); 732 } 733 734 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 735 { 736 /* We currently use nothing arch-specific in upper 32 bits */ 737 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 738 return -EINVAL; 739 740 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 741 case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg); 742 case KVM_REG_ARM_FW: return kvm_arm_get_fw_reg(vcpu, reg); 743 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg); 744 } 745 746 if (is_timer_reg(reg->id)) 747 return get_timer_reg(vcpu, reg); 748 749 return kvm_arm_sys_reg_get_reg(vcpu, reg); 750 } 751 752 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 753 { 754 /* We currently use nothing arch-specific in upper 32 bits */ 755 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 756 return -EINVAL; 757 758 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 759 case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg); 760 case KVM_REG_ARM_FW: return kvm_arm_set_fw_reg(vcpu, reg); 761 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg); 762 } 763 764 if (is_timer_reg(reg->id)) 765 return set_timer_reg(vcpu, reg); 766 767 return kvm_arm_sys_reg_set_reg(vcpu, reg); 768 } 769 770 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 771 struct kvm_sregs *sregs) 772 { 773 return -EINVAL; 774 } 775 776 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 777 struct kvm_sregs *sregs) 778 { 779 return -EINVAL; 780 } 781 782 int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 783 struct kvm_vcpu_events *events) 784 { 785 events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE); 786 events->exception.serror_has_esr = cpus_have_const_cap(ARM64_HAS_RAS_EXTN); 787 788 if (events->exception.serror_pending && events->exception.serror_has_esr) 789 events->exception.serror_esr = vcpu_get_vsesr(vcpu); 790 791 /* 792 * We never return a pending ext_dabt here because we deliver it to 793 * the virtual CPU directly when setting the event and it's no longer 794 * 'pending' at this point. 795 */ 796 797 return 0; 798 } 799 800 int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 801 struct kvm_vcpu_events *events) 802 { 803 bool serror_pending = events->exception.serror_pending; 804 bool has_esr = events->exception.serror_has_esr; 805 bool ext_dabt_pending = events->exception.ext_dabt_pending; 806 807 if (serror_pending && has_esr) { 808 if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) 809 return -EINVAL; 810 811 if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK)) 812 kvm_set_sei_esr(vcpu, events->exception.serror_esr); 813 else 814 return -EINVAL; 815 } else if (serror_pending) { 816 kvm_inject_vabt(vcpu); 817 } 818 819 if (ext_dabt_pending) 820 kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); 821 822 return 0; 823 } 824 825 int __attribute_const__ kvm_target_cpu(void) 826 { 827 unsigned long implementor = read_cpuid_implementor(); 828 unsigned long part_number = read_cpuid_part_number(); 829 830 switch (implementor) { 831 case ARM_CPU_IMP_ARM: 832 switch (part_number) { 833 case ARM_CPU_PART_AEM_V8: 834 return KVM_ARM_TARGET_AEM_V8; 835 case ARM_CPU_PART_FOUNDATION: 836 return KVM_ARM_TARGET_FOUNDATION_V8; 837 case ARM_CPU_PART_CORTEX_A53: 838 return KVM_ARM_TARGET_CORTEX_A53; 839 case ARM_CPU_PART_CORTEX_A57: 840 return KVM_ARM_TARGET_CORTEX_A57; 841 } 842 break; 843 case ARM_CPU_IMP_APM: 844 switch (part_number) { 845 case APM_CPU_PART_POTENZA: 846 return KVM_ARM_TARGET_XGENE_POTENZA; 847 } 848 break; 849 } 850 851 /* Return a default generic target */ 852 return KVM_ARM_TARGET_GENERIC_V8; 853 } 854 855 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) 856 { 857 int target = kvm_target_cpu(); 858 859 if (target < 0) 860 return -ENODEV; 861 862 memset(init, 0, sizeof(*init)); 863 864 /* 865 * For now, we don't return any features. 866 * In future, we might use features to return target 867 * specific features available for the preferred 868 * target type. 869 */ 870 init->target = (__u32)target; 871 872 return 0; 873 } 874 875 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 876 { 877 return -EINVAL; 878 } 879 880 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 881 { 882 return -EINVAL; 883 } 884 885 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 886 struct kvm_translation *tr) 887 { 888 return -EINVAL; 889 } 890 891 #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ 892 KVM_GUESTDBG_USE_SW_BP | \ 893 KVM_GUESTDBG_USE_HW | \ 894 KVM_GUESTDBG_SINGLESTEP) 895 896 /** 897 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging 898 * @kvm: pointer to the KVM struct 899 * @kvm_guest_debug: the ioctl data buffer 900 * 901 * This sets up and enables the VM for guest debugging. Userspace 902 * passes in a control flag to enable different debug types and 903 * potentially other architecture specific information in the rest of 904 * the structure. 905 */ 906 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 907 struct kvm_guest_debug *dbg) 908 { 909 int ret = 0; 910 911 trace_kvm_set_guest_debug(vcpu, dbg->control); 912 913 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) { 914 ret = -EINVAL; 915 goto out; 916 } 917 918 if (dbg->control & KVM_GUESTDBG_ENABLE) { 919 vcpu->guest_debug = dbg->control; 920 921 /* Hardware assisted Break and Watch points */ 922 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) { 923 vcpu->arch.external_debug_state = dbg->arch; 924 } 925 926 } else { 927 /* If not enabled clear all flags */ 928 vcpu->guest_debug = 0; 929 } 930 931 out: 932 return ret; 933 } 934 935 int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, 936 struct kvm_device_attr *attr) 937 { 938 int ret; 939 940 switch (attr->group) { 941 case KVM_ARM_VCPU_PMU_V3_CTRL: 942 ret = kvm_arm_pmu_v3_set_attr(vcpu, attr); 943 break; 944 case KVM_ARM_VCPU_TIMER_CTRL: 945 ret = kvm_arm_timer_set_attr(vcpu, attr); 946 break; 947 case KVM_ARM_VCPU_PVTIME_CTRL: 948 ret = kvm_arm_pvtime_set_attr(vcpu, attr); 949 break; 950 default: 951 ret = -ENXIO; 952 break; 953 } 954 955 return ret; 956 } 957 958 int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, 959 struct kvm_device_attr *attr) 960 { 961 int ret; 962 963 switch (attr->group) { 964 case KVM_ARM_VCPU_PMU_V3_CTRL: 965 ret = kvm_arm_pmu_v3_get_attr(vcpu, attr); 966 break; 967 case KVM_ARM_VCPU_TIMER_CTRL: 968 ret = kvm_arm_timer_get_attr(vcpu, attr); 969 break; 970 case KVM_ARM_VCPU_PVTIME_CTRL: 971 ret = kvm_arm_pvtime_get_attr(vcpu, attr); 972 break; 973 default: 974 ret = -ENXIO; 975 break; 976 } 977 978 return ret; 979 } 980 981 int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, 982 struct kvm_device_attr *attr) 983 { 984 int ret; 985 986 switch (attr->group) { 987 case KVM_ARM_VCPU_PMU_V3_CTRL: 988 ret = kvm_arm_pmu_v3_has_attr(vcpu, attr); 989 break; 990 case KVM_ARM_VCPU_TIMER_CTRL: 991 ret = kvm_arm_timer_has_attr(vcpu, attr); 992 break; 993 case KVM_ARM_VCPU_PVTIME_CTRL: 994 ret = kvm_arm_pvtime_has_attr(vcpu, attr); 995 break; 996 default: 997 ret = -ENXIO; 998 break; 999 } 1000 1001 return ret; 1002 } 1003