1 /* 2 * ARM implementation of KVM hooks 3 * 4 * Copyright Christoffer Dall 2009-2010 5 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems 6 * Copyright Alex Bennée 2014, Linaro 7 * 8 * This work is licensed under the terms of the GNU GPL, version 2 or later. 9 * See the COPYING file in the top-level directory. 10 * 11 */ 12 13 #include "qemu/osdep.h" 14 #include <sys/ioctl.h> 15 16 #include <linux/kvm.h> 17 18 #include "qemu/timer.h" 19 #include "qemu/error-report.h" 20 #include "qemu/main-loop.h" 21 #include "qom/object.h" 22 #include "qapi/error.h" 23 #include "system/system.h" 24 #include "system/runstate.h" 25 #include "system/kvm.h" 26 #include "system/kvm_int.h" 27 #include "kvm_arm.h" 28 #include "cpu.h" 29 #include "cpu-sysregs.h" 30 #include "trace.h" 31 #include "internals.h" 32 #include "hw/pci/pci.h" 33 #include "exec/memattrs.h" 34 #include "system/address-spaces.h" 35 #include "gdbstub/enums.h" 36 #include "hw/boards.h" 37 #include "hw/irq.h" 38 #include "qapi/visitor.h" 39 #include "qemu/log.h" 40 #include "hw/acpi/acpi.h" 41 #include "hw/acpi/ghes.h" 42 #include "target/arm/gtimer.h" 43 #include "migration/blocker.h" 44 45 const KVMCapabilityInfo kvm_arch_required_capabilities[] = { 46 KVM_CAP_INFO(DEVICE_CTRL), 47 KVM_CAP_LAST_INFO 48 }; 49 50 static bool cap_has_mp_state; 51 static bool cap_has_inject_serror_esr; 52 static bool cap_has_inject_ext_dabt; 53 54 /** 55 * ARMHostCPUFeatures: information about the host CPU (identified 56 * by asking the host kernel) 57 */ 58 typedef struct ARMHostCPUFeatures { 59 ARMISARegisters isar; 60 uint64_t features; 61 uint32_t target; 62 const char *dtb_compatible; 63 } ARMHostCPUFeatures; 64 65 static ARMHostCPUFeatures arm_host_cpu_features; 66 67 /** 68 * kvm_arm_vcpu_init: 69 * @cpu: ARMCPU 70 * 71 * Initialize (or reinitialize) the VCPU by invoking the 72 * KVM_ARM_VCPU_INIT ioctl with the CPU type and feature 73 * bitmask specified in the CPUState. 74 * 75 * Returns: 0 if success else < 0 error code 76 */ 77 static int kvm_arm_vcpu_init(ARMCPU *cpu) 78 { 79 struct kvm_vcpu_init init; 80 81 init.target = cpu->kvm_target; 82 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features)); 83 84 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_INIT, &init); 85 } 86 87 /** 88 * kvm_arm_vcpu_finalize: 89 * @cpu: ARMCPU 90 * @feature: feature to finalize 91 * 92 * Finalizes the configuration of the specified VCPU feature by 93 * invoking the KVM_ARM_VCPU_FINALIZE ioctl. Features requiring 94 * this are documented in the "KVM_ARM_VCPU_FINALIZE" section of 95 * KVM's API documentation. 96 * 97 * Returns: 0 if success else < 0 error code 98 */ 99 static int kvm_arm_vcpu_finalize(ARMCPU *cpu, int feature) 100 { 101 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_FINALIZE, &feature); 102 } 103 104 bool kvm_arm_create_scratch_host_vcpu(int *fdarray, 105 struct kvm_vcpu_init *init) 106 { 107 int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1; 108 int max_vm_pa_size; 109 110 kvmfd = qemu_open_old("/dev/kvm", O_RDWR); 111 if (kvmfd < 0) { 112 goto err; 113 } 114 max_vm_pa_size = ioctl(kvmfd, KVM_CHECK_EXTENSION, KVM_CAP_ARM_VM_IPA_SIZE); 115 if (max_vm_pa_size < 0) { 116 max_vm_pa_size = 0; 117 } 118 do { 119 vmfd = ioctl(kvmfd, KVM_CREATE_VM, max_vm_pa_size); 120 } while (vmfd == -1 && errno == EINTR); 121 if (vmfd < 0) { 122 goto err; 123 } 124 125 /* 126 * The MTE capability must be enabled by the VMM before creating 127 * any VCPUs in order to allow the MTE bits of the ID_AA64PFR1 128 * register to be probed correctly, as they are masked if MTE 129 * is not enabled. 130 */ 131 if (kvm_arm_mte_supported()) { 132 KVMState kvm_state; 133 134 kvm_state.fd = kvmfd; 135 kvm_state.vmfd = vmfd; 136 kvm_vm_enable_cap(&kvm_state, KVM_CAP_ARM_MTE, 0); 137 } 138 139 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0); 140 if (cpufd < 0) { 141 goto err; 142 } 143 144 if (!init) { 145 /* Caller doesn't want the VCPU to be initialized, so skip it */ 146 goto finish; 147 } 148 149 if (init->target == -1) { 150 struct kvm_vcpu_init preferred; 151 152 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred); 153 if (ret < 0) { 154 goto err; 155 } 156 init->target = preferred.target; 157 } 158 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init); 159 if (ret < 0) { 160 goto err; 161 } 162 163 finish: 164 fdarray[0] = kvmfd; 165 fdarray[1] = vmfd; 166 fdarray[2] = cpufd; 167 168 return true; 169 170 err: 171 if (cpufd >= 0) { 172 close(cpufd); 173 } 174 if (vmfd >= 0) { 175 close(vmfd); 176 } 177 if (kvmfd >= 0) { 178 close(kvmfd); 179 } 180 181 return false; 182 } 183 184 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray) 185 { 186 int i; 187 188 for (i = 2; i >= 0; i--) { 189 close(fdarray[i]); 190 } 191 } 192 193 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id) 194 { 195 uint64_t ret; 196 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret }; 197 int err; 198 199 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); 200 err = ioctl(fd, KVM_GET_ONE_REG, &idreg); 201 if (err < 0) { 202 return -1; 203 } 204 *pret = ret; 205 return 0; 206 } 207 208 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id) 209 { 210 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret }; 211 212 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); 213 return ioctl(fd, KVM_GET_ONE_REG, &idreg); 214 } 215 216 static bool kvm_arm_pauth_supported(void) 217 { 218 return (kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_ADDRESS) && 219 kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_GENERIC)); 220 } 221 222 223 static uint64_t idregs_sysreg_to_kvm_reg(ARMSysRegs sysreg) 224 { 225 return ARM64_SYS_REG((sysreg & CP_REG_ARM64_SYSREG_OP0_MASK) >> CP_REG_ARM64_SYSREG_OP0_SHIFT, 226 (sysreg & CP_REG_ARM64_SYSREG_OP1_MASK) >> CP_REG_ARM64_SYSREG_OP1_SHIFT, 227 (sysreg & CP_REG_ARM64_SYSREG_CRN_MASK) >> CP_REG_ARM64_SYSREG_CRN_SHIFT, 228 (sysreg & CP_REG_ARM64_SYSREG_CRM_MASK) >> CP_REG_ARM64_SYSREG_CRM_SHIFT, 229 (sysreg & CP_REG_ARM64_SYSREG_OP2_MASK) >> CP_REG_ARM64_SYSREG_OP2_SHIFT); 230 } 231 232 /* read a sysreg value and store it in the idregs */ 233 static int get_host_cpu_reg(int fd, ARMHostCPUFeatures *ahcf, 234 ARMIDRegisterIdx index) 235 { 236 uint64_t *reg; 237 int ret; 238 239 reg = &ahcf->isar.idregs[index]; 240 ret = read_sys_reg64(fd, reg, 241 idregs_sysreg_to_kvm_reg(id_register_sysreg[index])); 242 return ret; 243 } 244 245 static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) 246 { 247 /* Identify the feature bits corresponding to the host CPU, and 248 * fill out the ARMHostCPUClass fields accordingly. To do this 249 * we have to create a scratch VM, create a single CPU inside it, 250 * and then query that CPU for the relevant ID registers. 251 */ 252 int fdarray[3]; 253 bool sve_supported; 254 bool pmu_supported = false; 255 uint64_t features = 0; 256 int err; 257 258 /* 259 * target = -1 informs kvm_arm_create_scratch_host_vcpu() 260 * to use the preferred target 261 */ 262 struct kvm_vcpu_init init = { .target = -1, }; 263 264 /* 265 * Ask for SVE if supported, so that we can query ID_AA64ZFR0, 266 * which is otherwise RAZ. 267 */ 268 sve_supported = kvm_arm_sve_supported(); 269 if (sve_supported) { 270 init.features[0] |= 1 << KVM_ARM_VCPU_SVE; 271 } 272 273 /* 274 * Ask for Pointer Authentication if supported, so that we get 275 * the unsanitized field values for AA64ISAR1_EL1. 276 */ 277 if (kvm_arm_pauth_supported()) { 278 init.features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS | 279 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC); 280 } 281 282 if (kvm_arm_pmu_supported()) { 283 init.features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; 284 pmu_supported = true; 285 features |= 1ULL << ARM_FEATURE_PMU; 286 } 287 288 if (!kvm_arm_create_scratch_host_vcpu(fdarray, &init)) { 289 return false; 290 } 291 292 ahcf->target = init.target; 293 ahcf->dtb_compatible = "arm,armv8"; 294 int fd = fdarray[2]; 295 296 err = get_host_cpu_reg(fd, ahcf, ID_AA64PFR0_EL1_IDX); 297 if (unlikely(err < 0)) { 298 /* 299 * Before v4.15, the kernel only exposed a limited number of system 300 * registers, not including any of the interesting AArch64 ID regs. 301 * For the most part we could leave these fields as zero with minimal 302 * effect, since this does not affect the values seen by the guest. 303 * 304 * However, it could cause problems down the line for QEMU, 305 * so provide a minimal v8.0 default. 306 * 307 * ??? Could read MIDR and use knowledge from cpu64.c. 308 * ??? Could map a page of memory into our temp guest and 309 * run the tiniest of hand-crafted kernels to extract 310 * the values seen by the guest. 311 * ??? Either of these sounds like too much effort just 312 * to work around running a modern host kernel. 313 */ 314 SET_IDREG(&ahcf->isar, ID_AA64PFR0, 0x00000011); /* EL1&0, AArch64 only */ 315 err = 0; 316 } else { 317 err |= get_host_cpu_reg(fd, ahcf, ID_AA64PFR1_EL1_IDX); 318 err |= get_host_cpu_reg(fd, ahcf, ID_AA64SMFR0_EL1_IDX); 319 err |= get_host_cpu_reg(fd, ahcf, ID_AA64DFR0_EL1_IDX); 320 err |= get_host_cpu_reg(fd, ahcf, ID_AA64DFR1_EL1_IDX); 321 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR0_EL1_IDX); 322 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR1_EL1_IDX); 323 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR2_EL1_IDX); 324 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR0_EL1_IDX); 325 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR1_EL1_IDX); 326 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR2_EL1_IDX); 327 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR3_EL1_IDX); 328 329 /* 330 * Note that if AArch32 support is not present in the host, 331 * the AArch32 sysregs are present to be read, but will 332 * return UNKNOWN values. This is neither better nor worse 333 * than skipping the reads and leaving 0, as we must avoid 334 * considering the values in every case. 335 */ 336 err |= get_host_cpu_reg(fd, ahcf, ID_PFR0_EL1_IDX); 337 err |= get_host_cpu_reg(fd, ahcf, ID_PFR1_EL1_IDX); 338 err |= get_host_cpu_reg(fd, ahcf, ID_DFR0_EL1_IDX); 339 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR0_EL1_IDX); 340 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR1_EL1_IDX); 341 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR2_EL1_IDX); 342 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR3_EL1_IDX); 343 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR0_EL1_IDX); 344 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR1_EL1_IDX); 345 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR2_EL1_IDX); 346 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR3_EL1_IDX); 347 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR4_EL1_IDX); 348 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR5_EL1_IDX); 349 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR6_EL1_IDX); 350 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR4_EL1_IDX); 351 352 err |= read_sys_reg32(fd, &ahcf->isar.mvfr0, 353 ARM64_SYS_REG(3, 0, 0, 3, 0)); 354 err |= read_sys_reg32(fd, &ahcf->isar.mvfr1, 355 ARM64_SYS_REG(3, 0, 0, 3, 1)); 356 err |= read_sys_reg32(fd, &ahcf->isar.mvfr2, 357 ARM64_SYS_REG(3, 0, 0, 3, 2)); 358 err |= get_host_cpu_reg(fd, ahcf, ID_PFR2_EL1_IDX); 359 err |= get_host_cpu_reg(fd, ahcf, ID_DFR1_EL1_IDX); 360 err |= get_host_cpu_reg(fd, ahcf, ID_MMFR5_EL1_IDX); 361 362 /* 363 * DBGDIDR is a bit complicated because the kernel doesn't 364 * provide an accessor for it in 64-bit mode, which is what this 365 * scratch VM is in, and there's no architected "64-bit sysreg 366 * which reads the same as the 32-bit register" the way there is 367 * for other ID registers. Instead we synthesize a value from the 368 * AArch64 ID_AA64DFR0, the same way the kernel code in 369 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does. 370 * We only do this if the CPU supports AArch32 at EL1. 371 */ 372 if (FIELD_EX32_IDREG(&ahcf->isar, ID_AA64PFR0, EL1) >= 2) { 373 int wrps = FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, WRPS); 374 int brps = FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, BRPS); 375 int ctx_cmps = 376 FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, CTX_CMPS); 377 int version = 6; /* ARMv8 debug architecture */ 378 bool has_el3 = 379 !!FIELD_EX32_IDREG(&ahcf->isar, ID_AA64PFR0, EL3); 380 uint32_t dbgdidr = 0; 381 382 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps); 383 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps); 384 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps); 385 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version); 386 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3); 387 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3); 388 dbgdidr |= (1 << 15); /* RES1 bit */ 389 ahcf->isar.dbgdidr = dbgdidr; 390 } 391 392 if (pmu_supported) { 393 /* PMCR_EL0 is only accessible if the vCPU has feature PMU_V3 */ 394 err |= read_sys_reg64(fd, &ahcf->isar.reset_pmcr_el0, 395 ARM64_SYS_REG(3, 3, 9, 12, 0)); 396 } 397 398 if (sve_supported) { 399 /* 400 * There is a range of kernels between kernel commit 73433762fcae 401 * and f81cb2c3ad41 which have a bug where the kernel doesn't 402 * expose SYS_ID_AA64ZFR0_EL1 via the ONE_REG API unless the VM has 403 * enabled SVE support, which resulted in an error rather than RAZ. 404 * So only read the register if we set KVM_ARM_VCPU_SVE above. 405 */ 406 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ZFR0_EL1_IDX); 407 } 408 } 409 410 kvm_arm_destroy_scratch_host_vcpu(fdarray); 411 412 if (err < 0) { 413 return false; 414 } 415 416 /* 417 * We can assume any KVM supporting CPU is at least a v8 418 * with VFPv4+Neon; this in turn implies most of the other 419 * feature bits. 420 */ 421 features |= 1ULL << ARM_FEATURE_V8; 422 features |= 1ULL << ARM_FEATURE_NEON; 423 features |= 1ULL << ARM_FEATURE_AARCH64; 424 features |= 1ULL << ARM_FEATURE_GENERIC_TIMER; 425 426 ahcf->features = features; 427 428 return true; 429 } 430 431 void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu) 432 { 433 CPUARMState *env = &cpu->env; 434 435 if (!arm_host_cpu_features.dtb_compatible) { 436 if (!kvm_enabled() || 437 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) { 438 /* We can't report this error yet, so flag that we need to 439 * in arm_cpu_realizefn(). 440 */ 441 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE; 442 cpu->host_cpu_probe_failed = true; 443 return; 444 } 445 } 446 447 cpu->kvm_target = arm_host_cpu_features.target; 448 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible; 449 cpu->isar = arm_host_cpu_features.isar; 450 env->features = arm_host_cpu_features.features; 451 } 452 453 static bool kvm_no_adjvtime_get(Object *obj, Error **errp) 454 { 455 return !ARM_CPU(obj)->kvm_adjvtime; 456 } 457 458 static void kvm_no_adjvtime_set(Object *obj, bool value, Error **errp) 459 { 460 ARM_CPU(obj)->kvm_adjvtime = !value; 461 } 462 463 static bool kvm_steal_time_get(Object *obj, Error **errp) 464 { 465 return ARM_CPU(obj)->kvm_steal_time != ON_OFF_AUTO_OFF; 466 } 467 468 static void kvm_steal_time_set(Object *obj, bool value, Error **errp) 469 { 470 ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 471 } 472 473 /* KVM VCPU properties should be prefixed with "kvm-". */ 474 void kvm_arm_add_vcpu_properties(ARMCPU *cpu) 475 { 476 CPUARMState *env = &cpu->env; 477 Object *obj = OBJECT(cpu); 478 479 if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) { 480 cpu->kvm_adjvtime = true; 481 object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get, 482 kvm_no_adjvtime_set); 483 object_property_set_description(obj, "kvm-no-adjvtime", 484 "Set on to disable the adjustment of " 485 "the virtual counter. VM stopped time " 486 "will be counted."); 487 } 488 489 cpu->kvm_steal_time = ON_OFF_AUTO_AUTO; 490 object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get, 491 kvm_steal_time_set); 492 object_property_set_description(obj, "kvm-steal-time", 493 "Set off to disable KVM steal time."); 494 } 495 496 bool kvm_arm_pmu_supported(void) 497 { 498 return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3); 499 } 500 501 int kvm_arm_get_max_vm_ipa_size(MachineState *ms, bool *fixed_ipa) 502 { 503 KVMState *s = KVM_STATE(ms->accelerator); 504 int ret; 505 506 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE); 507 *fixed_ipa = ret <= 0; 508 509 return ret > 0 ? ret : 40; 510 } 511 512 int kvm_arch_get_default_type(MachineState *ms) 513 { 514 bool fixed_ipa; 515 int size = kvm_arm_get_max_vm_ipa_size(ms, &fixed_ipa); 516 return fixed_ipa ? 0 : size; 517 } 518 519 int kvm_arch_init(MachineState *ms, KVMState *s) 520 { 521 int ret = 0; 522 /* For ARM interrupt delivery is always asynchronous, 523 * whether we are using an in-kernel VGIC or not. 524 */ 525 kvm_async_interrupts_allowed = true; 526 527 /* 528 * PSCI wakes up secondary cores, so we always need to 529 * have vCPUs waiting in kernel space 530 */ 531 kvm_halt_in_kernel_allowed = true; 532 533 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE); 534 535 /* Check whether user space can specify guest syndrome value */ 536 cap_has_inject_serror_esr = 537 kvm_check_extension(s, KVM_CAP_ARM_INJECT_SERROR_ESR); 538 539 if (ms->smp.cpus > 256 && 540 !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) { 541 error_report("Using more than 256 vcpus requires a host kernel " 542 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2"); 543 ret = -EINVAL; 544 } 545 546 if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) { 547 if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) { 548 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap"); 549 } else { 550 /* Set status for supporting the external dabt injection */ 551 cap_has_inject_ext_dabt = kvm_check_extension(s, 552 KVM_CAP_ARM_INJECT_EXT_DABT); 553 } 554 } 555 556 if (s->kvm_eager_split_size) { 557 uint32_t sizes; 558 559 sizes = kvm_vm_check_extension(s, KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES); 560 if (!sizes) { 561 s->kvm_eager_split_size = 0; 562 warn_report("Eager Page Split support not available"); 563 } else if (!(s->kvm_eager_split_size & sizes)) { 564 error_report("Eager Page Split requested chunk size not valid"); 565 ret = -EINVAL; 566 } else { 567 ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE, 0, 568 s->kvm_eager_split_size); 569 if (ret < 0) { 570 error_report("Enabling of Eager Page Split failed: %s", 571 strerror(-ret)); 572 } 573 } 574 } 575 576 max_hw_wps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_WPS); 577 hw_watchpoints = g_array_sized_new(true, true, 578 sizeof(HWWatchpoint), max_hw_wps); 579 580 max_hw_bps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_BPS); 581 hw_breakpoints = g_array_sized_new(true, true, 582 sizeof(HWBreakpoint), max_hw_bps); 583 584 return ret; 585 } 586 587 unsigned long kvm_arch_vcpu_id(CPUState *cpu) 588 { 589 return cpu->cpu_index; 590 } 591 592 /* We track all the KVM devices which need their memory addresses 593 * passing to the kernel in a list of these structures. 594 * When board init is complete we run through the list and 595 * tell the kernel the base addresses of the memory regions. 596 * We use a MemoryListener to track mapping and unmapping of 597 * the regions during board creation, so the board models don't 598 * need to do anything special for the KVM case. 599 * 600 * Sometimes the address must be OR'ed with some other fields 601 * (for example for KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION). 602 * @kda_addr_ormask aims at storing the value of those fields. 603 */ 604 typedef struct KVMDevice { 605 struct kvm_arm_device_addr kda; 606 struct kvm_device_attr kdattr; 607 uint64_t kda_addr_ormask; 608 MemoryRegion *mr; 609 QSLIST_ENTRY(KVMDevice) entries; 610 int dev_fd; 611 } KVMDevice; 612 613 static QSLIST_HEAD(, KVMDevice) kvm_devices_head; 614 615 static void kvm_arm_devlistener_add(MemoryListener *listener, 616 MemoryRegionSection *section) 617 { 618 KVMDevice *kd; 619 620 QSLIST_FOREACH(kd, &kvm_devices_head, entries) { 621 if (section->mr == kd->mr) { 622 kd->kda.addr = section->offset_within_address_space; 623 } 624 } 625 } 626 627 static void kvm_arm_devlistener_del(MemoryListener *listener, 628 MemoryRegionSection *section) 629 { 630 KVMDevice *kd; 631 632 QSLIST_FOREACH(kd, &kvm_devices_head, entries) { 633 if (section->mr == kd->mr) { 634 kd->kda.addr = -1; 635 } 636 } 637 } 638 639 static MemoryListener devlistener = { 640 .name = "kvm-arm", 641 .region_add = kvm_arm_devlistener_add, 642 .region_del = kvm_arm_devlistener_del, 643 .priority = MEMORY_LISTENER_PRIORITY_MIN, 644 }; 645 646 static void kvm_arm_set_device_addr(KVMDevice *kd) 647 { 648 struct kvm_device_attr *attr = &kd->kdattr; 649 int ret; 650 uint64_t addr = kd->kda.addr; 651 652 addr |= kd->kda_addr_ormask; 653 attr->addr = (uintptr_t)&addr; 654 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr); 655 656 if (ret < 0) { 657 fprintf(stderr, "Failed to set device address: %s\n", 658 strerror(-ret)); 659 abort(); 660 } 661 } 662 663 static void kvm_arm_machine_init_done(Notifier *notifier, void *data) 664 { 665 KVMDevice *kd, *tkd; 666 667 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) { 668 if (kd->kda.addr != -1) { 669 kvm_arm_set_device_addr(kd); 670 } 671 memory_region_unref(kd->mr); 672 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries); 673 g_free(kd); 674 } 675 memory_listener_unregister(&devlistener); 676 } 677 678 static Notifier notify = { 679 .notify = kvm_arm_machine_init_done, 680 }; 681 682 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group, 683 uint64_t attr, int dev_fd, uint64_t addr_ormask) 684 { 685 KVMDevice *kd; 686 687 if (!kvm_irqchip_in_kernel()) { 688 return; 689 } 690 691 if (QSLIST_EMPTY(&kvm_devices_head)) { 692 memory_listener_register(&devlistener, &address_space_memory); 693 qemu_add_machine_init_done_notifier(¬ify); 694 } 695 kd = g_new0(KVMDevice, 1); 696 kd->mr = mr; 697 kd->kda.id = devid; 698 kd->kda.addr = -1; 699 kd->kdattr.flags = 0; 700 kd->kdattr.group = group; 701 kd->kdattr.attr = attr; 702 kd->dev_fd = dev_fd; 703 kd->kda_addr_ormask = addr_ormask; 704 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries); 705 memory_region_ref(kd->mr); 706 } 707 708 static int compare_u64(const void *a, const void *b) 709 { 710 if (*(uint64_t *)a > *(uint64_t *)b) { 711 return 1; 712 } 713 if (*(uint64_t *)a < *(uint64_t *)b) { 714 return -1; 715 } 716 return 0; 717 } 718 719 /* 720 * cpreg_values are sorted in ascending order by KVM register ID 721 * (see kvm_arm_init_cpreg_list). This allows us to cheaply find 722 * the storage for a KVM register by ID with a binary search. 723 */ 724 static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx) 725 { 726 uint64_t *res; 727 728 res = bsearch(®idx, cpu->cpreg_indexes, cpu->cpreg_array_len, 729 sizeof(uint64_t), compare_u64); 730 assert(res); 731 732 return &cpu->cpreg_values[res - cpu->cpreg_indexes]; 733 } 734 735 /** 736 * kvm_arm_reg_syncs_via_cpreg_list: 737 * @regidx: KVM register index 738 * 739 * Return true if this KVM register should be synchronized via the 740 * cpreg list of arbitrary system registers, false if it is synchronized 741 * by hand using code in kvm_arch_get/put_registers(). 742 */ 743 static bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) 744 { 745 switch (regidx & KVM_REG_ARM_COPROC_MASK) { 746 case KVM_REG_ARM_CORE: 747 case KVM_REG_ARM64_SVE: 748 return false; 749 default: 750 return true; 751 } 752 } 753 754 /** 755 * kvm_arm_init_cpreg_list: 756 * @cpu: ARMCPU 757 * 758 * Initialize the ARMCPU cpreg list according to the kernel's 759 * definition of what CPU registers it knows about (and throw away 760 * the previous TCG-created cpreg list). 761 * 762 * Returns: 0 if success, else < 0 error code 763 */ 764 static int kvm_arm_init_cpreg_list(ARMCPU *cpu) 765 { 766 struct kvm_reg_list rl; 767 struct kvm_reg_list *rlp; 768 int i, ret, arraylen; 769 CPUState *cs = CPU(cpu); 770 771 rl.n = 0; 772 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl); 773 if (ret != -E2BIG) { 774 return ret; 775 } 776 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t)); 777 rlp->n = rl.n; 778 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp); 779 if (ret) { 780 goto out; 781 } 782 /* Sort the list we get back from the kernel, since cpreg_tuples 783 * must be in strictly ascending order. 784 */ 785 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64); 786 787 for (i = 0, arraylen = 0; i < rlp->n; i++) { 788 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) { 789 continue; 790 } 791 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) { 792 case KVM_REG_SIZE_U32: 793 case KVM_REG_SIZE_U64: 794 break; 795 default: 796 fprintf(stderr, "Can't handle size of register in kernel list\n"); 797 ret = -EINVAL; 798 goto out; 799 } 800 801 arraylen++; 802 } 803 804 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen); 805 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen); 806 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes, 807 arraylen); 808 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values, 809 arraylen); 810 cpu->cpreg_array_len = arraylen; 811 cpu->cpreg_vmstate_array_len = arraylen; 812 813 for (i = 0, arraylen = 0; i < rlp->n; i++) { 814 uint64_t regidx = rlp->reg[i]; 815 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) { 816 continue; 817 } 818 cpu->cpreg_indexes[arraylen] = regidx; 819 arraylen++; 820 } 821 assert(cpu->cpreg_array_len == arraylen); 822 823 if (!write_kvmstate_to_list(cpu)) { 824 /* Shouldn't happen unless kernel is inconsistent about 825 * what registers exist. 826 */ 827 fprintf(stderr, "Initial read of kernel register state failed\n"); 828 ret = -EINVAL; 829 goto out; 830 } 831 832 out: 833 g_free(rlp); 834 return ret; 835 } 836 837 /** 838 * kvm_arm_cpreg_level: 839 * @regidx: KVM register index 840 * 841 * Return the level of this coprocessor/system register. Return value is 842 * either KVM_PUT_RUNTIME_STATE, KVM_PUT_RESET_STATE, or KVM_PUT_FULL_STATE. 843 */ 844 static int kvm_arm_cpreg_level(uint64_t regidx) 845 { 846 /* 847 * All system registers are assumed to be level KVM_PUT_RUNTIME_STATE. 848 * If a register should be written less often, you must add it here 849 * with a state of either KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE. 850 */ 851 switch (regidx) { 852 case KVM_REG_ARM_TIMER_CNT: 853 case KVM_REG_ARM_PTIMER_CNT: 854 return KVM_PUT_FULL_STATE; 855 } 856 return KVM_PUT_RUNTIME_STATE; 857 } 858 859 bool write_kvmstate_to_list(ARMCPU *cpu) 860 { 861 CPUState *cs = CPU(cpu); 862 int i; 863 bool ok = true; 864 865 for (i = 0; i < cpu->cpreg_array_len; i++) { 866 uint64_t regidx = cpu->cpreg_indexes[i]; 867 uint32_t v32; 868 int ret; 869 870 switch (regidx & KVM_REG_SIZE_MASK) { 871 case KVM_REG_SIZE_U32: 872 ret = kvm_get_one_reg(cs, regidx, &v32); 873 if (!ret) { 874 cpu->cpreg_values[i] = v32; 875 } 876 break; 877 case KVM_REG_SIZE_U64: 878 ret = kvm_get_one_reg(cs, regidx, cpu->cpreg_values + i); 879 break; 880 default: 881 g_assert_not_reached(); 882 } 883 if (ret) { 884 ok = false; 885 } 886 } 887 return ok; 888 } 889 890 bool write_list_to_kvmstate(ARMCPU *cpu, int level) 891 { 892 CPUState *cs = CPU(cpu); 893 int i; 894 bool ok = true; 895 896 for (i = 0; i < cpu->cpreg_array_len; i++) { 897 uint64_t regidx = cpu->cpreg_indexes[i]; 898 uint32_t v32; 899 int ret; 900 901 if (kvm_arm_cpreg_level(regidx) > level) { 902 continue; 903 } 904 905 switch (regidx & KVM_REG_SIZE_MASK) { 906 case KVM_REG_SIZE_U32: 907 v32 = cpu->cpreg_values[i]; 908 ret = kvm_set_one_reg(cs, regidx, &v32); 909 break; 910 case KVM_REG_SIZE_U64: 911 ret = kvm_set_one_reg(cs, regidx, cpu->cpreg_values + i); 912 break; 913 default: 914 g_assert_not_reached(); 915 } 916 if (ret) { 917 /* We might fail for "unknown register" and also for 918 * "you tried to set a register which is constant with 919 * a different value from what it actually contains". 920 */ 921 ok = false; 922 } 923 } 924 return ok; 925 } 926 927 void kvm_arm_cpu_pre_save(ARMCPU *cpu) 928 { 929 /* KVM virtual time adjustment */ 930 if (cpu->kvm_vtime_dirty) { 931 *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT) = cpu->kvm_vtime; 932 } 933 } 934 935 bool kvm_arm_cpu_post_load(ARMCPU *cpu) 936 { 937 if (!write_list_to_kvmstate(cpu, KVM_PUT_FULL_STATE)) { 938 return false; 939 } 940 /* Note that it's OK for the TCG side not to know about 941 * every register in the list; KVM is authoritative if 942 * we're using it. 943 */ 944 write_list_to_cpustate(cpu); 945 946 /* KVM virtual time adjustment */ 947 if (cpu->kvm_adjvtime) { 948 cpu->kvm_vtime = *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT); 949 cpu->kvm_vtime_dirty = true; 950 } 951 952 return true; 953 } 954 955 void kvm_arm_reset_vcpu(ARMCPU *cpu) 956 { 957 int ret; 958 959 /* Re-init VCPU so that all registers are set to 960 * their respective reset values. 961 */ 962 ret = kvm_arm_vcpu_init(cpu); 963 if (ret < 0) { 964 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret)); 965 abort(); 966 } 967 if (!write_kvmstate_to_list(cpu)) { 968 fprintf(stderr, "write_kvmstate_to_list failed\n"); 969 abort(); 970 } 971 /* 972 * Sync the reset values also into the CPUState. This is necessary 973 * because the next thing we do will be a kvm_arch_put_registers() 974 * which will update the list values from the CPUState before copying 975 * the list values back to KVM. It's OK to ignore failure returns here 976 * for the same reason we do so in kvm_arch_get_registers(). 977 */ 978 write_list_to_cpustate(cpu); 979 } 980 981 /* 982 * Update KVM's MP_STATE based on what QEMU thinks it is 983 */ 984 static int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu) 985 { 986 if (cap_has_mp_state) { 987 struct kvm_mp_state mp_state = { 988 .mp_state = (cpu->power_state == PSCI_OFF) ? 989 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE 990 }; 991 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state); 992 } 993 return 0; 994 } 995 996 /* 997 * Sync the KVM MP_STATE into QEMU 998 */ 999 static int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu) 1000 { 1001 if (cap_has_mp_state) { 1002 struct kvm_mp_state mp_state; 1003 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state); 1004 if (ret) { 1005 return ret; 1006 } 1007 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ? 1008 PSCI_OFF : PSCI_ON; 1009 } 1010 return 0; 1011 } 1012 1013 /** 1014 * kvm_arm_get_virtual_time: 1015 * @cpu: ARMCPU 1016 * 1017 * Gets the VCPU's virtual counter and stores it in the KVM CPU state. 1018 */ 1019 static void kvm_arm_get_virtual_time(ARMCPU *cpu) 1020 { 1021 int ret; 1022 1023 if (cpu->kvm_vtime_dirty) { 1024 return; 1025 } 1026 1027 ret = kvm_get_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime); 1028 if (ret) { 1029 error_report("Failed to get KVM_REG_ARM_TIMER_CNT"); 1030 abort(); 1031 } 1032 1033 cpu->kvm_vtime_dirty = true; 1034 } 1035 1036 /** 1037 * kvm_arm_put_virtual_time: 1038 * @cpu: ARMCPU 1039 * 1040 * Sets the VCPU's virtual counter to the value stored in the KVM CPU state. 1041 */ 1042 static void kvm_arm_put_virtual_time(ARMCPU *cpu) 1043 { 1044 int ret; 1045 1046 if (!cpu->kvm_vtime_dirty) { 1047 return; 1048 } 1049 1050 ret = kvm_set_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime); 1051 if (ret) { 1052 error_report("Failed to set KVM_REG_ARM_TIMER_CNT"); 1053 abort(); 1054 } 1055 1056 cpu->kvm_vtime_dirty = false; 1057 } 1058 1059 /** 1060 * kvm_put_vcpu_events: 1061 * @cpu: ARMCPU 1062 * 1063 * Put VCPU related state to kvm. 1064 * 1065 * Returns: 0 if success else < 0 error code 1066 */ 1067 static int kvm_put_vcpu_events(ARMCPU *cpu) 1068 { 1069 CPUARMState *env = &cpu->env; 1070 struct kvm_vcpu_events events; 1071 int ret; 1072 1073 if (!kvm_has_vcpu_events()) { 1074 return 0; 1075 } 1076 1077 memset(&events, 0, sizeof(events)); 1078 events.exception.serror_pending = env->serror.pending; 1079 1080 /* Inject SError to guest with specified syndrome if host kernel 1081 * supports it, otherwise inject SError without syndrome. 1082 */ 1083 if (cap_has_inject_serror_esr) { 1084 events.exception.serror_has_esr = env->serror.has_esr; 1085 events.exception.serror_esr = env->serror.esr; 1086 } 1087 1088 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events); 1089 if (ret) { 1090 error_report("failed to put vcpu events"); 1091 } 1092 1093 return ret; 1094 } 1095 1096 /** 1097 * kvm_get_vcpu_events: 1098 * @cpu: ARMCPU 1099 * 1100 * Get VCPU related state from kvm. 1101 * 1102 * Returns: 0 if success else < 0 error code 1103 */ 1104 static int kvm_get_vcpu_events(ARMCPU *cpu) 1105 { 1106 CPUARMState *env = &cpu->env; 1107 struct kvm_vcpu_events events; 1108 int ret; 1109 1110 if (!kvm_has_vcpu_events()) { 1111 return 0; 1112 } 1113 1114 memset(&events, 0, sizeof(events)); 1115 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events); 1116 if (ret) { 1117 error_report("failed to get vcpu events"); 1118 return ret; 1119 } 1120 1121 env->serror.pending = events.exception.serror_pending; 1122 env->serror.has_esr = events.exception.serror_has_esr; 1123 env->serror.esr = events.exception.serror_esr; 1124 1125 return 0; 1126 } 1127 1128 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0) 1129 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2) 1130 1131 /* 1132 * ESR_EL1 1133 * ISS encoding 1134 * AARCH64: DFSC, bits [5:0] 1135 * AARCH32: 1136 * TTBCR.EAE == 0 1137 * FS[4] - DFSR[10] 1138 * FS[3:0] - DFSR[3:0] 1139 * TTBCR.EAE == 1 1140 * FS, bits [5:0] 1141 */ 1142 #define ESR_DFSC(aarch64, lpae, v) \ 1143 ((aarch64 || (lpae)) ? ((v) & 0x3F) \ 1144 : (((v) >> 6) | ((v) & 0x1F))) 1145 1146 #define ESR_DFSC_EXTABT(aarch64, lpae) \ 1147 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8) 1148 1149 /** 1150 * kvm_arm_verify_ext_dabt_pending: 1151 * @cpu: ARMCPU 1152 * 1153 * Verify the fault status code wrt the Ext DABT injection 1154 * 1155 * Returns: true if the fault status code is as expected, false otherwise 1156 */ 1157 static bool kvm_arm_verify_ext_dabt_pending(ARMCPU *cpu) 1158 { 1159 CPUState *cs = CPU(cpu); 1160 uint64_t dfsr_val; 1161 1162 if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) { 1163 CPUARMState *env = &cpu->env; 1164 int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64); 1165 int lpae = 0; 1166 1167 if (!aarch64_mode) { 1168 uint64_t ttbcr; 1169 1170 if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) { 1171 lpae = arm_feature(env, ARM_FEATURE_LPAE) 1172 && (ttbcr & TTBCR_EAE); 1173 } 1174 } 1175 /* 1176 * The verification here is based on the DFSC bits 1177 * of the ESR_EL1 reg only 1178 */ 1179 return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) == 1180 ESR_DFSC_EXTABT(aarch64_mode, lpae)); 1181 } 1182 return false; 1183 } 1184 1185 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run) 1186 { 1187 ARMCPU *cpu = ARM_CPU(cs); 1188 CPUARMState *env = &cpu->env; 1189 1190 if (unlikely(env->ext_dabt_raised)) { 1191 /* 1192 * Verifying that the ext DABT has been properly injected, 1193 * otherwise risking indefinitely re-running the faulting instruction 1194 * Covering a very narrow case for kernels 5.5..5.5.4 1195 * when injected abort was misconfigured to be 1196 * an IMPLEMENTATION DEFINED exception (for 32-bit EL1) 1197 */ 1198 if (!arm_feature(env, ARM_FEATURE_AARCH64) && 1199 unlikely(!kvm_arm_verify_ext_dabt_pending(cpu))) { 1200 1201 error_report("Data abort exception with no valid ISS generated by " 1202 "guest memory access. KVM unable to emulate faulting " 1203 "instruction. Failed to inject an external data abort " 1204 "into the guest."); 1205 abort(); 1206 } 1207 /* Clear the status */ 1208 env->ext_dabt_raised = 0; 1209 } 1210 } 1211 1212 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run) 1213 { 1214 ARMCPU *cpu; 1215 uint32_t switched_level; 1216 1217 if (kvm_irqchip_in_kernel()) { 1218 /* 1219 * We only need to sync timer states with user-space interrupt 1220 * controllers, so return early and save cycles if we don't. 1221 */ 1222 return MEMTXATTRS_UNSPECIFIED; 1223 } 1224 1225 cpu = ARM_CPU(cs); 1226 1227 /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */ 1228 if (run->s.regs.device_irq_level != cpu->device_irq_level) { 1229 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level; 1230 1231 bql_lock(); 1232 1233 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) { 1234 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT], 1235 !!(run->s.regs.device_irq_level & 1236 KVM_ARM_DEV_EL1_VTIMER)); 1237 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER; 1238 } 1239 1240 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) { 1241 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS], 1242 !!(run->s.regs.device_irq_level & 1243 KVM_ARM_DEV_EL1_PTIMER)); 1244 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER; 1245 } 1246 1247 if (switched_level & KVM_ARM_DEV_PMU) { 1248 qemu_set_irq(cpu->pmu_interrupt, 1249 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU)); 1250 switched_level &= ~KVM_ARM_DEV_PMU; 1251 } 1252 1253 if (switched_level) { 1254 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n", 1255 __func__, switched_level); 1256 } 1257 1258 /* We also mark unknown levels as processed to not waste cycles */ 1259 cpu->device_irq_level = run->s.regs.device_irq_level; 1260 bql_unlock(); 1261 } 1262 1263 return MEMTXATTRS_UNSPECIFIED; 1264 } 1265 1266 static void kvm_arm_vm_state_change(void *opaque, bool running, RunState state) 1267 { 1268 ARMCPU *cpu = opaque; 1269 1270 if (running) { 1271 if (cpu->kvm_adjvtime) { 1272 kvm_arm_put_virtual_time(cpu); 1273 } 1274 } else { 1275 if (cpu->kvm_adjvtime) { 1276 kvm_arm_get_virtual_time(cpu); 1277 } 1278 } 1279 } 1280 1281 /** 1282 * kvm_arm_handle_dabt_nisv: 1283 * @cpu: ARMCPU 1284 * @esr_iss: ISS encoding (limited) for the exception from Data Abort 1285 * ISV bit set to '0b0' -> no valid instruction syndrome 1286 * @fault_ipa: faulting address for the synchronous data abort 1287 * 1288 * Returns: 0 if the exception has been handled, < 0 otherwise 1289 */ 1290 static int kvm_arm_handle_dabt_nisv(ARMCPU *cpu, uint64_t esr_iss, 1291 uint64_t fault_ipa) 1292 { 1293 CPUARMState *env = &cpu->env; 1294 /* 1295 * Request KVM to inject the external data abort into the guest 1296 */ 1297 if (cap_has_inject_ext_dabt) { 1298 struct kvm_vcpu_events events = { }; 1299 /* 1300 * The external data abort event will be handled immediately by KVM 1301 * using the address fault that triggered the exit on given VCPU. 1302 * Requesting injection of the external data abort does not rely 1303 * on any other VCPU state. Therefore, in this particular case, the VCPU 1304 * synchronization can be exceptionally skipped. 1305 */ 1306 events.exception.ext_dabt_pending = 1; 1307 /* KVM_CAP_ARM_INJECT_EXT_DABT implies KVM_CAP_VCPU_EVENTS */ 1308 if (!kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events)) { 1309 env->ext_dabt_raised = 1; 1310 return 0; 1311 } 1312 } else { 1313 error_report("Data abort exception triggered by guest memory access " 1314 "at physical address: 0x" TARGET_FMT_lx, 1315 (target_ulong)fault_ipa); 1316 error_printf("KVM unable to emulate faulting instruction.\n"); 1317 } 1318 return -1; 1319 } 1320 1321 /** 1322 * kvm_arm_handle_debug: 1323 * @cpu: ARMCPU 1324 * @debug_exit: debug part of the KVM exit structure 1325 * 1326 * Returns: TRUE if the debug exception was handled. 1327 * 1328 * See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register 1329 * 1330 * To minimise translating between kernel and user-space the kernel 1331 * ABI just provides user-space with the full exception syndrome 1332 * register value to be decoded in QEMU. 1333 */ 1334 static bool kvm_arm_handle_debug(ARMCPU *cpu, 1335 struct kvm_debug_exit_arch *debug_exit) 1336 { 1337 int hsr_ec = syn_get_ec(debug_exit->hsr); 1338 CPUState *cs = CPU(cpu); 1339 CPUARMState *env = &cpu->env; 1340 1341 /* Ensure PC is synchronised */ 1342 kvm_cpu_synchronize_state(cs); 1343 1344 switch (hsr_ec) { 1345 case EC_SOFTWARESTEP: 1346 if (cs->singlestep_enabled) { 1347 return true; 1348 } else { 1349 /* 1350 * The kernel should have suppressed the guest's ability to 1351 * single step at this point so something has gone wrong. 1352 */ 1353 error_report("%s: guest single-step while debugging unsupported" 1354 " (%"PRIx64", %"PRIx32")", 1355 __func__, env->pc, debug_exit->hsr); 1356 return false; 1357 } 1358 break; 1359 case EC_AA64_BKPT: 1360 if (kvm_find_sw_breakpoint(cs, env->pc)) { 1361 return true; 1362 } 1363 break; 1364 case EC_BREAKPOINT: 1365 if (find_hw_breakpoint(cs, env->pc)) { 1366 return true; 1367 } 1368 break; 1369 case EC_WATCHPOINT: 1370 { 1371 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far); 1372 if (wp) { 1373 cs->watchpoint_hit = wp; 1374 return true; 1375 } 1376 break; 1377 } 1378 default: 1379 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")", 1380 __func__, debug_exit->hsr, env->pc); 1381 } 1382 1383 /* If we are not handling the debug exception it must belong to 1384 * the guest. Let's re-use the existing TCG interrupt code to set 1385 * everything up properly. 1386 */ 1387 cs->exception_index = EXCP_BKPT; 1388 env->exception.syndrome = debug_exit->hsr; 1389 env->exception.vaddress = debug_exit->far; 1390 env->exception.target_el = 1; 1391 bql_lock(); 1392 arm_cpu_do_interrupt(cs); 1393 bql_unlock(); 1394 1395 return false; 1396 } 1397 1398 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) 1399 { 1400 ARMCPU *cpu = ARM_CPU(cs); 1401 int ret = 0; 1402 1403 switch (run->exit_reason) { 1404 case KVM_EXIT_DEBUG: 1405 if (kvm_arm_handle_debug(cpu, &run->debug.arch)) { 1406 ret = EXCP_DEBUG; 1407 } /* otherwise return to guest */ 1408 break; 1409 case KVM_EXIT_ARM_NISV: 1410 /* External DABT with no valid iss to decode */ 1411 ret = kvm_arm_handle_dabt_nisv(cpu, run->arm_nisv.esr_iss, 1412 run->arm_nisv.fault_ipa); 1413 break; 1414 default: 1415 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n", 1416 __func__, run->exit_reason); 1417 break; 1418 } 1419 return ret; 1420 } 1421 1422 bool kvm_arch_stop_on_emulation_error(CPUState *cs) 1423 { 1424 return true; 1425 } 1426 1427 int kvm_arch_process_async_events(CPUState *cs) 1428 { 1429 return 0; 1430 } 1431 1432 /** 1433 * kvm_arm_hw_debug_active: 1434 * @cpu: ARMCPU 1435 * 1436 * Return: TRUE if any hardware breakpoints in use. 1437 */ 1438 static bool kvm_arm_hw_debug_active(ARMCPU *cpu) 1439 { 1440 return ((cur_hw_wps > 0) || (cur_hw_bps > 0)); 1441 } 1442 1443 /** 1444 * kvm_arm_copy_hw_debug_data: 1445 * @ptr: kvm_guest_debug_arch structure 1446 * 1447 * Copy the architecture specific debug registers into the 1448 * kvm_guest_debug ioctl structure. 1449 */ 1450 static void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr) 1451 { 1452 int i; 1453 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch)); 1454 1455 for (i = 0; i < max_hw_wps; i++) { 1456 HWWatchpoint *wp = get_hw_wp(i); 1457 ptr->dbg_wcr[i] = wp->wcr; 1458 ptr->dbg_wvr[i] = wp->wvr; 1459 } 1460 for (i = 0; i < max_hw_bps; i++) { 1461 HWBreakpoint *bp = get_hw_bp(i); 1462 ptr->dbg_bcr[i] = bp->bcr; 1463 ptr->dbg_bvr[i] = bp->bvr; 1464 } 1465 } 1466 1467 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg) 1468 { 1469 if (kvm_sw_breakpoints_active(cs)) { 1470 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP; 1471 } 1472 if (kvm_arm_hw_debug_active(ARM_CPU(cs))) { 1473 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW; 1474 kvm_arm_copy_hw_debug_data(&dbg->arch); 1475 } 1476 } 1477 1478 void kvm_arch_init_irq_routing(KVMState *s) 1479 { 1480 } 1481 1482 int kvm_arch_irqchip_create(KVMState *s) 1483 { 1484 if (kvm_kernel_irqchip_split()) { 1485 error_report("-machine kernel_irqchip=split is not supported on ARM."); 1486 exit(1); 1487 } 1488 1489 /* If we can create the VGIC using the newer device control API, we 1490 * let the device do this when it initializes itself, otherwise we 1491 * fall back to the old API */ 1492 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL); 1493 } 1494 1495 int kvm_arm_vgic_probe(void) 1496 { 1497 int val = 0; 1498 1499 if (kvm_create_device(kvm_state, 1500 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) { 1501 val |= KVM_ARM_VGIC_V3; 1502 } 1503 if (kvm_create_device(kvm_state, 1504 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) { 1505 val |= KVM_ARM_VGIC_V2; 1506 } 1507 return val; 1508 } 1509 1510 int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level) 1511 { 1512 int kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq; 1513 int cpu_idx1 = cpu % 256; 1514 int cpu_idx2 = cpu / 256; 1515 1516 kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) | 1517 (cpu_idx2 << KVM_ARM_IRQ_VCPU2_SHIFT); 1518 1519 return kvm_set_irq(kvm_state, kvm_irq, !!level); 1520 } 1521 1522 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, 1523 uint64_t address, uint32_t data, PCIDevice *dev) 1524 { 1525 AddressSpace *as = pci_device_iommu_address_space(dev); 1526 hwaddr xlat, len, doorbell_gpa; 1527 MemoryRegionSection mrs; 1528 MemoryRegion *mr; 1529 1530 if (as == &address_space_memory) { 1531 return 0; 1532 } 1533 1534 /* MSI doorbell address is translated by an IOMMU */ 1535 1536 RCU_READ_LOCK_GUARD(); 1537 1538 mr = address_space_translate(as, address, &xlat, &len, true, 1539 MEMTXATTRS_UNSPECIFIED); 1540 1541 if (!mr) { 1542 return 1; 1543 } 1544 1545 mrs = memory_region_find(mr, xlat, 1); 1546 1547 if (!mrs.mr) { 1548 return 1; 1549 } 1550 1551 doorbell_gpa = mrs.offset_within_address_space; 1552 memory_region_unref(mrs.mr); 1553 1554 route->u.msi.address_lo = doorbell_gpa; 1555 route->u.msi.address_hi = doorbell_gpa >> 32; 1556 1557 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa); 1558 1559 return 0; 1560 } 1561 1562 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route, 1563 int vector, PCIDevice *dev) 1564 { 1565 return 0; 1566 } 1567 1568 int kvm_arch_release_virq_post(int virq) 1569 { 1570 return 0; 1571 } 1572 1573 int kvm_arch_msi_data_to_gsi(uint32_t data) 1574 { 1575 return (data - 32) & 0xffff; 1576 } 1577 1578 static void kvm_arch_get_eager_split_size(Object *obj, Visitor *v, 1579 const char *name, void *opaque, 1580 Error **errp) 1581 { 1582 KVMState *s = KVM_STATE(obj); 1583 uint64_t value = s->kvm_eager_split_size; 1584 1585 visit_type_size(v, name, &value, errp); 1586 } 1587 1588 static void kvm_arch_set_eager_split_size(Object *obj, Visitor *v, 1589 const char *name, void *opaque, 1590 Error **errp) 1591 { 1592 KVMState *s = KVM_STATE(obj); 1593 uint64_t value; 1594 1595 if (s->fd != -1) { 1596 error_setg(errp, "Unable to set early-split-size after KVM has been initialized"); 1597 return; 1598 } 1599 1600 if (!visit_type_size(v, name, &value, errp)) { 1601 return; 1602 } 1603 1604 if (value && !is_power_of_2(value)) { 1605 error_setg(errp, "early-split-size must be a power of two"); 1606 return; 1607 } 1608 1609 s->kvm_eager_split_size = value; 1610 } 1611 1612 void kvm_arch_accel_class_init(ObjectClass *oc) 1613 { 1614 object_class_property_add(oc, "eager-split-size", "size", 1615 kvm_arch_get_eager_split_size, 1616 kvm_arch_set_eager_split_size, NULL, NULL); 1617 1618 object_class_property_set_description(oc, "eager-split-size", 1619 "Eager Page Split chunk size for hugepages. (default: 0, disabled)"); 1620 } 1621 1622 int kvm_arch_insert_hw_breakpoint(vaddr addr, vaddr len, int type) 1623 { 1624 switch (type) { 1625 case GDB_BREAKPOINT_HW: 1626 return insert_hw_breakpoint(addr); 1627 break; 1628 case GDB_WATCHPOINT_READ: 1629 case GDB_WATCHPOINT_WRITE: 1630 case GDB_WATCHPOINT_ACCESS: 1631 return insert_hw_watchpoint(addr, len, type); 1632 default: 1633 return -ENOSYS; 1634 } 1635 } 1636 1637 int kvm_arch_remove_hw_breakpoint(vaddr addr, vaddr len, int type) 1638 { 1639 switch (type) { 1640 case GDB_BREAKPOINT_HW: 1641 return delete_hw_breakpoint(addr); 1642 case GDB_WATCHPOINT_READ: 1643 case GDB_WATCHPOINT_WRITE: 1644 case GDB_WATCHPOINT_ACCESS: 1645 return delete_hw_watchpoint(addr, len, type); 1646 default: 1647 return -ENOSYS; 1648 } 1649 } 1650 1651 void kvm_arch_remove_all_hw_breakpoints(void) 1652 { 1653 if (cur_hw_wps > 0) { 1654 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps); 1655 } 1656 if (cur_hw_bps > 0) { 1657 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps); 1658 } 1659 } 1660 1661 static bool kvm_arm_set_device_attr(ARMCPU *cpu, struct kvm_device_attr *attr, 1662 const char *name) 1663 { 1664 int err; 1665 1666 err = kvm_vcpu_ioctl(CPU(cpu), KVM_HAS_DEVICE_ATTR, attr); 1667 if (err != 0) { 1668 error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err)); 1669 return false; 1670 } 1671 1672 err = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEVICE_ATTR, attr); 1673 if (err != 0) { 1674 error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err)); 1675 return false; 1676 } 1677 1678 return true; 1679 } 1680 1681 void kvm_arm_pmu_init(ARMCPU *cpu) 1682 { 1683 struct kvm_device_attr attr = { 1684 .group = KVM_ARM_VCPU_PMU_V3_CTRL, 1685 .attr = KVM_ARM_VCPU_PMU_V3_INIT, 1686 }; 1687 1688 if (!cpu->has_pmu) { 1689 return; 1690 } 1691 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) { 1692 error_report("failed to init PMU"); 1693 abort(); 1694 } 1695 } 1696 1697 void kvm_arm_pmu_set_irq(ARMCPU *cpu, int irq) 1698 { 1699 struct kvm_device_attr attr = { 1700 .group = KVM_ARM_VCPU_PMU_V3_CTRL, 1701 .addr = (intptr_t)&irq, 1702 .attr = KVM_ARM_VCPU_PMU_V3_IRQ, 1703 }; 1704 1705 if (!cpu->has_pmu) { 1706 return; 1707 } 1708 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) { 1709 error_report("failed to set irq for PMU"); 1710 abort(); 1711 } 1712 } 1713 1714 void kvm_arm_pvtime_init(ARMCPU *cpu, uint64_t ipa) 1715 { 1716 struct kvm_device_attr attr = { 1717 .group = KVM_ARM_VCPU_PVTIME_CTRL, 1718 .attr = KVM_ARM_VCPU_PVTIME_IPA, 1719 .addr = (uint64_t)&ipa, 1720 }; 1721 1722 if (cpu->kvm_steal_time == ON_OFF_AUTO_OFF) { 1723 return; 1724 } 1725 if (!kvm_arm_set_device_attr(cpu, &attr, "PVTIME IPA")) { 1726 error_report("failed to init PVTIME IPA"); 1727 abort(); 1728 } 1729 } 1730 1731 void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp) 1732 { 1733 bool has_steal_time = kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME); 1734 1735 if (cpu->kvm_steal_time == ON_OFF_AUTO_AUTO) { 1736 if (!has_steal_time || !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { 1737 cpu->kvm_steal_time = ON_OFF_AUTO_OFF; 1738 } else { 1739 cpu->kvm_steal_time = ON_OFF_AUTO_ON; 1740 } 1741 } else if (cpu->kvm_steal_time == ON_OFF_AUTO_ON) { 1742 if (!has_steal_time) { 1743 error_setg(errp, "'kvm-steal-time' cannot be enabled " 1744 "on this host"); 1745 return; 1746 } else if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { 1747 /* 1748 * DEN0057A chapter 2 says "This specification only covers 1749 * systems in which the Execution state of the hypervisor 1750 * as well as EL1 of virtual machines is AArch64.". And, 1751 * to ensure that, the smc/hvc calls are only specified as 1752 * smc64/hvc64. 1753 */ 1754 error_setg(errp, "'kvm-steal-time' cannot be enabled " 1755 "for AArch32 guests"); 1756 return; 1757 } 1758 } 1759 } 1760 1761 bool kvm_arm_aarch32_supported(void) 1762 { 1763 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT); 1764 } 1765 1766 bool kvm_arm_sve_supported(void) 1767 { 1768 return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE); 1769 } 1770 1771 bool kvm_arm_mte_supported(void) 1772 { 1773 return kvm_check_extension(kvm_state, KVM_CAP_ARM_MTE); 1774 } 1775 1776 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); 1777 1778 uint32_t kvm_arm_sve_get_vls(ARMCPU *cpu) 1779 { 1780 /* Only call this function if kvm_arm_sve_supported() returns true. */ 1781 static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS]; 1782 static bool probed; 1783 uint32_t vq = 0; 1784 int i; 1785 1786 /* 1787 * KVM ensures all host CPUs support the same set of vector lengths. 1788 * So we only need to create the scratch VCPUs once and then cache 1789 * the results. 1790 */ 1791 if (!probed) { 1792 struct kvm_vcpu_init init = { 1793 .target = -1, 1794 .features[0] = (1 << KVM_ARM_VCPU_SVE), 1795 }; 1796 struct kvm_one_reg reg = { 1797 .id = KVM_REG_ARM64_SVE_VLS, 1798 .addr = (uint64_t)&vls[0], 1799 }; 1800 int fdarray[3], ret; 1801 1802 probed = true; 1803 1804 if (!kvm_arm_create_scratch_host_vcpu(fdarray, &init)) { 1805 error_report("failed to create scratch VCPU with SVE enabled"); 1806 abort(); 1807 } 1808 ret = ioctl(fdarray[2], KVM_GET_ONE_REG, ®); 1809 kvm_arm_destroy_scratch_host_vcpu(fdarray); 1810 if (ret) { 1811 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s", 1812 strerror(errno)); 1813 abort(); 1814 } 1815 1816 for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) { 1817 if (vls[i]) { 1818 vq = 64 - clz64(vls[i]) + i * 64; 1819 break; 1820 } 1821 } 1822 if (vq > ARM_MAX_VQ) { 1823 warn_report("KVM supports vector lengths larger than " 1824 "QEMU can enable"); 1825 vls[0] &= MAKE_64BIT_MASK(0, ARM_MAX_VQ); 1826 } 1827 } 1828 1829 return vls[0]; 1830 } 1831 1832 static int kvm_arm_sve_set_vls(ARMCPU *cpu) 1833 { 1834 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq.map }; 1835 1836 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX); 1837 1838 return kvm_set_one_reg(CPU(cpu), KVM_REG_ARM64_SVE_VLS, &vls[0]); 1839 } 1840 1841 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5 1842 1843 int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp) 1844 { 1845 return 0; 1846 } 1847 1848 int kvm_arch_init_vcpu(CPUState *cs) 1849 { 1850 int ret; 1851 uint64_t mpidr; 1852 ARMCPU *cpu = ARM_CPU(cs); 1853 CPUARMState *env = &cpu->env; 1854 uint64_t psciver; 1855 1856 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE) { 1857 error_report("KVM is not supported for this guest CPU type"); 1858 return -EINVAL; 1859 } 1860 1861 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cpu); 1862 1863 /* Determine init features for this CPU */ 1864 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features)); 1865 if (cs->start_powered_off) { 1866 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF; 1867 } 1868 if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) { 1869 cpu->psci_version = QEMU_PSCI_VERSION_0_2; 1870 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2; 1871 } 1872 if (!arm_feature(env, ARM_FEATURE_AARCH64)) { 1873 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT; 1874 } 1875 if (cpu->has_pmu) { 1876 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; 1877 } 1878 if (cpu_isar_feature(aa64_sve, cpu)) { 1879 assert(kvm_arm_sve_supported()); 1880 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE; 1881 } 1882 if (cpu_isar_feature(aa64_pauth, cpu)) { 1883 cpu->kvm_init_features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS | 1884 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC); 1885 } 1886 1887 /* Do KVM_ARM_VCPU_INIT ioctl */ 1888 ret = kvm_arm_vcpu_init(cpu); 1889 if (ret) { 1890 return ret; 1891 } 1892 1893 if (cpu_isar_feature(aa64_sve, cpu)) { 1894 ret = kvm_arm_sve_set_vls(cpu); 1895 if (ret) { 1896 return ret; 1897 } 1898 ret = kvm_arm_vcpu_finalize(cpu, KVM_ARM_VCPU_SVE); 1899 if (ret) { 1900 return ret; 1901 } 1902 } 1903 1904 /* 1905 * KVM reports the exact PSCI version it is implementing via a 1906 * special sysreg. If it is present, use its contents to determine 1907 * what to report to the guest in the dtb (it is the PSCI version, 1908 * in the same 15-bits major 16-bits minor format that PSCI_VERSION 1909 * returns). 1910 */ 1911 if (!kvm_get_one_reg(cs, KVM_REG_ARM_PSCI_VERSION, &psciver)) { 1912 cpu->psci_version = psciver; 1913 } 1914 1915 /* 1916 * When KVM is in use, PSCI is emulated in-kernel and not by qemu. 1917 * Currently KVM has its own idea about MPIDR assignment, so we 1918 * override our defaults with what we get from KVM. 1919 */ 1920 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr); 1921 if (ret) { 1922 return ret; 1923 } 1924 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK; 1925 1926 return kvm_arm_init_cpreg_list(cpu); 1927 } 1928 1929 int kvm_arch_destroy_vcpu(CPUState *cs) 1930 { 1931 return 0; 1932 } 1933 1934 /* Callers must hold the iothread mutex lock */ 1935 static void kvm_inject_arm_sea(CPUState *c) 1936 { 1937 ARMCPU *cpu = ARM_CPU(c); 1938 CPUARMState *env = &cpu->env; 1939 uint32_t esr; 1940 bool same_el; 1941 1942 c->exception_index = EXCP_DATA_ABORT; 1943 env->exception.target_el = 1; 1944 1945 /* 1946 * Set the DFSC to synchronous external abort and set FnV to not valid, 1947 * this will tell guest the FAR_ELx is UNKNOWN for this abort. 1948 */ 1949 same_el = arm_current_el(env) == env->exception.target_el; 1950 esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10); 1951 1952 env->exception.syndrome = esr; 1953 1954 arm_cpu_do_interrupt(c); 1955 } 1956 1957 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ 1958 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 1959 1960 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \ 1961 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 1962 1963 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \ 1964 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 1965 1966 static int kvm_arch_put_fpsimd(CPUState *cs) 1967 { 1968 CPUARMState *env = &ARM_CPU(cs)->env; 1969 int i, ret; 1970 1971 for (i = 0; i < 32; i++) { 1972 uint64_t *q = aa64_vfp_qreg(env, i); 1973 #if HOST_BIG_ENDIAN 1974 uint64_t fp_val[2] = { q[1], q[0] }; 1975 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), 1976 fp_val); 1977 #else 1978 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q); 1979 #endif 1980 if (ret) { 1981 return ret; 1982 } 1983 } 1984 1985 return 0; 1986 } 1987 1988 /* 1989 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits 1990 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard 1991 * code the slice index to zero for now as it's unlikely we'll need more than 1992 * one slice for quite some time. 1993 */ 1994 static int kvm_arch_put_sve(CPUState *cs) 1995 { 1996 ARMCPU *cpu = ARM_CPU(cs); 1997 CPUARMState *env = &cpu->env; 1998 uint64_t tmp[ARM_MAX_VQ * 2]; 1999 uint64_t *r; 2000 int n, ret; 2001 2002 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { 2003 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); 2004 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r); 2005 if (ret) { 2006 return ret; 2007 } 2008 } 2009 2010 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { 2011 r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], 2012 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2013 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r); 2014 if (ret) { 2015 return ret; 2016 } 2017 } 2018 2019 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], 2020 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2021 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r); 2022 if (ret) { 2023 return ret; 2024 } 2025 2026 return 0; 2027 } 2028 2029 int kvm_arch_put_registers(CPUState *cs, int level, Error **errp) 2030 { 2031 uint64_t val; 2032 uint32_t fpr; 2033 int i, ret; 2034 unsigned int el; 2035 2036 ARMCPU *cpu = ARM_CPU(cs); 2037 CPUARMState *env = &cpu->env; 2038 2039 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the 2040 * AArch64 registers before pushing them out to 64-bit KVM. 2041 */ 2042 if (!is_a64(env)) { 2043 aarch64_sync_32_to_64(env); 2044 } 2045 2046 for (i = 0; i < 31; i++) { 2047 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), 2048 &env->xregs[i]); 2049 if (ret) { 2050 return ret; 2051 } 2052 } 2053 2054 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the 2055 * QEMU side we keep the current SP in xregs[31] as well. 2056 */ 2057 aarch64_save_sp(env, 1); 2058 2059 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]); 2060 if (ret) { 2061 return ret; 2062 } 2063 2064 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]); 2065 if (ret) { 2066 return ret; 2067 } 2068 2069 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */ 2070 if (is_a64(env)) { 2071 val = pstate_read(env); 2072 } else { 2073 val = cpsr_read(env); 2074 } 2075 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val); 2076 if (ret) { 2077 return ret; 2078 } 2079 2080 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc); 2081 if (ret) { 2082 return ret; 2083 } 2084 2085 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]); 2086 if (ret) { 2087 return ret; 2088 } 2089 2090 /* Saved Program State Registers 2091 * 2092 * Before we restore from the banked_spsr[] array we need to 2093 * ensure that any modifications to env->spsr are correctly 2094 * reflected in the banks. 2095 */ 2096 el = arm_current_el(env); 2097 if (el > 0 && !is_a64(env)) { 2098 i = bank_number(env->uncached_cpsr & CPSR_M); 2099 env->banked_spsr[i] = env->spsr; 2100 } 2101 2102 /* KVM 0-4 map to QEMU banks 1-5 */ 2103 for (i = 0; i < KVM_NR_SPSR; i++) { 2104 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(spsr[i]), 2105 &env->banked_spsr[i + 1]); 2106 if (ret) { 2107 return ret; 2108 } 2109 } 2110 2111 if (cpu_isar_feature(aa64_sve, cpu)) { 2112 ret = kvm_arch_put_sve(cs); 2113 } else { 2114 ret = kvm_arch_put_fpsimd(cs); 2115 } 2116 if (ret) { 2117 return ret; 2118 } 2119 2120 fpr = vfp_get_fpsr(env); 2121 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr); 2122 if (ret) { 2123 return ret; 2124 } 2125 2126 fpr = vfp_get_fpcr(env); 2127 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr); 2128 if (ret) { 2129 return ret; 2130 } 2131 2132 write_cpustate_to_list(cpu, true); 2133 2134 if (!write_list_to_kvmstate(cpu, level)) { 2135 return -EINVAL; 2136 } 2137 2138 /* 2139 * Setting VCPU events should be triggered after syncing the registers 2140 * to avoid overwriting potential changes made by KVM upon calling 2141 * KVM_SET_VCPU_EVENTS ioctl 2142 */ 2143 ret = kvm_put_vcpu_events(cpu); 2144 if (ret) { 2145 return ret; 2146 } 2147 2148 return kvm_arm_sync_mpstate_to_kvm(cpu); 2149 } 2150 2151 static int kvm_arch_get_fpsimd(CPUState *cs) 2152 { 2153 CPUARMState *env = &ARM_CPU(cs)->env; 2154 int i, ret; 2155 2156 for (i = 0; i < 32; i++) { 2157 uint64_t *q = aa64_vfp_qreg(env, i); 2158 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q); 2159 if (ret) { 2160 return ret; 2161 } else { 2162 #if HOST_BIG_ENDIAN 2163 uint64_t t; 2164 t = q[0], q[0] = q[1], q[1] = t; 2165 #endif 2166 } 2167 } 2168 2169 return 0; 2170 } 2171 2172 /* 2173 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits 2174 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard 2175 * code the slice index to zero for now as it's unlikely we'll need more than 2176 * one slice for quite some time. 2177 */ 2178 static int kvm_arch_get_sve(CPUState *cs) 2179 { 2180 ARMCPU *cpu = ARM_CPU(cs); 2181 CPUARMState *env = &cpu->env; 2182 uint64_t *r; 2183 int n, ret; 2184 2185 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { 2186 r = &env->vfp.zregs[n].d[0]; 2187 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r); 2188 if (ret) { 2189 return ret; 2190 } 2191 sve_bswap64(r, r, cpu->sve_max_vq * 2); 2192 } 2193 2194 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { 2195 r = &env->vfp.pregs[n].p[0]; 2196 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r); 2197 if (ret) { 2198 return ret; 2199 } 2200 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2201 } 2202 2203 r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; 2204 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r); 2205 if (ret) { 2206 return ret; 2207 } 2208 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2209 2210 return 0; 2211 } 2212 2213 int kvm_arch_get_registers(CPUState *cs, Error **errp) 2214 { 2215 uint64_t val; 2216 unsigned int el; 2217 uint32_t fpr; 2218 int i, ret; 2219 2220 ARMCPU *cpu = ARM_CPU(cs); 2221 CPUARMState *env = &cpu->env; 2222 2223 for (i = 0; i < 31; i++) { 2224 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), 2225 &env->xregs[i]); 2226 if (ret) { 2227 return ret; 2228 } 2229 } 2230 2231 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]); 2232 if (ret) { 2233 return ret; 2234 } 2235 2236 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]); 2237 if (ret) { 2238 return ret; 2239 } 2240 2241 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val); 2242 if (ret) { 2243 return ret; 2244 } 2245 2246 env->aarch64 = ((val & PSTATE_nRW) == 0); 2247 if (is_a64(env)) { 2248 pstate_write(env, val); 2249 } else { 2250 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw); 2251 } 2252 2253 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the 2254 * QEMU side we keep the current SP in xregs[31] as well. 2255 */ 2256 aarch64_restore_sp(env, 1); 2257 2258 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc); 2259 if (ret) { 2260 return ret; 2261 } 2262 2263 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the 2264 * incoming AArch64 regs received from 64-bit KVM. 2265 * We must perform this after all of the registers have been acquired from 2266 * the kernel. 2267 */ 2268 if (!is_a64(env)) { 2269 aarch64_sync_64_to_32(env); 2270 } 2271 2272 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]); 2273 if (ret) { 2274 return ret; 2275 } 2276 2277 /* Fetch the SPSR registers 2278 * 2279 * KVM SPSRs 0-4 map to QEMU banks 1-5 2280 */ 2281 for (i = 0; i < KVM_NR_SPSR; i++) { 2282 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(spsr[i]), 2283 &env->banked_spsr[i + 1]); 2284 if (ret) { 2285 return ret; 2286 } 2287 } 2288 2289 el = arm_current_el(env); 2290 if (el > 0 && !is_a64(env)) { 2291 i = bank_number(env->uncached_cpsr & CPSR_M); 2292 env->spsr = env->banked_spsr[i]; 2293 } 2294 2295 if (cpu_isar_feature(aa64_sve, cpu)) { 2296 ret = kvm_arch_get_sve(cs); 2297 } else { 2298 ret = kvm_arch_get_fpsimd(cs); 2299 } 2300 if (ret) { 2301 return ret; 2302 } 2303 2304 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr); 2305 if (ret) { 2306 return ret; 2307 } 2308 vfp_set_fpsr(env, fpr); 2309 2310 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr); 2311 if (ret) { 2312 return ret; 2313 } 2314 vfp_set_fpcr(env, fpr); 2315 2316 ret = kvm_get_vcpu_events(cpu); 2317 if (ret) { 2318 return ret; 2319 } 2320 2321 if (!write_kvmstate_to_list(cpu)) { 2322 return -EINVAL; 2323 } 2324 /* Note that it's OK to have registers which aren't in CPUState, 2325 * so we can ignore a failure return here. 2326 */ 2327 write_list_to_cpustate(cpu); 2328 2329 ret = kvm_arm_sync_mpstate_to_qemu(cpu); 2330 2331 /* TODO: other registers */ 2332 return ret; 2333 } 2334 2335 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr) 2336 { 2337 ram_addr_t ram_addr; 2338 hwaddr paddr; 2339 2340 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO); 2341 2342 if (acpi_ghes_present() && addr) { 2343 ram_addr = qemu_ram_addr_from_host(addr); 2344 if (ram_addr != RAM_ADDR_INVALID && 2345 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) { 2346 kvm_hwpoison_page_add(ram_addr); 2347 /* 2348 * If this is a BUS_MCEERR_AR, we know we have been called 2349 * synchronously from the vCPU thread, so we can easily 2350 * synchronize the state and inject an error. 2351 * 2352 * TODO: we currently don't tell the guest at all about 2353 * BUS_MCEERR_AO. In that case we might either be being 2354 * called synchronously from the vCPU thread, or a bit 2355 * later from the main thread, so doing the injection of 2356 * the error would be more complicated. 2357 */ 2358 if (code == BUS_MCEERR_AR) { 2359 kvm_cpu_synchronize_state(c); 2360 if (!acpi_ghes_memory_errors(ACPI_HEST_SRC_ID_SEA, paddr)) { 2361 kvm_inject_arm_sea(c); 2362 } else { 2363 error_report("failed to record the error"); 2364 abort(); 2365 } 2366 } 2367 return; 2368 } 2369 if (code == BUS_MCEERR_AO) { 2370 error_report("Hardware memory error at addr %p for memory used by " 2371 "QEMU itself instead of guest system!", addr); 2372 } 2373 } 2374 2375 if (code == BUS_MCEERR_AR) { 2376 error_report("Hardware memory error!"); 2377 exit(1); 2378 } 2379 } 2380 2381 /* C6.6.29 BRK instruction */ 2382 static const uint32_t brk_insn = 0xd4200000; 2383 2384 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 2385 { 2386 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) || 2387 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) { 2388 return -EINVAL; 2389 } 2390 return 0; 2391 } 2392 2393 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 2394 { 2395 static uint32_t brk; 2396 2397 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) || 2398 brk != brk_insn || 2399 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) { 2400 return -EINVAL; 2401 } 2402 return 0; 2403 } 2404 2405 void kvm_arm_enable_mte(Object *cpuobj, Error **errp) 2406 { 2407 static bool tried_to_enable; 2408 static bool succeeded_to_enable; 2409 Error *mte_migration_blocker = NULL; 2410 ARMCPU *cpu = ARM_CPU(cpuobj); 2411 int ret; 2412 2413 if (!tried_to_enable) { 2414 /* 2415 * MTE on KVM is enabled on a per-VM basis (and retrying doesn't make 2416 * sense), and we only want a single migration blocker as well. 2417 */ 2418 tried_to_enable = true; 2419 2420 ret = kvm_vm_enable_cap(kvm_state, KVM_CAP_ARM_MTE, 0); 2421 if (ret) { 2422 error_setg_errno(errp, -ret, "Failed to enable KVM_CAP_ARM_MTE"); 2423 return; 2424 } 2425 2426 /* TODO: Add migration support with MTE enabled */ 2427 error_setg(&mte_migration_blocker, 2428 "Live migration disabled due to MTE enabled"); 2429 if (migrate_add_blocker(&mte_migration_blocker, errp)) { 2430 error_free(mte_migration_blocker); 2431 return; 2432 } 2433 2434 succeeded_to_enable = true; 2435 } 2436 2437 if (succeeded_to_enable) { 2438 cpu->kvm_mte = true; 2439 } 2440 } 2441 2442 void arm_cpu_kvm_set_irq(void *arm_cpu, int irq, int level) 2443 { 2444 ARMCPU *cpu = arm_cpu; 2445 CPUARMState *env = &cpu->env; 2446 CPUState *cs = CPU(cpu); 2447 uint32_t linestate_bit; 2448 int irq_id; 2449 2450 switch (irq) { 2451 case ARM_CPU_IRQ: 2452 irq_id = KVM_ARM_IRQ_CPU_IRQ; 2453 linestate_bit = CPU_INTERRUPT_HARD; 2454 break; 2455 case ARM_CPU_FIQ: 2456 irq_id = KVM_ARM_IRQ_CPU_FIQ; 2457 linestate_bit = CPU_INTERRUPT_FIQ; 2458 break; 2459 default: 2460 g_assert_not_reached(); 2461 } 2462 2463 if (level) { 2464 env->irq_line_state |= linestate_bit; 2465 } else { 2466 env->irq_line_state &= ~linestate_bit; 2467 } 2468 kvm_arm_set_irq(cs->cpu_index, KVM_ARM_IRQ_TYPE_CPU, irq_id, !!level); 2469 } 2470