1 /* 2 * ARM implementation of KVM hooks 3 * 4 * Copyright Christoffer Dall 2009-2010 5 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems 6 * Copyright Alex Bennée 2014, Linaro 7 * 8 * This work is licensed under the terms of the GNU GPL, version 2 or later. 9 * See the COPYING file in the top-level directory. 10 * 11 */ 12 13 #include "qemu/osdep.h" 14 #include <sys/ioctl.h> 15 16 #include <linux/kvm.h> 17 18 #include "qemu/timer.h" 19 #include "qemu/error-report.h" 20 #include "qemu/main-loop.h" 21 #include "qom/object.h" 22 #include "qapi/error.h" 23 #include "system/system.h" 24 #include "system/runstate.h" 25 #include "system/kvm.h" 26 #include "system/kvm_int.h" 27 #include "kvm_arm.h" 28 #include "cpu.h" 29 #include "cpu-sysregs.h" 30 #include "trace.h" 31 #include "internals.h" 32 #include "hw/pci/pci.h" 33 #include "exec/memattrs.h" 34 #include "system/address-spaces.h" 35 #include "gdbstub/enums.h" 36 #include "hw/boards.h" 37 #include "hw/irq.h" 38 #include "qapi/visitor.h" 39 #include "qemu/log.h" 40 #include "hw/acpi/acpi.h" 41 #include "hw/acpi/ghes.h" 42 #include "target/arm/gtimer.h" 43 #include "migration/blocker.h" 44 45 const KVMCapabilityInfo kvm_arch_required_capabilities[] = { 46 KVM_CAP_INFO(DEVICE_CTRL), 47 KVM_CAP_LAST_INFO 48 }; 49 50 static bool cap_has_mp_state; 51 static bool cap_has_inject_serror_esr; 52 static bool cap_has_inject_ext_dabt; 53 54 /** 55 * ARMHostCPUFeatures: information about the host CPU (identified 56 * by asking the host kernel) 57 */ 58 typedef struct ARMHostCPUFeatures { 59 ARMISARegisters isar; 60 uint64_t features; 61 uint32_t target; 62 const char *dtb_compatible; 63 } ARMHostCPUFeatures; 64 65 static ARMHostCPUFeatures arm_host_cpu_features; 66 67 /** 68 * kvm_arm_vcpu_init: 69 * @cpu: ARMCPU 70 * 71 * Initialize (or reinitialize) the VCPU by invoking the 72 * KVM_ARM_VCPU_INIT ioctl with the CPU type and feature 73 * bitmask specified in the CPUState. 74 * 75 * Returns: 0 if success else < 0 error code 76 */ 77 static int kvm_arm_vcpu_init(ARMCPU *cpu) 78 { 79 struct kvm_vcpu_init init; 80 81 init.target = cpu->kvm_target; 82 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features)); 83 84 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_INIT, &init); 85 } 86 87 /** 88 * kvm_arm_vcpu_finalize: 89 * @cpu: ARMCPU 90 * @feature: feature to finalize 91 * 92 * Finalizes the configuration of the specified VCPU feature by 93 * invoking the KVM_ARM_VCPU_FINALIZE ioctl. Features requiring 94 * this are documented in the "KVM_ARM_VCPU_FINALIZE" section of 95 * KVM's API documentation. 96 * 97 * Returns: 0 if success else < 0 error code 98 */ 99 static int kvm_arm_vcpu_finalize(ARMCPU *cpu, int feature) 100 { 101 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_FINALIZE, &feature); 102 } 103 104 bool kvm_arm_create_scratch_host_vcpu(int *fdarray, 105 struct kvm_vcpu_init *init) 106 { 107 int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1; 108 int max_vm_pa_size; 109 110 kvmfd = qemu_open_old("/dev/kvm", O_RDWR); 111 if (kvmfd < 0) { 112 goto err; 113 } 114 max_vm_pa_size = ioctl(kvmfd, KVM_CHECK_EXTENSION, KVM_CAP_ARM_VM_IPA_SIZE); 115 if (max_vm_pa_size < 0) { 116 max_vm_pa_size = 0; 117 } 118 do { 119 vmfd = ioctl(kvmfd, KVM_CREATE_VM, max_vm_pa_size); 120 } while (vmfd == -1 && errno == EINTR); 121 if (vmfd < 0) { 122 goto err; 123 } 124 125 /* 126 * The MTE capability must be enabled by the VMM before creating 127 * any VCPUs in order to allow the MTE bits of the ID_AA64PFR1 128 * register to be probed correctly, as they are masked if MTE 129 * is not enabled. 130 */ 131 if (kvm_arm_mte_supported()) { 132 KVMState kvm_state; 133 134 kvm_state.fd = kvmfd; 135 kvm_state.vmfd = vmfd; 136 kvm_vm_enable_cap(&kvm_state, KVM_CAP_ARM_MTE, 0); 137 } 138 139 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0); 140 if (cpufd < 0) { 141 goto err; 142 } 143 144 if (!init) { 145 /* Caller doesn't want the VCPU to be initialized, so skip it */ 146 goto finish; 147 } 148 149 if (init->target == -1) { 150 struct kvm_vcpu_init preferred; 151 152 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred); 153 if (ret < 0) { 154 goto err; 155 } 156 init->target = preferred.target; 157 } 158 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init); 159 if (ret < 0) { 160 goto err; 161 } 162 163 finish: 164 fdarray[0] = kvmfd; 165 fdarray[1] = vmfd; 166 fdarray[2] = cpufd; 167 168 return true; 169 170 err: 171 if (cpufd >= 0) { 172 close(cpufd); 173 } 174 if (vmfd >= 0) { 175 close(vmfd); 176 } 177 if (kvmfd >= 0) { 178 close(kvmfd); 179 } 180 181 return false; 182 } 183 184 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray) 185 { 186 int i; 187 188 for (i = 2; i >= 0; i--) { 189 close(fdarray[i]); 190 } 191 } 192 193 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id) 194 { 195 uint64_t ret; 196 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret }; 197 int err; 198 199 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); 200 err = ioctl(fd, KVM_GET_ONE_REG, &idreg); 201 if (err < 0) { 202 return -1; 203 } 204 *pret = ret; 205 return 0; 206 } 207 208 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id) 209 { 210 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret }; 211 212 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); 213 return ioctl(fd, KVM_GET_ONE_REG, &idreg); 214 } 215 216 static bool kvm_arm_pauth_supported(void) 217 { 218 return (kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_ADDRESS) && 219 kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_GENERIC)); 220 } 221 222 223 static uint64_t idregs_sysreg_to_kvm_reg(ARMSysRegs sysreg) 224 { 225 return ARM64_SYS_REG((sysreg & CP_REG_ARM64_SYSREG_OP0_MASK) >> CP_REG_ARM64_SYSREG_OP0_SHIFT, 226 (sysreg & CP_REG_ARM64_SYSREG_OP1_MASK) >> CP_REG_ARM64_SYSREG_OP1_SHIFT, 227 (sysreg & CP_REG_ARM64_SYSREG_CRN_MASK) >> CP_REG_ARM64_SYSREG_CRN_SHIFT, 228 (sysreg & CP_REG_ARM64_SYSREG_CRM_MASK) >> CP_REG_ARM64_SYSREG_CRM_SHIFT, 229 (sysreg & CP_REG_ARM64_SYSREG_OP2_MASK) >> CP_REG_ARM64_SYSREG_OP2_SHIFT); 230 } 231 232 /* read a sysreg value and store it in the idregs */ 233 static int get_host_cpu_reg(int fd, ARMHostCPUFeatures *ahcf, ARMIDRegisterIdx index) 234 { 235 uint64_t *reg; 236 int ret; 237 238 reg = &ahcf->isar.idregs[index]; 239 ret = read_sys_reg64(fd, reg, 240 idregs_sysreg_to_kvm_reg(id_register_sysreg[index])); 241 return ret; 242 } 243 244 static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) 245 { 246 /* Identify the feature bits corresponding to the host CPU, and 247 * fill out the ARMHostCPUClass fields accordingly. To do this 248 * we have to create a scratch VM, create a single CPU inside it, 249 * and then query that CPU for the relevant ID registers. 250 */ 251 int fdarray[3]; 252 bool sve_supported; 253 bool pmu_supported = false; 254 uint64_t features = 0; 255 int err; 256 257 /* 258 * target = -1 informs kvm_arm_create_scratch_host_vcpu() 259 * to use the preferred target 260 */ 261 struct kvm_vcpu_init init = { .target = -1, }; 262 263 /* 264 * Ask for SVE if supported, so that we can query ID_AA64ZFR0, 265 * which is otherwise RAZ. 266 */ 267 sve_supported = kvm_arm_sve_supported(); 268 if (sve_supported) { 269 init.features[0] |= 1 << KVM_ARM_VCPU_SVE; 270 } 271 272 /* 273 * Ask for Pointer Authentication if supported, so that we get 274 * the unsanitized field values for AA64ISAR1_EL1. 275 */ 276 if (kvm_arm_pauth_supported()) { 277 init.features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS | 278 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC); 279 } 280 281 if (kvm_arm_pmu_supported()) { 282 init.features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; 283 pmu_supported = true; 284 features |= 1ULL << ARM_FEATURE_PMU; 285 } 286 287 if (!kvm_arm_create_scratch_host_vcpu(fdarray, &init)) { 288 return false; 289 } 290 291 ahcf->target = init.target; 292 ahcf->dtb_compatible = "arm,arm-v8"; 293 int fd = fdarray[2]; 294 295 err = get_host_cpu_reg(fd, ahcf, ID_AA64PFR0_EL1_IDX); 296 if (unlikely(err < 0)) { 297 /* 298 * Before v4.15, the kernel only exposed a limited number of system 299 * registers, not including any of the interesting AArch64 ID regs. 300 * For the most part we could leave these fields as zero with minimal 301 * effect, since this does not affect the values seen by the guest. 302 * 303 * However, it could cause problems down the line for QEMU, 304 * so provide a minimal v8.0 default. 305 * 306 * ??? Could read MIDR and use knowledge from cpu64.c. 307 * ??? Could map a page of memory into our temp guest and 308 * run the tiniest of hand-crafted kernels to extract 309 * the values seen by the guest. 310 * ??? Either of these sounds like too much effort just 311 * to work around running a modern host kernel. 312 */ 313 SET_IDREG(&ahcf->isar, ID_AA64PFR0, 0x00000011); /* EL1&0, AArch64 only */ 314 err = 0; 315 } else { 316 err |= get_host_cpu_reg(fd, ahcf, ID_AA64PFR1_EL1_IDX); 317 err |= get_host_cpu_reg(fd, ahcf, ID_AA64SMFR0_EL1_IDX); 318 err |= get_host_cpu_reg(fd, ahcf, ID_AA64DFR0_EL1_IDX); 319 err |= get_host_cpu_reg(fd, ahcf, ID_AA64DFR1_EL1_IDX); 320 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR0_EL1_IDX); 321 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR1_EL1_IDX); 322 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ISAR2_EL1_IDX); 323 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR0_EL1_IDX); 324 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR1_EL1_IDX); 325 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR2_EL1_IDX); 326 err |= get_host_cpu_reg(fd, ahcf, ID_AA64MMFR3_EL1_IDX); 327 328 /* 329 * Note that if AArch32 support is not present in the host, 330 * the AArch32 sysregs are present to be read, but will 331 * return UNKNOWN values. This is neither better nor worse 332 * than skipping the reads and leaving 0, as we must avoid 333 * considering the values in every case. 334 */ 335 err |= get_host_cpu_reg(fd, ahcf, ID_PFR0_EL1_IDX); 336 err |= get_host_cpu_reg(fd, ahcf, ID_PFR1_EL1_IDX); 337 err |= get_host_cpu_reg(fd, ahcf, ID_DFR0_EL1_IDX); 338 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr0, 339 ARM64_SYS_REG(3, 0, 0, 1, 4)); 340 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr1, 341 ARM64_SYS_REG(3, 0, 0, 1, 5)); 342 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr2, 343 ARM64_SYS_REG(3, 0, 0, 1, 6)); 344 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr3, 345 ARM64_SYS_REG(3, 0, 0, 1, 7)); 346 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR0_EL1_IDX); 347 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR1_EL1_IDX); 348 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR2_EL1_IDX); 349 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR3_EL1_IDX); 350 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR4_EL1_IDX); 351 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR5_EL1_IDX); 352 err |= get_host_cpu_reg(fd, ahcf, ID_ISAR6_EL1_IDX); 353 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr4, 354 ARM64_SYS_REG(3, 0, 0, 2, 6)); 355 356 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr0, 357 ARM64_SYS_REG(3, 0, 0, 3, 0)); 358 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr1, 359 ARM64_SYS_REG(3, 0, 0, 3, 1)); 360 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr2, 361 ARM64_SYS_REG(3, 0, 0, 3, 2)); 362 err |= get_host_cpu_reg(fd, ahcf, ID_PFR2_EL1_IDX); 363 err |= get_host_cpu_reg(fd, ahcf, ID_DFR1_EL1_IDX); 364 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr5, 365 ARM64_SYS_REG(3, 0, 0, 3, 6)); 366 367 /* 368 * DBGDIDR is a bit complicated because the kernel doesn't 369 * provide an accessor for it in 64-bit mode, which is what this 370 * scratch VM is in, and there's no architected "64-bit sysreg 371 * which reads the same as the 32-bit register" the way there is 372 * for other ID registers. Instead we synthesize a value from the 373 * AArch64 ID_AA64DFR0, the same way the kernel code in 374 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does. 375 * We only do this if the CPU supports AArch32 at EL1. 376 */ 377 if (FIELD_EX32_IDREG(&ahcf->isar, ID_AA64PFR0, EL1) >= 2) { 378 int wrps = FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, WRPS); 379 int brps = FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, BRPS); 380 int ctx_cmps = 381 FIELD_EX64_IDREG(&ahcf->isar, ID_AA64DFR0, CTX_CMPS); 382 int version = 6; /* ARMv8 debug architecture */ 383 bool has_el3 = 384 !!FIELD_EX32_IDREG(&ahcf->isar, ID_AA64PFR0, EL3); 385 uint32_t dbgdidr = 0; 386 387 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps); 388 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps); 389 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps); 390 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version); 391 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3); 392 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3); 393 dbgdidr |= (1 << 15); /* RES1 bit */ 394 ahcf->isar.dbgdidr = dbgdidr; 395 } 396 397 if (pmu_supported) { 398 /* PMCR_EL0 is only accessible if the vCPU has feature PMU_V3 */ 399 err |= read_sys_reg64(fdarray[2], &ahcf->isar.reset_pmcr_el0, 400 ARM64_SYS_REG(3, 3, 9, 12, 0)); 401 } 402 403 if (sve_supported) { 404 /* 405 * There is a range of kernels between kernel commit 73433762fcae 406 * and f81cb2c3ad41 which have a bug where the kernel doesn't 407 * expose SYS_ID_AA64ZFR0_EL1 via the ONE_REG API unless the VM has 408 * enabled SVE support, which resulted in an error rather than RAZ. 409 * So only read the register if we set KVM_ARM_VCPU_SVE above. 410 */ 411 err |= get_host_cpu_reg(fd, ahcf, ID_AA64ZFR0_EL1_IDX); 412 } 413 } 414 415 kvm_arm_destroy_scratch_host_vcpu(fdarray); 416 417 if (err < 0) { 418 return false; 419 } 420 421 /* 422 * We can assume any KVM supporting CPU is at least a v8 423 * with VFPv4+Neon; this in turn implies most of the other 424 * feature bits. 425 */ 426 features |= 1ULL << ARM_FEATURE_V8; 427 features |= 1ULL << ARM_FEATURE_NEON; 428 features |= 1ULL << ARM_FEATURE_AARCH64; 429 features |= 1ULL << ARM_FEATURE_GENERIC_TIMER; 430 431 ahcf->features = features; 432 433 return true; 434 } 435 436 void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu) 437 { 438 CPUARMState *env = &cpu->env; 439 440 if (!arm_host_cpu_features.dtb_compatible) { 441 if (!kvm_enabled() || 442 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) { 443 /* We can't report this error yet, so flag that we need to 444 * in arm_cpu_realizefn(). 445 */ 446 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE; 447 cpu->host_cpu_probe_failed = true; 448 return; 449 } 450 } 451 452 cpu->kvm_target = arm_host_cpu_features.target; 453 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible; 454 cpu->isar = arm_host_cpu_features.isar; 455 env->features = arm_host_cpu_features.features; 456 } 457 458 static bool kvm_no_adjvtime_get(Object *obj, Error **errp) 459 { 460 return !ARM_CPU(obj)->kvm_adjvtime; 461 } 462 463 static void kvm_no_adjvtime_set(Object *obj, bool value, Error **errp) 464 { 465 ARM_CPU(obj)->kvm_adjvtime = !value; 466 } 467 468 static bool kvm_steal_time_get(Object *obj, Error **errp) 469 { 470 return ARM_CPU(obj)->kvm_steal_time != ON_OFF_AUTO_OFF; 471 } 472 473 static void kvm_steal_time_set(Object *obj, bool value, Error **errp) 474 { 475 ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 476 } 477 478 /* KVM VCPU properties should be prefixed with "kvm-". */ 479 void kvm_arm_add_vcpu_properties(ARMCPU *cpu) 480 { 481 CPUARMState *env = &cpu->env; 482 Object *obj = OBJECT(cpu); 483 484 if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) { 485 cpu->kvm_adjvtime = true; 486 object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get, 487 kvm_no_adjvtime_set); 488 object_property_set_description(obj, "kvm-no-adjvtime", 489 "Set on to disable the adjustment of " 490 "the virtual counter. VM stopped time " 491 "will be counted."); 492 } 493 494 cpu->kvm_steal_time = ON_OFF_AUTO_AUTO; 495 object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get, 496 kvm_steal_time_set); 497 object_property_set_description(obj, "kvm-steal-time", 498 "Set off to disable KVM steal time."); 499 } 500 501 bool kvm_arm_pmu_supported(void) 502 { 503 return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3); 504 } 505 506 int kvm_arm_get_max_vm_ipa_size(MachineState *ms, bool *fixed_ipa) 507 { 508 KVMState *s = KVM_STATE(ms->accelerator); 509 int ret; 510 511 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE); 512 *fixed_ipa = ret <= 0; 513 514 return ret > 0 ? ret : 40; 515 } 516 517 int kvm_arch_get_default_type(MachineState *ms) 518 { 519 bool fixed_ipa; 520 int size = kvm_arm_get_max_vm_ipa_size(ms, &fixed_ipa); 521 return fixed_ipa ? 0 : size; 522 } 523 524 int kvm_arch_init(MachineState *ms, KVMState *s) 525 { 526 int ret = 0; 527 /* For ARM interrupt delivery is always asynchronous, 528 * whether we are using an in-kernel VGIC or not. 529 */ 530 kvm_async_interrupts_allowed = true; 531 532 /* 533 * PSCI wakes up secondary cores, so we always need to 534 * have vCPUs waiting in kernel space 535 */ 536 kvm_halt_in_kernel_allowed = true; 537 538 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE); 539 540 /* Check whether user space can specify guest syndrome value */ 541 cap_has_inject_serror_esr = 542 kvm_check_extension(s, KVM_CAP_ARM_INJECT_SERROR_ESR); 543 544 if (ms->smp.cpus > 256 && 545 !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) { 546 error_report("Using more than 256 vcpus requires a host kernel " 547 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2"); 548 ret = -EINVAL; 549 } 550 551 if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) { 552 if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) { 553 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap"); 554 } else { 555 /* Set status for supporting the external dabt injection */ 556 cap_has_inject_ext_dabt = kvm_check_extension(s, 557 KVM_CAP_ARM_INJECT_EXT_DABT); 558 } 559 } 560 561 if (s->kvm_eager_split_size) { 562 uint32_t sizes; 563 564 sizes = kvm_vm_check_extension(s, KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES); 565 if (!sizes) { 566 s->kvm_eager_split_size = 0; 567 warn_report("Eager Page Split support not available"); 568 } else if (!(s->kvm_eager_split_size & sizes)) { 569 error_report("Eager Page Split requested chunk size not valid"); 570 ret = -EINVAL; 571 } else { 572 ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE, 0, 573 s->kvm_eager_split_size); 574 if (ret < 0) { 575 error_report("Enabling of Eager Page Split failed: %s", 576 strerror(-ret)); 577 } 578 } 579 } 580 581 max_hw_wps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_WPS); 582 hw_watchpoints = g_array_sized_new(true, true, 583 sizeof(HWWatchpoint), max_hw_wps); 584 585 max_hw_bps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_BPS); 586 hw_breakpoints = g_array_sized_new(true, true, 587 sizeof(HWBreakpoint), max_hw_bps); 588 589 return ret; 590 } 591 592 unsigned long kvm_arch_vcpu_id(CPUState *cpu) 593 { 594 return cpu->cpu_index; 595 } 596 597 /* We track all the KVM devices which need their memory addresses 598 * passing to the kernel in a list of these structures. 599 * When board init is complete we run through the list and 600 * tell the kernel the base addresses of the memory regions. 601 * We use a MemoryListener to track mapping and unmapping of 602 * the regions during board creation, so the board models don't 603 * need to do anything special for the KVM case. 604 * 605 * Sometimes the address must be OR'ed with some other fields 606 * (for example for KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION). 607 * @kda_addr_ormask aims at storing the value of those fields. 608 */ 609 typedef struct KVMDevice { 610 struct kvm_arm_device_addr kda; 611 struct kvm_device_attr kdattr; 612 uint64_t kda_addr_ormask; 613 MemoryRegion *mr; 614 QSLIST_ENTRY(KVMDevice) entries; 615 int dev_fd; 616 } KVMDevice; 617 618 static QSLIST_HEAD(, KVMDevice) kvm_devices_head; 619 620 static void kvm_arm_devlistener_add(MemoryListener *listener, 621 MemoryRegionSection *section) 622 { 623 KVMDevice *kd; 624 625 QSLIST_FOREACH(kd, &kvm_devices_head, entries) { 626 if (section->mr == kd->mr) { 627 kd->kda.addr = section->offset_within_address_space; 628 } 629 } 630 } 631 632 static void kvm_arm_devlistener_del(MemoryListener *listener, 633 MemoryRegionSection *section) 634 { 635 KVMDevice *kd; 636 637 QSLIST_FOREACH(kd, &kvm_devices_head, entries) { 638 if (section->mr == kd->mr) { 639 kd->kda.addr = -1; 640 } 641 } 642 } 643 644 static MemoryListener devlistener = { 645 .name = "kvm-arm", 646 .region_add = kvm_arm_devlistener_add, 647 .region_del = kvm_arm_devlistener_del, 648 .priority = MEMORY_LISTENER_PRIORITY_MIN, 649 }; 650 651 static void kvm_arm_set_device_addr(KVMDevice *kd) 652 { 653 struct kvm_device_attr *attr = &kd->kdattr; 654 int ret; 655 uint64_t addr = kd->kda.addr; 656 657 addr |= kd->kda_addr_ormask; 658 attr->addr = (uintptr_t)&addr; 659 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr); 660 661 if (ret < 0) { 662 fprintf(stderr, "Failed to set device address: %s\n", 663 strerror(-ret)); 664 abort(); 665 } 666 } 667 668 static void kvm_arm_machine_init_done(Notifier *notifier, void *data) 669 { 670 KVMDevice *kd, *tkd; 671 672 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) { 673 if (kd->kda.addr != -1) { 674 kvm_arm_set_device_addr(kd); 675 } 676 memory_region_unref(kd->mr); 677 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries); 678 g_free(kd); 679 } 680 memory_listener_unregister(&devlistener); 681 } 682 683 static Notifier notify = { 684 .notify = kvm_arm_machine_init_done, 685 }; 686 687 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group, 688 uint64_t attr, int dev_fd, uint64_t addr_ormask) 689 { 690 KVMDevice *kd; 691 692 if (!kvm_irqchip_in_kernel()) { 693 return; 694 } 695 696 if (QSLIST_EMPTY(&kvm_devices_head)) { 697 memory_listener_register(&devlistener, &address_space_memory); 698 qemu_add_machine_init_done_notifier(¬ify); 699 } 700 kd = g_new0(KVMDevice, 1); 701 kd->mr = mr; 702 kd->kda.id = devid; 703 kd->kda.addr = -1; 704 kd->kdattr.flags = 0; 705 kd->kdattr.group = group; 706 kd->kdattr.attr = attr; 707 kd->dev_fd = dev_fd; 708 kd->kda_addr_ormask = addr_ormask; 709 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries); 710 memory_region_ref(kd->mr); 711 } 712 713 static int compare_u64(const void *a, const void *b) 714 { 715 if (*(uint64_t *)a > *(uint64_t *)b) { 716 return 1; 717 } 718 if (*(uint64_t *)a < *(uint64_t *)b) { 719 return -1; 720 } 721 return 0; 722 } 723 724 /* 725 * cpreg_values are sorted in ascending order by KVM register ID 726 * (see kvm_arm_init_cpreg_list). This allows us to cheaply find 727 * the storage for a KVM register by ID with a binary search. 728 */ 729 static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx) 730 { 731 uint64_t *res; 732 733 res = bsearch(®idx, cpu->cpreg_indexes, cpu->cpreg_array_len, 734 sizeof(uint64_t), compare_u64); 735 assert(res); 736 737 return &cpu->cpreg_values[res - cpu->cpreg_indexes]; 738 } 739 740 /** 741 * kvm_arm_reg_syncs_via_cpreg_list: 742 * @regidx: KVM register index 743 * 744 * Return true if this KVM register should be synchronized via the 745 * cpreg list of arbitrary system registers, false if it is synchronized 746 * by hand using code in kvm_arch_get/put_registers(). 747 */ 748 static bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) 749 { 750 switch (regidx & KVM_REG_ARM_COPROC_MASK) { 751 case KVM_REG_ARM_CORE: 752 case KVM_REG_ARM64_SVE: 753 return false; 754 default: 755 return true; 756 } 757 } 758 759 /** 760 * kvm_arm_init_cpreg_list: 761 * @cpu: ARMCPU 762 * 763 * Initialize the ARMCPU cpreg list according to the kernel's 764 * definition of what CPU registers it knows about (and throw away 765 * the previous TCG-created cpreg list). 766 * 767 * Returns: 0 if success, else < 0 error code 768 */ 769 static int kvm_arm_init_cpreg_list(ARMCPU *cpu) 770 { 771 struct kvm_reg_list rl; 772 struct kvm_reg_list *rlp; 773 int i, ret, arraylen; 774 CPUState *cs = CPU(cpu); 775 776 rl.n = 0; 777 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl); 778 if (ret != -E2BIG) { 779 return ret; 780 } 781 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t)); 782 rlp->n = rl.n; 783 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp); 784 if (ret) { 785 goto out; 786 } 787 /* Sort the list we get back from the kernel, since cpreg_tuples 788 * must be in strictly ascending order. 789 */ 790 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64); 791 792 for (i = 0, arraylen = 0; i < rlp->n; i++) { 793 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) { 794 continue; 795 } 796 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) { 797 case KVM_REG_SIZE_U32: 798 case KVM_REG_SIZE_U64: 799 break; 800 default: 801 fprintf(stderr, "Can't handle size of register in kernel list\n"); 802 ret = -EINVAL; 803 goto out; 804 } 805 806 arraylen++; 807 } 808 809 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen); 810 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen); 811 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes, 812 arraylen); 813 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values, 814 arraylen); 815 cpu->cpreg_array_len = arraylen; 816 cpu->cpreg_vmstate_array_len = arraylen; 817 818 for (i = 0, arraylen = 0; i < rlp->n; i++) { 819 uint64_t regidx = rlp->reg[i]; 820 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) { 821 continue; 822 } 823 cpu->cpreg_indexes[arraylen] = regidx; 824 arraylen++; 825 } 826 assert(cpu->cpreg_array_len == arraylen); 827 828 if (!write_kvmstate_to_list(cpu)) { 829 /* Shouldn't happen unless kernel is inconsistent about 830 * what registers exist. 831 */ 832 fprintf(stderr, "Initial read of kernel register state failed\n"); 833 ret = -EINVAL; 834 goto out; 835 } 836 837 out: 838 g_free(rlp); 839 return ret; 840 } 841 842 /** 843 * kvm_arm_cpreg_level: 844 * @regidx: KVM register index 845 * 846 * Return the level of this coprocessor/system register. Return value is 847 * either KVM_PUT_RUNTIME_STATE, KVM_PUT_RESET_STATE, or KVM_PUT_FULL_STATE. 848 */ 849 static int kvm_arm_cpreg_level(uint64_t regidx) 850 { 851 /* 852 * All system registers are assumed to be level KVM_PUT_RUNTIME_STATE. 853 * If a register should be written less often, you must add it here 854 * with a state of either KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE. 855 */ 856 switch (regidx) { 857 case KVM_REG_ARM_TIMER_CNT: 858 case KVM_REG_ARM_PTIMER_CNT: 859 return KVM_PUT_FULL_STATE; 860 } 861 return KVM_PUT_RUNTIME_STATE; 862 } 863 864 bool write_kvmstate_to_list(ARMCPU *cpu) 865 { 866 CPUState *cs = CPU(cpu); 867 int i; 868 bool ok = true; 869 870 for (i = 0; i < cpu->cpreg_array_len; i++) { 871 uint64_t regidx = cpu->cpreg_indexes[i]; 872 uint32_t v32; 873 int ret; 874 875 switch (regidx & KVM_REG_SIZE_MASK) { 876 case KVM_REG_SIZE_U32: 877 ret = kvm_get_one_reg(cs, regidx, &v32); 878 if (!ret) { 879 cpu->cpreg_values[i] = v32; 880 } 881 break; 882 case KVM_REG_SIZE_U64: 883 ret = kvm_get_one_reg(cs, regidx, cpu->cpreg_values + i); 884 break; 885 default: 886 g_assert_not_reached(); 887 } 888 if (ret) { 889 ok = false; 890 } 891 } 892 return ok; 893 } 894 895 bool write_list_to_kvmstate(ARMCPU *cpu, int level) 896 { 897 CPUState *cs = CPU(cpu); 898 int i; 899 bool ok = true; 900 901 for (i = 0; i < cpu->cpreg_array_len; i++) { 902 uint64_t regidx = cpu->cpreg_indexes[i]; 903 uint32_t v32; 904 int ret; 905 906 if (kvm_arm_cpreg_level(regidx) > level) { 907 continue; 908 } 909 910 switch (regidx & KVM_REG_SIZE_MASK) { 911 case KVM_REG_SIZE_U32: 912 v32 = cpu->cpreg_values[i]; 913 ret = kvm_set_one_reg(cs, regidx, &v32); 914 break; 915 case KVM_REG_SIZE_U64: 916 ret = kvm_set_one_reg(cs, regidx, cpu->cpreg_values + i); 917 break; 918 default: 919 g_assert_not_reached(); 920 } 921 if (ret) { 922 /* We might fail for "unknown register" and also for 923 * "you tried to set a register which is constant with 924 * a different value from what it actually contains". 925 */ 926 ok = false; 927 } 928 } 929 return ok; 930 } 931 932 void kvm_arm_cpu_pre_save(ARMCPU *cpu) 933 { 934 /* KVM virtual time adjustment */ 935 if (cpu->kvm_vtime_dirty) { 936 *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT) = cpu->kvm_vtime; 937 } 938 } 939 940 bool kvm_arm_cpu_post_load(ARMCPU *cpu) 941 { 942 if (!write_list_to_kvmstate(cpu, KVM_PUT_FULL_STATE)) { 943 return false; 944 } 945 /* Note that it's OK for the TCG side not to know about 946 * every register in the list; KVM is authoritative if 947 * we're using it. 948 */ 949 write_list_to_cpustate(cpu); 950 951 /* KVM virtual time adjustment */ 952 if (cpu->kvm_adjvtime) { 953 cpu->kvm_vtime = *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT); 954 cpu->kvm_vtime_dirty = true; 955 } 956 957 return true; 958 } 959 960 void kvm_arm_reset_vcpu(ARMCPU *cpu) 961 { 962 int ret; 963 964 /* Re-init VCPU so that all registers are set to 965 * their respective reset values. 966 */ 967 ret = kvm_arm_vcpu_init(cpu); 968 if (ret < 0) { 969 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret)); 970 abort(); 971 } 972 if (!write_kvmstate_to_list(cpu)) { 973 fprintf(stderr, "write_kvmstate_to_list failed\n"); 974 abort(); 975 } 976 /* 977 * Sync the reset values also into the CPUState. This is necessary 978 * because the next thing we do will be a kvm_arch_put_registers() 979 * which will update the list values from the CPUState before copying 980 * the list values back to KVM. It's OK to ignore failure returns here 981 * for the same reason we do so in kvm_arch_get_registers(). 982 */ 983 write_list_to_cpustate(cpu); 984 } 985 986 /* 987 * Update KVM's MP_STATE based on what QEMU thinks it is 988 */ 989 static int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu) 990 { 991 if (cap_has_mp_state) { 992 struct kvm_mp_state mp_state = { 993 .mp_state = (cpu->power_state == PSCI_OFF) ? 994 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE 995 }; 996 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state); 997 } 998 return 0; 999 } 1000 1001 /* 1002 * Sync the KVM MP_STATE into QEMU 1003 */ 1004 static int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu) 1005 { 1006 if (cap_has_mp_state) { 1007 struct kvm_mp_state mp_state; 1008 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state); 1009 if (ret) { 1010 return ret; 1011 } 1012 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ? 1013 PSCI_OFF : PSCI_ON; 1014 } 1015 return 0; 1016 } 1017 1018 /** 1019 * kvm_arm_get_virtual_time: 1020 * @cpu: ARMCPU 1021 * 1022 * Gets the VCPU's virtual counter and stores it in the KVM CPU state. 1023 */ 1024 static void kvm_arm_get_virtual_time(ARMCPU *cpu) 1025 { 1026 int ret; 1027 1028 if (cpu->kvm_vtime_dirty) { 1029 return; 1030 } 1031 1032 ret = kvm_get_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime); 1033 if (ret) { 1034 error_report("Failed to get KVM_REG_ARM_TIMER_CNT"); 1035 abort(); 1036 } 1037 1038 cpu->kvm_vtime_dirty = true; 1039 } 1040 1041 /** 1042 * kvm_arm_put_virtual_time: 1043 * @cpu: ARMCPU 1044 * 1045 * Sets the VCPU's virtual counter to the value stored in the KVM CPU state. 1046 */ 1047 static void kvm_arm_put_virtual_time(ARMCPU *cpu) 1048 { 1049 int ret; 1050 1051 if (!cpu->kvm_vtime_dirty) { 1052 return; 1053 } 1054 1055 ret = kvm_set_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime); 1056 if (ret) { 1057 error_report("Failed to set KVM_REG_ARM_TIMER_CNT"); 1058 abort(); 1059 } 1060 1061 cpu->kvm_vtime_dirty = false; 1062 } 1063 1064 /** 1065 * kvm_put_vcpu_events: 1066 * @cpu: ARMCPU 1067 * 1068 * Put VCPU related state to kvm. 1069 * 1070 * Returns: 0 if success else < 0 error code 1071 */ 1072 static int kvm_put_vcpu_events(ARMCPU *cpu) 1073 { 1074 CPUARMState *env = &cpu->env; 1075 struct kvm_vcpu_events events; 1076 int ret; 1077 1078 if (!kvm_has_vcpu_events()) { 1079 return 0; 1080 } 1081 1082 memset(&events, 0, sizeof(events)); 1083 events.exception.serror_pending = env->serror.pending; 1084 1085 /* Inject SError to guest with specified syndrome if host kernel 1086 * supports it, otherwise inject SError without syndrome. 1087 */ 1088 if (cap_has_inject_serror_esr) { 1089 events.exception.serror_has_esr = env->serror.has_esr; 1090 events.exception.serror_esr = env->serror.esr; 1091 } 1092 1093 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events); 1094 if (ret) { 1095 error_report("failed to put vcpu events"); 1096 } 1097 1098 return ret; 1099 } 1100 1101 /** 1102 * kvm_get_vcpu_events: 1103 * @cpu: ARMCPU 1104 * 1105 * Get VCPU related state from kvm. 1106 * 1107 * Returns: 0 if success else < 0 error code 1108 */ 1109 static int kvm_get_vcpu_events(ARMCPU *cpu) 1110 { 1111 CPUARMState *env = &cpu->env; 1112 struct kvm_vcpu_events events; 1113 int ret; 1114 1115 if (!kvm_has_vcpu_events()) { 1116 return 0; 1117 } 1118 1119 memset(&events, 0, sizeof(events)); 1120 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events); 1121 if (ret) { 1122 error_report("failed to get vcpu events"); 1123 return ret; 1124 } 1125 1126 env->serror.pending = events.exception.serror_pending; 1127 env->serror.has_esr = events.exception.serror_has_esr; 1128 env->serror.esr = events.exception.serror_esr; 1129 1130 return 0; 1131 } 1132 1133 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0) 1134 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2) 1135 1136 /* 1137 * ESR_EL1 1138 * ISS encoding 1139 * AARCH64: DFSC, bits [5:0] 1140 * AARCH32: 1141 * TTBCR.EAE == 0 1142 * FS[4] - DFSR[10] 1143 * FS[3:0] - DFSR[3:0] 1144 * TTBCR.EAE == 1 1145 * FS, bits [5:0] 1146 */ 1147 #define ESR_DFSC(aarch64, lpae, v) \ 1148 ((aarch64 || (lpae)) ? ((v) & 0x3F) \ 1149 : (((v) >> 6) | ((v) & 0x1F))) 1150 1151 #define ESR_DFSC_EXTABT(aarch64, lpae) \ 1152 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8) 1153 1154 /** 1155 * kvm_arm_verify_ext_dabt_pending: 1156 * @cpu: ARMCPU 1157 * 1158 * Verify the fault status code wrt the Ext DABT injection 1159 * 1160 * Returns: true if the fault status code is as expected, false otherwise 1161 */ 1162 static bool kvm_arm_verify_ext_dabt_pending(ARMCPU *cpu) 1163 { 1164 CPUState *cs = CPU(cpu); 1165 uint64_t dfsr_val; 1166 1167 if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) { 1168 CPUARMState *env = &cpu->env; 1169 int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64); 1170 int lpae = 0; 1171 1172 if (!aarch64_mode) { 1173 uint64_t ttbcr; 1174 1175 if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) { 1176 lpae = arm_feature(env, ARM_FEATURE_LPAE) 1177 && (ttbcr & TTBCR_EAE); 1178 } 1179 } 1180 /* 1181 * The verification here is based on the DFSC bits 1182 * of the ESR_EL1 reg only 1183 */ 1184 return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) == 1185 ESR_DFSC_EXTABT(aarch64_mode, lpae)); 1186 } 1187 return false; 1188 } 1189 1190 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run) 1191 { 1192 ARMCPU *cpu = ARM_CPU(cs); 1193 CPUARMState *env = &cpu->env; 1194 1195 if (unlikely(env->ext_dabt_raised)) { 1196 /* 1197 * Verifying that the ext DABT has been properly injected, 1198 * otherwise risking indefinitely re-running the faulting instruction 1199 * Covering a very narrow case for kernels 5.5..5.5.4 1200 * when injected abort was misconfigured to be 1201 * an IMPLEMENTATION DEFINED exception (for 32-bit EL1) 1202 */ 1203 if (!arm_feature(env, ARM_FEATURE_AARCH64) && 1204 unlikely(!kvm_arm_verify_ext_dabt_pending(cpu))) { 1205 1206 error_report("Data abort exception with no valid ISS generated by " 1207 "guest memory access. KVM unable to emulate faulting " 1208 "instruction. Failed to inject an external data abort " 1209 "into the guest."); 1210 abort(); 1211 } 1212 /* Clear the status */ 1213 env->ext_dabt_raised = 0; 1214 } 1215 } 1216 1217 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run) 1218 { 1219 ARMCPU *cpu; 1220 uint32_t switched_level; 1221 1222 if (kvm_irqchip_in_kernel()) { 1223 /* 1224 * We only need to sync timer states with user-space interrupt 1225 * controllers, so return early and save cycles if we don't. 1226 */ 1227 return MEMTXATTRS_UNSPECIFIED; 1228 } 1229 1230 cpu = ARM_CPU(cs); 1231 1232 /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */ 1233 if (run->s.regs.device_irq_level != cpu->device_irq_level) { 1234 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level; 1235 1236 bql_lock(); 1237 1238 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) { 1239 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT], 1240 !!(run->s.regs.device_irq_level & 1241 KVM_ARM_DEV_EL1_VTIMER)); 1242 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER; 1243 } 1244 1245 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) { 1246 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS], 1247 !!(run->s.regs.device_irq_level & 1248 KVM_ARM_DEV_EL1_PTIMER)); 1249 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER; 1250 } 1251 1252 if (switched_level & KVM_ARM_DEV_PMU) { 1253 qemu_set_irq(cpu->pmu_interrupt, 1254 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU)); 1255 switched_level &= ~KVM_ARM_DEV_PMU; 1256 } 1257 1258 if (switched_level) { 1259 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n", 1260 __func__, switched_level); 1261 } 1262 1263 /* We also mark unknown levels as processed to not waste cycles */ 1264 cpu->device_irq_level = run->s.regs.device_irq_level; 1265 bql_unlock(); 1266 } 1267 1268 return MEMTXATTRS_UNSPECIFIED; 1269 } 1270 1271 static void kvm_arm_vm_state_change(void *opaque, bool running, RunState state) 1272 { 1273 ARMCPU *cpu = opaque; 1274 1275 if (running) { 1276 if (cpu->kvm_adjvtime) { 1277 kvm_arm_put_virtual_time(cpu); 1278 } 1279 } else { 1280 if (cpu->kvm_adjvtime) { 1281 kvm_arm_get_virtual_time(cpu); 1282 } 1283 } 1284 } 1285 1286 /** 1287 * kvm_arm_handle_dabt_nisv: 1288 * @cpu: ARMCPU 1289 * @esr_iss: ISS encoding (limited) for the exception from Data Abort 1290 * ISV bit set to '0b0' -> no valid instruction syndrome 1291 * @fault_ipa: faulting address for the synchronous data abort 1292 * 1293 * Returns: 0 if the exception has been handled, < 0 otherwise 1294 */ 1295 static int kvm_arm_handle_dabt_nisv(ARMCPU *cpu, uint64_t esr_iss, 1296 uint64_t fault_ipa) 1297 { 1298 CPUARMState *env = &cpu->env; 1299 /* 1300 * Request KVM to inject the external data abort into the guest 1301 */ 1302 if (cap_has_inject_ext_dabt) { 1303 struct kvm_vcpu_events events = { }; 1304 /* 1305 * The external data abort event will be handled immediately by KVM 1306 * using the address fault that triggered the exit on given VCPU. 1307 * Requesting injection of the external data abort does not rely 1308 * on any other VCPU state. Therefore, in this particular case, the VCPU 1309 * synchronization can be exceptionally skipped. 1310 */ 1311 events.exception.ext_dabt_pending = 1; 1312 /* KVM_CAP_ARM_INJECT_EXT_DABT implies KVM_CAP_VCPU_EVENTS */ 1313 if (!kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events)) { 1314 env->ext_dabt_raised = 1; 1315 return 0; 1316 } 1317 } else { 1318 error_report("Data abort exception triggered by guest memory access " 1319 "at physical address: 0x" TARGET_FMT_lx, 1320 (target_ulong)fault_ipa); 1321 error_printf("KVM unable to emulate faulting instruction.\n"); 1322 } 1323 return -1; 1324 } 1325 1326 /** 1327 * kvm_arm_handle_debug: 1328 * @cpu: ARMCPU 1329 * @debug_exit: debug part of the KVM exit structure 1330 * 1331 * Returns: TRUE if the debug exception was handled. 1332 * 1333 * See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register 1334 * 1335 * To minimise translating between kernel and user-space the kernel 1336 * ABI just provides user-space with the full exception syndrome 1337 * register value to be decoded in QEMU. 1338 */ 1339 static bool kvm_arm_handle_debug(ARMCPU *cpu, 1340 struct kvm_debug_exit_arch *debug_exit) 1341 { 1342 int hsr_ec = syn_get_ec(debug_exit->hsr); 1343 CPUState *cs = CPU(cpu); 1344 CPUARMState *env = &cpu->env; 1345 1346 /* Ensure PC is synchronised */ 1347 kvm_cpu_synchronize_state(cs); 1348 1349 switch (hsr_ec) { 1350 case EC_SOFTWARESTEP: 1351 if (cs->singlestep_enabled) { 1352 return true; 1353 } else { 1354 /* 1355 * The kernel should have suppressed the guest's ability to 1356 * single step at this point so something has gone wrong. 1357 */ 1358 error_report("%s: guest single-step while debugging unsupported" 1359 " (%"PRIx64", %"PRIx32")", 1360 __func__, env->pc, debug_exit->hsr); 1361 return false; 1362 } 1363 break; 1364 case EC_AA64_BKPT: 1365 if (kvm_find_sw_breakpoint(cs, env->pc)) { 1366 return true; 1367 } 1368 break; 1369 case EC_BREAKPOINT: 1370 if (find_hw_breakpoint(cs, env->pc)) { 1371 return true; 1372 } 1373 break; 1374 case EC_WATCHPOINT: 1375 { 1376 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far); 1377 if (wp) { 1378 cs->watchpoint_hit = wp; 1379 return true; 1380 } 1381 break; 1382 } 1383 default: 1384 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")", 1385 __func__, debug_exit->hsr, env->pc); 1386 } 1387 1388 /* If we are not handling the debug exception it must belong to 1389 * the guest. Let's re-use the existing TCG interrupt code to set 1390 * everything up properly. 1391 */ 1392 cs->exception_index = EXCP_BKPT; 1393 env->exception.syndrome = debug_exit->hsr; 1394 env->exception.vaddress = debug_exit->far; 1395 env->exception.target_el = 1; 1396 bql_lock(); 1397 arm_cpu_do_interrupt(cs); 1398 bql_unlock(); 1399 1400 return false; 1401 } 1402 1403 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) 1404 { 1405 ARMCPU *cpu = ARM_CPU(cs); 1406 int ret = 0; 1407 1408 switch (run->exit_reason) { 1409 case KVM_EXIT_DEBUG: 1410 if (kvm_arm_handle_debug(cpu, &run->debug.arch)) { 1411 ret = EXCP_DEBUG; 1412 } /* otherwise return to guest */ 1413 break; 1414 case KVM_EXIT_ARM_NISV: 1415 /* External DABT with no valid iss to decode */ 1416 ret = kvm_arm_handle_dabt_nisv(cpu, run->arm_nisv.esr_iss, 1417 run->arm_nisv.fault_ipa); 1418 break; 1419 default: 1420 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n", 1421 __func__, run->exit_reason); 1422 break; 1423 } 1424 return ret; 1425 } 1426 1427 bool kvm_arch_stop_on_emulation_error(CPUState *cs) 1428 { 1429 return true; 1430 } 1431 1432 int kvm_arch_process_async_events(CPUState *cs) 1433 { 1434 return 0; 1435 } 1436 1437 /** 1438 * kvm_arm_hw_debug_active: 1439 * @cpu: ARMCPU 1440 * 1441 * Return: TRUE if any hardware breakpoints in use. 1442 */ 1443 static bool kvm_arm_hw_debug_active(ARMCPU *cpu) 1444 { 1445 return ((cur_hw_wps > 0) || (cur_hw_bps > 0)); 1446 } 1447 1448 /** 1449 * kvm_arm_copy_hw_debug_data: 1450 * @ptr: kvm_guest_debug_arch structure 1451 * 1452 * Copy the architecture specific debug registers into the 1453 * kvm_guest_debug ioctl structure. 1454 */ 1455 static void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr) 1456 { 1457 int i; 1458 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch)); 1459 1460 for (i = 0; i < max_hw_wps; i++) { 1461 HWWatchpoint *wp = get_hw_wp(i); 1462 ptr->dbg_wcr[i] = wp->wcr; 1463 ptr->dbg_wvr[i] = wp->wvr; 1464 } 1465 for (i = 0; i < max_hw_bps; i++) { 1466 HWBreakpoint *bp = get_hw_bp(i); 1467 ptr->dbg_bcr[i] = bp->bcr; 1468 ptr->dbg_bvr[i] = bp->bvr; 1469 } 1470 } 1471 1472 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg) 1473 { 1474 if (kvm_sw_breakpoints_active(cs)) { 1475 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP; 1476 } 1477 if (kvm_arm_hw_debug_active(ARM_CPU(cs))) { 1478 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW; 1479 kvm_arm_copy_hw_debug_data(&dbg->arch); 1480 } 1481 } 1482 1483 void kvm_arch_init_irq_routing(KVMState *s) 1484 { 1485 } 1486 1487 int kvm_arch_irqchip_create(KVMState *s) 1488 { 1489 if (kvm_kernel_irqchip_split()) { 1490 error_report("-machine kernel_irqchip=split is not supported on ARM."); 1491 exit(1); 1492 } 1493 1494 /* If we can create the VGIC using the newer device control API, we 1495 * let the device do this when it initializes itself, otherwise we 1496 * fall back to the old API */ 1497 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL); 1498 } 1499 1500 int kvm_arm_vgic_probe(void) 1501 { 1502 int val = 0; 1503 1504 if (kvm_create_device(kvm_state, 1505 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) { 1506 val |= KVM_ARM_VGIC_V3; 1507 } 1508 if (kvm_create_device(kvm_state, 1509 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) { 1510 val |= KVM_ARM_VGIC_V2; 1511 } 1512 return val; 1513 } 1514 1515 int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level) 1516 { 1517 int kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq; 1518 int cpu_idx1 = cpu % 256; 1519 int cpu_idx2 = cpu / 256; 1520 1521 kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) | 1522 (cpu_idx2 << KVM_ARM_IRQ_VCPU2_SHIFT); 1523 1524 return kvm_set_irq(kvm_state, kvm_irq, !!level); 1525 } 1526 1527 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route, 1528 uint64_t address, uint32_t data, PCIDevice *dev) 1529 { 1530 AddressSpace *as = pci_device_iommu_address_space(dev); 1531 hwaddr xlat, len, doorbell_gpa; 1532 MemoryRegionSection mrs; 1533 MemoryRegion *mr; 1534 1535 if (as == &address_space_memory) { 1536 return 0; 1537 } 1538 1539 /* MSI doorbell address is translated by an IOMMU */ 1540 1541 RCU_READ_LOCK_GUARD(); 1542 1543 mr = address_space_translate(as, address, &xlat, &len, true, 1544 MEMTXATTRS_UNSPECIFIED); 1545 1546 if (!mr) { 1547 return 1; 1548 } 1549 1550 mrs = memory_region_find(mr, xlat, 1); 1551 1552 if (!mrs.mr) { 1553 return 1; 1554 } 1555 1556 doorbell_gpa = mrs.offset_within_address_space; 1557 memory_region_unref(mrs.mr); 1558 1559 route->u.msi.address_lo = doorbell_gpa; 1560 route->u.msi.address_hi = doorbell_gpa >> 32; 1561 1562 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa); 1563 1564 return 0; 1565 } 1566 1567 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route, 1568 int vector, PCIDevice *dev) 1569 { 1570 return 0; 1571 } 1572 1573 int kvm_arch_release_virq_post(int virq) 1574 { 1575 return 0; 1576 } 1577 1578 int kvm_arch_msi_data_to_gsi(uint32_t data) 1579 { 1580 return (data - 32) & 0xffff; 1581 } 1582 1583 static void kvm_arch_get_eager_split_size(Object *obj, Visitor *v, 1584 const char *name, void *opaque, 1585 Error **errp) 1586 { 1587 KVMState *s = KVM_STATE(obj); 1588 uint64_t value = s->kvm_eager_split_size; 1589 1590 visit_type_size(v, name, &value, errp); 1591 } 1592 1593 static void kvm_arch_set_eager_split_size(Object *obj, Visitor *v, 1594 const char *name, void *opaque, 1595 Error **errp) 1596 { 1597 KVMState *s = KVM_STATE(obj); 1598 uint64_t value; 1599 1600 if (s->fd != -1) { 1601 error_setg(errp, "Unable to set early-split-size after KVM has been initialized"); 1602 return; 1603 } 1604 1605 if (!visit_type_size(v, name, &value, errp)) { 1606 return; 1607 } 1608 1609 if (value && !is_power_of_2(value)) { 1610 error_setg(errp, "early-split-size must be a power of two"); 1611 return; 1612 } 1613 1614 s->kvm_eager_split_size = value; 1615 } 1616 1617 void kvm_arch_accel_class_init(ObjectClass *oc) 1618 { 1619 object_class_property_add(oc, "eager-split-size", "size", 1620 kvm_arch_get_eager_split_size, 1621 kvm_arch_set_eager_split_size, NULL, NULL); 1622 1623 object_class_property_set_description(oc, "eager-split-size", 1624 "Eager Page Split chunk size for hugepages. (default: 0, disabled)"); 1625 } 1626 1627 int kvm_arch_insert_hw_breakpoint(vaddr addr, vaddr len, int type) 1628 { 1629 switch (type) { 1630 case GDB_BREAKPOINT_HW: 1631 return insert_hw_breakpoint(addr); 1632 break; 1633 case GDB_WATCHPOINT_READ: 1634 case GDB_WATCHPOINT_WRITE: 1635 case GDB_WATCHPOINT_ACCESS: 1636 return insert_hw_watchpoint(addr, len, type); 1637 default: 1638 return -ENOSYS; 1639 } 1640 } 1641 1642 int kvm_arch_remove_hw_breakpoint(vaddr addr, vaddr len, int type) 1643 { 1644 switch (type) { 1645 case GDB_BREAKPOINT_HW: 1646 return delete_hw_breakpoint(addr); 1647 case GDB_WATCHPOINT_READ: 1648 case GDB_WATCHPOINT_WRITE: 1649 case GDB_WATCHPOINT_ACCESS: 1650 return delete_hw_watchpoint(addr, len, type); 1651 default: 1652 return -ENOSYS; 1653 } 1654 } 1655 1656 void kvm_arch_remove_all_hw_breakpoints(void) 1657 { 1658 if (cur_hw_wps > 0) { 1659 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps); 1660 } 1661 if (cur_hw_bps > 0) { 1662 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps); 1663 } 1664 } 1665 1666 static bool kvm_arm_set_device_attr(ARMCPU *cpu, struct kvm_device_attr *attr, 1667 const char *name) 1668 { 1669 int err; 1670 1671 err = kvm_vcpu_ioctl(CPU(cpu), KVM_HAS_DEVICE_ATTR, attr); 1672 if (err != 0) { 1673 error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err)); 1674 return false; 1675 } 1676 1677 err = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEVICE_ATTR, attr); 1678 if (err != 0) { 1679 error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err)); 1680 return false; 1681 } 1682 1683 return true; 1684 } 1685 1686 void kvm_arm_pmu_init(ARMCPU *cpu) 1687 { 1688 struct kvm_device_attr attr = { 1689 .group = KVM_ARM_VCPU_PMU_V3_CTRL, 1690 .attr = KVM_ARM_VCPU_PMU_V3_INIT, 1691 }; 1692 1693 if (!cpu->has_pmu) { 1694 return; 1695 } 1696 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) { 1697 error_report("failed to init PMU"); 1698 abort(); 1699 } 1700 } 1701 1702 void kvm_arm_pmu_set_irq(ARMCPU *cpu, int irq) 1703 { 1704 struct kvm_device_attr attr = { 1705 .group = KVM_ARM_VCPU_PMU_V3_CTRL, 1706 .addr = (intptr_t)&irq, 1707 .attr = KVM_ARM_VCPU_PMU_V3_IRQ, 1708 }; 1709 1710 if (!cpu->has_pmu) { 1711 return; 1712 } 1713 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) { 1714 error_report("failed to set irq for PMU"); 1715 abort(); 1716 } 1717 } 1718 1719 void kvm_arm_pvtime_init(ARMCPU *cpu, uint64_t ipa) 1720 { 1721 struct kvm_device_attr attr = { 1722 .group = KVM_ARM_VCPU_PVTIME_CTRL, 1723 .attr = KVM_ARM_VCPU_PVTIME_IPA, 1724 .addr = (uint64_t)&ipa, 1725 }; 1726 1727 if (cpu->kvm_steal_time == ON_OFF_AUTO_OFF) { 1728 return; 1729 } 1730 if (!kvm_arm_set_device_attr(cpu, &attr, "PVTIME IPA")) { 1731 error_report("failed to init PVTIME IPA"); 1732 abort(); 1733 } 1734 } 1735 1736 void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp) 1737 { 1738 bool has_steal_time = kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME); 1739 1740 if (cpu->kvm_steal_time == ON_OFF_AUTO_AUTO) { 1741 if (!has_steal_time || !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { 1742 cpu->kvm_steal_time = ON_OFF_AUTO_OFF; 1743 } else { 1744 cpu->kvm_steal_time = ON_OFF_AUTO_ON; 1745 } 1746 } else if (cpu->kvm_steal_time == ON_OFF_AUTO_ON) { 1747 if (!has_steal_time) { 1748 error_setg(errp, "'kvm-steal-time' cannot be enabled " 1749 "on this host"); 1750 return; 1751 } else if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { 1752 /* 1753 * DEN0057A chapter 2 says "This specification only covers 1754 * systems in which the Execution state of the hypervisor 1755 * as well as EL1 of virtual machines is AArch64.". And, 1756 * to ensure that, the smc/hvc calls are only specified as 1757 * smc64/hvc64. 1758 */ 1759 error_setg(errp, "'kvm-steal-time' cannot be enabled " 1760 "for AArch32 guests"); 1761 return; 1762 } 1763 } 1764 } 1765 1766 bool kvm_arm_aarch32_supported(void) 1767 { 1768 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT); 1769 } 1770 1771 bool kvm_arm_sve_supported(void) 1772 { 1773 return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE); 1774 } 1775 1776 bool kvm_arm_mte_supported(void) 1777 { 1778 return kvm_check_extension(kvm_state, KVM_CAP_ARM_MTE); 1779 } 1780 1781 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); 1782 1783 uint32_t kvm_arm_sve_get_vls(ARMCPU *cpu) 1784 { 1785 /* Only call this function if kvm_arm_sve_supported() returns true. */ 1786 static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS]; 1787 static bool probed; 1788 uint32_t vq = 0; 1789 int i; 1790 1791 /* 1792 * KVM ensures all host CPUs support the same set of vector lengths. 1793 * So we only need to create the scratch VCPUs once and then cache 1794 * the results. 1795 */ 1796 if (!probed) { 1797 struct kvm_vcpu_init init = { 1798 .target = -1, 1799 .features[0] = (1 << KVM_ARM_VCPU_SVE), 1800 }; 1801 struct kvm_one_reg reg = { 1802 .id = KVM_REG_ARM64_SVE_VLS, 1803 .addr = (uint64_t)&vls[0], 1804 }; 1805 int fdarray[3], ret; 1806 1807 probed = true; 1808 1809 if (!kvm_arm_create_scratch_host_vcpu(fdarray, &init)) { 1810 error_report("failed to create scratch VCPU with SVE enabled"); 1811 abort(); 1812 } 1813 ret = ioctl(fdarray[2], KVM_GET_ONE_REG, ®); 1814 kvm_arm_destroy_scratch_host_vcpu(fdarray); 1815 if (ret) { 1816 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s", 1817 strerror(errno)); 1818 abort(); 1819 } 1820 1821 for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) { 1822 if (vls[i]) { 1823 vq = 64 - clz64(vls[i]) + i * 64; 1824 break; 1825 } 1826 } 1827 if (vq > ARM_MAX_VQ) { 1828 warn_report("KVM supports vector lengths larger than " 1829 "QEMU can enable"); 1830 vls[0] &= MAKE_64BIT_MASK(0, ARM_MAX_VQ); 1831 } 1832 } 1833 1834 return vls[0]; 1835 } 1836 1837 static int kvm_arm_sve_set_vls(ARMCPU *cpu) 1838 { 1839 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq.map }; 1840 1841 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX); 1842 1843 return kvm_set_one_reg(CPU(cpu), KVM_REG_ARM64_SVE_VLS, &vls[0]); 1844 } 1845 1846 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5 1847 1848 int kvm_arch_pre_create_vcpu(CPUState *cpu, Error **errp) 1849 { 1850 return 0; 1851 } 1852 1853 int kvm_arch_init_vcpu(CPUState *cs) 1854 { 1855 int ret; 1856 uint64_t mpidr; 1857 ARMCPU *cpu = ARM_CPU(cs); 1858 CPUARMState *env = &cpu->env; 1859 uint64_t psciver; 1860 1861 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE) { 1862 error_report("KVM is not supported for this guest CPU type"); 1863 return -EINVAL; 1864 } 1865 1866 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cpu); 1867 1868 /* Determine init features for this CPU */ 1869 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features)); 1870 if (cs->start_powered_off) { 1871 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF; 1872 } 1873 if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) { 1874 cpu->psci_version = QEMU_PSCI_VERSION_0_2; 1875 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2; 1876 } 1877 if (!arm_feature(env, ARM_FEATURE_AARCH64)) { 1878 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT; 1879 } 1880 if (cpu->has_pmu) { 1881 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; 1882 } 1883 if (cpu_isar_feature(aa64_sve, cpu)) { 1884 assert(kvm_arm_sve_supported()); 1885 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE; 1886 } 1887 if (cpu_isar_feature(aa64_pauth, cpu)) { 1888 cpu->kvm_init_features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS | 1889 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC); 1890 } 1891 1892 /* Do KVM_ARM_VCPU_INIT ioctl */ 1893 ret = kvm_arm_vcpu_init(cpu); 1894 if (ret) { 1895 return ret; 1896 } 1897 1898 if (cpu_isar_feature(aa64_sve, cpu)) { 1899 ret = kvm_arm_sve_set_vls(cpu); 1900 if (ret) { 1901 return ret; 1902 } 1903 ret = kvm_arm_vcpu_finalize(cpu, KVM_ARM_VCPU_SVE); 1904 if (ret) { 1905 return ret; 1906 } 1907 } 1908 1909 /* 1910 * KVM reports the exact PSCI version it is implementing via a 1911 * special sysreg. If it is present, use its contents to determine 1912 * what to report to the guest in the dtb (it is the PSCI version, 1913 * in the same 15-bits major 16-bits minor format that PSCI_VERSION 1914 * returns). 1915 */ 1916 if (!kvm_get_one_reg(cs, KVM_REG_ARM_PSCI_VERSION, &psciver)) { 1917 cpu->psci_version = psciver; 1918 } 1919 1920 /* 1921 * When KVM is in use, PSCI is emulated in-kernel and not by qemu. 1922 * Currently KVM has its own idea about MPIDR assignment, so we 1923 * override our defaults with what we get from KVM. 1924 */ 1925 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr); 1926 if (ret) { 1927 return ret; 1928 } 1929 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK; 1930 1931 return kvm_arm_init_cpreg_list(cpu); 1932 } 1933 1934 int kvm_arch_destroy_vcpu(CPUState *cs) 1935 { 1936 return 0; 1937 } 1938 1939 /* Callers must hold the iothread mutex lock */ 1940 static void kvm_inject_arm_sea(CPUState *c) 1941 { 1942 ARMCPU *cpu = ARM_CPU(c); 1943 CPUARMState *env = &cpu->env; 1944 uint32_t esr; 1945 bool same_el; 1946 1947 c->exception_index = EXCP_DATA_ABORT; 1948 env->exception.target_el = 1; 1949 1950 /* 1951 * Set the DFSC to synchronous external abort and set FnV to not valid, 1952 * this will tell guest the FAR_ELx is UNKNOWN for this abort. 1953 */ 1954 same_el = arm_current_el(env) == env->exception.target_el; 1955 esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10); 1956 1957 env->exception.syndrome = esr; 1958 1959 arm_cpu_do_interrupt(c); 1960 } 1961 1962 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ 1963 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 1964 1965 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \ 1966 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 1967 1968 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \ 1969 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 1970 1971 static int kvm_arch_put_fpsimd(CPUState *cs) 1972 { 1973 CPUARMState *env = &ARM_CPU(cs)->env; 1974 int i, ret; 1975 1976 for (i = 0; i < 32; i++) { 1977 uint64_t *q = aa64_vfp_qreg(env, i); 1978 #if HOST_BIG_ENDIAN 1979 uint64_t fp_val[2] = { q[1], q[0] }; 1980 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), 1981 fp_val); 1982 #else 1983 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q); 1984 #endif 1985 if (ret) { 1986 return ret; 1987 } 1988 } 1989 1990 return 0; 1991 } 1992 1993 /* 1994 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits 1995 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard 1996 * code the slice index to zero for now as it's unlikely we'll need more than 1997 * one slice for quite some time. 1998 */ 1999 static int kvm_arch_put_sve(CPUState *cs) 2000 { 2001 ARMCPU *cpu = ARM_CPU(cs); 2002 CPUARMState *env = &cpu->env; 2003 uint64_t tmp[ARM_MAX_VQ * 2]; 2004 uint64_t *r; 2005 int n, ret; 2006 2007 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { 2008 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); 2009 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r); 2010 if (ret) { 2011 return ret; 2012 } 2013 } 2014 2015 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { 2016 r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], 2017 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2018 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r); 2019 if (ret) { 2020 return ret; 2021 } 2022 } 2023 2024 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], 2025 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2026 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r); 2027 if (ret) { 2028 return ret; 2029 } 2030 2031 return 0; 2032 } 2033 2034 int kvm_arch_put_registers(CPUState *cs, int level, Error **errp) 2035 { 2036 uint64_t val; 2037 uint32_t fpr; 2038 int i, ret; 2039 unsigned int el; 2040 2041 ARMCPU *cpu = ARM_CPU(cs); 2042 CPUARMState *env = &cpu->env; 2043 2044 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the 2045 * AArch64 registers before pushing them out to 64-bit KVM. 2046 */ 2047 if (!is_a64(env)) { 2048 aarch64_sync_32_to_64(env); 2049 } 2050 2051 for (i = 0; i < 31; i++) { 2052 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), 2053 &env->xregs[i]); 2054 if (ret) { 2055 return ret; 2056 } 2057 } 2058 2059 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the 2060 * QEMU side we keep the current SP in xregs[31] as well. 2061 */ 2062 aarch64_save_sp(env, 1); 2063 2064 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]); 2065 if (ret) { 2066 return ret; 2067 } 2068 2069 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]); 2070 if (ret) { 2071 return ret; 2072 } 2073 2074 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */ 2075 if (is_a64(env)) { 2076 val = pstate_read(env); 2077 } else { 2078 val = cpsr_read(env); 2079 } 2080 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val); 2081 if (ret) { 2082 return ret; 2083 } 2084 2085 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc); 2086 if (ret) { 2087 return ret; 2088 } 2089 2090 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]); 2091 if (ret) { 2092 return ret; 2093 } 2094 2095 /* Saved Program State Registers 2096 * 2097 * Before we restore from the banked_spsr[] array we need to 2098 * ensure that any modifications to env->spsr are correctly 2099 * reflected in the banks. 2100 */ 2101 el = arm_current_el(env); 2102 if (el > 0 && !is_a64(env)) { 2103 i = bank_number(env->uncached_cpsr & CPSR_M); 2104 env->banked_spsr[i] = env->spsr; 2105 } 2106 2107 /* KVM 0-4 map to QEMU banks 1-5 */ 2108 for (i = 0; i < KVM_NR_SPSR; i++) { 2109 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(spsr[i]), 2110 &env->banked_spsr[i + 1]); 2111 if (ret) { 2112 return ret; 2113 } 2114 } 2115 2116 if (cpu_isar_feature(aa64_sve, cpu)) { 2117 ret = kvm_arch_put_sve(cs); 2118 } else { 2119 ret = kvm_arch_put_fpsimd(cs); 2120 } 2121 if (ret) { 2122 return ret; 2123 } 2124 2125 fpr = vfp_get_fpsr(env); 2126 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr); 2127 if (ret) { 2128 return ret; 2129 } 2130 2131 fpr = vfp_get_fpcr(env); 2132 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr); 2133 if (ret) { 2134 return ret; 2135 } 2136 2137 write_cpustate_to_list(cpu, true); 2138 2139 if (!write_list_to_kvmstate(cpu, level)) { 2140 return -EINVAL; 2141 } 2142 2143 /* 2144 * Setting VCPU events should be triggered after syncing the registers 2145 * to avoid overwriting potential changes made by KVM upon calling 2146 * KVM_SET_VCPU_EVENTS ioctl 2147 */ 2148 ret = kvm_put_vcpu_events(cpu); 2149 if (ret) { 2150 return ret; 2151 } 2152 2153 return kvm_arm_sync_mpstate_to_kvm(cpu); 2154 } 2155 2156 static int kvm_arch_get_fpsimd(CPUState *cs) 2157 { 2158 CPUARMState *env = &ARM_CPU(cs)->env; 2159 int i, ret; 2160 2161 for (i = 0; i < 32; i++) { 2162 uint64_t *q = aa64_vfp_qreg(env, i); 2163 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q); 2164 if (ret) { 2165 return ret; 2166 } else { 2167 #if HOST_BIG_ENDIAN 2168 uint64_t t; 2169 t = q[0], q[0] = q[1], q[1] = t; 2170 #endif 2171 } 2172 } 2173 2174 return 0; 2175 } 2176 2177 /* 2178 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits 2179 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard 2180 * code the slice index to zero for now as it's unlikely we'll need more than 2181 * one slice for quite some time. 2182 */ 2183 static int kvm_arch_get_sve(CPUState *cs) 2184 { 2185 ARMCPU *cpu = ARM_CPU(cs); 2186 CPUARMState *env = &cpu->env; 2187 uint64_t *r; 2188 int n, ret; 2189 2190 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { 2191 r = &env->vfp.zregs[n].d[0]; 2192 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r); 2193 if (ret) { 2194 return ret; 2195 } 2196 sve_bswap64(r, r, cpu->sve_max_vq * 2); 2197 } 2198 2199 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { 2200 r = &env->vfp.pregs[n].p[0]; 2201 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r); 2202 if (ret) { 2203 return ret; 2204 } 2205 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2206 } 2207 2208 r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; 2209 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r); 2210 if (ret) { 2211 return ret; 2212 } 2213 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 2214 2215 return 0; 2216 } 2217 2218 int kvm_arch_get_registers(CPUState *cs, Error **errp) 2219 { 2220 uint64_t val; 2221 unsigned int el; 2222 uint32_t fpr; 2223 int i, ret; 2224 2225 ARMCPU *cpu = ARM_CPU(cs); 2226 CPUARMState *env = &cpu->env; 2227 2228 for (i = 0; i < 31; i++) { 2229 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]), 2230 &env->xregs[i]); 2231 if (ret) { 2232 return ret; 2233 } 2234 } 2235 2236 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]); 2237 if (ret) { 2238 return ret; 2239 } 2240 2241 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]); 2242 if (ret) { 2243 return ret; 2244 } 2245 2246 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val); 2247 if (ret) { 2248 return ret; 2249 } 2250 2251 env->aarch64 = ((val & PSTATE_nRW) == 0); 2252 if (is_a64(env)) { 2253 pstate_write(env, val); 2254 } else { 2255 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw); 2256 } 2257 2258 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the 2259 * QEMU side we keep the current SP in xregs[31] as well. 2260 */ 2261 aarch64_restore_sp(env, 1); 2262 2263 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc); 2264 if (ret) { 2265 return ret; 2266 } 2267 2268 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the 2269 * incoming AArch64 regs received from 64-bit KVM. 2270 * We must perform this after all of the registers have been acquired from 2271 * the kernel. 2272 */ 2273 if (!is_a64(env)) { 2274 aarch64_sync_64_to_32(env); 2275 } 2276 2277 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]); 2278 if (ret) { 2279 return ret; 2280 } 2281 2282 /* Fetch the SPSR registers 2283 * 2284 * KVM SPSRs 0-4 map to QEMU banks 1-5 2285 */ 2286 for (i = 0; i < KVM_NR_SPSR; i++) { 2287 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(spsr[i]), 2288 &env->banked_spsr[i + 1]); 2289 if (ret) { 2290 return ret; 2291 } 2292 } 2293 2294 el = arm_current_el(env); 2295 if (el > 0 && !is_a64(env)) { 2296 i = bank_number(env->uncached_cpsr & CPSR_M); 2297 env->spsr = env->banked_spsr[i]; 2298 } 2299 2300 if (cpu_isar_feature(aa64_sve, cpu)) { 2301 ret = kvm_arch_get_sve(cs); 2302 } else { 2303 ret = kvm_arch_get_fpsimd(cs); 2304 } 2305 if (ret) { 2306 return ret; 2307 } 2308 2309 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr); 2310 if (ret) { 2311 return ret; 2312 } 2313 vfp_set_fpsr(env, fpr); 2314 2315 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr); 2316 if (ret) { 2317 return ret; 2318 } 2319 vfp_set_fpcr(env, fpr); 2320 2321 ret = kvm_get_vcpu_events(cpu); 2322 if (ret) { 2323 return ret; 2324 } 2325 2326 if (!write_kvmstate_to_list(cpu)) { 2327 return -EINVAL; 2328 } 2329 /* Note that it's OK to have registers which aren't in CPUState, 2330 * so we can ignore a failure return here. 2331 */ 2332 write_list_to_cpustate(cpu); 2333 2334 ret = kvm_arm_sync_mpstate_to_qemu(cpu); 2335 2336 /* TODO: other registers */ 2337 return ret; 2338 } 2339 2340 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr) 2341 { 2342 ram_addr_t ram_addr; 2343 hwaddr paddr; 2344 2345 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO); 2346 2347 if (acpi_ghes_present() && addr) { 2348 ram_addr = qemu_ram_addr_from_host(addr); 2349 if (ram_addr != RAM_ADDR_INVALID && 2350 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) { 2351 kvm_hwpoison_page_add(ram_addr); 2352 /* 2353 * If this is a BUS_MCEERR_AR, we know we have been called 2354 * synchronously from the vCPU thread, so we can easily 2355 * synchronize the state and inject an error. 2356 * 2357 * TODO: we currently don't tell the guest at all about 2358 * BUS_MCEERR_AO. In that case we might either be being 2359 * called synchronously from the vCPU thread, or a bit 2360 * later from the main thread, so doing the injection of 2361 * the error would be more complicated. 2362 */ 2363 if (code == BUS_MCEERR_AR) { 2364 kvm_cpu_synchronize_state(c); 2365 if (!acpi_ghes_memory_errors(ACPI_HEST_SRC_ID_SEA, paddr)) { 2366 kvm_inject_arm_sea(c); 2367 } else { 2368 error_report("failed to record the error"); 2369 abort(); 2370 } 2371 } 2372 return; 2373 } 2374 if (code == BUS_MCEERR_AO) { 2375 error_report("Hardware memory error at addr %p for memory used by " 2376 "QEMU itself instead of guest system!", addr); 2377 } 2378 } 2379 2380 if (code == BUS_MCEERR_AR) { 2381 error_report("Hardware memory error!"); 2382 exit(1); 2383 } 2384 } 2385 2386 /* C6.6.29 BRK instruction */ 2387 static const uint32_t brk_insn = 0xd4200000; 2388 2389 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 2390 { 2391 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) || 2392 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) { 2393 return -EINVAL; 2394 } 2395 return 0; 2396 } 2397 2398 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 2399 { 2400 static uint32_t brk; 2401 2402 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) || 2403 brk != brk_insn || 2404 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) { 2405 return -EINVAL; 2406 } 2407 return 0; 2408 } 2409 2410 void kvm_arm_enable_mte(Object *cpuobj, Error **errp) 2411 { 2412 static bool tried_to_enable; 2413 static bool succeeded_to_enable; 2414 Error *mte_migration_blocker = NULL; 2415 ARMCPU *cpu = ARM_CPU(cpuobj); 2416 int ret; 2417 2418 if (!tried_to_enable) { 2419 /* 2420 * MTE on KVM is enabled on a per-VM basis (and retrying doesn't make 2421 * sense), and we only want a single migration blocker as well. 2422 */ 2423 tried_to_enable = true; 2424 2425 ret = kvm_vm_enable_cap(kvm_state, KVM_CAP_ARM_MTE, 0); 2426 if (ret) { 2427 error_setg_errno(errp, -ret, "Failed to enable KVM_CAP_ARM_MTE"); 2428 return; 2429 } 2430 2431 /* TODO: Add migration support with MTE enabled */ 2432 error_setg(&mte_migration_blocker, 2433 "Live migration disabled due to MTE enabled"); 2434 if (migrate_add_blocker(&mte_migration_blocker, errp)) { 2435 error_free(mte_migration_blocker); 2436 return; 2437 } 2438 2439 succeeded_to_enable = true; 2440 } 2441 2442 if (succeeded_to_enable) { 2443 cpu->kvm_mte = true; 2444 } 2445 } 2446 2447 void arm_cpu_kvm_set_irq(void *arm_cpu, int irq, int level) 2448 { 2449 ARMCPU *cpu = arm_cpu; 2450 CPUARMState *env = &cpu->env; 2451 CPUState *cs = CPU(cpu); 2452 uint32_t linestate_bit; 2453 int irq_id; 2454 2455 switch (irq) { 2456 case ARM_CPU_IRQ: 2457 irq_id = KVM_ARM_IRQ_CPU_IRQ; 2458 linestate_bit = CPU_INTERRUPT_HARD; 2459 break; 2460 case ARM_CPU_FIQ: 2461 irq_id = KVM_ARM_IRQ_CPU_FIQ; 2462 linestate_bit = CPU_INTERRUPT_FIQ; 2463 break; 2464 default: 2465 g_assert_not_reached(); 2466 } 2467 2468 if (level) { 2469 env->irq_line_state |= linestate_bit; 2470 } else { 2471 env->irq_line_state &= ~linestate_bit; 2472 } 2473 kvm_arm_set_irq(cs->cpu_index, KVM_ARM_IRQ_TYPE_CPU, irq_id, !!level); 2474 } 2475