1 /* 2 * ARM implementation of KVM hooks, 64 bit specific code 3 * 4 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems 5 * Copyright Alex Bennée 2014, Linaro 6 * 7 * This work is licensed under the terms of the GNU GPL, version 2 or later. 8 * See the COPYING file in the top-level directory. 9 * 10 */ 11 12 #include "qemu/osdep.h" 13 #include <sys/ioctl.h> 14 #include <sys/ptrace.h> 15 16 #include <linux/elf.h> 17 #include <linux/kvm.h> 18 19 #include "qemu-common.h" 20 #include "cpu.h" 21 #include "qemu/timer.h" 22 #include "qemu/error-report.h" 23 #include "qemu/host-utils.h" 24 #include "qemu/main-loop.h" 25 #include "exec/gdbstub.h" 26 #include "sysemu/runstate.h" 27 #include "sysemu/kvm.h" 28 #include "sysemu/kvm_int.h" 29 #include "kvm_arm.h" 30 #include "internals.h" 31 32 static bool have_guest_debug; 33 34 /* 35 * Although the ARM implementation of hardware assisted debugging 36 * allows for different breakpoints per-core, the current GDB 37 * interface treats them as a global pool of registers (which seems to 38 * be the case for x86, ppc and s390). As a result we store one copy 39 * of registers which is used for all active cores. 40 * 41 * Write access is serialised by virtue of the GDB protocol which 42 * updates things. Read access (i.e. when the values are copied to the 43 * vCPU) is also gated by GDB's run control. 44 * 45 * This is not unreasonable as most of the time debugging kernels you 46 * never know which core will eventually execute your function. 47 */ 48 49 typedef struct { 50 uint64_t bcr; 51 uint64_t bvr; 52 } HWBreakpoint; 53 54 /* The watchpoint registers can cover more area than the requested 55 * watchpoint so we need to store the additional information 56 * somewhere. We also need to supply a CPUWatchpoint to the GDB stub 57 * when the watchpoint is hit. 58 */ 59 typedef struct { 60 uint64_t wcr; 61 uint64_t wvr; 62 CPUWatchpoint details; 63 } HWWatchpoint; 64 65 /* Maximum and current break/watch point counts */ 66 int max_hw_bps, max_hw_wps; 67 GArray *hw_breakpoints, *hw_watchpoints; 68 69 #define cur_hw_wps (hw_watchpoints->len) 70 #define cur_hw_bps (hw_breakpoints->len) 71 #define get_hw_bp(i) (&g_array_index(hw_breakpoints, HWBreakpoint, i)) 72 #define get_hw_wp(i) (&g_array_index(hw_watchpoints, HWWatchpoint, i)) 73 74 /** 75 * kvm_arm_init_debug() - check for guest debug capabilities 76 * @cs: CPUState 77 * 78 * kvm_check_extension returns the number of debug registers we have 79 * or 0 if we have none. 80 * 81 */ 82 static void kvm_arm_init_debug(CPUState *cs) 83 { 84 have_guest_debug = kvm_check_extension(cs->kvm_state, 85 KVM_CAP_SET_GUEST_DEBUG); 86 87 max_hw_wps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_WPS); 88 hw_watchpoints = g_array_sized_new(true, true, 89 sizeof(HWWatchpoint), max_hw_wps); 90 91 max_hw_bps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_BPS); 92 hw_breakpoints = g_array_sized_new(true, true, 93 sizeof(HWBreakpoint), max_hw_bps); 94 return; 95 } 96 97 /** 98 * insert_hw_breakpoint() 99 * @addr: address of breakpoint 100 * 101 * See ARM ARM D2.9.1 for details but here we are only going to create 102 * simple un-linked breakpoints (i.e. we don't chain breakpoints 103 * together to match address and context or vmid). The hardware is 104 * capable of fancier matching but that will require exposing that 105 * fanciness to GDB's interface 106 * 107 * DBGBCR<n>_EL1, Debug Breakpoint Control Registers 108 * 109 * 31 24 23 20 19 16 15 14 13 12 9 8 5 4 3 2 1 0 110 * +------+------+-------+-----+----+------+-----+------+-----+---+ 111 * | RES0 | BT | LBN | SSC | HMC| RES0 | BAS | RES0 | PMC | E | 112 * +------+------+-------+-----+----+------+-----+------+-----+---+ 113 * 114 * BT: Breakpoint type (0 = unlinked address match) 115 * LBN: Linked BP number (0 = unused) 116 * SSC/HMC/PMC: Security, Higher and Priv access control (Table D-12) 117 * BAS: Byte Address Select (RES1 for AArch64) 118 * E: Enable bit 119 * 120 * DBGBVR<n>_EL1, Debug Breakpoint Value Registers 121 * 122 * 63 53 52 49 48 2 1 0 123 * +------+-----------+----------+-----+ 124 * | RESS | VA[52:49] | VA[48:2] | 0 0 | 125 * +------+-----------+----------+-----+ 126 * 127 * Depending on the addressing mode bits the top bits of the register 128 * are a sign extension of the highest applicable VA bit. Some 129 * versions of GDB don't do it correctly so we ensure they are correct 130 * here so future PC comparisons will work properly. 131 */ 132 133 static int insert_hw_breakpoint(target_ulong addr) 134 { 135 HWBreakpoint brk = { 136 .bcr = 0x1, /* BCR E=1, enable */ 137 .bvr = sextract64(addr, 0, 53) 138 }; 139 140 if (cur_hw_bps >= max_hw_bps) { 141 return -ENOBUFS; 142 } 143 144 brk.bcr = deposit32(brk.bcr, 1, 2, 0x3); /* PMC = 11 */ 145 brk.bcr = deposit32(brk.bcr, 5, 4, 0xf); /* BAS = RES1 */ 146 147 g_array_append_val(hw_breakpoints, brk); 148 149 return 0; 150 } 151 152 /** 153 * delete_hw_breakpoint() 154 * @pc: address of breakpoint 155 * 156 * Delete a breakpoint and shuffle any above down 157 */ 158 159 static int delete_hw_breakpoint(target_ulong pc) 160 { 161 int i; 162 for (i = 0; i < hw_breakpoints->len; i++) { 163 HWBreakpoint *brk = get_hw_bp(i); 164 if (brk->bvr == pc) { 165 g_array_remove_index(hw_breakpoints, i); 166 return 0; 167 } 168 } 169 return -ENOENT; 170 } 171 172 /** 173 * insert_hw_watchpoint() 174 * @addr: address of watch point 175 * @len: size of area 176 * @type: type of watch point 177 * 178 * See ARM ARM D2.10. As with the breakpoints we can do some advanced 179 * stuff if we want to. The watch points can be linked with the break 180 * points above to make them context aware. However for simplicity 181 * currently we only deal with simple read/write watch points. 182 * 183 * D7.3.11 DBGWCR<n>_EL1, Debug Watchpoint Control Registers 184 * 185 * 31 29 28 24 23 21 20 19 16 15 14 13 12 5 4 3 2 1 0 186 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+ 187 * | RES0 | MASK | RES0 | WT | LBN | SSC | HMC | BAS | LSC | PAC | E | 188 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+ 189 * 190 * MASK: num bits addr mask (0=none,01/10=res,11=3 bits (8 bytes)) 191 * WT: 0 - unlinked, 1 - linked (not currently used) 192 * LBN: Linked BP number (not currently used) 193 * SSC/HMC/PAC: Security, Higher and Priv access control (Table D2-11) 194 * BAS: Byte Address Select 195 * LSC: Load/Store control (01: load, 10: store, 11: both) 196 * E: Enable 197 * 198 * The bottom 2 bits of the value register are masked. Therefore to 199 * break on any sizes smaller than an unaligned word you need to set 200 * MASK=0, BAS=bit per byte in question. For larger regions (^2) you 201 * need to ensure you mask the address as required and set BAS=0xff 202 */ 203 204 static int insert_hw_watchpoint(target_ulong addr, 205 target_ulong len, int type) 206 { 207 HWWatchpoint wp = { 208 .wcr = 1, /* E=1, enable */ 209 .wvr = addr & (~0x7ULL), 210 .details = { .vaddr = addr, .len = len } 211 }; 212 213 if (cur_hw_wps >= max_hw_wps) { 214 return -ENOBUFS; 215 } 216 217 /* 218 * HMC=0 SSC=0 PAC=3 will hit EL0 or EL1, any security state, 219 * valid whether EL3 is implemented or not 220 */ 221 wp.wcr = deposit32(wp.wcr, 1, 2, 3); 222 223 switch (type) { 224 case GDB_WATCHPOINT_READ: 225 wp.wcr = deposit32(wp.wcr, 3, 2, 1); 226 wp.details.flags = BP_MEM_READ; 227 break; 228 case GDB_WATCHPOINT_WRITE: 229 wp.wcr = deposit32(wp.wcr, 3, 2, 2); 230 wp.details.flags = BP_MEM_WRITE; 231 break; 232 case GDB_WATCHPOINT_ACCESS: 233 wp.wcr = deposit32(wp.wcr, 3, 2, 3); 234 wp.details.flags = BP_MEM_ACCESS; 235 break; 236 default: 237 g_assert_not_reached(); 238 break; 239 } 240 if (len <= 8) { 241 /* we align the address and set the bits in BAS */ 242 int off = addr & 0x7; 243 int bas = (1 << len) - 1; 244 245 wp.wcr = deposit32(wp.wcr, 5 + off, 8 - off, bas); 246 } else { 247 /* For ranges above 8 bytes we need to be a power of 2 */ 248 if (is_power_of_2(len)) { 249 int bits = ctz64(len); 250 251 wp.wvr &= ~((1 << bits) - 1); 252 wp.wcr = deposit32(wp.wcr, 24, 4, bits); 253 wp.wcr = deposit32(wp.wcr, 5, 8, 0xff); 254 } else { 255 return -ENOBUFS; 256 } 257 } 258 259 g_array_append_val(hw_watchpoints, wp); 260 return 0; 261 } 262 263 264 static bool check_watchpoint_in_range(int i, target_ulong addr) 265 { 266 HWWatchpoint *wp = get_hw_wp(i); 267 uint64_t addr_top, addr_bottom = wp->wvr; 268 int bas = extract32(wp->wcr, 5, 8); 269 int mask = extract32(wp->wcr, 24, 4); 270 271 if (mask) { 272 addr_top = addr_bottom + (1 << mask); 273 } else { 274 /* BAS must be contiguous but can offset against the base 275 * address in DBGWVR */ 276 addr_bottom = addr_bottom + ctz32(bas); 277 addr_top = addr_bottom + clo32(bas); 278 } 279 280 if (addr >= addr_bottom && addr <= addr_top) { 281 return true; 282 } 283 284 return false; 285 } 286 287 /** 288 * delete_hw_watchpoint() 289 * @addr: address of breakpoint 290 * 291 * Delete a breakpoint and shuffle any above down 292 */ 293 294 static int delete_hw_watchpoint(target_ulong addr, 295 target_ulong len, int type) 296 { 297 int i; 298 for (i = 0; i < cur_hw_wps; i++) { 299 if (check_watchpoint_in_range(i, addr)) { 300 g_array_remove_index(hw_watchpoints, i); 301 return 0; 302 } 303 } 304 return -ENOENT; 305 } 306 307 308 int kvm_arch_insert_hw_breakpoint(target_ulong addr, 309 target_ulong len, int type) 310 { 311 switch (type) { 312 case GDB_BREAKPOINT_HW: 313 return insert_hw_breakpoint(addr); 314 break; 315 case GDB_WATCHPOINT_READ: 316 case GDB_WATCHPOINT_WRITE: 317 case GDB_WATCHPOINT_ACCESS: 318 return insert_hw_watchpoint(addr, len, type); 319 default: 320 return -ENOSYS; 321 } 322 } 323 324 int kvm_arch_remove_hw_breakpoint(target_ulong addr, 325 target_ulong len, int type) 326 { 327 switch (type) { 328 case GDB_BREAKPOINT_HW: 329 return delete_hw_breakpoint(addr); 330 break; 331 case GDB_WATCHPOINT_READ: 332 case GDB_WATCHPOINT_WRITE: 333 case GDB_WATCHPOINT_ACCESS: 334 return delete_hw_watchpoint(addr, len, type); 335 default: 336 return -ENOSYS; 337 } 338 } 339 340 341 void kvm_arch_remove_all_hw_breakpoints(void) 342 { 343 if (cur_hw_wps > 0) { 344 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps); 345 } 346 if (cur_hw_bps > 0) { 347 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps); 348 } 349 } 350 351 void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr) 352 { 353 int i; 354 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch)); 355 356 for (i = 0; i < max_hw_wps; i++) { 357 HWWatchpoint *wp = get_hw_wp(i); 358 ptr->dbg_wcr[i] = wp->wcr; 359 ptr->dbg_wvr[i] = wp->wvr; 360 } 361 for (i = 0; i < max_hw_bps; i++) { 362 HWBreakpoint *bp = get_hw_bp(i); 363 ptr->dbg_bcr[i] = bp->bcr; 364 ptr->dbg_bvr[i] = bp->bvr; 365 } 366 } 367 368 bool kvm_arm_hw_debug_active(CPUState *cs) 369 { 370 return ((cur_hw_wps > 0) || (cur_hw_bps > 0)); 371 } 372 373 static bool find_hw_breakpoint(CPUState *cpu, target_ulong pc) 374 { 375 int i; 376 377 for (i = 0; i < cur_hw_bps; i++) { 378 HWBreakpoint *bp = get_hw_bp(i); 379 if (bp->bvr == pc) { 380 return true; 381 } 382 } 383 return false; 384 } 385 386 static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu, target_ulong addr) 387 { 388 int i; 389 390 for (i = 0; i < cur_hw_wps; i++) { 391 if (check_watchpoint_in_range(i, addr)) { 392 return &get_hw_wp(i)->details; 393 } 394 } 395 return NULL; 396 } 397 398 static bool kvm_arm_pmu_set_attr(CPUState *cs, struct kvm_device_attr *attr) 399 { 400 int err; 401 402 err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr); 403 if (err != 0) { 404 error_report("PMU: KVM_HAS_DEVICE_ATTR: %s", strerror(-err)); 405 return false; 406 } 407 408 err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr); 409 if (err != 0) { 410 error_report("PMU: KVM_SET_DEVICE_ATTR: %s", strerror(-err)); 411 return false; 412 } 413 414 return true; 415 } 416 417 void kvm_arm_pmu_init(CPUState *cs) 418 { 419 struct kvm_device_attr attr = { 420 .group = KVM_ARM_VCPU_PMU_V3_CTRL, 421 .attr = KVM_ARM_VCPU_PMU_V3_INIT, 422 }; 423 424 if (!ARM_CPU(cs)->has_pmu) { 425 return; 426 } 427 if (!kvm_arm_pmu_set_attr(cs, &attr)) { 428 error_report("failed to init PMU"); 429 abort(); 430 } 431 } 432 433 void kvm_arm_pmu_set_irq(CPUState *cs, int irq) 434 { 435 struct kvm_device_attr attr = { 436 .group = KVM_ARM_VCPU_PMU_V3_CTRL, 437 .addr = (intptr_t)&irq, 438 .attr = KVM_ARM_VCPU_PMU_V3_IRQ, 439 }; 440 441 if (!ARM_CPU(cs)->has_pmu) { 442 return; 443 } 444 if (!kvm_arm_pmu_set_attr(cs, &attr)) { 445 error_report("failed to set irq for PMU"); 446 abort(); 447 } 448 } 449 450 static inline void set_feature(uint64_t *features, int feature) 451 { 452 *features |= 1ULL << feature; 453 } 454 455 static inline void unset_feature(uint64_t *features, int feature) 456 { 457 *features &= ~(1ULL << feature); 458 } 459 460 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id) 461 { 462 uint64_t ret; 463 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret }; 464 int err; 465 466 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); 467 err = ioctl(fd, KVM_GET_ONE_REG, &idreg); 468 if (err < 0) { 469 return -1; 470 } 471 *pret = ret; 472 return 0; 473 } 474 475 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id) 476 { 477 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret }; 478 479 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64); 480 return ioctl(fd, KVM_GET_ONE_REG, &idreg); 481 } 482 483 bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) 484 { 485 /* Identify the feature bits corresponding to the host CPU, and 486 * fill out the ARMHostCPUClass fields accordingly. To do this 487 * we have to create a scratch VM, create a single CPU inside it, 488 * and then query that CPU for the relevant ID registers. 489 */ 490 int fdarray[3]; 491 bool sve_supported; 492 uint64_t features = 0; 493 uint64_t t; 494 int err; 495 496 /* Old kernels may not know about the PREFERRED_TARGET ioctl: however 497 * we know these will only support creating one kind of guest CPU, 498 * which is its preferred CPU type. Fortunately these old kernels 499 * support only a very limited number of CPUs. 500 */ 501 static const uint32_t cpus_to_try[] = { 502 KVM_ARM_TARGET_AEM_V8, 503 KVM_ARM_TARGET_FOUNDATION_V8, 504 KVM_ARM_TARGET_CORTEX_A57, 505 QEMU_KVM_ARM_TARGET_NONE 506 }; 507 /* 508 * target = -1 informs kvm_arm_create_scratch_host_vcpu() 509 * to use the preferred target 510 */ 511 struct kvm_vcpu_init init = { .target = -1, }; 512 513 if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) { 514 return false; 515 } 516 517 ahcf->target = init.target; 518 ahcf->dtb_compatible = "arm,arm-v8"; 519 520 err = read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr0, 521 ARM64_SYS_REG(3, 0, 0, 4, 0)); 522 if (unlikely(err < 0)) { 523 /* 524 * Before v4.15, the kernel only exposed a limited number of system 525 * registers, not including any of the interesting AArch64 ID regs. 526 * For the most part we could leave these fields as zero with minimal 527 * effect, since this does not affect the values seen by the guest. 528 * 529 * However, it could cause problems down the line for QEMU, 530 * so provide a minimal v8.0 default. 531 * 532 * ??? Could read MIDR and use knowledge from cpu64.c. 533 * ??? Could map a page of memory into our temp guest and 534 * run the tiniest of hand-crafted kernels to extract 535 * the values seen by the guest. 536 * ??? Either of these sounds like too much effort just 537 * to work around running a modern host kernel. 538 */ 539 ahcf->isar.id_aa64pfr0 = 0x00000011; /* EL1&0, AArch64 only */ 540 err = 0; 541 } else { 542 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1, 543 ARM64_SYS_REG(3, 0, 0, 4, 1)); 544 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0, 545 ARM64_SYS_REG(3, 0, 0, 5, 0)); 546 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1, 547 ARM64_SYS_REG(3, 0, 0, 5, 1)); 548 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar0, 549 ARM64_SYS_REG(3, 0, 0, 6, 0)); 550 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar1, 551 ARM64_SYS_REG(3, 0, 0, 6, 1)); 552 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr0, 553 ARM64_SYS_REG(3, 0, 0, 7, 0)); 554 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr1, 555 ARM64_SYS_REG(3, 0, 0, 7, 1)); 556 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr2, 557 ARM64_SYS_REG(3, 0, 0, 7, 2)); 558 559 /* 560 * Note that if AArch32 support is not present in the host, 561 * the AArch32 sysregs are present to be read, but will 562 * return UNKNOWN values. This is neither better nor worse 563 * than skipping the reads and leaving 0, as we must avoid 564 * considering the values in every case. 565 */ 566 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr0, 567 ARM64_SYS_REG(3, 0, 0, 1, 2)); 568 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr0, 569 ARM64_SYS_REG(3, 0, 0, 1, 4)); 570 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr1, 571 ARM64_SYS_REG(3, 0, 0, 1, 5)); 572 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr2, 573 ARM64_SYS_REG(3, 0, 0, 1, 6)); 574 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr3, 575 ARM64_SYS_REG(3, 0, 0, 1, 7)); 576 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar0, 577 ARM64_SYS_REG(3, 0, 0, 2, 0)); 578 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar1, 579 ARM64_SYS_REG(3, 0, 0, 2, 1)); 580 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar2, 581 ARM64_SYS_REG(3, 0, 0, 2, 2)); 582 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar3, 583 ARM64_SYS_REG(3, 0, 0, 2, 3)); 584 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar4, 585 ARM64_SYS_REG(3, 0, 0, 2, 4)); 586 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar5, 587 ARM64_SYS_REG(3, 0, 0, 2, 5)); 588 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr4, 589 ARM64_SYS_REG(3, 0, 0, 2, 6)); 590 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar6, 591 ARM64_SYS_REG(3, 0, 0, 2, 7)); 592 593 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr0, 594 ARM64_SYS_REG(3, 0, 0, 3, 0)); 595 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr1, 596 ARM64_SYS_REG(3, 0, 0, 3, 1)); 597 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr2, 598 ARM64_SYS_REG(3, 0, 0, 3, 2)); 599 600 /* 601 * DBGDIDR is a bit complicated because the kernel doesn't 602 * provide an accessor for it in 64-bit mode, which is what this 603 * scratch VM is in, and there's no architected "64-bit sysreg 604 * which reads the same as the 32-bit register" the way there is 605 * for other ID registers. Instead we synthesize a value from the 606 * AArch64 ID_AA64DFR0, the same way the kernel code in 607 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does. 608 * We only do this if the CPU supports AArch32 at EL1. 609 */ 610 if (FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL1) >= 2) { 611 int wrps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, WRPS); 612 int brps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, BRPS); 613 int ctx_cmps = 614 FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, CTX_CMPS); 615 int version = 6; /* ARMv8 debug architecture */ 616 bool has_el3 = 617 !!FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL3); 618 uint32_t dbgdidr = 0; 619 620 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps); 621 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps); 622 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps); 623 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version); 624 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3); 625 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3); 626 dbgdidr |= (1 << 15); /* RES1 bit */ 627 ahcf->isar.dbgdidr = dbgdidr; 628 } 629 } 630 631 sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) > 0; 632 633 kvm_arm_destroy_scratch_host_vcpu(fdarray); 634 635 if (err < 0) { 636 return false; 637 } 638 639 /* Add feature bits that can't appear until after VCPU init. */ 640 if (sve_supported) { 641 t = ahcf->isar.id_aa64pfr0; 642 t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1); 643 ahcf->isar.id_aa64pfr0 = t; 644 } 645 646 /* 647 * We can assume any KVM supporting CPU is at least a v8 648 * with VFPv4+Neon; this in turn implies most of the other 649 * feature bits. 650 */ 651 set_feature(&features, ARM_FEATURE_V8); 652 set_feature(&features, ARM_FEATURE_NEON); 653 set_feature(&features, ARM_FEATURE_AARCH64); 654 set_feature(&features, ARM_FEATURE_PMU); 655 set_feature(&features, ARM_FEATURE_GENERIC_TIMER); 656 657 ahcf->features = features; 658 659 return true; 660 } 661 662 bool kvm_arm_aarch32_supported(CPUState *cpu) 663 { 664 KVMState *s = KVM_STATE(current_accel()); 665 666 return kvm_check_extension(s, KVM_CAP_ARM_EL1_32BIT); 667 } 668 669 bool kvm_arm_sve_supported(CPUState *cpu) 670 { 671 KVMState *s = KVM_STATE(current_accel()); 672 673 return kvm_check_extension(s, KVM_CAP_ARM_SVE); 674 } 675 676 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); 677 678 void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) 679 { 680 /* Only call this function if kvm_arm_sve_supported() returns true. */ 681 static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS]; 682 static bool probed; 683 uint32_t vq = 0; 684 int i, j; 685 686 bitmap_clear(map, 0, ARM_MAX_VQ); 687 688 /* 689 * KVM ensures all host CPUs support the same set of vector lengths. 690 * So we only need to create the scratch VCPUs once and then cache 691 * the results. 692 */ 693 if (!probed) { 694 struct kvm_vcpu_init init = { 695 .target = -1, 696 .features[0] = (1 << KVM_ARM_VCPU_SVE), 697 }; 698 struct kvm_one_reg reg = { 699 .id = KVM_REG_ARM64_SVE_VLS, 700 .addr = (uint64_t)&vls[0], 701 }; 702 int fdarray[3], ret; 703 704 probed = true; 705 706 if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, &init)) { 707 error_report("failed to create scratch VCPU with SVE enabled"); 708 abort(); 709 } 710 ret = ioctl(fdarray[2], KVM_GET_ONE_REG, ®); 711 kvm_arm_destroy_scratch_host_vcpu(fdarray); 712 if (ret) { 713 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s", 714 strerror(errno)); 715 abort(); 716 } 717 718 for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) { 719 if (vls[i]) { 720 vq = 64 - clz64(vls[i]) + i * 64; 721 break; 722 } 723 } 724 if (vq > ARM_MAX_VQ) { 725 warn_report("KVM supports vector lengths larger than " 726 "QEMU can enable"); 727 } 728 } 729 730 for (i = 0; i < KVM_ARM64_SVE_VLS_WORDS; ++i) { 731 if (!vls[i]) { 732 continue; 733 } 734 for (j = 1; j <= 64; ++j) { 735 vq = j + i * 64; 736 if (vq > ARM_MAX_VQ) { 737 return; 738 } 739 if (vls[i] & (1UL << (j - 1))) { 740 set_bit(vq - 1, map); 741 } 742 } 743 } 744 } 745 746 static int kvm_arm_sve_set_vls(CPUState *cs) 747 { 748 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = {0}; 749 struct kvm_one_reg reg = { 750 .id = KVM_REG_ARM64_SVE_VLS, 751 .addr = (uint64_t)&vls[0], 752 }; 753 ARMCPU *cpu = ARM_CPU(cs); 754 uint32_t vq; 755 int i, j; 756 757 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX); 758 759 for (vq = 1; vq <= cpu->sve_max_vq; ++vq) { 760 if (test_bit(vq - 1, cpu->sve_vq_map)) { 761 i = (vq - 1) / 64; 762 j = (vq - 1) % 64; 763 vls[i] |= 1UL << j; 764 } 765 } 766 767 return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 768 } 769 770 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5 771 772 int kvm_arch_init_vcpu(CPUState *cs) 773 { 774 int ret; 775 uint64_t mpidr; 776 ARMCPU *cpu = ARM_CPU(cs); 777 CPUARMState *env = &cpu->env; 778 779 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE || 780 !object_dynamic_cast(OBJECT(cpu), TYPE_AARCH64_CPU)) { 781 error_report("KVM is not supported for this guest CPU type"); 782 return -EINVAL; 783 } 784 785 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cs); 786 787 /* Determine init features for this CPU */ 788 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features)); 789 if (cpu->start_powered_off) { 790 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF; 791 } 792 if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) { 793 cpu->psci_version = 2; 794 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2; 795 } 796 if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) { 797 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT; 798 } 799 if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PMU_V3)) { 800 cpu->has_pmu = false; 801 } 802 if (cpu->has_pmu) { 803 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3; 804 } else { 805 unset_feature(&env->features, ARM_FEATURE_PMU); 806 } 807 if (cpu_isar_feature(aa64_sve, cpu)) { 808 assert(kvm_arm_sve_supported(cs)); 809 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE; 810 } 811 812 /* Do KVM_ARM_VCPU_INIT ioctl */ 813 ret = kvm_arm_vcpu_init(cs); 814 if (ret) { 815 return ret; 816 } 817 818 if (cpu_isar_feature(aa64_sve, cpu)) { 819 ret = kvm_arm_sve_set_vls(cs); 820 if (ret) { 821 return ret; 822 } 823 ret = kvm_arm_vcpu_finalize(cs, KVM_ARM_VCPU_SVE); 824 if (ret) { 825 return ret; 826 } 827 } 828 829 /* 830 * When KVM is in use, PSCI is emulated in-kernel and not by qemu. 831 * Currently KVM has its own idea about MPIDR assignment, so we 832 * override our defaults with what we get from KVM. 833 */ 834 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr); 835 if (ret) { 836 return ret; 837 } 838 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK; 839 840 kvm_arm_init_debug(cs); 841 842 /* Check whether user space can specify guest syndrome value */ 843 kvm_arm_init_serror_injection(cs); 844 845 return kvm_arm_init_cpreg_list(cpu); 846 } 847 848 int kvm_arch_destroy_vcpu(CPUState *cs) 849 { 850 return 0; 851 } 852 853 bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx) 854 { 855 /* Return true if the regidx is a register we should synchronize 856 * via the cpreg_tuples array (ie is not a core or sve reg that 857 * we sync by hand in kvm_arch_get/put_registers()) 858 */ 859 switch (regidx & KVM_REG_ARM_COPROC_MASK) { 860 case KVM_REG_ARM_CORE: 861 case KVM_REG_ARM64_SVE: 862 return false; 863 default: 864 return true; 865 } 866 } 867 868 typedef struct CPRegStateLevel { 869 uint64_t regidx; 870 int level; 871 } CPRegStateLevel; 872 873 /* All system registers not listed in the following table are assumed to be 874 * of the level KVM_PUT_RUNTIME_STATE. If a register should be written less 875 * often, you must add it to this table with a state of either 876 * KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE. 877 */ 878 static const CPRegStateLevel non_runtime_cpregs[] = { 879 { KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE }, 880 }; 881 882 int kvm_arm_cpreg_level(uint64_t regidx) 883 { 884 int i; 885 886 for (i = 0; i < ARRAY_SIZE(non_runtime_cpregs); i++) { 887 const CPRegStateLevel *l = &non_runtime_cpregs[i]; 888 if (l->regidx == regidx) { 889 return l->level; 890 } 891 } 892 893 return KVM_PUT_RUNTIME_STATE; 894 } 895 896 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ 897 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 898 899 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \ 900 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 901 902 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \ 903 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) 904 905 static int kvm_arch_put_fpsimd(CPUState *cs) 906 { 907 CPUARMState *env = &ARM_CPU(cs)->env; 908 struct kvm_one_reg reg; 909 int i, ret; 910 911 for (i = 0; i < 32; i++) { 912 uint64_t *q = aa64_vfp_qreg(env, i); 913 #ifdef HOST_WORDS_BIGENDIAN 914 uint64_t fp_val[2] = { q[1], q[0] }; 915 reg.addr = (uintptr_t)fp_val; 916 #else 917 reg.addr = (uintptr_t)q; 918 #endif 919 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]); 920 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 921 if (ret) { 922 return ret; 923 } 924 } 925 926 return 0; 927 } 928 929 /* 930 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits 931 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard 932 * code the slice index to zero for now as it's unlikely we'll need more than 933 * one slice for quite some time. 934 */ 935 static int kvm_arch_put_sve(CPUState *cs) 936 { 937 ARMCPU *cpu = ARM_CPU(cs); 938 CPUARMState *env = &cpu->env; 939 uint64_t tmp[ARM_MAX_VQ * 2]; 940 uint64_t *r; 941 struct kvm_one_reg reg; 942 int n, ret; 943 944 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { 945 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2); 946 reg.addr = (uintptr_t)r; 947 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); 948 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 949 if (ret) { 950 return ret; 951 } 952 } 953 954 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { 955 r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0], 956 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 957 reg.addr = (uintptr_t)r; 958 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); 959 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 960 if (ret) { 961 return ret; 962 } 963 } 964 965 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0], 966 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 967 reg.addr = (uintptr_t)r; 968 reg.id = KVM_REG_ARM64_SVE_FFR(0); 969 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 970 if (ret) { 971 return ret; 972 } 973 974 return 0; 975 } 976 977 int kvm_arch_put_registers(CPUState *cs, int level) 978 { 979 struct kvm_one_reg reg; 980 uint64_t val; 981 uint32_t fpr; 982 int i, ret; 983 unsigned int el; 984 985 ARMCPU *cpu = ARM_CPU(cs); 986 CPUARMState *env = &cpu->env; 987 988 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the 989 * AArch64 registers before pushing them out to 64-bit KVM. 990 */ 991 if (!is_a64(env)) { 992 aarch64_sync_32_to_64(env); 993 } 994 995 for (i = 0; i < 31; i++) { 996 reg.id = AARCH64_CORE_REG(regs.regs[i]); 997 reg.addr = (uintptr_t) &env->xregs[i]; 998 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 999 if (ret) { 1000 return ret; 1001 } 1002 } 1003 1004 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the 1005 * QEMU side we keep the current SP in xregs[31] as well. 1006 */ 1007 aarch64_save_sp(env, 1); 1008 1009 reg.id = AARCH64_CORE_REG(regs.sp); 1010 reg.addr = (uintptr_t) &env->sp_el[0]; 1011 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1012 if (ret) { 1013 return ret; 1014 } 1015 1016 reg.id = AARCH64_CORE_REG(sp_el1); 1017 reg.addr = (uintptr_t) &env->sp_el[1]; 1018 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1019 if (ret) { 1020 return ret; 1021 } 1022 1023 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */ 1024 if (is_a64(env)) { 1025 val = pstate_read(env); 1026 } else { 1027 val = cpsr_read(env); 1028 } 1029 reg.id = AARCH64_CORE_REG(regs.pstate); 1030 reg.addr = (uintptr_t) &val; 1031 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1032 if (ret) { 1033 return ret; 1034 } 1035 1036 reg.id = AARCH64_CORE_REG(regs.pc); 1037 reg.addr = (uintptr_t) &env->pc; 1038 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1039 if (ret) { 1040 return ret; 1041 } 1042 1043 reg.id = AARCH64_CORE_REG(elr_el1); 1044 reg.addr = (uintptr_t) &env->elr_el[1]; 1045 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1046 if (ret) { 1047 return ret; 1048 } 1049 1050 /* Saved Program State Registers 1051 * 1052 * Before we restore from the banked_spsr[] array we need to 1053 * ensure that any modifications to env->spsr are correctly 1054 * reflected in the banks. 1055 */ 1056 el = arm_current_el(env); 1057 if (el > 0 && !is_a64(env)) { 1058 i = bank_number(env->uncached_cpsr & CPSR_M); 1059 env->banked_spsr[i] = env->spsr; 1060 } 1061 1062 /* KVM 0-4 map to QEMU banks 1-5 */ 1063 for (i = 0; i < KVM_NR_SPSR; i++) { 1064 reg.id = AARCH64_CORE_REG(spsr[i]); 1065 reg.addr = (uintptr_t) &env->banked_spsr[i + 1]; 1066 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1067 if (ret) { 1068 return ret; 1069 } 1070 } 1071 1072 if (cpu_isar_feature(aa64_sve, cpu)) { 1073 ret = kvm_arch_put_sve(cs); 1074 } else { 1075 ret = kvm_arch_put_fpsimd(cs); 1076 } 1077 if (ret) { 1078 return ret; 1079 } 1080 1081 reg.addr = (uintptr_t)(&fpr); 1082 fpr = vfp_get_fpsr(env); 1083 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); 1084 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1085 if (ret) { 1086 return ret; 1087 } 1088 1089 reg.addr = (uintptr_t)(&fpr); 1090 fpr = vfp_get_fpcr(env); 1091 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); 1092 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 1093 if (ret) { 1094 return ret; 1095 } 1096 1097 write_cpustate_to_list(cpu, true); 1098 1099 if (!write_list_to_kvmstate(cpu, level)) { 1100 return -EINVAL; 1101 } 1102 1103 /* 1104 * Setting VCPU events should be triggered after syncing the registers 1105 * to avoid overwriting potential changes made by KVM upon calling 1106 * KVM_SET_VCPU_EVENTS ioctl 1107 */ 1108 ret = kvm_put_vcpu_events(cpu); 1109 if (ret) { 1110 return ret; 1111 } 1112 1113 kvm_arm_sync_mpstate_to_kvm(cpu); 1114 1115 return ret; 1116 } 1117 1118 static int kvm_arch_get_fpsimd(CPUState *cs) 1119 { 1120 CPUARMState *env = &ARM_CPU(cs)->env; 1121 struct kvm_one_reg reg; 1122 int i, ret; 1123 1124 for (i = 0; i < 32; i++) { 1125 uint64_t *q = aa64_vfp_qreg(env, i); 1126 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]); 1127 reg.addr = (uintptr_t)q; 1128 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1129 if (ret) { 1130 return ret; 1131 } else { 1132 #ifdef HOST_WORDS_BIGENDIAN 1133 uint64_t t; 1134 t = q[0], q[0] = q[1], q[1] = t; 1135 #endif 1136 } 1137 } 1138 1139 return 0; 1140 } 1141 1142 /* 1143 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits 1144 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard 1145 * code the slice index to zero for now as it's unlikely we'll need more than 1146 * one slice for quite some time. 1147 */ 1148 static int kvm_arch_get_sve(CPUState *cs) 1149 { 1150 ARMCPU *cpu = ARM_CPU(cs); 1151 CPUARMState *env = &cpu->env; 1152 struct kvm_one_reg reg; 1153 uint64_t *r; 1154 int n, ret; 1155 1156 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) { 1157 r = &env->vfp.zregs[n].d[0]; 1158 reg.addr = (uintptr_t)r; 1159 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0); 1160 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1161 if (ret) { 1162 return ret; 1163 } 1164 sve_bswap64(r, r, cpu->sve_max_vq * 2); 1165 } 1166 1167 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) { 1168 r = &env->vfp.pregs[n].p[0]; 1169 reg.addr = (uintptr_t)r; 1170 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0); 1171 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1172 if (ret) { 1173 return ret; 1174 } 1175 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 1176 } 1177 1178 r = &env->vfp.pregs[FFR_PRED_NUM].p[0]; 1179 reg.addr = (uintptr_t)r; 1180 reg.id = KVM_REG_ARM64_SVE_FFR(0); 1181 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1182 if (ret) { 1183 return ret; 1184 } 1185 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8)); 1186 1187 return 0; 1188 } 1189 1190 int kvm_arch_get_registers(CPUState *cs) 1191 { 1192 struct kvm_one_reg reg; 1193 uint64_t val; 1194 unsigned int el; 1195 uint32_t fpr; 1196 int i, ret; 1197 1198 ARMCPU *cpu = ARM_CPU(cs); 1199 CPUARMState *env = &cpu->env; 1200 1201 for (i = 0; i < 31; i++) { 1202 reg.id = AARCH64_CORE_REG(regs.regs[i]); 1203 reg.addr = (uintptr_t) &env->xregs[i]; 1204 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1205 if (ret) { 1206 return ret; 1207 } 1208 } 1209 1210 reg.id = AARCH64_CORE_REG(regs.sp); 1211 reg.addr = (uintptr_t) &env->sp_el[0]; 1212 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1213 if (ret) { 1214 return ret; 1215 } 1216 1217 reg.id = AARCH64_CORE_REG(sp_el1); 1218 reg.addr = (uintptr_t) &env->sp_el[1]; 1219 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1220 if (ret) { 1221 return ret; 1222 } 1223 1224 reg.id = AARCH64_CORE_REG(regs.pstate); 1225 reg.addr = (uintptr_t) &val; 1226 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1227 if (ret) { 1228 return ret; 1229 } 1230 1231 env->aarch64 = ((val & PSTATE_nRW) == 0); 1232 if (is_a64(env)) { 1233 pstate_write(env, val); 1234 } else { 1235 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw); 1236 } 1237 1238 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the 1239 * QEMU side we keep the current SP in xregs[31] as well. 1240 */ 1241 aarch64_restore_sp(env, 1); 1242 1243 reg.id = AARCH64_CORE_REG(regs.pc); 1244 reg.addr = (uintptr_t) &env->pc; 1245 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1246 if (ret) { 1247 return ret; 1248 } 1249 1250 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the 1251 * incoming AArch64 regs received from 64-bit KVM. 1252 * We must perform this after all of the registers have been acquired from 1253 * the kernel. 1254 */ 1255 if (!is_a64(env)) { 1256 aarch64_sync_64_to_32(env); 1257 } 1258 1259 reg.id = AARCH64_CORE_REG(elr_el1); 1260 reg.addr = (uintptr_t) &env->elr_el[1]; 1261 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1262 if (ret) { 1263 return ret; 1264 } 1265 1266 /* Fetch the SPSR registers 1267 * 1268 * KVM SPSRs 0-4 map to QEMU banks 1-5 1269 */ 1270 for (i = 0; i < KVM_NR_SPSR; i++) { 1271 reg.id = AARCH64_CORE_REG(spsr[i]); 1272 reg.addr = (uintptr_t) &env->banked_spsr[i + 1]; 1273 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1274 if (ret) { 1275 return ret; 1276 } 1277 } 1278 1279 el = arm_current_el(env); 1280 if (el > 0 && !is_a64(env)) { 1281 i = bank_number(env->uncached_cpsr & CPSR_M); 1282 env->spsr = env->banked_spsr[i]; 1283 } 1284 1285 if (cpu_isar_feature(aa64_sve, cpu)) { 1286 ret = kvm_arch_get_sve(cs); 1287 } else { 1288 ret = kvm_arch_get_fpsimd(cs); 1289 } 1290 if (ret) { 1291 return ret; 1292 } 1293 1294 reg.addr = (uintptr_t)(&fpr); 1295 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr); 1296 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1297 if (ret) { 1298 return ret; 1299 } 1300 vfp_set_fpsr(env, fpr); 1301 1302 reg.addr = (uintptr_t)(&fpr); 1303 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr); 1304 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 1305 if (ret) { 1306 return ret; 1307 } 1308 vfp_set_fpcr(env, fpr); 1309 1310 ret = kvm_get_vcpu_events(cpu); 1311 if (ret) { 1312 return ret; 1313 } 1314 1315 if (!write_kvmstate_to_list(cpu)) { 1316 return -EINVAL; 1317 } 1318 /* Note that it's OK to have registers which aren't in CPUState, 1319 * so we can ignore a failure return here. 1320 */ 1321 write_list_to_cpustate(cpu); 1322 1323 kvm_arm_sync_mpstate_to_qemu(cpu); 1324 1325 /* TODO: other registers */ 1326 return ret; 1327 } 1328 1329 /* C6.6.29 BRK instruction */ 1330 static const uint32_t brk_insn = 0xd4200000; 1331 1332 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 1333 { 1334 if (have_guest_debug) { 1335 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) || 1336 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) { 1337 return -EINVAL; 1338 } 1339 return 0; 1340 } else { 1341 error_report("guest debug not supported on this kernel"); 1342 return -EINVAL; 1343 } 1344 } 1345 1346 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp) 1347 { 1348 static uint32_t brk; 1349 1350 if (have_guest_debug) { 1351 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) || 1352 brk != brk_insn || 1353 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) { 1354 return -EINVAL; 1355 } 1356 return 0; 1357 } else { 1358 error_report("guest debug not supported on this kernel"); 1359 return -EINVAL; 1360 } 1361 } 1362 1363 /* See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register 1364 * 1365 * To minimise translating between kernel and user-space the kernel 1366 * ABI just provides user-space with the full exception syndrome 1367 * register value to be decoded in QEMU. 1368 */ 1369 1370 bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit) 1371 { 1372 int hsr_ec = syn_get_ec(debug_exit->hsr); 1373 ARMCPU *cpu = ARM_CPU(cs); 1374 CPUClass *cc = CPU_GET_CLASS(cs); 1375 CPUARMState *env = &cpu->env; 1376 1377 /* Ensure PC is synchronised */ 1378 kvm_cpu_synchronize_state(cs); 1379 1380 switch (hsr_ec) { 1381 case EC_SOFTWARESTEP: 1382 if (cs->singlestep_enabled) { 1383 return true; 1384 } else { 1385 /* 1386 * The kernel should have suppressed the guest's ability to 1387 * single step at this point so something has gone wrong. 1388 */ 1389 error_report("%s: guest single-step while debugging unsupported" 1390 " (%"PRIx64", %"PRIx32")", 1391 __func__, env->pc, debug_exit->hsr); 1392 return false; 1393 } 1394 break; 1395 case EC_AA64_BKPT: 1396 if (kvm_find_sw_breakpoint(cs, env->pc)) { 1397 return true; 1398 } 1399 break; 1400 case EC_BREAKPOINT: 1401 if (find_hw_breakpoint(cs, env->pc)) { 1402 return true; 1403 } 1404 break; 1405 case EC_WATCHPOINT: 1406 { 1407 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far); 1408 if (wp) { 1409 cs->watchpoint_hit = wp; 1410 return true; 1411 } 1412 break; 1413 } 1414 default: 1415 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")", 1416 __func__, debug_exit->hsr, env->pc); 1417 } 1418 1419 /* If we are not handling the debug exception it must belong to 1420 * the guest. Let's re-use the existing TCG interrupt code to set 1421 * everything up properly. 1422 */ 1423 cs->exception_index = EXCP_BKPT; 1424 env->exception.syndrome = debug_exit->hsr; 1425 env->exception.vaddress = debug_exit->far; 1426 env->exception.target_el = 1; 1427 qemu_mutex_lock_iothread(); 1428 cc->do_interrupt(cs); 1429 qemu_mutex_unlock_iothread(); 1430 1431 return false; 1432 } 1433