1 /* 2 * QEMU KVM support 3 * 4 * Copyright IBM, Corp. 2008 5 * Red Hat, Inc. 2008 6 * 7 * Authors: 8 * Anthony Liguori <aliguori@us.ibm.com> 9 * Glauber Costa <gcosta@redhat.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2 or later. 12 * See the COPYING file in the top-level directory. 13 * 14 */ 15 16 #include "qemu/osdep.h" 17 #include <sys/ioctl.h> 18 #include <poll.h> 19 20 #include <linux/kvm.h> 21 22 #include "qemu/atomic.h" 23 #include "qemu/option.h" 24 #include "qemu/config-file.h" 25 #include "qemu/error-report.h" 26 #include "qapi/error.h" 27 #include "hw/pci/msi.h" 28 #include "hw/pci/msix.h" 29 #include "hw/s390x/adapter.h" 30 #include "exec/gdbstub.h" 31 #include "sysemu/kvm_int.h" 32 #include "sysemu/runstate.h" 33 #include "sysemu/cpus.h" 34 #include "sysemu/accel-blocker.h" 35 #include "qemu/bswap.h" 36 #include "exec/memory.h" 37 #include "exec/ram_addr.h" 38 #include "qemu/event_notifier.h" 39 #include "qemu/main-loop.h" 40 #include "trace.h" 41 #include "hw/irq.h" 42 #include "qapi/visitor.h" 43 #include "qapi/qapi-types-common.h" 44 #include "qapi/qapi-visit-common.h" 45 #include "sysemu/reset.h" 46 #include "qemu/guest-random.h" 47 #include "sysemu/hw_accel.h" 48 #include "kvm-cpus.h" 49 #include "sysemu/dirtylimit.h" 50 #include "qemu/range.h" 51 52 #include "hw/boards.h" 53 #include "sysemu/stats.h" 54 55 /* This check must be after config-host.h is included */ 56 #ifdef CONFIG_EVENTFD 57 #include <sys/eventfd.h> 58 #endif 59 60 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We 61 * need to use the real host PAGE_SIZE, as that's what KVM will use. 62 */ 63 #ifdef PAGE_SIZE 64 #undef PAGE_SIZE 65 #endif 66 #define PAGE_SIZE qemu_real_host_page_size() 67 68 #ifndef KVM_GUESTDBG_BLOCKIRQ 69 #define KVM_GUESTDBG_BLOCKIRQ 0 70 #endif 71 72 struct KVMParkedVcpu { 73 unsigned long vcpu_id; 74 int kvm_fd; 75 QLIST_ENTRY(KVMParkedVcpu) node; 76 }; 77 78 KVMState *kvm_state; 79 bool kvm_kernel_irqchip; 80 bool kvm_split_irqchip; 81 bool kvm_async_interrupts_allowed; 82 bool kvm_halt_in_kernel_allowed; 83 bool kvm_resamplefds_allowed; 84 bool kvm_msi_via_irqfd_allowed; 85 bool kvm_gsi_routing_allowed; 86 bool kvm_gsi_direct_mapping; 87 bool kvm_allowed; 88 bool kvm_readonly_mem_allowed; 89 bool kvm_vm_attributes_allowed; 90 bool kvm_msi_use_devid; 91 static bool kvm_has_guest_debug; 92 static int kvm_sstep_flags; 93 static bool kvm_immediate_exit; 94 static hwaddr kvm_max_slot_size = ~0; 95 96 static const KVMCapabilityInfo kvm_required_capabilites[] = { 97 KVM_CAP_INFO(USER_MEMORY), 98 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS), 99 KVM_CAP_INFO(JOIN_MEMORY_REGIONS_WORKS), 100 KVM_CAP_INFO(INTERNAL_ERROR_DATA), 101 KVM_CAP_INFO(IOEVENTFD), 102 KVM_CAP_INFO(IOEVENTFD_ANY_LENGTH), 103 KVM_CAP_LAST_INFO 104 }; 105 106 static NotifierList kvm_irqchip_change_notifiers = 107 NOTIFIER_LIST_INITIALIZER(kvm_irqchip_change_notifiers); 108 109 struct KVMResampleFd { 110 int gsi; 111 EventNotifier *resample_event; 112 QLIST_ENTRY(KVMResampleFd) node; 113 }; 114 typedef struct KVMResampleFd KVMResampleFd; 115 116 /* 117 * Only used with split irqchip where we need to do the resample fd 118 * kick for the kernel from userspace. 119 */ 120 static QLIST_HEAD(, KVMResampleFd) kvm_resample_fd_list = 121 QLIST_HEAD_INITIALIZER(kvm_resample_fd_list); 122 123 static QemuMutex kml_slots_lock; 124 125 #define kvm_slots_lock() qemu_mutex_lock(&kml_slots_lock) 126 #define kvm_slots_unlock() qemu_mutex_unlock(&kml_slots_lock) 127 128 static void kvm_slot_init_dirty_bitmap(KVMSlot *mem); 129 130 static inline void kvm_resample_fd_remove(int gsi) 131 { 132 KVMResampleFd *rfd; 133 134 QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { 135 if (rfd->gsi == gsi) { 136 QLIST_REMOVE(rfd, node); 137 g_free(rfd); 138 break; 139 } 140 } 141 } 142 143 static inline void kvm_resample_fd_insert(int gsi, EventNotifier *event) 144 { 145 KVMResampleFd *rfd = g_new0(KVMResampleFd, 1); 146 147 rfd->gsi = gsi; 148 rfd->resample_event = event; 149 150 QLIST_INSERT_HEAD(&kvm_resample_fd_list, rfd, node); 151 } 152 153 void kvm_resample_fd_notify(int gsi) 154 { 155 KVMResampleFd *rfd; 156 157 QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { 158 if (rfd->gsi == gsi) { 159 event_notifier_set(rfd->resample_event); 160 trace_kvm_resample_fd_notify(gsi); 161 return; 162 } 163 } 164 } 165 166 unsigned int kvm_get_max_memslots(void) 167 { 168 KVMState *s = KVM_STATE(current_accel()); 169 170 return s->nr_slots; 171 } 172 173 unsigned int kvm_get_free_memslots(void) 174 { 175 unsigned int used_slots = 0; 176 KVMState *s = kvm_state; 177 int i; 178 179 kvm_slots_lock(); 180 for (i = 0; i < s->nr_as; i++) { 181 if (!s->as[i].ml) { 182 continue; 183 } 184 used_slots = MAX(used_slots, s->as[i].ml->nr_used_slots); 185 } 186 kvm_slots_unlock(); 187 188 return s->nr_slots - used_slots; 189 } 190 191 /* Called with KVMMemoryListener.slots_lock held */ 192 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml) 193 { 194 KVMState *s = kvm_state; 195 int i; 196 197 for (i = 0; i < s->nr_slots; i++) { 198 if (kml->slots[i].memory_size == 0) { 199 return &kml->slots[i]; 200 } 201 } 202 203 return NULL; 204 } 205 206 /* Called with KVMMemoryListener.slots_lock held */ 207 static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml) 208 { 209 KVMSlot *slot = kvm_get_free_slot(kml); 210 211 if (slot) { 212 return slot; 213 } 214 215 fprintf(stderr, "%s: no free slot available\n", __func__); 216 abort(); 217 } 218 219 static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml, 220 hwaddr start_addr, 221 hwaddr size) 222 { 223 KVMState *s = kvm_state; 224 int i; 225 226 for (i = 0; i < s->nr_slots; i++) { 227 KVMSlot *mem = &kml->slots[i]; 228 229 if (start_addr == mem->start_addr && size == mem->memory_size) { 230 return mem; 231 } 232 } 233 234 return NULL; 235 } 236 237 /* 238 * Calculate and align the start address and the size of the section. 239 * Return the size. If the size is 0, the aligned section is empty. 240 */ 241 static hwaddr kvm_align_section(MemoryRegionSection *section, 242 hwaddr *start) 243 { 244 hwaddr size = int128_get64(section->size); 245 hwaddr delta, aligned; 246 247 /* kvm works in page size chunks, but the function may be called 248 with sub-page size and unaligned start address. Pad the start 249 address to next and truncate size to previous page boundary. */ 250 aligned = ROUND_UP(section->offset_within_address_space, 251 qemu_real_host_page_size()); 252 delta = aligned - section->offset_within_address_space; 253 *start = aligned; 254 if (delta > size) { 255 return 0; 256 } 257 258 return (size - delta) & qemu_real_host_page_mask(); 259 } 260 261 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram, 262 hwaddr *phys_addr) 263 { 264 KVMMemoryListener *kml = &s->memory_listener; 265 int i, ret = 0; 266 267 kvm_slots_lock(); 268 for (i = 0; i < s->nr_slots; i++) { 269 KVMSlot *mem = &kml->slots[i]; 270 271 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) { 272 *phys_addr = mem->start_addr + (ram - mem->ram); 273 ret = 1; 274 break; 275 } 276 } 277 kvm_slots_unlock(); 278 279 return ret; 280 } 281 282 static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new) 283 { 284 KVMState *s = kvm_state; 285 struct kvm_userspace_memory_region mem; 286 int ret; 287 288 mem.slot = slot->slot | (kml->as_id << 16); 289 mem.guest_phys_addr = slot->start_addr; 290 mem.userspace_addr = (unsigned long)slot->ram; 291 mem.flags = slot->flags; 292 293 if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) { 294 /* Set the slot size to 0 before setting the slot to the desired 295 * value. This is needed based on KVM commit 75d61fbc. */ 296 mem.memory_size = 0; 297 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 298 if (ret < 0) { 299 goto err; 300 } 301 } 302 mem.memory_size = slot->memory_size; 303 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 304 slot->old_flags = mem.flags; 305 err: 306 trace_kvm_set_user_memory(mem.slot, mem.flags, mem.guest_phys_addr, 307 mem.memory_size, mem.userspace_addr, ret); 308 if (ret < 0) { 309 error_report("%s: KVM_SET_USER_MEMORY_REGION failed, slot=%d," 310 " start=0x%" PRIx64 ", size=0x%" PRIx64 ": %s", 311 __func__, mem.slot, slot->start_addr, 312 (uint64_t)mem.memory_size, strerror(errno)); 313 } 314 return ret; 315 } 316 317 static int do_kvm_destroy_vcpu(CPUState *cpu) 318 { 319 KVMState *s = kvm_state; 320 long mmap_size; 321 struct KVMParkedVcpu *vcpu = NULL; 322 int ret = 0; 323 324 trace_kvm_destroy_vcpu(); 325 326 ret = kvm_arch_destroy_vcpu(cpu); 327 if (ret < 0) { 328 goto err; 329 } 330 331 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 332 if (mmap_size < 0) { 333 ret = mmap_size; 334 trace_kvm_failed_get_vcpu_mmap_size(); 335 goto err; 336 } 337 338 ret = munmap(cpu->kvm_run, mmap_size); 339 if (ret < 0) { 340 goto err; 341 } 342 343 if (cpu->kvm_dirty_gfns) { 344 ret = munmap(cpu->kvm_dirty_gfns, s->kvm_dirty_ring_bytes); 345 if (ret < 0) { 346 goto err; 347 } 348 } 349 350 vcpu = g_malloc0(sizeof(*vcpu)); 351 vcpu->vcpu_id = kvm_arch_vcpu_id(cpu); 352 vcpu->kvm_fd = cpu->kvm_fd; 353 QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); 354 err: 355 return ret; 356 } 357 358 void kvm_destroy_vcpu(CPUState *cpu) 359 { 360 if (do_kvm_destroy_vcpu(cpu) < 0) { 361 error_report("kvm_destroy_vcpu failed"); 362 exit(EXIT_FAILURE); 363 } 364 } 365 366 static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id) 367 { 368 struct KVMParkedVcpu *cpu; 369 370 QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { 371 if (cpu->vcpu_id == vcpu_id) { 372 int kvm_fd; 373 374 QLIST_REMOVE(cpu, node); 375 kvm_fd = cpu->kvm_fd; 376 g_free(cpu); 377 return kvm_fd; 378 } 379 } 380 381 return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id); 382 } 383 384 int kvm_init_vcpu(CPUState *cpu, Error **errp) 385 { 386 KVMState *s = kvm_state; 387 long mmap_size; 388 int ret; 389 390 trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); 391 392 ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu)); 393 if (ret < 0) { 394 error_setg_errno(errp, -ret, "kvm_init_vcpu: kvm_get_vcpu failed (%lu)", 395 kvm_arch_vcpu_id(cpu)); 396 goto err; 397 } 398 399 cpu->kvm_fd = ret; 400 cpu->kvm_state = s; 401 cpu->vcpu_dirty = true; 402 cpu->dirty_pages = 0; 403 cpu->throttle_us_per_full = 0; 404 405 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 406 if (mmap_size < 0) { 407 ret = mmap_size; 408 error_setg_errno(errp, -mmap_size, 409 "kvm_init_vcpu: KVM_GET_VCPU_MMAP_SIZE failed"); 410 goto err; 411 } 412 413 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, 414 cpu->kvm_fd, 0); 415 if (cpu->kvm_run == MAP_FAILED) { 416 ret = -errno; 417 error_setg_errno(errp, ret, 418 "kvm_init_vcpu: mmap'ing vcpu state failed (%lu)", 419 kvm_arch_vcpu_id(cpu)); 420 goto err; 421 } 422 423 if (s->coalesced_mmio && !s->coalesced_mmio_ring) { 424 s->coalesced_mmio_ring = 425 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE; 426 } 427 428 if (s->kvm_dirty_ring_size) { 429 /* Use MAP_SHARED to share pages with the kernel */ 430 cpu->kvm_dirty_gfns = mmap(NULL, s->kvm_dirty_ring_bytes, 431 PROT_READ | PROT_WRITE, MAP_SHARED, 432 cpu->kvm_fd, 433 PAGE_SIZE * KVM_DIRTY_LOG_PAGE_OFFSET); 434 if (cpu->kvm_dirty_gfns == MAP_FAILED) { 435 ret = -errno; 436 goto err; 437 } 438 } 439 440 ret = kvm_arch_init_vcpu(cpu); 441 if (ret < 0) { 442 error_setg_errno(errp, -ret, 443 "kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)", 444 kvm_arch_vcpu_id(cpu)); 445 } 446 cpu->kvm_vcpu_stats_fd = kvm_vcpu_ioctl(cpu, KVM_GET_STATS_FD, NULL); 447 448 err: 449 return ret; 450 } 451 452 /* 453 * dirty pages logging control 454 */ 455 456 static int kvm_mem_flags(MemoryRegion *mr) 457 { 458 bool readonly = mr->readonly || memory_region_is_romd(mr); 459 int flags = 0; 460 461 if (memory_region_get_dirty_log_mask(mr) != 0) { 462 flags |= KVM_MEM_LOG_DIRTY_PAGES; 463 } 464 if (readonly && kvm_readonly_mem_allowed) { 465 flags |= KVM_MEM_READONLY; 466 } 467 return flags; 468 } 469 470 /* Called with KVMMemoryListener.slots_lock held */ 471 static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem, 472 MemoryRegion *mr) 473 { 474 mem->flags = kvm_mem_flags(mr); 475 476 /* If nothing changed effectively, no need to issue ioctl */ 477 if (mem->flags == mem->old_flags) { 478 return 0; 479 } 480 481 kvm_slot_init_dirty_bitmap(mem); 482 return kvm_set_user_memory_region(kml, mem, false); 483 } 484 485 static int kvm_section_update_flags(KVMMemoryListener *kml, 486 MemoryRegionSection *section) 487 { 488 hwaddr start_addr, size, slot_size; 489 KVMSlot *mem; 490 int ret = 0; 491 492 size = kvm_align_section(section, &start_addr); 493 if (!size) { 494 return 0; 495 } 496 497 kvm_slots_lock(); 498 499 while (size && !ret) { 500 slot_size = MIN(kvm_max_slot_size, size); 501 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 502 if (!mem) { 503 /* We don't have a slot if we want to trap every access. */ 504 goto out; 505 } 506 507 ret = kvm_slot_update_flags(kml, mem, section->mr); 508 start_addr += slot_size; 509 size -= slot_size; 510 } 511 512 out: 513 kvm_slots_unlock(); 514 return ret; 515 } 516 517 static void kvm_log_start(MemoryListener *listener, 518 MemoryRegionSection *section, 519 int old, int new) 520 { 521 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 522 int r; 523 524 if (old != 0) { 525 return; 526 } 527 528 r = kvm_section_update_flags(kml, section); 529 if (r < 0) { 530 abort(); 531 } 532 } 533 534 static void kvm_log_stop(MemoryListener *listener, 535 MemoryRegionSection *section, 536 int old, int new) 537 { 538 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 539 int r; 540 541 if (new != 0) { 542 return; 543 } 544 545 r = kvm_section_update_flags(kml, section); 546 if (r < 0) { 547 abort(); 548 } 549 } 550 551 /* get kvm's dirty pages bitmap and update qemu's */ 552 static void kvm_slot_sync_dirty_pages(KVMSlot *slot) 553 { 554 ram_addr_t start = slot->ram_start_offset; 555 ram_addr_t pages = slot->memory_size / qemu_real_host_page_size(); 556 557 cpu_physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); 558 } 559 560 static void kvm_slot_reset_dirty_pages(KVMSlot *slot) 561 { 562 memset(slot->dirty_bmap, 0, slot->dirty_bmap_size); 563 } 564 565 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1)) 566 567 /* Allocate the dirty bitmap for a slot */ 568 static void kvm_slot_init_dirty_bitmap(KVMSlot *mem) 569 { 570 if (!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) || mem->dirty_bmap) { 571 return; 572 } 573 574 /* 575 * XXX bad kernel interface alert 576 * For dirty bitmap, kernel allocates array of size aligned to 577 * bits-per-long. But for case when the kernel is 64bits and 578 * the userspace is 32bits, userspace can't align to the same 579 * bits-per-long, since sizeof(long) is different between kernel 580 * and user space. This way, userspace will provide buffer which 581 * may be 4 bytes less than the kernel will use, resulting in 582 * userspace memory corruption (which is not detectable by valgrind 583 * too, in most cases). 584 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in 585 * a hope that sizeof(long) won't become >8 any time soon. 586 * 587 * Note: the granule of kvm dirty log is qemu_real_host_page_size. 588 * And mem->memory_size is aligned to it (otherwise this mem can't 589 * be registered to KVM). 590 */ 591 hwaddr bitmap_size = ALIGN(mem->memory_size / qemu_real_host_page_size(), 592 /*HOST_LONG_BITS*/ 64) / 8; 593 mem->dirty_bmap = g_malloc0(bitmap_size); 594 mem->dirty_bmap_size = bitmap_size; 595 } 596 597 /* 598 * Sync dirty bitmap from kernel to KVMSlot.dirty_bmap, return true if 599 * succeeded, false otherwise 600 */ 601 static bool kvm_slot_get_dirty_log(KVMState *s, KVMSlot *slot) 602 { 603 struct kvm_dirty_log d = {}; 604 int ret; 605 606 d.dirty_bitmap = slot->dirty_bmap; 607 d.slot = slot->slot | (slot->as_id << 16); 608 ret = kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d); 609 610 if (ret == -ENOENT) { 611 /* kernel does not have dirty bitmap in this slot */ 612 ret = 0; 613 } 614 if (ret) { 615 error_report_once("%s: KVM_GET_DIRTY_LOG failed with %d", 616 __func__, ret); 617 } 618 return ret == 0; 619 } 620 621 /* Should be with all slots_lock held for the address spaces. */ 622 static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id, 623 uint32_t slot_id, uint64_t offset) 624 { 625 KVMMemoryListener *kml; 626 KVMSlot *mem; 627 628 if (as_id >= s->nr_as) { 629 return; 630 } 631 632 kml = s->as[as_id].ml; 633 mem = &kml->slots[slot_id]; 634 635 if (!mem->memory_size || offset >= 636 (mem->memory_size / qemu_real_host_page_size())) { 637 return; 638 } 639 640 set_bit(offset, mem->dirty_bmap); 641 } 642 643 static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn) 644 { 645 /* 646 * Read the flags before the value. Pairs with barrier in 647 * KVM's kvm_dirty_ring_push() function. 648 */ 649 return qatomic_load_acquire(&gfn->flags) == KVM_DIRTY_GFN_F_DIRTY; 650 } 651 652 static void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn) 653 { 654 /* 655 * Use a store-release so that the CPU that executes KVM_RESET_DIRTY_RINGS 656 * sees the full content of the ring: 657 * 658 * CPU0 CPU1 CPU2 659 * ------------------------------------------------------------------------------ 660 * fill gfn0 661 * store-rel flags for gfn0 662 * load-acq flags for gfn0 663 * store-rel RESET for gfn0 664 * ioctl(RESET_RINGS) 665 * load-acq flags for gfn0 666 * check if flags have RESET 667 * 668 * The synchronization goes from CPU2 to CPU0 to CPU1. 669 */ 670 qatomic_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET); 671 } 672 673 /* 674 * Should be with all slots_lock held for the address spaces. It returns the 675 * dirty page we've collected on this dirty ring. 676 */ 677 static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu) 678 { 679 struct kvm_dirty_gfn *dirty_gfns = cpu->kvm_dirty_gfns, *cur; 680 uint32_t ring_size = s->kvm_dirty_ring_size; 681 uint32_t count = 0, fetch = cpu->kvm_fetch_index; 682 683 /* 684 * It's possible that we race with vcpu creation code where the vcpu is 685 * put onto the vcpus list but not yet initialized the dirty ring 686 * structures. If so, skip it. 687 */ 688 if (!cpu->created) { 689 return 0; 690 } 691 692 assert(dirty_gfns && ring_size); 693 trace_kvm_dirty_ring_reap_vcpu(cpu->cpu_index); 694 695 while (true) { 696 cur = &dirty_gfns[fetch % ring_size]; 697 if (!dirty_gfn_is_dirtied(cur)) { 698 break; 699 } 700 kvm_dirty_ring_mark_page(s, cur->slot >> 16, cur->slot & 0xffff, 701 cur->offset); 702 dirty_gfn_set_collected(cur); 703 trace_kvm_dirty_ring_page(cpu->cpu_index, fetch, cur->offset); 704 fetch++; 705 count++; 706 } 707 cpu->kvm_fetch_index = fetch; 708 cpu->dirty_pages += count; 709 710 return count; 711 } 712 713 /* Must be with slots_lock held */ 714 static uint64_t kvm_dirty_ring_reap_locked(KVMState *s, CPUState* cpu) 715 { 716 int ret; 717 uint64_t total = 0; 718 int64_t stamp; 719 720 stamp = get_clock(); 721 722 if (cpu) { 723 total = kvm_dirty_ring_reap_one(s, cpu); 724 } else { 725 CPU_FOREACH(cpu) { 726 total += kvm_dirty_ring_reap_one(s, cpu); 727 } 728 } 729 730 if (total) { 731 ret = kvm_vm_ioctl(s, KVM_RESET_DIRTY_RINGS); 732 assert(ret == total); 733 } 734 735 stamp = get_clock() - stamp; 736 737 if (total) { 738 trace_kvm_dirty_ring_reap(total, stamp / 1000); 739 } 740 741 return total; 742 } 743 744 /* 745 * Currently for simplicity, we must hold BQL before calling this. We can 746 * consider to drop the BQL if we're clear with all the race conditions. 747 */ 748 static uint64_t kvm_dirty_ring_reap(KVMState *s, CPUState *cpu) 749 { 750 uint64_t total; 751 752 /* 753 * We need to lock all kvm slots for all address spaces here, 754 * because: 755 * 756 * (1) We need to mark dirty for dirty bitmaps in multiple slots 757 * and for tons of pages, so it's better to take the lock here 758 * once rather than once per page. And more importantly, 759 * 760 * (2) We must _NOT_ publish dirty bits to the other threads 761 * (e.g., the migration thread) via the kvm memory slot dirty 762 * bitmaps before correctly re-protect those dirtied pages. 763 * Otherwise we can have potential risk of data corruption if 764 * the page data is read in the other thread before we do 765 * reset below. 766 */ 767 kvm_slots_lock(); 768 total = kvm_dirty_ring_reap_locked(s, cpu); 769 kvm_slots_unlock(); 770 771 return total; 772 } 773 774 static void do_kvm_cpu_synchronize_kick(CPUState *cpu, run_on_cpu_data arg) 775 { 776 /* No need to do anything */ 777 } 778 779 /* 780 * Kick all vcpus out in a synchronized way. When returned, we 781 * guarantee that every vcpu has been kicked and at least returned to 782 * userspace once. 783 */ 784 static void kvm_cpu_synchronize_kick_all(void) 785 { 786 CPUState *cpu; 787 788 CPU_FOREACH(cpu) { 789 run_on_cpu(cpu, do_kvm_cpu_synchronize_kick, RUN_ON_CPU_NULL); 790 } 791 } 792 793 /* 794 * Flush all the existing dirty pages to the KVM slot buffers. When 795 * this call returns, we guarantee that all the touched dirty pages 796 * before calling this function have been put into the per-kvmslot 797 * dirty bitmap. 798 * 799 * This function must be called with BQL held. 800 */ 801 static void kvm_dirty_ring_flush(void) 802 { 803 trace_kvm_dirty_ring_flush(0); 804 /* 805 * The function needs to be serialized. Since this function 806 * should always be with BQL held, serialization is guaranteed. 807 * However, let's be sure of it. 808 */ 809 assert(bql_locked()); 810 /* 811 * First make sure to flush the hardware buffers by kicking all 812 * vcpus out in a synchronous way. 813 */ 814 kvm_cpu_synchronize_kick_all(); 815 kvm_dirty_ring_reap(kvm_state, NULL); 816 trace_kvm_dirty_ring_flush(1); 817 } 818 819 /** 820 * kvm_physical_sync_dirty_bitmap - Sync dirty bitmap from kernel space 821 * 822 * This function will first try to fetch dirty bitmap from the kernel, 823 * and then updates qemu's dirty bitmap. 824 * 825 * NOTE: caller must be with kml->slots_lock held. 826 * 827 * @kml: the KVM memory listener object 828 * @section: the memory section to sync the dirty bitmap with 829 */ 830 static void kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, 831 MemoryRegionSection *section) 832 { 833 KVMState *s = kvm_state; 834 KVMSlot *mem; 835 hwaddr start_addr, size; 836 hwaddr slot_size; 837 838 size = kvm_align_section(section, &start_addr); 839 while (size) { 840 slot_size = MIN(kvm_max_slot_size, size); 841 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 842 if (!mem) { 843 /* We don't have a slot if we want to trap every access. */ 844 return; 845 } 846 if (kvm_slot_get_dirty_log(s, mem)) { 847 kvm_slot_sync_dirty_pages(mem); 848 } 849 start_addr += slot_size; 850 size -= slot_size; 851 } 852 } 853 854 /* Alignment requirement for KVM_CLEAR_DIRTY_LOG - 64 pages */ 855 #define KVM_CLEAR_LOG_SHIFT 6 856 #define KVM_CLEAR_LOG_ALIGN (qemu_real_host_page_size() << KVM_CLEAR_LOG_SHIFT) 857 #define KVM_CLEAR_LOG_MASK (-KVM_CLEAR_LOG_ALIGN) 858 859 static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start, 860 uint64_t size) 861 { 862 KVMState *s = kvm_state; 863 uint64_t end, bmap_start, start_delta, bmap_npages; 864 struct kvm_clear_dirty_log d; 865 unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size(); 866 int ret; 867 868 /* 869 * We need to extend either the start or the size or both to 870 * satisfy the KVM interface requirement. Firstly, do the start 871 * page alignment on 64 host pages 872 */ 873 bmap_start = start & KVM_CLEAR_LOG_MASK; 874 start_delta = start - bmap_start; 875 bmap_start /= psize; 876 877 /* 878 * The kernel interface has restriction on the size too, that either: 879 * 880 * (1) the size is 64 host pages aligned (just like the start), or 881 * (2) the size fills up until the end of the KVM memslot. 882 */ 883 bmap_npages = DIV_ROUND_UP(size + start_delta, KVM_CLEAR_LOG_ALIGN) 884 << KVM_CLEAR_LOG_SHIFT; 885 end = mem->memory_size / psize; 886 if (bmap_npages > end - bmap_start) { 887 bmap_npages = end - bmap_start; 888 } 889 start_delta /= psize; 890 891 /* 892 * Prepare the bitmap to clear dirty bits. Here we must guarantee 893 * that we won't clear any unknown dirty bits otherwise we might 894 * accidentally clear some set bits which are not yet synced from 895 * the kernel into QEMU's bitmap, then we'll lose track of the 896 * guest modifications upon those pages (which can directly lead 897 * to guest data loss or panic after migration). 898 * 899 * Layout of the KVMSlot.dirty_bmap: 900 * 901 * |<-------- bmap_npages -----------..>| 902 * [1] 903 * start_delta size 904 * |----------------|-------------|------------------|------------| 905 * ^ ^ ^ ^ 906 * | | | | 907 * start bmap_start (start) end 908 * of memslot of memslot 909 * 910 * [1] bmap_npages can be aligned to either 64 pages or the end of slot 911 */ 912 913 assert(bmap_start % BITS_PER_LONG == 0); 914 /* We should never do log_clear before log_sync */ 915 assert(mem->dirty_bmap); 916 if (start_delta || bmap_npages - size / psize) { 917 /* Slow path - we need to manipulate a temp bitmap */ 918 bmap_clear = bitmap_new(bmap_npages); 919 bitmap_copy_with_src_offset(bmap_clear, mem->dirty_bmap, 920 bmap_start, start_delta + size / psize); 921 /* 922 * We need to fill the holes at start because that was not 923 * specified by the caller and we extended the bitmap only for 924 * 64 pages alignment 925 */ 926 bitmap_clear(bmap_clear, 0, start_delta); 927 d.dirty_bitmap = bmap_clear; 928 } else { 929 /* 930 * Fast path - both start and size align well with BITS_PER_LONG 931 * (or the end of memory slot) 932 */ 933 d.dirty_bitmap = mem->dirty_bmap + BIT_WORD(bmap_start); 934 } 935 936 d.first_page = bmap_start; 937 /* It should never overflow. If it happens, say something */ 938 assert(bmap_npages <= UINT32_MAX); 939 d.num_pages = bmap_npages; 940 d.slot = mem->slot | (as_id << 16); 941 942 ret = kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d); 943 if (ret < 0 && ret != -ENOENT) { 944 error_report("%s: KVM_CLEAR_DIRTY_LOG failed, slot=%d, " 945 "start=0x%"PRIx64", size=0x%"PRIx32", errno=%d", 946 __func__, d.slot, (uint64_t)d.first_page, 947 (uint32_t)d.num_pages, ret); 948 } else { 949 ret = 0; 950 trace_kvm_clear_dirty_log(d.slot, d.first_page, d.num_pages); 951 } 952 953 /* 954 * After we have updated the remote dirty bitmap, we update the 955 * cached bitmap as well for the memslot, then if another user 956 * clears the same region we know we shouldn't clear it again on 957 * the remote otherwise it's data loss as well. 958 */ 959 bitmap_clear(mem->dirty_bmap, bmap_start + start_delta, 960 size / psize); 961 /* This handles the NULL case well */ 962 g_free(bmap_clear); 963 return ret; 964 } 965 966 967 /** 968 * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range 969 * 970 * NOTE: this will be a no-op if we haven't enabled manual dirty log 971 * protection in the host kernel because in that case this operation 972 * will be done within log_sync(). 973 * 974 * @kml: the kvm memory listener 975 * @section: the memory range to clear dirty bitmap 976 */ 977 static int kvm_physical_log_clear(KVMMemoryListener *kml, 978 MemoryRegionSection *section) 979 { 980 KVMState *s = kvm_state; 981 uint64_t start, size, offset, count; 982 KVMSlot *mem; 983 int ret = 0, i; 984 985 if (!s->manual_dirty_log_protect) { 986 /* No need to do explicit clear */ 987 return ret; 988 } 989 990 start = section->offset_within_address_space; 991 size = int128_get64(section->size); 992 993 if (!size) { 994 /* Nothing more we can do... */ 995 return ret; 996 } 997 998 kvm_slots_lock(); 999 1000 for (i = 0; i < s->nr_slots; i++) { 1001 mem = &kml->slots[i]; 1002 /* Discard slots that are empty or do not overlap the section */ 1003 if (!mem->memory_size || 1004 mem->start_addr > start + size - 1 || 1005 start > mem->start_addr + mem->memory_size - 1) { 1006 continue; 1007 } 1008 1009 if (start >= mem->start_addr) { 1010 /* The slot starts before section or is aligned to it. */ 1011 offset = start - mem->start_addr; 1012 count = MIN(mem->memory_size - offset, size); 1013 } else { 1014 /* The slot starts after section. */ 1015 offset = 0; 1016 count = MIN(mem->memory_size, size - (mem->start_addr - start)); 1017 } 1018 ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count); 1019 if (ret < 0) { 1020 break; 1021 } 1022 } 1023 1024 kvm_slots_unlock(); 1025 1026 return ret; 1027 } 1028 1029 static void kvm_coalesce_mmio_region(MemoryListener *listener, 1030 MemoryRegionSection *secion, 1031 hwaddr start, hwaddr size) 1032 { 1033 KVMState *s = kvm_state; 1034 1035 if (s->coalesced_mmio) { 1036 struct kvm_coalesced_mmio_zone zone; 1037 1038 zone.addr = start; 1039 zone.size = size; 1040 zone.pad = 0; 1041 1042 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); 1043 } 1044 } 1045 1046 static void kvm_uncoalesce_mmio_region(MemoryListener *listener, 1047 MemoryRegionSection *secion, 1048 hwaddr start, hwaddr size) 1049 { 1050 KVMState *s = kvm_state; 1051 1052 if (s->coalesced_mmio) { 1053 struct kvm_coalesced_mmio_zone zone; 1054 1055 zone.addr = start; 1056 zone.size = size; 1057 zone.pad = 0; 1058 1059 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); 1060 } 1061 } 1062 1063 static void kvm_coalesce_pio_add(MemoryListener *listener, 1064 MemoryRegionSection *section, 1065 hwaddr start, hwaddr size) 1066 { 1067 KVMState *s = kvm_state; 1068 1069 if (s->coalesced_pio) { 1070 struct kvm_coalesced_mmio_zone zone; 1071 1072 zone.addr = start; 1073 zone.size = size; 1074 zone.pio = 1; 1075 1076 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); 1077 } 1078 } 1079 1080 static void kvm_coalesce_pio_del(MemoryListener *listener, 1081 MemoryRegionSection *section, 1082 hwaddr start, hwaddr size) 1083 { 1084 KVMState *s = kvm_state; 1085 1086 if (s->coalesced_pio) { 1087 struct kvm_coalesced_mmio_zone zone; 1088 1089 zone.addr = start; 1090 zone.size = size; 1091 zone.pio = 1; 1092 1093 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); 1094 } 1095 } 1096 1097 int kvm_check_extension(KVMState *s, unsigned int extension) 1098 { 1099 int ret; 1100 1101 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension); 1102 if (ret < 0) { 1103 ret = 0; 1104 } 1105 1106 return ret; 1107 } 1108 1109 int kvm_vm_check_extension(KVMState *s, unsigned int extension) 1110 { 1111 int ret; 1112 1113 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension); 1114 if (ret < 0) { 1115 /* VM wide version not implemented, use global one instead */ 1116 ret = kvm_check_extension(s, extension); 1117 } 1118 1119 return ret; 1120 } 1121 1122 /* 1123 * We track the poisoned pages to be able to: 1124 * - replace them on VM reset 1125 * - block a migration for a VM with a poisoned page 1126 */ 1127 typedef struct HWPoisonPage { 1128 ram_addr_t ram_addr; 1129 QLIST_ENTRY(HWPoisonPage) list; 1130 } HWPoisonPage; 1131 1132 static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list = 1133 QLIST_HEAD_INITIALIZER(hwpoison_page_list); 1134 1135 static void kvm_unpoison_all(void *param) 1136 { 1137 HWPoisonPage *page, *next_page; 1138 1139 QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) { 1140 QLIST_REMOVE(page, list); 1141 qemu_ram_remap(page->ram_addr, TARGET_PAGE_SIZE); 1142 g_free(page); 1143 } 1144 } 1145 1146 void kvm_hwpoison_page_add(ram_addr_t ram_addr) 1147 { 1148 HWPoisonPage *page; 1149 1150 QLIST_FOREACH(page, &hwpoison_page_list, list) { 1151 if (page->ram_addr == ram_addr) { 1152 return; 1153 } 1154 } 1155 page = g_new(HWPoisonPage, 1); 1156 page->ram_addr = ram_addr; 1157 QLIST_INSERT_HEAD(&hwpoison_page_list, page, list); 1158 } 1159 1160 bool kvm_hwpoisoned_mem(void) 1161 { 1162 return !QLIST_EMPTY(&hwpoison_page_list); 1163 } 1164 1165 static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size) 1166 { 1167 #if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN 1168 /* The kernel expects ioeventfd values in HOST_BIG_ENDIAN 1169 * endianness, but the memory core hands them in target endianness. 1170 * For example, PPC is always treated as big-endian even if running 1171 * on KVM and on PPC64LE. Correct here. 1172 */ 1173 switch (size) { 1174 case 2: 1175 val = bswap16(val); 1176 break; 1177 case 4: 1178 val = bswap32(val); 1179 break; 1180 } 1181 #endif 1182 return val; 1183 } 1184 1185 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val, 1186 bool assign, uint32_t size, bool datamatch) 1187 { 1188 int ret; 1189 struct kvm_ioeventfd iofd = { 1190 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 1191 .addr = addr, 1192 .len = size, 1193 .flags = 0, 1194 .fd = fd, 1195 }; 1196 1197 trace_kvm_set_ioeventfd_mmio(fd, (uint64_t)addr, val, assign, size, 1198 datamatch); 1199 if (!kvm_enabled()) { 1200 return -ENOSYS; 1201 } 1202 1203 if (datamatch) { 1204 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 1205 } 1206 if (!assign) { 1207 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 1208 } 1209 1210 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd); 1211 1212 if (ret < 0) { 1213 return -errno; 1214 } 1215 1216 return 0; 1217 } 1218 1219 static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val, 1220 bool assign, uint32_t size, bool datamatch) 1221 { 1222 struct kvm_ioeventfd kick = { 1223 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 1224 .addr = addr, 1225 .flags = KVM_IOEVENTFD_FLAG_PIO, 1226 .len = size, 1227 .fd = fd, 1228 }; 1229 int r; 1230 trace_kvm_set_ioeventfd_pio(fd, addr, val, assign, size, datamatch); 1231 if (!kvm_enabled()) { 1232 return -ENOSYS; 1233 } 1234 if (datamatch) { 1235 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 1236 } 1237 if (!assign) { 1238 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 1239 } 1240 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick); 1241 if (r < 0) { 1242 return r; 1243 } 1244 return 0; 1245 } 1246 1247 1248 static const KVMCapabilityInfo * 1249 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list) 1250 { 1251 while (list->name) { 1252 if (!kvm_check_extension(s, list->value)) { 1253 return list; 1254 } 1255 list++; 1256 } 1257 return NULL; 1258 } 1259 1260 void kvm_set_max_memslot_size(hwaddr max_slot_size) 1261 { 1262 g_assert( 1263 ROUND_UP(max_slot_size, qemu_real_host_page_size()) == max_slot_size 1264 ); 1265 kvm_max_slot_size = max_slot_size; 1266 } 1267 1268 /* Called with KVMMemoryListener.slots_lock held */ 1269 static void kvm_set_phys_mem(KVMMemoryListener *kml, 1270 MemoryRegionSection *section, bool add) 1271 { 1272 KVMSlot *mem; 1273 int err; 1274 MemoryRegion *mr = section->mr; 1275 bool writable = !mr->readonly && !mr->rom_device; 1276 hwaddr start_addr, size, slot_size, mr_offset; 1277 ram_addr_t ram_start_offset; 1278 void *ram; 1279 1280 if (!memory_region_is_ram(mr)) { 1281 if (writable || !kvm_readonly_mem_allowed) { 1282 return; 1283 } else if (!mr->romd_mode) { 1284 /* If the memory device is not in romd_mode, then we actually want 1285 * to remove the kvm memory slot so all accesses will trap. */ 1286 add = false; 1287 } 1288 } 1289 1290 size = kvm_align_section(section, &start_addr); 1291 if (!size) { 1292 return; 1293 } 1294 1295 /* The offset of the kvmslot within the memory region */ 1296 mr_offset = section->offset_within_region + start_addr - 1297 section->offset_within_address_space; 1298 1299 /* use aligned delta to align the ram address and offset */ 1300 ram = memory_region_get_ram_ptr(mr) + mr_offset; 1301 ram_start_offset = memory_region_get_ram_addr(mr) + mr_offset; 1302 1303 if (!add) { 1304 do { 1305 slot_size = MIN(kvm_max_slot_size, size); 1306 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 1307 if (!mem) { 1308 return; 1309 } 1310 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1311 /* 1312 * NOTE: We should be aware of the fact that here we're only 1313 * doing a best effort to sync dirty bits. No matter whether 1314 * we're using dirty log or dirty ring, we ignored two facts: 1315 * 1316 * (1) dirty bits can reside in hardware buffers (PML) 1317 * 1318 * (2) after we collected dirty bits here, pages can be dirtied 1319 * again before we do the final KVM_SET_USER_MEMORY_REGION to 1320 * remove the slot. 1321 * 1322 * Not easy. Let's cross the fingers until it's fixed. 1323 */ 1324 if (kvm_state->kvm_dirty_ring_size) { 1325 kvm_dirty_ring_reap_locked(kvm_state, NULL); 1326 if (kvm_state->kvm_dirty_ring_with_bitmap) { 1327 kvm_slot_sync_dirty_pages(mem); 1328 kvm_slot_get_dirty_log(kvm_state, mem); 1329 } 1330 } else { 1331 kvm_slot_get_dirty_log(kvm_state, mem); 1332 } 1333 kvm_slot_sync_dirty_pages(mem); 1334 } 1335 1336 /* unregister the slot */ 1337 g_free(mem->dirty_bmap); 1338 mem->dirty_bmap = NULL; 1339 mem->memory_size = 0; 1340 mem->flags = 0; 1341 err = kvm_set_user_memory_region(kml, mem, false); 1342 if (err) { 1343 fprintf(stderr, "%s: error unregistering slot: %s\n", 1344 __func__, strerror(-err)); 1345 abort(); 1346 } 1347 start_addr += slot_size; 1348 size -= slot_size; 1349 kml->nr_used_slots--; 1350 } while (size); 1351 return; 1352 } 1353 1354 /* register the new slot */ 1355 do { 1356 slot_size = MIN(kvm_max_slot_size, size); 1357 mem = kvm_alloc_slot(kml); 1358 mem->as_id = kml->as_id; 1359 mem->memory_size = slot_size; 1360 mem->start_addr = start_addr; 1361 mem->ram_start_offset = ram_start_offset; 1362 mem->ram = ram; 1363 mem->flags = kvm_mem_flags(mr); 1364 kvm_slot_init_dirty_bitmap(mem); 1365 err = kvm_set_user_memory_region(kml, mem, true); 1366 if (err) { 1367 fprintf(stderr, "%s: error registering slot: %s\n", __func__, 1368 strerror(-err)); 1369 abort(); 1370 } 1371 start_addr += slot_size; 1372 ram_start_offset += slot_size; 1373 ram += slot_size; 1374 size -= slot_size; 1375 kml->nr_used_slots++; 1376 } while (size); 1377 } 1378 1379 static void *kvm_dirty_ring_reaper_thread(void *data) 1380 { 1381 KVMState *s = data; 1382 struct KVMDirtyRingReaper *r = &s->reaper; 1383 1384 rcu_register_thread(); 1385 1386 trace_kvm_dirty_ring_reaper("init"); 1387 1388 while (true) { 1389 r->reaper_state = KVM_DIRTY_RING_REAPER_WAIT; 1390 trace_kvm_dirty_ring_reaper("wait"); 1391 /* 1392 * TODO: provide a smarter timeout rather than a constant? 1393 */ 1394 sleep(1); 1395 1396 /* keep sleeping so that dirtylimit not be interfered by reaper */ 1397 if (dirtylimit_in_service()) { 1398 continue; 1399 } 1400 1401 trace_kvm_dirty_ring_reaper("wakeup"); 1402 r->reaper_state = KVM_DIRTY_RING_REAPER_REAPING; 1403 1404 bql_lock(); 1405 kvm_dirty_ring_reap(s, NULL); 1406 bql_unlock(); 1407 1408 r->reaper_iteration++; 1409 } 1410 1411 trace_kvm_dirty_ring_reaper("exit"); 1412 1413 rcu_unregister_thread(); 1414 1415 return NULL; 1416 } 1417 1418 static void kvm_dirty_ring_reaper_init(KVMState *s) 1419 { 1420 struct KVMDirtyRingReaper *r = &s->reaper; 1421 1422 qemu_thread_create(&r->reaper_thr, "kvm-reaper", 1423 kvm_dirty_ring_reaper_thread, 1424 s, QEMU_THREAD_JOINABLE); 1425 } 1426 1427 static int kvm_dirty_ring_init(KVMState *s) 1428 { 1429 uint32_t ring_size = s->kvm_dirty_ring_size; 1430 uint64_t ring_bytes = ring_size * sizeof(struct kvm_dirty_gfn); 1431 unsigned int capability = KVM_CAP_DIRTY_LOG_RING; 1432 int ret; 1433 1434 s->kvm_dirty_ring_size = 0; 1435 s->kvm_dirty_ring_bytes = 0; 1436 1437 /* Bail if the dirty ring size isn't specified */ 1438 if (!ring_size) { 1439 return 0; 1440 } 1441 1442 /* 1443 * Read the max supported pages. Fall back to dirty logging mode 1444 * if the dirty ring isn't supported. 1445 */ 1446 ret = kvm_vm_check_extension(s, capability); 1447 if (ret <= 0) { 1448 capability = KVM_CAP_DIRTY_LOG_RING_ACQ_REL; 1449 ret = kvm_vm_check_extension(s, capability); 1450 } 1451 1452 if (ret <= 0) { 1453 warn_report("KVM dirty ring not available, using bitmap method"); 1454 return 0; 1455 } 1456 1457 if (ring_bytes > ret) { 1458 error_report("KVM dirty ring size %" PRIu32 " too big " 1459 "(maximum is %ld). Please use a smaller value.", 1460 ring_size, (long)ret / sizeof(struct kvm_dirty_gfn)); 1461 return -EINVAL; 1462 } 1463 1464 ret = kvm_vm_enable_cap(s, capability, 0, ring_bytes); 1465 if (ret) { 1466 error_report("Enabling of KVM dirty ring failed: %s. " 1467 "Suggested minimum value is 1024.", strerror(-ret)); 1468 return -EIO; 1469 } 1470 1471 /* Enable the backup bitmap if it is supported */ 1472 ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP); 1473 if (ret > 0) { 1474 ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP, 0); 1475 if (ret) { 1476 error_report("Enabling of KVM dirty ring's backup bitmap failed: " 1477 "%s. ", strerror(-ret)); 1478 return -EIO; 1479 } 1480 1481 s->kvm_dirty_ring_with_bitmap = true; 1482 } 1483 1484 s->kvm_dirty_ring_size = ring_size; 1485 s->kvm_dirty_ring_bytes = ring_bytes; 1486 1487 return 0; 1488 } 1489 1490 static void kvm_region_add(MemoryListener *listener, 1491 MemoryRegionSection *section) 1492 { 1493 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1494 KVMMemoryUpdate *update; 1495 1496 update = g_new0(KVMMemoryUpdate, 1); 1497 update->section = *section; 1498 1499 QSIMPLEQ_INSERT_TAIL(&kml->transaction_add, update, next); 1500 } 1501 1502 static void kvm_region_del(MemoryListener *listener, 1503 MemoryRegionSection *section) 1504 { 1505 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1506 KVMMemoryUpdate *update; 1507 1508 update = g_new0(KVMMemoryUpdate, 1); 1509 update->section = *section; 1510 1511 QSIMPLEQ_INSERT_TAIL(&kml->transaction_del, update, next); 1512 } 1513 1514 static void kvm_region_commit(MemoryListener *listener) 1515 { 1516 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, 1517 listener); 1518 KVMMemoryUpdate *u1, *u2; 1519 bool need_inhibit = false; 1520 1521 if (QSIMPLEQ_EMPTY(&kml->transaction_add) && 1522 QSIMPLEQ_EMPTY(&kml->transaction_del)) { 1523 return; 1524 } 1525 1526 /* 1527 * We have to be careful when regions to add overlap with ranges to remove. 1528 * We have to simulate atomic KVM memslot updates by making sure no ioctl() 1529 * is currently active. 1530 * 1531 * The lists are order by addresses, so it's easy to find overlaps. 1532 */ 1533 u1 = QSIMPLEQ_FIRST(&kml->transaction_del); 1534 u2 = QSIMPLEQ_FIRST(&kml->transaction_add); 1535 while (u1 && u2) { 1536 Range r1, r2; 1537 1538 range_init_nofail(&r1, u1->section.offset_within_address_space, 1539 int128_get64(u1->section.size)); 1540 range_init_nofail(&r2, u2->section.offset_within_address_space, 1541 int128_get64(u2->section.size)); 1542 1543 if (range_overlaps_range(&r1, &r2)) { 1544 need_inhibit = true; 1545 break; 1546 } 1547 if (range_lob(&r1) < range_lob(&r2)) { 1548 u1 = QSIMPLEQ_NEXT(u1, next); 1549 } else { 1550 u2 = QSIMPLEQ_NEXT(u2, next); 1551 } 1552 } 1553 1554 kvm_slots_lock(); 1555 if (need_inhibit) { 1556 accel_ioctl_inhibit_begin(); 1557 } 1558 1559 /* Remove all memslots before adding the new ones. */ 1560 while (!QSIMPLEQ_EMPTY(&kml->transaction_del)) { 1561 u1 = QSIMPLEQ_FIRST(&kml->transaction_del); 1562 QSIMPLEQ_REMOVE_HEAD(&kml->transaction_del, next); 1563 1564 kvm_set_phys_mem(kml, &u1->section, false); 1565 memory_region_unref(u1->section.mr); 1566 1567 g_free(u1); 1568 } 1569 while (!QSIMPLEQ_EMPTY(&kml->transaction_add)) { 1570 u1 = QSIMPLEQ_FIRST(&kml->transaction_add); 1571 QSIMPLEQ_REMOVE_HEAD(&kml->transaction_add, next); 1572 1573 memory_region_ref(u1->section.mr); 1574 kvm_set_phys_mem(kml, &u1->section, true); 1575 1576 g_free(u1); 1577 } 1578 1579 if (need_inhibit) { 1580 accel_ioctl_inhibit_end(); 1581 } 1582 kvm_slots_unlock(); 1583 } 1584 1585 static void kvm_log_sync(MemoryListener *listener, 1586 MemoryRegionSection *section) 1587 { 1588 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1589 1590 kvm_slots_lock(); 1591 kvm_physical_sync_dirty_bitmap(kml, section); 1592 kvm_slots_unlock(); 1593 } 1594 1595 static void kvm_log_sync_global(MemoryListener *l, bool last_stage) 1596 { 1597 KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener); 1598 KVMState *s = kvm_state; 1599 KVMSlot *mem; 1600 int i; 1601 1602 /* Flush all kernel dirty addresses into KVMSlot dirty bitmap */ 1603 kvm_dirty_ring_flush(); 1604 1605 /* 1606 * TODO: make this faster when nr_slots is big while there are 1607 * only a few used slots (small VMs). 1608 */ 1609 kvm_slots_lock(); 1610 for (i = 0; i < s->nr_slots; i++) { 1611 mem = &kml->slots[i]; 1612 if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1613 kvm_slot_sync_dirty_pages(mem); 1614 1615 if (s->kvm_dirty_ring_with_bitmap && last_stage && 1616 kvm_slot_get_dirty_log(s, mem)) { 1617 kvm_slot_sync_dirty_pages(mem); 1618 } 1619 1620 /* 1621 * This is not needed by KVM_GET_DIRTY_LOG because the 1622 * ioctl will unconditionally overwrite the whole region. 1623 * However kvm dirty ring has no such side effect. 1624 */ 1625 kvm_slot_reset_dirty_pages(mem); 1626 } 1627 } 1628 kvm_slots_unlock(); 1629 } 1630 1631 static void kvm_log_clear(MemoryListener *listener, 1632 MemoryRegionSection *section) 1633 { 1634 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1635 int r; 1636 1637 r = kvm_physical_log_clear(kml, section); 1638 if (r < 0) { 1639 error_report_once("%s: kvm log clear failed: mr=%s " 1640 "offset=%"HWADDR_PRIx" size=%"PRIx64, __func__, 1641 section->mr->name, section->offset_within_region, 1642 int128_get64(section->size)); 1643 abort(); 1644 } 1645 } 1646 1647 static void kvm_mem_ioeventfd_add(MemoryListener *listener, 1648 MemoryRegionSection *section, 1649 bool match_data, uint64_t data, 1650 EventNotifier *e) 1651 { 1652 int fd = event_notifier_get_fd(e); 1653 int r; 1654 1655 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 1656 data, true, int128_get64(section->size), 1657 match_data); 1658 if (r < 0) { 1659 fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", 1660 __func__, strerror(-r), -r); 1661 abort(); 1662 } 1663 } 1664 1665 static void kvm_mem_ioeventfd_del(MemoryListener *listener, 1666 MemoryRegionSection *section, 1667 bool match_data, uint64_t data, 1668 EventNotifier *e) 1669 { 1670 int fd = event_notifier_get_fd(e); 1671 int r; 1672 1673 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 1674 data, false, int128_get64(section->size), 1675 match_data); 1676 if (r < 0) { 1677 fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", 1678 __func__, strerror(-r), -r); 1679 abort(); 1680 } 1681 } 1682 1683 static void kvm_io_ioeventfd_add(MemoryListener *listener, 1684 MemoryRegionSection *section, 1685 bool match_data, uint64_t data, 1686 EventNotifier *e) 1687 { 1688 int fd = event_notifier_get_fd(e); 1689 int r; 1690 1691 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 1692 data, true, int128_get64(section->size), 1693 match_data); 1694 if (r < 0) { 1695 fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", 1696 __func__, strerror(-r), -r); 1697 abort(); 1698 } 1699 } 1700 1701 static void kvm_io_ioeventfd_del(MemoryListener *listener, 1702 MemoryRegionSection *section, 1703 bool match_data, uint64_t data, 1704 EventNotifier *e) 1705 1706 { 1707 int fd = event_notifier_get_fd(e); 1708 int r; 1709 1710 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 1711 data, false, int128_get64(section->size), 1712 match_data); 1713 if (r < 0) { 1714 fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", 1715 __func__, strerror(-r), -r); 1716 abort(); 1717 } 1718 } 1719 1720 void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, 1721 AddressSpace *as, int as_id, const char *name) 1722 { 1723 int i; 1724 1725 kml->slots = g_new0(KVMSlot, s->nr_slots); 1726 kml->as_id = as_id; 1727 1728 for (i = 0; i < s->nr_slots; i++) { 1729 kml->slots[i].slot = i; 1730 } 1731 1732 QSIMPLEQ_INIT(&kml->transaction_add); 1733 QSIMPLEQ_INIT(&kml->transaction_del); 1734 1735 kml->listener.region_add = kvm_region_add; 1736 kml->listener.region_del = kvm_region_del; 1737 kml->listener.commit = kvm_region_commit; 1738 kml->listener.log_start = kvm_log_start; 1739 kml->listener.log_stop = kvm_log_stop; 1740 kml->listener.priority = MEMORY_LISTENER_PRIORITY_ACCEL; 1741 kml->listener.name = name; 1742 1743 if (s->kvm_dirty_ring_size) { 1744 kml->listener.log_sync_global = kvm_log_sync_global; 1745 } else { 1746 kml->listener.log_sync = kvm_log_sync; 1747 kml->listener.log_clear = kvm_log_clear; 1748 } 1749 1750 memory_listener_register(&kml->listener, as); 1751 1752 for (i = 0; i < s->nr_as; ++i) { 1753 if (!s->as[i].as) { 1754 s->as[i].as = as; 1755 s->as[i].ml = kml; 1756 break; 1757 } 1758 } 1759 } 1760 1761 static MemoryListener kvm_io_listener = { 1762 .name = "kvm-io", 1763 .coalesced_io_add = kvm_coalesce_pio_add, 1764 .coalesced_io_del = kvm_coalesce_pio_del, 1765 .eventfd_add = kvm_io_ioeventfd_add, 1766 .eventfd_del = kvm_io_ioeventfd_del, 1767 .priority = MEMORY_LISTENER_PRIORITY_DEV_BACKEND, 1768 }; 1769 1770 int kvm_set_irq(KVMState *s, int irq, int level) 1771 { 1772 struct kvm_irq_level event; 1773 int ret; 1774 1775 assert(kvm_async_interrupts_enabled()); 1776 1777 event.level = level; 1778 event.irq = irq; 1779 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event); 1780 if (ret < 0) { 1781 perror("kvm_set_irq"); 1782 abort(); 1783 } 1784 1785 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status; 1786 } 1787 1788 #ifdef KVM_CAP_IRQ_ROUTING 1789 typedef struct KVMMSIRoute { 1790 struct kvm_irq_routing_entry kroute; 1791 QTAILQ_ENTRY(KVMMSIRoute) entry; 1792 } KVMMSIRoute; 1793 1794 static void set_gsi(KVMState *s, unsigned int gsi) 1795 { 1796 set_bit(gsi, s->used_gsi_bitmap); 1797 } 1798 1799 static void clear_gsi(KVMState *s, unsigned int gsi) 1800 { 1801 clear_bit(gsi, s->used_gsi_bitmap); 1802 } 1803 1804 void kvm_init_irq_routing(KVMState *s) 1805 { 1806 int gsi_count; 1807 1808 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1; 1809 if (gsi_count > 0) { 1810 /* Round up so we can search ints using ffs */ 1811 s->used_gsi_bitmap = bitmap_new(gsi_count); 1812 s->gsi_count = gsi_count; 1813 } 1814 1815 s->irq_routes = g_malloc0(sizeof(*s->irq_routes)); 1816 s->nr_allocated_irq_routes = 0; 1817 1818 kvm_arch_init_irq_routing(s); 1819 } 1820 1821 void kvm_irqchip_commit_routes(KVMState *s) 1822 { 1823 int ret; 1824 1825 if (kvm_gsi_direct_mapping()) { 1826 return; 1827 } 1828 1829 if (!kvm_gsi_routing_enabled()) { 1830 return; 1831 } 1832 1833 s->irq_routes->flags = 0; 1834 trace_kvm_irqchip_commit_routes(); 1835 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes); 1836 assert(ret == 0); 1837 } 1838 1839 static void kvm_add_routing_entry(KVMState *s, 1840 struct kvm_irq_routing_entry *entry) 1841 { 1842 struct kvm_irq_routing_entry *new; 1843 int n, size; 1844 1845 if (s->irq_routes->nr == s->nr_allocated_irq_routes) { 1846 n = s->nr_allocated_irq_routes * 2; 1847 if (n < 64) { 1848 n = 64; 1849 } 1850 size = sizeof(struct kvm_irq_routing); 1851 size += n * sizeof(*new); 1852 s->irq_routes = g_realloc(s->irq_routes, size); 1853 s->nr_allocated_irq_routes = n; 1854 } 1855 n = s->irq_routes->nr++; 1856 new = &s->irq_routes->entries[n]; 1857 1858 *new = *entry; 1859 1860 set_gsi(s, entry->gsi); 1861 } 1862 1863 static int kvm_update_routing_entry(KVMState *s, 1864 struct kvm_irq_routing_entry *new_entry) 1865 { 1866 struct kvm_irq_routing_entry *entry; 1867 int n; 1868 1869 for (n = 0; n < s->irq_routes->nr; n++) { 1870 entry = &s->irq_routes->entries[n]; 1871 if (entry->gsi != new_entry->gsi) { 1872 continue; 1873 } 1874 1875 if(!memcmp(entry, new_entry, sizeof *entry)) { 1876 return 0; 1877 } 1878 1879 *entry = *new_entry; 1880 1881 return 0; 1882 } 1883 1884 return -ESRCH; 1885 } 1886 1887 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin) 1888 { 1889 struct kvm_irq_routing_entry e = {}; 1890 1891 assert(pin < s->gsi_count); 1892 1893 e.gsi = irq; 1894 e.type = KVM_IRQ_ROUTING_IRQCHIP; 1895 e.flags = 0; 1896 e.u.irqchip.irqchip = irqchip; 1897 e.u.irqchip.pin = pin; 1898 kvm_add_routing_entry(s, &e); 1899 } 1900 1901 void kvm_irqchip_release_virq(KVMState *s, int virq) 1902 { 1903 struct kvm_irq_routing_entry *e; 1904 int i; 1905 1906 if (kvm_gsi_direct_mapping()) { 1907 return; 1908 } 1909 1910 for (i = 0; i < s->irq_routes->nr; i++) { 1911 e = &s->irq_routes->entries[i]; 1912 if (e->gsi == virq) { 1913 s->irq_routes->nr--; 1914 *e = s->irq_routes->entries[s->irq_routes->nr]; 1915 } 1916 } 1917 clear_gsi(s, virq); 1918 kvm_arch_release_virq_post(virq); 1919 trace_kvm_irqchip_release_virq(virq); 1920 } 1921 1922 void kvm_irqchip_add_change_notifier(Notifier *n) 1923 { 1924 notifier_list_add(&kvm_irqchip_change_notifiers, n); 1925 } 1926 1927 void kvm_irqchip_remove_change_notifier(Notifier *n) 1928 { 1929 notifier_remove(n); 1930 } 1931 1932 void kvm_irqchip_change_notify(void) 1933 { 1934 notifier_list_notify(&kvm_irqchip_change_notifiers, NULL); 1935 } 1936 1937 static int kvm_irqchip_get_virq(KVMState *s) 1938 { 1939 int next_virq; 1940 1941 /* Return the lowest unused GSI in the bitmap */ 1942 next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count); 1943 if (next_virq >= s->gsi_count) { 1944 return -ENOSPC; 1945 } else { 1946 return next_virq; 1947 } 1948 } 1949 1950 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 1951 { 1952 struct kvm_msi msi; 1953 1954 msi.address_lo = (uint32_t)msg.address; 1955 msi.address_hi = msg.address >> 32; 1956 msi.data = le32_to_cpu(msg.data); 1957 msi.flags = 0; 1958 memset(msi.pad, 0, sizeof(msi.pad)); 1959 1960 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi); 1961 } 1962 1963 int kvm_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *dev) 1964 { 1965 struct kvm_irq_routing_entry kroute = {}; 1966 int virq; 1967 KVMState *s = c->s; 1968 MSIMessage msg = {0, 0}; 1969 1970 if (pci_available && dev) { 1971 msg = pci_get_msi_message(dev, vector); 1972 } 1973 1974 if (kvm_gsi_direct_mapping()) { 1975 return kvm_arch_msi_data_to_gsi(msg.data); 1976 } 1977 1978 if (!kvm_gsi_routing_enabled()) { 1979 return -ENOSYS; 1980 } 1981 1982 virq = kvm_irqchip_get_virq(s); 1983 if (virq < 0) { 1984 return virq; 1985 } 1986 1987 kroute.gsi = virq; 1988 kroute.type = KVM_IRQ_ROUTING_MSI; 1989 kroute.flags = 0; 1990 kroute.u.msi.address_lo = (uint32_t)msg.address; 1991 kroute.u.msi.address_hi = msg.address >> 32; 1992 kroute.u.msi.data = le32_to_cpu(msg.data); 1993 if (pci_available && kvm_msi_devid_required()) { 1994 kroute.flags = KVM_MSI_VALID_DEVID; 1995 kroute.u.msi.devid = pci_requester_id(dev); 1996 } 1997 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 1998 kvm_irqchip_release_virq(s, virq); 1999 return -EINVAL; 2000 } 2001 2002 if (s->irq_routes->nr < s->gsi_count) { 2003 trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A", 2004 vector, virq); 2005 2006 kvm_add_routing_entry(s, &kroute); 2007 kvm_arch_add_msi_route_post(&kroute, vector, dev); 2008 c->changes++; 2009 } else { 2010 kvm_irqchip_release_virq(s, virq); 2011 return -ENOSPC; 2012 } 2013 2014 return virq; 2015 } 2016 2017 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, 2018 PCIDevice *dev) 2019 { 2020 struct kvm_irq_routing_entry kroute = {}; 2021 2022 if (kvm_gsi_direct_mapping()) { 2023 return 0; 2024 } 2025 2026 if (!kvm_irqchip_in_kernel()) { 2027 return -ENOSYS; 2028 } 2029 2030 kroute.gsi = virq; 2031 kroute.type = KVM_IRQ_ROUTING_MSI; 2032 kroute.flags = 0; 2033 kroute.u.msi.address_lo = (uint32_t)msg.address; 2034 kroute.u.msi.address_hi = msg.address >> 32; 2035 kroute.u.msi.data = le32_to_cpu(msg.data); 2036 if (pci_available && kvm_msi_devid_required()) { 2037 kroute.flags = KVM_MSI_VALID_DEVID; 2038 kroute.u.msi.devid = pci_requester_id(dev); 2039 } 2040 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 2041 return -EINVAL; 2042 } 2043 2044 trace_kvm_irqchip_update_msi_route(virq); 2045 2046 return kvm_update_routing_entry(s, &kroute); 2047 } 2048 2049 static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, 2050 EventNotifier *resample, int virq, 2051 bool assign) 2052 { 2053 int fd = event_notifier_get_fd(event); 2054 int rfd = resample ? event_notifier_get_fd(resample) : -1; 2055 2056 struct kvm_irqfd irqfd = { 2057 .fd = fd, 2058 .gsi = virq, 2059 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN, 2060 }; 2061 2062 if (rfd != -1) { 2063 assert(assign); 2064 if (kvm_irqchip_is_split()) { 2065 /* 2066 * When the slow irqchip (e.g. IOAPIC) is in the 2067 * userspace, KVM kernel resamplefd will not work because 2068 * the EOI of the interrupt will be delivered to userspace 2069 * instead, so the KVM kernel resamplefd kick will be 2070 * skipped. The userspace here mimics what the kernel 2071 * provides with resamplefd, remember the resamplefd and 2072 * kick it when we receive EOI of this IRQ. 2073 * 2074 * This is hackery because IOAPIC is mostly bypassed 2075 * (except EOI broadcasts) when irqfd is used. However 2076 * this can bring much performance back for split irqchip 2077 * with INTx IRQs (for VFIO, this gives 93% perf of the 2078 * full fast path, which is 46% perf boost comparing to 2079 * the INTx slow path). 2080 */ 2081 kvm_resample_fd_insert(virq, resample); 2082 } else { 2083 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE; 2084 irqfd.resamplefd = rfd; 2085 } 2086 } else if (!assign) { 2087 if (kvm_irqchip_is_split()) { 2088 kvm_resample_fd_remove(virq); 2089 } 2090 } 2091 2092 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd); 2093 } 2094 2095 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) 2096 { 2097 struct kvm_irq_routing_entry kroute = {}; 2098 int virq; 2099 2100 if (!kvm_gsi_routing_enabled()) { 2101 return -ENOSYS; 2102 } 2103 2104 virq = kvm_irqchip_get_virq(s); 2105 if (virq < 0) { 2106 return virq; 2107 } 2108 2109 kroute.gsi = virq; 2110 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER; 2111 kroute.flags = 0; 2112 kroute.u.adapter.summary_addr = adapter->summary_addr; 2113 kroute.u.adapter.ind_addr = adapter->ind_addr; 2114 kroute.u.adapter.summary_offset = adapter->summary_offset; 2115 kroute.u.adapter.ind_offset = adapter->ind_offset; 2116 kroute.u.adapter.adapter_id = adapter->adapter_id; 2117 2118 kvm_add_routing_entry(s, &kroute); 2119 2120 return virq; 2121 } 2122 2123 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) 2124 { 2125 struct kvm_irq_routing_entry kroute = {}; 2126 int virq; 2127 2128 if (!kvm_gsi_routing_enabled()) { 2129 return -ENOSYS; 2130 } 2131 if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) { 2132 return -ENOSYS; 2133 } 2134 virq = kvm_irqchip_get_virq(s); 2135 if (virq < 0) { 2136 return virq; 2137 } 2138 2139 kroute.gsi = virq; 2140 kroute.type = KVM_IRQ_ROUTING_HV_SINT; 2141 kroute.flags = 0; 2142 kroute.u.hv_sint.vcpu = vcpu; 2143 kroute.u.hv_sint.sint = sint; 2144 2145 kvm_add_routing_entry(s, &kroute); 2146 kvm_irqchip_commit_routes(s); 2147 2148 return virq; 2149 } 2150 2151 #else /* !KVM_CAP_IRQ_ROUTING */ 2152 2153 void kvm_init_irq_routing(KVMState *s) 2154 { 2155 } 2156 2157 void kvm_irqchip_release_virq(KVMState *s, int virq) 2158 { 2159 } 2160 2161 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 2162 { 2163 abort(); 2164 } 2165 2166 int kvm_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *dev) 2167 { 2168 return -ENOSYS; 2169 } 2170 2171 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) 2172 { 2173 return -ENOSYS; 2174 } 2175 2176 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) 2177 { 2178 return -ENOSYS; 2179 } 2180 2181 static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, 2182 EventNotifier *resample, int virq, 2183 bool assign) 2184 { 2185 abort(); 2186 } 2187 2188 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg) 2189 { 2190 return -ENOSYS; 2191 } 2192 #endif /* !KVM_CAP_IRQ_ROUTING */ 2193 2194 int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 2195 EventNotifier *rn, int virq) 2196 { 2197 return kvm_irqchip_assign_irqfd(s, n, rn, virq, true); 2198 } 2199 2200 int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 2201 int virq) 2202 { 2203 return kvm_irqchip_assign_irqfd(s, n, NULL, virq, false); 2204 } 2205 2206 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, 2207 EventNotifier *rn, qemu_irq irq) 2208 { 2209 gpointer key, gsi; 2210 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 2211 2212 if (!found) { 2213 return -ENXIO; 2214 } 2215 return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi)); 2216 } 2217 2218 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, 2219 qemu_irq irq) 2220 { 2221 gpointer key, gsi; 2222 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 2223 2224 if (!found) { 2225 return -ENXIO; 2226 } 2227 return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi)); 2228 } 2229 2230 void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi) 2231 { 2232 g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi)); 2233 } 2234 2235 static void kvm_irqchip_create(KVMState *s) 2236 { 2237 int ret; 2238 2239 assert(s->kernel_irqchip_split != ON_OFF_AUTO_AUTO); 2240 if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) { 2241 ; 2242 } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) { 2243 ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0); 2244 if (ret < 0) { 2245 fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret)); 2246 exit(1); 2247 } 2248 } else { 2249 return; 2250 } 2251 2252 if (kvm_check_extension(s, KVM_CAP_IRQFD) <= 0) { 2253 fprintf(stderr, "kvm: irqfd not implemented\n"); 2254 exit(1); 2255 } 2256 2257 /* First probe and see if there's a arch-specific hook to create the 2258 * in-kernel irqchip for us */ 2259 ret = kvm_arch_irqchip_create(s); 2260 if (ret == 0) { 2261 if (s->kernel_irqchip_split == ON_OFF_AUTO_ON) { 2262 error_report("Split IRQ chip mode not supported."); 2263 exit(1); 2264 } else { 2265 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP); 2266 } 2267 } 2268 if (ret < 0) { 2269 fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret)); 2270 exit(1); 2271 } 2272 2273 kvm_kernel_irqchip = true; 2274 /* If we have an in-kernel IRQ chip then we must have asynchronous 2275 * interrupt delivery (though the reverse is not necessarily true) 2276 */ 2277 kvm_async_interrupts_allowed = true; 2278 kvm_halt_in_kernel_allowed = true; 2279 2280 kvm_init_irq_routing(s); 2281 2282 s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal); 2283 } 2284 2285 /* Find number of supported CPUs using the recommended 2286 * procedure from the kernel API documentation to cope with 2287 * older kernels that may be missing capabilities. 2288 */ 2289 static int kvm_recommended_vcpus(KVMState *s) 2290 { 2291 int ret = kvm_vm_check_extension(s, KVM_CAP_NR_VCPUS); 2292 return (ret) ? ret : 4; 2293 } 2294 2295 static int kvm_max_vcpus(KVMState *s) 2296 { 2297 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS); 2298 return (ret) ? ret : kvm_recommended_vcpus(s); 2299 } 2300 2301 static int kvm_max_vcpu_id(KVMState *s) 2302 { 2303 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID); 2304 return (ret) ? ret : kvm_max_vcpus(s); 2305 } 2306 2307 bool kvm_vcpu_id_is_valid(int vcpu_id) 2308 { 2309 KVMState *s = KVM_STATE(current_accel()); 2310 return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s); 2311 } 2312 2313 bool kvm_dirty_ring_enabled(void) 2314 { 2315 return kvm_state->kvm_dirty_ring_size ? true : false; 2316 } 2317 2318 static void query_stats_cb(StatsResultList **result, StatsTarget target, 2319 strList *names, strList *targets, Error **errp); 2320 static void query_stats_schemas_cb(StatsSchemaList **result, Error **errp); 2321 2322 uint32_t kvm_dirty_ring_size(void) 2323 { 2324 return kvm_state->kvm_dirty_ring_size; 2325 } 2326 2327 static int kvm_init(MachineState *ms) 2328 { 2329 MachineClass *mc = MACHINE_GET_CLASS(ms); 2330 static const char upgrade_note[] = 2331 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n" 2332 "(see http://sourceforge.net/projects/kvm).\n"; 2333 const struct { 2334 const char *name; 2335 int num; 2336 } num_cpus[] = { 2337 { "SMP", ms->smp.cpus }, 2338 { "hotpluggable", ms->smp.max_cpus }, 2339 { /* end of list */ } 2340 }, *nc = num_cpus; 2341 int soft_vcpus_limit, hard_vcpus_limit; 2342 KVMState *s; 2343 const KVMCapabilityInfo *missing_cap; 2344 int ret; 2345 int type; 2346 uint64_t dirty_log_manual_caps; 2347 2348 qemu_mutex_init(&kml_slots_lock); 2349 2350 s = KVM_STATE(ms->accelerator); 2351 2352 /* 2353 * On systems where the kernel can support different base page 2354 * sizes, host page size may be different from TARGET_PAGE_SIZE, 2355 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum 2356 * page size for the system though. 2357 */ 2358 assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size()); 2359 2360 s->sigmask_len = 8; 2361 accel_blocker_init(); 2362 2363 #ifdef TARGET_KVM_HAVE_GUEST_DEBUG 2364 QTAILQ_INIT(&s->kvm_sw_breakpoints); 2365 #endif 2366 QLIST_INIT(&s->kvm_parked_vcpus); 2367 s->fd = qemu_open_old(s->device ?: "/dev/kvm", O_RDWR); 2368 if (s->fd == -1) { 2369 fprintf(stderr, "Could not access KVM kernel module: %m\n"); 2370 ret = -errno; 2371 goto err; 2372 } 2373 2374 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0); 2375 if (ret < KVM_API_VERSION) { 2376 if (ret >= 0) { 2377 ret = -EINVAL; 2378 } 2379 fprintf(stderr, "kvm version too old\n"); 2380 goto err; 2381 } 2382 2383 if (ret > KVM_API_VERSION) { 2384 ret = -EINVAL; 2385 fprintf(stderr, "kvm version not supported\n"); 2386 goto err; 2387 } 2388 2389 kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT); 2390 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS); 2391 2392 /* If unspecified, use the default value */ 2393 if (!s->nr_slots) { 2394 s->nr_slots = 32; 2395 } 2396 2397 s->nr_as = kvm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE); 2398 if (s->nr_as <= 1) { 2399 s->nr_as = 1; 2400 } 2401 s->as = g_new0(struct KVMAs, s->nr_as); 2402 2403 if (object_property_find(OBJECT(current_machine), "kvm-type")) { 2404 g_autofree char *kvm_type = object_property_get_str(OBJECT(current_machine), 2405 "kvm-type", 2406 &error_abort); 2407 type = mc->kvm_type(ms, kvm_type); 2408 } else if (mc->kvm_type) { 2409 type = mc->kvm_type(ms, NULL); 2410 } else { 2411 type = kvm_arch_get_default_type(ms); 2412 } 2413 2414 if (type < 0) { 2415 ret = -EINVAL; 2416 goto err; 2417 } 2418 2419 do { 2420 ret = kvm_ioctl(s, KVM_CREATE_VM, type); 2421 } while (ret == -EINTR); 2422 2423 if (ret < 0) { 2424 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret, 2425 strerror(-ret)); 2426 2427 #ifdef TARGET_S390X 2428 if (ret == -EINVAL) { 2429 fprintf(stderr, 2430 "Host kernel setup problem detected. Please verify:\n"); 2431 fprintf(stderr, "- for kernels supporting the switch_amode or" 2432 " user_mode parameters, whether\n"); 2433 fprintf(stderr, 2434 " user space is running in primary address space\n"); 2435 fprintf(stderr, 2436 "- for kernels supporting the vm.allocate_pgste sysctl, " 2437 "whether it is enabled\n"); 2438 } 2439 #elif defined(TARGET_PPC) 2440 if (ret == -EINVAL) { 2441 fprintf(stderr, 2442 "PPC KVM module is not loaded. Try modprobe kvm_%s.\n", 2443 (type == 2) ? "pr" : "hv"); 2444 } 2445 #endif 2446 goto err; 2447 } 2448 2449 s->vmfd = ret; 2450 2451 /* check the vcpu limits */ 2452 soft_vcpus_limit = kvm_recommended_vcpus(s); 2453 hard_vcpus_limit = kvm_max_vcpus(s); 2454 2455 while (nc->name) { 2456 if (nc->num > soft_vcpus_limit) { 2457 warn_report("Number of %s cpus requested (%d) exceeds " 2458 "the recommended cpus supported by KVM (%d)", 2459 nc->name, nc->num, soft_vcpus_limit); 2460 2461 if (nc->num > hard_vcpus_limit) { 2462 fprintf(stderr, "Number of %s cpus requested (%d) exceeds " 2463 "the maximum cpus supported by KVM (%d)\n", 2464 nc->name, nc->num, hard_vcpus_limit); 2465 exit(1); 2466 } 2467 } 2468 nc++; 2469 } 2470 2471 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites); 2472 if (!missing_cap) { 2473 missing_cap = 2474 kvm_check_extension_list(s, kvm_arch_required_capabilities); 2475 } 2476 if (missing_cap) { 2477 ret = -EINVAL; 2478 fprintf(stderr, "kvm does not support %s\n%s", 2479 missing_cap->name, upgrade_note); 2480 goto err; 2481 } 2482 2483 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO); 2484 s->coalesced_pio = s->coalesced_mmio && 2485 kvm_check_extension(s, KVM_CAP_COALESCED_PIO); 2486 2487 /* 2488 * Enable KVM dirty ring if supported, otherwise fall back to 2489 * dirty logging mode 2490 */ 2491 ret = kvm_dirty_ring_init(s); 2492 if (ret < 0) { 2493 goto err; 2494 } 2495 2496 /* 2497 * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 is not needed when dirty ring is 2498 * enabled. More importantly, KVM_DIRTY_LOG_INITIALLY_SET will assume no 2499 * page is wr-protected initially, which is against how kvm dirty ring is 2500 * usage - kvm dirty ring requires all pages are wr-protected at the very 2501 * beginning. Enabling this feature for dirty ring causes data corruption. 2502 * 2503 * TODO: Without KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and kvm clear dirty log, 2504 * we may expect a higher stall time when starting the migration. In the 2505 * future we can enable KVM_CLEAR_DIRTY_LOG to work with dirty ring too: 2506 * instead of clearing dirty bit, it can be a way to explicitly wr-protect 2507 * guest pages. 2508 */ 2509 if (!s->kvm_dirty_ring_size) { 2510 dirty_log_manual_caps = 2511 kvm_check_extension(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); 2512 dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | 2513 KVM_DIRTY_LOG_INITIALLY_SET); 2514 s->manual_dirty_log_protect = dirty_log_manual_caps; 2515 if (dirty_log_manual_caps) { 2516 ret = kvm_vm_enable_cap(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 0, 2517 dirty_log_manual_caps); 2518 if (ret) { 2519 warn_report("Trying to enable capability %"PRIu64" of " 2520 "KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 but failed. " 2521 "Falling back to the legacy mode. ", 2522 dirty_log_manual_caps); 2523 s->manual_dirty_log_protect = 0; 2524 } 2525 } 2526 } 2527 2528 #ifdef KVM_CAP_VCPU_EVENTS 2529 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS); 2530 #endif 2531 s->max_nested_state_len = kvm_check_extension(s, KVM_CAP_NESTED_STATE); 2532 2533 s->irq_set_ioctl = KVM_IRQ_LINE; 2534 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) { 2535 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS; 2536 } 2537 2538 kvm_readonly_mem_allowed = 2539 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0); 2540 2541 kvm_resamplefds_allowed = 2542 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0); 2543 2544 kvm_vm_attributes_allowed = 2545 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0); 2546 2547 #ifdef TARGET_KVM_HAVE_GUEST_DEBUG 2548 kvm_has_guest_debug = 2549 (kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG) > 0); 2550 #endif 2551 2552 kvm_sstep_flags = 0; 2553 if (kvm_has_guest_debug) { 2554 kvm_sstep_flags = SSTEP_ENABLE; 2555 2556 #if defined TARGET_KVM_HAVE_GUEST_DEBUG 2557 int guest_debug_flags = 2558 kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG2); 2559 2560 if (guest_debug_flags & KVM_GUESTDBG_BLOCKIRQ) { 2561 kvm_sstep_flags |= SSTEP_NOIRQ; 2562 } 2563 #endif 2564 } 2565 2566 kvm_state = s; 2567 2568 ret = kvm_arch_init(ms, s); 2569 if (ret < 0) { 2570 goto err; 2571 } 2572 2573 if (s->kernel_irqchip_split == ON_OFF_AUTO_AUTO) { 2574 s->kernel_irqchip_split = mc->default_kernel_irqchip_split ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 2575 } 2576 2577 qemu_register_reset(kvm_unpoison_all, NULL); 2578 2579 if (s->kernel_irqchip_allowed) { 2580 kvm_irqchip_create(s); 2581 } 2582 2583 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add; 2584 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del; 2585 s->memory_listener.listener.coalesced_io_add = kvm_coalesce_mmio_region; 2586 s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region; 2587 2588 kvm_memory_listener_register(s, &s->memory_listener, 2589 &address_space_memory, 0, "kvm-memory"); 2590 memory_listener_register(&kvm_io_listener, 2591 &address_space_io); 2592 2593 s->sync_mmu = !!kvm_vm_check_extension(kvm_state, KVM_CAP_SYNC_MMU); 2594 if (!s->sync_mmu) { 2595 ret = ram_block_discard_disable(true); 2596 assert(!ret); 2597 } 2598 2599 if (s->kvm_dirty_ring_size) { 2600 kvm_dirty_ring_reaper_init(s); 2601 } 2602 2603 if (kvm_check_extension(kvm_state, KVM_CAP_BINARY_STATS_FD)) { 2604 add_stats_callbacks(STATS_PROVIDER_KVM, query_stats_cb, 2605 query_stats_schemas_cb); 2606 } 2607 2608 return 0; 2609 2610 err: 2611 assert(ret < 0); 2612 if (s->vmfd >= 0) { 2613 close(s->vmfd); 2614 } 2615 if (s->fd != -1) { 2616 close(s->fd); 2617 } 2618 g_free(s->as); 2619 g_free(s->memory_listener.slots); 2620 2621 return ret; 2622 } 2623 2624 void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len) 2625 { 2626 s->sigmask_len = sigmask_len; 2627 } 2628 2629 static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction, 2630 int size, uint32_t count) 2631 { 2632 int i; 2633 uint8_t *ptr = data; 2634 2635 for (i = 0; i < count; i++) { 2636 address_space_rw(&address_space_io, port, attrs, 2637 ptr, size, 2638 direction == KVM_EXIT_IO_OUT); 2639 ptr += size; 2640 } 2641 } 2642 2643 static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run) 2644 { 2645 int i; 2646 2647 fprintf(stderr, "KVM internal error. Suberror: %d\n", 2648 run->internal.suberror); 2649 2650 for (i = 0; i < run->internal.ndata; ++i) { 2651 fprintf(stderr, "extra data[%d]: 0x%016"PRIx64"\n", 2652 i, (uint64_t)run->internal.data[i]); 2653 } 2654 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) { 2655 fprintf(stderr, "emulation failure\n"); 2656 if (!kvm_arch_stop_on_emulation_error(cpu)) { 2657 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2658 return EXCP_INTERRUPT; 2659 } 2660 } 2661 /* FIXME: Should trigger a qmp message to let management know 2662 * something went wrong. 2663 */ 2664 return -1; 2665 } 2666 2667 void kvm_flush_coalesced_mmio_buffer(void) 2668 { 2669 KVMState *s = kvm_state; 2670 2671 if (!s || s->coalesced_flush_in_progress) { 2672 return; 2673 } 2674 2675 s->coalesced_flush_in_progress = true; 2676 2677 if (s->coalesced_mmio_ring) { 2678 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring; 2679 while (ring->first != ring->last) { 2680 struct kvm_coalesced_mmio *ent; 2681 2682 ent = &ring->coalesced_mmio[ring->first]; 2683 2684 if (ent->pio == 1) { 2685 address_space_write(&address_space_io, ent->phys_addr, 2686 MEMTXATTRS_UNSPECIFIED, ent->data, 2687 ent->len); 2688 } else { 2689 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len); 2690 } 2691 smp_wmb(); 2692 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX; 2693 } 2694 } 2695 2696 s->coalesced_flush_in_progress = false; 2697 } 2698 2699 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg) 2700 { 2701 if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) { 2702 int ret = kvm_arch_get_registers(cpu); 2703 if (ret) { 2704 error_report("Failed to get registers: %s", strerror(-ret)); 2705 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2706 vm_stop(RUN_STATE_INTERNAL_ERROR); 2707 } 2708 2709 cpu->vcpu_dirty = true; 2710 } 2711 } 2712 2713 void kvm_cpu_synchronize_state(CPUState *cpu) 2714 { 2715 if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) { 2716 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL); 2717 } 2718 } 2719 2720 static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg) 2721 { 2722 int ret = kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE); 2723 if (ret) { 2724 error_report("Failed to put registers after reset: %s", strerror(-ret)); 2725 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2726 vm_stop(RUN_STATE_INTERNAL_ERROR); 2727 } 2728 2729 cpu->vcpu_dirty = false; 2730 } 2731 2732 void kvm_cpu_synchronize_post_reset(CPUState *cpu) 2733 { 2734 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); 2735 } 2736 2737 static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) 2738 { 2739 int ret = kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE); 2740 if (ret) { 2741 error_report("Failed to put registers after init: %s", strerror(-ret)); 2742 exit(1); 2743 } 2744 2745 cpu->vcpu_dirty = false; 2746 } 2747 2748 void kvm_cpu_synchronize_post_init(CPUState *cpu) 2749 { 2750 if (!kvm_state->guest_state_protected) { 2751 /* 2752 * This runs before the machine_init_done notifiers, and is the last 2753 * opportunity to synchronize the state of confidential guests. 2754 */ 2755 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL); 2756 } 2757 } 2758 2759 static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg) 2760 { 2761 cpu->vcpu_dirty = true; 2762 } 2763 2764 void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu) 2765 { 2766 run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL); 2767 } 2768 2769 #ifdef KVM_HAVE_MCE_INJECTION 2770 static __thread void *pending_sigbus_addr; 2771 static __thread int pending_sigbus_code; 2772 static __thread bool have_sigbus_pending; 2773 #endif 2774 2775 static void kvm_cpu_kick(CPUState *cpu) 2776 { 2777 qatomic_set(&cpu->kvm_run->immediate_exit, 1); 2778 } 2779 2780 static void kvm_cpu_kick_self(void) 2781 { 2782 if (kvm_immediate_exit) { 2783 kvm_cpu_kick(current_cpu); 2784 } else { 2785 qemu_cpu_kick_self(); 2786 } 2787 } 2788 2789 static void kvm_eat_signals(CPUState *cpu) 2790 { 2791 struct timespec ts = { 0, 0 }; 2792 siginfo_t siginfo; 2793 sigset_t waitset; 2794 sigset_t chkset; 2795 int r; 2796 2797 if (kvm_immediate_exit) { 2798 qatomic_set(&cpu->kvm_run->immediate_exit, 0); 2799 /* Write kvm_run->immediate_exit before the cpu->exit_request 2800 * write in kvm_cpu_exec. 2801 */ 2802 smp_wmb(); 2803 return; 2804 } 2805 2806 sigemptyset(&waitset); 2807 sigaddset(&waitset, SIG_IPI); 2808 2809 do { 2810 r = sigtimedwait(&waitset, &siginfo, &ts); 2811 if (r == -1 && !(errno == EAGAIN || errno == EINTR)) { 2812 perror("sigtimedwait"); 2813 exit(1); 2814 } 2815 2816 r = sigpending(&chkset); 2817 if (r == -1) { 2818 perror("sigpending"); 2819 exit(1); 2820 } 2821 } while (sigismember(&chkset, SIG_IPI)); 2822 } 2823 2824 int kvm_cpu_exec(CPUState *cpu) 2825 { 2826 struct kvm_run *run = cpu->kvm_run; 2827 int ret, run_ret; 2828 2829 trace_kvm_cpu_exec(); 2830 2831 if (kvm_arch_process_async_events(cpu)) { 2832 qatomic_set(&cpu->exit_request, 0); 2833 return EXCP_HLT; 2834 } 2835 2836 bql_unlock(); 2837 cpu_exec_start(cpu); 2838 2839 do { 2840 MemTxAttrs attrs; 2841 2842 if (cpu->vcpu_dirty) { 2843 ret = kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE); 2844 if (ret) { 2845 error_report("Failed to put registers after init: %s", 2846 strerror(-ret)); 2847 ret = -1; 2848 break; 2849 } 2850 2851 cpu->vcpu_dirty = false; 2852 } 2853 2854 kvm_arch_pre_run(cpu, run); 2855 if (qatomic_read(&cpu->exit_request)) { 2856 trace_kvm_interrupt_exit_request(); 2857 /* 2858 * KVM requires us to reenter the kernel after IO exits to complete 2859 * instruction emulation. This self-signal will ensure that we 2860 * leave ASAP again. 2861 */ 2862 kvm_cpu_kick_self(); 2863 } 2864 2865 /* Read cpu->exit_request before KVM_RUN reads run->immediate_exit. 2866 * Matching barrier in kvm_eat_signals. 2867 */ 2868 smp_rmb(); 2869 2870 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0); 2871 2872 attrs = kvm_arch_post_run(cpu, run); 2873 2874 #ifdef KVM_HAVE_MCE_INJECTION 2875 if (unlikely(have_sigbus_pending)) { 2876 bql_lock(); 2877 kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code, 2878 pending_sigbus_addr); 2879 have_sigbus_pending = false; 2880 bql_unlock(); 2881 } 2882 #endif 2883 2884 if (run_ret < 0) { 2885 if (run_ret == -EINTR || run_ret == -EAGAIN) { 2886 trace_kvm_io_window_exit(); 2887 kvm_eat_signals(cpu); 2888 ret = EXCP_INTERRUPT; 2889 break; 2890 } 2891 fprintf(stderr, "error: kvm run failed %s\n", 2892 strerror(-run_ret)); 2893 #ifdef TARGET_PPC 2894 if (run_ret == -EBUSY) { 2895 fprintf(stderr, 2896 "This is probably because your SMT is enabled.\n" 2897 "VCPU can only run on primary threads with all " 2898 "secondary threads offline.\n"); 2899 } 2900 #endif 2901 ret = -1; 2902 break; 2903 } 2904 2905 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason); 2906 switch (run->exit_reason) { 2907 case KVM_EXIT_IO: 2908 /* Called outside BQL */ 2909 kvm_handle_io(run->io.port, attrs, 2910 (uint8_t *)run + run->io.data_offset, 2911 run->io.direction, 2912 run->io.size, 2913 run->io.count); 2914 ret = 0; 2915 break; 2916 case KVM_EXIT_MMIO: 2917 /* Called outside BQL */ 2918 address_space_rw(&address_space_memory, 2919 run->mmio.phys_addr, attrs, 2920 run->mmio.data, 2921 run->mmio.len, 2922 run->mmio.is_write); 2923 ret = 0; 2924 break; 2925 case KVM_EXIT_IRQ_WINDOW_OPEN: 2926 ret = EXCP_INTERRUPT; 2927 break; 2928 case KVM_EXIT_SHUTDOWN: 2929 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 2930 ret = EXCP_INTERRUPT; 2931 break; 2932 case KVM_EXIT_UNKNOWN: 2933 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n", 2934 (uint64_t)run->hw.hardware_exit_reason); 2935 ret = -1; 2936 break; 2937 case KVM_EXIT_INTERNAL_ERROR: 2938 ret = kvm_handle_internal_error(cpu, run); 2939 break; 2940 case KVM_EXIT_DIRTY_RING_FULL: 2941 /* 2942 * We shouldn't continue if the dirty ring of this vcpu is 2943 * still full. Got kicked by KVM_RESET_DIRTY_RINGS. 2944 */ 2945 trace_kvm_dirty_ring_full(cpu->cpu_index); 2946 bql_lock(); 2947 /* 2948 * We throttle vCPU by making it sleep once it exit from kernel 2949 * due to dirty ring full. In the dirtylimit scenario, reaping 2950 * all vCPUs after a single vCPU dirty ring get full result in 2951 * the miss of sleep, so just reap the ring-fulled vCPU. 2952 */ 2953 if (dirtylimit_in_service()) { 2954 kvm_dirty_ring_reap(kvm_state, cpu); 2955 } else { 2956 kvm_dirty_ring_reap(kvm_state, NULL); 2957 } 2958 bql_unlock(); 2959 dirtylimit_vcpu_execute(cpu); 2960 ret = 0; 2961 break; 2962 case KVM_EXIT_SYSTEM_EVENT: 2963 trace_kvm_run_exit_system_event(cpu->cpu_index, run->system_event.type); 2964 switch (run->system_event.type) { 2965 case KVM_SYSTEM_EVENT_SHUTDOWN: 2966 qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); 2967 ret = EXCP_INTERRUPT; 2968 break; 2969 case KVM_SYSTEM_EVENT_RESET: 2970 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 2971 ret = EXCP_INTERRUPT; 2972 break; 2973 case KVM_SYSTEM_EVENT_CRASH: 2974 kvm_cpu_synchronize_state(cpu); 2975 bql_lock(); 2976 qemu_system_guest_panicked(cpu_get_crash_info(cpu)); 2977 bql_unlock(); 2978 ret = 0; 2979 break; 2980 default: 2981 ret = kvm_arch_handle_exit(cpu, run); 2982 break; 2983 } 2984 break; 2985 default: 2986 ret = kvm_arch_handle_exit(cpu, run); 2987 break; 2988 } 2989 } while (ret == 0); 2990 2991 cpu_exec_end(cpu); 2992 bql_lock(); 2993 2994 if (ret < 0) { 2995 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2996 vm_stop(RUN_STATE_INTERNAL_ERROR); 2997 } 2998 2999 qatomic_set(&cpu->exit_request, 0); 3000 return ret; 3001 } 3002 3003 int kvm_ioctl(KVMState *s, int type, ...) 3004 { 3005 int ret; 3006 void *arg; 3007 va_list ap; 3008 3009 va_start(ap, type); 3010 arg = va_arg(ap, void *); 3011 va_end(ap); 3012 3013 trace_kvm_ioctl(type, arg); 3014 ret = ioctl(s->fd, type, arg); 3015 if (ret == -1) { 3016 ret = -errno; 3017 } 3018 return ret; 3019 } 3020 3021 int kvm_vm_ioctl(KVMState *s, int type, ...) 3022 { 3023 int ret; 3024 void *arg; 3025 va_list ap; 3026 3027 va_start(ap, type); 3028 arg = va_arg(ap, void *); 3029 va_end(ap); 3030 3031 trace_kvm_vm_ioctl(type, arg); 3032 accel_ioctl_begin(); 3033 ret = ioctl(s->vmfd, type, arg); 3034 accel_ioctl_end(); 3035 if (ret == -1) { 3036 ret = -errno; 3037 } 3038 return ret; 3039 } 3040 3041 int kvm_vcpu_ioctl(CPUState *cpu, int type, ...) 3042 { 3043 int ret; 3044 void *arg; 3045 va_list ap; 3046 3047 va_start(ap, type); 3048 arg = va_arg(ap, void *); 3049 va_end(ap); 3050 3051 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg); 3052 accel_cpu_ioctl_begin(cpu); 3053 ret = ioctl(cpu->kvm_fd, type, arg); 3054 accel_cpu_ioctl_end(cpu); 3055 if (ret == -1) { 3056 ret = -errno; 3057 } 3058 return ret; 3059 } 3060 3061 int kvm_device_ioctl(int fd, int type, ...) 3062 { 3063 int ret; 3064 void *arg; 3065 va_list ap; 3066 3067 va_start(ap, type); 3068 arg = va_arg(ap, void *); 3069 va_end(ap); 3070 3071 trace_kvm_device_ioctl(fd, type, arg); 3072 accel_ioctl_begin(); 3073 ret = ioctl(fd, type, arg); 3074 accel_ioctl_end(); 3075 if (ret == -1) { 3076 ret = -errno; 3077 } 3078 return ret; 3079 } 3080 3081 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr) 3082 { 3083 int ret; 3084 struct kvm_device_attr attribute = { 3085 .group = group, 3086 .attr = attr, 3087 }; 3088 3089 if (!kvm_vm_attributes_allowed) { 3090 return 0; 3091 } 3092 3093 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute); 3094 /* kvm returns 0 on success for HAS_DEVICE_ATTR */ 3095 return ret ? 0 : 1; 3096 } 3097 3098 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr) 3099 { 3100 struct kvm_device_attr attribute = { 3101 .group = group, 3102 .attr = attr, 3103 .flags = 0, 3104 }; 3105 3106 return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1; 3107 } 3108 3109 int kvm_device_access(int fd, int group, uint64_t attr, 3110 void *val, bool write, Error **errp) 3111 { 3112 struct kvm_device_attr kvmattr; 3113 int err; 3114 3115 kvmattr.flags = 0; 3116 kvmattr.group = group; 3117 kvmattr.attr = attr; 3118 kvmattr.addr = (uintptr_t)val; 3119 3120 err = kvm_device_ioctl(fd, 3121 write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR, 3122 &kvmattr); 3123 if (err < 0) { 3124 error_setg_errno(errp, -err, 3125 "KVM_%s_DEVICE_ATTR failed: Group %d " 3126 "attr 0x%016" PRIx64, 3127 write ? "SET" : "GET", group, attr); 3128 } 3129 return err; 3130 } 3131 3132 bool kvm_has_sync_mmu(void) 3133 { 3134 return kvm_state->sync_mmu; 3135 } 3136 3137 int kvm_has_vcpu_events(void) 3138 { 3139 return kvm_state->vcpu_events; 3140 } 3141 3142 int kvm_max_nested_state_length(void) 3143 { 3144 return kvm_state->max_nested_state_len; 3145 } 3146 3147 int kvm_has_gsi_routing(void) 3148 { 3149 #ifdef KVM_CAP_IRQ_ROUTING 3150 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING); 3151 #else 3152 return false; 3153 #endif 3154 } 3155 3156 bool kvm_arm_supports_user_irq(void) 3157 { 3158 return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ); 3159 } 3160 3161 #ifdef TARGET_KVM_HAVE_GUEST_DEBUG 3162 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu, vaddr pc) 3163 { 3164 struct kvm_sw_breakpoint *bp; 3165 3166 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) { 3167 if (bp->pc == pc) { 3168 return bp; 3169 } 3170 } 3171 return NULL; 3172 } 3173 3174 int kvm_sw_breakpoints_active(CPUState *cpu) 3175 { 3176 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints); 3177 } 3178 3179 struct kvm_set_guest_debug_data { 3180 struct kvm_guest_debug dbg; 3181 int err; 3182 }; 3183 3184 static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data) 3185 { 3186 struct kvm_set_guest_debug_data *dbg_data = 3187 (struct kvm_set_guest_debug_data *) data.host_ptr; 3188 3189 dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG, 3190 &dbg_data->dbg); 3191 } 3192 3193 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap) 3194 { 3195 struct kvm_set_guest_debug_data data; 3196 3197 data.dbg.control = reinject_trap; 3198 3199 if (cpu->singlestep_enabled) { 3200 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP; 3201 3202 if (cpu->singlestep_enabled & SSTEP_NOIRQ) { 3203 data.dbg.control |= KVM_GUESTDBG_BLOCKIRQ; 3204 } 3205 } 3206 kvm_arch_update_guest_debug(cpu, &data.dbg); 3207 3208 run_on_cpu(cpu, kvm_invoke_set_guest_debug, 3209 RUN_ON_CPU_HOST_PTR(&data)); 3210 return data.err; 3211 } 3212 3213 bool kvm_supports_guest_debug(void) 3214 { 3215 /* probed during kvm_init() */ 3216 return kvm_has_guest_debug; 3217 } 3218 3219 int kvm_insert_breakpoint(CPUState *cpu, int type, vaddr addr, vaddr len) 3220 { 3221 struct kvm_sw_breakpoint *bp; 3222 int err; 3223 3224 if (type == GDB_BREAKPOINT_SW) { 3225 bp = kvm_find_sw_breakpoint(cpu, addr); 3226 if (bp) { 3227 bp->use_count++; 3228 return 0; 3229 } 3230 3231 bp = g_new(struct kvm_sw_breakpoint, 1); 3232 bp->pc = addr; 3233 bp->use_count = 1; 3234 err = kvm_arch_insert_sw_breakpoint(cpu, bp); 3235 if (err) { 3236 g_free(bp); 3237 return err; 3238 } 3239 3240 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 3241 } else { 3242 err = kvm_arch_insert_hw_breakpoint(addr, len, type); 3243 if (err) { 3244 return err; 3245 } 3246 } 3247 3248 CPU_FOREACH(cpu) { 3249 err = kvm_update_guest_debug(cpu, 0); 3250 if (err) { 3251 return err; 3252 } 3253 } 3254 return 0; 3255 } 3256 3257 int kvm_remove_breakpoint(CPUState *cpu, int type, vaddr addr, vaddr len) 3258 { 3259 struct kvm_sw_breakpoint *bp; 3260 int err; 3261 3262 if (type == GDB_BREAKPOINT_SW) { 3263 bp = kvm_find_sw_breakpoint(cpu, addr); 3264 if (!bp) { 3265 return -ENOENT; 3266 } 3267 3268 if (bp->use_count > 1) { 3269 bp->use_count--; 3270 return 0; 3271 } 3272 3273 err = kvm_arch_remove_sw_breakpoint(cpu, bp); 3274 if (err) { 3275 return err; 3276 } 3277 3278 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 3279 g_free(bp); 3280 } else { 3281 err = kvm_arch_remove_hw_breakpoint(addr, len, type); 3282 if (err) { 3283 return err; 3284 } 3285 } 3286 3287 CPU_FOREACH(cpu) { 3288 err = kvm_update_guest_debug(cpu, 0); 3289 if (err) { 3290 return err; 3291 } 3292 } 3293 return 0; 3294 } 3295 3296 void kvm_remove_all_breakpoints(CPUState *cpu) 3297 { 3298 struct kvm_sw_breakpoint *bp, *next; 3299 KVMState *s = cpu->kvm_state; 3300 CPUState *tmpcpu; 3301 3302 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) { 3303 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) { 3304 /* Try harder to find a CPU that currently sees the breakpoint. */ 3305 CPU_FOREACH(tmpcpu) { 3306 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) { 3307 break; 3308 } 3309 } 3310 } 3311 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry); 3312 g_free(bp); 3313 } 3314 kvm_arch_remove_all_hw_breakpoints(); 3315 3316 CPU_FOREACH(cpu) { 3317 kvm_update_guest_debug(cpu, 0); 3318 } 3319 } 3320 3321 #endif /* !TARGET_KVM_HAVE_GUEST_DEBUG */ 3322 3323 static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset) 3324 { 3325 KVMState *s = kvm_state; 3326 struct kvm_signal_mask *sigmask; 3327 int r; 3328 3329 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset)); 3330 3331 sigmask->len = s->sigmask_len; 3332 memcpy(sigmask->sigset, sigset, sizeof(*sigset)); 3333 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask); 3334 g_free(sigmask); 3335 3336 return r; 3337 } 3338 3339 static void kvm_ipi_signal(int sig) 3340 { 3341 if (current_cpu) { 3342 assert(kvm_immediate_exit); 3343 kvm_cpu_kick(current_cpu); 3344 } 3345 } 3346 3347 void kvm_init_cpu_signals(CPUState *cpu) 3348 { 3349 int r; 3350 sigset_t set; 3351 struct sigaction sigact; 3352 3353 memset(&sigact, 0, sizeof(sigact)); 3354 sigact.sa_handler = kvm_ipi_signal; 3355 sigaction(SIG_IPI, &sigact, NULL); 3356 3357 pthread_sigmask(SIG_BLOCK, NULL, &set); 3358 #if defined KVM_HAVE_MCE_INJECTION 3359 sigdelset(&set, SIGBUS); 3360 pthread_sigmask(SIG_SETMASK, &set, NULL); 3361 #endif 3362 sigdelset(&set, SIG_IPI); 3363 if (kvm_immediate_exit) { 3364 r = pthread_sigmask(SIG_SETMASK, &set, NULL); 3365 } else { 3366 r = kvm_set_signal_mask(cpu, &set); 3367 } 3368 if (r) { 3369 fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r)); 3370 exit(1); 3371 } 3372 } 3373 3374 /* Called asynchronously in VCPU thread. */ 3375 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr) 3376 { 3377 #ifdef KVM_HAVE_MCE_INJECTION 3378 if (have_sigbus_pending) { 3379 return 1; 3380 } 3381 have_sigbus_pending = true; 3382 pending_sigbus_addr = addr; 3383 pending_sigbus_code = code; 3384 qatomic_set(&cpu->exit_request, 1); 3385 return 0; 3386 #else 3387 return 1; 3388 #endif 3389 } 3390 3391 /* Called synchronously (via signalfd) in main thread. */ 3392 int kvm_on_sigbus(int code, void *addr) 3393 { 3394 #ifdef KVM_HAVE_MCE_INJECTION 3395 /* Action required MCE kills the process if SIGBUS is blocked. Because 3396 * that's what happens in the I/O thread, where we handle MCE via signalfd, 3397 * we can only get action optional here. 3398 */ 3399 assert(code != BUS_MCEERR_AR); 3400 kvm_arch_on_sigbus_vcpu(first_cpu, code, addr); 3401 return 0; 3402 #else 3403 return 1; 3404 #endif 3405 } 3406 3407 int kvm_create_device(KVMState *s, uint64_t type, bool test) 3408 { 3409 int ret; 3410 struct kvm_create_device create_dev; 3411 3412 create_dev.type = type; 3413 create_dev.fd = -1; 3414 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0; 3415 3416 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) { 3417 return -ENOTSUP; 3418 } 3419 3420 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev); 3421 if (ret) { 3422 return ret; 3423 } 3424 3425 return test ? 0 : create_dev.fd; 3426 } 3427 3428 bool kvm_device_supported(int vmfd, uint64_t type) 3429 { 3430 struct kvm_create_device create_dev = { 3431 .type = type, 3432 .fd = -1, 3433 .flags = KVM_CREATE_DEVICE_TEST, 3434 }; 3435 3436 if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) { 3437 return false; 3438 } 3439 3440 return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0); 3441 } 3442 3443 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source) 3444 { 3445 struct kvm_one_reg reg; 3446 int r; 3447 3448 reg.id = id; 3449 reg.addr = (uintptr_t) source; 3450 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 3451 if (r) { 3452 trace_kvm_failed_reg_set(id, strerror(-r)); 3453 } 3454 return r; 3455 } 3456 3457 int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target) 3458 { 3459 struct kvm_one_reg reg; 3460 int r; 3461 3462 reg.id = id; 3463 reg.addr = (uintptr_t) target; 3464 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 3465 if (r) { 3466 trace_kvm_failed_reg_get(id, strerror(-r)); 3467 } 3468 return r; 3469 } 3470 3471 static bool kvm_accel_has_memory(MachineState *ms, AddressSpace *as, 3472 hwaddr start_addr, hwaddr size) 3473 { 3474 KVMState *kvm = KVM_STATE(ms->accelerator); 3475 int i; 3476 3477 for (i = 0; i < kvm->nr_as; ++i) { 3478 if (kvm->as[i].as == as && kvm->as[i].ml) { 3479 size = MIN(kvm_max_slot_size, size); 3480 return NULL != kvm_lookup_matching_slot(kvm->as[i].ml, 3481 start_addr, size); 3482 } 3483 } 3484 3485 return false; 3486 } 3487 3488 static void kvm_get_kvm_shadow_mem(Object *obj, Visitor *v, 3489 const char *name, void *opaque, 3490 Error **errp) 3491 { 3492 KVMState *s = KVM_STATE(obj); 3493 int64_t value = s->kvm_shadow_mem; 3494 3495 visit_type_int(v, name, &value, errp); 3496 } 3497 3498 static void kvm_set_kvm_shadow_mem(Object *obj, Visitor *v, 3499 const char *name, void *opaque, 3500 Error **errp) 3501 { 3502 KVMState *s = KVM_STATE(obj); 3503 int64_t value; 3504 3505 if (s->fd != -1) { 3506 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3507 return; 3508 } 3509 3510 if (!visit_type_int(v, name, &value, errp)) { 3511 return; 3512 } 3513 3514 s->kvm_shadow_mem = value; 3515 } 3516 3517 static void kvm_set_kernel_irqchip(Object *obj, Visitor *v, 3518 const char *name, void *opaque, 3519 Error **errp) 3520 { 3521 KVMState *s = KVM_STATE(obj); 3522 OnOffSplit mode; 3523 3524 if (s->fd != -1) { 3525 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3526 return; 3527 } 3528 3529 if (!visit_type_OnOffSplit(v, name, &mode, errp)) { 3530 return; 3531 } 3532 switch (mode) { 3533 case ON_OFF_SPLIT_ON: 3534 s->kernel_irqchip_allowed = true; 3535 s->kernel_irqchip_required = true; 3536 s->kernel_irqchip_split = ON_OFF_AUTO_OFF; 3537 break; 3538 case ON_OFF_SPLIT_OFF: 3539 s->kernel_irqchip_allowed = false; 3540 s->kernel_irqchip_required = false; 3541 s->kernel_irqchip_split = ON_OFF_AUTO_OFF; 3542 break; 3543 case ON_OFF_SPLIT_SPLIT: 3544 s->kernel_irqchip_allowed = true; 3545 s->kernel_irqchip_required = true; 3546 s->kernel_irqchip_split = ON_OFF_AUTO_ON; 3547 break; 3548 default: 3549 /* The value was checked in visit_type_OnOffSplit() above. If 3550 * we get here, then something is wrong in QEMU. 3551 */ 3552 abort(); 3553 } 3554 } 3555 3556 bool kvm_kernel_irqchip_allowed(void) 3557 { 3558 return kvm_state->kernel_irqchip_allowed; 3559 } 3560 3561 bool kvm_kernel_irqchip_required(void) 3562 { 3563 return kvm_state->kernel_irqchip_required; 3564 } 3565 3566 bool kvm_kernel_irqchip_split(void) 3567 { 3568 return kvm_state->kernel_irqchip_split == ON_OFF_AUTO_ON; 3569 } 3570 3571 static void kvm_get_dirty_ring_size(Object *obj, Visitor *v, 3572 const char *name, void *opaque, 3573 Error **errp) 3574 { 3575 KVMState *s = KVM_STATE(obj); 3576 uint32_t value = s->kvm_dirty_ring_size; 3577 3578 visit_type_uint32(v, name, &value, errp); 3579 } 3580 3581 static void kvm_set_dirty_ring_size(Object *obj, Visitor *v, 3582 const char *name, void *opaque, 3583 Error **errp) 3584 { 3585 KVMState *s = KVM_STATE(obj); 3586 uint32_t value; 3587 3588 if (s->fd != -1) { 3589 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3590 return; 3591 } 3592 3593 if (!visit_type_uint32(v, name, &value, errp)) { 3594 return; 3595 } 3596 if (value & (value - 1)) { 3597 error_setg(errp, "dirty-ring-size must be a power of two."); 3598 return; 3599 } 3600 3601 s->kvm_dirty_ring_size = value; 3602 } 3603 3604 static char *kvm_get_device(Object *obj, 3605 Error **errp G_GNUC_UNUSED) 3606 { 3607 KVMState *s = KVM_STATE(obj); 3608 3609 return g_strdup(s->device); 3610 } 3611 3612 static void kvm_set_device(Object *obj, 3613 const char *value, 3614 Error **errp G_GNUC_UNUSED) 3615 { 3616 KVMState *s = KVM_STATE(obj); 3617 3618 g_free(s->device); 3619 s->device = g_strdup(value); 3620 } 3621 3622 static void kvm_accel_instance_init(Object *obj) 3623 { 3624 KVMState *s = KVM_STATE(obj); 3625 3626 s->fd = -1; 3627 s->vmfd = -1; 3628 s->kvm_shadow_mem = -1; 3629 s->kernel_irqchip_allowed = true; 3630 s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; 3631 /* KVM dirty ring is by default off */ 3632 s->kvm_dirty_ring_size = 0; 3633 s->kvm_dirty_ring_with_bitmap = false; 3634 s->kvm_eager_split_size = 0; 3635 s->notify_vmexit = NOTIFY_VMEXIT_OPTION_RUN; 3636 s->notify_window = 0; 3637 s->xen_version = 0; 3638 s->xen_gnttab_max_frames = 64; 3639 s->xen_evtchn_max_pirq = 256; 3640 s->device = NULL; 3641 } 3642 3643 /** 3644 * kvm_gdbstub_sstep_flags(): 3645 * 3646 * Returns: SSTEP_* flags that KVM supports for guest debug. The 3647 * support is probed during kvm_init() 3648 */ 3649 static int kvm_gdbstub_sstep_flags(void) 3650 { 3651 return kvm_sstep_flags; 3652 } 3653 3654 static void kvm_accel_class_init(ObjectClass *oc, void *data) 3655 { 3656 AccelClass *ac = ACCEL_CLASS(oc); 3657 ac->name = "KVM"; 3658 ac->init_machine = kvm_init; 3659 ac->has_memory = kvm_accel_has_memory; 3660 ac->allowed = &kvm_allowed; 3661 ac->gdbstub_supported_sstep_flags = kvm_gdbstub_sstep_flags; 3662 3663 object_class_property_add(oc, "kernel-irqchip", "on|off|split", 3664 NULL, kvm_set_kernel_irqchip, 3665 NULL, NULL); 3666 object_class_property_set_description(oc, "kernel-irqchip", 3667 "Configure KVM in-kernel irqchip"); 3668 3669 object_class_property_add(oc, "kvm-shadow-mem", "int", 3670 kvm_get_kvm_shadow_mem, kvm_set_kvm_shadow_mem, 3671 NULL, NULL); 3672 object_class_property_set_description(oc, "kvm-shadow-mem", 3673 "KVM shadow MMU size"); 3674 3675 object_class_property_add(oc, "dirty-ring-size", "uint32", 3676 kvm_get_dirty_ring_size, kvm_set_dirty_ring_size, 3677 NULL, NULL); 3678 object_class_property_set_description(oc, "dirty-ring-size", 3679 "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)"); 3680 3681 object_class_property_add_str(oc, "device", kvm_get_device, kvm_set_device); 3682 object_class_property_set_description(oc, "device", 3683 "Path to the device node to use (default: /dev/kvm)"); 3684 3685 kvm_arch_accel_class_init(oc); 3686 } 3687 3688 static const TypeInfo kvm_accel_type = { 3689 .name = TYPE_KVM_ACCEL, 3690 .parent = TYPE_ACCEL, 3691 .instance_init = kvm_accel_instance_init, 3692 .class_init = kvm_accel_class_init, 3693 .instance_size = sizeof(KVMState), 3694 }; 3695 3696 static void kvm_type_init(void) 3697 { 3698 type_register_static(&kvm_accel_type); 3699 } 3700 3701 type_init(kvm_type_init); 3702 3703 typedef struct StatsArgs { 3704 union StatsResultsType { 3705 StatsResultList **stats; 3706 StatsSchemaList **schema; 3707 } result; 3708 strList *names; 3709 Error **errp; 3710 } StatsArgs; 3711 3712 static StatsList *add_kvmstat_entry(struct kvm_stats_desc *pdesc, 3713 uint64_t *stats_data, 3714 StatsList *stats_list, 3715 Error **errp) 3716 { 3717 3718 Stats *stats; 3719 uint64List *val_list = NULL; 3720 3721 /* Only add stats that we understand. */ 3722 switch (pdesc->flags & KVM_STATS_TYPE_MASK) { 3723 case KVM_STATS_TYPE_CUMULATIVE: 3724 case KVM_STATS_TYPE_INSTANT: 3725 case KVM_STATS_TYPE_PEAK: 3726 case KVM_STATS_TYPE_LINEAR_HIST: 3727 case KVM_STATS_TYPE_LOG_HIST: 3728 break; 3729 default: 3730 return stats_list; 3731 } 3732 3733 switch (pdesc->flags & KVM_STATS_UNIT_MASK) { 3734 case KVM_STATS_UNIT_NONE: 3735 case KVM_STATS_UNIT_BYTES: 3736 case KVM_STATS_UNIT_CYCLES: 3737 case KVM_STATS_UNIT_SECONDS: 3738 case KVM_STATS_UNIT_BOOLEAN: 3739 break; 3740 default: 3741 return stats_list; 3742 } 3743 3744 switch (pdesc->flags & KVM_STATS_BASE_MASK) { 3745 case KVM_STATS_BASE_POW10: 3746 case KVM_STATS_BASE_POW2: 3747 break; 3748 default: 3749 return stats_list; 3750 } 3751 3752 /* Alloc and populate data list */ 3753 stats = g_new0(Stats, 1); 3754 stats->name = g_strdup(pdesc->name); 3755 stats->value = g_new0(StatsValue, 1);; 3756 3757 if ((pdesc->flags & KVM_STATS_UNIT_MASK) == KVM_STATS_UNIT_BOOLEAN) { 3758 stats->value->u.boolean = *stats_data; 3759 stats->value->type = QTYPE_QBOOL; 3760 } else if (pdesc->size == 1) { 3761 stats->value->u.scalar = *stats_data; 3762 stats->value->type = QTYPE_QNUM; 3763 } else { 3764 int i; 3765 for (i = 0; i < pdesc->size; i++) { 3766 QAPI_LIST_PREPEND(val_list, stats_data[i]); 3767 } 3768 stats->value->u.list = val_list; 3769 stats->value->type = QTYPE_QLIST; 3770 } 3771 3772 QAPI_LIST_PREPEND(stats_list, stats); 3773 return stats_list; 3774 } 3775 3776 static StatsSchemaValueList *add_kvmschema_entry(struct kvm_stats_desc *pdesc, 3777 StatsSchemaValueList *list, 3778 Error **errp) 3779 { 3780 StatsSchemaValueList *schema_entry = g_new0(StatsSchemaValueList, 1); 3781 schema_entry->value = g_new0(StatsSchemaValue, 1); 3782 3783 switch (pdesc->flags & KVM_STATS_TYPE_MASK) { 3784 case KVM_STATS_TYPE_CUMULATIVE: 3785 schema_entry->value->type = STATS_TYPE_CUMULATIVE; 3786 break; 3787 case KVM_STATS_TYPE_INSTANT: 3788 schema_entry->value->type = STATS_TYPE_INSTANT; 3789 break; 3790 case KVM_STATS_TYPE_PEAK: 3791 schema_entry->value->type = STATS_TYPE_PEAK; 3792 break; 3793 case KVM_STATS_TYPE_LINEAR_HIST: 3794 schema_entry->value->type = STATS_TYPE_LINEAR_HISTOGRAM; 3795 schema_entry->value->bucket_size = pdesc->bucket_size; 3796 schema_entry->value->has_bucket_size = true; 3797 break; 3798 case KVM_STATS_TYPE_LOG_HIST: 3799 schema_entry->value->type = STATS_TYPE_LOG2_HISTOGRAM; 3800 break; 3801 default: 3802 goto exit; 3803 } 3804 3805 switch (pdesc->flags & KVM_STATS_UNIT_MASK) { 3806 case KVM_STATS_UNIT_NONE: 3807 break; 3808 case KVM_STATS_UNIT_BOOLEAN: 3809 schema_entry->value->has_unit = true; 3810 schema_entry->value->unit = STATS_UNIT_BOOLEAN; 3811 break; 3812 case KVM_STATS_UNIT_BYTES: 3813 schema_entry->value->has_unit = true; 3814 schema_entry->value->unit = STATS_UNIT_BYTES; 3815 break; 3816 case KVM_STATS_UNIT_CYCLES: 3817 schema_entry->value->has_unit = true; 3818 schema_entry->value->unit = STATS_UNIT_CYCLES; 3819 break; 3820 case KVM_STATS_UNIT_SECONDS: 3821 schema_entry->value->has_unit = true; 3822 schema_entry->value->unit = STATS_UNIT_SECONDS; 3823 break; 3824 default: 3825 goto exit; 3826 } 3827 3828 schema_entry->value->exponent = pdesc->exponent; 3829 if (pdesc->exponent) { 3830 switch (pdesc->flags & KVM_STATS_BASE_MASK) { 3831 case KVM_STATS_BASE_POW10: 3832 schema_entry->value->has_base = true; 3833 schema_entry->value->base = 10; 3834 break; 3835 case KVM_STATS_BASE_POW2: 3836 schema_entry->value->has_base = true; 3837 schema_entry->value->base = 2; 3838 break; 3839 default: 3840 goto exit; 3841 } 3842 } 3843 3844 schema_entry->value->name = g_strdup(pdesc->name); 3845 schema_entry->next = list; 3846 return schema_entry; 3847 exit: 3848 g_free(schema_entry->value); 3849 g_free(schema_entry); 3850 return list; 3851 } 3852 3853 /* Cached stats descriptors */ 3854 typedef struct StatsDescriptors { 3855 const char *ident; /* cache key, currently the StatsTarget */ 3856 struct kvm_stats_desc *kvm_stats_desc; 3857 struct kvm_stats_header kvm_stats_header; 3858 QTAILQ_ENTRY(StatsDescriptors) next; 3859 } StatsDescriptors; 3860 3861 static QTAILQ_HEAD(, StatsDescriptors) stats_descriptors = 3862 QTAILQ_HEAD_INITIALIZER(stats_descriptors); 3863 3864 /* 3865 * Return the descriptors for 'target', that either have already been read 3866 * or are retrieved from 'stats_fd'. 3867 */ 3868 static StatsDescriptors *find_stats_descriptors(StatsTarget target, int stats_fd, 3869 Error **errp) 3870 { 3871 StatsDescriptors *descriptors; 3872 const char *ident; 3873 struct kvm_stats_desc *kvm_stats_desc; 3874 struct kvm_stats_header *kvm_stats_header; 3875 size_t size_desc; 3876 ssize_t ret; 3877 3878 ident = StatsTarget_str(target); 3879 QTAILQ_FOREACH(descriptors, &stats_descriptors, next) { 3880 if (g_str_equal(descriptors->ident, ident)) { 3881 return descriptors; 3882 } 3883 } 3884 3885 descriptors = g_new0(StatsDescriptors, 1); 3886 3887 /* Read stats header */ 3888 kvm_stats_header = &descriptors->kvm_stats_header; 3889 ret = pread(stats_fd, kvm_stats_header, sizeof(*kvm_stats_header), 0); 3890 if (ret != sizeof(*kvm_stats_header)) { 3891 error_setg(errp, "KVM stats: failed to read stats header: " 3892 "expected %zu actual %zu", 3893 sizeof(*kvm_stats_header), ret); 3894 g_free(descriptors); 3895 return NULL; 3896 } 3897 size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; 3898 3899 /* Read stats descriptors */ 3900 kvm_stats_desc = g_malloc0_n(kvm_stats_header->num_desc, size_desc); 3901 ret = pread(stats_fd, kvm_stats_desc, 3902 size_desc * kvm_stats_header->num_desc, 3903 kvm_stats_header->desc_offset); 3904 3905 if (ret != size_desc * kvm_stats_header->num_desc) { 3906 error_setg(errp, "KVM stats: failed to read stats descriptors: " 3907 "expected %zu actual %zu", 3908 size_desc * kvm_stats_header->num_desc, ret); 3909 g_free(descriptors); 3910 g_free(kvm_stats_desc); 3911 return NULL; 3912 } 3913 descriptors->kvm_stats_desc = kvm_stats_desc; 3914 descriptors->ident = ident; 3915 QTAILQ_INSERT_TAIL(&stats_descriptors, descriptors, next); 3916 return descriptors; 3917 } 3918 3919 static void query_stats(StatsResultList **result, StatsTarget target, 3920 strList *names, int stats_fd, CPUState *cpu, 3921 Error **errp) 3922 { 3923 struct kvm_stats_desc *kvm_stats_desc; 3924 struct kvm_stats_header *kvm_stats_header; 3925 StatsDescriptors *descriptors; 3926 g_autofree uint64_t *stats_data = NULL; 3927 struct kvm_stats_desc *pdesc; 3928 StatsList *stats_list = NULL; 3929 size_t size_desc, size_data = 0; 3930 ssize_t ret; 3931 int i; 3932 3933 descriptors = find_stats_descriptors(target, stats_fd, errp); 3934 if (!descriptors) { 3935 return; 3936 } 3937 3938 kvm_stats_header = &descriptors->kvm_stats_header; 3939 kvm_stats_desc = descriptors->kvm_stats_desc; 3940 size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; 3941 3942 /* Tally the total data size; read schema data */ 3943 for (i = 0; i < kvm_stats_header->num_desc; ++i) { 3944 pdesc = (void *)kvm_stats_desc + i * size_desc; 3945 size_data += pdesc->size * sizeof(*stats_data); 3946 } 3947 3948 stats_data = g_malloc0(size_data); 3949 ret = pread(stats_fd, stats_data, size_data, kvm_stats_header->data_offset); 3950 3951 if (ret != size_data) { 3952 error_setg(errp, "KVM stats: failed to read data: " 3953 "expected %zu actual %zu", size_data, ret); 3954 return; 3955 } 3956 3957 for (i = 0; i < kvm_stats_header->num_desc; ++i) { 3958 uint64_t *stats; 3959 pdesc = (void *)kvm_stats_desc + i * size_desc; 3960 3961 /* Add entry to the list */ 3962 stats = (void *)stats_data + pdesc->offset; 3963 if (!apply_str_list_filter(pdesc->name, names)) { 3964 continue; 3965 } 3966 stats_list = add_kvmstat_entry(pdesc, stats, stats_list, errp); 3967 } 3968 3969 if (!stats_list) { 3970 return; 3971 } 3972 3973 switch (target) { 3974 case STATS_TARGET_VM: 3975 add_stats_entry(result, STATS_PROVIDER_KVM, NULL, stats_list); 3976 break; 3977 case STATS_TARGET_VCPU: 3978 add_stats_entry(result, STATS_PROVIDER_KVM, 3979 cpu->parent_obj.canonical_path, 3980 stats_list); 3981 break; 3982 default: 3983 g_assert_not_reached(); 3984 } 3985 } 3986 3987 static void query_stats_schema(StatsSchemaList **result, StatsTarget target, 3988 int stats_fd, Error **errp) 3989 { 3990 struct kvm_stats_desc *kvm_stats_desc; 3991 struct kvm_stats_header *kvm_stats_header; 3992 StatsDescriptors *descriptors; 3993 struct kvm_stats_desc *pdesc; 3994 StatsSchemaValueList *stats_list = NULL; 3995 size_t size_desc; 3996 int i; 3997 3998 descriptors = find_stats_descriptors(target, stats_fd, errp); 3999 if (!descriptors) { 4000 return; 4001 } 4002 4003 kvm_stats_header = &descriptors->kvm_stats_header; 4004 kvm_stats_desc = descriptors->kvm_stats_desc; 4005 size_desc = sizeof(*kvm_stats_desc) + kvm_stats_header->name_size; 4006 4007 /* Tally the total data size; read schema data */ 4008 for (i = 0; i < kvm_stats_header->num_desc; ++i) { 4009 pdesc = (void *)kvm_stats_desc + i * size_desc; 4010 stats_list = add_kvmschema_entry(pdesc, stats_list, errp); 4011 } 4012 4013 add_stats_schema(result, STATS_PROVIDER_KVM, target, stats_list); 4014 } 4015 4016 static void query_stats_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args) 4017 { 4018 int stats_fd = cpu->kvm_vcpu_stats_fd; 4019 Error *local_err = NULL; 4020 4021 if (stats_fd == -1) { 4022 error_setg_errno(&local_err, errno, "KVM stats: ioctl failed"); 4023 error_propagate(kvm_stats_args->errp, local_err); 4024 return; 4025 } 4026 query_stats(kvm_stats_args->result.stats, STATS_TARGET_VCPU, 4027 kvm_stats_args->names, stats_fd, cpu, 4028 kvm_stats_args->errp); 4029 } 4030 4031 static void query_stats_schema_vcpu(CPUState *cpu, StatsArgs *kvm_stats_args) 4032 { 4033 int stats_fd = cpu->kvm_vcpu_stats_fd; 4034 Error *local_err = NULL; 4035 4036 if (stats_fd == -1) { 4037 error_setg_errno(&local_err, errno, "KVM stats: ioctl failed"); 4038 error_propagate(kvm_stats_args->errp, local_err); 4039 return; 4040 } 4041 query_stats_schema(kvm_stats_args->result.schema, STATS_TARGET_VCPU, stats_fd, 4042 kvm_stats_args->errp); 4043 } 4044 4045 static void query_stats_cb(StatsResultList **result, StatsTarget target, 4046 strList *names, strList *targets, Error **errp) 4047 { 4048 KVMState *s = kvm_state; 4049 CPUState *cpu; 4050 int stats_fd; 4051 4052 switch (target) { 4053 case STATS_TARGET_VM: 4054 { 4055 stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL); 4056 if (stats_fd == -1) { 4057 error_setg_errno(errp, errno, "KVM stats: ioctl failed"); 4058 return; 4059 } 4060 query_stats(result, target, names, stats_fd, NULL, errp); 4061 close(stats_fd); 4062 break; 4063 } 4064 case STATS_TARGET_VCPU: 4065 { 4066 StatsArgs stats_args; 4067 stats_args.result.stats = result; 4068 stats_args.names = names; 4069 stats_args.errp = errp; 4070 CPU_FOREACH(cpu) { 4071 if (!apply_str_list_filter(cpu->parent_obj.canonical_path, targets)) { 4072 continue; 4073 } 4074 query_stats_vcpu(cpu, &stats_args); 4075 } 4076 break; 4077 } 4078 default: 4079 break; 4080 } 4081 } 4082 4083 void query_stats_schemas_cb(StatsSchemaList **result, Error **errp) 4084 { 4085 StatsArgs stats_args; 4086 KVMState *s = kvm_state; 4087 int stats_fd; 4088 4089 stats_fd = kvm_vm_ioctl(s, KVM_GET_STATS_FD, NULL); 4090 if (stats_fd == -1) { 4091 error_setg_errno(errp, errno, "KVM stats: ioctl failed"); 4092 return; 4093 } 4094 query_stats_schema(result, STATS_TARGET_VM, stats_fd, errp); 4095 close(stats_fd); 4096 4097 if (first_cpu) { 4098 stats_args.result.schema = result; 4099 stats_args.errp = errp; 4100 query_stats_schema_vcpu(first_cpu, &stats_args); 4101 } 4102 } 4103 4104 void kvm_mark_guest_state_protected(void) 4105 { 4106 kvm_state->guest_state_protected = true; 4107 } 4108