1 /* 2 * QEMU KVM support 3 * 4 * Copyright IBM, Corp. 2008 5 * Red Hat, Inc. 2008 6 * 7 * Authors: 8 * Anthony Liguori <aliguori@us.ibm.com> 9 * Glauber Costa <gcosta@redhat.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2 or later. 12 * See the COPYING file in the top-level directory. 13 * 14 */ 15 16 #include "qemu/osdep.h" 17 #include <sys/ioctl.h> 18 #include <poll.h> 19 20 #include <linux/kvm.h> 21 22 #include "qemu/atomic.h" 23 #include "qemu/option.h" 24 #include "qemu/config-file.h" 25 #include "qemu/error-report.h" 26 #include "qapi/error.h" 27 #include "hw/pci/msi.h" 28 #include "hw/pci/msix.h" 29 #include "hw/s390x/adapter.h" 30 #include "exec/gdbstub.h" 31 #include "sysemu/kvm_int.h" 32 #include "sysemu/runstate.h" 33 #include "sysemu/cpus.h" 34 #include "qemu/bswap.h" 35 #include "exec/memory.h" 36 #include "exec/ram_addr.h" 37 #include "qemu/event_notifier.h" 38 #include "qemu/main-loop.h" 39 #include "trace.h" 40 #include "hw/irq.h" 41 #include "qapi/visitor.h" 42 #include "qapi/qapi-types-common.h" 43 #include "qapi/qapi-visit-common.h" 44 #include "sysemu/reset.h" 45 #include "qemu/guest-random.h" 46 #include "sysemu/hw_accel.h" 47 #include "kvm-cpus.h" 48 49 #include "hw/boards.h" 50 51 /* This check must be after config-host.h is included */ 52 #ifdef CONFIG_EVENTFD 53 #include <sys/eventfd.h> 54 #endif 55 56 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We 57 * need to use the real host PAGE_SIZE, as that's what KVM will use. 58 */ 59 #ifdef PAGE_SIZE 60 #undef PAGE_SIZE 61 #endif 62 #define PAGE_SIZE qemu_real_host_page_size() 63 64 #ifndef KVM_GUESTDBG_BLOCKIRQ 65 #define KVM_GUESTDBG_BLOCKIRQ 0 66 #endif 67 68 //#define DEBUG_KVM 69 70 #ifdef DEBUG_KVM 71 #define DPRINTF(fmt, ...) \ 72 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0) 73 #else 74 #define DPRINTF(fmt, ...) \ 75 do { } while (0) 76 #endif 77 78 #define KVM_MSI_HASHTAB_SIZE 256 79 80 struct KVMParkedVcpu { 81 unsigned long vcpu_id; 82 int kvm_fd; 83 QLIST_ENTRY(KVMParkedVcpu) node; 84 }; 85 86 enum KVMDirtyRingReaperState { 87 KVM_DIRTY_RING_REAPER_NONE = 0, 88 /* The reaper is sleeping */ 89 KVM_DIRTY_RING_REAPER_WAIT, 90 /* The reaper is reaping for dirty pages */ 91 KVM_DIRTY_RING_REAPER_REAPING, 92 }; 93 94 /* 95 * KVM reaper instance, responsible for collecting the KVM dirty bits 96 * via the dirty ring. 97 */ 98 struct KVMDirtyRingReaper { 99 /* The reaper thread */ 100 QemuThread reaper_thr; 101 volatile uint64_t reaper_iteration; /* iteration number of reaper thr */ 102 volatile enum KVMDirtyRingReaperState reaper_state; /* reap thr state */ 103 }; 104 105 struct KVMState 106 { 107 AccelState parent_obj; 108 109 int nr_slots; 110 int fd; 111 int vmfd; 112 int coalesced_mmio; 113 int coalesced_pio; 114 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; 115 bool coalesced_flush_in_progress; 116 int vcpu_events; 117 int robust_singlestep; 118 int debugregs; 119 #ifdef KVM_CAP_SET_GUEST_DEBUG 120 QTAILQ_HEAD(, kvm_sw_breakpoint) kvm_sw_breakpoints; 121 #endif 122 int max_nested_state_len; 123 int many_ioeventfds; 124 int intx_set_mask; 125 int kvm_shadow_mem; 126 bool kernel_irqchip_allowed; 127 bool kernel_irqchip_required; 128 OnOffAuto kernel_irqchip_split; 129 bool sync_mmu; 130 uint64_t manual_dirty_log_protect; 131 /* The man page (and posix) say ioctl numbers are signed int, but 132 * they're not. Linux, glibc and *BSD all treat ioctl numbers as 133 * unsigned, and treating them as signed here can break things */ 134 unsigned irq_set_ioctl; 135 unsigned int sigmask_len; 136 GHashTable *gsimap; 137 #ifdef KVM_CAP_IRQ_ROUTING 138 struct kvm_irq_routing *irq_routes; 139 int nr_allocated_irq_routes; 140 unsigned long *used_gsi_bitmap; 141 unsigned int gsi_count; 142 QTAILQ_HEAD(, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; 143 #endif 144 KVMMemoryListener memory_listener; 145 QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; 146 147 /* For "info mtree -f" to tell if an MR is registered in KVM */ 148 int nr_as; 149 struct KVMAs { 150 KVMMemoryListener *ml; 151 AddressSpace *as; 152 } *as; 153 uint64_t kvm_dirty_ring_bytes; /* Size of the per-vcpu dirty ring */ 154 uint32_t kvm_dirty_ring_size; /* Number of dirty GFNs per ring */ 155 struct KVMDirtyRingReaper reaper; 156 }; 157 158 KVMState *kvm_state; 159 bool kvm_kernel_irqchip; 160 bool kvm_split_irqchip; 161 bool kvm_async_interrupts_allowed; 162 bool kvm_halt_in_kernel_allowed; 163 bool kvm_eventfds_allowed; 164 bool kvm_irqfds_allowed; 165 bool kvm_resamplefds_allowed; 166 bool kvm_msi_via_irqfd_allowed; 167 bool kvm_gsi_routing_allowed; 168 bool kvm_gsi_direct_mapping; 169 bool kvm_allowed; 170 bool kvm_readonly_mem_allowed; 171 bool kvm_vm_attributes_allowed; 172 bool kvm_direct_msi_allowed; 173 bool kvm_ioeventfd_any_length_allowed; 174 bool kvm_msi_use_devid; 175 bool kvm_has_guest_debug; 176 int kvm_sstep_flags; 177 static bool kvm_immediate_exit; 178 static hwaddr kvm_max_slot_size = ~0; 179 180 static const KVMCapabilityInfo kvm_required_capabilites[] = { 181 KVM_CAP_INFO(USER_MEMORY), 182 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS), 183 KVM_CAP_INFO(JOIN_MEMORY_REGIONS_WORKS), 184 KVM_CAP_LAST_INFO 185 }; 186 187 static NotifierList kvm_irqchip_change_notifiers = 188 NOTIFIER_LIST_INITIALIZER(kvm_irqchip_change_notifiers); 189 190 struct KVMResampleFd { 191 int gsi; 192 EventNotifier *resample_event; 193 QLIST_ENTRY(KVMResampleFd) node; 194 }; 195 typedef struct KVMResampleFd KVMResampleFd; 196 197 /* 198 * Only used with split irqchip where we need to do the resample fd 199 * kick for the kernel from userspace. 200 */ 201 static QLIST_HEAD(, KVMResampleFd) kvm_resample_fd_list = 202 QLIST_HEAD_INITIALIZER(kvm_resample_fd_list); 203 204 static QemuMutex kml_slots_lock; 205 206 #define kvm_slots_lock() qemu_mutex_lock(&kml_slots_lock) 207 #define kvm_slots_unlock() qemu_mutex_unlock(&kml_slots_lock) 208 209 static void kvm_slot_init_dirty_bitmap(KVMSlot *mem); 210 211 static inline void kvm_resample_fd_remove(int gsi) 212 { 213 KVMResampleFd *rfd; 214 215 QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { 216 if (rfd->gsi == gsi) { 217 QLIST_REMOVE(rfd, node); 218 g_free(rfd); 219 break; 220 } 221 } 222 } 223 224 static inline void kvm_resample_fd_insert(int gsi, EventNotifier *event) 225 { 226 KVMResampleFd *rfd = g_new0(KVMResampleFd, 1); 227 228 rfd->gsi = gsi; 229 rfd->resample_event = event; 230 231 QLIST_INSERT_HEAD(&kvm_resample_fd_list, rfd, node); 232 } 233 234 void kvm_resample_fd_notify(int gsi) 235 { 236 KVMResampleFd *rfd; 237 238 QLIST_FOREACH(rfd, &kvm_resample_fd_list, node) { 239 if (rfd->gsi == gsi) { 240 event_notifier_set(rfd->resample_event); 241 trace_kvm_resample_fd_notify(gsi); 242 return; 243 } 244 } 245 } 246 247 int kvm_get_max_memslots(void) 248 { 249 KVMState *s = KVM_STATE(current_accel()); 250 251 return s->nr_slots; 252 } 253 254 /* Called with KVMMemoryListener.slots_lock held */ 255 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml) 256 { 257 KVMState *s = kvm_state; 258 int i; 259 260 for (i = 0; i < s->nr_slots; i++) { 261 if (kml->slots[i].memory_size == 0) { 262 return &kml->slots[i]; 263 } 264 } 265 266 return NULL; 267 } 268 269 bool kvm_has_free_slot(MachineState *ms) 270 { 271 KVMState *s = KVM_STATE(ms->accelerator); 272 bool result; 273 KVMMemoryListener *kml = &s->memory_listener; 274 275 kvm_slots_lock(); 276 result = !!kvm_get_free_slot(kml); 277 kvm_slots_unlock(); 278 279 return result; 280 } 281 282 /* Called with KVMMemoryListener.slots_lock held */ 283 static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml) 284 { 285 KVMSlot *slot = kvm_get_free_slot(kml); 286 287 if (slot) { 288 return slot; 289 } 290 291 fprintf(stderr, "%s: no free slot available\n", __func__); 292 abort(); 293 } 294 295 static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml, 296 hwaddr start_addr, 297 hwaddr size) 298 { 299 KVMState *s = kvm_state; 300 int i; 301 302 for (i = 0; i < s->nr_slots; i++) { 303 KVMSlot *mem = &kml->slots[i]; 304 305 if (start_addr == mem->start_addr && size == mem->memory_size) { 306 return mem; 307 } 308 } 309 310 return NULL; 311 } 312 313 /* 314 * Calculate and align the start address and the size of the section. 315 * Return the size. If the size is 0, the aligned section is empty. 316 */ 317 static hwaddr kvm_align_section(MemoryRegionSection *section, 318 hwaddr *start) 319 { 320 hwaddr size = int128_get64(section->size); 321 hwaddr delta, aligned; 322 323 /* kvm works in page size chunks, but the function may be called 324 with sub-page size and unaligned start address. Pad the start 325 address to next and truncate size to previous page boundary. */ 326 aligned = ROUND_UP(section->offset_within_address_space, 327 qemu_real_host_page_size()); 328 delta = aligned - section->offset_within_address_space; 329 *start = aligned; 330 if (delta > size) { 331 return 0; 332 } 333 334 return (size - delta) & qemu_real_host_page_mask(); 335 } 336 337 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram, 338 hwaddr *phys_addr) 339 { 340 KVMMemoryListener *kml = &s->memory_listener; 341 int i, ret = 0; 342 343 kvm_slots_lock(); 344 for (i = 0; i < s->nr_slots; i++) { 345 KVMSlot *mem = &kml->slots[i]; 346 347 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) { 348 *phys_addr = mem->start_addr + (ram - mem->ram); 349 ret = 1; 350 break; 351 } 352 } 353 kvm_slots_unlock(); 354 355 return ret; 356 } 357 358 static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new) 359 { 360 KVMState *s = kvm_state; 361 struct kvm_userspace_memory_region mem; 362 int ret; 363 364 mem.slot = slot->slot | (kml->as_id << 16); 365 mem.guest_phys_addr = slot->start_addr; 366 mem.userspace_addr = (unsigned long)slot->ram; 367 mem.flags = slot->flags; 368 369 if (slot->memory_size && !new && (mem.flags ^ slot->old_flags) & KVM_MEM_READONLY) { 370 /* Set the slot size to 0 before setting the slot to the desired 371 * value. This is needed based on KVM commit 75d61fbc. */ 372 mem.memory_size = 0; 373 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 374 if (ret < 0) { 375 goto err; 376 } 377 } 378 mem.memory_size = slot->memory_size; 379 ret = kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 380 slot->old_flags = mem.flags; 381 err: 382 trace_kvm_set_user_memory(mem.slot, mem.flags, mem.guest_phys_addr, 383 mem.memory_size, mem.userspace_addr, ret); 384 if (ret < 0) { 385 error_report("%s: KVM_SET_USER_MEMORY_REGION failed, slot=%d," 386 " start=0x%" PRIx64 ", size=0x%" PRIx64 ": %s", 387 __func__, mem.slot, slot->start_addr, 388 (uint64_t)mem.memory_size, strerror(errno)); 389 } 390 return ret; 391 } 392 393 static int do_kvm_destroy_vcpu(CPUState *cpu) 394 { 395 KVMState *s = kvm_state; 396 long mmap_size; 397 struct KVMParkedVcpu *vcpu = NULL; 398 int ret = 0; 399 400 DPRINTF("kvm_destroy_vcpu\n"); 401 402 ret = kvm_arch_destroy_vcpu(cpu); 403 if (ret < 0) { 404 goto err; 405 } 406 407 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 408 if (mmap_size < 0) { 409 ret = mmap_size; 410 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n"); 411 goto err; 412 } 413 414 ret = munmap(cpu->kvm_run, mmap_size); 415 if (ret < 0) { 416 goto err; 417 } 418 419 if (cpu->kvm_dirty_gfns) { 420 ret = munmap(cpu->kvm_dirty_gfns, s->kvm_dirty_ring_bytes); 421 if (ret < 0) { 422 goto err; 423 } 424 } 425 426 vcpu = g_malloc0(sizeof(*vcpu)); 427 vcpu->vcpu_id = kvm_arch_vcpu_id(cpu); 428 vcpu->kvm_fd = cpu->kvm_fd; 429 QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); 430 err: 431 return ret; 432 } 433 434 void kvm_destroy_vcpu(CPUState *cpu) 435 { 436 if (do_kvm_destroy_vcpu(cpu) < 0) { 437 error_report("kvm_destroy_vcpu failed"); 438 exit(EXIT_FAILURE); 439 } 440 } 441 442 static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id) 443 { 444 struct KVMParkedVcpu *cpu; 445 446 QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { 447 if (cpu->vcpu_id == vcpu_id) { 448 int kvm_fd; 449 450 QLIST_REMOVE(cpu, node); 451 kvm_fd = cpu->kvm_fd; 452 g_free(cpu); 453 return kvm_fd; 454 } 455 } 456 457 return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id); 458 } 459 460 int kvm_init_vcpu(CPUState *cpu, Error **errp) 461 { 462 KVMState *s = kvm_state; 463 long mmap_size; 464 int ret; 465 466 trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu)); 467 468 ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu)); 469 if (ret < 0) { 470 error_setg_errno(errp, -ret, "kvm_init_vcpu: kvm_get_vcpu failed (%lu)", 471 kvm_arch_vcpu_id(cpu)); 472 goto err; 473 } 474 475 cpu->kvm_fd = ret; 476 cpu->kvm_state = s; 477 cpu->vcpu_dirty = true; 478 cpu->dirty_pages = 0; 479 480 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 481 if (mmap_size < 0) { 482 ret = mmap_size; 483 error_setg_errno(errp, -mmap_size, 484 "kvm_init_vcpu: KVM_GET_VCPU_MMAP_SIZE failed"); 485 goto err; 486 } 487 488 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, 489 cpu->kvm_fd, 0); 490 if (cpu->kvm_run == MAP_FAILED) { 491 ret = -errno; 492 error_setg_errno(errp, ret, 493 "kvm_init_vcpu: mmap'ing vcpu state failed (%lu)", 494 kvm_arch_vcpu_id(cpu)); 495 goto err; 496 } 497 498 if (s->coalesced_mmio && !s->coalesced_mmio_ring) { 499 s->coalesced_mmio_ring = 500 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE; 501 } 502 503 if (s->kvm_dirty_ring_size) { 504 /* Use MAP_SHARED to share pages with the kernel */ 505 cpu->kvm_dirty_gfns = mmap(NULL, s->kvm_dirty_ring_bytes, 506 PROT_READ | PROT_WRITE, MAP_SHARED, 507 cpu->kvm_fd, 508 PAGE_SIZE * KVM_DIRTY_LOG_PAGE_OFFSET); 509 if (cpu->kvm_dirty_gfns == MAP_FAILED) { 510 ret = -errno; 511 DPRINTF("mmap'ing vcpu dirty gfns failed: %d\n", ret); 512 goto err; 513 } 514 } 515 516 ret = kvm_arch_init_vcpu(cpu); 517 if (ret < 0) { 518 error_setg_errno(errp, -ret, 519 "kvm_init_vcpu: kvm_arch_init_vcpu failed (%lu)", 520 kvm_arch_vcpu_id(cpu)); 521 } 522 err: 523 return ret; 524 } 525 526 /* 527 * dirty pages logging control 528 */ 529 530 static int kvm_mem_flags(MemoryRegion *mr) 531 { 532 bool readonly = mr->readonly || memory_region_is_romd(mr); 533 int flags = 0; 534 535 if (memory_region_get_dirty_log_mask(mr) != 0) { 536 flags |= KVM_MEM_LOG_DIRTY_PAGES; 537 } 538 if (readonly && kvm_readonly_mem_allowed) { 539 flags |= KVM_MEM_READONLY; 540 } 541 return flags; 542 } 543 544 /* Called with KVMMemoryListener.slots_lock held */ 545 static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem, 546 MemoryRegion *mr) 547 { 548 mem->flags = kvm_mem_flags(mr); 549 550 /* If nothing changed effectively, no need to issue ioctl */ 551 if (mem->flags == mem->old_flags) { 552 return 0; 553 } 554 555 kvm_slot_init_dirty_bitmap(mem); 556 return kvm_set_user_memory_region(kml, mem, false); 557 } 558 559 static int kvm_section_update_flags(KVMMemoryListener *kml, 560 MemoryRegionSection *section) 561 { 562 hwaddr start_addr, size, slot_size; 563 KVMSlot *mem; 564 int ret = 0; 565 566 size = kvm_align_section(section, &start_addr); 567 if (!size) { 568 return 0; 569 } 570 571 kvm_slots_lock(); 572 573 while (size && !ret) { 574 slot_size = MIN(kvm_max_slot_size, size); 575 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 576 if (!mem) { 577 /* We don't have a slot if we want to trap every access. */ 578 goto out; 579 } 580 581 ret = kvm_slot_update_flags(kml, mem, section->mr); 582 start_addr += slot_size; 583 size -= slot_size; 584 } 585 586 out: 587 kvm_slots_unlock(); 588 return ret; 589 } 590 591 static void kvm_log_start(MemoryListener *listener, 592 MemoryRegionSection *section, 593 int old, int new) 594 { 595 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 596 int r; 597 598 if (old != 0) { 599 return; 600 } 601 602 r = kvm_section_update_flags(kml, section); 603 if (r < 0) { 604 abort(); 605 } 606 } 607 608 static void kvm_log_stop(MemoryListener *listener, 609 MemoryRegionSection *section, 610 int old, int new) 611 { 612 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 613 int r; 614 615 if (new != 0) { 616 return; 617 } 618 619 r = kvm_section_update_flags(kml, section); 620 if (r < 0) { 621 abort(); 622 } 623 } 624 625 /* get kvm's dirty pages bitmap and update qemu's */ 626 static void kvm_slot_sync_dirty_pages(KVMSlot *slot) 627 { 628 ram_addr_t start = slot->ram_start_offset; 629 ram_addr_t pages = slot->memory_size / qemu_real_host_page_size(); 630 631 cpu_physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); 632 } 633 634 static void kvm_slot_reset_dirty_pages(KVMSlot *slot) 635 { 636 memset(slot->dirty_bmap, 0, slot->dirty_bmap_size); 637 } 638 639 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1)) 640 641 /* Allocate the dirty bitmap for a slot */ 642 static void kvm_slot_init_dirty_bitmap(KVMSlot *mem) 643 { 644 if (!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) || mem->dirty_bmap) { 645 return; 646 } 647 648 /* 649 * XXX bad kernel interface alert 650 * For dirty bitmap, kernel allocates array of size aligned to 651 * bits-per-long. But for case when the kernel is 64bits and 652 * the userspace is 32bits, userspace can't align to the same 653 * bits-per-long, since sizeof(long) is different between kernel 654 * and user space. This way, userspace will provide buffer which 655 * may be 4 bytes less than the kernel will use, resulting in 656 * userspace memory corruption (which is not detectable by valgrind 657 * too, in most cases). 658 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in 659 * a hope that sizeof(long) won't become >8 any time soon. 660 * 661 * Note: the granule of kvm dirty log is qemu_real_host_page_size. 662 * And mem->memory_size is aligned to it (otherwise this mem can't 663 * be registered to KVM). 664 */ 665 hwaddr bitmap_size = ALIGN(mem->memory_size / qemu_real_host_page_size(), 666 /*HOST_LONG_BITS*/ 64) / 8; 667 mem->dirty_bmap = g_malloc0(bitmap_size); 668 mem->dirty_bmap_size = bitmap_size; 669 } 670 671 /* 672 * Sync dirty bitmap from kernel to KVMSlot.dirty_bmap, return true if 673 * succeeded, false otherwise 674 */ 675 static bool kvm_slot_get_dirty_log(KVMState *s, KVMSlot *slot) 676 { 677 struct kvm_dirty_log d = {}; 678 int ret; 679 680 d.dirty_bitmap = slot->dirty_bmap; 681 d.slot = slot->slot | (slot->as_id << 16); 682 ret = kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d); 683 684 if (ret == -ENOENT) { 685 /* kernel does not have dirty bitmap in this slot */ 686 ret = 0; 687 } 688 if (ret) { 689 error_report_once("%s: KVM_GET_DIRTY_LOG failed with %d", 690 __func__, ret); 691 } 692 return ret == 0; 693 } 694 695 /* Should be with all slots_lock held for the address spaces. */ 696 static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id, 697 uint32_t slot_id, uint64_t offset) 698 { 699 KVMMemoryListener *kml; 700 KVMSlot *mem; 701 702 if (as_id >= s->nr_as) { 703 return; 704 } 705 706 kml = s->as[as_id].ml; 707 mem = &kml->slots[slot_id]; 708 709 if (!mem->memory_size || offset >= 710 (mem->memory_size / qemu_real_host_page_size())) { 711 return; 712 } 713 714 set_bit(offset, mem->dirty_bmap); 715 } 716 717 static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn) 718 { 719 return gfn->flags == KVM_DIRTY_GFN_F_DIRTY; 720 } 721 722 static void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn) 723 { 724 gfn->flags = KVM_DIRTY_GFN_F_RESET; 725 } 726 727 /* 728 * Should be with all slots_lock held for the address spaces. It returns the 729 * dirty page we've collected on this dirty ring. 730 */ 731 static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu) 732 { 733 struct kvm_dirty_gfn *dirty_gfns = cpu->kvm_dirty_gfns, *cur; 734 uint32_t ring_size = s->kvm_dirty_ring_size; 735 uint32_t count = 0, fetch = cpu->kvm_fetch_index; 736 737 assert(dirty_gfns && ring_size); 738 trace_kvm_dirty_ring_reap_vcpu(cpu->cpu_index); 739 740 while (true) { 741 cur = &dirty_gfns[fetch % ring_size]; 742 if (!dirty_gfn_is_dirtied(cur)) { 743 break; 744 } 745 kvm_dirty_ring_mark_page(s, cur->slot >> 16, cur->slot & 0xffff, 746 cur->offset); 747 dirty_gfn_set_collected(cur); 748 trace_kvm_dirty_ring_page(cpu->cpu_index, fetch, cur->offset); 749 fetch++; 750 count++; 751 } 752 cpu->kvm_fetch_index = fetch; 753 cpu->dirty_pages += count; 754 755 return count; 756 } 757 758 /* Must be with slots_lock held */ 759 static uint64_t kvm_dirty_ring_reap_locked(KVMState *s) 760 { 761 int ret; 762 CPUState *cpu; 763 uint64_t total = 0; 764 int64_t stamp; 765 766 stamp = get_clock(); 767 768 CPU_FOREACH(cpu) { 769 total += kvm_dirty_ring_reap_one(s, cpu); 770 } 771 772 if (total) { 773 ret = kvm_vm_ioctl(s, KVM_RESET_DIRTY_RINGS); 774 assert(ret == total); 775 } 776 777 stamp = get_clock() - stamp; 778 779 if (total) { 780 trace_kvm_dirty_ring_reap(total, stamp / 1000); 781 } 782 783 return total; 784 } 785 786 /* 787 * Currently for simplicity, we must hold BQL before calling this. We can 788 * consider to drop the BQL if we're clear with all the race conditions. 789 */ 790 static uint64_t kvm_dirty_ring_reap(KVMState *s) 791 { 792 uint64_t total; 793 794 /* 795 * We need to lock all kvm slots for all address spaces here, 796 * because: 797 * 798 * (1) We need to mark dirty for dirty bitmaps in multiple slots 799 * and for tons of pages, so it's better to take the lock here 800 * once rather than once per page. And more importantly, 801 * 802 * (2) We must _NOT_ publish dirty bits to the other threads 803 * (e.g., the migration thread) via the kvm memory slot dirty 804 * bitmaps before correctly re-protect those dirtied pages. 805 * Otherwise we can have potential risk of data corruption if 806 * the page data is read in the other thread before we do 807 * reset below. 808 */ 809 kvm_slots_lock(); 810 total = kvm_dirty_ring_reap_locked(s); 811 kvm_slots_unlock(); 812 813 return total; 814 } 815 816 static void do_kvm_cpu_synchronize_kick(CPUState *cpu, run_on_cpu_data arg) 817 { 818 /* No need to do anything */ 819 } 820 821 /* 822 * Kick all vcpus out in a synchronized way. When returned, we 823 * guarantee that every vcpu has been kicked and at least returned to 824 * userspace once. 825 */ 826 static void kvm_cpu_synchronize_kick_all(void) 827 { 828 CPUState *cpu; 829 830 CPU_FOREACH(cpu) { 831 run_on_cpu(cpu, do_kvm_cpu_synchronize_kick, RUN_ON_CPU_NULL); 832 } 833 } 834 835 /* 836 * Flush all the existing dirty pages to the KVM slot buffers. When 837 * this call returns, we guarantee that all the touched dirty pages 838 * before calling this function have been put into the per-kvmslot 839 * dirty bitmap. 840 * 841 * This function must be called with BQL held. 842 */ 843 static void kvm_dirty_ring_flush(void) 844 { 845 trace_kvm_dirty_ring_flush(0); 846 /* 847 * The function needs to be serialized. Since this function 848 * should always be with BQL held, serialization is guaranteed. 849 * However, let's be sure of it. 850 */ 851 assert(qemu_mutex_iothread_locked()); 852 /* 853 * First make sure to flush the hardware buffers by kicking all 854 * vcpus out in a synchronous way. 855 */ 856 kvm_cpu_synchronize_kick_all(); 857 kvm_dirty_ring_reap(kvm_state); 858 trace_kvm_dirty_ring_flush(1); 859 } 860 861 /** 862 * kvm_physical_sync_dirty_bitmap - Sync dirty bitmap from kernel space 863 * 864 * This function will first try to fetch dirty bitmap from the kernel, 865 * and then updates qemu's dirty bitmap. 866 * 867 * NOTE: caller must be with kml->slots_lock held. 868 * 869 * @kml: the KVM memory listener object 870 * @section: the memory section to sync the dirty bitmap with 871 */ 872 static void kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, 873 MemoryRegionSection *section) 874 { 875 KVMState *s = kvm_state; 876 KVMSlot *mem; 877 hwaddr start_addr, size; 878 hwaddr slot_size; 879 880 size = kvm_align_section(section, &start_addr); 881 while (size) { 882 slot_size = MIN(kvm_max_slot_size, size); 883 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 884 if (!mem) { 885 /* We don't have a slot if we want to trap every access. */ 886 return; 887 } 888 if (kvm_slot_get_dirty_log(s, mem)) { 889 kvm_slot_sync_dirty_pages(mem); 890 } 891 start_addr += slot_size; 892 size -= slot_size; 893 } 894 } 895 896 /* Alignment requirement for KVM_CLEAR_DIRTY_LOG - 64 pages */ 897 #define KVM_CLEAR_LOG_SHIFT 6 898 #define KVM_CLEAR_LOG_ALIGN (qemu_real_host_page_size() << KVM_CLEAR_LOG_SHIFT) 899 #define KVM_CLEAR_LOG_MASK (-KVM_CLEAR_LOG_ALIGN) 900 901 static int kvm_log_clear_one_slot(KVMSlot *mem, int as_id, uint64_t start, 902 uint64_t size) 903 { 904 KVMState *s = kvm_state; 905 uint64_t end, bmap_start, start_delta, bmap_npages; 906 struct kvm_clear_dirty_log d; 907 unsigned long *bmap_clear = NULL, psize = qemu_real_host_page_size(); 908 int ret; 909 910 /* 911 * We need to extend either the start or the size or both to 912 * satisfy the KVM interface requirement. Firstly, do the start 913 * page alignment on 64 host pages 914 */ 915 bmap_start = start & KVM_CLEAR_LOG_MASK; 916 start_delta = start - bmap_start; 917 bmap_start /= psize; 918 919 /* 920 * The kernel interface has restriction on the size too, that either: 921 * 922 * (1) the size is 64 host pages aligned (just like the start), or 923 * (2) the size fills up until the end of the KVM memslot. 924 */ 925 bmap_npages = DIV_ROUND_UP(size + start_delta, KVM_CLEAR_LOG_ALIGN) 926 << KVM_CLEAR_LOG_SHIFT; 927 end = mem->memory_size / psize; 928 if (bmap_npages > end - bmap_start) { 929 bmap_npages = end - bmap_start; 930 } 931 start_delta /= psize; 932 933 /* 934 * Prepare the bitmap to clear dirty bits. Here we must guarantee 935 * that we won't clear any unknown dirty bits otherwise we might 936 * accidentally clear some set bits which are not yet synced from 937 * the kernel into QEMU's bitmap, then we'll lose track of the 938 * guest modifications upon those pages (which can directly lead 939 * to guest data loss or panic after migration). 940 * 941 * Layout of the KVMSlot.dirty_bmap: 942 * 943 * |<-------- bmap_npages -----------..>| 944 * [1] 945 * start_delta size 946 * |----------------|-------------|------------------|------------| 947 * ^ ^ ^ ^ 948 * | | | | 949 * start bmap_start (start) end 950 * of memslot of memslot 951 * 952 * [1] bmap_npages can be aligned to either 64 pages or the end of slot 953 */ 954 955 assert(bmap_start % BITS_PER_LONG == 0); 956 /* We should never do log_clear before log_sync */ 957 assert(mem->dirty_bmap); 958 if (start_delta || bmap_npages - size / psize) { 959 /* Slow path - we need to manipulate a temp bitmap */ 960 bmap_clear = bitmap_new(bmap_npages); 961 bitmap_copy_with_src_offset(bmap_clear, mem->dirty_bmap, 962 bmap_start, start_delta + size / psize); 963 /* 964 * We need to fill the holes at start because that was not 965 * specified by the caller and we extended the bitmap only for 966 * 64 pages alignment 967 */ 968 bitmap_clear(bmap_clear, 0, start_delta); 969 d.dirty_bitmap = bmap_clear; 970 } else { 971 /* 972 * Fast path - both start and size align well with BITS_PER_LONG 973 * (or the end of memory slot) 974 */ 975 d.dirty_bitmap = mem->dirty_bmap + BIT_WORD(bmap_start); 976 } 977 978 d.first_page = bmap_start; 979 /* It should never overflow. If it happens, say something */ 980 assert(bmap_npages <= UINT32_MAX); 981 d.num_pages = bmap_npages; 982 d.slot = mem->slot | (as_id << 16); 983 984 ret = kvm_vm_ioctl(s, KVM_CLEAR_DIRTY_LOG, &d); 985 if (ret < 0 && ret != -ENOENT) { 986 error_report("%s: KVM_CLEAR_DIRTY_LOG failed, slot=%d, " 987 "start=0x%"PRIx64", size=0x%"PRIx32", errno=%d", 988 __func__, d.slot, (uint64_t)d.first_page, 989 (uint32_t)d.num_pages, ret); 990 } else { 991 ret = 0; 992 trace_kvm_clear_dirty_log(d.slot, d.first_page, d.num_pages); 993 } 994 995 /* 996 * After we have updated the remote dirty bitmap, we update the 997 * cached bitmap as well for the memslot, then if another user 998 * clears the same region we know we shouldn't clear it again on 999 * the remote otherwise it's data loss as well. 1000 */ 1001 bitmap_clear(mem->dirty_bmap, bmap_start + start_delta, 1002 size / psize); 1003 /* This handles the NULL case well */ 1004 g_free(bmap_clear); 1005 return ret; 1006 } 1007 1008 1009 /** 1010 * kvm_physical_log_clear - Clear the kernel's dirty bitmap for range 1011 * 1012 * NOTE: this will be a no-op if we haven't enabled manual dirty log 1013 * protection in the host kernel because in that case this operation 1014 * will be done within log_sync(). 1015 * 1016 * @kml: the kvm memory listener 1017 * @section: the memory range to clear dirty bitmap 1018 */ 1019 static int kvm_physical_log_clear(KVMMemoryListener *kml, 1020 MemoryRegionSection *section) 1021 { 1022 KVMState *s = kvm_state; 1023 uint64_t start, size, offset, count; 1024 KVMSlot *mem; 1025 int ret = 0, i; 1026 1027 if (!s->manual_dirty_log_protect) { 1028 /* No need to do explicit clear */ 1029 return ret; 1030 } 1031 1032 start = section->offset_within_address_space; 1033 size = int128_get64(section->size); 1034 1035 if (!size) { 1036 /* Nothing more we can do... */ 1037 return ret; 1038 } 1039 1040 kvm_slots_lock(); 1041 1042 for (i = 0; i < s->nr_slots; i++) { 1043 mem = &kml->slots[i]; 1044 /* Discard slots that are empty or do not overlap the section */ 1045 if (!mem->memory_size || 1046 mem->start_addr > start + size - 1 || 1047 start > mem->start_addr + mem->memory_size - 1) { 1048 continue; 1049 } 1050 1051 if (start >= mem->start_addr) { 1052 /* The slot starts before section or is aligned to it. */ 1053 offset = start - mem->start_addr; 1054 count = MIN(mem->memory_size - offset, size); 1055 } else { 1056 /* The slot starts after section. */ 1057 offset = 0; 1058 count = MIN(mem->memory_size, size - (mem->start_addr - start)); 1059 } 1060 ret = kvm_log_clear_one_slot(mem, kml->as_id, offset, count); 1061 if (ret < 0) { 1062 break; 1063 } 1064 } 1065 1066 kvm_slots_unlock(); 1067 1068 return ret; 1069 } 1070 1071 static void kvm_coalesce_mmio_region(MemoryListener *listener, 1072 MemoryRegionSection *secion, 1073 hwaddr start, hwaddr size) 1074 { 1075 KVMState *s = kvm_state; 1076 1077 if (s->coalesced_mmio) { 1078 struct kvm_coalesced_mmio_zone zone; 1079 1080 zone.addr = start; 1081 zone.size = size; 1082 zone.pad = 0; 1083 1084 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); 1085 } 1086 } 1087 1088 static void kvm_uncoalesce_mmio_region(MemoryListener *listener, 1089 MemoryRegionSection *secion, 1090 hwaddr start, hwaddr size) 1091 { 1092 KVMState *s = kvm_state; 1093 1094 if (s->coalesced_mmio) { 1095 struct kvm_coalesced_mmio_zone zone; 1096 1097 zone.addr = start; 1098 zone.size = size; 1099 zone.pad = 0; 1100 1101 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); 1102 } 1103 } 1104 1105 static void kvm_coalesce_pio_add(MemoryListener *listener, 1106 MemoryRegionSection *section, 1107 hwaddr start, hwaddr size) 1108 { 1109 KVMState *s = kvm_state; 1110 1111 if (s->coalesced_pio) { 1112 struct kvm_coalesced_mmio_zone zone; 1113 1114 zone.addr = start; 1115 zone.size = size; 1116 zone.pio = 1; 1117 1118 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); 1119 } 1120 } 1121 1122 static void kvm_coalesce_pio_del(MemoryListener *listener, 1123 MemoryRegionSection *section, 1124 hwaddr start, hwaddr size) 1125 { 1126 KVMState *s = kvm_state; 1127 1128 if (s->coalesced_pio) { 1129 struct kvm_coalesced_mmio_zone zone; 1130 1131 zone.addr = start; 1132 zone.size = size; 1133 zone.pio = 1; 1134 1135 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); 1136 } 1137 } 1138 1139 static MemoryListener kvm_coalesced_pio_listener = { 1140 .name = "kvm-coalesced-pio", 1141 .coalesced_io_add = kvm_coalesce_pio_add, 1142 .coalesced_io_del = kvm_coalesce_pio_del, 1143 }; 1144 1145 int kvm_check_extension(KVMState *s, unsigned int extension) 1146 { 1147 int ret; 1148 1149 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension); 1150 if (ret < 0) { 1151 ret = 0; 1152 } 1153 1154 return ret; 1155 } 1156 1157 int kvm_vm_check_extension(KVMState *s, unsigned int extension) 1158 { 1159 int ret; 1160 1161 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension); 1162 if (ret < 0) { 1163 /* VM wide version not implemented, use global one instead */ 1164 ret = kvm_check_extension(s, extension); 1165 } 1166 1167 return ret; 1168 } 1169 1170 typedef struct HWPoisonPage { 1171 ram_addr_t ram_addr; 1172 QLIST_ENTRY(HWPoisonPage) list; 1173 } HWPoisonPage; 1174 1175 static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list = 1176 QLIST_HEAD_INITIALIZER(hwpoison_page_list); 1177 1178 static void kvm_unpoison_all(void *param) 1179 { 1180 HWPoisonPage *page, *next_page; 1181 1182 QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) { 1183 QLIST_REMOVE(page, list); 1184 qemu_ram_remap(page->ram_addr, TARGET_PAGE_SIZE); 1185 g_free(page); 1186 } 1187 } 1188 1189 void kvm_hwpoison_page_add(ram_addr_t ram_addr) 1190 { 1191 HWPoisonPage *page; 1192 1193 QLIST_FOREACH(page, &hwpoison_page_list, list) { 1194 if (page->ram_addr == ram_addr) { 1195 return; 1196 } 1197 } 1198 page = g_new(HWPoisonPage, 1); 1199 page->ram_addr = ram_addr; 1200 QLIST_INSERT_HEAD(&hwpoison_page_list, page, list); 1201 } 1202 1203 static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size) 1204 { 1205 #if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN 1206 /* The kernel expects ioeventfd values in HOST_BIG_ENDIAN 1207 * endianness, but the memory core hands them in target endianness. 1208 * For example, PPC is always treated as big-endian even if running 1209 * on KVM and on PPC64LE. Correct here. 1210 */ 1211 switch (size) { 1212 case 2: 1213 val = bswap16(val); 1214 break; 1215 case 4: 1216 val = bswap32(val); 1217 break; 1218 } 1219 #endif 1220 return val; 1221 } 1222 1223 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val, 1224 bool assign, uint32_t size, bool datamatch) 1225 { 1226 int ret; 1227 struct kvm_ioeventfd iofd = { 1228 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 1229 .addr = addr, 1230 .len = size, 1231 .flags = 0, 1232 .fd = fd, 1233 }; 1234 1235 trace_kvm_set_ioeventfd_mmio(fd, (uint64_t)addr, val, assign, size, 1236 datamatch); 1237 if (!kvm_enabled()) { 1238 return -ENOSYS; 1239 } 1240 1241 if (datamatch) { 1242 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 1243 } 1244 if (!assign) { 1245 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 1246 } 1247 1248 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd); 1249 1250 if (ret < 0) { 1251 return -errno; 1252 } 1253 1254 return 0; 1255 } 1256 1257 static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val, 1258 bool assign, uint32_t size, bool datamatch) 1259 { 1260 struct kvm_ioeventfd kick = { 1261 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 1262 .addr = addr, 1263 .flags = KVM_IOEVENTFD_FLAG_PIO, 1264 .len = size, 1265 .fd = fd, 1266 }; 1267 int r; 1268 trace_kvm_set_ioeventfd_pio(fd, addr, val, assign, size, datamatch); 1269 if (!kvm_enabled()) { 1270 return -ENOSYS; 1271 } 1272 if (datamatch) { 1273 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 1274 } 1275 if (!assign) { 1276 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 1277 } 1278 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick); 1279 if (r < 0) { 1280 return r; 1281 } 1282 return 0; 1283 } 1284 1285 1286 static int kvm_check_many_ioeventfds(void) 1287 { 1288 /* Userspace can use ioeventfd for io notification. This requires a host 1289 * that supports eventfd(2) and an I/O thread; since eventfd does not 1290 * support SIGIO it cannot interrupt the vcpu. 1291 * 1292 * Older kernels have a 6 device limit on the KVM io bus. Find out so we 1293 * can avoid creating too many ioeventfds. 1294 */ 1295 #if defined(CONFIG_EVENTFD) 1296 int ioeventfds[7]; 1297 int i, ret = 0; 1298 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) { 1299 ioeventfds[i] = eventfd(0, EFD_CLOEXEC); 1300 if (ioeventfds[i] < 0) { 1301 break; 1302 } 1303 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true); 1304 if (ret < 0) { 1305 close(ioeventfds[i]); 1306 break; 1307 } 1308 } 1309 1310 /* Decide whether many devices are supported or not */ 1311 ret = i == ARRAY_SIZE(ioeventfds); 1312 1313 while (i-- > 0) { 1314 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true); 1315 close(ioeventfds[i]); 1316 } 1317 return ret; 1318 #else 1319 return 0; 1320 #endif 1321 } 1322 1323 static const KVMCapabilityInfo * 1324 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list) 1325 { 1326 while (list->name) { 1327 if (!kvm_check_extension(s, list->value)) { 1328 return list; 1329 } 1330 list++; 1331 } 1332 return NULL; 1333 } 1334 1335 void kvm_set_max_memslot_size(hwaddr max_slot_size) 1336 { 1337 g_assert( 1338 ROUND_UP(max_slot_size, qemu_real_host_page_size()) == max_slot_size 1339 ); 1340 kvm_max_slot_size = max_slot_size; 1341 } 1342 1343 static void kvm_set_phys_mem(KVMMemoryListener *kml, 1344 MemoryRegionSection *section, bool add) 1345 { 1346 KVMSlot *mem; 1347 int err; 1348 MemoryRegion *mr = section->mr; 1349 bool writeable = !mr->readonly && !mr->rom_device; 1350 hwaddr start_addr, size, slot_size, mr_offset; 1351 ram_addr_t ram_start_offset; 1352 void *ram; 1353 1354 if (!memory_region_is_ram(mr)) { 1355 if (writeable || !kvm_readonly_mem_allowed) { 1356 return; 1357 } else if (!mr->romd_mode) { 1358 /* If the memory device is not in romd_mode, then we actually want 1359 * to remove the kvm memory slot so all accesses will trap. */ 1360 add = false; 1361 } 1362 } 1363 1364 size = kvm_align_section(section, &start_addr); 1365 if (!size) { 1366 return; 1367 } 1368 1369 /* The offset of the kvmslot within the memory region */ 1370 mr_offset = section->offset_within_region + start_addr - 1371 section->offset_within_address_space; 1372 1373 /* use aligned delta to align the ram address and offset */ 1374 ram = memory_region_get_ram_ptr(mr) + mr_offset; 1375 ram_start_offset = memory_region_get_ram_addr(mr) + mr_offset; 1376 1377 kvm_slots_lock(); 1378 1379 if (!add) { 1380 do { 1381 slot_size = MIN(kvm_max_slot_size, size); 1382 mem = kvm_lookup_matching_slot(kml, start_addr, slot_size); 1383 if (!mem) { 1384 goto out; 1385 } 1386 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1387 /* 1388 * NOTE: We should be aware of the fact that here we're only 1389 * doing a best effort to sync dirty bits. No matter whether 1390 * we're using dirty log or dirty ring, we ignored two facts: 1391 * 1392 * (1) dirty bits can reside in hardware buffers (PML) 1393 * 1394 * (2) after we collected dirty bits here, pages can be dirtied 1395 * again before we do the final KVM_SET_USER_MEMORY_REGION to 1396 * remove the slot. 1397 * 1398 * Not easy. Let's cross the fingers until it's fixed. 1399 */ 1400 if (kvm_state->kvm_dirty_ring_size) { 1401 kvm_dirty_ring_reap_locked(kvm_state); 1402 } else { 1403 kvm_slot_get_dirty_log(kvm_state, mem); 1404 } 1405 kvm_slot_sync_dirty_pages(mem); 1406 } 1407 1408 /* unregister the slot */ 1409 g_free(mem->dirty_bmap); 1410 mem->dirty_bmap = NULL; 1411 mem->memory_size = 0; 1412 mem->flags = 0; 1413 err = kvm_set_user_memory_region(kml, mem, false); 1414 if (err) { 1415 fprintf(stderr, "%s: error unregistering slot: %s\n", 1416 __func__, strerror(-err)); 1417 abort(); 1418 } 1419 start_addr += slot_size; 1420 size -= slot_size; 1421 } while (size); 1422 goto out; 1423 } 1424 1425 /* register the new slot */ 1426 do { 1427 slot_size = MIN(kvm_max_slot_size, size); 1428 mem = kvm_alloc_slot(kml); 1429 mem->as_id = kml->as_id; 1430 mem->memory_size = slot_size; 1431 mem->start_addr = start_addr; 1432 mem->ram_start_offset = ram_start_offset; 1433 mem->ram = ram; 1434 mem->flags = kvm_mem_flags(mr); 1435 kvm_slot_init_dirty_bitmap(mem); 1436 err = kvm_set_user_memory_region(kml, mem, true); 1437 if (err) { 1438 fprintf(stderr, "%s: error registering slot: %s\n", __func__, 1439 strerror(-err)); 1440 abort(); 1441 } 1442 start_addr += slot_size; 1443 ram_start_offset += slot_size; 1444 ram += slot_size; 1445 size -= slot_size; 1446 } while (size); 1447 1448 out: 1449 kvm_slots_unlock(); 1450 } 1451 1452 static void *kvm_dirty_ring_reaper_thread(void *data) 1453 { 1454 KVMState *s = data; 1455 struct KVMDirtyRingReaper *r = &s->reaper; 1456 1457 rcu_register_thread(); 1458 1459 trace_kvm_dirty_ring_reaper("init"); 1460 1461 while (true) { 1462 r->reaper_state = KVM_DIRTY_RING_REAPER_WAIT; 1463 trace_kvm_dirty_ring_reaper("wait"); 1464 /* 1465 * TODO: provide a smarter timeout rather than a constant? 1466 */ 1467 sleep(1); 1468 1469 trace_kvm_dirty_ring_reaper("wakeup"); 1470 r->reaper_state = KVM_DIRTY_RING_REAPER_REAPING; 1471 1472 qemu_mutex_lock_iothread(); 1473 kvm_dirty_ring_reap(s); 1474 qemu_mutex_unlock_iothread(); 1475 1476 r->reaper_iteration++; 1477 } 1478 1479 trace_kvm_dirty_ring_reaper("exit"); 1480 1481 rcu_unregister_thread(); 1482 1483 return NULL; 1484 } 1485 1486 static int kvm_dirty_ring_reaper_init(KVMState *s) 1487 { 1488 struct KVMDirtyRingReaper *r = &s->reaper; 1489 1490 qemu_thread_create(&r->reaper_thr, "kvm-reaper", 1491 kvm_dirty_ring_reaper_thread, 1492 s, QEMU_THREAD_JOINABLE); 1493 1494 return 0; 1495 } 1496 1497 static void kvm_region_add(MemoryListener *listener, 1498 MemoryRegionSection *section) 1499 { 1500 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1501 1502 memory_region_ref(section->mr); 1503 kvm_set_phys_mem(kml, section, true); 1504 } 1505 1506 static void kvm_region_del(MemoryListener *listener, 1507 MemoryRegionSection *section) 1508 { 1509 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1510 1511 kvm_set_phys_mem(kml, section, false); 1512 memory_region_unref(section->mr); 1513 } 1514 1515 static void kvm_log_sync(MemoryListener *listener, 1516 MemoryRegionSection *section) 1517 { 1518 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1519 1520 kvm_slots_lock(); 1521 kvm_physical_sync_dirty_bitmap(kml, section); 1522 kvm_slots_unlock(); 1523 } 1524 1525 static void kvm_log_sync_global(MemoryListener *l) 1526 { 1527 KVMMemoryListener *kml = container_of(l, KVMMemoryListener, listener); 1528 KVMState *s = kvm_state; 1529 KVMSlot *mem; 1530 int i; 1531 1532 /* Flush all kernel dirty addresses into KVMSlot dirty bitmap */ 1533 kvm_dirty_ring_flush(); 1534 1535 /* 1536 * TODO: make this faster when nr_slots is big while there are 1537 * only a few used slots (small VMs). 1538 */ 1539 kvm_slots_lock(); 1540 for (i = 0; i < s->nr_slots; i++) { 1541 mem = &kml->slots[i]; 1542 if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 1543 kvm_slot_sync_dirty_pages(mem); 1544 /* 1545 * This is not needed by KVM_GET_DIRTY_LOG because the 1546 * ioctl will unconditionally overwrite the whole region. 1547 * However kvm dirty ring has no such side effect. 1548 */ 1549 kvm_slot_reset_dirty_pages(mem); 1550 } 1551 } 1552 kvm_slots_unlock(); 1553 } 1554 1555 static void kvm_log_clear(MemoryListener *listener, 1556 MemoryRegionSection *section) 1557 { 1558 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 1559 int r; 1560 1561 r = kvm_physical_log_clear(kml, section); 1562 if (r < 0) { 1563 error_report_once("%s: kvm log clear failed: mr=%s " 1564 "offset=%"HWADDR_PRIx" size=%"PRIx64, __func__, 1565 section->mr->name, section->offset_within_region, 1566 int128_get64(section->size)); 1567 abort(); 1568 } 1569 } 1570 1571 static void kvm_mem_ioeventfd_add(MemoryListener *listener, 1572 MemoryRegionSection *section, 1573 bool match_data, uint64_t data, 1574 EventNotifier *e) 1575 { 1576 int fd = event_notifier_get_fd(e); 1577 int r; 1578 1579 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 1580 data, true, int128_get64(section->size), 1581 match_data); 1582 if (r < 0) { 1583 fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", 1584 __func__, strerror(-r), -r); 1585 abort(); 1586 } 1587 } 1588 1589 static void kvm_mem_ioeventfd_del(MemoryListener *listener, 1590 MemoryRegionSection *section, 1591 bool match_data, uint64_t data, 1592 EventNotifier *e) 1593 { 1594 int fd = event_notifier_get_fd(e); 1595 int r; 1596 1597 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 1598 data, false, int128_get64(section->size), 1599 match_data); 1600 if (r < 0) { 1601 fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", 1602 __func__, strerror(-r), -r); 1603 abort(); 1604 } 1605 } 1606 1607 static void kvm_io_ioeventfd_add(MemoryListener *listener, 1608 MemoryRegionSection *section, 1609 bool match_data, uint64_t data, 1610 EventNotifier *e) 1611 { 1612 int fd = event_notifier_get_fd(e); 1613 int r; 1614 1615 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 1616 data, true, int128_get64(section->size), 1617 match_data); 1618 if (r < 0) { 1619 fprintf(stderr, "%s: error adding ioeventfd: %s (%d)\n", 1620 __func__, strerror(-r), -r); 1621 abort(); 1622 } 1623 } 1624 1625 static void kvm_io_ioeventfd_del(MemoryListener *listener, 1626 MemoryRegionSection *section, 1627 bool match_data, uint64_t data, 1628 EventNotifier *e) 1629 1630 { 1631 int fd = event_notifier_get_fd(e); 1632 int r; 1633 1634 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 1635 data, false, int128_get64(section->size), 1636 match_data); 1637 if (r < 0) { 1638 fprintf(stderr, "%s: error deleting ioeventfd: %s (%d)\n", 1639 __func__, strerror(-r), -r); 1640 abort(); 1641 } 1642 } 1643 1644 void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, 1645 AddressSpace *as, int as_id, const char *name) 1646 { 1647 int i; 1648 1649 kml->slots = g_new0(KVMSlot, s->nr_slots); 1650 kml->as_id = as_id; 1651 1652 for (i = 0; i < s->nr_slots; i++) { 1653 kml->slots[i].slot = i; 1654 } 1655 1656 kml->listener.region_add = kvm_region_add; 1657 kml->listener.region_del = kvm_region_del; 1658 kml->listener.log_start = kvm_log_start; 1659 kml->listener.log_stop = kvm_log_stop; 1660 kml->listener.priority = 10; 1661 kml->listener.name = name; 1662 1663 if (s->kvm_dirty_ring_size) { 1664 kml->listener.log_sync_global = kvm_log_sync_global; 1665 } else { 1666 kml->listener.log_sync = kvm_log_sync; 1667 kml->listener.log_clear = kvm_log_clear; 1668 } 1669 1670 memory_listener_register(&kml->listener, as); 1671 1672 for (i = 0; i < s->nr_as; ++i) { 1673 if (!s->as[i].as) { 1674 s->as[i].as = as; 1675 s->as[i].ml = kml; 1676 break; 1677 } 1678 } 1679 } 1680 1681 static MemoryListener kvm_io_listener = { 1682 .name = "kvm-io", 1683 .eventfd_add = kvm_io_ioeventfd_add, 1684 .eventfd_del = kvm_io_ioeventfd_del, 1685 .priority = 10, 1686 }; 1687 1688 int kvm_set_irq(KVMState *s, int irq, int level) 1689 { 1690 struct kvm_irq_level event; 1691 int ret; 1692 1693 assert(kvm_async_interrupts_enabled()); 1694 1695 event.level = level; 1696 event.irq = irq; 1697 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event); 1698 if (ret < 0) { 1699 perror("kvm_set_irq"); 1700 abort(); 1701 } 1702 1703 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status; 1704 } 1705 1706 #ifdef KVM_CAP_IRQ_ROUTING 1707 typedef struct KVMMSIRoute { 1708 struct kvm_irq_routing_entry kroute; 1709 QTAILQ_ENTRY(KVMMSIRoute) entry; 1710 } KVMMSIRoute; 1711 1712 static void set_gsi(KVMState *s, unsigned int gsi) 1713 { 1714 set_bit(gsi, s->used_gsi_bitmap); 1715 } 1716 1717 static void clear_gsi(KVMState *s, unsigned int gsi) 1718 { 1719 clear_bit(gsi, s->used_gsi_bitmap); 1720 } 1721 1722 void kvm_init_irq_routing(KVMState *s) 1723 { 1724 int gsi_count, i; 1725 1726 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1; 1727 if (gsi_count > 0) { 1728 /* Round up so we can search ints using ffs */ 1729 s->used_gsi_bitmap = bitmap_new(gsi_count); 1730 s->gsi_count = gsi_count; 1731 } 1732 1733 s->irq_routes = g_malloc0(sizeof(*s->irq_routes)); 1734 s->nr_allocated_irq_routes = 0; 1735 1736 if (!kvm_direct_msi_allowed) { 1737 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) { 1738 QTAILQ_INIT(&s->msi_hashtab[i]); 1739 } 1740 } 1741 1742 kvm_arch_init_irq_routing(s); 1743 } 1744 1745 void kvm_irqchip_commit_routes(KVMState *s) 1746 { 1747 int ret; 1748 1749 if (kvm_gsi_direct_mapping()) { 1750 return; 1751 } 1752 1753 if (!kvm_gsi_routing_enabled()) { 1754 return; 1755 } 1756 1757 s->irq_routes->flags = 0; 1758 trace_kvm_irqchip_commit_routes(); 1759 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes); 1760 assert(ret == 0); 1761 } 1762 1763 static void kvm_add_routing_entry(KVMState *s, 1764 struct kvm_irq_routing_entry *entry) 1765 { 1766 struct kvm_irq_routing_entry *new; 1767 int n, size; 1768 1769 if (s->irq_routes->nr == s->nr_allocated_irq_routes) { 1770 n = s->nr_allocated_irq_routes * 2; 1771 if (n < 64) { 1772 n = 64; 1773 } 1774 size = sizeof(struct kvm_irq_routing); 1775 size += n * sizeof(*new); 1776 s->irq_routes = g_realloc(s->irq_routes, size); 1777 s->nr_allocated_irq_routes = n; 1778 } 1779 n = s->irq_routes->nr++; 1780 new = &s->irq_routes->entries[n]; 1781 1782 *new = *entry; 1783 1784 set_gsi(s, entry->gsi); 1785 } 1786 1787 static int kvm_update_routing_entry(KVMState *s, 1788 struct kvm_irq_routing_entry *new_entry) 1789 { 1790 struct kvm_irq_routing_entry *entry; 1791 int n; 1792 1793 for (n = 0; n < s->irq_routes->nr; n++) { 1794 entry = &s->irq_routes->entries[n]; 1795 if (entry->gsi != new_entry->gsi) { 1796 continue; 1797 } 1798 1799 if(!memcmp(entry, new_entry, sizeof *entry)) { 1800 return 0; 1801 } 1802 1803 *entry = *new_entry; 1804 1805 return 0; 1806 } 1807 1808 return -ESRCH; 1809 } 1810 1811 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin) 1812 { 1813 struct kvm_irq_routing_entry e = {}; 1814 1815 assert(pin < s->gsi_count); 1816 1817 e.gsi = irq; 1818 e.type = KVM_IRQ_ROUTING_IRQCHIP; 1819 e.flags = 0; 1820 e.u.irqchip.irqchip = irqchip; 1821 e.u.irqchip.pin = pin; 1822 kvm_add_routing_entry(s, &e); 1823 } 1824 1825 void kvm_irqchip_release_virq(KVMState *s, int virq) 1826 { 1827 struct kvm_irq_routing_entry *e; 1828 int i; 1829 1830 if (kvm_gsi_direct_mapping()) { 1831 return; 1832 } 1833 1834 for (i = 0; i < s->irq_routes->nr; i++) { 1835 e = &s->irq_routes->entries[i]; 1836 if (e->gsi == virq) { 1837 s->irq_routes->nr--; 1838 *e = s->irq_routes->entries[s->irq_routes->nr]; 1839 } 1840 } 1841 clear_gsi(s, virq); 1842 kvm_arch_release_virq_post(virq); 1843 trace_kvm_irqchip_release_virq(virq); 1844 } 1845 1846 void kvm_irqchip_add_change_notifier(Notifier *n) 1847 { 1848 notifier_list_add(&kvm_irqchip_change_notifiers, n); 1849 } 1850 1851 void kvm_irqchip_remove_change_notifier(Notifier *n) 1852 { 1853 notifier_remove(n); 1854 } 1855 1856 void kvm_irqchip_change_notify(void) 1857 { 1858 notifier_list_notify(&kvm_irqchip_change_notifiers, NULL); 1859 } 1860 1861 static unsigned int kvm_hash_msi(uint32_t data) 1862 { 1863 /* This is optimized for IA32 MSI layout. However, no other arch shall 1864 * repeat the mistake of not providing a direct MSI injection API. */ 1865 return data & 0xff; 1866 } 1867 1868 static void kvm_flush_dynamic_msi_routes(KVMState *s) 1869 { 1870 KVMMSIRoute *route, *next; 1871 unsigned int hash; 1872 1873 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) { 1874 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) { 1875 kvm_irqchip_release_virq(s, route->kroute.gsi); 1876 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry); 1877 g_free(route); 1878 } 1879 } 1880 } 1881 1882 static int kvm_irqchip_get_virq(KVMState *s) 1883 { 1884 int next_virq; 1885 1886 /* 1887 * PIC and IOAPIC share the first 16 GSI numbers, thus the available 1888 * GSI numbers are more than the number of IRQ route. Allocating a GSI 1889 * number can succeed even though a new route entry cannot be added. 1890 * When this happens, flush dynamic MSI entries to free IRQ route entries. 1891 */ 1892 if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) { 1893 kvm_flush_dynamic_msi_routes(s); 1894 } 1895 1896 /* Return the lowest unused GSI in the bitmap */ 1897 next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count); 1898 if (next_virq >= s->gsi_count) { 1899 return -ENOSPC; 1900 } else { 1901 return next_virq; 1902 } 1903 } 1904 1905 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg) 1906 { 1907 unsigned int hash = kvm_hash_msi(msg.data); 1908 KVMMSIRoute *route; 1909 1910 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) { 1911 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address && 1912 route->kroute.u.msi.address_hi == (msg.address >> 32) && 1913 route->kroute.u.msi.data == le32_to_cpu(msg.data)) { 1914 return route; 1915 } 1916 } 1917 return NULL; 1918 } 1919 1920 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 1921 { 1922 struct kvm_msi msi; 1923 KVMMSIRoute *route; 1924 1925 if (kvm_direct_msi_allowed) { 1926 msi.address_lo = (uint32_t)msg.address; 1927 msi.address_hi = msg.address >> 32; 1928 msi.data = le32_to_cpu(msg.data); 1929 msi.flags = 0; 1930 memset(msi.pad, 0, sizeof(msi.pad)); 1931 1932 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi); 1933 } 1934 1935 route = kvm_lookup_msi_route(s, msg); 1936 if (!route) { 1937 int virq; 1938 1939 virq = kvm_irqchip_get_virq(s); 1940 if (virq < 0) { 1941 return virq; 1942 } 1943 1944 route = g_new0(KVMMSIRoute, 1); 1945 route->kroute.gsi = virq; 1946 route->kroute.type = KVM_IRQ_ROUTING_MSI; 1947 route->kroute.flags = 0; 1948 route->kroute.u.msi.address_lo = (uint32_t)msg.address; 1949 route->kroute.u.msi.address_hi = msg.address >> 32; 1950 route->kroute.u.msi.data = le32_to_cpu(msg.data); 1951 1952 kvm_add_routing_entry(s, &route->kroute); 1953 kvm_irqchip_commit_routes(s); 1954 1955 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route, 1956 entry); 1957 } 1958 1959 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI); 1960 1961 return kvm_set_irq(s, route->kroute.gsi, 1); 1962 } 1963 1964 int kvm_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *dev) 1965 { 1966 struct kvm_irq_routing_entry kroute = {}; 1967 int virq; 1968 KVMState *s = c->s; 1969 MSIMessage msg = {0, 0}; 1970 1971 if (pci_available && dev) { 1972 msg = pci_get_msi_message(dev, vector); 1973 } 1974 1975 if (kvm_gsi_direct_mapping()) { 1976 return kvm_arch_msi_data_to_gsi(msg.data); 1977 } 1978 1979 if (!kvm_gsi_routing_enabled()) { 1980 return -ENOSYS; 1981 } 1982 1983 virq = kvm_irqchip_get_virq(s); 1984 if (virq < 0) { 1985 return virq; 1986 } 1987 1988 kroute.gsi = virq; 1989 kroute.type = KVM_IRQ_ROUTING_MSI; 1990 kroute.flags = 0; 1991 kroute.u.msi.address_lo = (uint32_t)msg.address; 1992 kroute.u.msi.address_hi = msg.address >> 32; 1993 kroute.u.msi.data = le32_to_cpu(msg.data); 1994 if (pci_available && kvm_msi_devid_required()) { 1995 kroute.flags = KVM_MSI_VALID_DEVID; 1996 kroute.u.msi.devid = pci_requester_id(dev); 1997 } 1998 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 1999 kvm_irqchip_release_virq(s, virq); 2000 return -EINVAL; 2001 } 2002 2003 trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A", 2004 vector, virq); 2005 2006 kvm_add_routing_entry(s, &kroute); 2007 kvm_arch_add_msi_route_post(&kroute, vector, dev); 2008 c->changes++; 2009 2010 return virq; 2011 } 2012 2013 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, 2014 PCIDevice *dev) 2015 { 2016 struct kvm_irq_routing_entry kroute = {}; 2017 2018 if (kvm_gsi_direct_mapping()) { 2019 return 0; 2020 } 2021 2022 if (!kvm_irqchip_in_kernel()) { 2023 return -ENOSYS; 2024 } 2025 2026 kroute.gsi = virq; 2027 kroute.type = KVM_IRQ_ROUTING_MSI; 2028 kroute.flags = 0; 2029 kroute.u.msi.address_lo = (uint32_t)msg.address; 2030 kroute.u.msi.address_hi = msg.address >> 32; 2031 kroute.u.msi.data = le32_to_cpu(msg.data); 2032 if (pci_available && kvm_msi_devid_required()) { 2033 kroute.flags = KVM_MSI_VALID_DEVID; 2034 kroute.u.msi.devid = pci_requester_id(dev); 2035 } 2036 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 2037 return -EINVAL; 2038 } 2039 2040 trace_kvm_irqchip_update_msi_route(virq); 2041 2042 return kvm_update_routing_entry(s, &kroute); 2043 } 2044 2045 static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, 2046 EventNotifier *resample, int virq, 2047 bool assign) 2048 { 2049 int fd = event_notifier_get_fd(event); 2050 int rfd = resample ? event_notifier_get_fd(resample) : -1; 2051 2052 struct kvm_irqfd irqfd = { 2053 .fd = fd, 2054 .gsi = virq, 2055 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN, 2056 }; 2057 2058 if (rfd != -1) { 2059 assert(assign); 2060 if (kvm_irqchip_is_split()) { 2061 /* 2062 * When the slow irqchip (e.g. IOAPIC) is in the 2063 * userspace, KVM kernel resamplefd will not work because 2064 * the EOI of the interrupt will be delivered to userspace 2065 * instead, so the KVM kernel resamplefd kick will be 2066 * skipped. The userspace here mimics what the kernel 2067 * provides with resamplefd, remember the resamplefd and 2068 * kick it when we receive EOI of this IRQ. 2069 * 2070 * This is hackery because IOAPIC is mostly bypassed 2071 * (except EOI broadcasts) when irqfd is used. However 2072 * this can bring much performance back for split irqchip 2073 * with INTx IRQs (for VFIO, this gives 93% perf of the 2074 * full fast path, which is 46% perf boost comparing to 2075 * the INTx slow path). 2076 */ 2077 kvm_resample_fd_insert(virq, resample); 2078 } else { 2079 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE; 2080 irqfd.resamplefd = rfd; 2081 } 2082 } else if (!assign) { 2083 if (kvm_irqchip_is_split()) { 2084 kvm_resample_fd_remove(virq); 2085 } 2086 } 2087 2088 if (!kvm_irqfds_enabled()) { 2089 return -ENOSYS; 2090 } 2091 2092 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd); 2093 } 2094 2095 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) 2096 { 2097 struct kvm_irq_routing_entry kroute = {}; 2098 int virq; 2099 2100 if (!kvm_gsi_routing_enabled()) { 2101 return -ENOSYS; 2102 } 2103 2104 virq = kvm_irqchip_get_virq(s); 2105 if (virq < 0) { 2106 return virq; 2107 } 2108 2109 kroute.gsi = virq; 2110 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER; 2111 kroute.flags = 0; 2112 kroute.u.adapter.summary_addr = adapter->summary_addr; 2113 kroute.u.adapter.ind_addr = adapter->ind_addr; 2114 kroute.u.adapter.summary_offset = adapter->summary_offset; 2115 kroute.u.adapter.ind_offset = adapter->ind_offset; 2116 kroute.u.adapter.adapter_id = adapter->adapter_id; 2117 2118 kvm_add_routing_entry(s, &kroute); 2119 2120 return virq; 2121 } 2122 2123 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) 2124 { 2125 struct kvm_irq_routing_entry kroute = {}; 2126 int virq; 2127 2128 if (!kvm_gsi_routing_enabled()) { 2129 return -ENOSYS; 2130 } 2131 if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) { 2132 return -ENOSYS; 2133 } 2134 virq = kvm_irqchip_get_virq(s); 2135 if (virq < 0) { 2136 return virq; 2137 } 2138 2139 kroute.gsi = virq; 2140 kroute.type = KVM_IRQ_ROUTING_HV_SINT; 2141 kroute.flags = 0; 2142 kroute.u.hv_sint.vcpu = vcpu; 2143 kroute.u.hv_sint.sint = sint; 2144 2145 kvm_add_routing_entry(s, &kroute); 2146 kvm_irqchip_commit_routes(s); 2147 2148 return virq; 2149 } 2150 2151 #else /* !KVM_CAP_IRQ_ROUTING */ 2152 2153 void kvm_init_irq_routing(KVMState *s) 2154 { 2155 } 2156 2157 void kvm_irqchip_release_virq(KVMState *s, int virq) 2158 { 2159 } 2160 2161 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 2162 { 2163 abort(); 2164 } 2165 2166 int kvm_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *dev) 2167 { 2168 return -ENOSYS; 2169 } 2170 2171 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) 2172 { 2173 return -ENOSYS; 2174 } 2175 2176 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) 2177 { 2178 return -ENOSYS; 2179 } 2180 2181 static int kvm_irqchip_assign_irqfd(KVMState *s, EventNotifier *event, 2182 EventNotifier *resample, int virq, 2183 bool assign) 2184 { 2185 abort(); 2186 } 2187 2188 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg) 2189 { 2190 return -ENOSYS; 2191 } 2192 #endif /* !KVM_CAP_IRQ_ROUTING */ 2193 2194 int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 2195 EventNotifier *rn, int virq) 2196 { 2197 return kvm_irqchip_assign_irqfd(s, n, rn, virq, true); 2198 } 2199 2200 int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 2201 int virq) 2202 { 2203 return kvm_irqchip_assign_irqfd(s, n, NULL, virq, false); 2204 } 2205 2206 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, 2207 EventNotifier *rn, qemu_irq irq) 2208 { 2209 gpointer key, gsi; 2210 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 2211 2212 if (!found) { 2213 return -ENXIO; 2214 } 2215 return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi)); 2216 } 2217 2218 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, 2219 qemu_irq irq) 2220 { 2221 gpointer key, gsi; 2222 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 2223 2224 if (!found) { 2225 return -ENXIO; 2226 } 2227 return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi)); 2228 } 2229 2230 void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi) 2231 { 2232 g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi)); 2233 } 2234 2235 static void kvm_irqchip_create(KVMState *s) 2236 { 2237 int ret; 2238 2239 assert(s->kernel_irqchip_split != ON_OFF_AUTO_AUTO); 2240 if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) { 2241 ; 2242 } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) { 2243 ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0); 2244 if (ret < 0) { 2245 fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret)); 2246 exit(1); 2247 } 2248 } else { 2249 return; 2250 } 2251 2252 /* First probe and see if there's a arch-specific hook to create the 2253 * in-kernel irqchip for us */ 2254 ret = kvm_arch_irqchip_create(s); 2255 if (ret == 0) { 2256 if (s->kernel_irqchip_split == ON_OFF_AUTO_ON) { 2257 perror("Split IRQ chip mode not supported."); 2258 exit(1); 2259 } else { 2260 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP); 2261 } 2262 } 2263 if (ret < 0) { 2264 fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret)); 2265 exit(1); 2266 } 2267 2268 kvm_kernel_irqchip = true; 2269 /* If we have an in-kernel IRQ chip then we must have asynchronous 2270 * interrupt delivery (though the reverse is not necessarily true) 2271 */ 2272 kvm_async_interrupts_allowed = true; 2273 kvm_halt_in_kernel_allowed = true; 2274 2275 kvm_init_irq_routing(s); 2276 2277 s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal); 2278 } 2279 2280 /* Find number of supported CPUs using the recommended 2281 * procedure from the kernel API documentation to cope with 2282 * older kernels that may be missing capabilities. 2283 */ 2284 static int kvm_recommended_vcpus(KVMState *s) 2285 { 2286 int ret = kvm_vm_check_extension(s, KVM_CAP_NR_VCPUS); 2287 return (ret) ? ret : 4; 2288 } 2289 2290 static int kvm_max_vcpus(KVMState *s) 2291 { 2292 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS); 2293 return (ret) ? ret : kvm_recommended_vcpus(s); 2294 } 2295 2296 static int kvm_max_vcpu_id(KVMState *s) 2297 { 2298 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID); 2299 return (ret) ? ret : kvm_max_vcpus(s); 2300 } 2301 2302 bool kvm_vcpu_id_is_valid(int vcpu_id) 2303 { 2304 KVMState *s = KVM_STATE(current_accel()); 2305 return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s); 2306 } 2307 2308 bool kvm_dirty_ring_enabled(void) 2309 { 2310 return kvm_state->kvm_dirty_ring_size ? true : false; 2311 } 2312 2313 static int kvm_init(MachineState *ms) 2314 { 2315 MachineClass *mc = MACHINE_GET_CLASS(ms); 2316 static const char upgrade_note[] = 2317 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n" 2318 "(see http://sourceforge.net/projects/kvm).\n"; 2319 struct { 2320 const char *name; 2321 int num; 2322 } num_cpus[] = { 2323 { "SMP", ms->smp.cpus }, 2324 { "hotpluggable", ms->smp.max_cpus }, 2325 { NULL, } 2326 }, *nc = num_cpus; 2327 int soft_vcpus_limit, hard_vcpus_limit; 2328 KVMState *s; 2329 const KVMCapabilityInfo *missing_cap; 2330 int ret; 2331 int type = 0; 2332 uint64_t dirty_log_manual_caps; 2333 2334 qemu_mutex_init(&kml_slots_lock); 2335 2336 s = KVM_STATE(ms->accelerator); 2337 2338 /* 2339 * On systems where the kernel can support different base page 2340 * sizes, host page size may be different from TARGET_PAGE_SIZE, 2341 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum 2342 * page size for the system though. 2343 */ 2344 assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size()); 2345 2346 s->sigmask_len = 8; 2347 2348 #ifdef KVM_CAP_SET_GUEST_DEBUG 2349 QTAILQ_INIT(&s->kvm_sw_breakpoints); 2350 #endif 2351 QLIST_INIT(&s->kvm_parked_vcpus); 2352 s->fd = qemu_open_old("/dev/kvm", O_RDWR); 2353 if (s->fd == -1) { 2354 fprintf(stderr, "Could not access KVM kernel module: %m\n"); 2355 ret = -errno; 2356 goto err; 2357 } 2358 2359 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0); 2360 if (ret < KVM_API_VERSION) { 2361 if (ret >= 0) { 2362 ret = -EINVAL; 2363 } 2364 fprintf(stderr, "kvm version too old\n"); 2365 goto err; 2366 } 2367 2368 if (ret > KVM_API_VERSION) { 2369 ret = -EINVAL; 2370 fprintf(stderr, "kvm version not supported\n"); 2371 goto err; 2372 } 2373 2374 kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT); 2375 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS); 2376 2377 /* If unspecified, use the default value */ 2378 if (!s->nr_slots) { 2379 s->nr_slots = 32; 2380 } 2381 2382 s->nr_as = kvm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE); 2383 if (s->nr_as <= 1) { 2384 s->nr_as = 1; 2385 } 2386 s->as = g_new0(struct KVMAs, s->nr_as); 2387 2388 if (object_property_find(OBJECT(current_machine), "kvm-type")) { 2389 g_autofree char *kvm_type = object_property_get_str(OBJECT(current_machine), 2390 "kvm-type", 2391 &error_abort); 2392 type = mc->kvm_type(ms, kvm_type); 2393 } else if (mc->kvm_type) { 2394 type = mc->kvm_type(ms, NULL); 2395 } 2396 2397 do { 2398 ret = kvm_ioctl(s, KVM_CREATE_VM, type); 2399 } while (ret == -EINTR); 2400 2401 if (ret < 0) { 2402 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret, 2403 strerror(-ret)); 2404 2405 #ifdef TARGET_S390X 2406 if (ret == -EINVAL) { 2407 fprintf(stderr, 2408 "Host kernel setup problem detected. Please verify:\n"); 2409 fprintf(stderr, "- for kernels supporting the switch_amode or" 2410 " user_mode parameters, whether\n"); 2411 fprintf(stderr, 2412 " user space is running in primary address space\n"); 2413 fprintf(stderr, 2414 "- for kernels supporting the vm.allocate_pgste sysctl, " 2415 "whether it is enabled\n"); 2416 } 2417 #elif defined(TARGET_PPC) 2418 if (ret == -EINVAL) { 2419 fprintf(stderr, 2420 "PPC KVM module is not loaded. Try modprobe kvm_%s.\n", 2421 (type == 2) ? "pr" : "hv"); 2422 } 2423 #endif 2424 goto err; 2425 } 2426 2427 s->vmfd = ret; 2428 2429 /* check the vcpu limits */ 2430 soft_vcpus_limit = kvm_recommended_vcpus(s); 2431 hard_vcpus_limit = kvm_max_vcpus(s); 2432 2433 while (nc->name) { 2434 if (nc->num > soft_vcpus_limit) { 2435 warn_report("Number of %s cpus requested (%d) exceeds " 2436 "the recommended cpus supported by KVM (%d)", 2437 nc->name, nc->num, soft_vcpus_limit); 2438 2439 if (nc->num > hard_vcpus_limit) { 2440 fprintf(stderr, "Number of %s cpus requested (%d) exceeds " 2441 "the maximum cpus supported by KVM (%d)\n", 2442 nc->name, nc->num, hard_vcpus_limit); 2443 exit(1); 2444 } 2445 } 2446 nc++; 2447 } 2448 2449 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites); 2450 if (!missing_cap) { 2451 missing_cap = 2452 kvm_check_extension_list(s, kvm_arch_required_capabilities); 2453 } 2454 if (missing_cap) { 2455 ret = -EINVAL; 2456 fprintf(stderr, "kvm does not support %s\n%s", 2457 missing_cap->name, upgrade_note); 2458 goto err; 2459 } 2460 2461 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO); 2462 s->coalesced_pio = s->coalesced_mmio && 2463 kvm_check_extension(s, KVM_CAP_COALESCED_PIO); 2464 2465 /* 2466 * Enable KVM dirty ring if supported, otherwise fall back to 2467 * dirty logging mode 2468 */ 2469 if (s->kvm_dirty_ring_size > 0) { 2470 uint64_t ring_bytes; 2471 2472 ring_bytes = s->kvm_dirty_ring_size * sizeof(struct kvm_dirty_gfn); 2473 2474 /* Read the max supported pages */ 2475 ret = kvm_vm_check_extension(s, KVM_CAP_DIRTY_LOG_RING); 2476 if (ret > 0) { 2477 if (ring_bytes > ret) { 2478 error_report("KVM dirty ring size %" PRIu32 " too big " 2479 "(maximum is %ld). Please use a smaller value.", 2480 s->kvm_dirty_ring_size, 2481 (long)ret / sizeof(struct kvm_dirty_gfn)); 2482 ret = -EINVAL; 2483 goto err; 2484 } 2485 2486 ret = kvm_vm_enable_cap(s, KVM_CAP_DIRTY_LOG_RING, 0, ring_bytes); 2487 if (ret) { 2488 error_report("Enabling of KVM dirty ring failed: %s. " 2489 "Suggested minimum value is 1024.", strerror(-ret)); 2490 goto err; 2491 } 2492 2493 s->kvm_dirty_ring_bytes = ring_bytes; 2494 } else { 2495 warn_report("KVM dirty ring not available, using bitmap method"); 2496 s->kvm_dirty_ring_size = 0; 2497 } 2498 } 2499 2500 /* 2501 * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 is not needed when dirty ring is 2502 * enabled. More importantly, KVM_DIRTY_LOG_INITIALLY_SET will assume no 2503 * page is wr-protected initially, which is against how kvm dirty ring is 2504 * usage - kvm dirty ring requires all pages are wr-protected at the very 2505 * beginning. Enabling this feature for dirty ring causes data corruption. 2506 * 2507 * TODO: Without KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and kvm clear dirty log, 2508 * we may expect a higher stall time when starting the migration. In the 2509 * future we can enable KVM_CLEAR_DIRTY_LOG to work with dirty ring too: 2510 * instead of clearing dirty bit, it can be a way to explicitly wr-protect 2511 * guest pages. 2512 */ 2513 if (!s->kvm_dirty_ring_size) { 2514 dirty_log_manual_caps = 2515 kvm_check_extension(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); 2516 dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | 2517 KVM_DIRTY_LOG_INITIALLY_SET); 2518 s->manual_dirty_log_protect = dirty_log_manual_caps; 2519 if (dirty_log_manual_caps) { 2520 ret = kvm_vm_enable_cap(s, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2, 0, 2521 dirty_log_manual_caps); 2522 if (ret) { 2523 warn_report("Trying to enable capability %"PRIu64" of " 2524 "KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 but failed. " 2525 "Falling back to the legacy mode. ", 2526 dirty_log_manual_caps); 2527 s->manual_dirty_log_protect = 0; 2528 } 2529 } 2530 } 2531 2532 #ifdef KVM_CAP_VCPU_EVENTS 2533 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS); 2534 #endif 2535 2536 s->robust_singlestep = 2537 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP); 2538 2539 #ifdef KVM_CAP_DEBUGREGS 2540 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS); 2541 #endif 2542 2543 s->max_nested_state_len = kvm_check_extension(s, KVM_CAP_NESTED_STATE); 2544 2545 #ifdef KVM_CAP_IRQ_ROUTING 2546 kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0); 2547 #endif 2548 2549 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3); 2550 2551 s->irq_set_ioctl = KVM_IRQ_LINE; 2552 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) { 2553 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS; 2554 } 2555 2556 kvm_readonly_mem_allowed = 2557 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0); 2558 2559 kvm_eventfds_allowed = 2560 (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0); 2561 2562 kvm_irqfds_allowed = 2563 (kvm_check_extension(s, KVM_CAP_IRQFD) > 0); 2564 2565 kvm_resamplefds_allowed = 2566 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0); 2567 2568 kvm_vm_attributes_allowed = 2569 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0); 2570 2571 kvm_ioeventfd_any_length_allowed = 2572 (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0); 2573 2574 #ifdef KVM_CAP_SET_GUEST_DEBUG 2575 kvm_has_guest_debug = 2576 (kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG) > 0); 2577 #endif 2578 2579 kvm_sstep_flags = 0; 2580 if (kvm_has_guest_debug) { 2581 kvm_sstep_flags = SSTEP_ENABLE; 2582 2583 #if defined KVM_CAP_SET_GUEST_DEBUG2 2584 int guest_debug_flags = 2585 kvm_check_extension(s, KVM_CAP_SET_GUEST_DEBUG2); 2586 2587 if (guest_debug_flags & KVM_GUESTDBG_BLOCKIRQ) { 2588 kvm_sstep_flags |= SSTEP_NOIRQ; 2589 } 2590 #endif 2591 } 2592 2593 kvm_state = s; 2594 2595 ret = kvm_arch_init(ms, s); 2596 if (ret < 0) { 2597 goto err; 2598 } 2599 2600 if (s->kernel_irqchip_split == ON_OFF_AUTO_AUTO) { 2601 s->kernel_irqchip_split = mc->default_kernel_irqchip_split ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 2602 } 2603 2604 qemu_register_reset(kvm_unpoison_all, NULL); 2605 2606 if (s->kernel_irqchip_allowed) { 2607 kvm_irqchip_create(s); 2608 } 2609 2610 if (kvm_eventfds_allowed) { 2611 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add; 2612 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del; 2613 } 2614 s->memory_listener.listener.coalesced_io_add = kvm_coalesce_mmio_region; 2615 s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region; 2616 2617 kvm_memory_listener_register(s, &s->memory_listener, 2618 &address_space_memory, 0, "kvm-memory"); 2619 if (kvm_eventfds_allowed) { 2620 memory_listener_register(&kvm_io_listener, 2621 &address_space_io); 2622 } 2623 memory_listener_register(&kvm_coalesced_pio_listener, 2624 &address_space_io); 2625 2626 s->many_ioeventfds = kvm_check_many_ioeventfds(); 2627 2628 s->sync_mmu = !!kvm_vm_check_extension(kvm_state, KVM_CAP_SYNC_MMU); 2629 if (!s->sync_mmu) { 2630 ret = ram_block_discard_disable(true); 2631 assert(!ret); 2632 } 2633 2634 if (s->kvm_dirty_ring_size) { 2635 ret = kvm_dirty_ring_reaper_init(s); 2636 if (ret) { 2637 goto err; 2638 } 2639 } 2640 2641 return 0; 2642 2643 err: 2644 assert(ret < 0); 2645 if (s->vmfd >= 0) { 2646 close(s->vmfd); 2647 } 2648 if (s->fd != -1) { 2649 close(s->fd); 2650 } 2651 g_free(s->memory_listener.slots); 2652 2653 return ret; 2654 } 2655 2656 void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len) 2657 { 2658 s->sigmask_len = sigmask_len; 2659 } 2660 2661 static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction, 2662 int size, uint32_t count) 2663 { 2664 int i; 2665 uint8_t *ptr = data; 2666 2667 for (i = 0; i < count; i++) { 2668 address_space_rw(&address_space_io, port, attrs, 2669 ptr, size, 2670 direction == KVM_EXIT_IO_OUT); 2671 ptr += size; 2672 } 2673 } 2674 2675 static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run) 2676 { 2677 fprintf(stderr, "KVM internal error. Suberror: %d\n", 2678 run->internal.suberror); 2679 2680 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) { 2681 int i; 2682 2683 for (i = 0; i < run->internal.ndata; ++i) { 2684 fprintf(stderr, "extra data[%d]: 0x%016"PRIx64"\n", 2685 i, (uint64_t)run->internal.data[i]); 2686 } 2687 } 2688 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) { 2689 fprintf(stderr, "emulation failure\n"); 2690 if (!kvm_arch_stop_on_emulation_error(cpu)) { 2691 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2692 return EXCP_INTERRUPT; 2693 } 2694 } 2695 /* FIXME: Should trigger a qmp message to let management know 2696 * something went wrong. 2697 */ 2698 return -1; 2699 } 2700 2701 void kvm_flush_coalesced_mmio_buffer(void) 2702 { 2703 KVMState *s = kvm_state; 2704 2705 if (s->coalesced_flush_in_progress) { 2706 return; 2707 } 2708 2709 s->coalesced_flush_in_progress = true; 2710 2711 if (s->coalesced_mmio_ring) { 2712 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring; 2713 while (ring->first != ring->last) { 2714 struct kvm_coalesced_mmio *ent; 2715 2716 ent = &ring->coalesced_mmio[ring->first]; 2717 2718 if (ent->pio == 1) { 2719 address_space_write(&address_space_io, ent->phys_addr, 2720 MEMTXATTRS_UNSPECIFIED, ent->data, 2721 ent->len); 2722 } else { 2723 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len); 2724 } 2725 smp_wmb(); 2726 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX; 2727 } 2728 } 2729 2730 s->coalesced_flush_in_progress = false; 2731 } 2732 2733 bool kvm_cpu_check_are_resettable(void) 2734 { 2735 return kvm_arch_cpu_check_are_resettable(); 2736 } 2737 2738 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg) 2739 { 2740 if (!cpu->vcpu_dirty) { 2741 kvm_arch_get_registers(cpu); 2742 cpu->vcpu_dirty = true; 2743 } 2744 } 2745 2746 void kvm_cpu_synchronize_state(CPUState *cpu) 2747 { 2748 if (!cpu->vcpu_dirty) { 2749 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL); 2750 } 2751 } 2752 2753 static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg) 2754 { 2755 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE); 2756 cpu->vcpu_dirty = false; 2757 } 2758 2759 void kvm_cpu_synchronize_post_reset(CPUState *cpu) 2760 { 2761 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); 2762 } 2763 2764 static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) 2765 { 2766 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE); 2767 cpu->vcpu_dirty = false; 2768 } 2769 2770 void kvm_cpu_synchronize_post_init(CPUState *cpu) 2771 { 2772 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL); 2773 } 2774 2775 static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg) 2776 { 2777 cpu->vcpu_dirty = true; 2778 } 2779 2780 void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu) 2781 { 2782 run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL); 2783 } 2784 2785 #ifdef KVM_HAVE_MCE_INJECTION 2786 static __thread void *pending_sigbus_addr; 2787 static __thread int pending_sigbus_code; 2788 static __thread bool have_sigbus_pending; 2789 #endif 2790 2791 static void kvm_cpu_kick(CPUState *cpu) 2792 { 2793 qatomic_set(&cpu->kvm_run->immediate_exit, 1); 2794 } 2795 2796 static void kvm_cpu_kick_self(void) 2797 { 2798 if (kvm_immediate_exit) { 2799 kvm_cpu_kick(current_cpu); 2800 } else { 2801 qemu_cpu_kick_self(); 2802 } 2803 } 2804 2805 static void kvm_eat_signals(CPUState *cpu) 2806 { 2807 struct timespec ts = { 0, 0 }; 2808 siginfo_t siginfo; 2809 sigset_t waitset; 2810 sigset_t chkset; 2811 int r; 2812 2813 if (kvm_immediate_exit) { 2814 qatomic_set(&cpu->kvm_run->immediate_exit, 0); 2815 /* Write kvm_run->immediate_exit before the cpu->exit_request 2816 * write in kvm_cpu_exec. 2817 */ 2818 smp_wmb(); 2819 return; 2820 } 2821 2822 sigemptyset(&waitset); 2823 sigaddset(&waitset, SIG_IPI); 2824 2825 do { 2826 r = sigtimedwait(&waitset, &siginfo, &ts); 2827 if (r == -1 && !(errno == EAGAIN || errno == EINTR)) { 2828 perror("sigtimedwait"); 2829 exit(1); 2830 } 2831 2832 r = sigpending(&chkset); 2833 if (r == -1) { 2834 perror("sigpending"); 2835 exit(1); 2836 } 2837 } while (sigismember(&chkset, SIG_IPI)); 2838 } 2839 2840 int kvm_cpu_exec(CPUState *cpu) 2841 { 2842 struct kvm_run *run = cpu->kvm_run; 2843 int ret, run_ret; 2844 2845 DPRINTF("kvm_cpu_exec()\n"); 2846 2847 if (kvm_arch_process_async_events(cpu)) { 2848 qatomic_set(&cpu->exit_request, 0); 2849 return EXCP_HLT; 2850 } 2851 2852 qemu_mutex_unlock_iothread(); 2853 cpu_exec_start(cpu); 2854 2855 do { 2856 MemTxAttrs attrs; 2857 2858 if (cpu->vcpu_dirty) { 2859 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE); 2860 cpu->vcpu_dirty = false; 2861 } 2862 2863 kvm_arch_pre_run(cpu, run); 2864 if (qatomic_read(&cpu->exit_request)) { 2865 DPRINTF("interrupt exit requested\n"); 2866 /* 2867 * KVM requires us to reenter the kernel after IO exits to complete 2868 * instruction emulation. This self-signal will ensure that we 2869 * leave ASAP again. 2870 */ 2871 kvm_cpu_kick_self(); 2872 } 2873 2874 /* Read cpu->exit_request before KVM_RUN reads run->immediate_exit. 2875 * Matching barrier in kvm_eat_signals. 2876 */ 2877 smp_rmb(); 2878 2879 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0); 2880 2881 attrs = kvm_arch_post_run(cpu, run); 2882 2883 #ifdef KVM_HAVE_MCE_INJECTION 2884 if (unlikely(have_sigbus_pending)) { 2885 qemu_mutex_lock_iothread(); 2886 kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code, 2887 pending_sigbus_addr); 2888 have_sigbus_pending = false; 2889 qemu_mutex_unlock_iothread(); 2890 } 2891 #endif 2892 2893 if (run_ret < 0) { 2894 if (run_ret == -EINTR || run_ret == -EAGAIN) { 2895 DPRINTF("io window exit\n"); 2896 kvm_eat_signals(cpu); 2897 ret = EXCP_INTERRUPT; 2898 break; 2899 } 2900 fprintf(stderr, "error: kvm run failed %s\n", 2901 strerror(-run_ret)); 2902 #ifdef TARGET_PPC 2903 if (run_ret == -EBUSY) { 2904 fprintf(stderr, 2905 "This is probably because your SMT is enabled.\n" 2906 "VCPU can only run on primary threads with all " 2907 "secondary threads offline.\n"); 2908 } 2909 #endif 2910 ret = -1; 2911 break; 2912 } 2913 2914 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason); 2915 switch (run->exit_reason) { 2916 case KVM_EXIT_IO: 2917 DPRINTF("handle_io\n"); 2918 /* Called outside BQL */ 2919 kvm_handle_io(run->io.port, attrs, 2920 (uint8_t *)run + run->io.data_offset, 2921 run->io.direction, 2922 run->io.size, 2923 run->io.count); 2924 ret = 0; 2925 break; 2926 case KVM_EXIT_MMIO: 2927 DPRINTF("handle_mmio\n"); 2928 /* Called outside BQL */ 2929 address_space_rw(&address_space_memory, 2930 run->mmio.phys_addr, attrs, 2931 run->mmio.data, 2932 run->mmio.len, 2933 run->mmio.is_write); 2934 ret = 0; 2935 break; 2936 case KVM_EXIT_IRQ_WINDOW_OPEN: 2937 DPRINTF("irq_window_open\n"); 2938 ret = EXCP_INTERRUPT; 2939 break; 2940 case KVM_EXIT_SHUTDOWN: 2941 DPRINTF("shutdown\n"); 2942 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 2943 ret = EXCP_INTERRUPT; 2944 break; 2945 case KVM_EXIT_UNKNOWN: 2946 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n", 2947 (uint64_t)run->hw.hardware_exit_reason); 2948 ret = -1; 2949 break; 2950 case KVM_EXIT_INTERNAL_ERROR: 2951 ret = kvm_handle_internal_error(cpu, run); 2952 break; 2953 case KVM_EXIT_DIRTY_RING_FULL: 2954 /* 2955 * We shouldn't continue if the dirty ring of this vcpu is 2956 * still full. Got kicked by KVM_RESET_DIRTY_RINGS. 2957 */ 2958 trace_kvm_dirty_ring_full(cpu->cpu_index); 2959 qemu_mutex_lock_iothread(); 2960 kvm_dirty_ring_reap(kvm_state); 2961 qemu_mutex_unlock_iothread(); 2962 ret = 0; 2963 break; 2964 case KVM_EXIT_SYSTEM_EVENT: 2965 switch (run->system_event.type) { 2966 case KVM_SYSTEM_EVENT_SHUTDOWN: 2967 qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); 2968 ret = EXCP_INTERRUPT; 2969 break; 2970 case KVM_SYSTEM_EVENT_RESET: 2971 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 2972 ret = EXCP_INTERRUPT; 2973 break; 2974 case KVM_SYSTEM_EVENT_CRASH: 2975 kvm_cpu_synchronize_state(cpu); 2976 qemu_mutex_lock_iothread(); 2977 qemu_system_guest_panicked(cpu_get_crash_info(cpu)); 2978 qemu_mutex_unlock_iothread(); 2979 ret = 0; 2980 break; 2981 default: 2982 DPRINTF("kvm_arch_handle_exit\n"); 2983 ret = kvm_arch_handle_exit(cpu, run); 2984 break; 2985 } 2986 break; 2987 default: 2988 DPRINTF("kvm_arch_handle_exit\n"); 2989 ret = kvm_arch_handle_exit(cpu, run); 2990 break; 2991 } 2992 } while (ret == 0); 2993 2994 cpu_exec_end(cpu); 2995 qemu_mutex_lock_iothread(); 2996 2997 if (ret < 0) { 2998 cpu_dump_state(cpu, stderr, CPU_DUMP_CODE); 2999 vm_stop(RUN_STATE_INTERNAL_ERROR); 3000 } 3001 3002 qatomic_set(&cpu->exit_request, 0); 3003 return ret; 3004 } 3005 3006 int kvm_ioctl(KVMState *s, int type, ...) 3007 { 3008 int ret; 3009 void *arg; 3010 va_list ap; 3011 3012 va_start(ap, type); 3013 arg = va_arg(ap, void *); 3014 va_end(ap); 3015 3016 trace_kvm_ioctl(type, arg); 3017 ret = ioctl(s->fd, type, arg); 3018 if (ret == -1) { 3019 ret = -errno; 3020 } 3021 return ret; 3022 } 3023 3024 int kvm_vm_ioctl(KVMState *s, int type, ...) 3025 { 3026 int ret; 3027 void *arg; 3028 va_list ap; 3029 3030 va_start(ap, type); 3031 arg = va_arg(ap, void *); 3032 va_end(ap); 3033 3034 trace_kvm_vm_ioctl(type, arg); 3035 ret = ioctl(s->vmfd, type, arg); 3036 if (ret == -1) { 3037 ret = -errno; 3038 } 3039 return ret; 3040 } 3041 3042 int kvm_vcpu_ioctl(CPUState *cpu, int type, ...) 3043 { 3044 int ret; 3045 void *arg; 3046 va_list ap; 3047 3048 va_start(ap, type); 3049 arg = va_arg(ap, void *); 3050 va_end(ap); 3051 3052 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg); 3053 ret = ioctl(cpu->kvm_fd, type, arg); 3054 if (ret == -1) { 3055 ret = -errno; 3056 } 3057 return ret; 3058 } 3059 3060 int kvm_device_ioctl(int fd, int type, ...) 3061 { 3062 int ret; 3063 void *arg; 3064 va_list ap; 3065 3066 va_start(ap, type); 3067 arg = va_arg(ap, void *); 3068 va_end(ap); 3069 3070 trace_kvm_device_ioctl(fd, type, arg); 3071 ret = ioctl(fd, type, arg); 3072 if (ret == -1) { 3073 ret = -errno; 3074 } 3075 return ret; 3076 } 3077 3078 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr) 3079 { 3080 int ret; 3081 struct kvm_device_attr attribute = { 3082 .group = group, 3083 .attr = attr, 3084 }; 3085 3086 if (!kvm_vm_attributes_allowed) { 3087 return 0; 3088 } 3089 3090 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute); 3091 /* kvm returns 0 on success for HAS_DEVICE_ATTR */ 3092 return ret ? 0 : 1; 3093 } 3094 3095 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr) 3096 { 3097 struct kvm_device_attr attribute = { 3098 .group = group, 3099 .attr = attr, 3100 .flags = 0, 3101 }; 3102 3103 return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1; 3104 } 3105 3106 int kvm_device_access(int fd, int group, uint64_t attr, 3107 void *val, bool write, Error **errp) 3108 { 3109 struct kvm_device_attr kvmattr; 3110 int err; 3111 3112 kvmattr.flags = 0; 3113 kvmattr.group = group; 3114 kvmattr.attr = attr; 3115 kvmattr.addr = (uintptr_t)val; 3116 3117 err = kvm_device_ioctl(fd, 3118 write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR, 3119 &kvmattr); 3120 if (err < 0) { 3121 error_setg_errno(errp, -err, 3122 "KVM_%s_DEVICE_ATTR failed: Group %d " 3123 "attr 0x%016" PRIx64, 3124 write ? "SET" : "GET", group, attr); 3125 } 3126 return err; 3127 } 3128 3129 bool kvm_has_sync_mmu(void) 3130 { 3131 return kvm_state->sync_mmu; 3132 } 3133 3134 int kvm_has_vcpu_events(void) 3135 { 3136 return kvm_state->vcpu_events; 3137 } 3138 3139 int kvm_has_robust_singlestep(void) 3140 { 3141 return kvm_state->robust_singlestep; 3142 } 3143 3144 int kvm_has_debugregs(void) 3145 { 3146 return kvm_state->debugregs; 3147 } 3148 3149 int kvm_max_nested_state_length(void) 3150 { 3151 return kvm_state->max_nested_state_len; 3152 } 3153 3154 int kvm_has_many_ioeventfds(void) 3155 { 3156 if (!kvm_enabled()) { 3157 return 0; 3158 } 3159 return kvm_state->many_ioeventfds; 3160 } 3161 3162 int kvm_has_gsi_routing(void) 3163 { 3164 #ifdef KVM_CAP_IRQ_ROUTING 3165 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING); 3166 #else 3167 return false; 3168 #endif 3169 } 3170 3171 int kvm_has_intx_set_mask(void) 3172 { 3173 return kvm_state->intx_set_mask; 3174 } 3175 3176 bool kvm_arm_supports_user_irq(void) 3177 { 3178 return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ); 3179 } 3180 3181 #ifdef KVM_CAP_SET_GUEST_DEBUG 3182 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu, 3183 target_ulong pc) 3184 { 3185 struct kvm_sw_breakpoint *bp; 3186 3187 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) { 3188 if (bp->pc == pc) { 3189 return bp; 3190 } 3191 } 3192 return NULL; 3193 } 3194 3195 int kvm_sw_breakpoints_active(CPUState *cpu) 3196 { 3197 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints); 3198 } 3199 3200 struct kvm_set_guest_debug_data { 3201 struct kvm_guest_debug dbg; 3202 int err; 3203 }; 3204 3205 static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data) 3206 { 3207 struct kvm_set_guest_debug_data *dbg_data = 3208 (struct kvm_set_guest_debug_data *) data.host_ptr; 3209 3210 dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG, 3211 &dbg_data->dbg); 3212 } 3213 3214 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap) 3215 { 3216 struct kvm_set_guest_debug_data data; 3217 3218 data.dbg.control = reinject_trap; 3219 3220 if (cpu->singlestep_enabled) { 3221 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP; 3222 3223 if (cpu->singlestep_enabled & SSTEP_NOIRQ) { 3224 data.dbg.control |= KVM_GUESTDBG_BLOCKIRQ; 3225 } 3226 } 3227 kvm_arch_update_guest_debug(cpu, &data.dbg); 3228 3229 run_on_cpu(cpu, kvm_invoke_set_guest_debug, 3230 RUN_ON_CPU_HOST_PTR(&data)); 3231 return data.err; 3232 } 3233 3234 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr, 3235 target_ulong len, int type) 3236 { 3237 struct kvm_sw_breakpoint *bp; 3238 int err; 3239 3240 if (type == GDB_BREAKPOINT_SW) { 3241 bp = kvm_find_sw_breakpoint(cpu, addr); 3242 if (bp) { 3243 bp->use_count++; 3244 return 0; 3245 } 3246 3247 bp = g_new(struct kvm_sw_breakpoint, 1); 3248 bp->pc = addr; 3249 bp->use_count = 1; 3250 err = kvm_arch_insert_sw_breakpoint(cpu, bp); 3251 if (err) { 3252 g_free(bp); 3253 return err; 3254 } 3255 3256 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 3257 } else { 3258 err = kvm_arch_insert_hw_breakpoint(addr, len, type); 3259 if (err) { 3260 return err; 3261 } 3262 } 3263 3264 CPU_FOREACH(cpu) { 3265 err = kvm_update_guest_debug(cpu, 0); 3266 if (err) { 3267 return err; 3268 } 3269 } 3270 return 0; 3271 } 3272 3273 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr, 3274 target_ulong len, int type) 3275 { 3276 struct kvm_sw_breakpoint *bp; 3277 int err; 3278 3279 if (type == GDB_BREAKPOINT_SW) { 3280 bp = kvm_find_sw_breakpoint(cpu, addr); 3281 if (!bp) { 3282 return -ENOENT; 3283 } 3284 3285 if (bp->use_count > 1) { 3286 bp->use_count--; 3287 return 0; 3288 } 3289 3290 err = kvm_arch_remove_sw_breakpoint(cpu, bp); 3291 if (err) { 3292 return err; 3293 } 3294 3295 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 3296 g_free(bp); 3297 } else { 3298 err = kvm_arch_remove_hw_breakpoint(addr, len, type); 3299 if (err) { 3300 return err; 3301 } 3302 } 3303 3304 CPU_FOREACH(cpu) { 3305 err = kvm_update_guest_debug(cpu, 0); 3306 if (err) { 3307 return err; 3308 } 3309 } 3310 return 0; 3311 } 3312 3313 void kvm_remove_all_breakpoints(CPUState *cpu) 3314 { 3315 struct kvm_sw_breakpoint *bp, *next; 3316 KVMState *s = cpu->kvm_state; 3317 CPUState *tmpcpu; 3318 3319 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) { 3320 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) { 3321 /* Try harder to find a CPU that currently sees the breakpoint. */ 3322 CPU_FOREACH(tmpcpu) { 3323 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) { 3324 break; 3325 } 3326 } 3327 } 3328 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry); 3329 g_free(bp); 3330 } 3331 kvm_arch_remove_all_hw_breakpoints(); 3332 3333 CPU_FOREACH(cpu) { 3334 kvm_update_guest_debug(cpu, 0); 3335 } 3336 } 3337 3338 #else /* !KVM_CAP_SET_GUEST_DEBUG */ 3339 3340 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap) 3341 { 3342 return -EINVAL; 3343 } 3344 3345 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr, 3346 target_ulong len, int type) 3347 { 3348 return -EINVAL; 3349 } 3350 3351 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr, 3352 target_ulong len, int type) 3353 { 3354 return -EINVAL; 3355 } 3356 3357 void kvm_remove_all_breakpoints(CPUState *cpu) 3358 { 3359 } 3360 #endif /* !KVM_CAP_SET_GUEST_DEBUG */ 3361 3362 static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset) 3363 { 3364 KVMState *s = kvm_state; 3365 struct kvm_signal_mask *sigmask; 3366 int r; 3367 3368 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset)); 3369 3370 sigmask->len = s->sigmask_len; 3371 memcpy(sigmask->sigset, sigset, sizeof(*sigset)); 3372 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask); 3373 g_free(sigmask); 3374 3375 return r; 3376 } 3377 3378 static void kvm_ipi_signal(int sig) 3379 { 3380 if (current_cpu) { 3381 assert(kvm_immediate_exit); 3382 kvm_cpu_kick(current_cpu); 3383 } 3384 } 3385 3386 void kvm_init_cpu_signals(CPUState *cpu) 3387 { 3388 int r; 3389 sigset_t set; 3390 struct sigaction sigact; 3391 3392 memset(&sigact, 0, sizeof(sigact)); 3393 sigact.sa_handler = kvm_ipi_signal; 3394 sigaction(SIG_IPI, &sigact, NULL); 3395 3396 pthread_sigmask(SIG_BLOCK, NULL, &set); 3397 #if defined KVM_HAVE_MCE_INJECTION 3398 sigdelset(&set, SIGBUS); 3399 pthread_sigmask(SIG_SETMASK, &set, NULL); 3400 #endif 3401 sigdelset(&set, SIG_IPI); 3402 if (kvm_immediate_exit) { 3403 r = pthread_sigmask(SIG_SETMASK, &set, NULL); 3404 } else { 3405 r = kvm_set_signal_mask(cpu, &set); 3406 } 3407 if (r) { 3408 fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r)); 3409 exit(1); 3410 } 3411 } 3412 3413 /* Called asynchronously in VCPU thread. */ 3414 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr) 3415 { 3416 #ifdef KVM_HAVE_MCE_INJECTION 3417 if (have_sigbus_pending) { 3418 return 1; 3419 } 3420 have_sigbus_pending = true; 3421 pending_sigbus_addr = addr; 3422 pending_sigbus_code = code; 3423 qatomic_set(&cpu->exit_request, 1); 3424 return 0; 3425 #else 3426 return 1; 3427 #endif 3428 } 3429 3430 /* Called synchronously (via signalfd) in main thread. */ 3431 int kvm_on_sigbus(int code, void *addr) 3432 { 3433 #ifdef KVM_HAVE_MCE_INJECTION 3434 /* Action required MCE kills the process if SIGBUS is blocked. Because 3435 * that's what happens in the I/O thread, where we handle MCE via signalfd, 3436 * we can only get action optional here. 3437 */ 3438 assert(code != BUS_MCEERR_AR); 3439 kvm_arch_on_sigbus_vcpu(first_cpu, code, addr); 3440 return 0; 3441 #else 3442 return 1; 3443 #endif 3444 } 3445 3446 int kvm_create_device(KVMState *s, uint64_t type, bool test) 3447 { 3448 int ret; 3449 struct kvm_create_device create_dev; 3450 3451 create_dev.type = type; 3452 create_dev.fd = -1; 3453 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0; 3454 3455 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) { 3456 return -ENOTSUP; 3457 } 3458 3459 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev); 3460 if (ret) { 3461 return ret; 3462 } 3463 3464 return test ? 0 : create_dev.fd; 3465 } 3466 3467 bool kvm_device_supported(int vmfd, uint64_t type) 3468 { 3469 struct kvm_create_device create_dev = { 3470 .type = type, 3471 .fd = -1, 3472 .flags = KVM_CREATE_DEVICE_TEST, 3473 }; 3474 3475 if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) { 3476 return false; 3477 } 3478 3479 return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0); 3480 } 3481 3482 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source) 3483 { 3484 struct kvm_one_reg reg; 3485 int r; 3486 3487 reg.id = id; 3488 reg.addr = (uintptr_t) source; 3489 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 3490 if (r) { 3491 trace_kvm_failed_reg_set(id, strerror(-r)); 3492 } 3493 return r; 3494 } 3495 3496 int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target) 3497 { 3498 struct kvm_one_reg reg; 3499 int r; 3500 3501 reg.id = id; 3502 reg.addr = (uintptr_t) target; 3503 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 3504 if (r) { 3505 trace_kvm_failed_reg_get(id, strerror(-r)); 3506 } 3507 return r; 3508 } 3509 3510 static bool kvm_accel_has_memory(MachineState *ms, AddressSpace *as, 3511 hwaddr start_addr, hwaddr size) 3512 { 3513 KVMState *kvm = KVM_STATE(ms->accelerator); 3514 int i; 3515 3516 for (i = 0; i < kvm->nr_as; ++i) { 3517 if (kvm->as[i].as == as && kvm->as[i].ml) { 3518 size = MIN(kvm_max_slot_size, size); 3519 return NULL != kvm_lookup_matching_slot(kvm->as[i].ml, 3520 start_addr, size); 3521 } 3522 } 3523 3524 return false; 3525 } 3526 3527 static void kvm_get_kvm_shadow_mem(Object *obj, Visitor *v, 3528 const char *name, void *opaque, 3529 Error **errp) 3530 { 3531 KVMState *s = KVM_STATE(obj); 3532 int64_t value = s->kvm_shadow_mem; 3533 3534 visit_type_int(v, name, &value, errp); 3535 } 3536 3537 static void kvm_set_kvm_shadow_mem(Object *obj, Visitor *v, 3538 const char *name, void *opaque, 3539 Error **errp) 3540 { 3541 KVMState *s = KVM_STATE(obj); 3542 int64_t value; 3543 3544 if (s->fd != -1) { 3545 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3546 return; 3547 } 3548 3549 if (!visit_type_int(v, name, &value, errp)) { 3550 return; 3551 } 3552 3553 s->kvm_shadow_mem = value; 3554 } 3555 3556 static void kvm_set_kernel_irqchip(Object *obj, Visitor *v, 3557 const char *name, void *opaque, 3558 Error **errp) 3559 { 3560 KVMState *s = KVM_STATE(obj); 3561 OnOffSplit mode; 3562 3563 if (s->fd != -1) { 3564 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3565 return; 3566 } 3567 3568 if (!visit_type_OnOffSplit(v, name, &mode, errp)) { 3569 return; 3570 } 3571 switch (mode) { 3572 case ON_OFF_SPLIT_ON: 3573 s->kernel_irqchip_allowed = true; 3574 s->kernel_irqchip_required = true; 3575 s->kernel_irqchip_split = ON_OFF_AUTO_OFF; 3576 break; 3577 case ON_OFF_SPLIT_OFF: 3578 s->kernel_irqchip_allowed = false; 3579 s->kernel_irqchip_required = false; 3580 s->kernel_irqchip_split = ON_OFF_AUTO_OFF; 3581 break; 3582 case ON_OFF_SPLIT_SPLIT: 3583 s->kernel_irqchip_allowed = true; 3584 s->kernel_irqchip_required = true; 3585 s->kernel_irqchip_split = ON_OFF_AUTO_ON; 3586 break; 3587 default: 3588 /* The value was checked in visit_type_OnOffSplit() above. If 3589 * we get here, then something is wrong in QEMU. 3590 */ 3591 abort(); 3592 } 3593 } 3594 3595 bool kvm_kernel_irqchip_allowed(void) 3596 { 3597 return kvm_state->kernel_irqchip_allowed; 3598 } 3599 3600 bool kvm_kernel_irqchip_required(void) 3601 { 3602 return kvm_state->kernel_irqchip_required; 3603 } 3604 3605 bool kvm_kernel_irqchip_split(void) 3606 { 3607 return kvm_state->kernel_irqchip_split == ON_OFF_AUTO_ON; 3608 } 3609 3610 static void kvm_get_dirty_ring_size(Object *obj, Visitor *v, 3611 const char *name, void *opaque, 3612 Error **errp) 3613 { 3614 KVMState *s = KVM_STATE(obj); 3615 uint32_t value = s->kvm_dirty_ring_size; 3616 3617 visit_type_uint32(v, name, &value, errp); 3618 } 3619 3620 static void kvm_set_dirty_ring_size(Object *obj, Visitor *v, 3621 const char *name, void *opaque, 3622 Error **errp) 3623 { 3624 KVMState *s = KVM_STATE(obj); 3625 Error *error = NULL; 3626 uint32_t value; 3627 3628 if (s->fd != -1) { 3629 error_setg(errp, "Cannot set properties after the accelerator has been initialized"); 3630 return; 3631 } 3632 3633 visit_type_uint32(v, name, &value, &error); 3634 if (error) { 3635 error_propagate(errp, error); 3636 return; 3637 } 3638 if (value & (value - 1)) { 3639 error_setg(errp, "dirty-ring-size must be a power of two."); 3640 return; 3641 } 3642 3643 s->kvm_dirty_ring_size = value; 3644 } 3645 3646 static void kvm_accel_instance_init(Object *obj) 3647 { 3648 KVMState *s = KVM_STATE(obj); 3649 3650 s->fd = -1; 3651 s->vmfd = -1; 3652 s->kvm_shadow_mem = -1; 3653 s->kernel_irqchip_allowed = true; 3654 s->kernel_irqchip_split = ON_OFF_AUTO_AUTO; 3655 /* KVM dirty ring is by default off */ 3656 s->kvm_dirty_ring_size = 0; 3657 } 3658 3659 static void kvm_accel_class_init(ObjectClass *oc, void *data) 3660 { 3661 AccelClass *ac = ACCEL_CLASS(oc); 3662 ac->name = "KVM"; 3663 ac->init_machine = kvm_init; 3664 ac->has_memory = kvm_accel_has_memory; 3665 ac->allowed = &kvm_allowed; 3666 3667 object_class_property_add(oc, "kernel-irqchip", "on|off|split", 3668 NULL, kvm_set_kernel_irqchip, 3669 NULL, NULL); 3670 object_class_property_set_description(oc, "kernel-irqchip", 3671 "Configure KVM in-kernel irqchip"); 3672 3673 object_class_property_add(oc, "kvm-shadow-mem", "int", 3674 kvm_get_kvm_shadow_mem, kvm_set_kvm_shadow_mem, 3675 NULL, NULL); 3676 object_class_property_set_description(oc, "kvm-shadow-mem", 3677 "KVM shadow MMU size"); 3678 3679 object_class_property_add(oc, "dirty-ring-size", "uint32", 3680 kvm_get_dirty_ring_size, kvm_set_dirty_ring_size, 3681 NULL, NULL); 3682 object_class_property_set_description(oc, "dirty-ring-size", 3683 "Size of KVM dirty page ring buffer (default: 0, i.e. use bitmap)"); 3684 } 3685 3686 static const TypeInfo kvm_accel_type = { 3687 .name = TYPE_KVM_ACCEL, 3688 .parent = TYPE_ACCEL, 3689 .instance_init = kvm_accel_instance_init, 3690 .class_init = kvm_accel_class_init, 3691 .instance_size = sizeof(KVMState), 3692 }; 3693 3694 static void kvm_type_init(void) 3695 { 3696 type_register_static(&kvm_accel_type); 3697 } 3698 3699 type_init(kvm_type_init); 3700