1 /* 2 * QEMU KVM support 3 * 4 * Copyright IBM, Corp. 2008 5 * Red Hat, Inc. 2008 6 * 7 * Authors: 8 * Anthony Liguori <aliguori@us.ibm.com> 9 * Glauber Costa <gcosta@redhat.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2 or later. 12 * See the COPYING file in the top-level directory. 13 * 14 */ 15 16 #include "qemu/osdep.h" 17 #include <sys/ioctl.h> 18 19 #include <linux/kvm.h> 20 21 #include "qemu-common.h" 22 #include "qemu/atomic.h" 23 #include "qemu/option.h" 24 #include "qemu/config-file.h" 25 #include "qemu/error-report.h" 26 #include "qapi/error.h" 27 #include "hw/hw.h" 28 #include "hw/pci/msi.h" 29 #include "hw/pci/msix.h" 30 #include "hw/s390x/adapter.h" 31 #include "exec/gdbstub.h" 32 #include "sysemu/kvm_int.h" 33 #include "sysemu/cpus.h" 34 #include "qemu/bswap.h" 35 #include "exec/memory.h" 36 #include "exec/ram_addr.h" 37 #include "exec/address-spaces.h" 38 #include "qemu/event_notifier.h" 39 #include "trace.h" 40 #include "hw/irq.h" 41 42 #include "hw/boards.h" 43 44 /* This check must be after config-host.h is included */ 45 #ifdef CONFIG_EVENTFD 46 #include <sys/eventfd.h> 47 #endif 48 49 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We 50 * need to use the real host PAGE_SIZE, as that's what KVM will use. 51 */ 52 #define PAGE_SIZE getpagesize() 53 54 //#define DEBUG_KVM 55 56 #ifdef DEBUG_KVM 57 #define DPRINTF(fmt, ...) \ 58 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0) 59 #else 60 #define DPRINTF(fmt, ...) \ 61 do { } while (0) 62 #endif 63 64 #define KVM_MSI_HASHTAB_SIZE 256 65 66 struct KVMParkedVcpu { 67 unsigned long vcpu_id; 68 int kvm_fd; 69 QLIST_ENTRY(KVMParkedVcpu) node; 70 }; 71 72 struct KVMState 73 { 74 AccelState parent_obj; 75 76 int nr_slots; 77 int fd; 78 int vmfd; 79 int coalesced_mmio; 80 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; 81 bool coalesced_flush_in_progress; 82 int broken_set_mem_region; 83 int vcpu_events; 84 int robust_singlestep; 85 int debugregs; 86 #ifdef KVM_CAP_SET_GUEST_DEBUG 87 struct kvm_sw_breakpoint_head kvm_sw_breakpoints; 88 #endif 89 int many_ioeventfds; 90 int intx_set_mask; 91 /* The man page (and posix) say ioctl numbers are signed int, but 92 * they're not. Linux, glibc and *BSD all treat ioctl numbers as 93 * unsigned, and treating them as signed here can break things */ 94 unsigned irq_set_ioctl; 95 unsigned int sigmask_len; 96 GHashTable *gsimap; 97 #ifdef KVM_CAP_IRQ_ROUTING 98 struct kvm_irq_routing *irq_routes; 99 int nr_allocated_irq_routes; 100 unsigned long *used_gsi_bitmap; 101 unsigned int gsi_count; 102 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE]; 103 #endif 104 KVMMemoryListener memory_listener; 105 QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus; 106 }; 107 108 KVMState *kvm_state; 109 bool kvm_kernel_irqchip; 110 bool kvm_split_irqchip; 111 bool kvm_async_interrupts_allowed; 112 bool kvm_halt_in_kernel_allowed; 113 bool kvm_eventfds_allowed; 114 bool kvm_irqfds_allowed; 115 bool kvm_resamplefds_allowed; 116 bool kvm_msi_via_irqfd_allowed; 117 bool kvm_gsi_routing_allowed; 118 bool kvm_gsi_direct_mapping; 119 bool kvm_allowed; 120 bool kvm_readonly_mem_allowed; 121 bool kvm_vm_attributes_allowed; 122 bool kvm_direct_msi_allowed; 123 bool kvm_ioeventfd_any_length_allowed; 124 bool kvm_msi_use_devid; 125 static bool kvm_immediate_exit; 126 127 static const KVMCapabilityInfo kvm_required_capabilites[] = { 128 KVM_CAP_INFO(USER_MEMORY), 129 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS), 130 KVM_CAP_LAST_INFO 131 }; 132 133 int kvm_get_max_memslots(void) 134 { 135 KVMState *s = KVM_STATE(current_machine->accelerator); 136 137 return s->nr_slots; 138 } 139 140 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml) 141 { 142 KVMState *s = kvm_state; 143 int i; 144 145 for (i = 0; i < s->nr_slots; i++) { 146 if (kml->slots[i].memory_size == 0) { 147 return &kml->slots[i]; 148 } 149 } 150 151 return NULL; 152 } 153 154 bool kvm_has_free_slot(MachineState *ms) 155 { 156 KVMState *s = KVM_STATE(ms->accelerator); 157 158 return kvm_get_free_slot(&s->memory_listener); 159 } 160 161 static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml) 162 { 163 KVMSlot *slot = kvm_get_free_slot(kml); 164 165 if (slot) { 166 return slot; 167 } 168 169 fprintf(stderr, "%s: no free slot available\n", __func__); 170 abort(); 171 } 172 173 static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml, 174 hwaddr start_addr, 175 hwaddr end_addr) 176 { 177 KVMState *s = kvm_state; 178 int i; 179 180 for (i = 0; i < s->nr_slots; i++) { 181 KVMSlot *mem = &kml->slots[i]; 182 183 if (start_addr == mem->start_addr && 184 end_addr == mem->start_addr + mem->memory_size) { 185 return mem; 186 } 187 } 188 189 return NULL; 190 } 191 192 /* 193 * Find overlapping slot with lowest start address 194 */ 195 static KVMSlot *kvm_lookup_overlapping_slot(KVMMemoryListener *kml, 196 hwaddr start_addr, 197 hwaddr end_addr) 198 { 199 KVMState *s = kvm_state; 200 KVMSlot *found = NULL; 201 int i; 202 203 for (i = 0; i < s->nr_slots; i++) { 204 KVMSlot *mem = &kml->slots[i]; 205 206 if (mem->memory_size == 0 || 207 (found && found->start_addr < mem->start_addr)) { 208 continue; 209 } 210 211 if (end_addr > mem->start_addr && 212 start_addr < mem->start_addr + mem->memory_size) { 213 found = mem; 214 } 215 } 216 217 return found; 218 } 219 220 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram, 221 hwaddr *phys_addr) 222 { 223 KVMMemoryListener *kml = &s->memory_listener; 224 int i; 225 226 for (i = 0; i < s->nr_slots; i++) { 227 KVMSlot *mem = &kml->slots[i]; 228 229 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) { 230 *phys_addr = mem->start_addr + (ram - mem->ram); 231 return 1; 232 } 233 } 234 235 return 0; 236 } 237 238 static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot) 239 { 240 KVMState *s = kvm_state; 241 struct kvm_userspace_memory_region mem; 242 243 mem.slot = slot->slot | (kml->as_id << 16); 244 mem.guest_phys_addr = slot->start_addr; 245 mem.userspace_addr = (unsigned long)slot->ram; 246 mem.flags = slot->flags; 247 248 if (slot->memory_size && mem.flags & KVM_MEM_READONLY) { 249 /* Set the slot size to 0 before setting the slot to the desired 250 * value. This is needed based on KVM commit 75d61fbc. */ 251 mem.memory_size = 0; 252 kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 253 } 254 mem.memory_size = slot->memory_size; 255 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem); 256 } 257 258 int kvm_destroy_vcpu(CPUState *cpu) 259 { 260 KVMState *s = kvm_state; 261 long mmap_size; 262 struct KVMParkedVcpu *vcpu = NULL; 263 int ret = 0; 264 265 DPRINTF("kvm_destroy_vcpu\n"); 266 267 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 268 if (mmap_size < 0) { 269 ret = mmap_size; 270 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n"); 271 goto err; 272 } 273 274 ret = munmap(cpu->kvm_run, mmap_size); 275 if (ret < 0) { 276 goto err; 277 } 278 279 vcpu = g_malloc0(sizeof(*vcpu)); 280 vcpu->vcpu_id = kvm_arch_vcpu_id(cpu); 281 vcpu->kvm_fd = cpu->kvm_fd; 282 QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node); 283 err: 284 return ret; 285 } 286 287 static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id) 288 { 289 struct KVMParkedVcpu *cpu; 290 291 QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) { 292 if (cpu->vcpu_id == vcpu_id) { 293 int kvm_fd; 294 295 QLIST_REMOVE(cpu, node); 296 kvm_fd = cpu->kvm_fd; 297 g_free(cpu); 298 return kvm_fd; 299 } 300 } 301 302 return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id); 303 } 304 305 int kvm_init_vcpu(CPUState *cpu) 306 { 307 KVMState *s = kvm_state; 308 long mmap_size; 309 int ret; 310 311 DPRINTF("kvm_init_vcpu\n"); 312 313 ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu)); 314 if (ret < 0) { 315 DPRINTF("kvm_create_vcpu failed\n"); 316 goto err; 317 } 318 319 cpu->kvm_fd = ret; 320 cpu->kvm_state = s; 321 cpu->vcpu_dirty = true; 322 323 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0); 324 if (mmap_size < 0) { 325 ret = mmap_size; 326 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n"); 327 goto err; 328 } 329 330 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, 331 cpu->kvm_fd, 0); 332 if (cpu->kvm_run == MAP_FAILED) { 333 ret = -errno; 334 DPRINTF("mmap'ing vcpu state failed\n"); 335 goto err; 336 } 337 338 if (s->coalesced_mmio && !s->coalesced_mmio_ring) { 339 s->coalesced_mmio_ring = 340 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE; 341 } 342 343 ret = kvm_arch_init_vcpu(cpu); 344 err: 345 return ret; 346 } 347 348 /* 349 * dirty pages logging control 350 */ 351 352 static int kvm_mem_flags(MemoryRegion *mr) 353 { 354 bool readonly = mr->readonly || memory_region_is_romd(mr); 355 int flags = 0; 356 357 if (memory_region_get_dirty_log_mask(mr) != 0) { 358 flags |= KVM_MEM_LOG_DIRTY_PAGES; 359 } 360 if (readonly && kvm_readonly_mem_allowed) { 361 flags |= KVM_MEM_READONLY; 362 } 363 return flags; 364 } 365 366 static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem, 367 MemoryRegion *mr) 368 { 369 int old_flags; 370 371 old_flags = mem->flags; 372 mem->flags = kvm_mem_flags(mr); 373 374 /* If nothing changed effectively, no need to issue ioctl */ 375 if (mem->flags == old_flags) { 376 return 0; 377 } 378 379 return kvm_set_user_memory_region(kml, mem); 380 } 381 382 static int kvm_section_update_flags(KVMMemoryListener *kml, 383 MemoryRegionSection *section) 384 { 385 hwaddr phys_addr = section->offset_within_address_space; 386 ram_addr_t size = int128_get64(section->size); 387 KVMSlot *mem = kvm_lookup_matching_slot(kml, phys_addr, phys_addr + size); 388 389 if (mem == NULL) { 390 return 0; 391 } else { 392 return kvm_slot_update_flags(kml, mem, section->mr); 393 } 394 } 395 396 static void kvm_log_start(MemoryListener *listener, 397 MemoryRegionSection *section, 398 int old, int new) 399 { 400 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 401 int r; 402 403 if (old != 0) { 404 return; 405 } 406 407 r = kvm_section_update_flags(kml, section); 408 if (r < 0) { 409 abort(); 410 } 411 } 412 413 static void kvm_log_stop(MemoryListener *listener, 414 MemoryRegionSection *section, 415 int old, int new) 416 { 417 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 418 int r; 419 420 if (new != 0) { 421 return; 422 } 423 424 r = kvm_section_update_flags(kml, section); 425 if (r < 0) { 426 abort(); 427 } 428 } 429 430 /* get kvm's dirty pages bitmap and update qemu's */ 431 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section, 432 unsigned long *bitmap) 433 { 434 ram_addr_t start = section->offset_within_region + 435 memory_region_get_ram_addr(section->mr); 436 ram_addr_t pages = int128_get64(section->size) / getpagesize(); 437 438 cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages); 439 return 0; 440 } 441 442 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1)) 443 444 /** 445 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space 446 * This function updates qemu's dirty bitmap using 447 * memory_region_set_dirty(). This means all bits are set 448 * to dirty. 449 * 450 * @start_add: start of logged region. 451 * @end_addr: end of logged region. 452 */ 453 static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml, 454 MemoryRegionSection *section) 455 { 456 KVMState *s = kvm_state; 457 unsigned long size, allocated_size = 0; 458 struct kvm_dirty_log d = {}; 459 KVMSlot *mem; 460 int ret = 0; 461 hwaddr start_addr = section->offset_within_address_space; 462 hwaddr end_addr = start_addr + int128_get64(section->size); 463 464 d.dirty_bitmap = NULL; 465 while (start_addr < end_addr) { 466 mem = kvm_lookup_overlapping_slot(kml, start_addr, end_addr); 467 if (mem == NULL) { 468 break; 469 } 470 471 /* XXX bad kernel interface alert 472 * For dirty bitmap, kernel allocates array of size aligned to 473 * bits-per-long. But for case when the kernel is 64bits and 474 * the userspace is 32bits, userspace can't align to the same 475 * bits-per-long, since sizeof(long) is different between kernel 476 * and user space. This way, userspace will provide buffer which 477 * may be 4 bytes less than the kernel will use, resulting in 478 * userspace memory corruption (which is not detectable by valgrind 479 * too, in most cases). 480 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in 481 * a hope that sizeof(long) won't become >8 any time soon. 482 */ 483 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), 484 /*HOST_LONG_BITS*/ 64) / 8; 485 if (!d.dirty_bitmap) { 486 d.dirty_bitmap = g_malloc(size); 487 } else if (size > allocated_size) { 488 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size); 489 } 490 allocated_size = size; 491 memset(d.dirty_bitmap, 0, allocated_size); 492 493 d.slot = mem->slot | (kml->as_id << 16); 494 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) { 495 DPRINTF("ioctl failed %d\n", errno); 496 ret = -1; 497 break; 498 } 499 500 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap); 501 start_addr = mem->start_addr + mem->memory_size; 502 } 503 g_free(d.dirty_bitmap); 504 505 return ret; 506 } 507 508 static void kvm_coalesce_mmio_region(MemoryListener *listener, 509 MemoryRegionSection *secion, 510 hwaddr start, hwaddr size) 511 { 512 KVMState *s = kvm_state; 513 514 if (s->coalesced_mmio) { 515 struct kvm_coalesced_mmio_zone zone; 516 517 zone.addr = start; 518 zone.size = size; 519 zone.pad = 0; 520 521 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone); 522 } 523 } 524 525 static void kvm_uncoalesce_mmio_region(MemoryListener *listener, 526 MemoryRegionSection *secion, 527 hwaddr start, hwaddr size) 528 { 529 KVMState *s = kvm_state; 530 531 if (s->coalesced_mmio) { 532 struct kvm_coalesced_mmio_zone zone; 533 534 zone.addr = start; 535 zone.size = size; 536 zone.pad = 0; 537 538 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone); 539 } 540 } 541 542 int kvm_check_extension(KVMState *s, unsigned int extension) 543 { 544 int ret; 545 546 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension); 547 if (ret < 0) { 548 ret = 0; 549 } 550 551 return ret; 552 } 553 554 int kvm_vm_check_extension(KVMState *s, unsigned int extension) 555 { 556 int ret; 557 558 ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension); 559 if (ret < 0) { 560 /* VM wide version not implemented, use global one instead */ 561 ret = kvm_check_extension(s, extension); 562 } 563 564 return ret; 565 } 566 567 static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size) 568 { 569 #if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN) 570 /* The kernel expects ioeventfd values in HOST_WORDS_BIGENDIAN 571 * endianness, but the memory core hands them in target endianness. 572 * For example, PPC is always treated as big-endian even if running 573 * on KVM and on PPC64LE. Correct here. 574 */ 575 switch (size) { 576 case 2: 577 val = bswap16(val); 578 break; 579 case 4: 580 val = bswap32(val); 581 break; 582 } 583 #endif 584 return val; 585 } 586 587 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val, 588 bool assign, uint32_t size, bool datamatch) 589 { 590 int ret; 591 struct kvm_ioeventfd iofd = { 592 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 593 .addr = addr, 594 .len = size, 595 .flags = 0, 596 .fd = fd, 597 }; 598 599 if (!kvm_enabled()) { 600 return -ENOSYS; 601 } 602 603 if (datamatch) { 604 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 605 } 606 if (!assign) { 607 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 608 } 609 610 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd); 611 612 if (ret < 0) { 613 return -errno; 614 } 615 616 return 0; 617 } 618 619 static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val, 620 bool assign, uint32_t size, bool datamatch) 621 { 622 struct kvm_ioeventfd kick = { 623 .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0, 624 .addr = addr, 625 .flags = KVM_IOEVENTFD_FLAG_PIO, 626 .len = size, 627 .fd = fd, 628 }; 629 int r; 630 if (!kvm_enabled()) { 631 return -ENOSYS; 632 } 633 if (datamatch) { 634 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH; 635 } 636 if (!assign) { 637 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN; 638 } 639 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick); 640 if (r < 0) { 641 return r; 642 } 643 return 0; 644 } 645 646 647 static int kvm_check_many_ioeventfds(void) 648 { 649 /* Userspace can use ioeventfd for io notification. This requires a host 650 * that supports eventfd(2) and an I/O thread; since eventfd does not 651 * support SIGIO it cannot interrupt the vcpu. 652 * 653 * Older kernels have a 6 device limit on the KVM io bus. Find out so we 654 * can avoid creating too many ioeventfds. 655 */ 656 #if defined(CONFIG_EVENTFD) 657 int ioeventfds[7]; 658 int i, ret = 0; 659 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) { 660 ioeventfds[i] = eventfd(0, EFD_CLOEXEC); 661 if (ioeventfds[i] < 0) { 662 break; 663 } 664 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true); 665 if (ret < 0) { 666 close(ioeventfds[i]); 667 break; 668 } 669 } 670 671 /* Decide whether many devices are supported or not */ 672 ret = i == ARRAY_SIZE(ioeventfds); 673 674 while (i-- > 0) { 675 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true); 676 close(ioeventfds[i]); 677 } 678 return ret; 679 #else 680 return 0; 681 #endif 682 } 683 684 static const KVMCapabilityInfo * 685 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list) 686 { 687 while (list->name) { 688 if (!kvm_check_extension(s, list->value)) { 689 return list; 690 } 691 list++; 692 } 693 return NULL; 694 } 695 696 static void kvm_set_phys_mem(KVMMemoryListener *kml, 697 MemoryRegionSection *section, bool add) 698 { 699 KVMState *s = kvm_state; 700 KVMSlot *mem, old; 701 int err; 702 MemoryRegion *mr = section->mr; 703 bool writeable = !mr->readonly && !mr->rom_device; 704 hwaddr start_addr = section->offset_within_address_space; 705 ram_addr_t size = int128_get64(section->size); 706 void *ram = NULL; 707 unsigned delta; 708 709 /* kvm works in page size chunks, but the function may be called 710 with sub-page size and unaligned start address. Pad the start 711 address to next and truncate size to previous page boundary. */ 712 delta = qemu_real_host_page_size - (start_addr & ~qemu_real_host_page_mask); 713 delta &= ~qemu_real_host_page_mask; 714 if (delta > size) { 715 return; 716 } 717 start_addr += delta; 718 size -= delta; 719 size &= qemu_real_host_page_mask; 720 if (!size || (start_addr & ~qemu_real_host_page_mask)) { 721 return; 722 } 723 724 if (!memory_region_is_ram(mr)) { 725 if (writeable || !kvm_readonly_mem_allowed) { 726 return; 727 } else if (!mr->romd_mode) { 728 /* If the memory device is not in romd_mode, then we actually want 729 * to remove the kvm memory slot so all accesses will trap. */ 730 add = false; 731 } 732 } 733 734 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta; 735 736 while (1) { 737 mem = kvm_lookup_overlapping_slot(kml, start_addr, start_addr + size); 738 if (!mem) { 739 break; 740 } 741 742 if (add && start_addr >= mem->start_addr && 743 (start_addr + size <= mem->start_addr + mem->memory_size) && 744 (ram - start_addr == mem->ram - mem->start_addr)) { 745 /* The new slot fits into the existing one and comes with 746 * identical parameters - update flags and done. */ 747 kvm_slot_update_flags(kml, mem, mr); 748 return; 749 } 750 751 old = *mem; 752 753 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { 754 kvm_physical_sync_dirty_bitmap(kml, section); 755 } 756 757 /* unregister the overlapping slot */ 758 mem->memory_size = 0; 759 err = kvm_set_user_memory_region(kml, mem); 760 if (err) { 761 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n", 762 __func__, strerror(-err)); 763 abort(); 764 } 765 766 /* Workaround for older KVM versions: we can't join slots, even not by 767 * unregistering the previous ones and then registering the larger 768 * slot. We have to maintain the existing fragmentation. Sigh. 769 * 770 * This workaround assumes that the new slot starts at the same 771 * address as the first existing one. If not or if some overlapping 772 * slot comes around later, we will fail (not seen in practice so far) 773 * - and actually require a recent KVM version. */ 774 if (s->broken_set_mem_region && 775 old.start_addr == start_addr && old.memory_size < size && add) { 776 mem = kvm_alloc_slot(kml); 777 mem->memory_size = old.memory_size; 778 mem->start_addr = old.start_addr; 779 mem->ram = old.ram; 780 mem->flags = kvm_mem_flags(mr); 781 782 err = kvm_set_user_memory_region(kml, mem); 783 if (err) { 784 fprintf(stderr, "%s: error updating slot: %s\n", __func__, 785 strerror(-err)); 786 abort(); 787 } 788 789 start_addr += old.memory_size; 790 ram += old.memory_size; 791 size -= old.memory_size; 792 continue; 793 } 794 795 /* register prefix slot */ 796 if (old.start_addr < start_addr) { 797 mem = kvm_alloc_slot(kml); 798 mem->memory_size = start_addr - old.start_addr; 799 mem->start_addr = old.start_addr; 800 mem->ram = old.ram; 801 mem->flags = kvm_mem_flags(mr); 802 803 err = kvm_set_user_memory_region(kml, mem); 804 if (err) { 805 fprintf(stderr, "%s: error registering prefix slot: %s\n", 806 __func__, strerror(-err)); 807 #ifdef TARGET_PPC 808 fprintf(stderr, "%s: This is probably because your kernel's " \ 809 "PAGE_SIZE is too big. Please try to use 4k " \ 810 "PAGE_SIZE!\n", __func__); 811 #endif 812 abort(); 813 } 814 } 815 816 /* register suffix slot */ 817 if (old.start_addr + old.memory_size > start_addr + size) { 818 ram_addr_t size_delta; 819 820 mem = kvm_alloc_slot(kml); 821 mem->start_addr = start_addr + size; 822 size_delta = mem->start_addr - old.start_addr; 823 mem->memory_size = old.memory_size - size_delta; 824 mem->ram = old.ram + size_delta; 825 mem->flags = kvm_mem_flags(mr); 826 827 err = kvm_set_user_memory_region(kml, mem); 828 if (err) { 829 fprintf(stderr, "%s: error registering suffix slot: %s\n", 830 __func__, strerror(-err)); 831 abort(); 832 } 833 } 834 } 835 836 /* in case the KVM bug workaround already "consumed" the new slot */ 837 if (!size) { 838 return; 839 } 840 if (!add) { 841 return; 842 } 843 mem = kvm_alloc_slot(kml); 844 mem->memory_size = size; 845 mem->start_addr = start_addr; 846 mem->ram = ram; 847 mem->flags = kvm_mem_flags(mr); 848 849 err = kvm_set_user_memory_region(kml, mem); 850 if (err) { 851 fprintf(stderr, "%s: error registering slot: %s\n", __func__, 852 strerror(-err)); 853 abort(); 854 } 855 } 856 857 static void kvm_region_add(MemoryListener *listener, 858 MemoryRegionSection *section) 859 { 860 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 861 862 memory_region_ref(section->mr); 863 kvm_set_phys_mem(kml, section, true); 864 } 865 866 static void kvm_region_del(MemoryListener *listener, 867 MemoryRegionSection *section) 868 { 869 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 870 871 kvm_set_phys_mem(kml, section, false); 872 memory_region_unref(section->mr); 873 } 874 875 static void kvm_log_sync(MemoryListener *listener, 876 MemoryRegionSection *section) 877 { 878 KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener); 879 int r; 880 881 r = kvm_physical_sync_dirty_bitmap(kml, section); 882 if (r < 0) { 883 abort(); 884 } 885 } 886 887 static void kvm_mem_ioeventfd_add(MemoryListener *listener, 888 MemoryRegionSection *section, 889 bool match_data, uint64_t data, 890 EventNotifier *e) 891 { 892 int fd = event_notifier_get_fd(e); 893 int r; 894 895 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 896 data, true, int128_get64(section->size), 897 match_data); 898 if (r < 0) { 899 fprintf(stderr, "%s: error adding ioeventfd: %s\n", 900 __func__, strerror(-r)); 901 abort(); 902 } 903 } 904 905 static void kvm_mem_ioeventfd_del(MemoryListener *listener, 906 MemoryRegionSection *section, 907 bool match_data, uint64_t data, 908 EventNotifier *e) 909 { 910 int fd = event_notifier_get_fd(e); 911 int r; 912 913 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space, 914 data, false, int128_get64(section->size), 915 match_data); 916 if (r < 0) { 917 abort(); 918 } 919 } 920 921 static void kvm_io_ioeventfd_add(MemoryListener *listener, 922 MemoryRegionSection *section, 923 bool match_data, uint64_t data, 924 EventNotifier *e) 925 { 926 int fd = event_notifier_get_fd(e); 927 int r; 928 929 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 930 data, true, int128_get64(section->size), 931 match_data); 932 if (r < 0) { 933 fprintf(stderr, "%s: error adding ioeventfd: %s\n", 934 __func__, strerror(-r)); 935 abort(); 936 } 937 } 938 939 static void kvm_io_ioeventfd_del(MemoryListener *listener, 940 MemoryRegionSection *section, 941 bool match_data, uint64_t data, 942 EventNotifier *e) 943 944 { 945 int fd = event_notifier_get_fd(e); 946 int r; 947 948 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space, 949 data, false, int128_get64(section->size), 950 match_data); 951 if (r < 0) { 952 abort(); 953 } 954 } 955 956 void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, 957 AddressSpace *as, int as_id) 958 { 959 int i; 960 961 kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot)); 962 kml->as_id = as_id; 963 964 for (i = 0; i < s->nr_slots; i++) { 965 kml->slots[i].slot = i; 966 } 967 968 kml->listener.region_add = kvm_region_add; 969 kml->listener.region_del = kvm_region_del; 970 kml->listener.log_start = kvm_log_start; 971 kml->listener.log_stop = kvm_log_stop; 972 kml->listener.log_sync = kvm_log_sync; 973 kml->listener.priority = 10; 974 975 memory_listener_register(&kml->listener, as); 976 } 977 978 static MemoryListener kvm_io_listener = { 979 .eventfd_add = kvm_io_ioeventfd_add, 980 .eventfd_del = kvm_io_ioeventfd_del, 981 .priority = 10, 982 }; 983 984 int kvm_set_irq(KVMState *s, int irq, int level) 985 { 986 struct kvm_irq_level event; 987 int ret; 988 989 assert(kvm_async_interrupts_enabled()); 990 991 event.level = level; 992 event.irq = irq; 993 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event); 994 if (ret < 0) { 995 perror("kvm_set_irq"); 996 abort(); 997 } 998 999 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status; 1000 } 1001 1002 #ifdef KVM_CAP_IRQ_ROUTING 1003 typedef struct KVMMSIRoute { 1004 struct kvm_irq_routing_entry kroute; 1005 QTAILQ_ENTRY(KVMMSIRoute) entry; 1006 } KVMMSIRoute; 1007 1008 static void set_gsi(KVMState *s, unsigned int gsi) 1009 { 1010 set_bit(gsi, s->used_gsi_bitmap); 1011 } 1012 1013 static void clear_gsi(KVMState *s, unsigned int gsi) 1014 { 1015 clear_bit(gsi, s->used_gsi_bitmap); 1016 } 1017 1018 void kvm_init_irq_routing(KVMState *s) 1019 { 1020 int gsi_count, i; 1021 1022 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1; 1023 if (gsi_count > 0) { 1024 /* Round up so we can search ints using ffs */ 1025 s->used_gsi_bitmap = bitmap_new(gsi_count); 1026 s->gsi_count = gsi_count; 1027 } 1028 1029 s->irq_routes = g_malloc0(sizeof(*s->irq_routes)); 1030 s->nr_allocated_irq_routes = 0; 1031 1032 if (!kvm_direct_msi_allowed) { 1033 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) { 1034 QTAILQ_INIT(&s->msi_hashtab[i]); 1035 } 1036 } 1037 1038 kvm_arch_init_irq_routing(s); 1039 } 1040 1041 void kvm_irqchip_commit_routes(KVMState *s) 1042 { 1043 int ret; 1044 1045 if (kvm_gsi_direct_mapping()) { 1046 return; 1047 } 1048 1049 if (!kvm_gsi_routing_enabled()) { 1050 return; 1051 } 1052 1053 s->irq_routes->flags = 0; 1054 trace_kvm_irqchip_commit_routes(); 1055 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes); 1056 assert(ret == 0); 1057 } 1058 1059 static void kvm_add_routing_entry(KVMState *s, 1060 struct kvm_irq_routing_entry *entry) 1061 { 1062 struct kvm_irq_routing_entry *new; 1063 int n, size; 1064 1065 if (s->irq_routes->nr == s->nr_allocated_irq_routes) { 1066 n = s->nr_allocated_irq_routes * 2; 1067 if (n < 64) { 1068 n = 64; 1069 } 1070 size = sizeof(struct kvm_irq_routing); 1071 size += n * sizeof(*new); 1072 s->irq_routes = g_realloc(s->irq_routes, size); 1073 s->nr_allocated_irq_routes = n; 1074 } 1075 n = s->irq_routes->nr++; 1076 new = &s->irq_routes->entries[n]; 1077 1078 *new = *entry; 1079 1080 set_gsi(s, entry->gsi); 1081 } 1082 1083 static int kvm_update_routing_entry(KVMState *s, 1084 struct kvm_irq_routing_entry *new_entry) 1085 { 1086 struct kvm_irq_routing_entry *entry; 1087 int n; 1088 1089 for (n = 0; n < s->irq_routes->nr; n++) { 1090 entry = &s->irq_routes->entries[n]; 1091 if (entry->gsi != new_entry->gsi) { 1092 continue; 1093 } 1094 1095 if(!memcmp(entry, new_entry, sizeof *entry)) { 1096 return 0; 1097 } 1098 1099 *entry = *new_entry; 1100 1101 return 0; 1102 } 1103 1104 return -ESRCH; 1105 } 1106 1107 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin) 1108 { 1109 struct kvm_irq_routing_entry e = {}; 1110 1111 assert(pin < s->gsi_count); 1112 1113 e.gsi = irq; 1114 e.type = KVM_IRQ_ROUTING_IRQCHIP; 1115 e.flags = 0; 1116 e.u.irqchip.irqchip = irqchip; 1117 e.u.irqchip.pin = pin; 1118 kvm_add_routing_entry(s, &e); 1119 } 1120 1121 void kvm_irqchip_release_virq(KVMState *s, int virq) 1122 { 1123 struct kvm_irq_routing_entry *e; 1124 int i; 1125 1126 if (kvm_gsi_direct_mapping()) { 1127 return; 1128 } 1129 1130 for (i = 0; i < s->irq_routes->nr; i++) { 1131 e = &s->irq_routes->entries[i]; 1132 if (e->gsi == virq) { 1133 s->irq_routes->nr--; 1134 *e = s->irq_routes->entries[s->irq_routes->nr]; 1135 } 1136 } 1137 clear_gsi(s, virq); 1138 kvm_arch_release_virq_post(virq); 1139 trace_kvm_irqchip_release_virq(virq); 1140 } 1141 1142 static unsigned int kvm_hash_msi(uint32_t data) 1143 { 1144 /* This is optimized for IA32 MSI layout. However, no other arch shall 1145 * repeat the mistake of not providing a direct MSI injection API. */ 1146 return data & 0xff; 1147 } 1148 1149 static void kvm_flush_dynamic_msi_routes(KVMState *s) 1150 { 1151 KVMMSIRoute *route, *next; 1152 unsigned int hash; 1153 1154 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) { 1155 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) { 1156 kvm_irqchip_release_virq(s, route->kroute.gsi); 1157 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry); 1158 g_free(route); 1159 } 1160 } 1161 } 1162 1163 static int kvm_irqchip_get_virq(KVMState *s) 1164 { 1165 int next_virq; 1166 1167 /* 1168 * PIC and IOAPIC share the first 16 GSI numbers, thus the available 1169 * GSI numbers are more than the number of IRQ route. Allocating a GSI 1170 * number can succeed even though a new route entry cannot be added. 1171 * When this happens, flush dynamic MSI entries to free IRQ route entries. 1172 */ 1173 if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) { 1174 kvm_flush_dynamic_msi_routes(s); 1175 } 1176 1177 /* Return the lowest unused GSI in the bitmap */ 1178 next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count); 1179 if (next_virq >= s->gsi_count) { 1180 return -ENOSPC; 1181 } else { 1182 return next_virq; 1183 } 1184 } 1185 1186 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg) 1187 { 1188 unsigned int hash = kvm_hash_msi(msg.data); 1189 KVMMSIRoute *route; 1190 1191 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) { 1192 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address && 1193 route->kroute.u.msi.address_hi == (msg.address >> 32) && 1194 route->kroute.u.msi.data == le32_to_cpu(msg.data)) { 1195 return route; 1196 } 1197 } 1198 return NULL; 1199 } 1200 1201 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 1202 { 1203 struct kvm_msi msi; 1204 KVMMSIRoute *route; 1205 1206 if (kvm_direct_msi_allowed) { 1207 msi.address_lo = (uint32_t)msg.address; 1208 msi.address_hi = msg.address >> 32; 1209 msi.data = le32_to_cpu(msg.data); 1210 msi.flags = 0; 1211 memset(msi.pad, 0, sizeof(msi.pad)); 1212 1213 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi); 1214 } 1215 1216 route = kvm_lookup_msi_route(s, msg); 1217 if (!route) { 1218 int virq; 1219 1220 virq = kvm_irqchip_get_virq(s); 1221 if (virq < 0) { 1222 return virq; 1223 } 1224 1225 route = g_malloc0(sizeof(KVMMSIRoute)); 1226 route->kroute.gsi = virq; 1227 route->kroute.type = KVM_IRQ_ROUTING_MSI; 1228 route->kroute.flags = 0; 1229 route->kroute.u.msi.address_lo = (uint32_t)msg.address; 1230 route->kroute.u.msi.address_hi = msg.address >> 32; 1231 route->kroute.u.msi.data = le32_to_cpu(msg.data); 1232 1233 kvm_add_routing_entry(s, &route->kroute); 1234 kvm_irqchip_commit_routes(s); 1235 1236 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route, 1237 entry); 1238 } 1239 1240 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI); 1241 1242 return kvm_set_irq(s, route->kroute.gsi, 1); 1243 } 1244 1245 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev) 1246 { 1247 struct kvm_irq_routing_entry kroute = {}; 1248 int virq; 1249 MSIMessage msg = {0, 0}; 1250 1251 if (pci_available && dev) { 1252 msg = pci_get_msi_message(dev, vector); 1253 } 1254 1255 if (kvm_gsi_direct_mapping()) { 1256 return kvm_arch_msi_data_to_gsi(msg.data); 1257 } 1258 1259 if (!kvm_gsi_routing_enabled()) { 1260 return -ENOSYS; 1261 } 1262 1263 virq = kvm_irqchip_get_virq(s); 1264 if (virq < 0) { 1265 return virq; 1266 } 1267 1268 kroute.gsi = virq; 1269 kroute.type = KVM_IRQ_ROUTING_MSI; 1270 kroute.flags = 0; 1271 kroute.u.msi.address_lo = (uint32_t)msg.address; 1272 kroute.u.msi.address_hi = msg.address >> 32; 1273 kroute.u.msi.data = le32_to_cpu(msg.data); 1274 if (pci_available && kvm_msi_devid_required()) { 1275 kroute.flags = KVM_MSI_VALID_DEVID; 1276 kroute.u.msi.devid = pci_requester_id(dev); 1277 } 1278 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 1279 kvm_irqchip_release_virq(s, virq); 1280 return -EINVAL; 1281 } 1282 1283 trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A", 1284 vector, virq); 1285 1286 kvm_add_routing_entry(s, &kroute); 1287 kvm_arch_add_msi_route_post(&kroute, vector, dev); 1288 kvm_irqchip_commit_routes(s); 1289 1290 return virq; 1291 } 1292 1293 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg, 1294 PCIDevice *dev) 1295 { 1296 struct kvm_irq_routing_entry kroute = {}; 1297 1298 if (kvm_gsi_direct_mapping()) { 1299 return 0; 1300 } 1301 1302 if (!kvm_irqchip_in_kernel()) { 1303 return -ENOSYS; 1304 } 1305 1306 kroute.gsi = virq; 1307 kroute.type = KVM_IRQ_ROUTING_MSI; 1308 kroute.flags = 0; 1309 kroute.u.msi.address_lo = (uint32_t)msg.address; 1310 kroute.u.msi.address_hi = msg.address >> 32; 1311 kroute.u.msi.data = le32_to_cpu(msg.data); 1312 if (pci_available && kvm_msi_devid_required()) { 1313 kroute.flags = KVM_MSI_VALID_DEVID; 1314 kroute.u.msi.devid = pci_requester_id(dev); 1315 } 1316 if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) { 1317 return -EINVAL; 1318 } 1319 1320 trace_kvm_irqchip_update_msi_route(virq); 1321 1322 return kvm_update_routing_entry(s, &kroute); 1323 } 1324 1325 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq, 1326 bool assign) 1327 { 1328 struct kvm_irqfd irqfd = { 1329 .fd = fd, 1330 .gsi = virq, 1331 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN, 1332 }; 1333 1334 if (rfd != -1) { 1335 irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE; 1336 irqfd.resamplefd = rfd; 1337 } 1338 1339 if (!kvm_irqfds_enabled()) { 1340 return -ENOSYS; 1341 } 1342 1343 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd); 1344 } 1345 1346 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) 1347 { 1348 struct kvm_irq_routing_entry kroute = {}; 1349 int virq; 1350 1351 if (!kvm_gsi_routing_enabled()) { 1352 return -ENOSYS; 1353 } 1354 1355 virq = kvm_irqchip_get_virq(s); 1356 if (virq < 0) { 1357 return virq; 1358 } 1359 1360 kroute.gsi = virq; 1361 kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER; 1362 kroute.flags = 0; 1363 kroute.u.adapter.summary_addr = adapter->summary_addr; 1364 kroute.u.adapter.ind_addr = adapter->ind_addr; 1365 kroute.u.adapter.summary_offset = adapter->summary_offset; 1366 kroute.u.adapter.ind_offset = adapter->ind_offset; 1367 kroute.u.adapter.adapter_id = adapter->adapter_id; 1368 1369 kvm_add_routing_entry(s, &kroute); 1370 1371 return virq; 1372 } 1373 1374 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) 1375 { 1376 struct kvm_irq_routing_entry kroute = {}; 1377 int virq; 1378 1379 if (!kvm_gsi_routing_enabled()) { 1380 return -ENOSYS; 1381 } 1382 if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) { 1383 return -ENOSYS; 1384 } 1385 virq = kvm_irqchip_get_virq(s); 1386 if (virq < 0) { 1387 return virq; 1388 } 1389 1390 kroute.gsi = virq; 1391 kroute.type = KVM_IRQ_ROUTING_HV_SINT; 1392 kroute.flags = 0; 1393 kroute.u.hv_sint.vcpu = vcpu; 1394 kroute.u.hv_sint.sint = sint; 1395 1396 kvm_add_routing_entry(s, &kroute); 1397 kvm_irqchip_commit_routes(s); 1398 1399 return virq; 1400 } 1401 1402 #else /* !KVM_CAP_IRQ_ROUTING */ 1403 1404 void kvm_init_irq_routing(KVMState *s) 1405 { 1406 } 1407 1408 void kvm_irqchip_release_virq(KVMState *s, int virq) 1409 { 1410 } 1411 1412 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg) 1413 { 1414 abort(); 1415 } 1416 1417 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev) 1418 { 1419 return -ENOSYS; 1420 } 1421 1422 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter) 1423 { 1424 return -ENOSYS; 1425 } 1426 1427 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint) 1428 { 1429 return -ENOSYS; 1430 } 1431 1432 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign) 1433 { 1434 abort(); 1435 } 1436 1437 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg) 1438 { 1439 return -ENOSYS; 1440 } 1441 #endif /* !KVM_CAP_IRQ_ROUTING */ 1442 1443 int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 1444 EventNotifier *rn, int virq) 1445 { 1446 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), 1447 rn ? event_notifier_get_fd(rn) : -1, virq, true); 1448 } 1449 1450 int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n, 1451 int virq) 1452 { 1453 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq, 1454 false); 1455 } 1456 1457 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, 1458 EventNotifier *rn, qemu_irq irq) 1459 { 1460 gpointer key, gsi; 1461 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 1462 1463 if (!found) { 1464 return -ENXIO; 1465 } 1466 return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi)); 1467 } 1468 1469 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, 1470 qemu_irq irq) 1471 { 1472 gpointer key, gsi; 1473 gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi); 1474 1475 if (!found) { 1476 return -ENXIO; 1477 } 1478 return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi)); 1479 } 1480 1481 void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi) 1482 { 1483 g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi)); 1484 } 1485 1486 static void kvm_irqchip_create(MachineState *machine, KVMState *s) 1487 { 1488 int ret; 1489 1490 if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) { 1491 ; 1492 } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) { 1493 ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0); 1494 if (ret < 0) { 1495 fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret)); 1496 exit(1); 1497 } 1498 } else { 1499 return; 1500 } 1501 1502 /* First probe and see if there's a arch-specific hook to create the 1503 * in-kernel irqchip for us */ 1504 ret = kvm_arch_irqchip_create(machine, s); 1505 if (ret == 0) { 1506 if (machine_kernel_irqchip_split(machine)) { 1507 perror("Split IRQ chip mode not supported."); 1508 exit(1); 1509 } else { 1510 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP); 1511 } 1512 } 1513 if (ret < 0) { 1514 fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret)); 1515 exit(1); 1516 } 1517 1518 kvm_kernel_irqchip = true; 1519 /* If we have an in-kernel IRQ chip then we must have asynchronous 1520 * interrupt delivery (though the reverse is not necessarily true) 1521 */ 1522 kvm_async_interrupts_allowed = true; 1523 kvm_halt_in_kernel_allowed = true; 1524 1525 kvm_init_irq_routing(s); 1526 1527 s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal); 1528 } 1529 1530 /* Find number of supported CPUs using the recommended 1531 * procedure from the kernel API documentation to cope with 1532 * older kernels that may be missing capabilities. 1533 */ 1534 static int kvm_recommended_vcpus(KVMState *s) 1535 { 1536 int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS); 1537 return (ret) ? ret : 4; 1538 } 1539 1540 static int kvm_max_vcpus(KVMState *s) 1541 { 1542 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS); 1543 return (ret) ? ret : kvm_recommended_vcpus(s); 1544 } 1545 1546 static int kvm_max_vcpu_id(KVMState *s) 1547 { 1548 int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID); 1549 return (ret) ? ret : kvm_max_vcpus(s); 1550 } 1551 1552 bool kvm_vcpu_id_is_valid(int vcpu_id) 1553 { 1554 KVMState *s = KVM_STATE(current_machine->accelerator); 1555 return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s); 1556 } 1557 1558 static int kvm_init(MachineState *ms) 1559 { 1560 MachineClass *mc = MACHINE_GET_CLASS(ms); 1561 static const char upgrade_note[] = 1562 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n" 1563 "(see http://sourceforge.net/projects/kvm).\n"; 1564 struct { 1565 const char *name; 1566 int num; 1567 } num_cpus[] = { 1568 { "SMP", smp_cpus }, 1569 { "hotpluggable", max_cpus }, 1570 { NULL, } 1571 }, *nc = num_cpus; 1572 int soft_vcpus_limit, hard_vcpus_limit; 1573 KVMState *s; 1574 const KVMCapabilityInfo *missing_cap; 1575 int ret; 1576 int type = 0; 1577 const char *kvm_type; 1578 1579 s = KVM_STATE(ms->accelerator); 1580 1581 /* 1582 * On systems where the kernel can support different base page 1583 * sizes, host page size may be different from TARGET_PAGE_SIZE, 1584 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum 1585 * page size for the system though. 1586 */ 1587 assert(TARGET_PAGE_SIZE <= getpagesize()); 1588 1589 s->sigmask_len = 8; 1590 1591 #ifdef KVM_CAP_SET_GUEST_DEBUG 1592 QTAILQ_INIT(&s->kvm_sw_breakpoints); 1593 #endif 1594 QLIST_INIT(&s->kvm_parked_vcpus); 1595 s->vmfd = -1; 1596 s->fd = qemu_open("/dev/kvm", O_RDWR); 1597 if (s->fd == -1) { 1598 fprintf(stderr, "Could not access KVM kernel module: %m\n"); 1599 ret = -errno; 1600 goto err; 1601 } 1602 1603 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0); 1604 if (ret < KVM_API_VERSION) { 1605 if (ret >= 0) { 1606 ret = -EINVAL; 1607 } 1608 fprintf(stderr, "kvm version too old\n"); 1609 goto err; 1610 } 1611 1612 if (ret > KVM_API_VERSION) { 1613 ret = -EINVAL; 1614 fprintf(stderr, "kvm version not supported\n"); 1615 goto err; 1616 } 1617 1618 kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT); 1619 s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS); 1620 1621 /* If unspecified, use the default value */ 1622 if (!s->nr_slots) { 1623 s->nr_slots = 32; 1624 } 1625 1626 /* check the vcpu limits */ 1627 soft_vcpus_limit = kvm_recommended_vcpus(s); 1628 hard_vcpus_limit = kvm_max_vcpus(s); 1629 1630 while (nc->name) { 1631 if (nc->num > soft_vcpus_limit) { 1632 fprintf(stderr, 1633 "Warning: Number of %s cpus requested (%d) exceeds " 1634 "the recommended cpus supported by KVM (%d)\n", 1635 nc->name, nc->num, soft_vcpus_limit); 1636 1637 if (nc->num > hard_vcpus_limit) { 1638 fprintf(stderr, "Number of %s cpus requested (%d) exceeds " 1639 "the maximum cpus supported by KVM (%d)\n", 1640 nc->name, nc->num, hard_vcpus_limit); 1641 exit(1); 1642 } 1643 } 1644 nc++; 1645 } 1646 1647 kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type"); 1648 if (mc->kvm_type) { 1649 type = mc->kvm_type(kvm_type); 1650 } else if (kvm_type) { 1651 ret = -EINVAL; 1652 fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type); 1653 goto err; 1654 } 1655 1656 do { 1657 ret = kvm_ioctl(s, KVM_CREATE_VM, type); 1658 } while (ret == -EINTR); 1659 1660 if (ret < 0) { 1661 fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret, 1662 strerror(-ret)); 1663 1664 #ifdef TARGET_S390X 1665 if (ret == -EINVAL) { 1666 fprintf(stderr, 1667 "Host kernel setup problem detected. Please verify:\n"); 1668 fprintf(stderr, "- for kernels supporting the switch_amode or" 1669 " user_mode parameters, whether\n"); 1670 fprintf(stderr, 1671 " user space is running in primary address space\n"); 1672 fprintf(stderr, 1673 "- for kernels supporting the vm.allocate_pgste sysctl, " 1674 "whether it is enabled\n"); 1675 } 1676 #endif 1677 goto err; 1678 } 1679 1680 s->vmfd = ret; 1681 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites); 1682 if (!missing_cap) { 1683 missing_cap = 1684 kvm_check_extension_list(s, kvm_arch_required_capabilities); 1685 } 1686 if (missing_cap) { 1687 ret = -EINVAL; 1688 fprintf(stderr, "kvm does not support %s\n%s", 1689 missing_cap->name, upgrade_note); 1690 goto err; 1691 } 1692 1693 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO); 1694 1695 s->broken_set_mem_region = 1; 1696 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS); 1697 if (ret > 0) { 1698 s->broken_set_mem_region = 0; 1699 } 1700 1701 #ifdef KVM_CAP_VCPU_EVENTS 1702 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS); 1703 #endif 1704 1705 s->robust_singlestep = 1706 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP); 1707 1708 #ifdef KVM_CAP_DEBUGREGS 1709 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS); 1710 #endif 1711 1712 #ifdef KVM_CAP_IRQ_ROUTING 1713 kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0); 1714 #endif 1715 1716 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3); 1717 1718 s->irq_set_ioctl = KVM_IRQ_LINE; 1719 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) { 1720 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS; 1721 } 1722 1723 #ifdef KVM_CAP_READONLY_MEM 1724 kvm_readonly_mem_allowed = 1725 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0); 1726 #endif 1727 1728 kvm_eventfds_allowed = 1729 (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0); 1730 1731 kvm_irqfds_allowed = 1732 (kvm_check_extension(s, KVM_CAP_IRQFD) > 0); 1733 1734 kvm_resamplefds_allowed = 1735 (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0); 1736 1737 kvm_vm_attributes_allowed = 1738 (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0); 1739 1740 kvm_ioeventfd_any_length_allowed = 1741 (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0); 1742 1743 kvm_state = s; 1744 1745 ret = kvm_arch_init(ms, s); 1746 if (ret < 0) { 1747 goto err; 1748 } 1749 1750 if (machine_kernel_irqchip_allowed(ms)) { 1751 kvm_irqchip_create(ms, s); 1752 } 1753 1754 if (kvm_eventfds_allowed) { 1755 s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add; 1756 s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del; 1757 } 1758 s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region; 1759 s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region; 1760 1761 kvm_memory_listener_register(s, &s->memory_listener, 1762 &address_space_memory, 0); 1763 memory_listener_register(&kvm_io_listener, 1764 &address_space_io); 1765 1766 s->many_ioeventfds = kvm_check_many_ioeventfds(); 1767 1768 return 0; 1769 1770 err: 1771 assert(ret < 0); 1772 if (s->vmfd >= 0) { 1773 close(s->vmfd); 1774 } 1775 if (s->fd != -1) { 1776 close(s->fd); 1777 } 1778 g_free(s->memory_listener.slots); 1779 1780 return ret; 1781 } 1782 1783 void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len) 1784 { 1785 s->sigmask_len = sigmask_len; 1786 } 1787 1788 static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction, 1789 int size, uint32_t count) 1790 { 1791 int i; 1792 uint8_t *ptr = data; 1793 1794 for (i = 0; i < count; i++) { 1795 address_space_rw(&address_space_io, port, attrs, 1796 ptr, size, 1797 direction == KVM_EXIT_IO_OUT); 1798 ptr += size; 1799 } 1800 } 1801 1802 static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run) 1803 { 1804 fprintf(stderr, "KVM internal error. Suberror: %d\n", 1805 run->internal.suberror); 1806 1807 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) { 1808 int i; 1809 1810 for (i = 0; i < run->internal.ndata; ++i) { 1811 fprintf(stderr, "extra data[%d]: %"PRIx64"\n", 1812 i, (uint64_t)run->internal.data[i]); 1813 } 1814 } 1815 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) { 1816 fprintf(stderr, "emulation failure\n"); 1817 if (!kvm_arch_stop_on_emulation_error(cpu)) { 1818 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE); 1819 return EXCP_INTERRUPT; 1820 } 1821 } 1822 /* FIXME: Should trigger a qmp message to let management know 1823 * something went wrong. 1824 */ 1825 return -1; 1826 } 1827 1828 void kvm_flush_coalesced_mmio_buffer(void) 1829 { 1830 KVMState *s = kvm_state; 1831 1832 if (s->coalesced_flush_in_progress) { 1833 return; 1834 } 1835 1836 s->coalesced_flush_in_progress = true; 1837 1838 if (s->coalesced_mmio_ring) { 1839 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring; 1840 while (ring->first != ring->last) { 1841 struct kvm_coalesced_mmio *ent; 1842 1843 ent = &ring->coalesced_mmio[ring->first]; 1844 1845 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len); 1846 smp_wmb(); 1847 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX; 1848 } 1849 } 1850 1851 s->coalesced_flush_in_progress = false; 1852 } 1853 1854 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg) 1855 { 1856 if (!cpu->vcpu_dirty) { 1857 kvm_arch_get_registers(cpu); 1858 cpu->vcpu_dirty = true; 1859 } 1860 } 1861 1862 void kvm_cpu_synchronize_state(CPUState *cpu) 1863 { 1864 if (!cpu->vcpu_dirty) { 1865 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL); 1866 } 1867 } 1868 1869 static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg) 1870 { 1871 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE); 1872 cpu->vcpu_dirty = false; 1873 } 1874 1875 void kvm_cpu_synchronize_post_reset(CPUState *cpu) 1876 { 1877 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL); 1878 } 1879 1880 static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg) 1881 { 1882 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE); 1883 cpu->vcpu_dirty = false; 1884 } 1885 1886 void kvm_cpu_synchronize_post_init(CPUState *cpu) 1887 { 1888 run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL); 1889 } 1890 1891 static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg) 1892 { 1893 cpu->vcpu_dirty = true; 1894 } 1895 1896 void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu) 1897 { 1898 run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL); 1899 } 1900 1901 #ifdef KVM_HAVE_MCE_INJECTION 1902 static __thread void *pending_sigbus_addr; 1903 static __thread int pending_sigbus_code; 1904 static __thread bool have_sigbus_pending; 1905 #endif 1906 1907 static void kvm_cpu_kick(CPUState *cpu) 1908 { 1909 atomic_set(&cpu->kvm_run->immediate_exit, 1); 1910 } 1911 1912 static void kvm_cpu_kick_self(void) 1913 { 1914 if (kvm_immediate_exit) { 1915 kvm_cpu_kick(current_cpu); 1916 } else { 1917 qemu_cpu_kick_self(); 1918 } 1919 } 1920 1921 static void kvm_eat_signals(CPUState *cpu) 1922 { 1923 struct timespec ts = { 0, 0 }; 1924 siginfo_t siginfo; 1925 sigset_t waitset; 1926 sigset_t chkset; 1927 int r; 1928 1929 if (kvm_immediate_exit) { 1930 atomic_set(&cpu->kvm_run->immediate_exit, 0); 1931 /* Write kvm_run->immediate_exit before the cpu->exit_request 1932 * write in kvm_cpu_exec. 1933 */ 1934 smp_wmb(); 1935 return; 1936 } 1937 1938 sigemptyset(&waitset); 1939 sigaddset(&waitset, SIG_IPI); 1940 1941 do { 1942 r = sigtimedwait(&waitset, &siginfo, &ts); 1943 if (r == -1 && !(errno == EAGAIN || errno == EINTR)) { 1944 perror("sigtimedwait"); 1945 exit(1); 1946 } 1947 1948 r = sigpending(&chkset); 1949 if (r == -1) { 1950 perror("sigpending"); 1951 exit(1); 1952 } 1953 } while (sigismember(&chkset, SIG_IPI)); 1954 } 1955 1956 int kvm_cpu_exec(CPUState *cpu) 1957 { 1958 struct kvm_run *run = cpu->kvm_run; 1959 int ret, run_ret; 1960 1961 DPRINTF("kvm_cpu_exec()\n"); 1962 1963 if (kvm_arch_process_async_events(cpu)) { 1964 atomic_set(&cpu->exit_request, 0); 1965 return EXCP_HLT; 1966 } 1967 1968 qemu_mutex_unlock_iothread(); 1969 cpu_exec_start(cpu); 1970 1971 do { 1972 MemTxAttrs attrs; 1973 1974 if (cpu->vcpu_dirty) { 1975 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE); 1976 cpu->vcpu_dirty = false; 1977 } 1978 1979 kvm_arch_pre_run(cpu, run); 1980 if (atomic_read(&cpu->exit_request)) { 1981 DPRINTF("interrupt exit requested\n"); 1982 /* 1983 * KVM requires us to reenter the kernel after IO exits to complete 1984 * instruction emulation. This self-signal will ensure that we 1985 * leave ASAP again. 1986 */ 1987 kvm_cpu_kick_self(); 1988 } 1989 1990 /* Read cpu->exit_request before KVM_RUN reads run->immediate_exit. 1991 * Matching barrier in kvm_eat_signals. 1992 */ 1993 smp_rmb(); 1994 1995 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0); 1996 1997 attrs = kvm_arch_post_run(cpu, run); 1998 1999 #ifdef KVM_HAVE_MCE_INJECTION 2000 if (unlikely(have_sigbus_pending)) { 2001 qemu_mutex_lock_iothread(); 2002 kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code, 2003 pending_sigbus_addr); 2004 have_sigbus_pending = false; 2005 qemu_mutex_unlock_iothread(); 2006 } 2007 #endif 2008 2009 if (run_ret < 0) { 2010 if (run_ret == -EINTR || run_ret == -EAGAIN) { 2011 DPRINTF("io window exit\n"); 2012 kvm_eat_signals(cpu); 2013 ret = EXCP_INTERRUPT; 2014 break; 2015 } 2016 fprintf(stderr, "error: kvm run failed %s\n", 2017 strerror(-run_ret)); 2018 #ifdef TARGET_PPC 2019 if (run_ret == -EBUSY) { 2020 fprintf(stderr, 2021 "This is probably because your SMT is enabled.\n" 2022 "VCPU can only run on primary threads with all " 2023 "secondary threads offline.\n"); 2024 } 2025 #endif 2026 ret = -1; 2027 break; 2028 } 2029 2030 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason); 2031 switch (run->exit_reason) { 2032 case KVM_EXIT_IO: 2033 DPRINTF("handle_io\n"); 2034 /* Called outside BQL */ 2035 kvm_handle_io(run->io.port, attrs, 2036 (uint8_t *)run + run->io.data_offset, 2037 run->io.direction, 2038 run->io.size, 2039 run->io.count); 2040 ret = 0; 2041 break; 2042 case KVM_EXIT_MMIO: 2043 DPRINTF("handle_mmio\n"); 2044 /* Called outside BQL */ 2045 address_space_rw(&address_space_memory, 2046 run->mmio.phys_addr, attrs, 2047 run->mmio.data, 2048 run->mmio.len, 2049 run->mmio.is_write); 2050 ret = 0; 2051 break; 2052 case KVM_EXIT_IRQ_WINDOW_OPEN: 2053 DPRINTF("irq_window_open\n"); 2054 ret = EXCP_INTERRUPT; 2055 break; 2056 case KVM_EXIT_SHUTDOWN: 2057 DPRINTF("shutdown\n"); 2058 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 2059 ret = EXCP_INTERRUPT; 2060 break; 2061 case KVM_EXIT_UNKNOWN: 2062 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n", 2063 (uint64_t)run->hw.hardware_exit_reason); 2064 ret = -1; 2065 break; 2066 case KVM_EXIT_INTERNAL_ERROR: 2067 ret = kvm_handle_internal_error(cpu, run); 2068 break; 2069 case KVM_EXIT_SYSTEM_EVENT: 2070 switch (run->system_event.type) { 2071 case KVM_SYSTEM_EVENT_SHUTDOWN: 2072 qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); 2073 ret = EXCP_INTERRUPT; 2074 break; 2075 case KVM_SYSTEM_EVENT_RESET: 2076 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); 2077 ret = EXCP_INTERRUPT; 2078 break; 2079 case KVM_SYSTEM_EVENT_CRASH: 2080 kvm_cpu_synchronize_state(cpu); 2081 qemu_mutex_lock_iothread(); 2082 qemu_system_guest_panicked(cpu_get_crash_info(cpu)); 2083 qemu_mutex_unlock_iothread(); 2084 ret = 0; 2085 break; 2086 default: 2087 DPRINTF("kvm_arch_handle_exit\n"); 2088 ret = kvm_arch_handle_exit(cpu, run); 2089 break; 2090 } 2091 break; 2092 default: 2093 DPRINTF("kvm_arch_handle_exit\n"); 2094 ret = kvm_arch_handle_exit(cpu, run); 2095 break; 2096 } 2097 } while (ret == 0); 2098 2099 cpu_exec_end(cpu); 2100 qemu_mutex_lock_iothread(); 2101 2102 if (ret < 0) { 2103 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE); 2104 vm_stop(RUN_STATE_INTERNAL_ERROR); 2105 } 2106 2107 atomic_set(&cpu->exit_request, 0); 2108 return ret; 2109 } 2110 2111 int kvm_ioctl(KVMState *s, int type, ...) 2112 { 2113 int ret; 2114 void *arg; 2115 va_list ap; 2116 2117 va_start(ap, type); 2118 arg = va_arg(ap, void *); 2119 va_end(ap); 2120 2121 trace_kvm_ioctl(type, arg); 2122 ret = ioctl(s->fd, type, arg); 2123 if (ret == -1) { 2124 ret = -errno; 2125 } 2126 return ret; 2127 } 2128 2129 int kvm_vm_ioctl(KVMState *s, int type, ...) 2130 { 2131 int ret; 2132 void *arg; 2133 va_list ap; 2134 2135 va_start(ap, type); 2136 arg = va_arg(ap, void *); 2137 va_end(ap); 2138 2139 trace_kvm_vm_ioctl(type, arg); 2140 ret = ioctl(s->vmfd, type, arg); 2141 if (ret == -1) { 2142 ret = -errno; 2143 } 2144 return ret; 2145 } 2146 2147 int kvm_vcpu_ioctl(CPUState *cpu, int type, ...) 2148 { 2149 int ret; 2150 void *arg; 2151 va_list ap; 2152 2153 va_start(ap, type); 2154 arg = va_arg(ap, void *); 2155 va_end(ap); 2156 2157 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg); 2158 ret = ioctl(cpu->kvm_fd, type, arg); 2159 if (ret == -1) { 2160 ret = -errno; 2161 } 2162 return ret; 2163 } 2164 2165 int kvm_device_ioctl(int fd, int type, ...) 2166 { 2167 int ret; 2168 void *arg; 2169 va_list ap; 2170 2171 va_start(ap, type); 2172 arg = va_arg(ap, void *); 2173 va_end(ap); 2174 2175 trace_kvm_device_ioctl(fd, type, arg); 2176 ret = ioctl(fd, type, arg); 2177 if (ret == -1) { 2178 ret = -errno; 2179 } 2180 return ret; 2181 } 2182 2183 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr) 2184 { 2185 int ret; 2186 struct kvm_device_attr attribute = { 2187 .group = group, 2188 .attr = attr, 2189 }; 2190 2191 if (!kvm_vm_attributes_allowed) { 2192 return 0; 2193 } 2194 2195 ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute); 2196 /* kvm returns 0 on success for HAS_DEVICE_ATTR */ 2197 return ret ? 0 : 1; 2198 } 2199 2200 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr) 2201 { 2202 struct kvm_device_attr attribute = { 2203 .group = group, 2204 .attr = attr, 2205 .flags = 0, 2206 }; 2207 2208 return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1; 2209 } 2210 2211 int kvm_device_access(int fd, int group, uint64_t attr, 2212 void *val, bool write, Error **errp) 2213 { 2214 struct kvm_device_attr kvmattr; 2215 int err; 2216 2217 kvmattr.flags = 0; 2218 kvmattr.group = group; 2219 kvmattr.attr = attr; 2220 kvmattr.addr = (uintptr_t)val; 2221 2222 err = kvm_device_ioctl(fd, 2223 write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR, 2224 &kvmattr); 2225 if (err < 0) { 2226 error_setg_errno(errp, -err, 2227 "KVM_%s_DEVICE_ATTR failed: Group %d " 2228 "attr 0x%016" PRIx64, 2229 write ? "SET" : "GET", group, attr); 2230 } 2231 return err; 2232 } 2233 2234 /* Return 1 on success, 0 on failure */ 2235 int kvm_has_sync_mmu(void) 2236 { 2237 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU); 2238 } 2239 2240 int kvm_has_vcpu_events(void) 2241 { 2242 return kvm_state->vcpu_events; 2243 } 2244 2245 int kvm_has_robust_singlestep(void) 2246 { 2247 return kvm_state->robust_singlestep; 2248 } 2249 2250 int kvm_has_debugregs(void) 2251 { 2252 return kvm_state->debugregs; 2253 } 2254 2255 int kvm_has_many_ioeventfds(void) 2256 { 2257 if (!kvm_enabled()) { 2258 return 0; 2259 } 2260 return kvm_state->many_ioeventfds; 2261 } 2262 2263 int kvm_has_gsi_routing(void) 2264 { 2265 #ifdef KVM_CAP_IRQ_ROUTING 2266 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING); 2267 #else 2268 return false; 2269 #endif 2270 } 2271 2272 int kvm_has_intx_set_mask(void) 2273 { 2274 return kvm_state->intx_set_mask; 2275 } 2276 2277 bool kvm_arm_supports_user_irq(void) 2278 { 2279 return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ); 2280 } 2281 2282 #ifdef KVM_CAP_SET_GUEST_DEBUG 2283 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu, 2284 target_ulong pc) 2285 { 2286 struct kvm_sw_breakpoint *bp; 2287 2288 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) { 2289 if (bp->pc == pc) { 2290 return bp; 2291 } 2292 } 2293 return NULL; 2294 } 2295 2296 int kvm_sw_breakpoints_active(CPUState *cpu) 2297 { 2298 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints); 2299 } 2300 2301 struct kvm_set_guest_debug_data { 2302 struct kvm_guest_debug dbg; 2303 int err; 2304 }; 2305 2306 static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data) 2307 { 2308 struct kvm_set_guest_debug_data *dbg_data = 2309 (struct kvm_set_guest_debug_data *) data.host_ptr; 2310 2311 dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG, 2312 &dbg_data->dbg); 2313 } 2314 2315 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap) 2316 { 2317 struct kvm_set_guest_debug_data data; 2318 2319 data.dbg.control = reinject_trap; 2320 2321 if (cpu->singlestep_enabled) { 2322 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP; 2323 } 2324 kvm_arch_update_guest_debug(cpu, &data.dbg); 2325 2326 run_on_cpu(cpu, kvm_invoke_set_guest_debug, 2327 RUN_ON_CPU_HOST_PTR(&data)); 2328 return data.err; 2329 } 2330 2331 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr, 2332 target_ulong len, int type) 2333 { 2334 struct kvm_sw_breakpoint *bp; 2335 int err; 2336 2337 if (type == GDB_BREAKPOINT_SW) { 2338 bp = kvm_find_sw_breakpoint(cpu, addr); 2339 if (bp) { 2340 bp->use_count++; 2341 return 0; 2342 } 2343 2344 bp = g_malloc(sizeof(struct kvm_sw_breakpoint)); 2345 bp->pc = addr; 2346 bp->use_count = 1; 2347 err = kvm_arch_insert_sw_breakpoint(cpu, bp); 2348 if (err) { 2349 g_free(bp); 2350 return err; 2351 } 2352 2353 QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 2354 } else { 2355 err = kvm_arch_insert_hw_breakpoint(addr, len, type); 2356 if (err) { 2357 return err; 2358 } 2359 } 2360 2361 CPU_FOREACH(cpu) { 2362 err = kvm_update_guest_debug(cpu, 0); 2363 if (err) { 2364 return err; 2365 } 2366 } 2367 return 0; 2368 } 2369 2370 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr, 2371 target_ulong len, int type) 2372 { 2373 struct kvm_sw_breakpoint *bp; 2374 int err; 2375 2376 if (type == GDB_BREAKPOINT_SW) { 2377 bp = kvm_find_sw_breakpoint(cpu, addr); 2378 if (!bp) { 2379 return -ENOENT; 2380 } 2381 2382 if (bp->use_count > 1) { 2383 bp->use_count--; 2384 return 0; 2385 } 2386 2387 err = kvm_arch_remove_sw_breakpoint(cpu, bp); 2388 if (err) { 2389 return err; 2390 } 2391 2392 QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry); 2393 g_free(bp); 2394 } else { 2395 err = kvm_arch_remove_hw_breakpoint(addr, len, type); 2396 if (err) { 2397 return err; 2398 } 2399 } 2400 2401 CPU_FOREACH(cpu) { 2402 err = kvm_update_guest_debug(cpu, 0); 2403 if (err) { 2404 return err; 2405 } 2406 } 2407 return 0; 2408 } 2409 2410 void kvm_remove_all_breakpoints(CPUState *cpu) 2411 { 2412 struct kvm_sw_breakpoint *bp, *next; 2413 KVMState *s = cpu->kvm_state; 2414 CPUState *tmpcpu; 2415 2416 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) { 2417 if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) { 2418 /* Try harder to find a CPU that currently sees the breakpoint. */ 2419 CPU_FOREACH(tmpcpu) { 2420 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) { 2421 break; 2422 } 2423 } 2424 } 2425 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry); 2426 g_free(bp); 2427 } 2428 kvm_arch_remove_all_hw_breakpoints(); 2429 2430 CPU_FOREACH(cpu) { 2431 kvm_update_guest_debug(cpu, 0); 2432 } 2433 } 2434 2435 #else /* !KVM_CAP_SET_GUEST_DEBUG */ 2436 2437 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap) 2438 { 2439 return -EINVAL; 2440 } 2441 2442 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr, 2443 target_ulong len, int type) 2444 { 2445 return -EINVAL; 2446 } 2447 2448 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr, 2449 target_ulong len, int type) 2450 { 2451 return -EINVAL; 2452 } 2453 2454 void kvm_remove_all_breakpoints(CPUState *cpu) 2455 { 2456 } 2457 #endif /* !KVM_CAP_SET_GUEST_DEBUG */ 2458 2459 static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset) 2460 { 2461 KVMState *s = kvm_state; 2462 struct kvm_signal_mask *sigmask; 2463 int r; 2464 2465 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset)); 2466 2467 sigmask->len = s->sigmask_len; 2468 memcpy(sigmask->sigset, sigset, sizeof(*sigset)); 2469 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask); 2470 g_free(sigmask); 2471 2472 return r; 2473 } 2474 2475 static void kvm_ipi_signal(int sig) 2476 { 2477 if (current_cpu) { 2478 assert(kvm_immediate_exit); 2479 kvm_cpu_kick(current_cpu); 2480 } 2481 } 2482 2483 void kvm_init_cpu_signals(CPUState *cpu) 2484 { 2485 int r; 2486 sigset_t set; 2487 struct sigaction sigact; 2488 2489 memset(&sigact, 0, sizeof(sigact)); 2490 sigact.sa_handler = kvm_ipi_signal; 2491 sigaction(SIG_IPI, &sigact, NULL); 2492 2493 pthread_sigmask(SIG_BLOCK, NULL, &set); 2494 #if defined KVM_HAVE_MCE_INJECTION 2495 sigdelset(&set, SIGBUS); 2496 pthread_sigmask(SIG_SETMASK, &set, NULL); 2497 #endif 2498 sigdelset(&set, SIG_IPI); 2499 if (kvm_immediate_exit) { 2500 r = pthread_sigmask(SIG_SETMASK, &set, NULL); 2501 } else { 2502 r = kvm_set_signal_mask(cpu, &set); 2503 } 2504 if (r) { 2505 fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r)); 2506 exit(1); 2507 } 2508 } 2509 2510 /* Called asynchronously in VCPU thread. */ 2511 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr) 2512 { 2513 #ifdef KVM_HAVE_MCE_INJECTION 2514 if (have_sigbus_pending) { 2515 return 1; 2516 } 2517 have_sigbus_pending = true; 2518 pending_sigbus_addr = addr; 2519 pending_sigbus_code = code; 2520 atomic_set(&cpu->exit_request, 1); 2521 return 0; 2522 #else 2523 return 1; 2524 #endif 2525 } 2526 2527 /* Called synchronously (via signalfd) in main thread. */ 2528 int kvm_on_sigbus(int code, void *addr) 2529 { 2530 #ifdef KVM_HAVE_MCE_INJECTION 2531 /* Action required MCE kills the process if SIGBUS is blocked. Because 2532 * that's what happens in the I/O thread, where we handle MCE via signalfd, 2533 * we can only get action optional here. 2534 */ 2535 assert(code != BUS_MCEERR_AR); 2536 kvm_arch_on_sigbus_vcpu(first_cpu, code, addr); 2537 return 0; 2538 #else 2539 return 1; 2540 #endif 2541 } 2542 2543 int kvm_create_device(KVMState *s, uint64_t type, bool test) 2544 { 2545 int ret; 2546 struct kvm_create_device create_dev; 2547 2548 create_dev.type = type; 2549 create_dev.fd = -1; 2550 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0; 2551 2552 if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) { 2553 return -ENOTSUP; 2554 } 2555 2556 ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev); 2557 if (ret) { 2558 return ret; 2559 } 2560 2561 return test ? 0 : create_dev.fd; 2562 } 2563 2564 bool kvm_device_supported(int vmfd, uint64_t type) 2565 { 2566 struct kvm_create_device create_dev = { 2567 .type = type, 2568 .fd = -1, 2569 .flags = KVM_CREATE_DEVICE_TEST, 2570 }; 2571 2572 if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) { 2573 return false; 2574 } 2575 2576 return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0); 2577 } 2578 2579 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source) 2580 { 2581 struct kvm_one_reg reg; 2582 int r; 2583 2584 reg.id = id; 2585 reg.addr = (uintptr_t) source; 2586 r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); 2587 if (r) { 2588 trace_kvm_failed_reg_set(id, strerror(-r)); 2589 } 2590 return r; 2591 } 2592 2593 int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target) 2594 { 2595 struct kvm_one_reg reg; 2596 int r; 2597 2598 reg.id = id; 2599 reg.addr = (uintptr_t) target; 2600 r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®); 2601 if (r) { 2602 trace_kvm_failed_reg_get(id, strerror(-r)); 2603 } 2604 return r; 2605 } 2606 2607 static void kvm_accel_class_init(ObjectClass *oc, void *data) 2608 { 2609 AccelClass *ac = ACCEL_CLASS(oc); 2610 ac->name = "KVM"; 2611 ac->init_machine = kvm_init; 2612 ac->allowed = &kvm_allowed; 2613 } 2614 2615 static const TypeInfo kvm_accel_type = { 2616 .name = TYPE_KVM_ACCEL, 2617 .parent = TYPE_ACCEL, 2618 .class_init = kvm_accel_class_init, 2619 .instance_size = sizeof(KVMState), 2620 }; 2621 2622 static void kvm_type_init(void) 2623 { 2624 type_register_static(&kvm_accel_type); 2625 } 2626 2627 type_init(kvm_type_init); 2628