1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/pci.h" 30 #include "exec/address-spaces.h" 31 #include "exec/memory.h" 32 #include "exec/ram_addr.h" 33 #include "hw/hw.h" 34 #include "qemu/error-report.h" 35 #include "qemu/main-loop.h" 36 #include "qemu/range.h" 37 #include "sysemu/kvm.h" 38 #include "sysemu/reset.h" 39 #include "sysemu/runstate.h" 40 #include "trace.h" 41 #include "qapi/error.h" 42 #include "migration/migration.h" 43 #include "migration/misc.h" 44 #include "migration/blocker.h" 45 #include "migration/qemu-file.h" 46 #include "sysemu/tpm.h" 47 48 VFIODeviceList vfio_device_list = 49 QLIST_HEAD_INITIALIZER(vfio_device_list); 50 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 51 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 52 53 #ifdef CONFIG_KVM 54 /* 55 * We have a single VFIO pseudo device per KVM VM. Once created it lives 56 * for the life of the VM. Closing the file descriptor only drops our 57 * reference to it and the device's reference to kvm. Therefore once 58 * initialized, this file descriptor is only released on QEMU exit and 59 * we'll re-use it should another vfio device be attached before then. 60 */ 61 int vfio_kvm_device_fd = -1; 62 #endif 63 64 /* 65 * Device state interfaces 66 */ 67 68 bool vfio_mig_active(void) 69 { 70 VFIODevice *vbasedev; 71 72 if (QLIST_EMPTY(&vfio_device_list)) { 73 return false; 74 } 75 76 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 77 if (vbasedev->migration_blocker) { 78 return false; 79 } 80 } 81 return true; 82 } 83 84 static Error *multiple_devices_migration_blocker; 85 86 /* 87 * Multiple devices migration is allowed only if all devices support P2P 88 * migration. Single device migration is allowed regardless of P2P migration 89 * support. 90 */ 91 static bool vfio_multiple_devices_migration_is_supported(void) 92 { 93 VFIODevice *vbasedev; 94 unsigned int device_num = 0; 95 bool all_support_p2p = true; 96 97 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 98 if (vbasedev->migration) { 99 device_num++; 100 101 if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) { 102 all_support_p2p = false; 103 } 104 } 105 } 106 107 return all_support_p2p || device_num <= 1; 108 } 109 110 int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp) 111 { 112 int ret; 113 114 if (vfio_multiple_devices_migration_is_supported()) { 115 return 0; 116 } 117 118 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { 119 error_setg(errp, "Multiple VFIO devices migration is supported only if " 120 "all of them support P2P migration"); 121 return -EINVAL; 122 } 123 124 if (multiple_devices_migration_blocker) { 125 return 0; 126 } 127 128 error_setg(&multiple_devices_migration_blocker, 129 "Multiple VFIO devices migration is supported only if all of " 130 "them support P2P migration"); 131 ret = migrate_add_blocker(&multiple_devices_migration_blocker, errp); 132 133 return ret; 134 } 135 136 void vfio_unblock_multiple_devices_migration(void) 137 { 138 if (!multiple_devices_migration_blocker || 139 !vfio_multiple_devices_migration_is_supported()) { 140 return; 141 } 142 143 migrate_del_blocker(&multiple_devices_migration_blocker); 144 } 145 146 bool vfio_viommu_preset(VFIODevice *vbasedev) 147 { 148 return vbasedev->bcontainer->space->as != &address_space_memory; 149 } 150 151 static void vfio_set_migration_error(int err) 152 { 153 MigrationState *ms = migrate_get_current(); 154 155 if (migration_is_setup_or_active(ms->state)) { 156 WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { 157 if (ms->to_dst_file) { 158 qemu_file_set_error(ms->to_dst_file, err); 159 } 160 } 161 } 162 } 163 164 bool vfio_device_state_is_running(VFIODevice *vbasedev) 165 { 166 VFIOMigration *migration = vbasedev->migration; 167 168 return migration->device_state == VFIO_DEVICE_STATE_RUNNING || 169 migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P; 170 } 171 172 bool vfio_device_state_is_precopy(VFIODevice *vbasedev) 173 { 174 VFIOMigration *migration = vbasedev->migration; 175 176 return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY || 177 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P; 178 } 179 180 static bool vfio_devices_all_dirty_tracking(VFIOContainerBase *bcontainer) 181 { 182 VFIODevice *vbasedev; 183 MigrationState *ms = migrate_get_current(); 184 185 if (ms->state != MIGRATION_STATUS_ACTIVE && 186 ms->state != MIGRATION_STATUS_DEVICE) { 187 return false; 188 } 189 190 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 191 VFIOMigration *migration = vbasedev->migration; 192 193 if (!migration) { 194 return false; 195 } 196 197 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 198 (vfio_device_state_is_running(vbasedev) || 199 vfio_device_state_is_precopy(vbasedev))) { 200 return false; 201 } 202 } 203 return true; 204 } 205 206 bool vfio_devices_all_device_dirty_tracking(VFIOContainerBase *bcontainer) 207 { 208 VFIODevice *vbasedev; 209 210 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 211 if (!vbasedev->dirty_pages_supported) { 212 return false; 213 } 214 } 215 216 return true; 217 } 218 219 /* 220 * Check if all VFIO devices are running and migration is active, which is 221 * essentially equivalent to the migration being in pre-copy phase. 222 */ 223 bool vfio_devices_all_running_and_mig_active(VFIOContainerBase *bcontainer) 224 { 225 VFIODevice *vbasedev; 226 227 if (!migration_is_active(migrate_get_current())) { 228 return false; 229 } 230 231 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 232 VFIOMigration *migration = vbasedev->migration; 233 234 if (!migration) { 235 return false; 236 } 237 238 if (vfio_device_state_is_running(vbasedev) || 239 vfio_device_state_is_precopy(vbasedev)) { 240 continue; 241 } else { 242 return false; 243 } 244 } 245 return true; 246 } 247 248 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 249 { 250 return (!memory_region_is_ram(section->mr) && 251 !memory_region_is_iommu(section->mr)) || 252 memory_region_is_protected(section->mr) || 253 /* 254 * Sizing an enabled 64-bit BAR can cause spurious mappings to 255 * addresses in the upper part of the 64-bit address space. These 256 * are never accessed by the CPU and beyond the address width of 257 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 258 */ 259 section->offset_within_address_space & (1ULL << 63); 260 } 261 262 /* Called with rcu_read_lock held. */ 263 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 264 ram_addr_t *ram_addr, bool *read_only) 265 { 266 bool ret, mr_has_discard_manager; 267 268 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, 269 &mr_has_discard_manager); 270 if (ret && mr_has_discard_manager) { 271 /* 272 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 273 * pages will remain pinned inside vfio until unmapped, resulting in a 274 * higher memory consumption than expected. If memory would get 275 * populated again later, there would be an inconsistency between pages 276 * pinned by vfio and pages seen by QEMU. This is the case until 277 * unmapped from the IOMMU (e.g., during device reset). 278 * 279 * With malicious guests, we really only care about pinning more memory 280 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 281 * exceeded and can be used to mitigate this problem. 282 */ 283 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 284 " RAM (e.g., virtio-mem) works, however, malicious" 285 " guests can trigger pinning of more memory than" 286 " intended via an IOMMU. It's possible to mitigate " 287 " by setting/adjusting RLIMIT_MEMLOCK."); 288 } 289 return ret; 290 } 291 292 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 293 { 294 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 295 VFIOContainerBase *bcontainer = giommu->bcontainer; 296 hwaddr iova = iotlb->iova + giommu->iommu_offset; 297 void *vaddr; 298 int ret; 299 300 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 301 iova, iova + iotlb->addr_mask); 302 303 if (iotlb->target_as != &address_space_memory) { 304 error_report("Wrong target AS \"%s\", only system memory is allowed", 305 iotlb->target_as->name ? iotlb->target_as->name : "none"); 306 vfio_set_migration_error(-EINVAL); 307 return; 308 } 309 310 rcu_read_lock(); 311 312 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 313 bool read_only; 314 315 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only)) { 316 goto out; 317 } 318 /* 319 * vaddr is only valid until rcu_read_unlock(). But after 320 * vfio_dma_map has set up the mapping the pages will be 321 * pinned by the kernel. This makes sure that the RAM backend 322 * of vaddr will always be there, even if the memory object is 323 * destroyed and its backing memory munmap-ed. 324 */ 325 ret = vfio_container_dma_map(bcontainer, iova, 326 iotlb->addr_mask + 1, vaddr, 327 read_only); 328 if (ret) { 329 error_report("vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 330 "0x%"HWADDR_PRIx", %p) = %d (%s)", 331 bcontainer, iova, 332 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 333 } 334 } else { 335 ret = vfio_container_dma_unmap(bcontainer, iova, 336 iotlb->addr_mask + 1, iotlb); 337 if (ret) { 338 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 339 "0x%"HWADDR_PRIx") = %d (%s)", 340 bcontainer, iova, 341 iotlb->addr_mask + 1, ret, strerror(-ret)); 342 vfio_set_migration_error(ret); 343 } 344 } 345 out: 346 rcu_read_unlock(); 347 } 348 349 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 350 MemoryRegionSection *section) 351 { 352 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 353 listener); 354 VFIOContainerBase *bcontainer = vrdl->bcontainer; 355 const hwaddr size = int128_get64(section->size); 356 const hwaddr iova = section->offset_within_address_space; 357 int ret; 358 359 /* Unmap with a single call. */ 360 ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL); 361 if (ret) { 362 error_report("%s: vfio_container_dma_unmap() failed: %s", __func__, 363 strerror(-ret)); 364 } 365 } 366 367 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 368 MemoryRegionSection *section) 369 { 370 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 371 listener); 372 VFIOContainerBase *bcontainer = vrdl->bcontainer; 373 const hwaddr end = section->offset_within_region + 374 int128_get64(section->size); 375 hwaddr start, next, iova; 376 void *vaddr; 377 int ret; 378 379 /* 380 * Map in (aligned within memory region) minimum granularity, so we can 381 * unmap in minimum granularity later. 382 */ 383 for (start = section->offset_within_region; start < end; start = next) { 384 next = ROUND_UP(start + 1, vrdl->granularity); 385 next = MIN(next, end); 386 387 iova = start - section->offset_within_region + 388 section->offset_within_address_space; 389 vaddr = memory_region_get_ram_ptr(section->mr) + start; 390 391 ret = vfio_container_dma_map(bcontainer, iova, next - start, 392 vaddr, section->readonly); 393 if (ret) { 394 /* Rollback */ 395 vfio_ram_discard_notify_discard(rdl, section); 396 return ret; 397 } 398 } 399 return 0; 400 } 401 402 static void vfio_register_ram_discard_listener(VFIOContainerBase *bcontainer, 403 MemoryRegionSection *section) 404 { 405 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 406 VFIORamDiscardListener *vrdl; 407 408 /* Ignore some corner cases not relevant in practice. */ 409 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 410 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 411 TARGET_PAGE_SIZE)); 412 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 413 414 vrdl = g_new0(VFIORamDiscardListener, 1); 415 vrdl->bcontainer = bcontainer; 416 vrdl->mr = section->mr; 417 vrdl->offset_within_address_space = section->offset_within_address_space; 418 vrdl->size = int128_get64(section->size); 419 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 420 section->mr); 421 422 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 423 g_assert(bcontainer->pgsizes && 424 vrdl->granularity >= 1ULL << ctz64(bcontainer->pgsizes)); 425 426 ram_discard_listener_init(&vrdl->listener, 427 vfio_ram_discard_notify_populate, 428 vfio_ram_discard_notify_discard, true); 429 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 430 QLIST_INSERT_HEAD(&bcontainer->vrdl_list, vrdl, next); 431 432 /* 433 * Sanity-check if we have a theoretically problematic setup where we could 434 * exceed the maximum number of possible DMA mappings over time. We assume 435 * that each mapped section in the same address space as a RamDiscardManager 436 * section consumes exactly one DMA mapping, with the exception of 437 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 438 * in the same address space as RamDiscardManager sections. 439 * 440 * We assume that each section in the address space consumes one memslot. 441 * We take the number of KVM memory slots as a best guess for the maximum 442 * number of sections in the address space we could have over time, 443 * also consuming DMA mappings. 444 */ 445 if (bcontainer->dma_max_mappings) { 446 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 447 448 #ifdef CONFIG_KVM 449 if (kvm_enabled()) { 450 max_memslots = kvm_get_max_memslots(); 451 } 452 #endif 453 454 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 455 hwaddr start, end; 456 457 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 458 vrdl->granularity); 459 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 460 vrdl->granularity); 461 vrdl_mappings += (end - start) / vrdl->granularity; 462 vrdl_count++; 463 } 464 465 if (vrdl_mappings + max_memslots - vrdl_count > 466 bcontainer->dma_max_mappings) { 467 warn_report("%s: possibly running out of DMA mappings. E.g., try" 468 " increasing the 'block-size' of virtio-mem devies." 469 " Maximum possible DMA mappings: %d, Maximum possible" 470 " memslots: %d", __func__, bcontainer->dma_max_mappings, 471 max_memslots); 472 } 473 } 474 } 475 476 static void vfio_unregister_ram_discard_listener(VFIOContainerBase *bcontainer, 477 MemoryRegionSection *section) 478 { 479 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 480 VFIORamDiscardListener *vrdl = NULL; 481 482 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 483 if (vrdl->mr == section->mr && 484 vrdl->offset_within_address_space == 485 section->offset_within_address_space) { 486 break; 487 } 488 } 489 490 if (!vrdl) { 491 hw_error("vfio: Trying to unregister missing RAM discard listener"); 492 } 493 494 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 495 QLIST_REMOVE(vrdl, next); 496 g_free(vrdl); 497 } 498 499 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 500 { 501 MemoryRegion *mr = section->mr; 502 503 if (!TPM_IS_CRB(mr->owner)) { 504 return false; 505 } 506 507 /* this is a known safe misaligned region, just trace for debug purpose */ 508 trace_vfio_known_safe_misalignment(memory_region_name(mr), 509 section->offset_within_address_space, 510 section->offset_within_region, 511 qemu_real_host_page_size()); 512 return true; 513 } 514 515 static bool vfio_listener_valid_section(MemoryRegionSection *section, 516 const char *name) 517 { 518 if (vfio_listener_skipped_section(section)) { 519 trace_vfio_listener_region_skip(name, 520 section->offset_within_address_space, 521 section->offset_within_address_space + 522 int128_get64(int128_sub(section->size, int128_one()))); 523 return false; 524 } 525 526 if (unlikely((section->offset_within_address_space & 527 ~qemu_real_host_page_mask()) != 528 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 529 if (!vfio_known_safe_misalignment(section)) { 530 error_report("%s received unaligned region %s iova=0x%"PRIx64 531 " offset_within_region=0x%"PRIx64 532 " qemu_real_host_page_size=0x%"PRIxPTR, 533 __func__, memory_region_name(section->mr), 534 section->offset_within_address_space, 535 section->offset_within_region, 536 qemu_real_host_page_size()); 537 } 538 return false; 539 } 540 541 return true; 542 } 543 544 static bool vfio_get_section_iova_range(VFIOContainerBase *bcontainer, 545 MemoryRegionSection *section, 546 hwaddr *out_iova, hwaddr *out_end, 547 Int128 *out_llend) 548 { 549 Int128 llend; 550 hwaddr iova; 551 552 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 553 llend = int128_make64(section->offset_within_address_space); 554 llend = int128_add(llend, section->size); 555 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 556 557 if (int128_ge(int128_make64(iova), llend)) { 558 return false; 559 } 560 561 *out_iova = iova; 562 *out_end = int128_get64(int128_sub(llend, int128_one())); 563 if (out_llend) { 564 *out_llend = llend; 565 } 566 return true; 567 } 568 569 static void vfio_listener_region_add(MemoryListener *listener, 570 MemoryRegionSection *section) 571 { 572 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 573 listener); 574 hwaddr iova, end; 575 Int128 llend, llsize; 576 void *vaddr; 577 int ret; 578 Error *err = NULL; 579 580 if (!vfio_listener_valid_section(section, "region_add")) { 581 return; 582 } 583 584 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 585 &llend)) { 586 if (memory_region_is_ram_device(section->mr)) { 587 trace_vfio_listener_region_add_no_dma_map( 588 memory_region_name(section->mr), 589 section->offset_within_address_space, 590 int128_getlo(section->size), 591 qemu_real_host_page_size()); 592 } 593 return; 594 } 595 596 if (vfio_container_add_section_window(bcontainer, section, &err)) { 597 goto fail; 598 } 599 600 memory_region_ref(section->mr); 601 602 if (memory_region_is_iommu(section->mr)) { 603 VFIOGuestIOMMU *giommu; 604 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 605 int iommu_idx; 606 607 trace_vfio_listener_region_add_iommu(iova, end); 608 /* 609 * FIXME: For VFIO iommu types which have KVM acceleration to 610 * avoid bouncing all map/unmaps through qemu this way, this 611 * would be the right place to wire that up (tell the KVM 612 * device emulation the VFIO iommu handles to use). 613 */ 614 giommu = g_malloc0(sizeof(*giommu)); 615 giommu->iommu_mr = iommu_mr; 616 giommu->iommu_offset = section->offset_within_address_space - 617 section->offset_within_region; 618 giommu->bcontainer = bcontainer; 619 llend = int128_add(int128_make64(section->offset_within_region), 620 section->size); 621 llend = int128_sub(llend, int128_one()); 622 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 623 MEMTXATTRS_UNSPECIFIED); 624 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 625 IOMMU_NOTIFIER_IOTLB_EVENTS, 626 section->offset_within_region, 627 int128_get64(llend), 628 iommu_idx); 629 630 ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr, 631 bcontainer->pgsizes, 632 &err); 633 if (ret) { 634 g_free(giommu); 635 goto fail; 636 } 637 638 if (bcontainer->iova_ranges) { 639 ret = memory_region_iommu_set_iova_ranges(giommu->iommu_mr, 640 bcontainer->iova_ranges, 641 &err); 642 if (ret) { 643 g_free(giommu); 644 goto fail; 645 } 646 } 647 648 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 649 &err); 650 if (ret) { 651 g_free(giommu); 652 goto fail; 653 } 654 QLIST_INSERT_HEAD(&bcontainer->giommu_list, giommu, giommu_next); 655 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 656 657 return; 658 } 659 660 /* Here we assume that memory_region_is_ram(section->mr)==true */ 661 662 /* 663 * For RAM memory regions with a RamDiscardManager, we only want to map the 664 * actually populated parts - and update the mapping whenever we're notified 665 * about changes. 666 */ 667 if (memory_region_has_ram_discard_manager(section->mr)) { 668 vfio_register_ram_discard_listener(bcontainer, section); 669 return; 670 } 671 672 vaddr = memory_region_get_ram_ptr(section->mr) + 673 section->offset_within_region + 674 (iova - section->offset_within_address_space); 675 676 trace_vfio_listener_region_add_ram(iova, end, vaddr); 677 678 llsize = int128_sub(llend, int128_make64(iova)); 679 680 if (memory_region_is_ram_device(section->mr)) { 681 hwaddr pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 682 683 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 684 trace_vfio_listener_region_add_no_dma_map( 685 memory_region_name(section->mr), 686 section->offset_within_address_space, 687 int128_getlo(section->size), 688 pgmask + 1); 689 return; 690 } 691 } 692 693 ret = vfio_container_dma_map(bcontainer, iova, int128_get64(llsize), 694 vaddr, section->readonly); 695 if (ret) { 696 error_setg(&err, "vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 697 "0x%"HWADDR_PRIx", %p) = %d (%s)", 698 bcontainer, iova, int128_get64(llsize), vaddr, ret, 699 strerror(-ret)); 700 if (memory_region_is_ram_device(section->mr)) { 701 /* Allow unexpected mappings not to be fatal for RAM devices */ 702 error_report_err(err); 703 return; 704 } 705 goto fail; 706 } 707 708 return; 709 710 fail: 711 if (memory_region_is_ram_device(section->mr)) { 712 error_reportf_err(err, "PCI p2p may not work: "); 713 return; 714 } 715 /* 716 * On the initfn path, store the first error in the container so we 717 * can gracefully fail. Runtime, there's not much we can do other 718 * than throw a hardware error. 719 */ 720 if (!bcontainer->initialized) { 721 if (!bcontainer->error) { 722 error_propagate_prepend(&bcontainer->error, err, 723 "Region %s: ", 724 memory_region_name(section->mr)); 725 } else { 726 error_free(err); 727 } 728 } else { 729 error_report_err(err); 730 hw_error("vfio: DMA mapping failed, unable to continue"); 731 } 732 } 733 734 static void vfio_listener_region_del(MemoryListener *listener, 735 MemoryRegionSection *section) 736 { 737 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 738 listener); 739 hwaddr iova, end; 740 Int128 llend, llsize; 741 int ret; 742 bool try_unmap = true; 743 744 if (!vfio_listener_valid_section(section, "region_del")) { 745 return; 746 } 747 748 if (memory_region_is_iommu(section->mr)) { 749 VFIOGuestIOMMU *giommu; 750 751 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 752 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 753 giommu->n.start == section->offset_within_region) { 754 memory_region_unregister_iommu_notifier(section->mr, 755 &giommu->n); 756 QLIST_REMOVE(giommu, giommu_next); 757 g_free(giommu); 758 break; 759 } 760 } 761 762 /* 763 * FIXME: We assume the one big unmap below is adequate to 764 * remove any individual page mappings in the IOMMU which 765 * might have been copied into VFIO. This works for a page table 766 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 767 * That may not be true for all IOMMU types. 768 */ 769 } 770 771 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 772 &llend)) { 773 return; 774 } 775 776 llsize = int128_sub(llend, int128_make64(iova)); 777 778 trace_vfio_listener_region_del(iova, end); 779 780 if (memory_region_is_ram_device(section->mr)) { 781 hwaddr pgmask; 782 783 pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 784 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 785 } else if (memory_region_has_ram_discard_manager(section->mr)) { 786 vfio_unregister_ram_discard_listener(bcontainer, section); 787 /* Unregistering will trigger an unmap. */ 788 try_unmap = false; 789 } 790 791 if (try_unmap) { 792 if (int128_eq(llsize, int128_2_64())) { 793 /* The unmap ioctl doesn't accept a full 64-bit span. */ 794 llsize = int128_rshift(llsize, 1); 795 ret = vfio_container_dma_unmap(bcontainer, iova, 796 int128_get64(llsize), NULL); 797 if (ret) { 798 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 799 "0x%"HWADDR_PRIx") = %d (%s)", 800 bcontainer, iova, int128_get64(llsize), ret, 801 strerror(-ret)); 802 } 803 iova += int128_get64(llsize); 804 } 805 ret = vfio_container_dma_unmap(bcontainer, iova, 806 int128_get64(llsize), NULL); 807 if (ret) { 808 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 809 "0x%"HWADDR_PRIx") = %d (%s)", 810 bcontainer, iova, int128_get64(llsize), ret, 811 strerror(-ret)); 812 } 813 } 814 815 memory_region_unref(section->mr); 816 817 vfio_container_del_section_window(bcontainer, section); 818 } 819 820 typedef struct VFIODirtyRanges { 821 hwaddr min32; 822 hwaddr max32; 823 hwaddr min64; 824 hwaddr max64; 825 hwaddr minpci64; 826 hwaddr maxpci64; 827 } VFIODirtyRanges; 828 829 typedef struct VFIODirtyRangesListener { 830 VFIOContainerBase *bcontainer; 831 VFIODirtyRanges ranges; 832 MemoryListener listener; 833 } VFIODirtyRangesListener; 834 835 static bool vfio_section_is_vfio_pci(MemoryRegionSection *section, 836 VFIOContainerBase *bcontainer) 837 { 838 VFIOPCIDevice *pcidev; 839 VFIODevice *vbasedev; 840 Object *owner; 841 842 owner = memory_region_owner(section->mr); 843 844 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 845 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 846 continue; 847 } 848 pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 849 if (OBJECT(pcidev) == owner) { 850 return true; 851 } 852 } 853 854 return false; 855 } 856 857 static void vfio_dirty_tracking_update(MemoryListener *listener, 858 MemoryRegionSection *section) 859 { 860 VFIODirtyRangesListener *dirty = container_of(listener, 861 VFIODirtyRangesListener, 862 listener); 863 VFIODirtyRanges *range = &dirty->ranges; 864 hwaddr iova, end, *min, *max; 865 866 if (!vfio_listener_valid_section(section, "tracking_update") || 867 !vfio_get_section_iova_range(dirty->bcontainer, section, 868 &iova, &end, NULL)) { 869 return; 870 } 871 872 /* 873 * The address space passed to the dirty tracker is reduced to three ranges: 874 * one for 32-bit DMA ranges, one for 64-bit DMA ranges and one for the 875 * PCI 64-bit hole. 876 * 877 * The underlying reports of dirty will query a sub-interval of each of 878 * these ranges. 879 * 880 * The purpose of the three range handling is to handle known cases of big 881 * holes in the address space, like the x86 AMD 1T hole, and firmware (like 882 * OVMF) which may relocate the pci-hole64 to the end of the address space. 883 * The latter would otherwise generate large ranges for tracking, stressing 884 * the limits of supported hardware. The pci-hole32 will always be below 4G 885 * (overlapping or not) so it doesn't need special handling and is part of 886 * the 32-bit range. 887 * 888 * The alternative would be an IOVATree but that has a much bigger runtime 889 * overhead and unnecessary complexity. 890 */ 891 if (vfio_section_is_vfio_pci(section, dirty->bcontainer) && 892 iova >= UINT32_MAX) { 893 min = &range->minpci64; 894 max = &range->maxpci64; 895 } else { 896 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 897 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 898 } 899 if (*min > iova) { 900 *min = iova; 901 } 902 if (*max < end) { 903 *max = end; 904 } 905 906 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 907 return; 908 } 909 910 static const MemoryListener vfio_dirty_tracking_listener = { 911 .name = "vfio-tracking", 912 .region_add = vfio_dirty_tracking_update, 913 }; 914 915 static void vfio_dirty_tracking_init(VFIOContainerBase *bcontainer, 916 VFIODirtyRanges *ranges) 917 { 918 VFIODirtyRangesListener dirty; 919 920 memset(&dirty, 0, sizeof(dirty)); 921 dirty.ranges.min32 = UINT32_MAX; 922 dirty.ranges.min64 = UINT64_MAX; 923 dirty.ranges.minpci64 = UINT64_MAX; 924 dirty.listener = vfio_dirty_tracking_listener; 925 dirty.bcontainer = bcontainer; 926 927 memory_listener_register(&dirty.listener, 928 bcontainer->space->as); 929 930 *ranges = dirty.ranges; 931 932 /* 933 * The memory listener is synchronous, and used to calculate the range 934 * to dirty tracking. Unregister it after we are done as we are not 935 * interested in any follow-up updates. 936 */ 937 memory_listener_unregister(&dirty.listener); 938 } 939 940 static void vfio_devices_dma_logging_stop(VFIOContainerBase *bcontainer) 941 { 942 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 943 sizeof(uint64_t))] = {}; 944 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 945 VFIODevice *vbasedev; 946 947 feature->argsz = sizeof(buf); 948 feature->flags = VFIO_DEVICE_FEATURE_SET | 949 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 950 951 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 952 if (!vbasedev->dirty_tracking) { 953 continue; 954 } 955 956 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 957 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 958 vbasedev->name, -errno, strerror(errno)); 959 } 960 vbasedev->dirty_tracking = false; 961 } 962 } 963 964 static struct vfio_device_feature * 965 vfio_device_feature_dma_logging_start_create(VFIOContainerBase *bcontainer, 966 VFIODirtyRanges *tracking) 967 { 968 struct vfio_device_feature *feature; 969 size_t feature_size; 970 struct vfio_device_feature_dma_logging_control *control; 971 struct vfio_device_feature_dma_logging_range *ranges; 972 973 feature_size = sizeof(struct vfio_device_feature) + 974 sizeof(struct vfio_device_feature_dma_logging_control); 975 feature = g_try_malloc0(feature_size); 976 if (!feature) { 977 errno = ENOMEM; 978 return NULL; 979 } 980 feature->argsz = feature_size; 981 feature->flags = VFIO_DEVICE_FEATURE_SET | 982 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 983 984 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 985 control->page_size = qemu_real_host_page_size(); 986 987 /* 988 * DMA logging uAPI guarantees to support at least a number of ranges that 989 * fits into a single host kernel base page. 990 */ 991 control->num_ranges = !!tracking->max32 + !!tracking->max64 + 992 !!tracking->maxpci64; 993 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 994 control->num_ranges); 995 if (!ranges) { 996 g_free(feature); 997 errno = ENOMEM; 998 999 return NULL; 1000 } 1001 1002 control->ranges = (__u64)(uintptr_t)ranges; 1003 if (tracking->max32) { 1004 ranges->iova = tracking->min32; 1005 ranges->length = (tracking->max32 - tracking->min32) + 1; 1006 ranges++; 1007 } 1008 if (tracking->max64) { 1009 ranges->iova = tracking->min64; 1010 ranges->length = (tracking->max64 - tracking->min64) + 1; 1011 ranges++; 1012 } 1013 if (tracking->maxpci64) { 1014 ranges->iova = tracking->minpci64; 1015 ranges->length = (tracking->maxpci64 - tracking->minpci64) + 1; 1016 } 1017 1018 trace_vfio_device_dirty_tracking_start(control->num_ranges, 1019 tracking->min32, tracking->max32, 1020 tracking->min64, tracking->max64, 1021 tracking->minpci64, tracking->maxpci64); 1022 1023 return feature; 1024 } 1025 1026 static void vfio_device_feature_dma_logging_start_destroy( 1027 struct vfio_device_feature *feature) 1028 { 1029 struct vfio_device_feature_dma_logging_control *control = 1030 (struct vfio_device_feature_dma_logging_control *)feature->data; 1031 struct vfio_device_feature_dma_logging_range *ranges = 1032 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 1033 1034 g_free(ranges); 1035 g_free(feature); 1036 } 1037 1038 static int vfio_devices_dma_logging_start(VFIOContainerBase *bcontainer) 1039 { 1040 struct vfio_device_feature *feature; 1041 VFIODirtyRanges ranges; 1042 VFIODevice *vbasedev; 1043 int ret = 0; 1044 1045 vfio_dirty_tracking_init(bcontainer, &ranges); 1046 feature = vfio_device_feature_dma_logging_start_create(bcontainer, 1047 &ranges); 1048 if (!feature) { 1049 return -errno; 1050 } 1051 1052 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1053 if (vbasedev->dirty_tracking) { 1054 continue; 1055 } 1056 1057 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 1058 if (ret) { 1059 ret = -errno; 1060 error_report("%s: Failed to start DMA logging, err %d (%s)", 1061 vbasedev->name, ret, strerror(errno)); 1062 goto out; 1063 } 1064 vbasedev->dirty_tracking = true; 1065 } 1066 1067 out: 1068 if (ret) { 1069 vfio_devices_dma_logging_stop(bcontainer); 1070 } 1071 1072 vfio_device_feature_dma_logging_start_destroy(feature); 1073 1074 return ret; 1075 } 1076 1077 static void vfio_listener_log_global_start(MemoryListener *listener) 1078 { 1079 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1080 listener); 1081 int ret; 1082 1083 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1084 ret = vfio_devices_dma_logging_start(bcontainer); 1085 } else { 1086 ret = vfio_container_set_dirty_page_tracking(bcontainer, true); 1087 } 1088 1089 if (ret) { 1090 error_report("vfio: Could not start dirty page tracking, err: %d (%s)", 1091 ret, strerror(-ret)); 1092 vfio_set_migration_error(ret); 1093 } 1094 } 1095 1096 static void vfio_listener_log_global_stop(MemoryListener *listener) 1097 { 1098 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1099 listener); 1100 int ret = 0; 1101 1102 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1103 vfio_devices_dma_logging_stop(bcontainer); 1104 } else { 1105 ret = vfio_container_set_dirty_page_tracking(bcontainer, false); 1106 } 1107 1108 if (ret) { 1109 error_report("vfio: Could not stop dirty page tracking, err: %d (%s)", 1110 ret, strerror(-ret)); 1111 vfio_set_migration_error(ret); 1112 } 1113 } 1114 1115 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, 1116 hwaddr size, void *bitmap) 1117 { 1118 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 1119 sizeof(struct vfio_device_feature_dma_logging_report), 1120 sizeof(__u64))] = {}; 1121 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1122 struct vfio_device_feature_dma_logging_report *report = 1123 (struct vfio_device_feature_dma_logging_report *)feature->data; 1124 1125 report->iova = iova; 1126 report->length = size; 1127 report->page_size = qemu_real_host_page_size(); 1128 report->bitmap = (__u64)(uintptr_t)bitmap; 1129 1130 feature->argsz = sizeof(buf); 1131 feature->flags = VFIO_DEVICE_FEATURE_GET | 1132 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; 1133 1134 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1135 return -errno; 1136 } 1137 1138 return 0; 1139 } 1140 1141 int vfio_devices_query_dirty_bitmap(VFIOContainerBase *bcontainer, 1142 VFIOBitmap *vbmap, hwaddr iova, 1143 hwaddr size) 1144 { 1145 VFIODevice *vbasedev; 1146 int ret; 1147 1148 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1149 ret = vfio_device_dma_logging_report(vbasedev, iova, size, 1150 vbmap->bitmap); 1151 if (ret) { 1152 error_report("%s: Failed to get DMA logging report, iova: " 1153 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx 1154 ", err: %d (%s)", 1155 vbasedev->name, iova, size, ret, strerror(-ret)); 1156 1157 return ret; 1158 } 1159 } 1160 1161 return 0; 1162 } 1163 1164 int vfio_get_dirty_bitmap(VFIOContainerBase *bcontainer, uint64_t iova, 1165 uint64_t size, ram_addr_t ram_addr) 1166 { 1167 bool all_device_dirty_tracking = 1168 vfio_devices_all_device_dirty_tracking(bcontainer); 1169 uint64_t dirty_pages; 1170 VFIOBitmap vbmap; 1171 int ret; 1172 1173 if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { 1174 cpu_physical_memory_set_dirty_range(ram_addr, size, 1175 tcg_enabled() ? DIRTY_CLIENTS_ALL : 1176 DIRTY_CLIENTS_NOCODE); 1177 return 0; 1178 } 1179 1180 ret = vfio_bitmap_alloc(&vbmap, size); 1181 if (ret) { 1182 return ret; 1183 } 1184 1185 if (all_device_dirty_tracking) { 1186 ret = vfio_devices_query_dirty_bitmap(bcontainer, &vbmap, iova, size); 1187 } else { 1188 ret = vfio_container_query_dirty_bitmap(bcontainer, &vbmap, iova, size); 1189 } 1190 1191 if (ret) { 1192 goto out; 1193 } 1194 1195 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, 1196 vbmap.pages); 1197 1198 trace_vfio_get_dirty_bitmap(iova, size, vbmap.size, ram_addr, dirty_pages); 1199 out: 1200 g_free(vbmap.bitmap); 1201 1202 return ret; 1203 } 1204 1205 typedef struct { 1206 IOMMUNotifier n; 1207 VFIOGuestIOMMU *giommu; 1208 } vfio_giommu_dirty_notifier; 1209 1210 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1211 { 1212 vfio_giommu_dirty_notifier *gdn = container_of(n, 1213 vfio_giommu_dirty_notifier, n); 1214 VFIOGuestIOMMU *giommu = gdn->giommu; 1215 VFIOContainerBase *bcontainer = giommu->bcontainer; 1216 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1217 ram_addr_t translated_addr; 1218 int ret = -EINVAL; 1219 1220 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1221 1222 if (iotlb->target_as != &address_space_memory) { 1223 error_report("Wrong target AS \"%s\", only system memory is allowed", 1224 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1225 goto out; 1226 } 1227 1228 rcu_read_lock(); 1229 if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) { 1230 ret = vfio_get_dirty_bitmap(bcontainer, iova, iotlb->addr_mask + 1, 1231 translated_addr); 1232 if (ret) { 1233 error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1234 "0x%"HWADDR_PRIx") = %d (%s)", 1235 bcontainer, iova, iotlb->addr_mask + 1, ret, 1236 strerror(-ret)); 1237 } 1238 } 1239 rcu_read_unlock(); 1240 1241 out: 1242 if (ret) { 1243 vfio_set_migration_error(ret); 1244 } 1245 } 1246 1247 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1248 void *opaque) 1249 { 1250 const hwaddr size = int128_get64(section->size); 1251 const hwaddr iova = section->offset_within_address_space; 1252 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1253 section->offset_within_region; 1254 VFIORamDiscardListener *vrdl = opaque; 1255 1256 /* 1257 * Sync the whole mapped region (spanning multiple individual mappings) 1258 * in one go. 1259 */ 1260 return vfio_get_dirty_bitmap(vrdl->bcontainer, iova, size, ram_addr); 1261 } 1262 1263 static int 1264 vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainerBase *bcontainer, 1265 MemoryRegionSection *section) 1266 { 1267 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1268 VFIORamDiscardListener *vrdl = NULL; 1269 1270 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 1271 if (vrdl->mr == section->mr && 1272 vrdl->offset_within_address_space == 1273 section->offset_within_address_space) { 1274 break; 1275 } 1276 } 1277 1278 if (!vrdl) { 1279 hw_error("vfio: Trying to sync missing RAM discard listener"); 1280 } 1281 1282 /* 1283 * We only want/can synchronize the bitmap for actually mapped parts - 1284 * which correspond to populated parts. Replay all populated parts. 1285 */ 1286 return ram_discard_manager_replay_populated(rdm, section, 1287 vfio_ram_discard_get_dirty_bitmap, 1288 &vrdl); 1289 } 1290 1291 static int vfio_sync_dirty_bitmap(VFIOContainerBase *bcontainer, 1292 MemoryRegionSection *section) 1293 { 1294 ram_addr_t ram_addr; 1295 1296 if (memory_region_is_iommu(section->mr)) { 1297 VFIOGuestIOMMU *giommu; 1298 1299 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 1300 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1301 giommu->n.start == section->offset_within_region) { 1302 Int128 llend; 1303 vfio_giommu_dirty_notifier gdn = { .giommu = giommu }; 1304 int idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1305 MEMTXATTRS_UNSPECIFIED); 1306 1307 llend = int128_add(int128_make64(section->offset_within_region), 1308 section->size); 1309 llend = int128_sub(llend, int128_one()); 1310 1311 iommu_notifier_init(&gdn.n, 1312 vfio_iommu_map_dirty_notify, 1313 IOMMU_NOTIFIER_MAP, 1314 section->offset_within_region, 1315 int128_get64(llend), 1316 idx); 1317 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1318 break; 1319 } 1320 } 1321 return 0; 1322 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1323 return vfio_sync_ram_discard_listener_dirty_bitmap(bcontainer, section); 1324 } 1325 1326 ram_addr = memory_region_get_ram_addr(section->mr) + 1327 section->offset_within_region; 1328 1329 return vfio_get_dirty_bitmap(bcontainer, 1330 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1331 int128_get64(section->size), ram_addr); 1332 } 1333 1334 static void vfio_listener_log_sync(MemoryListener *listener, 1335 MemoryRegionSection *section) 1336 { 1337 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1338 listener); 1339 int ret; 1340 1341 if (vfio_listener_skipped_section(section)) { 1342 return; 1343 } 1344 1345 if (vfio_devices_all_dirty_tracking(bcontainer)) { 1346 ret = vfio_sync_dirty_bitmap(bcontainer, section); 1347 if (ret) { 1348 error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret, 1349 strerror(-ret)); 1350 vfio_set_migration_error(ret); 1351 } 1352 } 1353 } 1354 1355 const MemoryListener vfio_memory_listener = { 1356 .name = "vfio", 1357 .region_add = vfio_listener_region_add, 1358 .region_del = vfio_listener_region_del, 1359 .log_global_start = vfio_listener_log_global_start, 1360 .log_global_stop = vfio_listener_log_global_stop, 1361 .log_sync = vfio_listener_log_sync, 1362 }; 1363 1364 void vfio_reset_handler(void *opaque) 1365 { 1366 VFIODevice *vbasedev; 1367 1368 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1369 if (vbasedev->dev->realized) { 1370 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 1371 } 1372 } 1373 1374 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1375 if (vbasedev->dev->realized && vbasedev->needs_reset) { 1376 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 1377 } 1378 } 1379 } 1380 1381 int vfio_kvm_device_add_fd(int fd, Error **errp) 1382 { 1383 #ifdef CONFIG_KVM 1384 struct kvm_device_attr attr = { 1385 .group = KVM_DEV_VFIO_FILE, 1386 .attr = KVM_DEV_VFIO_FILE_ADD, 1387 .addr = (uint64_t)(unsigned long)&fd, 1388 }; 1389 1390 if (!kvm_enabled()) { 1391 return 0; 1392 } 1393 1394 if (vfio_kvm_device_fd < 0) { 1395 struct kvm_create_device cd = { 1396 .type = KVM_DEV_TYPE_VFIO, 1397 }; 1398 1399 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 1400 error_setg_errno(errp, errno, "Failed to create KVM VFIO device"); 1401 return -errno; 1402 } 1403 1404 vfio_kvm_device_fd = cd.fd; 1405 } 1406 1407 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1408 error_setg_errno(errp, errno, "Failed to add fd %d to KVM VFIO device", 1409 fd); 1410 return -errno; 1411 } 1412 #endif 1413 return 0; 1414 } 1415 1416 int vfio_kvm_device_del_fd(int fd, Error **errp) 1417 { 1418 #ifdef CONFIG_KVM 1419 struct kvm_device_attr attr = { 1420 .group = KVM_DEV_VFIO_FILE, 1421 .attr = KVM_DEV_VFIO_FILE_DEL, 1422 .addr = (uint64_t)(unsigned long)&fd, 1423 }; 1424 1425 if (vfio_kvm_device_fd < 0) { 1426 error_setg(errp, "KVM VFIO device isn't created yet"); 1427 return -EINVAL; 1428 } 1429 1430 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1431 error_setg_errno(errp, errno, 1432 "Failed to remove fd %d from KVM VFIO device", fd); 1433 return -errno; 1434 } 1435 #endif 1436 return 0; 1437 } 1438 1439 VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1440 { 1441 VFIOAddressSpace *space; 1442 1443 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1444 if (space->as == as) { 1445 return space; 1446 } 1447 } 1448 1449 /* No suitable VFIOAddressSpace, create a new one */ 1450 space = g_malloc0(sizeof(*space)); 1451 space->as = as; 1452 QLIST_INIT(&space->containers); 1453 1454 if (QLIST_EMPTY(&vfio_address_spaces)) { 1455 qemu_register_reset(vfio_reset_handler, NULL); 1456 } 1457 1458 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1459 1460 return space; 1461 } 1462 1463 void vfio_put_address_space(VFIOAddressSpace *space) 1464 { 1465 if (QLIST_EMPTY(&space->containers)) { 1466 QLIST_REMOVE(space, list); 1467 g_free(space); 1468 } 1469 if (QLIST_EMPTY(&vfio_address_spaces)) { 1470 qemu_unregister_reset(vfio_reset_handler, NULL); 1471 } 1472 } 1473 1474 struct vfio_device_info *vfio_get_device_info(int fd) 1475 { 1476 struct vfio_device_info *info; 1477 uint32_t argsz = sizeof(*info); 1478 1479 info = g_malloc0(argsz); 1480 1481 retry: 1482 info->argsz = argsz; 1483 1484 if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) { 1485 g_free(info); 1486 return NULL; 1487 } 1488 1489 if (info->argsz > argsz) { 1490 argsz = info->argsz; 1491 info = g_realloc(info, argsz); 1492 goto retry; 1493 } 1494 1495 return info; 1496 } 1497 1498 int vfio_attach_device(char *name, VFIODevice *vbasedev, 1499 AddressSpace *as, Error **errp) 1500 { 1501 const VFIOIOMMUOps *ops = &vfio_legacy_ops; 1502 1503 return ops->attach_device(name, vbasedev, as, errp); 1504 } 1505 1506 void vfio_detach_device(VFIODevice *vbasedev) 1507 { 1508 if (!vbasedev->bcontainer) { 1509 return; 1510 } 1511 vbasedev->bcontainer->ops->detach_device(vbasedev); 1512 } 1513