1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include CONFIG_DEVICES /* CONFIG_IOMMUFD */ 23 #include <sys/ioctl.h> 24 #ifdef CONFIG_KVM 25 #include <linux/kvm.h> 26 #endif 27 #include <linux/vfio.h> 28 29 #include "hw/vfio/vfio-common.h" 30 #include "hw/vfio/pci.h" 31 #include "exec/address-spaces.h" 32 #include "exec/memory.h" 33 #include "exec/ram_addr.h" 34 #include "hw/hw.h" 35 #include "qemu/error-report.h" 36 #include "qemu/main-loop.h" 37 #include "qemu/range.h" 38 #include "sysemu/kvm.h" 39 #include "sysemu/reset.h" 40 #include "sysemu/runstate.h" 41 #include "trace.h" 42 #include "qapi/error.h" 43 #include "migration/migration.h" 44 #include "migration/misc.h" 45 #include "migration/blocker.h" 46 #include "migration/qemu-file.h" 47 #include "sysemu/tpm.h" 48 49 VFIODeviceList vfio_device_list = 50 QLIST_HEAD_INITIALIZER(vfio_device_list); 51 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 52 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 53 54 #ifdef CONFIG_KVM 55 /* 56 * We have a single VFIO pseudo device per KVM VM. Once created it lives 57 * for the life of the VM. Closing the file descriptor only drops our 58 * reference to it and the device's reference to kvm. Therefore once 59 * initialized, this file descriptor is only released on QEMU exit and 60 * we'll re-use it should another vfio device be attached before then. 61 */ 62 int vfio_kvm_device_fd = -1; 63 #endif 64 65 /* 66 * Device state interfaces 67 */ 68 69 bool vfio_mig_active(void) 70 { 71 VFIODevice *vbasedev; 72 73 if (QLIST_EMPTY(&vfio_device_list)) { 74 return false; 75 } 76 77 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 78 if (vbasedev->migration_blocker) { 79 return false; 80 } 81 } 82 return true; 83 } 84 85 static Error *multiple_devices_migration_blocker; 86 87 /* 88 * Multiple devices migration is allowed only if all devices support P2P 89 * migration. Single device migration is allowed regardless of P2P migration 90 * support. 91 */ 92 static bool vfio_multiple_devices_migration_is_supported(void) 93 { 94 VFIODevice *vbasedev; 95 unsigned int device_num = 0; 96 bool all_support_p2p = true; 97 98 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 99 if (vbasedev->migration) { 100 device_num++; 101 102 if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) { 103 all_support_p2p = false; 104 } 105 } 106 } 107 108 return all_support_p2p || device_num <= 1; 109 } 110 111 int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp) 112 { 113 int ret; 114 115 if (vfio_multiple_devices_migration_is_supported()) { 116 return 0; 117 } 118 119 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { 120 error_setg(errp, "Multiple VFIO devices migration is supported only if " 121 "all of them support P2P migration"); 122 return -EINVAL; 123 } 124 125 if (multiple_devices_migration_blocker) { 126 return 0; 127 } 128 129 error_setg(&multiple_devices_migration_blocker, 130 "Multiple VFIO devices migration is supported only if all of " 131 "them support P2P migration"); 132 ret = migrate_add_blocker(&multiple_devices_migration_blocker, errp); 133 134 return ret; 135 } 136 137 void vfio_unblock_multiple_devices_migration(void) 138 { 139 if (!multiple_devices_migration_blocker || 140 !vfio_multiple_devices_migration_is_supported()) { 141 return; 142 } 143 144 migrate_del_blocker(&multiple_devices_migration_blocker); 145 } 146 147 bool vfio_viommu_preset(VFIODevice *vbasedev) 148 { 149 return vbasedev->bcontainer->space->as != &address_space_memory; 150 } 151 152 static void vfio_set_migration_error(int err) 153 { 154 MigrationState *ms = migrate_get_current(); 155 156 if (migration_is_setup_or_active(ms->state)) { 157 WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { 158 if (ms->to_dst_file) { 159 qemu_file_set_error(ms->to_dst_file, err); 160 } 161 } 162 } 163 } 164 165 bool vfio_device_state_is_running(VFIODevice *vbasedev) 166 { 167 VFIOMigration *migration = vbasedev->migration; 168 169 return migration->device_state == VFIO_DEVICE_STATE_RUNNING || 170 migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P; 171 } 172 173 bool vfio_device_state_is_precopy(VFIODevice *vbasedev) 174 { 175 VFIOMigration *migration = vbasedev->migration; 176 177 return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY || 178 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P; 179 } 180 181 static bool vfio_devices_all_dirty_tracking(VFIOContainerBase *bcontainer) 182 { 183 VFIODevice *vbasedev; 184 MigrationState *ms = migrate_get_current(); 185 186 if (ms->state != MIGRATION_STATUS_ACTIVE && 187 ms->state != MIGRATION_STATUS_DEVICE) { 188 return false; 189 } 190 191 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 192 VFIOMigration *migration = vbasedev->migration; 193 194 if (!migration) { 195 return false; 196 } 197 198 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 199 (vfio_device_state_is_running(vbasedev) || 200 vfio_device_state_is_precopy(vbasedev))) { 201 return false; 202 } 203 } 204 return true; 205 } 206 207 bool vfio_devices_all_device_dirty_tracking(const VFIOContainerBase *bcontainer) 208 { 209 VFIODevice *vbasedev; 210 211 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 212 if (!vbasedev->dirty_pages_supported) { 213 return false; 214 } 215 } 216 217 return true; 218 } 219 220 /* 221 * Check if all VFIO devices are running and migration is active, which is 222 * essentially equivalent to the migration being in pre-copy phase. 223 */ 224 bool 225 vfio_devices_all_running_and_mig_active(const VFIOContainerBase *bcontainer) 226 { 227 VFIODevice *vbasedev; 228 229 if (!migration_is_active(migrate_get_current())) { 230 return false; 231 } 232 233 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 234 VFIOMigration *migration = vbasedev->migration; 235 236 if (!migration) { 237 return false; 238 } 239 240 if (vfio_device_state_is_running(vbasedev) || 241 vfio_device_state_is_precopy(vbasedev)) { 242 continue; 243 } else { 244 return false; 245 } 246 } 247 return true; 248 } 249 250 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 251 { 252 return (!memory_region_is_ram(section->mr) && 253 !memory_region_is_iommu(section->mr)) || 254 memory_region_is_protected(section->mr) || 255 /* 256 * Sizing an enabled 64-bit BAR can cause spurious mappings to 257 * addresses in the upper part of the 64-bit address space. These 258 * are never accessed by the CPU and beyond the address width of 259 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 260 */ 261 section->offset_within_address_space & (1ULL << 63); 262 } 263 264 /* Called with rcu_read_lock held. */ 265 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 266 ram_addr_t *ram_addr, bool *read_only) 267 { 268 bool ret, mr_has_discard_manager; 269 270 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, 271 &mr_has_discard_manager); 272 if (ret && mr_has_discard_manager) { 273 /* 274 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 275 * pages will remain pinned inside vfio until unmapped, resulting in a 276 * higher memory consumption than expected. If memory would get 277 * populated again later, there would be an inconsistency between pages 278 * pinned by vfio and pages seen by QEMU. This is the case until 279 * unmapped from the IOMMU (e.g., during device reset). 280 * 281 * With malicious guests, we really only care about pinning more memory 282 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 283 * exceeded and can be used to mitigate this problem. 284 */ 285 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 286 " RAM (e.g., virtio-mem) works, however, malicious" 287 " guests can trigger pinning of more memory than" 288 " intended via an IOMMU. It's possible to mitigate " 289 " by setting/adjusting RLIMIT_MEMLOCK."); 290 } 291 return ret; 292 } 293 294 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 295 { 296 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 297 VFIOContainerBase *bcontainer = giommu->bcontainer; 298 hwaddr iova = iotlb->iova + giommu->iommu_offset; 299 void *vaddr; 300 int ret; 301 302 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 303 iova, iova + iotlb->addr_mask); 304 305 if (iotlb->target_as != &address_space_memory) { 306 error_report("Wrong target AS \"%s\", only system memory is allowed", 307 iotlb->target_as->name ? iotlb->target_as->name : "none"); 308 vfio_set_migration_error(-EINVAL); 309 return; 310 } 311 312 rcu_read_lock(); 313 314 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 315 bool read_only; 316 317 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only)) { 318 goto out; 319 } 320 /* 321 * vaddr is only valid until rcu_read_unlock(). But after 322 * vfio_dma_map has set up the mapping the pages will be 323 * pinned by the kernel. This makes sure that the RAM backend 324 * of vaddr will always be there, even if the memory object is 325 * destroyed and its backing memory munmap-ed. 326 */ 327 ret = vfio_container_dma_map(bcontainer, iova, 328 iotlb->addr_mask + 1, vaddr, 329 read_only); 330 if (ret) { 331 error_report("vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 332 "0x%"HWADDR_PRIx", %p) = %d (%s)", 333 bcontainer, iova, 334 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 335 } 336 } else { 337 ret = vfio_container_dma_unmap(bcontainer, iova, 338 iotlb->addr_mask + 1, iotlb); 339 if (ret) { 340 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 341 "0x%"HWADDR_PRIx") = %d (%s)", 342 bcontainer, iova, 343 iotlb->addr_mask + 1, ret, strerror(-ret)); 344 vfio_set_migration_error(ret); 345 } 346 } 347 out: 348 rcu_read_unlock(); 349 } 350 351 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 352 MemoryRegionSection *section) 353 { 354 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 355 listener); 356 VFIOContainerBase *bcontainer = vrdl->bcontainer; 357 const hwaddr size = int128_get64(section->size); 358 const hwaddr iova = section->offset_within_address_space; 359 int ret; 360 361 /* Unmap with a single call. */ 362 ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL); 363 if (ret) { 364 error_report("%s: vfio_container_dma_unmap() failed: %s", __func__, 365 strerror(-ret)); 366 } 367 } 368 369 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 370 MemoryRegionSection *section) 371 { 372 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 373 listener); 374 VFIOContainerBase *bcontainer = vrdl->bcontainer; 375 const hwaddr end = section->offset_within_region + 376 int128_get64(section->size); 377 hwaddr start, next, iova; 378 void *vaddr; 379 int ret; 380 381 /* 382 * Map in (aligned within memory region) minimum granularity, so we can 383 * unmap in minimum granularity later. 384 */ 385 for (start = section->offset_within_region; start < end; start = next) { 386 next = ROUND_UP(start + 1, vrdl->granularity); 387 next = MIN(next, end); 388 389 iova = start - section->offset_within_region + 390 section->offset_within_address_space; 391 vaddr = memory_region_get_ram_ptr(section->mr) + start; 392 393 ret = vfio_container_dma_map(bcontainer, iova, next - start, 394 vaddr, section->readonly); 395 if (ret) { 396 /* Rollback */ 397 vfio_ram_discard_notify_discard(rdl, section); 398 return ret; 399 } 400 } 401 return 0; 402 } 403 404 static void vfio_register_ram_discard_listener(VFIOContainerBase *bcontainer, 405 MemoryRegionSection *section) 406 { 407 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 408 VFIORamDiscardListener *vrdl; 409 410 /* Ignore some corner cases not relevant in practice. */ 411 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 412 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 413 TARGET_PAGE_SIZE)); 414 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 415 416 vrdl = g_new0(VFIORamDiscardListener, 1); 417 vrdl->bcontainer = bcontainer; 418 vrdl->mr = section->mr; 419 vrdl->offset_within_address_space = section->offset_within_address_space; 420 vrdl->size = int128_get64(section->size); 421 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 422 section->mr); 423 424 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 425 g_assert(bcontainer->pgsizes && 426 vrdl->granularity >= 1ULL << ctz64(bcontainer->pgsizes)); 427 428 ram_discard_listener_init(&vrdl->listener, 429 vfio_ram_discard_notify_populate, 430 vfio_ram_discard_notify_discard, true); 431 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 432 QLIST_INSERT_HEAD(&bcontainer->vrdl_list, vrdl, next); 433 434 /* 435 * Sanity-check if we have a theoretically problematic setup where we could 436 * exceed the maximum number of possible DMA mappings over time. We assume 437 * that each mapped section in the same address space as a RamDiscardManager 438 * section consumes exactly one DMA mapping, with the exception of 439 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 440 * in the same address space as RamDiscardManager sections. 441 * 442 * We assume that each section in the address space consumes one memslot. 443 * We take the number of KVM memory slots as a best guess for the maximum 444 * number of sections in the address space we could have over time, 445 * also consuming DMA mappings. 446 */ 447 if (bcontainer->dma_max_mappings) { 448 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 449 450 #ifdef CONFIG_KVM 451 if (kvm_enabled()) { 452 max_memslots = kvm_get_max_memslots(); 453 } 454 #endif 455 456 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 457 hwaddr start, end; 458 459 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 460 vrdl->granularity); 461 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 462 vrdl->granularity); 463 vrdl_mappings += (end - start) / vrdl->granularity; 464 vrdl_count++; 465 } 466 467 if (vrdl_mappings + max_memslots - vrdl_count > 468 bcontainer->dma_max_mappings) { 469 warn_report("%s: possibly running out of DMA mappings. E.g., try" 470 " increasing the 'block-size' of virtio-mem devies." 471 " Maximum possible DMA mappings: %d, Maximum possible" 472 " memslots: %d", __func__, bcontainer->dma_max_mappings, 473 max_memslots); 474 } 475 } 476 } 477 478 static void vfio_unregister_ram_discard_listener(VFIOContainerBase *bcontainer, 479 MemoryRegionSection *section) 480 { 481 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 482 VFIORamDiscardListener *vrdl = NULL; 483 484 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 485 if (vrdl->mr == section->mr && 486 vrdl->offset_within_address_space == 487 section->offset_within_address_space) { 488 break; 489 } 490 } 491 492 if (!vrdl) { 493 hw_error("vfio: Trying to unregister missing RAM discard listener"); 494 } 495 496 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 497 QLIST_REMOVE(vrdl, next); 498 g_free(vrdl); 499 } 500 501 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 502 { 503 MemoryRegion *mr = section->mr; 504 505 if (!TPM_IS_CRB(mr->owner)) { 506 return false; 507 } 508 509 /* this is a known safe misaligned region, just trace for debug purpose */ 510 trace_vfio_known_safe_misalignment(memory_region_name(mr), 511 section->offset_within_address_space, 512 section->offset_within_region, 513 qemu_real_host_page_size()); 514 return true; 515 } 516 517 static bool vfio_listener_valid_section(MemoryRegionSection *section, 518 const char *name) 519 { 520 if (vfio_listener_skipped_section(section)) { 521 trace_vfio_listener_region_skip(name, 522 section->offset_within_address_space, 523 section->offset_within_address_space + 524 int128_get64(int128_sub(section->size, int128_one()))); 525 return false; 526 } 527 528 if (unlikely((section->offset_within_address_space & 529 ~qemu_real_host_page_mask()) != 530 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 531 if (!vfio_known_safe_misalignment(section)) { 532 error_report("%s received unaligned region %s iova=0x%"PRIx64 533 " offset_within_region=0x%"PRIx64 534 " qemu_real_host_page_size=0x%"PRIxPTR, 535 __func__, memory_region_name(section->mr), 536 section->offset_within_address_space, 537 section->offset_within_region, 538 qemu_real_host_page_size()); 539 } 540 return false; 541 } 542 543 return true; 544 } 545 546 static bool vfio_get_section_iova_range(VFIOContainerBase *bcontainer, 547 MemoryRegionSection *section, 548 hwaddr *out_iova, hwaddr *out_end, 549 Int128 *out_llend) 550 { 551 Int128 llend; 552 hwaddr iova; 553 554 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 555 llend = int128_make64(section->offset_within_address_space); 556 llend = int128_add(llend, section->size); 557 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 558 559 if (int128_ge(int128_make64(iova), llend)) { 560 return false; 561 } 562 563 *out_iova = iova; 564 *out_end = int128_get64(int128_sub(llend, int128_one())); 565 if (out_llend) { 566 *out_llend = llend; 567 } 568 return true; 569 } 570 571 static void vfio_listener_region_add(MemoryListener *listener, 572 MemoryRegionSection *section) 573 { 574 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 575 listener); 576 hwaddr iova, end; 577 Int128 llend, llsize; 578 void *vaddr; 579 int ret; 580 Error *err = NULL; 581 582 if (!vfio_listener_valid_section(section, "region_add")) { 583 return; 584 } 585 586 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 587 &llend)) { 588 if (memory_region_is_ram_device(section->mr)) { 589 trace_vfio_listener_region_add_no_dma_map( 590 memory_region_name(section->mr), 591 section->offset_within_address_space, 592 int128_getlo(section->size), 593 qemu_real_host_page_size()); 594 } 595 return; 596 } 597 598 if (vfio_container_add_section_window(bcontainer, section, &err)) { 599 goto fail; 600 } 601 602 memory_region_ref(section->mr); 603 604 if (memory_region_is_iommu(section->mr)) { 605 VFIOGuestIOMMU *giommu; 606 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 607 int iommu_idx; 608 609 trace_vfio_listener_region_add_iommu(iova, end); 610 /* 611 * FIXME: For VFIO iommu types which have KVM acceleration to 612 * avoid bouncing all map/unmaps through qemu this way, this 613 * would be the right place to wire that up (tell the KVM 614 * device emulation the VFIO iommu handles to use). 615 */ 616 giommu = g_malloc0(sizeof(*giommu)); 617 giommu->iommu_mr = iommu_mr; 618 giommu->iommu_offset = section->offset_within_address_space - 619 section->offset_within_region; 620 giommu->bcontainer = bcontainer; 621 llend = int128_add(int128_make64(section->offset_within_region), 622 section->size); 623 llend = int128_sub(llend, int128_one()); 624 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 625 MEMTXATTRS_UNSPECIFIED); 626 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 627 IOMMU_NOTIFIER_IOTLB_EVENTS, 628 section->offset_within_region, 629 int128_get64(llend), 630 iommu_idx); 631 632 ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr, 633 bcontainer->pgsizes, 634 &err); 635 if (ret) { 636 g_free(giommu); 637 goto fail; 638 } 639 640 if (bcontainer->iova_ranges) { 641 ret = memory_region_iommu_set_iova_ranges(giommu->iommu_mr, 642 bcontainer->iova_ranges, 643 &err); 644 if (ret) { 645 g_free(giommu); 646 goto fail; 647 } 648 } 649 650 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 651 &err); 652 if (ret) { 653 g_free(giommu); 654 goto fail; 655 } 656 QLIST_INSERT_HEAD(&bcontainer->giommu_list, giommu, giommu_next); 657 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 658 659 return; 660 } 661 662 /* Here we assume that memory_region_is_ram(section->mr)==true */ 663 664 /* 665 * For RAM memory regions with a RamDiscardManager, we only want to map the 666 * actually populated parts - and update the mapping whenever we're notified 667 * about changes. 668 */ 669 if (memory_region_has_ram_discard_manager(section->mr)) { 670 vfio_register_ram_discard_listener(bcontainer, section); 671 return; 672 } 673 674 vaddr = memory_region_get_ram_ptr(section->mr) + 675 section->offset_within_region + 676 (iova - section->offset_within_address_space); 677 678 trace_vfio_listener_region_add_ram(iova, end, vaddr); 679 680 llsize = int128_sub(llend, int128_make64(iova)); 681 682 if (memory_region_is_ram_device(section->mr)) { 683 hwaddr pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 684 685 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 686 trace_vfio_listener_region_add_no_dma_map( 687 memory_region_name(section->mr), 688 section->offset_within_address_space, 689 int128_getlo(section->size), 690 pgmask + 1); 691 return; 692 } 693 } 694 695 ret = vfio_container_dma_map(bcontainer, iova, int128_get64(llsize), 696 vaddr, section->readonly); 697 if (ret) { 698 error_setg(&err, "vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 699 "0x%"HWADDR_PRIx", %p) = %d (%s)", 700 bcontainer, iova, int128_get64(llsize), vaddr, ret, 701 strerror(-ret)); 702 if (memory_region_is_ram_device(section->mr)) { 703 /* Allow unexpected mappings not to be fatal for RAM devices */ 704 error_report_err(err); 705 return; 706 } 707 goto fail; 708 } 709 710 return; 711 712 fail: 713 if (memory_region_is_ram_device(section->mr)) { 714 error_reportf_err(err, "PCI p2p may not work: "); 715 return; 716 } 717 /* 718 * On the initfn path, store the first error in the container so we 719 * can gracefully fail. Runtime, there's not much we can do other 720 * than throw a hardware error. 721 */ 722 if (!bcontainer->initialized) { 723 if (!bcontainer->error) { 724 error_propagate_prepend(&bcontainer->error, err, 725 "Region %s: ", 726 memory_region_name(section->mr)); 727 } else { 728 error_free(err); 729 } 730 } else { 731 error_report_err(err); 732 hw_error("vfio: DMA mapping failed, unable to continue"); 733 } 734 } 735 736 static void vfio_listener_region_del(MemoryListener *listener, 737 MemoryRegionSection *section) 738 { 739 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 740 listener); 741 hwaddr iova, end; 742 Int128 llend, llsize; 743 int ret; 744 bool try_unmap = true; 745 746 if (!vfio_listener_valid_section(section, "region_del")) { 747 return; 748 } 749 750 if (memory_region_is_iommu(section->mr)) { 751 VFIOGuestIOMMU *giommu; 752 753 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 754 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 755 giommu->n.start == section->offset_within_region) { 756 memory_region_unregister_iommu_notifier(section->mr, 757 &giommu->n); 758 QLIST_REMOVE(giommu, giommu_next); 759 g_free(giommu); 760 break; 761 } 762 } 763 764 /* 765 * FIXME: We assume the one big unmap below is adequate to 766 * remove any individual page mappings in the IOMMU which 767 * might have been copied into VFIO. This works for a page table 768 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 769 * That may not be true for all IOMMU types. 770 */ 771 } 772 773 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 774 &llend)) { 775 return; 776 } 777 778 llsize = int128_sub(llend, int128_make64(iova)); 779 780 trace_vfio_listener_region_del(iova, end); 781 782 if (memory_region_is_ram_device(section->mr)) { 783 hwaddr pgmask; 784 785 pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 786 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 787 } else if (memory_region_has_ram_discard_manager(section->mr)) { 788 vfio_unregister_ram_discard_listener(bcontainer, section); 789 /* Unregistering will trigger an unmap. */ 790 try_unmap = false; 791 } 792 793 if (try_unmap) { 794 if (int128_eq(llsize, int128_2_64())) { 795 /* The unmap ioctl doesn't accept a full 64-bit span. */ 796 llsize = int128_rshift(llsize, 1); 797 ret = vfio_container_dma_unmap(bcontainer, iova, 798 int128_get64(llsize), NULL); 799 if (ret) { 800 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 801 "0x%"HWADDR_PRIx") = %d (%s)", 802 bcontainer, iova, int128_get64(llsize), ret, 803 strerror(-ret)); 804 } 805 iova += int128_get64(llsize); 806 } 807 ret = vfio_container_dma_unmap(bcontainer, iova, 808 int128_get64(llsize), NULL); 809 if (ret) { 810 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 811 "0x%"HWADDR_PRIx") = %d (%s)", 812 bcontainer, iova, int128_get64(llsize), ret, 813 strerror(-ret)); 814 } 815 } 816 817 memory_region_unref(section->mr); 818 819 vfio_container_del_section_window(bcontainer, section); 820 } 821 822 typedef struct VFIODirtyRanges { 823 hwaddr min32; 824 hwaddr max32; 825 hwaddr min64; 826 hwaddr max64; 827 hwaddr minpci64; 828 hwaddr maxpci64; 829 } VFIODirtyRanges; 830 831 typedef struct VFIODirtyRangesListener { 832 VFIOContainerBase *bcontainer; 833 VFIODirtyRanges ranges; 834 MemoryListener listener; 835 } VFIODirtyRangesListener; 836 837 static bool vfio_section_is_vfio_pci(MemoryRegionSection *section, 838 VFIOContainerBase *bcontainer) 839 { 840 VFIOPCIDevice *pcidev; 841 VFIODevice *vbasedev; 842 Object *owner; 843 844 owner = memory_region_owner(section->mr); 845 846 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 847 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 848 continue; 849 } 850 pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 851 if (OBJECT(pcidev) == owner) { 852 return true; 853 } 854 } 855 856 return false; 857 } 858 859 static void vfio_dirty_tracking_update(MemoryListener *listener, 860 MemoryRegionSection *section) 861 { 862 VFIODirtyRangesListener *dirty = container_of(listener, 863 VFIODirtyRangesListener, 864 listener); 865 VFIODirtyRanges *range = &dirty->ranges; 866 hwaddr iova, end, *min, *max; 867 868 if (!vfio_listener_valid_section(section, "tracking_update") || 869 !vfio_get_section_iova_range(dirty->bcontainer, section, 870 &iova, &end, NULL)) { 871 return; 872 } 873 874 /* 875 * The address space passed to the dirty tracker is reduced to three ranges: 876 * one for 32-bit DMA ranges, one for 64-bit DMA ranges and one for the 877 * PCI 64-bit hole. 878 * 879 * The underlying reports of dirty will query a sub-interval of each of 880 * these ranges. 881 * 882 * The purpose of the three range handling is to handle known cases of big 883 * holes in the address space, like the x86 AMD 1T hole, and firmware (like 884 * OVMF) which may relocate the pci-hole64 to the end of the address space. 885 * The latter would otherwise generate large ranges for tracking, stressing 886 * the limits of supported hardware. The pci-hole32 will always be below 4G 887 * (overlapping or not) so it doesn't need special handling and is part of 888 * the 32-bit range. 889 * 890 * The alternative would be an IOVATree but that has a much bigger runtime 891 * overhead and unnecessary complexity. 892 */ 893 if (vfio_section_is_vfio_pci(section, dirty->bcontainer) && 894 iova >= UINT32_MAX) { 895 min = &range->minpci64; 896 max = &range->maxpci64; 897 } else { 898 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 899 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 900 } 901 if (*min > iova) { 902 *min = iova; 903 } 904 if (*max < end) { 905 *max = end; 906 } 907 908 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 909 return; 910 } 911 912 static const MemoryListener vfio_dirty_tracking_listener = { 913 .name = "vfio-tracking", 914 .region_add = vfio_dirty_tracking_update, 915 }; 916 917 static void vfio_dirty_tracking_init(VFIOContainerBase *bcontainer, 918 VFIODirtyRanges *ranges) 919 { 920 VFIODirtyRangesListener dirty; 921 922 memset(&dirty, 0, sizeof(dirty)); 923 dirty.ranges.min32 = UINT32_MAX; 924 dirty.ranges.min64 = UINT64_MAX; 925 dirty.ranges.minpci64 = UINT64_MAX; 926 dirty.listener = vfio_dirty_tracking_listener; 927 dirty.bcontainer = bcontainer; 928 929 memory_listener_register(&dirty.listener, 930 bcontainer->space->as); 931 932 *ranges = dirty.ranges; 933 934 /* 935 * The memory listener is synchronous, and used to calculate the range 936 * to dirty tracking. Unregister it after we are done as we are not 937 * interested in any follow-up updates. 938 */ 939 memory_listener_unregister(&dirty.listener); 940 } 941 942 static void vfio_devices_dma_logging_stop(VFIOContainerBase *bcontainer) 943 { 944 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 945 sizeof(uint64_t))] = {}; 946 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 947 VFIODevice *vbasedev; 948 949 feature->argsz = sizeof(buf); 950 feature->flags = VFIO_DEVICE_FEATURE_SET | 951 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 952 953 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 954 if (!vbasedev->dirty_tracking) { 955 continue; 956 } 957 958 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 959 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 960 vbasedev->name, -errno, strerror(errno)); 961 } 962 vbasedev->dirty_tracking = false; 963 } 964 } 965 966 static struct vfio_device_feature * 967 vfio_device_feature_dma_logging_start_create(VFIOContainerBase *bcontainer, 968 VFIODirtyRanges *tracking) 969 { 970 struct vfio_device_feature *feature; 971 size_t feature_size; 972 struct vfio_device_feature_dma_logging_control *control; 973 struct vfio_device_feature_dma_logging_range *ranges; 974 975 feature_size = sizeof(struct vfio_device_feature) + 976 sizeof(struct vfio_device_feature_dma_logging_control); 977 feature = g_try_malloc0(feature_size); 978 if (!feature) { 979 errno = ENOMEM; 980 return NULL; 981 } 982 feature->argsz = feature_size; 983 feature->flags = VFIO_DEVICE_FEATURE_SET | 984 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 985 986 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 987 control->page_size = qemu_real_host_page_size(); 988 989 /* 990 * DMA logging uAPI guarantees to support at least a number of ranges that 991 * fits into a single host kernel base page. 992 */ 993 control->num_ranges = !!tracking->max32 + !!tracking->max64 + 994 !!tracking->maxpci64; 995 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 996 control->num_ranges); 997 if (!ranges) { 998 g_free(feature); 999 errno = ENOMEM; 1000 1001 return NULL; 1002 } 1003 1004 control->ranges = (__u64)(uintptr_t)ranges; 1005 if (tracking->max32) { 1006 ranges->iova = tracking->min32; 1007 ranges->length = (tracking->max32 - tracking->min32) + 1; 1008 ranges++; 1009 } 1010 if (tracking->max64) { 1011 ranges->iova = tracking->min64; 1012 ranges->length = (tracking->max64 - tracking->min64) + 1; 1013 ranges++; 1014 } 1015 if (tracking->maxpci64) { 1016 ranges->iova = tracking->minpci64; 1017 ranges->length = (tracking->maxpci64 - tracking->minpci64) + 1; 1018 } 1019 1020 trace_vfio_device_dirty_tracking_start(control->num_ranges, 1021 tracking->min32, tracking->max32, 1022 tracking->min64, tracking->max64, 1023 tracking->minpci64, tracking->maxpci64); 1024 1025 return feature; 1026 } 1027 1028 static void vfio_device_feature_dma_logging_start_destroy( 1029 struct vfio_device_feature *feature) 1030 { 1031 struct vfio_device_feature_dma_logging_control *control = 1032 (struct vfio_device_feature_dma_logging_control *)feature->data; 1033 struct vfio_device_feature_dma_logging_range *ranges = 1034 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 1035 1036 g_free(ranges); 1037 g_free(feature); 1038 } 1039 1040 static int vfio_devices_dma_logging_start(VFIOContainerBase *bcontainer) 1041 { 1042 struct vfio_device_feature *feature; 1043 VFIODirtyRanges ranges; 1044 VFIODevice *vbasedev; 1045 int ret = 0; 1046 1047 vfio_dirty_tracking_init(bcontainer, &ranges); 1048 feature = vfio_device_feature_dma_logging_start_create(bcontainer, 1049 &ranges); 1050 if (!feature) { 1051 return -errno; 1052 } 1053 1054 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1055 if (vbasedev->dirty_tracking) { 1056 continue; 1057 } 1058 1059 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 1060 if (ret) { 1061 ret = -errno; 1062 error_report("%s: Failed to start DMA logging, err %d (%s)", 1063 vbasedev->name, ret, strerror(errno)); 1064 goto out; 1065 } 1066 vbasedev->dirty_tracking = true; 1067 } 1068 1069 out: 1070 if (ret) { 1071 vfio_devices_dma_logging_stop(bcontainer); 1072 } 1073 1074 vfio_device_feature_dma_logging_start_destroy(feature); 1075 1076 return ret; 1077 } 1078 1079 static void vfio_listener_log_global_start(MemoryListener *listener) 1080 { 1081 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1082 listener); 1083 int ret; 1084 1085 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1086 ret = vfio_devices_dma_logging_start(bcontainer); 1087 } else { 1088 ret = vfio_container_set_dirty_page_tracking(bcontainer, true); 1089 } 1090 1091 if (ret) { 1092 error_report("vfio: Could not start dirty page tracking, err: %d (%s)", 1093 ret, strerror(-ret)); 1094 vfio_set_migration_error(ret); 1095 } 1096 } 1097 1098 static void vfio_listener_log_global_stop(MemoryListener *listener) 1099 { 1100 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1101 listener); 1102 int ret = 0; 1103 1104 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1105 vfio_devices_dma_logging_stop(bcontainer); 1106 } else { 1107 ret = vfio_container_set_dirty_page_tracking(bcontainer, false); 1108 } 1109 1110 if (ret) { 1111 error_report("vfio: Could not stop dirty page tracking, err: %d (%s)", 1112 ret, strerror(-ret)); 1113 vfio_set_migration_error(ret); 1114 } 1115 } 1116 1117 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, 1118 hwaddr size, void *bitmap) 1119 { 1120 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 1121 sizeof(struct vfio_device_feature_dma_logging_report), 1122 sizeof(__u64))] = {}; 1123 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1124 struct vfio_device_feature_dma_logging_report *report = 1125 (struct vfio_device_feature_dma_logging_report *)feature->data; 1126 1127 report->iova = iova; 1128 report->length = size; 1129 report->page_size = qemu_real_host_page_size(); 1130 report->bitmap = (__u64)(uintptr_t)bitmap; 1131 1132 feature->argsz = sizeof(buf); 1133 feature->flags = VFIO_DEVICE_FEATURE_GET | 1134 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; 1135 1136 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1137 return -errno; 1138 } 1139 1140 return 0; 1141 } 1142 1143 int vfio_devices_query_dirty_bitmap(const VFIOContainerBase *bcontainer, 1144 VFIOBitmap *vbmap, hwaddr iova, 1145 hwaddr size) 1146 { 1147 VFIODevice *vbasedev; 1148 int ret; 1149 1150 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1151 ret = vfio_device_dma_logging_report(vbasedev, iova, size, 1152 vbmap->bitmap); 1153 if (ret) { 1154 error_report("%s: Failed to get DMA logging report, iova: " 1155 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx 1156 ", err: %d (%s)", 1157 vbasedev->name, iova, size, ret, strerror(-ret)); 1158 1159 return ret; 1160 } 1161 } 1162 1163 return 0; 1164 } 1165 1166 int vfio_get_dirty_bitmap(const VFIOContainerBase *bcontainer, uint64_t iova, 1167 uint64_t size, ram_addr_t ram_addr) 1168 { 1169 bool all_device_dirty_tracking = 1170 vfio_devices_all_device_dirty_tracking(bcontainer); 1171 uint64_t dirty_pages; 1172 VFIOBitmap vbmap; 1173 int ret; 1174 1175 if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { 1176 cpu_physical_memory_set_dirty_range(ram_addr, size, 1177 tcg_enabled() ? DIRTY_CLIENTS_ALL : 1178 DIRTY_CLIENTS_NOCODE); 1179 return 0; 1180 } 1181 1182 ret = vfio_bitmap_alloc(&vbmap, size); 1183 if (ret) { 1184 return ret; 1185 } 1186 1187 if (all_device_dirty_tracking) { 1188 ret = vfio_devices_query_dirty_bitmap(bcontainer, &vbmap, iova, size); 1189 } else { 1190 ret = vfio_container_query_dirty_bitmap(bcontainer, &vbmap, iova, size); 1191 } 1192 1193 if (ret) { 1194 goto out; 1195 } 1196 1197 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, 1198 vbmap.pages); 1199 1200 trace_vfio_get_dirty_bitmap(iova, size, vbmap.size, ram_addr, dirty_pages); 1201 out: 1202 g_free(vbmap.bitmap); 1203 1204 return ret; 1205 } 1206 1207 typedef struct { 1208 IOMMUNotifier n; 1209 VFIOGuestIOMMU *giommu; 1210 } vfio_giommu_dirty_notifier; 1211 1212 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1213 { 1214 vfio_giommu_dirty_notifier *gdn = container_of(n, 1215 vfio_giommu_dirty_notifier, n); 1216 VFIOGuestIOMMU *giommu = gdn->giommu; 1217 VFIOContainerBase *bcontainer = giommu->bcontainer; 1218 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1219 ram_addr_t translated_addr; 1220 int ret = -EINVAL; 1221 1222 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1223 1224 if (iotlb->target_as != &address_space_memory) { 1225 error_report("Wrong target AS \"%s\", only system memory is allowed", 1226 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1227 goto out; 1228 } 1229 1230 rcu_read_lock(); 1231 if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) { 1232 ret = vfio_get_dirty_bitmap(bcontainer, iova, iotlb->addr_mask + 1, 1233 translated_addr); 1234 if (ret) { 1235 error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1236 "0x%"HWADDR_PRIx") = %d (%s)", 1237 bcontainer, iova, iotlb->addr_mask + 1, ret, 1238 strerror(-ret)); 1239 } 1240 } 1241 rcu_read_unlock(); 1242 1243 out: 1244 if (ret) { 1245 vfio_set_migration_error(ret); 1246 } 1247 } 1248 1249 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1250 void *opaque) 1251 { 1252 const hwaddr size = int128_get64(section->size); 1253 const hwaddr iova = section->offset_within_address_space; 1254 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1255 section->offset_within_region; 1256 VFIORamDiscardListener *vrdl = opaque; 1257 1258 /* 1259 * Sync the whole mapped region (spanning multiple individual mappings) 1260 * in one go. 1261 */ 1262 return vfio_get_dirty_bitmap(vrdl->bcontainer, iova, size, ram_addr); 1263 } 1264 1265 static int 1266 vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainerBase *bcontainer, 1267 MemoryRegionSection *section) 1268 { 1269 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1270 VFIORamDiscardListener *vrdl = NULL; 1271 1272 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 1273 if (vrdl->mr == section->mr && 1274 vrdl->offset_within_address_space == 1275 section->offset_within_address_space) { 1276 break; 1277 } 1278 } 1279 1280 if (!vrdl) { 1281 hw_error("vfio: Trying to sync missing RAM discard listener"); 1282 } 1283 1284 /* 1285 * We only want/can synchronize the bitmap for actually mapped parts - 1286 * which correspond to populated parts. Replay all populated parts. 1287 */ 1288 return ram_discard_manager_replay_populated(rdm, section, 1289 vfio_ram_discard_get_dirty_bitmap, 1290 &vrdl); 1291 } 1292 1293 static int vfio_sync_dirty_bitmap(VFIOContainerBase *bcontainer, 1294 MemoryRegionSection *section) 1295 { 1296 ram_addr_t ram_addr; 1297 1298 if (memory_region_is_iommu(section->mr)) { 1299 VFIOGuestIOMMU *giommu; 1300 1301 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 1302 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1303 giommu->n.start == section->offset_within_region) { 1304 Int128 llend; 1305 vfio_giommu_dirty_notifier gdn = { .giommu = giommu }; 1306 int idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1307 MEMTXATTRS_UNSPECIFIED); 1308 1309 llend = int128_add(int128_make64(section->offset_within_region), 1310 section->size); 1311 llend = int128_sub(llend, int128_one()); 1312 1313 iommu_notifier_init(&gdn.n, 1314 vfio_iommu_map_dirty_notify, 1315 IOMMU_NOTIFIER_MAP, 1316 section->offset_within_region, 1317 int128_get64(llend), 1318 idx); 1319 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1320 break; 1321 } 1322 } 1323 return 0; 1324 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1325 return vfio_sync_ram_discard_listener_dirty_bitmap(bcontainer, section); 1326 } 1327 1328 ram_addr = memory_region_get_ram_addr(section->mr) + 1329 section->offset_within_region; 1330 1331 return vfio_get_dirty_bitmap(bcontainer, 1332 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1333 int128_get64(section->size), ram_addr); 1334 } 1335 1336 static void vfio_listener_log_sync(MemoryListener *listener, 1337 MemoryRegionSection *section) 1338 { 1339 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1340 listener); 1341 int ret; 1342 1343 if (vfio_listener_skipped_section(section)) { 1344 return; 1345 } 1346 1347 if (vfio_devices_all_dirty_tracking(bcontainer)) { 1348 ret = vfio_sync_dirty_bitmap(bcontainer, section); 1349 if (ret) { 1350 error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret, 1351 strerror(-ret)); 1352 vfio_set_migration_error(ret); 1353 } 1354 } 1355 } 1356 1357 const MemoryListener vfio_memory_listener = { 1358 .name = "vfio", 1359 .region_add = vfio_listener_region_add, 1360 .region_del = vfio_listener_region_del, 1361 .log_global_start = vfio_listener_log_global_start, 1362 .log_global_stop = vfio_listener_log_global_stop, 1363 .log_sync = vfio_listener_log_sync, 1364 }; 1365 1366 void vfio_reset_handler(void *opaque) 1367 { 1368 VFIODevice *vbasedev; 1369 1370 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1371 if (vbasedev->dev->realized) { 1372 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 1373 } 1374 } 1375 1376 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1377 if (vbasedev->dev->realized && vbasedev->needs_reset) { 1378 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 1379 } 1380 } 1381 } 1382 1383 int vfio_kvm_device_add_fd(int fd, Error **errp) 1384 { 1385 #ifdef CONFIG_KVM 1386 struct kvm_device_attr attr = { 1387 .group = KVM_DEV_VFIO_FILE, 1388 .attr = KVM_DEV_VFIO_FILE_ADD, 1389 .addr = (uint64_t)(unsigned long)&fd, 1390 }; 1391 1392 if (!kvm_enabled()) { 1393 return 0; 1394 } 1395 1396 if (vfio_kvm_device_fd < 0) { 1397 struct kvm_create_device cd = { 1398 .type = KVM_DEV_TYPE_VFIO, 1399 }; 1400 1401 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 1402 error_setg_errno(errp, errno, "Failed to create KVM VFIO device"); 1403 return -errno; 1404 } 1405 1406 vfio_kvm_device_fd = cd.fd; 1407 } 1408 1409 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1410 error_setg_errno(errp, errno, "Failed to add fd %d to KVM VFIO device", 1411 fd); 1412 return -errno; 1413 } 1414 #endif 1415 return 0; 1416 } 1417 1418 int vfio_kvm_device_del_fd(int fd, Error **errp) 1419 { 1420 #ifdef CONFIG_KVM 1421 struct kvm_device_attr attr = { 1422 .group = KVM_DEV_VFIO_FILE, 1423 .attr = KVM_DEV_VFIO_FILE_DEL, 1424 .addr = (uint64_t)(unsigned long)&fd, 1425 }; 1426 1427 if (vfio_kvm_device_fd < 0) { 1428 error_setg(errp, "KVM VFIO device isn't created yet"); 1429 return -EINVAL; 1430 } 1431 1432 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1433 error_setg_errno(errp, errno, 1434 "Failed to remove fd %d from KVM VFIO device", fd); 1435 return -errno; 1436 } 1437 #endif 1438 return 0; 1439 } 1440 1441 VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1442 { 1443 VFIOAddressSpace *space; 1444 1445 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1446 if (space->as == as) { 1447 return space; 1448 } 1449 } 1450 1451 /* No suitable VFIOAddressSpace, create a new one */ 1452 space = g_malloc0(sizeof(*space)); 1453 space->as = as; 1454 QLIST_INIT(&space->containers); 1455 1456 if (QLIST_EMPTY(&vfio_address_spaces)) { 1457 qemu_register_reset(vfio_reset_handler, NULL); 1458 } 1459 1460 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1461 1462 return space; 1463 } 1464 1465 void vfio_put_address_space(VFIOAddressSpace *space) 1466 { 1467 if (!QLIST_EMPTY(&space->containers)) { 1468 return; 1469 } 1470 1471 QLIST_REMOVE(space, list); 1472 g_free(space); 1473 1474 if (QLIST_EMPTY(&vfio_address_spaces)) { 1475 qemu_unregister_reset(vfio_reset_handler, NULL); 1476 } 1477 } 1478 1479 struct vfio_device_info *vfio_get_device_info(int fd) 1480 { 1481 struct vfio_device_info *info; 1482 uint32_t argsz = sizeof(*info); 1483 1484 info = g_malloc0(argsz); 1485 1486 retry: 1487 info->argsz = argsz; 1488 1489 if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) { 1490 g_free(info); 1491 return NULL; 1492 } 1493 1494 if (info->argsz > argsz) { 1495 argsz = info->argsz; 1496 info = g_realloc(info, argsz); 1497 goto retry; 1498 } 1499 1500 return info; 1501 } 1502 1503 int vfio_attach_device(char *name, VFIODevice *vbasedev, 1504 AddressSpace *as, Error **errp) 1505 { 1506 const VFIOIOMMUOps *ops = &vfio_legacy_ops; 1507 1508 #ifdef CONFIG_IOMMUFD 1509 if (vbasedev->iommufd) { 1510 ops = &vfio_iommufd_ops; 1511 } 1512 #endif 1513 return ops->attach_device(name, vbasedev, as, errp); 1514 } 1515 1516 void vfio_detach_device(VFIODevice *vbasedev) 1517 { 1518 if (!vbasedev->bcontainer) { 1519 return; 1520 } 1521 vbasedev->bcontainer->ops->detach_device(vbasedev); 1522 } 1523