1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/pci.h" 30 #include "exec/address-spaces.h" 31 #include "exec/memory.h" 32 #include "exec/ram_addr.h" 33 #include "hw/hw.h" 34 #include "qemu/error-report.h" 35 #include "qemu/main-loop.h" 36 #include "qemu/range.h" 37 #include "sysemu/kvm.h" 38 #include "sysemu/reset.h" 39 #include "sysemu/runstate.h" 40 #include "trace.h" 41 #include "qapi/error.h" 42 #include "migration/misc.h" 43 #include "migration/blocker.h" 44 #include "migration/qemu-file.h" 45 #include "sysemu/tpm.h" 46 47 VFIODeviceList vfio_device_list = 48 QLIST_HEAD_INITIALIZER(vfio_device_list); 49 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 50 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 51 52 #ifdef CONFIG_KVM 53 /* 54 * We have a single VFIO pseudo device per KVM VM. Once created it lives 55 * for the life of the VM. Closing the file descriptor only drops our 56 * reference to it and the device's reference to kvm. Therefore once 57 * initialized, this file descriptor is only released on QEMU exit and 58 * we'll re-use it should another vfio device be attached before then. 59 */ 60 int vfio_kvm_device_fd = -1; 61 #endif 62 63 /* 64 * Device state interfaces 65 */ 66 67 bool vfio_mig_active(void) 68 { 69 VFIODevice *vbasedev; 70 71 if (QLIST_EMPTY(&vfio_device_list)) { 72 return false; 73 } 74 75 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 76 if (vbasedev->migration_blocker) { 77 return false; 78 } 79 } 80 return true; 81 } 82 83 static Error *multiple_devices_migration_blocker; 84 85 /* 86 * Multiple devices migration is allowed only if all devices support P2P 87 * migration. Single device migration is allowed regardless of P2P migration 88 * support. 89 */ 90 static bool vfio_multiple_devices_migration_is_supported(void) 91 { 92 VFIODevice *vbasedev; 93 unsigned int device_num = 0; 94 bool all_support_p2p = true; 95 96 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 97 if (vbasedev->migration) { 98 device_num++; 99 100 if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) { 101 all_support_p2p = false; 102 } 103 } 104 } 105 106 return all_support_p2p || device_num <= 1; 107 } 108 109 int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp) 110 { 111 int ret; 112 113 if (vfio_multiple_devices_migration_is_supported()) { 114 return 0; 115 } 116 117 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { 118 error_setg(errp, "Multiple VFIO devices migration is supported only if " 119 "all of them support P2P migration"); 120 return -EINVAL; 121 } 122 123 if (multiple_devices_migration_blocker) { 124 return 0; 125 } 126 127 error_setg(&multiple_devices_migration_blocker, 128 "Multiple VFIO devices migration is supported only if all of " 129 "them support P2P migration"); 130 ret = migrate_add_blocker_normal(&multiple_devices_migration_blocker, errp); 131 132 return ret; 133 } 134 135 void vfio_unblock_multiple_devices_migration(void) 136 { 137 if (!multiple_devices_migration_blocker || 138 !vfio_multiple_devices_migration_is_supported()) { 139 return; 140 } 141 142 migrate_del_blocker(&multiple_devices_migration_blocker); 143 } 144 145 bool vfio_viommu_preset(VFIODevice *vbasedev) 146 { 147 return vbasedev->bcontainer->space->as != &address_space_memory; 148 } 149 150 static void vfio_set_migration_error(int ret) 151 { 152 if (migration_is_setup_or_active()) { 153 migration_file_set_error(ret, NULL); 154 } 155 } 156 157 bool vfio_device_state_is_running(VFIODevice *vbasedev) 158 { 159 VFIOMigration *migration = vbasedev->migration; 160 161 return migration->device_state == VFIO_DEVICE_STATE_RUNNING || 162 migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P; 163 } 164 165 bool vfio_device_state_is_precopy(VFIODevice *vbasedev) 166 { 167 VFIOMigration *migration = vbasedev->migration; 168 169 return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY || 170 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P; 171 } 172 173 static bool vfio_devices_all_dirty_tracking(VFIOContainerBase *bcontainer) 174 { 175 VFIODevice *vbasedev; 176 177 if (!migration_is_active() && !migration_is_device()) { 178 return false; 179 } 180 181 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 182 VFIOMigration *migration = vbasedev->migration; 183 184 if (!migration) { 185 return false; 186 } 187 188 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 189 (vfio_device_state_is_running(vbasedev) || 190 vfio_device_state_is_precopy(vbasedev))) { 191 return false; 192 } 193 } 194 return true; 195 } 196 197 bool vfio_devices_all_device_dirty_tracking(const VFIOContainerBase *bcontainer) 198 { 199 VFIODevice *vbasedev; 200 201 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 202 if (!vbasedev->dirty_pages_supported) { 203 return false; 204 } 205 } 206 207 return true; 208 } 209 210 /* 211 * Check if all VFIO devices are running and migration is active, which is 212 * essentially equivalent to the migration being in pre-copy phase. 213 */ 214 bool 215 vfio_devices_all_running_and_mig_active(const VFIOContainerBase *bcontainer) 216 { 217 VFIODevice *vbasedev; 218 219 if (!migration_is_active()) { 220 return false; 221 } 222 223 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 224 VFIOMigration *migration = vbasedev->migration; 225 226 if (!migration) { 227 return false; 228 } 229 230 if (vfio_device_state_is_running(vbasedev) || 231 vfio_device_state_is_precopy(vbasedev)) { 232 continue; 233 } else { 234 return false; 235 } 236 } 237 return true; 238 } 239 240 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 241 { 242 return (!memory_region_is_ram(section->mr) && 243 !memory_region_is_iommu(section->mr)) || 244 memory_region_is_protected(section->mr) || 245 /* 246 * Sizing an enabled 64-bit BAR can cause spurious mappings to 247 * addresses in the upper part of the 64-bit address space. These 248 * are never accessed by the CPU and beyond the address width of 249 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 250 */ 251 section->offset_within_address_space & (1ULL << 63); 252 } 253 254 /* Called with rcu_read_lock held. */ 255 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 256 ram_addr_t *ram_addr, bool *read_only, 257 Error **errp) 258 { 259 bool ret, mr_has_discard_manager; 260 261 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, 262 &mr_has_discard_manager, errp); 263 if (ret && mr_has_discard_manager) { 264 /* 265 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 266 * pages will remain pinned inside vfio until unmapped, resulting in a 267 * higher memory consumption than expected. If memory would get 268 * populated again later, there would be an inconsistency between pages 269 * pinned by vfio and pages seen by QEMU. This is the case until 270 * unmapped from the IOMMU (e.g., during device reset). 271 * 272 * With malicious guests, we really only care about pinning more memory 273 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 274 * exceeded and can be used to mitigate this problem. 275 */ 276 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 277 " RAM (e.g., virtio-mem) works, however, malicious" 278 " guests can trigger pinning of more memory than" 279 " intended via an IOMMU. It's possible to mitigate " 280 " by setting/adjusting RLIMIT_MEMLOCK."); 281 } 282 return ret; 283 } 284 285 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 286 { 287 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 288 VFIOContainerBase *bcontainer = giommu->bcontainer; 289 hwaddr iova = iotlb->iova + giommu->iommu_offset; 290 void *vaddr; 291 int ret; 292 Error *local_err = NULL; 293 294 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 295 iova, iova + iotlb->addr_mask); 296 297 if (iotlb->target_as != &address_space_memory) { 298 error_report("Wrong target AS \"%s\", only system memory is allowed", 299 iotlb->target_as->name ? iotlb->target_as->name : "none"); 300 vfio_set_migration_error(-EINVAL); 301 return; 302 } 303 304 rcu_read_lock(); 305 306 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 307 bool read_only; 308 309 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, &local_err)) { 310 error_report_err(local_err); 311 goto out; 312 } 313 /* 314 * vaddr is only valid until rcu_read_unlock(). But after 315 * vfio_dma_map has set up the mapping the pages will be 316 * pinned by the kernel. This makes sure that the RAM backend 317 * of vaddr will always be there, even if the memory object is 318 * destroyed and its backing memory munmap-ed. 319 */ 320 ret = vfio_container_dma_map(bcontainer, iova, 321 iotlb->addr_mask + 1, vaddr, 322 read_only); 323 if (ret) { 324 error_report("vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 325 "0x%"HWADDR_PRIx", %p) = %d (%s)", 326 bcontainer, iova, 327 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 328 } 329 } else { 330 ret = vfio_container_dma_unmap(bcontainer, iova, 331 iotlb->addr_mask + 1, iotlb); 332 if (ret) { 333 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 334 "0x%"HWADDR_PRIx") = %d (%s)", 335 bcontainer, iova, 336 iotlb->addr_mask + 1, ret, strerror(-ret)); 337 vfio_set_migration_error(ret); 338 } 339 } 340 out: 341 rcu_read_unlock(); 342 } 343 344 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 345 MemoryRegionSection *section) 346 { 347 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 348 listener); 349 VFIOContainerBase *bcontainer = vrdl->bcontainer; 350 const hwaddr size = int128_get64(section->size); 351 const hwaddr iova = section->offset_within_address_space; 352 int ret; 353 354 /* Unmap with a single call. */ 355 ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL); 356 if (ret) { 357 error_report("%s: vfio_container_dma_unmap() failed: %s", __func__, 358 strerror(-ret)); 359 } 360 } 361 362 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 363 MemoryRegionSection *section) 364 { 365 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 366 listener); 367 VFIOContainerBase *bcontainer = vrdl->bcontainer; 368 const hwaddr end = section->offset_within_region + 369 int128_get64(section->size); 370 hwaddr start, next, iova; 371 void *vaddr; 372 int ret; 373 374 /* 375 * Map in (aligned within memory region) minimum granularity, so we can 376 * unmap in minimum granularity later. 377 */ 378 for (start = section->offset_within_region; start < end; start = next) { 379 next = ROUND_UP(start + 1, vrdl->granularity); 380 next = MIN(next, end); 381 382 iova = start - section->offset_within_region + 383 section->offset_within_address_space; 384 vaddr = memory_region_get_ram_ptr(section->mr) + start; 385 386 ret = vfio_container_dma_map(bcontainer, iova, next - start, 387 vaddr, section->readonly); 388 if (ret) { 389 /* Rollback */ 390 vfio_ram_discard_notify_discard(rdl, section); 391 return ret; 392 } 393 } 394 return 0; 395 } 396 397 static void vfio_register_ram_discard_listener(VFIOContainerBase *bcontainer, 398 MemoryRegionSection *section) 399 { 400 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 401 VFIORamDiscardListener *vrdl; 402 403 /* Ignore some corner cases not relevant in practice. */ 404 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 405 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 406 TARGET_PAGE_SIZE)); 407 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 408 409 vrdl = g_new0(VFIORamDiscardListener, 1); 410 vrdl->bcontainer = bcontainer; 411 vrdl->mr = section->mr; 412 vrdl->offset_within_address_space = section->offset_within_address_space; 413 vrdl->size = int128_get64(section->size); 414 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 415 section->mr); 416 417 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 418 g_assert(bcontainer->pgsizes && 419 vrdl->granularity >= 1ULL << ctz64(bcontainer->pgsizes)); 420 421 ram_discard_listener_init(&vrdl->listener, 422 vfio_ram_discard_notify_populate, 423 vfio_ram_discard_notify_discard, true); 424 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 425 QLIST_INSERT_HEAD(&bcontainer->vrdl_list, vrdl, next); 426 427 /* 428 * Sanity-check if we have a theoretically problematic setup where we could 429 * exceed the maximum number of possible DMA mappings over time. We assume 430 * that each mapped section in the same address space as a RamDiscardManager 431 * section consumes exactly one DMA mapping, with the exception of 432 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 433 * in the same address space as RamDiscardManager sections. 434 * 435 * We assume that each section in the address space consumes one memslot. 436 * We take the number of KVM memory slots as a best guess for the maximum 437 * number of sections in the address space we could have over time, 438 * also consuming DMA mappings. 439 */ 440 if (bcontainer->dma_max_mappings) { 441 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 442 443 #ifdef CONFIG_KVM 444 if (kvm_enabled()) { 445 max_memslots = kvm_get_max_memslots(); 446 } 447 #endif 448 449 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 450 hwaddr start, end; 451 452 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 453 vrdl->granularity); 454 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 455 vrdl->granularity); 456 vrdl_mappings += (end - start) / vrdl->granularity; 457 vrdl_count++; 458 } 459 460 if (vrdl_mappings + max_memslots - vrdl_count > 461 bcontainer->dma_max_mappings) { 462 warn_report("%s: possibly running out of DMA mappings. E.g., try" 463 " increasing the 'block-size' of virtio-mem devies." 464 " Maximum possible DMA mappings: %d, Maximum possible" 465 " memslots: %d", __func__, bcontainer->dma_max_mappings, 466 max_memslots); 467 } 468 } 469 } 470 471 static void vfio_unregister_ram_discard_listener(VFIOContainerBase *bcontainer, 472 MemoryRegionSection *section) 473 { 474 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 475 VFIORamDiscardListener *vrdl = NULL; 476 477 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 478 if (vrdl->mr == section->mr && 479 vrdl->offset_within_address_space == 480 section->offset_within_address_space) { 481 break; 482 } 483 } 484 485 if (!vrdl) { 486 hw_error("vfio: Trying to unregister missing RAM discard listener"); 487 } 488 489 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 490 QLIST_REMOVE(vrdl, next); 491 g_free(vrdl); 492 } 493 494 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 495 { 496 MemoryRegion *mr = section->mr; 497 498 if (!TPM_IS_CRB(mr->owner)) { 499 return false; 500 } 501 502 /* this is a known safe misaligned region, just trace for debug purpose */ 503 trace_vfio_known_safe_misalignment(memory_region_name(mr), 504 section->offset_within_address_space, 505 section->offset_within_region, 506 qemu_real_host_page_size()); 507 return true; 508 } 509 510 static bool vfio_listener_valid_section(MemoryRegionSection *section, 511 const char *name) 512 { 513 if (vfio_listener_skipped_section(section)) { 514 trace_vfio_listener_region_skip(name, 515 section->offset_within_address_space, 516 section->offset_within_address_space + 517 int128_get64(int128_sub(section->size, int128_one()))); 518 return false; 519 } 520 521 if (unlikely((section->offset_within_address_space & 522 ~qemu_real_host_page_mask()) != 523 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 524 if (!vfio_known_safe_misalignment(section)) { 525 error_report("%s received unaligned region %s iova=0x%"PRIx64 526 " offset_within_region=0x%"PRIx64 527 " qemu_real_host_page_size=0x%"PRIxPTR, 528 __func__, memory_region_name(section->mr), 529 section->offset_within_address_space, 530 section->offset_within_region, 531 qemu_real_host_page_size()); 532 } 533 return false; 534 } 535 536 return true; 537 } 538 539 static bool vfio_get_section_iova_range(VFIOContainerBase *bcontainer, 540 MemoryRegionSection *section, 541 hwaddr *out_iova, hwaddr *out_end, 542 Int128 *out_llend) 543 { 544 Int128 llend; 545 hwaddr iova; 546 547 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 548 llend = int128_make64(section->offset_within_address_space); 549 llend = int128_add(llend, section->size); 550 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 551 552 if (int128_ge(int128_make64(iova), llend)) { 553 return false; 554 } 555 556 *out_iova = iova; 557 *out_end = int128_get64(int128_sub(llend, int128_one())); 558 if (out_llend) { 559 *out_llend = llend; 560 } 561 return true; 562 } 563 564 static void vfio_listener_region_add(MemoryListener *listener, 565 MemoryRegionSection *section) 566 { 567 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 568 listener); 569 hwaddr iova, end; 570 Int128 llend, llsize; 571 void *vaddr; 572 int ret; 573 Error *err = NULL; 574 575 if (!vfio_listener_valid_section(section, "region_add")) { 576 return; 577 } 578 579 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 580 &llend)) { 581 if (memory_region_is_ram_device(section->mr)) { 582 trace_vfio_listener_region_add_no_dma_map( 583 memory_region_name(section->mr), 584 section->offset_within_address_space, 585 int128_getlo(section->size), 586 qemu_real_host_page_size()); 587 } 588 return; 589 } 590 591 if (!vfio_container_add_section_window(bcontainer, section, &err)) { 592 goto fail; 593 } 594 595 memory_region_ref(section->mr); 596 597 if (memory_region_is_iommu(section->mr)) { 598 VFIOGuestIOMMU *giommu; 599 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 600 int iommu_idx; 601 602 trace_vfio_listener_region_add_iommu(iova, end); 603 /* 604 * FIXME: For VFIO iommu types which have KVM acceleration to 605 * avoid bouncing all map/unmaps through qemu this way, this 606 * would be the right place to wire that up (tell the KVM 607 * device emulation the VFIO iommu handles to use). 608 */ 609 giommu = g_malloc0(sizeof(*giommu)); 610 giommu->iommu_mr = iommu_mr; 611 giommu->iommu_offset = section->offset_within_address_space - 612 section->offset_within_region; 613 giommu->bcontainer = bcontainer; 614 llend = int128_add(int128_make64(section->offset_within_region), 615 section->size); 616 llend = int128_sub(llend, int128_one()); 617 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 618 MEMTXATTRS_UNSPECIFIED); 619 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 620 IOMMU_NOTIFIER_IOTLB_EVENTS, 621 section->offset_within_region, 622 int128_get64(llend), 623 iommu_idx); 624 625 ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr, 626 bcontainer->pgsizes, 627 &err); 628 if (ret) { 629 g_free(giommu); 630 goto fail; 631 } 632 633 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 634 &err); 635 if (ret) { 636 g_free(giommu); 637 goto fail; 638 } 639 QLIST_INSERT_HEAD(&bcontainer->giommu_list, giommu, giommu_next); 640 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 641 642 return; 643 } 644 645 /* Here we assume that memory_region_is_ram(section->mr)==true */ 646 647 /* 648 * For RAM memory regions with a RamDiscardManager, we only want to map the 649 * actually populated parts - and update the mapping whenever we're notified 650 * about changes. 651 */ 652 if (memory_region_has_ram_discard_manager(section->mr)) { 653 vfio_register_ram_discard_listener(bcontainer, section); 654 return; 655 } 656 657 vaddr = memory_region_get_ram_ptr(section->mr) + 658 section->offset_within_region + 659 (iova - section->offset_within_address_space); 660 661 trace_vfio_listener_region_add_ram(iova, end, vaddr); 662 663 llsize = int128_sub(llend, int128_make64(iova)); 664 665 if (memory_region_is_ram_device(section->mr)) { 666 hwaddr pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 667 668 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 669 trace_vfio_listener_region_add_no_dma_map( 670 memory_region_name(section->mr), 671 section->offset_within_address_space, 672 int128_getlo(section->size), 673 pgmask + 1); 674 return; 675 } 676 } 677 678 ret = vfio_container_dma_map(bcontainer, iova, int128_get64(llsize), 679 vaddr, section->readonly); 680 if (ret) { 681 error_setg(&err, "vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 682 "0x%"HWADDR_PRIx", %p) = %d (%s)", 683 bcontainer, iova, int128_get64(llsize), vaddr, ret, 684 strerror(-ret)); 685 if (memory_region_is_ram_device(section->mr)) { 686 /* Allow unexpected mappings not to be fatal for RAM devices */ 687 error_report_err(err); 688 return; 689 } 690 goto fail; 691 } 692 693 return; 694 695 fail: 696 if (memory_region_is_ram_device(section->mr)) { 697 error_reportf_err(err, "PCI p2p may not work: "); 698 return; 699 } 700 /* 701 * On the initfn path, store the first error in the container so we 702 * can gracefully fail. Runtime, there's not much we can do other 703 * than throw a hardware error. 704 */ 705 if (!bcontainer->initialized) { 706 if (!bcontainer->error) { 707 error_propagate_prepend(&bcontainer->error, err, 708 "Region %s: ", 709 memory_region_name(section->mr)); 710 } else { 711 error_free(err); 712 } 713 } else { 714 error_report_err(err); 715 hw_error("vfio: DMA mapping failed, unable to continue"); 716 } 717 } 718 719 static void vfio_listener_region_del(MemoryListener *listener, 720 MemoryRegionSection *section) 721 { 722 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 723 listener); 724 hwaddr iova, end; 725 Int128 llend, llsize; 726 int ret; 727 bool try_unmap = true; 728 729 if (!vfio_listener_valid_section(section, "region_del")) { 730 return; 731 } 732 733 if (memory_region_is_iommu(section->mr)) { 734 VFIOGuestIOMMU *giommu; 735 736 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 737 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 738 giommu->n.start == section->offset_within_region) { 739 memory_region_unregister_iommu_notifier(section->mr, 740 &giommu->n); 741 QLIST_REMOVE(giommu, giommu_next); 742 g_free(giommu); 743 break; 744 } 745 } 746 747 /* 748 * FIXME: We assume the one big unmap below is adequate to 749 * remove any individual page mappings in the IOMMU which 750 * might have been copied into VFIO. This works for a page table 751 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 752 * That may not be true for all IOMMU types. 753 */ 754 } 755 756 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 757 &llend)) { 758 return; 759 } 760 761 llsize = int128_sub(llend, int128_make64(iova)); 762 763 trace_vfio_listener_region_del(iova, end); 764 765 if (memory_region_is_ram_device(section->mr)) { 766 hwaddr pgmask; 767 768 pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 769 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 770 } else if (memory_region_has_ram_discard_manager(section->mr)) { 771 vfio_unregister_ram_discard_listener(bcontainer, section); 772 /* Unregistering will trigger an unmap. */ 773 try_unmap = false; 774 } 775 776 if (try_unmap) { 777 if (int128_eq(llsize, int128_2_64())) { 778 /* The unmap ioctl doesn't accept a full 64-bit span. */ 779 llsize = int128_rshift(llsize, 1); 780 ret = vfio_container_dma_unmap(bcontainer, iova, 781 int128_get64(llsize), NULL); 782 if (ret) { 783 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 784 "0x%"HWADDR_PRIx") = %d (%s)", 785 bcontainer, iova, int128_get64(llsize), ret, 786 strerror(-ret)); 787 } 788 iova += int128_get64(llsize); 789 } 790 ret = vfio_container_dma_unmap(bcontainer, iova, 791 int128_get64(llsize), NULL); 792 if (ret) { 793 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 794 "0x%"HWADDR_PRIx") = %d (%s)", 795 bcontainer, iova, int128_get64(llsize), ret, 796 strerror(-ret)); 797 } 798 } 799 800 memory_region_unref(section->mr); 801 802 vfio_container_del_section_window(bcontainer, section); 803 } 804 805 typedef struct VFIODirtyRanges { 806 hwaddr min32; 807 hwaddr max32; 808 hwaddr min64; 809 hwaddr max64; 810 hwaddr minpci64; 811 hwaddr maxpci64; 812 } VFIODirtyRanges; 813 814 typedef struct VFIODirtyRangesListener { 815 VFIOContainerBase *bcontainer; 816 VFIODirtyRanges ranges; 817 MemoryListener listener; 818 } VFIODirtyRangesListener; 819 820 static bool vfio_section_is_vfio_pci(MemoryRegionSection *section, 821 VFIOContainerBase *bcontainer) 822 { 823 VFIOPCIDevice *pcidev; 824 VFIODevice *vbasedev; 825 Object *owner; 826 827 owner = memory_region_owner(section->mr); 828 829 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 830 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 831 continue; 832 } 833 pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 834 if (OBJECT(pcidev) == owner) { 835 return true; 836 } 837 } 838 839 return false; 840 } 841 842 static void vfio_dirty_tracking_update_range(VFIODirtyRanges *range, 843 hwaddr iova, hwaddr end, 844 bool update_pci) 845 { 846 hwaddr *min, *max; 847 848 /* 849 * The address space passed to the dirty tracker is reduced to three ranges: 850 * one for 32-bit DMA ranges, one for 64-bit DMA ranges and one for the 851 * PCI 64-bit hole. 852 * 853 * The underlying reports of dirty will query a sub-interval of each of 854 * these ranges. 855 * 856 * The purpose of the three range handling is to handle known cases of big 857 * holes in the address space, like the x86 AMD 1T hole, and firmware (like 858 * OVMF) which may relocate the pci-hole64 to the end of the address space. 859 * The latter would otherwise generate large ranges for tracking, stressing 860 * the limits of supported hardware. The pci-hole32 will always be below 4G 861 * (overlapping or not) so it doesn't need special handling and is part of 862 * the 32-bit range. 863 * 864 * The alternative would be an IOVATree but that has a much bigger runtime 865 * overhead and unnecessary complexity. 866 */ 867 if (update_pci && iova >= UINT32_MAX) { 868 min = &range->minpci64; 869 max = &range->maxpci64; 870 } else { 871 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 872 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 873 } 874 if (*min > iova) { 875 *min = iova; 876 } 877 if (*max < end) { 878 *max = end; 879 } 880 881 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 882 } 883 884 static void vfio_dirty_tracking_update(MemoryListener *listener, 885 MemoryRegionSection *section) 886 { 887 VFIODirtyRangesListener *dirty = 888 container_of(listener, VFIODirtyRangesListener, listener); 889 hwaddr iova, end; 890 891 if (!vfio_listener_valid_section(section, "tracking_update") || 892 !vfio_get_section_iova_range(dirty->bcontainer, section, 893 &iova, &end, NULL)) { 894 return; 895 } 896 897 vfio_dirty_tracking_update_range(&dirty->ranges, iova, end, 898 vfio_section_is_vfio_pci(section, dirty->bcontainer)); 899 } 900 901 static const MemoryListener vfio_dirty_tracking_listener = { 902 .name = "vfio-tracking", 903 .region_add = vfio_dirty_tracking_update, 904 }; 905 906 static void vfio_dirty_tracking_init(VFIOContainerBase *bcontainer, 907 VFIODirtyRanges *ranges) 908 { 909 VFIODirtyRangesListener dirty; 910 911 memset(&dirty, 0, sizeof(dirty)); 912 dirty.ranges.min32 = UINT32_MAX; 913 dirty.ranges.min64 = UINT64_MAX; 914 dirty.ranges.minpci64 = UINT64_MAX; 915 dirty.listener = vfio_dirty_tracking_listener; 916 dirty.bcontainer = bcontainer; 917 918 memory_listener_register(&dirty.listener, 919 bcontainer->space->as); 920 921 *ranges = dirty.ranges; 922 923 /* 924 * The memory listener is synchronous, and used to calculate the range 925 * to dirty tracking. Unregister it after we are done as we are not 926 * interested in any follow-up updates. 927 */ 928 memory_listener_unregister(&dirty.listener); 929 } 930 931 static void vfio_devices_dma_logging_stop(VFIOContainerBase *bcontainer) 932 { 933 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 934 sizeof(uint64_t))] = {}; 935 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 936 VFIODevice *vbasedev; 937 938 feature->argsz = sizeof(buf); 939 feature->flags = VFIO_DEVICE_FEATURE_SET | 940 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 941 942 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 943 if (!vbasedev->dirty_tracking) { 944 continue; 945 } 946 947 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 948 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 949 vbasedev->name, -errno, strerror(errno)); 950 } 951 vbasedev->dirty_tracking = false; 952 } 953 } 954 955 static struct vfio_device_feature * 956 vfio_device_feature_dma_logging_start_create(VFIOContainerBase *bcontainer, 957 VFIODirtyRanges *tracking) 958 { 959 struct vfio_device_feature *feature; 960 size_t feature_size; 961 struct vfio_device_feature_dma_logging_control *control; 962 struct vfio_device_feature_dma_logging_range *ranges; 963 964 feature_size = sizeof(struct vfio_device_feature) + 965 sizeof(struct vfio_device_feature_dma_logging_control); 966 feature = g_try_malloc0(feature_size); 967 if (!feature) { 968 errno = ENOMEM; 969 return NULL; 970 } 971 feature->argsz = feature_size; 972 feature->flags = VFIO_DEVICE_FEATURE_SET | 973 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 974 975 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 976 control->page_size = qemu_real_host_page_size(); 977 978 /* 979 * DMA logging uAPI guarantees to support at least a number of ranges that 980 * fits into a single host kernel base page. 981 */ 982 control->num_ranges = !!tracking->max32 + !!tracking->max64 + 983 !!tracking->maxpci64; 984 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 985 control->num_ranges); 986 if (!ranges) { 987 g_free(feature); 988 errno = ENOMEM; 989 990 return NULL; 991 } 992 993 control->ranges = (uintptr_t)ranges; 994 if (tracking->max32) { 995 ranges->iova = tracking->min32; 996 ranges->length = (tracking->max32 - tracking->min32) + 1; 997 ranges++; 998 } 999 if (tracking->max64) { 1000 ranges->iova = tracking->min64; 1001 ranges->length = (tracking->max64 - tracking->min64) + 1; 1002 ranges++; 1003 } 1004 if (tracking->maxpci64) { 1005 ranges->iova = tracking->minpci64; 1006 ranges->length = (tracking->maxpci64 - tracking->minpci64) + 1; 1007 } 1008 1009 trace_vfio_device_dirty_tracking_start(control->num_ranges, 1010 tracking->min32, tracking->max32, 1011 tracking->min64, tracking->max64, 1012 tracking->minpci64, tracking->maxpci64); 1013 1014 return feature; 1015 } 1016 1017 static void vfio_device_feature_dma_logging_start_destroy( 1018 struct vfio_device_feature *feature) 1019 { 1020 struct vfio_device_feature_dma_logging_control *control = 1021 (struct vfio_device_feature_dma_logging_control *)feature->data; 1022 struct vfio_device_feature_dma_logging_range *ranges = 1023 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 1024 1025 g_free(ranges); 1026 g_free(feature); 1027 } 1028 1029 static bool vfio_devices_dma_logging_start(VFIOContainerBase *bcontainer, 1030 Error **errp) 1031 { 1032 struct vfio_device_feature *feature; 1033 VFIODirtyRanges ranges; 1034 VFIODevice *vbasedev; 1035 int ret = 0; 1036 1037 vfio_dirty_tracking_init(bcontainer, &ranges); 1038 feature = vfio_device_feature_dma_logging_start_create(bcontainer, 1039 &ranges); 1040 if (!feature) { 1041 error_setg_errno(errp, errno, "Failed to prepare DMA logging"); 1042 return false; 1043 } 1044 1045 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1046 if (vbasedev->dirty_tracking) { 1047 continue; 1048 } 1049 1050 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 1051 if (ret) { 1052 ret = -errno; 1053 error_setg_errno(errp, errno, "%s: Failed to start DMA logging", 1054 vbasedev->name); 1055 goto out; 1056 } 1057 vbasedev->dirty_tracking = true; 1058 } 1059 1060 out: 1061 if (ret) { 1062 vfio_devices_dma_logging_stop(bcontainer); 1063 } 1064 1065 vfio_device_feature_dma_logging_start_destroy(feature); 1066 1067 return ret == 0; 1068 } 1069 1070 static bool vfio_listener_log_global_start(MemoryListener *listener, 1071 Error **errp) 1072 { 1073 ERRP_GUARD(); 1074 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1075 listener); 1076 bool ret; 1077 1078 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1079 ret = vfio_devices_dma_logging_start(bcontainer, errp); 1080 } else { 1081 ret = vfio_container_set_dirty_page_tracking(bcontainer, true, errp) == 0; 1082 } 1083 1084 if (!ret) { 1085 error_prepend(errp, "vfio: Could not start dirty page tracking - "); 1086 } 1087 return ret; 1088 } 1089 1090 static void vfio_listener_log_global_stop(MemoryListener *listener) 1091 { 1092 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1093 listener); 1094 Error *local_err = NULL; 1095 int ret = 0; 1096 1097 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1098 vfio_devices_dma_logging_stop(bcontainer); 1099 } else { 1100 ret = vfio_container_set_dirty_page_tracking(bcontainer, false, 1101 &local_err); 1102 } 1103 1104 if (ret) { 1105 error_prepend(&local_err, 1106 "vfio: Could not stop dirty page tracking - "); 1107 error_report_err(local_err); 1108 vfio_set_migration_error(ret); 1109 } 1110 } 1111 1112 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, 1113 hwaddr size, void *bitmap) 1114 { 1115 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 1116 sizeof(struct vfio_device_feature_dma_logging_report), 1117 sizeof(uint64_t))] = {}; 1118 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1119 struct vfio_device_feature_dma_logging_report *report = 1120 (struct vfio_device_feature_dma_logging_report *)feature->data; 1121 1122 report->iova = iova; 1123 report->length = size; 1124 report->page_size = qemu_real_host_page_size(); 1125 report->bitmap = (uintptr_t)bitmap; 1126 1127 feature->argsz = sizeof(buf); 1128 feature->flags = VFIO_DEVICE_FEATURE_GET | 1129 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; 1130 1131 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1132 return -errno; 1133 } 1134 1135 return 0; 1136 } 1137 1138 int vfio_devices_query_dirty_bitmap(const VFIOContainerBase *bcontainer, 1139 VFIOBitmap *vbmap, hwaddr iova, hwaddr size, Error **errp) 1140 { 1141 VFIODevice *vbasedev; 1142 int ret; 1143 1144 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1145 ret = vfio_device_dma_logging_report(vbasedev, iova, size, 1146 vbmap->bitmap); 1147 if (ret) { 1148 error_setg_errno(errp, -ret, 1149 "%s: Failed to get DMA logging report, iova: " 1150 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx, 1151 vbasedev->name, iova, size); 1152 1153 return ret; 1154 } 1155 } 1156 1157 return 0; 1158 } 1159 1160 int vfio_get_dirty_bitmap(const VFIOContainerBase *bcontainer, uint64_t iova, 1161 uint64_t size, ram_addr_t ram_addr, Error **errp) 1162 { 1163 bool all_device_dirty_tracking = 1164 vfio_devices_all_device_dirty_tracking(bcontainer); 1165 uint64_t dirty_pages; 1166 VFIOBitmap vbmap; 1167 int ret; 1168 1169 if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { 1170 cpu_physical_memory_set_dirty_range(ram_addr, size, 1171 tcg_enabled() ? DIRTY_CLIENTS_ALL : 1172 DIRTY_CLIENTS_NOCODE); 1173 return 0; 1174 } 1175 1176 ret = vfio_bitmap_alloc(&vbmap, size); 1177 if (ret) { 1178 error_setg_errno(errp, -ret, 1179 "Failed to allocate dirty tracking bitmap"); 1180 return ret; 1181 } 1182 1183 if (all_device_dirty_tracking) { 1184 ret = vfio_devices_query_dirty_bitmap(bcontainer, &vbmap, iova, size, 1185 errp); 1186 } else { 1187 ret = vfio_container_query_dirty_bitmap(bcontainer, &vbmap, iova, size, 1188 errp); 1189 } 1190 1191 if (ret) { 1192 goto out; 1193 } 1194 1195 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, 1196 vbmap.pages); 1197 1198 trace_vfio_get_dirty_bitmap(iova, size, vbmap.size, ram_addr, dirty_pages); 1199 out: 1200 g_free(vbmap.bitmap); 1201 1202 return ret; 1203 } 1204 1205 typedef struct { 1206 IOMMUNotifier n; 1207 VFIOGuestIOMMU *giommu; 1208 } vfio_giommu_dirty_notifier; 1209 1210 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1211 { 1212 vfio_giommu_dirty_notifier *gdn = container_of(n, 1213 vfio_giommu_dirty_notifier, n); 1214 VFIOGuestIOMMU *giommu = gdn->giommu; 1215 VFIOContainerBase *bcontainer = giommu->bcontainer; 1216 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1217 ram_addr_t translated_addr; 1218 Error *local_err = NULL; 1219 int ret = -EINVAL; 1220 1221 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1222 1223 if (iotlb->target_as != &address_space_memory) { 1224 error_report("Wrong target AS \"%s\", only system memory is allowed", 1225 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1226 goto out; 1227 } 1228 1229 rcu_read_lock(); 1230 if (!vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL, &local_err)) { 1231 error_report_err(local_err); 1232 goto out_unlock; 1233 } 1234 1235 ret = vfio_get_dirty_bitmap(bcontainer, iova, iotlb->addr_mask + 1, 1236 translated_addr, &local_err); 1237 if (ret) { 1238 error_prepend(&local_err, 1239 "vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1240 "0x%"HWADDR_PRIx") failed - ", bcontainer, iova, 1241 iotlb->addr_mask + 1); 1242 error_report_err(local_err); 1243 } 1244 1245 out_unlock: 1246 rcu_read_unlock(); 1247 1248 out: 1249 if (ret) { 1250 vfio_set_migration_error(ret); 1251 } 1252 } 1253 1254 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1255 void *opaque) 1256 { 1257 const hwaddr size = int128_get64(section->size); 1258 const hwaddr iova = section->offset_within_address_space; 1259 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1260 section->offset_within_region; 1261 VFIORamDiscardListener *vrdl = opaque; 1262 Error *local_err = NULL; 1263 int ret; 1264 1265 /* 1266 * Sync the whole mapped region (spanning multiple individual mappings) 1267 * in one go. 1268 */ 1269 ret = vfio_get_dirty_bitmap(vrdl->bcontainer, iova, size, ram_addr, 1270 &local_err); 1271 if (ret) { 1272 error_report_err(local_err); 1273 } 1274 return ret; 1275 } 1276 1277 static int 1278 vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainerBase *bcontainer, 1279 MemoryRegionSection *section) 1280 { 1281 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1282 VFIORamDiscardListener *vrdl = NULL; 1283 1284 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 1285 if (vrdl->mr == section->mr && 1286 vrdl->offset_within_address_space == 1287 section->offset_within_address_space) { 1288 break; 1289 } 1290 } 1291 1292 if (!vrdl) { 1293 hw_error("vfio: Trying to sync missing RAM discard listener"); 1294 } 1295 1296 /* 1297 * We only want/can synchronize the bitmap for actually mapped parts - 1298 * which correspond to populated parts. Replay all populated parts. 1299 */ 1300 return ram_discard_manager_replay_populated(rdm, section, 1301 vfio_ram_discard_get_dirty_bitmap, 1302 &vrdl); 1303 } 1304 1305 static int vfio_sync_iommu_dirty_bitmap(VFIOContainerBase *bcontainer, 1306 MemoryRegionSection *section) 1307 { 1308 VFIOGuestIOMMU *giommu; 1309 bool found = false; 1310 Int128 llend; 1311 vfio_giommu_dirty_notifier gdn; 1312 int idx; 1313 1314 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 1315 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1316 giommu->n.start == section->offset_within_region) { 1317 found = true; 1318 break; 1319 } 1320 } 1321 1322 if (!found) { 1323 return 0; 1324 } 1325 1326 gdn.giommu = giommu; 1327 idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1328 MEMTXATTRS_UNSPECIFIED); 1329 1330 llend = int128_add(int128_make64(section->offset_within_region), 1331 section->size); 1332 llend = int128_sub(llend, int128_one()); 1333 1334 iommu_notifier_init(&gdn.n, vfio_iommu_map_dirty_notify, IOMMU_NOTIFIER_MAP, 1335 section->offset_within_region, int128_get64(llend), 1336 idx); 1337 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1338 1339 return 0; 1340 } 1341 1342 static int vfio_sync_dirty_bitmap(VFIOContainerBase *bcontainer, 1343 MemoryRegionSection *section, Error **errp) 1344 { 1345 ram_addr_t ram_addr; 1346 1347 if (memory_region_is_iommu(section->mr)) { 1348 return vfio_sync_iommu_dirty_bitmap(bcontainer, section); 1349 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1350 int ret; 1351 1352 ret = vfio_sync_ram_discard_listener_dirty_bitmap(bcontainer, section); 1353 if (ret) { 1354 error_setg(errp, 1355 "Failed to sync dirty bitmap with RAM discard listener"); 1356 } 1357 return ret; 1358 } 1359 1360 ram_addr = memory_region_get_ram_addr(section->mr) + 1361 section->offset_within_region; 1362 1363 return vfio_get_dirty_bitmap(bcontainer, 1364 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1365 int128_get64(section->size), ram_addr, errp); 1366 } 1367 1368 static void vfio_listener_log_sync(MemoryListener *listener, 1369 MemoryRegionSection *section) 1370 { 1371 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1372 listener); 1373 int ret; 1374 Error *local_err = NULL; 1375 1376 if (vfio_listener_skipped_section(section)) { 1377 return; 1378 } 1379 1380 if (vfio_devices_all_dirty_tracking(bcontainer)) { 1381 ret = vfio_sync_dirty_bitmap(bcontainer, section, &local_err); 1382 if (ret) { 1383 error_report_err(local_err); 1384 vfio_set_migration_error(ret); 1385 } 1386 } 1387 } 1388 1389 const MemoryListener vfio_memory_listener = { 1390 .name = "vfio", 1391 .region_add = vfio_listener_region_add, 1392 .region_del = vfio_listener_region_del, 1393 .log_global_start = vfio_listener_log_global_start, 1394 .log_global_stop = vfio_listener_log_global_stop, 1395 .log_sync = vfio_listener_log_sync, 1396 }; 1397 1398 void vfio_reset_handler(void *opaque) 1399 { 1400 VFIODevice *vbasedev; 1401 1402 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 1403 if (vbasedev->dev->realized) { 1404 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 1405 } 1406 } 1407 1408 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 1409 if (vbasedev->dev->realized && vbasedev->needs_reset) { 1410 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 1411 } 1412 } 1413 } 1414 1415 int vfio_kvm_device_add_fd(int fd, Error **errp) 1416 { 1417 #ifdef CONFIG_KVM 1418 struct kvm_device_attr attr = { 1419 .group = KVM_DEV_VFIO_FILE, 1420 .attr = KVM_DEV_VFIO_FILE_ADD, 1421 .addr = (uint64_t)(unsigned long)&fd, 1422 }; 1423 1424 if (!kvm_enabled()) { 1425 return 0; 1426 } 1427 1428 if (vfio_kvm_device_fd < 0) { 1429 struct kvm_create_device cd = { 1430 .type = KVM_DEV_TYPE_VFIO, 1431 }; 1432 1433 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 1434 error_setg_errno(errp, errno, "Failed to create KVM VFIO device"); 1435 return -errno; 1436 } 1437 1438 vfio_kvm_device_fd = cd.fd; 1439 } 1440 1441 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1442 error_setg_errno(errp, errno, "Failed to add fd %d to KVM VFIO device", 1443 fd); 1444 return -errno; 1445 } 1446 #endif 1447 return 0; 1448 } 1449 1450 int vfio_kvm_device_del_fd(int fd, Error **errp) 1451 { 1452 #ifdef CONFIG_KVM 1453 struct kvm_device_attr attr = { 1454 .group = KVM_DEV_VFIO_FILE, 1455 .attr = KVM_DEV_VFIO_FILE_DEL, 1456 .addr = (uint64_t)(unsigned long)&fd, 1457 }; 1458 1459 if (vfio_kvm_device_fd < 0) { 1460 error_setg(errp, "KVM VFIO device isn't created yet"); 1461 return -EINVAL; 1462 } 1463 1464 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1465 error_setg_errno(errp, errno, 1466 "Failed to remove fd %d from KVM VFIO device", fd); 1467 return -errno; 1468 } 1469 #endif 1470 return 0; 1471 } 1472 1473 VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1474 { 1475 VFIOAddressSpace *space; 1476 1477 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1478 if (space->as == as) { 1479 return space; 1480 } 1481 } 1482 1483 /* No suitable VFIOAddressSpace, create a new one */ 1484 space = g_malloc0(sizeof(*space)); 1485 space->as = as; 1486 QLIST_INIT(&space->containers); 1487 1488 if (QLIST_EMPTY(&vfio_address_spaces)) { 1489 qemu_register_reset(vfio_reset_handler, NULL); 1490 } 1491 1492 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1493 1494 return space; 1495 } 1496 1497 void vfio_put_address_space(VFIOAddressSpace *space) 1498 { 1499 if (!QLIST_EMPTY(&space->containers)) { 1500 return; 1501 } 1502 1503 QLIST_REMOVE(space, list); 1504 g_free(space); 1505 1506 if (QLIST_EMPTY(&vfio_address_spaces)) { 1507 qemu_unregister_reset(vfio_reset_handler, NULL); 1508 } 1509 } 1510 1511 void vfio_address_space_insert(VFIOAddressSpace *space, 1512 VFIOContainerBase *bcontainer) 1513 { 1514 QLIST_INSERT_HEAD(&space->containers, bcontainer, next); 1515 bcontainer->space = space; 1516 } 1517 1518 struct vfio_device_info *vfio_get_device_info(int fd) 1519 { 1520 struct vfio_device_info *info; 1521 uint32_t argsz = sizeof(*info); 1522 1523 info = g_malloc0(argsz); 1524 1525 retry: 1526 info->argsz = argsz; 1527 1528 if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) { 1529 g_free(info); 1530 return NULL; 1531 } 1532 1533 if (info->argsz > argsz) { 1534 argsz = info->argsz; 1535 info = g_realloc(info, argsz); 1536 goto retry; 1537 } 1538 1539 return info; 1540 } 1541 1542 bool vfio_attach_device(char *name, VFIODevice *vbasedev, 1543 AddressSpace *as, Error **errp) 1544 { 1545 const VFIOIOMMUClass *ops = 1546 VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_LEGACY)); 1547 HostIOMMUDevice *hiod; 1548 1549 if (vbasedev->iommufd) { 1550 ops = VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD)); 1551 } 1552 1553 assert(ops); 1554 1555 if (!ops->attach_device(name, vbasedev, as, errp)) { 1556 return false; 1557 } 1558 1559 hiod = HOST_IOMMU_DEVICE(object_new(ops->hiod_typename)); 1560 if (!HOST_IOMMU_DEVICE_GET_CLASS(hiod)->realize(hiod, vbasedev, errp)) { 1561 object_unref(hiod); 1562 ops->detach_device(vbasedev); 1563 return false; 1564 } 1565 vbasedev->hiod = hiod; 1566 1567 return true; 1568 } 1569 1570 void vfio_detach_device(VFIODevice *vbasedev) 1571 { 1572 if (!vbasedev->bcontainer) { 1573 return; 1574 } 1575 object_unref(vbasedev->hiod); 1576 VFIO_IOMMU_GET_CLASS(vbasedev->bcontainer)->detach_device(vbasedev); 1577 } 1578