1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/vfio.h" 30 #include "hw/vfio/pci.h" 31 #include "exec/address-spaces.h" 32 #include "exec/memory.h" 33 #include "exec/ram_addr.h" 34 #include "hw/hw.h" 35 #include "qemu/error-report.h" 36 #include "qemu/main-loop.h" 37 #include "qemu/range.h" 38 #include "sysemu/kvm.h" 39 #include "sysemu/reset.h" 40 #include "sysemu/runstate.h" 41 #include "trace.h" 42 #include "qapi/error.h" 43 #include "migration/migration.h" 44 #include "migration/misc.h" 45 #include "migration/blocker.h" 46 #include "migration/qemu-file.h" 47 #include "sysemu/tpm.h" 48 49 VFIODeviceList vfio_device_list = 50 QLIST_HEAD_INITIALIZER(vfio_device_list); 51 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 52 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 53 54 #ifdef CONFIG_KVM 55 /* 56 * We have a single VFIO pseudo device per KVM VM. Once created it lives 57 * for the life of the VM. Closing the file descriptor only drops our 58 * reference to it and the device's reference to kvm. Therefore once 59 * initialized, this file descriptor is only released on QEMU exit and 60 * we'll re-use it should another vfio device be attached before then. 61 */ 62 int vfio_kvm_device_fd = -1; 63 #endif 64 65 /* 66 * Device state interfaces 67 */ 68 69 bool vfio_mig_active(void) 70 { 71 VFIODevice *vbasedev; 72 73 if (QLIST_EMPTY(&vfio_device_list)) { 74 return false; 75 } 76 77 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 78 if (vbasedev->migration_blocker) { 79 return false; 80 } 81 } 82 return true; 83 } 84 85 static Error *multiple_devices_migration_blocker; 86 87 /* 88 * Multiple devices migration is allowed only if all devices support P2P 89 * migration. Single device migration is allowed regardless of P2P migration 90 * support. 91 */ 92 static bool vfio_multiple_devices_migration_is_supported(void) 93 { 94 VFIODevice *vbasedev; 95 unsigned int device_num = 0; 96 bool all_support_p2p = true; 97 98 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 99 if (vbasedev->migration) { 100 device_num++; 101 102 if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) { 103 all_support_p2p = false; 104 } 105 } 106 } 107 108 return all_support_p2p || device_num <= 1; 109 } 110 111 int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp) 112 { 113 int ret; 114 115 if (vfio_multiple_devices_migration_is_supported()) { 116 return 0; 117 } 118 119 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { 120 error_setg(errp, "Multiple VFIO devices migration is supported only if " 121 "all of them support P2P migration"); 122 return -EINVAL; 123 } 124 125 if (multiple_devices_migration_blocker) { 126 return 0; 127 } 128 129 error_setg(&multiple_devices_migration_blocker, 130 "Multiple VFIO devices migration is supported only if all of " 131 "them support P2P migration"); 132 ret = migrate_add_blocker(&multiple_devices_migration_blocker, errp); 133 134 return ret; 135 } 136 137 void vfio_unblock_multiple_devices_migration(void) 138 { 139 if (!multiple_devices_migration_blocker || 140 !vfio_multiple_devices_migration_is_supported()) { 141 return; 142 } 143 144 migrate_del_blocker(&multiple_devices_migration_blocker); 145 } 146 147 bool vfio_viommu_preset(VFIODevice *vbasedev) 148 { 149 return vbasedev->container->space->as != &address_space_memory; 150 } 151 152 static void vfio_set_migration_error(int err) 153 { 154 MigrationState *ms = migrate_get_current(); 155 156 if (migration_is_setup_or_active(ms->state)) { 157 WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { 158 if (ms->to_dst_file) { 159 qemu_file_set_error(ms->to_dst_file, err); 160 } 161 } 162 } 163 } 164 165 bool vfio_device_state_is_running(VFIODevice *vbasedev) 166 { 167 VFIOMigration *migration = vbasedev->migration; 168 169 return migration->device_state == VFIO_DEVICE_STATE_RUNNING || 170 migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P; 171 } 172 173 bool vfio_device_state_is_precopy(VFIODevice *vbasedev) 174 { 175 VFIOMigration *migration = vbasedev->migration; 176 177 return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY || 178 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P; 179 } 180 181 static bool vfio_devices_all_dirty_tracking(VFIOContainer *container) 182 { 183 VFIODevice *vbasedev; 184 MigrationState *ms = migrate_get_current(); 185 186 if (ms->state != MIGRATION_STATUS_ACTIVE && 187 ms->state != MIGRATION_STATUS_DEVICE) { 188 return false; 189 } 190 191 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 192 VFIOMigration *migration = vbasedev->migration; 193 194 if (!migration) { 195 return false; 196 } 197 198 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 199 (vfio_device_state_is_running(vbasedev) || 200 vfio_device_state_is_precopy(vbasedev))) { 201 return false; 202 } 203 } 204 return true; 205 } 206 207 bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container) 208 { 209 VFIODevice *vbasedev; 210 211 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 212 if (!vbasedev->dirty_pages_supported) { 213 return false; 214 } 215 } 216 217 return true; 218 } 219 220 /* 221 * Check if all VFIO devices are running and migration is active, which is 222 * essentially equivalent to the migration being in pre-copy phase. 223 */ 224 bool vfio_devices_all_running_and_mig_active(VFIOContainer *container) 225 { 226 VFIODevice *vbasedev; 227 228 if (!migration_is_active(migrate_get_current())) { 229 return false; 230 } 231 232 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 233 VFIOMigration *migration = vbasedev->migration; 234 235 if (!migration) { 236 return false; 237 } 238 239 if (vfio_device_state_is_running(vbasedev) || 240 vfio_device_state_is_precopy(vbasedev)) { 241 continue; 242 } else { 243 return false; 244 } 245 } 246 return true; 247 } 248 249 void vfio_host_win_add(VFIOContainer *container, hwaddr min_iova, 250 hwaddr max_iova, uint64_t iova_pgsizes) 251 { 252 VFIOHostDMAWindow *hostwin; 253 254 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 255 if (ranges_overlap(hostwin->min_iova, 256 hostwin->max_iova - hostwin->min_iova + 1, 257 min_iova, 258 max_iova - min_iova + 1)) { 259 hw_error("%s: Overlapped IOMMU are not enabled", __func__); 260 } 261 } 262 263 hostwin = g_malloc0(sizeof(*hostwin)); 264 265 hostwin->min_iova = min_iova; 266 hostwin->max_iova = max_iova; 267 hostwin->iova_pgsizes = iova_pgsizes; 268 QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next); 269 } 270 271 int vfio_host_win_del(VFIOContainer *container, 272 hwaddr min_iova, hwaddr max_iova) 273 { 274 VFIOHostDMAWindow *hostwin; 275 276 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 277 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) { 278 QLIST_REMOVE(hostwin, hostwin_next); 279 g_free(hostwin); 280 return 0; 281 } 282 } 283 284 return -1; 285 } 286 287 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 288 { 289 return (!memory_region_is_ram(section->mr) && 290 !memory_region_is_iommu(section->mr)) || 291 memory_region_is_protected(section->mr) || 292 /* 293 * Sizing an enabled 64-bit BAR can cause spurious mappings to 294 * addresses in the upper part of the 64-bit address space. These 295 * are never accessed by the CPU and beyond the address width of 296 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 297 */ 298 section->offset_within_address_space & (1ULL << 63); 299 } 300 301 /* Called with rcu_read_lock held. */ 302 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 303 ram_addr_t *ram_addr, bool *read_only) 304 { 305 bool ret, mr_has_discard_manager; 306 307 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, 308 &mr_has_discard_manager); 309 if (ret && mr_has_discard_manager) { 310 /* 311 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 312 * pages will remain pinned inside vfio until unmapped, resulting in a 313 * higher memory consumption than expected. If memory would get 314 * populated again later, there would be an inconsistency between pages 315 * pinned by vfio and pages seen by QEMU. This is the case until 316 * unmapped from the IOMMU (e.g., during device reset). 317 * 318 * With malicious guests, we really only care about pinning more memory 319 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 320 * exceeded and can be used to mitigate this problem. 321 */ 322 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 323 " RAM (e.g., virtio-mem) works, however, malicious" 324 " guests can trigger pinning of more memory than" 325 " intended via an IOMMU. It's possible to mitigate " 326 " by setting/adjusting RLIMIT_MEMLOCK."); 327 } 328 return ret; 329 } 330 331 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 332 { 333 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 334 VFIOContainer *container = giommu->container; 335 hwaddr iova = iotlb->iova + giommu->iommu_offset; 336 void *vaddr; 337 int ret; 338 339 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 340 iova, iova + iotlb->addr_mask); 341 342 if (iotlb->target_as != &address_space_memory) { 343 error_report("Wrong target AS \"%s\", only system memory is allowed", 344 iotlb->target_as->name ? iotlb->target_as->name : "none"); 345 vfio_set_migration_error(-EINVAL); 346 return; 347 } 348 349 rcu_read_lock(); 350 351 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 352 bool read_only; 353 354 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only)) { 355 goto out; 356 } 357 /* 358 * vaddr is only valid until rcu_read_unlock(). But after 359 * vfio_dma_map has set up the mapping the pages will be 360 * pinned by the kernel. This makes sure that the RAM backend 361 * of vaddr will always be there, even if the memory object is 362 * destroyed and its backing memory munmap-ed. 363 */ 364 ret = vfio_dma_map(container, iova, 365 iotlb->addr_mask + 1, vaddr, 366 read_only); 367 if (ret) { 368 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 369 "0x%"HWADDR_PRIx", %p) = %d (%s)", 370 container, iova, 371 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 372 } 373 } else { 374 ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1, iotlb); 375 if (ret) { 376 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 377 "0x%"HWADDR_PRIx") = %d (%s)", 378 container, iova, 379 iotlb->addr_mask + 1, ret, strerror(-ret)); 380 vfio_set_migration_error(ret); 381 } 382 } 383 out: 384 rcu_read_unlock(); 385 } 386 387 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 388 MemoryRegionSection *section) 389 { 390 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 391 listener); 392 const hwaddr size = int128_get64(section->size); 393 const hwaddr iova = section->offset_within_address_space; 394 int ret; 395 396 /* Unmap with a single call. */ 397 ret = vfio_dma_unmap(vrdl->container, iova, size , NULL); 398 if (ret) { 399 error_report("%s: vfio_dma_unmap() failed: %s", __func__, 400 strerror(-ret)); 401 } 402 } 403 404 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 405 MemoryRegionSection *section) 406 { 407 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 408 listener); 409 const hwaddr end = section->offset_within_region + 410 int128_get64(section->size); 411 hwaddr start, next, iova; 412 void *vaddr; 413 int ret; 414 415 /* 416 * Map in (aligned within memory region) minimum granularity, so we can 417 * unmap in minimum granularity later. 418 */ 419 for (start = section->offset_within_region; start < end; start = next) { 420 next = ROUND_UP(start + 1, vrdl->granularity); 421 next = MIN(next, end); 422 423 iova = start - section->offset_within_region + 424 section->offset_within_address_space; 425 vaddr = memory_region_get_ram_ptr(section->mr) + start; 426 427 ret = vfio_dma_map(vrdl->container, iova, next - start, 428 vaddr, section->readonly); 429 if (ret) { 430 /* Rollback */ 431 vfio_ram_discard_notify_discard(rdl, section); 432 return ret; 433 } 434 } 435 return 0; 436 } 437 438 static void vfio_register_ram_discard_listener(VFIOContainer *container, 439 MemoryRegionSection *section) 440 { 441 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 442 VFIORamDiscardListener *vrdl; 443 444 /* Ignore some corner cases not relevant in practice. */ 445 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 446 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 447 TARGET_PAGE_SIZE)); 448 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 449 450 vrdl = g_new0(VFIORamDiscardListener, 1); 451 vrdl->container = container; 452 vrdl->mr = section->mr; 453 vrdl->offset_within_address_space = section->offset_within_address_space; 454 vrdl->size = int128_get64(section->size); 455 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 456 section->mr); 457 458 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 459 g_assert(container->pgsizes && 460 vrdl->granularity >= 1ULL << ctz64(container->pgsizes)); 461 462 ram_discard_listener_init(&vrdl->listener, 463 vfio_ram_discard_notify_populate, 464 vfio_ram_discard_notify_discard, true); 465 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 466 QLIST_INSERT_HEAD(&container->vrdl_list, vrdl, next); 467 468 /* 469 * Sanity-check if we have a theoretically problematic setup where we could 470 * exceed the maximum number of possible DMA mappings over time. We assume 471 * that each mapped section in the same address space as a RamDiscardManager 472 * section consumes exactly one DMA mapping, with the exception of 473 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 474 * in the same address space as RamDiscardManager sections. 475 * 476 * We assume that each section in the address space consumes one memslot. 477 * We take the number of KVM memory slots as a best guess for the maximum 478 * number of sections in the address space we could have over time, 479 * also consuming DMA mappings. 480 */ 481 if (container->dma_max_mappings) { 482 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 483 484 #ifdef CONFIG_KVM 485 if (kvm_enabled()) { 486 max_memslots = kvm_get_max_memslots(); 487 } 488 #endif 489 490 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 491 hwaddr start, end; 492 493 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 494 vrdl->granularity); 495 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 496 vrdl->granularity); 497 vrdl_mappings += (end - start) / vrdl->granularity; 498 vrdl_count++; 499 } 500 501 if (vrdl_mappings + max_memslots - vrdl_count > 502 container->dma_max_mappings) { 503 warn_report("%s: possibly running out of DMA mappings. E.g., try" 504 " increasing the 'block-size' of virtio-mem devies." 505 " Maximum possible DMA mappings: %d, Maximum possible" 506 " memslots: %d", __func__, container->dma_max_mappings, 507 max_memslots); 508 } 509 } 510 } 511 512 static void vfio_unregister_ram_discard_listener(VFIOContainer *container, 513 MemoryRegionSection *section) 514 { 515 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 516 VFIORamDiscardListener *vrdl = NULL; 517 518 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 519 if (vrdl->mr == section->mr && 520 vrdl->offset_within_address_space == 521 section->offset_within_address_space) { 522 break; 523 } 524 } 525 526 if (!vrdl) { 527 hw_error("vfio: Trying to unregister missing RAM discard listener"); 528 } 529 530 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 531 QLIST_REMOVE(vrdl, next); 532 g_free(vrdl); 533 } 534 535 static VFIOHostDMAWindow *vfio_find_hostwin(VFIOContainer *container, 536 hwaddr iova, hwaddr end) 537 { 538 VFIOHostDMAWindow *hostwin; 539 bool hostwin_found = false; 540 541 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 542 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 543 hostwin_found = true; 544 break; 545 } 546 } 547 548 return hostwin_found ? hostwin : NULL; 549 } 550 551 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 552 { 553 MemoryRegion *mr = section->mr; 554 555 if (!TPM_IS_CRB(mr->owner)) { 556 return false; 557 } 558 559 /* this is a known safe misaligned region, just trace for debug purpose */ 560 trace_vfio_known_safe_misalignment(memory_region_name(mr), 561 section->offset_within_address_space, 562 section->offset_within_region, 563 qemu_real_host_page_size()); 564 return true; 565 } 566 567 static bool vfio_listener_valid_section(MemoryRegionSection *section, 568 const char *name) 569 { 570 if (vfio_listener_skipped_section(section)) { 571 trace_vfio_listener_region_skip(name, 572 section->offset_within_address_space, 573 section->offset_within_address_space + 574 int128_get64(int128_sub(section->size, int128_one()))); 575 return false; 576 } 577 578 if (unlikely((section->offset_within_address_space & 579 ~qemu_real_host_page_mask()) != 580 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 581 if (!vfio_known_safe_misalignment(section)) { 582 error_report("%s received unaligned region %s iova=0x%"PRIx64 583 " offset_within_region=0x%"PRIx64 584 " qemu_real_host_page_size=0x%"PRIxPTR, 585 __func__, memory_region_name(section->mr), 586 section->offset_within_address_space, 587 section->offset_within_region, 588 qemu_real_host_page_size()); 589 } 590 return false; 591 } 592 593 return true; 594 } 595 596 static bool vfio_get_section_iova_range(VFIOContainer *container, 597 MemoryRegionSection *section, 598 hwaddr *out_iova, hwaddr *out_end, 599 Int128 *out_llend) 600 { 601 Int128 llend; 602 hwaddr iova; 603 604 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 605 llend = int128_make64(section->offset_within_address_space); 606 llend = int128_add(llend, section->size); 607 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 608 609 if (int128_ge(int128_make64(iova), llend)) { 610 return false; 611 } 612 613 *out_iova = iova; 614 *out_end = int128_get64(int128_sub(llend, int128_one())); 615 if (out_llend) { 616 *out_llend = llend; 617 } 618 return true; 619 } 620 621 static void vfio_listener_region_add(MemoryListener *listener, 622 MemoryRegionSection *section) 623 { 624 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 625 hwaddr iova, end; 626 Int128 llend, llsize; 627 void *vaddr; 628 int ret; 629 VFIOHostDMAWindow *hostwin; 630 Error *err = NULL; 631 632 if (!vfio_listener_valid_section(section, "region_add")) { 633 return; 634 } 635 636 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) { 637 if (memory_region_is_ram_device(section->mr)) { 638 trace_vfio_listener_region_add_no_dma_map( 639 memory_region_name(section->mr), 640 section->offset_within_address_space, 641 int128_getlo(section->size), 642 qemu_real_host_page_size()); 643 } 644 return; 645 } 646 647 if (vfio_container_add_section_window(container, section, &err)) { 648 goto fail; 649 } 650 651 hostwin = vfio_find_hostwin(container, iova, end); 652 if (!hostwin) { 653 error_setg(&err, "Container %p can't map guest IOVA region" 654 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container, iova, end); 655 goto fail; 656 } 657 658 memory_region_ref(section->mr); 659 660 if (memory_region_is_iommu(section->mr)) { 661 VFIOGuestIOMMU *giommu; 662 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 663 int iommu_idx; 664 665 trace_vfio_listener_region_add_iommu(iova, end); 666 /* 667 * FIXME: For VFIO iommu types which have KVM acceleration to 668 * avoid bouncing all map/unmaps through qemu this way, this 669 * would be the right place to wire that up (tell the KVM 670 * device emulation the VFIO iommu handles to use). 671 */ 672 giommu = g_malloc0(sizeof(*giommu)); 673 giommu->iommu_mr = iommu_mr; 674 giommu->iommu_offset = section->offset_within_address_space - 675 section->offset_within_region; 676 giommu->container = container; 677 llend = int128_add(int128_make64(section->offset_within_region), 678 section->size); 679 llend = int128_sub(llend, int128_one()); 680 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 681 MEMTXATTRS_UNSPECIFIED); 682 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 683 IOMMU_NOTIFIER_IOTLB_EVENTS, 684 section->offset_within_region, 685 int128_get64(llend), 686 iommu_idx); 687 688 ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr, 689 container->pgsizes, 690 &err); 691 if (ret) { 692 g_free(giommu); 693 goto fail; 694 } 695 696 if (container->iova_ranges) { 697 ret = memory_region_iommu_set_iova_ranges(giommu->iommu_mr, 698 container->iova_ranges, &err); 699 if (ret) { 700 g_free(giommu); 701 goto fail; 702 } 703 } 704 705 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 706 &err); 707 if (ret) { 708 g_free(giommu); 709 goto fail; 710 } 711 QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next); 712 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 713 714 return; 715 } 716 717 /* Here we assume that memory_region_is_ram(section->mr)==true */ 718 719 /* 720 * For RAM memory regions with a RamDiscardManager, we only want to map the 721 * actually populated parts - and update the mapping whenever we're notified 722 * about changes. 723 */ 724 if (memory_region_has_ram_discard_manager(section->mr)) { 725 vfio_register_ram_discard_listener(container, section); 726 return; 727 } 728 729 vaddr = memory_region_get_ram_ptr(section->mr) + 730 section->offset_within_region + 731 (iova - section->offset_within_address_space); 732 733 trace_vfio_listener_region_add_ram(iova, end, vaddr); 734 735 llsize = int128_sub(llend, int128_make64(iova)); 736 737 if (memory_region_is_ram_device(section->mr)) { 738 hwaddr pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 739 740 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 741 trace_vfio_listener_region_add_no_dma_map( 742 memory_region_name(section->mr), 743 section->offset_within_address_space, 744 int128_getlo(section->size), 745 pgmask + 1); 746 return; 747 } 748 } 749 750 ret = vfio_dma_map(container, iova, int128_get64(llsize), 751 vaddr, section->readonly); 752 if (ret) { 753 error_setg(&err, "vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 754 "0x%"HWADDR_PRIx", %p) = %d (%s)", 755 container, iova, int128_get64(llsize), vaddr, ret, 756 strerror(-ret)); 757 if (memory_region_is_ram_device(section->mr)) { 758 /* Allow unexpected mappings not to be fatal for RAM devices */ 759 error_report_err(err); 760 return; 761 } 762 goto fail; 763 } 764 765 return; 766 767 fail: 768 if (memory_region_is_ram_device(section->mr)) { 769 error_reportf_err(err, "PCI p2p may not work: "); 770 return; 771 } 772 /* 773 * On the initfn path, store the first error in the container so we 774 * can gracefully fail. Runtime, there's not much we can do other 775 * than throw a hardware error. 776 */ 777 if (!container->initialized) { 778 if (!container->error) { 779 error_propagate_prepend(&container->error, err, 780 "Region %s: ", 781 memory_region_name(section->mr)); 782 } else { 783 error_free(err); 784 } 785 } else { 786 error_report_err(err); 787 hw_error("vfio: DMA mapping failed, unable to continue"); 788 } 789 } 790 791 static void vfio_listener_region_del(MemoryListener *listener, 792 MemoryRegionSection *section) 793 { 794 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 795 hwaddr iova, end; 796 Int128 llend, llsize; 797 int ret; 798 bool try_unmap = true; 799 800 if (!vfio_listener_valid_section(section, "region_del")) { 801 return; 802 } 803 804 if (memory_region_is_iommu(section->mr)) { 805 VFIOGuestIOMMU *giommu; 806 807 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 808 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 809 giommu->n.start == section->offset_within_region) { 810 memory_region_unregister_iommu_notifier(section->mr, 811 &giommu->n); 812 QLIST_REMOVE(giommu, giommu_next); 813 g_free(giommu); 814 break; 815 } 816 } 817 818 /* 819 * FIXME: We assume the one big unmap below is adequate to 820 * remove any individual page mappings in the IOMMU which 821 * might have been copied into VFIO. This works for a page table 822 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 823 * That may not be true for all IOMMU types. 824 */ 825 } 826 827 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) { 828 return; 829 } 830 831 llsize = int128_sub(llend, int128_make64(iova)); 832 833 trace_vfio_listener_region_del(iova, end); 834 835 if (memory_region_is_ram_device(section->mr)) { 836 hwaddr pgmask; 837 VFIOHostDMAWindow *hostwin; 838 839 hostwin = vfio_find_hostwin(container, iova, end); 840 assert(hostwin); /* or region_add() would have failed */ 841 842 pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 843 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 844 } else if (memory_region_has_ram_discard_manager(section->mr)) { 845 vfio_unregister_ram_discard_listener(container, section); 846 /* Unregistering will trigger an unmap. */ 847 try_unmap = false; 848 } 849 850 if (try_unmap) { 851 if (int128_eq(llsize, int128_2_64())) { 852 /* The unmap ioctl doesn't accept a full 64-bit span. */ 853 llsize = int128_rshift(llsize, 1); 854 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 855 if (ret) { 856 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 857 "0x%"HWADDR_PRIx") = %d (%s)", 858 container, iova, int128_get64(llsize), ret, 859 strerror(-ret)); 860 } 861 iova += int128_get64(llsize); 862 } 863 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 864 if (ret) { 865 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 866 "0x%"HWADDR_PRIx") = %d (%s)", 867 container, iova, int128_get64(llsize), ret, 868 strerror(-ret)); 869 } 870 } 871 872 memory_region_unref(section->mr); 873 874 vfio_container_del_section_window(container, section); 875 } 876 877 typedef struct VFIODirtyRanges { 878 hwaddr min32; 879 hwaddr max32; 880 hwaddr min64; 881 hwaddr max64; 882 hwaddr minpci64; 883 hwaddr maxpci64; 884 } VFIODirtyRanges; 885 886 typedef struct VFIODirtyRangesListener { 887 VFIOContainer *container; 888 VFIODirtyRanges ranges; 889 MemoryListener listener; 890 } VFIODirtyRangesListener; 891 892 static bool vfio_section_is_vfio_pci(MemoryRegionSection *section, 893 VFIOContainer *container) 894 { 895 VFIOPCIDevice *pcidev; 896 VFIODevice *vbasedev; 897 Object *owner; 898 899 owner = memory_region_owner(section->mr); 900 901 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 902 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 903 continue; 904 } 905 pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 906 if (OBJECT(pcidev) == owner) { 907 return true; 908 } 909 } 910 911 return false; 912 } 913 914 static void vfio_dirty_tracking_update(MemoryListener *listener, 915 MemoryRegionSection *section) 916 { 917 VFIODirtyRangesListener *dirty = container_of(listener, 918 VFIODirtyRangesListener, 919 listener); 920 VFIODirtyRanges *range = &dirty->ranges; 921 hwaddr iova, end, *min, *max; 922 923 if (!vfio_listener_valid_section(section, "tracking_update") || 924 !vfio_get_section_iova_range(dirty->container, section, 925 &iova, &end, NULL)) { 926 return; 927 } 928 929 /* 930 * The address space passed to the dirty tracker is reduced to three ranges: 931 * one for 32-bit DMA ranges, one for 64-bit DMA ranges and one for the 932 * PCI 64-bit hole. 933 * 934 * The underlying reports of dirty will query a sub-interval of each of 935 * these ranges. 936 * 937 * The purpose of the three range handling is to handle known cases of big 938 * holes in the address space, like the x86 AMD 1T hole, and firmware (like 939 * OVMF) which may relocate the pci-hole64 to the end of the address space. 940 * The latter would otherwise generate large ranges for tracking, stressing 941 * the limits of supported hardware. The pci-hole32 will always be below 4G 942 * (overlapping or not) so it doesn't need special handling and is part of 943 * the 32-bit range. 944 * 945 * The alternative would be an IOVATree but that has a much bigger runtime 946 * overhead and unnecessary complexity. 947 */ 948 if (vfio_section_is_vfio_pci(section, dirty->container) && 949 iova >= UINT32_MAX) { 950 min = &range->minpci64; 951 max = &range->maxpci64; 952 } else { 953 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 954 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 955 } 956 if (*min > iova) { 957 *min = iova; 958 } 959 if (*max < end) { 960 *max = end; 961 } 962 963 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 964 return; 965 } 966 967 static const MemoryListener vfio_dirty_tracking_listener = { 968 .name = "vfio-tracking", 969 .region_add = vfio_dirty_tracking_update, 970 }; 971 972 static void vfio_dirty_tracking_init(VFIOContainer *container, 973 VFIODirtyRanges *ranges) 974 { 975 VFIODirtyRangesListener dirty; 976 977 memset(&dirty, 0, sizeof(dirty)); 978 dirty.ranges.min32 = UINT32_MAX; 979 dirty.ranges.min64 = UINT64_MAX; 980 dirty.ranges.minpci64 = UINT64_MAX; 981 dirty.listener = vfio_dirty_tracking_listener; 982 dirty.container = container; 983 984 memory_listener_register(&dirty.listener, 985 container->space->as); 986 987 *ranges = dirty.ranges; 988 989 /* 990 * The memory listener is synchronous, and used to calculate the range 991 * to dirty tracking. Unregister it after we are done as we are not 992 * interested in any follow-up updates. 993 */ 994 memory_listener_unregister(&dirty.listener); 995 } 996 997 static void vfio_devices_dma_logging_stop(VFIOContainer *container) 998 { 999 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 1000 sizeof(uint64_t))] = {}; 1001 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1002 VFIODevice *vbasedev; 1003 1004 feature->argsz = sizeof(buf); 1005 feature->flags = VFIO_DEVICE_FEATURE_SET | 1006 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 1007 1008 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 1009 if (!vbasedev->dirty_tracking) { 1010 continue; 1011 } 1012 1013 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1014 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 1015 vbasedev->name, -errno, strerror(errno)); 1016 } 1017 vbasedev->dirty_tracking = false; 1018 } 1019 } 1020 1021 static struct vfio_device_feature * 1022 vfio_device_feature_dma_logging_start_create(VFIOContainer *container, 1023 VFIODirtyRanges *tracking) 1024 { 1025 struct vfio_device_feature *feature; 1026 size_t feature_size; 1027 struct vfio_device_feature_dma_logging_control *control; 1028 struct vfio_device_feature_dma_logging_range *ranges; 1029 1030 feature_size = sizeof(struct vfio_device_feature) + 1031 sizeof(struct vfio_device_feature_dma_logging_control); 1032 feature = g_try_malloc0(feature_size); 1033 if (!feature) { 1034 errno = ENOMEM; 1035 return NULL; 1036 } 1037 feature->argsz = feature_size; 1038 feature->flags = VFIO_DEVICE_FEATURE_SET | 1039 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 1040 1041 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 1042 control->page_size = qemu_real_host_page_size(); 1043 1044 /* 1045 * DMA logging uAPI guarantees to support at least a number of ranges that 1046 * fits into a single host kernel base page. 1047 */ 1048 control->num_ranges = !!tracking->max32 + !!tracking->max64 + 1049 !!tracking->maxpci64; 1050 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 1051 control->num_ranges); 1052 if (!ranges) { 1053 g_free(feature); 1054 errno = ENOMEM; 1055 1056 return NULL; 1057 } 1058 1059 control->ranges = (__u64)(uintptr_t)ranges; 1060 if (tracking->max32) { 1061 ranges->iova = tracking->min32; 1062 ranges->length = (tracking->max32 - tracking->min32) + 1; 1063 ranges++; 1064 } 1065 if (tracking->max64) { 1066 ranges->iova = tracking->min64; 1067 ranges->length = (tracking->max64 - tracking->min64) + 1; 1068 ranges++; 1069 } 1070 if (tracking->maxpci64) { 1071 ranges->iova = tracking->minpci64; 1072 ranges->length = (tracking->maxpci64 - tracking->minpci64) + 1; 1073 } 1074 1075 trace_vfio_device_dirty_tracking_start(control->num_ranges, 1076 tracking->min32, tracking->max32, 1077 tracking->min64, tracking->max64, 1078 tracking->minpci64, tracking->maxpci64); 1079 1080 return feature; 1081 } 1082 1083 static void vfio_device_feature_dma_logging_start_destroy( 1084 struct vfio_device_feature *feature) 1085 { 1086 struct vfio_device_feature_dma_logging_control *control = 1087 (struct vfio_device_feature_dma_logging_control *)feature->data; 1088 struct vfio_device_feature_dma_logging_range *ranges = 1089 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 1090 1091 g_free(ranges); 1092 g_free(feature); 1093 } 1094 1095 static int vfio_devices_dma_logging_start(VFIOContainer *container) 1096 { 1097 struct vfio_device_feature *feature; 1098 VFIODirtyRanges ranges; 1099 VFIODevice *vbasedev; 1100 int ret = 0; 1101 1102 vfio_dirty_tracking_init(container, &ranges); 1103 feature = vfio_device_feature_dma_logging_start_create(container, 1104 &ranges); 1105 if (!feature) { 1106 return -errno; 1107 } 1108 1109 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 1110 if (vbasedev->dirty_tracking) { 1111 continue; 1112 } 1113 1114 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 1115 if (ret) { 1116 ret = -errno; 1117 error_report("%s: Failed to start DMA logging, err %d (%s)", 1118 vbasedev->name, ret, strerror(errno)); 1119 goto out; 1120 } 1121 vbasedev->dirty_tracking = true; 1122 } 1123 1124 out: 1125 if (ret) { 1126 vfio_devices_dma_logging_stop(container); 1127 } 1128 1129 vfio_device_feature_dma_logging_start_destroy(feature); 1130 1131 return ret; 1132 } 1133 1134 static void vfio_listener_log_global_start(MemoryListener *listener) 1135 { 1136 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1137 int ret; 1138 1139 if (vfio_devices_all_device_dirty_tracking(container)) { 1140 ret = vfio_devices_dma_logging_start(container); 1141 } else { 1142 ret = vfio_set_dirty_page_tracking(container, true); 1143 } 1144 1145 if (ret) { 1146 error_report("vfio: Could not start dirty page tracking, err: %d (%s)", 1147 ret, strerror(-ret)); 1148 vfio_set_migration_error(ret); 1149 } 1150 } 1151 1152 static void vfio_listener_log_global_stop(MemoryListener *listener) 1153 { 1154 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1155 int ret = 0; 1156 1157 if (vfio_devices_all_device_dirty_tracking(container)) { 1158 vfio_devices_dma_logging_stop(container); 1159 } else { 1160 ret = vfio_set_dirty_page_tracking(container, false); 1161 } 1162 1163 if (ret) { 1164 error_report("vfio: Could not stop dirty page tracking, err: %d (%s)", 1165 ret, strerror(-ret)); 1166 vfio_set_migration_error(ret); 1167 } 1168 } 1169 1170 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, 1171 hwaddr size, void *bitmap) 1172 { 1173 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 1174 sizeof(struct vfio_device_feature_dma_logging_report), 1175 sizeof(__u64))] = {}; 1176 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1177 struct vfio_device_feature_dma_logging_report *report = 1178 (struct vfio_device_feature_dma_logging_report *)feature->data; 1179 1180 report->iova = iova; 1181 report->length = size; 1182 report->page_size = qemu_real_host_page_size(); 1183 report->bitmap = (__u64)(uintptr_t)bitmap; 1184 1185 feature->argsz = sizeof(buf); 1186 feature->flags = VFIO_DEVICE_FEATURE_GET | 1187 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; 1188 1189 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1190 return -errno; 1191 } 1192 1193 return 0; 1194 } 1195 1196 int vfio_devices_query_dirty_bitmap(VFIOContainer *container, 1197 VFIOBitmap *vbmap, hwaddr iova, 1198 hwaddr size) 1199 { 1200 VFIODevice *vbasedev; 1201 int ret; 1202 1203 QLIST_FOREACH(vbasedev, &container->device_list, container_next) { 1204 ret = vfio_device_dma_logging_report(vbasedev, iova, size, 1205 vbmap->bitmap); 1206 if (ret) { 1207 error_report("%s: Failed to get DMA logging report, iova: " 1208 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx 1209 ", err: %d (%s)", 1210 vbasedev->name, iova, size, ret, strerror(-ret)); 1211 1212 return ret; 1213 } 1214 } 1215 1216 return 0; 1217 } 1218 1219 int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova, 1220 uint64_t size, ram_addr_t ram_addr) 1221 { 1222 bool all_device_dirty_tracking = 1223 vfio_devices_all_device_dirty_tracking(container); 1224 uint64_t dirty_pages; 1225 VFIOBitmap vbmap; 1226 int ret; 1227 1228 if (!container->dirty_pages_supported && !all_device_dirty_tracking) { 1229 cpu_physical_memory_set_dirty_range(ram_addr, size, 1230 tcg_enabled() ? DIRTY_CLIENTS_ALL : 1231 DIRTY_CLIENTS_NOCODE); 1232 return 0; 1233 } 1234 1235 ret = vfio_bitmap_alloc(&vbmap, size); 1236 if (ret) { 1237 return ret; 1238 } 1239 1240 if (all_device_dirty_tracking) { 1241 ret = vfio_devices_query_dirty_bitmap(container, &vbmap, iova, size); 1242 } else { 1243 ret = vfio_query_dirty_bitmap(container, &vbmap, iova, size); 1244 } 1245 1246 if (ret) { 1247 goto out; 1248 } 1249 1250 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, 1251 vbmap.pages); 1252 1253 trace_vfio_get_dirty_bitmap(container->fd, iova, size, vbmap.size, 1254 ram_addr, dirty_pages); 1255 out: 1256 g_free(vbmap.bitmap); 1257 1258 return ret; 1259 } 1260 1261 typedef struct { 1262 IOMMUNotifier n; 1263 VFIOGuestIOMMU *giommu; 1264 } vfio_giommu_dirty_notifier; 1265 1266 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1267 { 1268 vfio_giommu_dirty_notifier *gdn = container_of(n, 1269 vfio_giommu_dirty_notifier, n); 1270 VFIOGuestIOMMU *giommu = gdn->giommu; 1271 VFIOContainer *container = giommu->container; 1272 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1273 ram_addr_t translated_addr; 1274 int ret = -EINVAL; 1275 1276 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1277 1278 if (iotlb->target_as != &address_space_memory) { 1279 error_report("Wrong target AS \"%s\", only system memory is allowed", 1280 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1281 goto out; 1282 } 1283 1284 rcu_read_lock(); 1285 if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) { 1286 ret = vfio_get_dirty_bitmap(container, iova, iotlb->addr_mask + 1, 1287 translated_addr); 1288 if (ret) { 1289 error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1290 "0x%"HWADDR_PRIx") = %d (%s)", 1291 container, iova, iotlb->addr_mask + 1, ret, 1292 strerror(-ret)); 1293 } 1294 } 1295 rcu_read_unlock(); 1296 1297 out: 1298 if (ret) { 1299 vfio_set_migration_error(ret); 1300 } 1301 } 1302 1303 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1304 void *opaque) 1305 { 1306 const hwaddr size = int128_get64(section->size); 1307 const hwaddr iova = section->offset_within_address_space; 1308 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1309 section->offset_within_region; 1310 VFIORamDiscardListener *vrdl = opaque; 1311 1312 /* 1313 * Sync the whole mapped region (spanning multiple individual mappings) 1314 * in one go. 1315 */ 1316 return vfio_get_dirty_bitmap(vrdl->container, iova, size, ram_addr); 1317 } 1318 1319 static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container, 1320 MemoryRegionSection *section) 1321 { 1322 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1323 VFIORamDiscardListener *vrdl = NULL; 1324 1325 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 1326 if (vrdl->mr == section->mr && 1327 vrdl->offset_within_address_space == 1328 section->offset_within_address_space) { 1329 break; 1330 } 1331 } 1332 1333 if (!vrdl) { 1334 hw_error("vfio: Trying to sync missing RAM discard listener"); 1335 } 1336 1337 /* 1338 * We only want/can synchronize the bitmap for actually mapped parts - 1339 * which correspond to populated parts. Replay all populated parts. 1340 */ 1341 return ram_discard_manager_replay_populated(rdm, section, 1342 vfio_ram_discard_get_dirty_bitmap, 1343 &vrdl); 1344 } 1345 1346 static int vfio_sync_dirty_bitmap(VFIOContainer *container, 1347 MemoryRegionSection *section) 1348 { 1349 ram_addr_t ram_addr; 1350 1351 if (memory_region_is_iommu(section->mr)) { 1352 VFIOGuestIOMMU *giommu; 1353 1354 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 1355 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1356 giommu->n.start == section->offset_within_region) { 1357 Int128 llend; 1358 vfio_giommu_dirty_notifier gdn = { .giommu = giommu }; 1359 int idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1360 MEMTXATTRS_UNSPECIFIED); 1361 1362 llend = int128_add(int128_make64(section->offset_within_region), 1363 section->size); 1364 llend = int128_sub(llend, int128_one()); 1365 1366 iommu_notifier_init(&gdn.n, 1367 vfio_iommu_map_dirty_notify, 1368 IOMMU_NOTIFIER_MAP, 1369 section->offset_within_region, 1370 int128_get64(llend), 1371 idx); 1372 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1373 break; 1374 } 1375 } 1376 return 0; 1377 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1378 return vfio_sync_ram_discard_listener_dirty_bitmap(container, section); 1379 } 1380 1381 ram_addr = memory_region_get_ram_addr(section->mr) + 1382 section->offset_within_region; 1383 1384 return vfio_get_dirty_bitmap(container, 1385 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1386 int128_get64(section->size), ram_addr); 1387 } 1388 1389 static void vfio_listener_log_sync(MemoryListener *listener, 1390 MemoryRegionSection *section) 1391 { 1392 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1393 int ret; 1394 1395 if (vfio_listener_skipped_section(section)) { 1396 return; 1397 } 1398 1399 if (vfio_devices_all_dirty_tracking(container)) { 1400 ret = vfio_sync_dirty_bitmap(container, section); 1401 if (ret) { 1402 error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret, 1403 strerror(-ret)); 1404 vfio_set_migration_error(ret); 1405 } 1406 } 1407 } 1408 1409 const MemoryListener vfio_memory_listener = { 1410 .name = "vfio", 1411 .region_add = vfio_listener_region_add, 1412 .region_del = vfio_listener_region_del, 1413 .log_global_start = vfio_listener_log_global_start, 1414 .log_global_stop = vfio_listener_log_global_stop, 1415 .log_sync = vfio_listener_log_sync, 1416 }; 1417 1418 void vfio_reset_handler(void *opaque) 1419 { 1420 VFIODevice *vbasedev; 1421 1422 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1423 if (vbasedev->dev->realized) { 1424 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 1425 } 1426 } 1427 1428 QLIST_FOREACH(vbasedev, &vfio_device_list, next) { 1429 if (vbasedev->dev->realized && vbasedev->needs_reset) { 1430 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 1431 } 1432 } 1433 } 1434 1435 int vfio_kvm_device_add_fd(int fd, Error **errp) 1436 { 1437 #ifdef CONFIG_KVM 1438 struct kvm_device_attr attr = { 1439 .group = KVM_DEV_VFIO_FILE, 1440 .attr = KVM_DEV_VFIO_FILE_ADD, 1441 .addr = (uint64_t)(unsigned long)&fd, 1442 }; 1443 1444 if (!kvm_enabled()) { 1445 return 0; 1446 } 1447 1448 if (vfio_kvm_device_fd < 0) { 1449 struct kvm_create_device cd = { 1450 .type = KVM_DEV_TYPE_VFIO, 1451 }; 1452 1453 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 1454 error_setg_errno(errp, errno, "Failed to create KVM VFIO device"); 1455 return -errno; 1456 } 1457 1458 vfio_kvm_device_fd = cd.fd; 1459 } 1460 1461 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1462 error_setg_errno(errp, errno, "Failed to add fd %d to KVM VFIO device", 1463 fd); 1464 return -errno; 1465 } 1466 #endif 1467 return 0; 1468 } 1469 1470 int vfio_kvm_device_del_fd(int fd, Error **errp) 1471 { 1472 #ifdef CONFIG_KVM 1473 struct kvm_device_attr attr = { 1474 .group = KVM_DEV_VFIO_FILE, 1475 .attr = KVM_DEV_VFIO_FILE_DEL, 1476 .addr = (uint64_t)(unsigned long)&fd, 1477 }; 1478 1479 if (vfio_kvm_device_fd < 0) { 1480 error_setg(errp, "KVM VFIO device isn't created yet"); 1481 return -EINVAL; 1482 } 1483 1484 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1485 error_setg_errno(errp, errno, 1486 "Failed to remove fd %d from KVM VFIO device", fd); 1487 return -errno; 1488 } 1489 #endif 1490 return 0; 1491 } 1492 1493 VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1494 { 1495 VFIOAddressSpace *space; 1496 1497 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1498 if (space->as == as) { 1499 return space; 1500 } 1501 } 1502 1503 /* No suitable VFIOAddressSpace, create a new one */ 1504 space = g_malloc0(sizeof(*space)); 1505 space->as = as; 1506 QLIST_INIT(&space->containers); 1507 1508 if (QLIST_EMPTY(&vfio_address_spaces)) { 1509 qemu_register_reset(vfio_reset_handler, NULL); 1510 } 1511 1512 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1513 1514 return space; 1515 } 1516 1517 void vfio_put_address_space(VFIOAddressSpace *space) 1518 { 1519 if (QLIST_EMPTY(&space->containers)) { 1520 QLIST_REMOVE(space, list); 1521 g_free(space); 1522 } 1523 if (QLIST_EMPTY(&vfio_address_spaces)) { 1524 qemu_unregister_reset(vfio_reset_handler, NULL); 1525 } 1526 } 1527 1528 struct vfio_device_info *vfio_get_device_info(int fd) 1529 { 1530 struct vfio_device_info *info; 1531 uint32_t argsz = sizeof(*info); 1532 1533 info = g_malloc0(argsz); 1534 1535 retry: 1536 info->argsz = argsz; 1537 1538 if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) { 1539 g_free(info); 1540 return NULL; 1541 } 1542 1543 if (info->argsz > argsz) { 1544 argsz = info->argsz; 1545 info = g_realloc(info, argsz); 1546 goto retry; 1547 } 1548 1549 return info; 1550 } 1551