1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/pci.h" 30 #include "exec/address-spaces.h" 31 #include "exec/memory.h" 32 #include "exec/ram_addr.h" 33 #include "hw/hw.h" 34 #include "qemu/error-report.h" 35 #include "qemu/main-loop.h" 36 #include "qemu/range.h" 37 #include "system/kvm.h" 38 #include "system/reset.h" 39 #include "system/runstate.h" 40 #include "trace.h" 41 #include "qapi/error.h" 42 #include "migration/misc.h" 43 #include "migration/blocker.h" 44 #include "migration/qemu-file.h" 45 #include "system/tpm.h" 46 47 VFIODeviceList vfio_device_list = 48 QLIST_HEAD_INITIALIZER(vfio_device_list); 49 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 50 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 51 52 #ifdef CONFIG_KVM 53 /* 54 * We have a single VFIO pseudo device per KVM VM. Once created it lives 55 * for the life of the VM. Closing the file descriptor only drops our 56 * reference to it and the device's reference to kvm. Therefore once 57 * initialized, this file descriptor is only released on QEMU exit and 58 * we'll re-use it should another vfio device be attached before then. 59 */ 60 int vfio_kvm_device_fd = -1; 61 #endif 62 63 /* 64 * Device state interfaces 65 */ 66 67 bool vfio_mig_active(void) 68 { 69 VFIODevice *vbasedev; 70 71 if (QLIST_EMPTY(&vfio_device_list)) { 72 return false; 73 } 74 75 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 76 if (vbasedev->migration_blocker) { 77 return false; 78 } 79 } 80 return true; 81 } 82 83 static Error *multiple_devices_migration_blocker; 84 85 /* 86 * Multiple devices migration is allowed only if all devices support P2P 87 * migration. Single device migration is allowed regardless of P2P migration 88 * support. 89 */ 90 static bool vfio_multiple_devices_migration_is_supported(void) 91 { 92 VFIODevice *vbasedev; 93 unsigned int device_num = 0; 94 bool all_support_p2p = true; 95 96 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 97 if (vbasedev->migration) { 98 device_num++; 99 100 if (!(vbasedev->migration->mig_flags & VFIO_MIGRATION_P2P)) { 101 all_support_p2p = false; 102 } 103 } 104 } 105 106 return all_support_p2p || device_num <= 1; 107 } 108 109 int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp) 110 { 111 int ret; 112 113 if (vfio_multiple_devices_migration_is_supported()) { 114 return 0; 115 } 116 117 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) { 118 error_setg(errp, "Multiple VFIO devices migration is supported only if " 119 "all of them support P2P migration"); 120 return -EINVAL; 121 } 122 123 if (multiple_devices_migration_blocker) { 124 return 0; 125 } 126 127 error_setg(&multiple_devices_migration_blocker, 128 "Multiple VFIO devices migration is supported only if all of " 129 "them support P2P migration"); 130 ret = migrate_add_blocker_normal(&multiple_devices_migration_blocker, errp); 131 132 return ret; 133 } 134 135 void vfio_unblock_multiple_devices_migration(void) 136 { 137 if (!multiple_devices_migration_blocker || 138 !vfio_multiple_devices_migration_is_supported()) { 139 return; 140 } 141 142 migrate_del_blocker(&multiple_devices_migration_blocker); 143 } 144 145 bool vfio_viommu_preset(VFIODevice *vbasedev) 146 { 147 return vbasedev->bcontainer->space->as != &address_space_memory; 148 } 149 150 static void vfio_set_migration_error(int ret) 151 { 152 if (migration_is_running()) { 153 migration_file_set_error(ret, NULL); 154 } 155 } 156 157 bool vfio_device_state_is_running(VFIODevice *vbasedev) 158 { 159 VFIOMigration *migration = vbasedev->migration; 160 161 return migration->device_state == VFIO_DEVICE_STATE_RUNNING || 162 migration->device_state == VFIO_DEVICE_STATE_RUNNING_P2P; 163 } 164 165 bool vfio_device_state_is_precopy(VFIODevice *vbasedev) 166 { 167 VFIOMigration *migration = vbasedev->migration; 168 169 return migration->device_state == VFIO_DEVICE_STATE_PRE_COPY || 170 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY_P2P; 171 } 172 173 static bool vfio_devices_all_device_dirty_tracking_started( 174 const VFIOContainerBase *bcontainer) 175 { 176 VFIODevice *vbasedev; 177 178 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 179 if (!vbasedev->dirty_tracking) { 180 return false; 181 } 182 } 183 184 return true; 185 } 186 187 bool vfio_devices_all_dirty_tracking_started( 188 const VFIOContainerBase *bcontainer) 189 { 190 return vfio_devices_all_device_dirty_tracking_started(bcontainer) || 191 bcontainer->dirty_pages_started; 192 } 193 194 static bool vfio_log_sync_needed(const VFIOContainerBase *bcontainer) 195 { 196 VFIODevice *vbasedev; 197 198 if (!vfio_devices_all_dirty_tracking_started(bcontainer)) { 199 return false; 200 } 201 202 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 203 VFIOMigration *migration = vbasedev->migration; 204 205 if (!migration) { 206 return false; 207 } 208 209 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 210 (vfio_device_state_is_running(vbasedev) || 211 vfio_device_state_is_precopy(vbasedev))) { 212 return false; 213 } 214 } 215 return true; 216 } 217 218 bool vfio_devices_all_device_dirty_tracking(const VFIOContainerBase *bcontainer) 219 { 220 VFIODevice *vbasedev; 221 222 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 223 if (vbasedev->device_dirty_page_tracking == ON_OFF_AUTO_OFF) { 224 return false; 225 } 226 if (!vbasedev->dirty_pages_supported) { 227 return false; 228 } 229 } 230 231 return true; 232 } 233 234 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 235 { 236 return (!memory_region_is_ram(section->mr) && 237 !memory_region_is_iommu(section->mr)) || 238 memory_region_is_protected(section->mr) || 239 /* 240 * Sizing an enabled 64-bit BAR can cause spurious mappings to 241 * addresses in the upper part of the 64-bit address space. These 242 * are never accessed by the CPU and beyond the address width of 243 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 244 */ 245 section->offset_within_address_space & (1ULL << 63); 246 } 247 248 /* Called with rcu_read_lock held. */ 249 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 250 ram_addr_t *ram_addr, bool *read_only, 251 Error **errp) 252 { 253 bool ret, mr_has_discard_manager; 254 255 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, 256 &mr_has_discard_manager, errp); 257 if (ret && mr_has_discard_manager) { 258 /* 259 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 260 * pages will remain pinned inside vfio until unmapped, resulting in a 261 * higher memory consumption than expected. If memory would get 262 * populated again later, there would be an inconsistency between pages 263 * pinned by vfio and pages seen by QEMU. This is the case until 264 * unmapped from the IOMMU (e.g., during device reset). 265 * 266 * With malicious guests, we really only care about pinning more memory 267 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 268 * exceeded and can be used to mitigate this problem. 269 */ 270 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 271 " RAM (e.g., virtio-mem) works, however, malicious" 272 " guests can trigger pinning of more memory than" 273 " intended via an IOMMU. It's possible to mitigate " 274 " by setting/adjusting RLIMIT_MEMLOCK."); 275 } 276 return ret; 277 } 278 279 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 280 { 281 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 282 VFIOContainerBase *bcontainer = giommu->bcontainer; 283 hwaddr iova = iotlb->iova + giommu->iommu_offset; 284 void *vaddr; 285 int ret; 286 Error *local_err = NULL; 287 288 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 289 iova, iova + iotlb->addr_mask); 290 291 if (iotlb->target_as != &address_space_memory) { 292 error_report("Wrong target AS \"%s\", only system memory is allowed", 293 iotlb->target_as->name ? iotlb->target_as->name : "none"); 294 vfio_set_migration_error(-EINVAL); 295 return; 296 } 297 298 rcu_read_lock(); 299 300 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 301 bool read_only; 302 303 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only, &local_err)) { 304 error_report_err(local_err); 305 goto out; 306 } 307 /* 308 * vaddr is only valid until rcu_read_unlock(). But after 309 * vfio_dma_map has set up the mapping the pages will be 310 * pinned by the kernel. This makes sure that the RAM backend 311 * of vaddr will always be there, even if the memory object is 312 * destroyed and its backing memory munmap-ed. 313 */ 314 ret = vfio_container_dma_map(bcontainer, iova, 315 iotlb->addr_mask + 1, vaddr, 316 read_only); 317 if (ret) { 318 error_report("vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 319 "0x%"HWADDR_PRIx", %p) = %d (%s)", 320 bcontainer, iova, 321 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 322 } 323 } else { 324 ret = vfio_container_dma_unmap(bcontainer, iova, 325 iotlb->addr_mask + 1, iotlb); 326 if (ret) { 327 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 328 "0x%"HWADDR_PRIx") = %d (%s)", 329 bcontainer, iova, 330 iotlb->addr_mask + 1, ret, strerror(-ret)); 331 vfio_set_migration_error(ret); 332 } 333 } 334 out: 335 rcu_read_unlock(); 336 } 337 338 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 339 MemoryRegionSection *section) 340 { 341 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 342 listener); 343 VFIOContainerBase *bcontainer = vrdl->bcontainer; 344 const hwaddr size = int128_get64(section->size); 345 const hwaddr iova = section->offset_within_address_space; 346 int ret; 347 348 /* Unmap with a single call. */ 349 ret = vfio_container_dma_unmap(bcontainer, iova, size , NULL); 350 if (ret) { 351 error_report("%s: vfio_container_dma_unmap() failed: %s", __func__, 352 strerror(-ret)); 353 } 354 } 355 356 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 357 MemoryRegionSection *section) 358 { 359 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 360 listener); 361 VFIOContainerBase *bcontainer = vrdl->bcontainer; 362 const hwaddr end = section->offset_within_region + 363 int128_get64(section->size); 364 hwaddr start, next, iova; 365 void *vaddr; 366 int ret; 367 368 /* 369 * Map in (aligned within memory region) minimum granularity, so we can 370 * unmap in minimum granularity later. 371 */ 372 for (start = section->offset_within_region; start < end; start = next) { 373 next = ROUND_UP(start + 1, vrdl->granularity); 374 next = MIN(next, end); 375 376 iova = start - section->offset_within_region + 377 section->offset_within_address_space; 378 vaddr = memory_region_get_ram_ptr(section->mr) + start; 379 380 ret = vfio_container_dma_map(bcontainer, iova, next - start, 381 vaddr, section->readonly); 382 if (ret) { 383 /* Rollback */ 384 vfio_ram_discard_notify_discard(rdl, section); 385 return ret; 386 } 387 } 388 return 0; 389 } 390 391 static void vfio_register_ram_discard_listener(VFIOContainerBase *bcontainer, 392 MemoryRegionSection *section) 393 { 394 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 395 VFIORamDiscardListener *vrdl; 396 397 /* Ignore some corner cases not relevant in practice. */ 398 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 399 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 400 TARGET_PAGE_SIZE)); 401 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 402 403 vrdl = g_new0(VFIORamDiscardListener, 1); 404 vrdl->bcontainer = bcontainer; 405 vrdl->mr = section->mr; 406 vrdl->offset_within_address_space = section->offset_within_address_space; 407 vrdl->size = int128_get64(section->size); 408 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 409 section->mr); 410 411 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 412 g_assert(bcontainer->pgsizes && 413 vrdl->granularity >= 1ULL << ctz64(bcontainer->pgsizes)); 414 415 ram_discard_listener_init(&vrdl->listener, 416 vfio_ram_discard_notify_populate, 417 vfio_ram_discard_notify_discard, true); 418 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 419 QLIST_INSERT_HEAD(&bcontainer->vrdl_list, vrdl, next); 420 421 /* 422 * Sanity-check if we have a theoretically problematic setup where we could 423 * exceed the maximum number of possible DMA mappings over time. We assume 424 * that each mapped section in the same address space as a RamDiscardManager 425 * section consumes exactly one DMA mapping, with the exception of 426 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 427 * in the same address space as RamDiscardManager sections. 428 * 429 * We assume that each section in the address space consumes one memslot. 430 * We take the number of KVM memory slots as a best guess for the maximum 431 * number of sections in the address space we could have over time, 432 * also consuming DMA mappings. 433 */ 434 if (bcontainer->dma_max_mappings) { 435 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 436 437 #ifdef CONFIG_KVM 438 if (kvm_enabled()) { 439 max_memslots = kvm_get_max_memslots(); 440 } 441 #endif 442 443 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 444 hwaddr start, end; 445 446 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 447 vrdl->granularity); 448 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 449 vrdl->granularity); 450 vrdl_mappings += (end - start) / vrdl->granularity; 451 vrdl_count++; 452 } 453 454 if (vrdl_mappings + max_memslots - vrdl_count > 455 bcontainer->dma_max_mappings) { 456 warn_report("%s: possibly running out of DMA mappings. E.g., try" 457 " increasing the 'block-size' of virtio-mem devies." 458 " Maximum possible DMA mappings: %d, Maximum possible" 459 " memslots: %d", __func__, bcontainer->dma_max_mappings, 460 max_memslots); 461 } 462 } 463 } 464 465 static void vfio_unregister_ram_discard_listener(VFIOContainerBase *bcontainer, 466 MemoryRegionSection *section) 467 { 468 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 469 VFIORamDiscardListener *vrdl = NULL; 470 471 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 472 if (vrdl->mr == section->mr && 473 vrdl->offset_within_address_space == 474 section->offset_within_address_space) { 475 break; 476 } 477 } 478 479 if (!vrdl) { 480 hw_error("vfio: Trying to unregister missing RAM discard listener"); 481 } 482 483 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 484 QLIST_REMOVE(vrdl, next); 485 g_free(vrdl); 486 } 487 488 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 489 { 490 MemoryRegion *mr = section->mr; 491 492 if (!TPM_IS_CRB(mr->owner)) { 493 return false; 494 } 495 496 /* this is a known safe misaligned region, just trace for debug purpose */ 497 trace_vfio_known_safe_misalignment(memory_region_name(mr), 498 section->offset_within_address_space, 499 section->offset_within_region, 500 qemu_real_host_page_size()); 501 return true; 502 } 503 504 static bool vfio_listener_valid_section(MemoryRegionSection *section, 505 const char *name) 506 { 507 if (vfio_listener_skipped_section(section)) { 508 trace_vfio_listener_region_skip(name, 509 section->offset_within_address_space, 510 section->offset_within_address_space + 511 int128_get64(int128_sub(section->size, int128_one()))); 512 return false; 513 } 514 515 if (unlikely((section->offset_within_address_space & 516 ~qemu_real_host_page_mask()) != 517 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 518 if (!vfio_known_safe_misalignment(section)) { 519 error_report("%s received unaligned region %s iova=0x%"PRIx64 520 " offset_within_region=0x%"PRIx64 521 " qemu_real_host_page_size=0x%"PRIxPTR, 522 __func__, memory_region_name(section->mr), 523 section->offset_within_address_space, 524 section->offset_within_region, 525 qemu_real_host_page_size()); 526 } 527 return false; 528 } 529 530 return true; 531 } 532 533 static bool vfio_get_section_iova_range(VFIOContainerBase *bcontainer, 534 MemoryRegionSection *section, 535 hwaddr *out_iova, hwaddr *out_end, 536 Int128 *out_llend) 537 { 538 Int128 llend; 539 hwaddr iova; 540 541 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 542 llend = int128_make64(section->offset_within_address_space); 543 llend = int128_add(llend, section->size); 544 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 545 546 if (int128_ge(int128_make64(iova), llend)) { 547 return false; 548 } 549 550 *out_iova = iova; 551 *out_end = int128_get64(int128_sub(llend, int128_one())); 552 if (out_llend) { 553 *out_llend = llend; 554 } 555 return true; 556 } 557 558 static void vfio_device_error_append(VFIODevice *vbasedev, Error **errp) 559 { 560 /* 561 * MMIO region mapping failures are not fatal but in this case PCI 562 * peer-to-peer transactions are broken. 563 */ 564 if (vbasedev && vbasedev->type == VFIO_DEVICE_TYPE_PCI) { 565 error_append_hint(errp, "%s: PCI peer-to-peer transactions " 566 "on BARs are not supported.\n", vbasedev->name); 567 } 568 } 569 570 static void vfio_listener_region_add(MemoryListener *listener, 571 MemoryRegionSection *section) 572 { 573 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 574 listener); 575 hwaddr iova, end; 576 Int128 llend, llsize; 577 void *vaddr; 578 int ret; 579 Error *err = NULL; 580 581 if (!vfio_listener_valid_section(section, "region_add")) { 582 return; 583 } 584 585 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 586 &llend)) { 587 if (memory_region_is_ram_device(section->mr)) { 588 trace_vfio_listener_region_add_no_dma_map( 589 memory_region_name(section->mr), 590 section->offset_within_address_space, 591 int128_getlo(section->size), 592 qemu_real_host_page_size()); 593 } 594 return; 595 } 596 597 /* PPC64/pseries machine only */ 598 if (!vfio_container_add_section_window(bcontainer, section, &err)) { 599 goto mmio_dma_error; 600 } 601 602 memory_region_ref(section->mr); 603 604 if (memory_region_is_iommu(section->mr)) { 605 VFIOGuestIOMMU *giommu; 606 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 607 int iommu_idx; 608 609 trace_vfio_listener_region_add_iommu(section->mr->name, iova, end); 610 /* 611 * FIXME: For VFIO iommu types which have KVM acceleration to 612 * avoid bouncing all map/unmaps through qemu this way, this 613 * would be the right place to wire that up (tell the KVM 614 * device emulation the VFIO iommu handles to use). 615 */ 616 giommu = g_malloc0(sizeof(*giommu)); 617 giommu->iommu_mr = iommu_mr; 618 giommu->iommu_offset = section->offset_within_address_space - 619 section->offset_within_region; 620 giommu->bcontainer = bcontainer; 621 llend = int128_add(int128_make64(section->offset_within_region), 622 section->size); 623 llend = int128_sub(llend, int128_one()); 624 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 625 MEMTXATTRS_UNSPECIFIED); 626 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 627 IOMMU_NOTIFIER_IOTLB_EVENTS, 628 section->offset_within_region, 629 int128_get64(llend), 630 iommu_idx); 631 632 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 633 &err); 634 if (ret) { 635 g_free(giommu); 636 goto fail; 637 } 638 QLIST_INSERT_HEAD(&bcontainer->giommu_list, giommu, giommu_next); 639 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 640 641 return; 642 } 643 644 /* Here we assume that memory_region_is_ram(section->mr)==true */ 645 646 /* 647 * For RAM memory regions with a RamDiscardManager, we only want to map the 648 * actually populated parts - and update the mapping whenever we're notified 649 * about changes. 650 */ 651 if (memory_region_has_ram_discard_manager(section->mr)) { 652 vfio_register_ram_discard_listener(bcontainer, section); 653 return; 654 } 655 656 vaddr = memory_region_get_ram_ptr(section->mr) + 657 section->offset_within_region + 658 (iova - section->offset_within_address_space); 659 660 trace_vfio_listener_region_add_ram(iova, end, vaddr); 661 662 llsize = int128_sub(llend, int128_make64(iova)); 663 664 if (memory_region_is_ram_device(section->mr)) { 665 hwaddr pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 666 667 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 668 trace_vfio_listener_region_add_no_dma_map( 669 memory_region_name(section->mr), 670 section->offset_within_address_space, 671 int128_getlo(section->size), 672 pgmask + 1); 673 return; 674 } 675 } 676 677 ret = vfio_container_dma_map(bcontainer, iova, int128_get64(llsize), 678 vaddr, section->readonly); 679 if (ret) { 680 error_setg(&err, "vfio_container_dma_map(%p, 0x%"HWADDR_PRIx", " 681 "0x%"HWADDR_PRIx", %p) = %d (%s)", 682 bcontainer, iova, int128_get64(llsize), vaddr, ret, 683 strerror(-ret)); 684 mmio_dma_error: 685 if (memory_region_is_ram_device(section->mr)) { 686 /* Allow unexpected mappings not to be fatal for RAM devices */ 687 VFIODevice *vbasedev = 688 vfio_get_vfio_device(memory_region_owner(section->mr)); 689 vfio_device_error_append(vbasedev, &err); 690 warn_report_err_once(err); 691 return; 692 } 693 goto fail; 694 } 695 696 return; 697 698 fail: 699 if (!bcontainer->initialized) { 700 /* 701 * At machine init time or when the device is attached to the 702 * VM, store the first error in the container so we can 703 * gracefully fail the device realize routine. 704 */ 705 if (!bcontainer->error) { 706 error_propagate_prepend(&bcontainer->error, err, 707 "Region %s: ", 708 memory_region_name(section->mr)); 709 } else { 710 error_free(err); 711 } 712 } else { 713 /* 714 * At runtime, there's not much we can do other than throw a 715 * hardware error. 716 */ 717 error_report_err(err); 718 hw_error("vfio: DMA mapping failed, unable to continue"); 719 } 720 } 721 722 static void vfio_listener_region_del(MemoryListener *listener, 723 MemoryRegionSection *section) 724 { 725 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 726 listener); 727 hwaddr iova, end; 728 Int128 llend, llsize; 729 int ret; 730 bool try_unmap = true; 731 732 if (!vfio_listener_valid_section(section, "region_del")) { 733 return; 734 } 735 736 if (memory_region_is_iommu(section->mr)) { 737 VFIOGuestIOMMU *giommu; 738 739 trace_vfio_listener_region_del_iommu(section->mr->name); 740 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 741 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 742 giommu->n.start == section->offset_within_region) { 743 memory_region_unregister_iommu_notifier(section->mr, 744 &giommu->n); 745 QLIST_REMOVE(giommu, giommu_next); 746 g_free(giommu); 747 break; 748 } 749 } 750 751 /* 752 * FIXME: We assume the one big unmap below is adequate to 753 * remove any individual page mappings in the IOMMU which 754 * might have been copied into VFIO. This works for a page table 755 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 756 * That may not be true for all IOMMU types. 757 */ 758 } 759 760 if (!vfio_get_section_iova_range(bcontainer, section, &iova, &end, 761 &llend)) { 762 return; 763 } 764 765 llsize = int128_sub(llend, int128_make64(iova)); 766 767 trace_vfio_listener_region_del(iova, end); 768 769 if (memory_region_is_ram_device(section->mr)) { 770 hwaddr pgmask; 771 772 pgmask = (1ULL << ctz64(bcontainer->pgsizes)) - 1; 773 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 774 } else if (memory_region_has_ram_discard_manager(section->mr)) { 775 vfio_unregister_ram_discard_listener(bcontainer, section); 776 /* Unregistering will trigger an unmap. */ 777 try_unmap = false; 778 } 779 780 if (try_unmap) { 781 if (int128_eq(llsize, int128_2_64())) { 782 /* The unmap ioctl doesn't accept a full 64-bit span. */ 783 llsize = int128_rshift(llsize, 1); 784 ret = vfio_container_dma_unmap(bcontainer, iova, 785 int128_get64(llsize), NULL); 786 if (ret) { 787 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 788 "0x%"HWADDR_PRIx") = %d (%s)", 789 bcontainer, iova, int128_get64(llsize), ret, 790 strerror(-ret)); 791 } 792 iova += int128_get64(llsize); 793 } 794 ret = vfio_container_dma_unmap(bcontainer, iova, 795 int128_get64(llsize), NULL); 796 if (ret) { 797 error_report("vfio_container_dma_unmap(%p, 0x%"HWADDR_PRIx", " 798 "0x%"HWADDR_PRIx") = %d (%s)", 799 bcontainer, iova, int128_get64(llsize), ret, 800 strerror(-ret)); 801 } 802 } 803 804 memory_region_unref(section->mr); 805 806 /* PPC64/pseries machine only */ 807 vfio_container_del_section_window(bcontainer, section); 808 } 809 810 typedef struct VFIODirtyRanges { 811 hwaddr min32; 812 hwaddr max32; 813 hwaddr min64; 814 hwaddr max64; 815 hwaddr minpci64; 816 hwaddr maxpci64; 817 } VFIODirtyRanges; 818 819 typedef struct VFIODirtyRangesListener { 820 VFIOContainerBase *bcontainer; 821 VFIODirtyRanges ranges; 822 MemoryListener listener; 823 } VFIODirtyRangesListener; 824 825 static bool vfio_section_is_vfio_pci(MemoryRegionSection *section, 826 VFIOContainerBase *bcontainer) 827 { 828 VFIOPCIDevice *pcidev; 829 VFIODevice *vbasedev; 830 Object *owner; 831 832 owner = memory_region_owner(section->mr); 833 834 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 835 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 836 continue; 837 } 838 pcidev = container_of(vbasedev, VFIOPCIDevice, vbasedev); 839 if (OBJECT(pcidev) == owner) { 840 return true; 841 } 842 } 843 844 return false; 845 } 846 847 static void vfio_dirty_tracking_update_range(VFIODirtyRanges *range, 848 hwaddr iova, hwaddr end, 849 bool update_pci) 850 { 851 hwaddr *min, *max; 852 853 /* 854 * The address space passed to the dirty tracker is reduced to three ranges: 855 * one for 32-bit DMA ranges, one for 64-bit DMA ranges and one for the 856 * PCI 64-bit hole. 857 * 858 * The underlying reports of dirty will query a sub-interval of each of 859 * these ranges. 860 * 861 * The purpose of the three range handling is to handle known cases of big 862 * holes in the address space, like the x86 AMD 1T hole, and firmware (like 863 * OVMF) which may relocate the pci-hole64 to the end of the address space. 864 * The latter would otherwise generate large ranges for tracking, stressing 865 * the limits of supported hardware. The pci-hole32 will always be below 4G 866 * (overlapping or not) so it doesn't need special handling and is part of 867 * the 32-bit range. 868 * 869 * The alternative would be an IOVATree but that has a much bigger runtime 870 * overhead and unnecessary complexity. 871 */ 872 if (update_pci && iova >= UINT32_MAX) { 873 min = &range->minpci64; 874 max = &range->maxpci64; 875 } else { 876 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 877 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 878 } 879 if (*min > iova) { 880 *min = iova; 881 } 882 if (*max < end) { 883 *max = end; 884 } 885 886 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 887 } 888 889 static void vfio_dirty_tracking_update(MemoryListener *listener, 890 MemoryRegionSection *section) 891 { 892 VFIODirtyRangesListener *dirty = 893 container_of(listener, VFIODirtyRangesListener, listener); 894 hwaddr iova, end; 895 896 if (!vfio_listener_valid_section(section, "tracking_update") || 897 !vfio_get_section_iova_range(dirty->bcontainer, section, 898 &iova, &end, NULL)) { 899 return; 900 } 901 902 vfio_dirty_tracking_update_range(&dirty->ranges, iova, end, 903 vfio_section_is_vfio_pci(section, dirty->bcontainer)); 904 } 905 906 static const MemoryListener vfio_dirty_tracking_listener = { 907 .name = "vfio-tracking", 908 .region_add = vfio_dirty_tracking_update, 909 }; 910 911 static void vfio_dirty_tracking_init(VFIOContainerBase *bcontainer, 912 VFIODirtyRanges *ranges) 913 { 914 VFIODirtyRangesListener dirty; 915 916 memset(&dirty, 0, sizeof(dirty)); 917 dirty.ranges.min32 = UINT32_MAX; 918 dirty.ranges.min64 = UINT64_MAX; 919 dirty.ranges.minpci64 = UINT64_MAX; 920 dirty.listener = vfio_dirty_tracking_listener; 921 dirty.bcontainer = bcontainer; 922 923 memory_listener_register(&dirty.listener, 924 bcontainer->space->as); 925 926 *ranges = dirty.ranges; 927 928 /* 929 * The memory listener is synchronous, and used to calculate the range 930 * to dirty tracking. Unregister it after we are done as we are not 931 * interested in any follow-up updates. 932 */ 933 memory_listener_unregister(&dirty.listener); 934 } 935 936 static void vfio_devices_dma_logging_stop(VFIOContainerBase *bcontainer) 937 { 938 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 939 sizeof(uint64_t))] = {}; 940 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 941 VFIODevice *vbasedev; 942 943 feature->argsz = sizeof(buf); 944 feature->flags = VFIO_DEVICE_FEATURE_SET | 945 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 946 947 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 948 if (!vbasedev->dirty_tracking) { 949 continue; 950 } 951 952 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 953 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 954 vbasedev->name, -errno, strerror(errno)); 955 } 956 vbasedev->dirty_tracking = false; 957 } 958 } 959 960 static struct vfio_device_feature * 961 vfio_device_feature_dma_logging_start_create(VFIOContainerBase *bcontainer, 962 VFIODirtyRanges *tracking) 963 { 964 struct vfio_device_feature *feature; 965 size_t feature_size; 966 struct vfio_device_feature_dma_logging_control *control; 967 struct vfio_device_feature_dma_logging_range *ranges; 968 969 feature_size = sizeof(struct vfio_device_feature) + 970 sizeof(struct vfio_device_feature_dma_logging_control); 971 feature = g_try_malloc0(feature_size); 972 if (!feature) { 973 errno = ENOMEM; 974 return NULL; 975 } 976 feature->argsz = feature_size; 977 feature->flags = VFIO_DEVICE_FEATURE_SET | 978 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 979 980 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 981 control->page_size = qemu_real_host_page_size(); 982 983 /* 984 * DMA logging uAPI guarantees to support at least a number of ranges that 985 * fits into a single host kernel base page. 986 */ 987 control->num_ranges = !!tracking->max32 + !!tracking->max64 + 988 !!tracking->maxpci64; 989 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 990 control->num_ranges); 991 if (!ranges) { 992 g_free(feature); 993 errno = ENOMEM; 994 995 return NULL; 996 } 997 998 control->ranges = (uintptr_t)ranges; 999 if (tracking->max32) { 1000 ranges->iova = tracking->min32; 1001 ranges->length = (tracking->max32 - tracking->min32) + 1; 1002 ranges++; 1003 } 1004 if (tracking->max64) { 1005 ranges->iova = tracking->min64; 1006 ranges->length = (tracking->max64 - tracking->min64) + 1; 1007 ranges++; 1008 } 1009 if (tracking->maxpci64) { 1010 ranges->iova = tracking->minpci64; 1011 ranges->length = (tracking->maxpci64 - tracking->minpci64) + 1; 1012 } 1013 1014 trace_vfio_device_dirty_tracking_start(control->num_ranges, 1015 tracking->min32, tracking->max32, 1016 tracking->min64, tracking->max64, 1017 tracking->minpci64, tracking->maxpci64); 1018 1019 return feature; 1020 } 1021 1022 static void vfio_device_feature_dma_logging_start_destroy( 1023 struct vfio_device_feature *feature) 1024 { 1025 struct vfio_device_feature_dma_logging_control *control = 1026 (struct vfio_device_feature_dma_logging_control *)feature->data; 1027 struct vfio_device_feature_dma_logging_range *ranges = 1028 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 1029 1030 g_free(ranges); 1031 g_free(feature); 1032 } 1033 1034 static bool vfio_devices_dma_logging_start(VFIOContainerBase *bcontainer, 1035 Error **errp) 1036 { 1037 struct vfio_device_feature *feature; 1038 VFIODirtyRanges ranges; 1039 VFIODevice *vbasedev; 1040 int ret = 0; 1041 1042 vfio_dirty_tracking_init(bcontainer, &ranges); 1043 feature = vfio_device_feature_dma_logging_start_create(bcontainer, 1044 &ranges); 1045 if (!feature) { 1046 error_setg_errno(errp, errno, "Failed to prepare DMA logging"); 1047 return false; 1048 } 1049 1050 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1051 if (vbasedev->dirty_tracking) { 1052 continue; 1053 } 1054 1055 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 1056 if (ret) { 1057 ret = -errno; 1058 error_setg_errno(errp, errno, "%s: Failed to start DMA logging", 1059 vbasedev->name); 1060 goto out; 1061 } 1062 vbasedev->dirty_tracking = true; 1063 } 1064 1065 out: 1066 if (ret) { 1067 vfio_devices_dma_logging_stop(bcontainer); 1068 } 1069 1070 vfio_device_feature_dma_logging_start_destroy(feature); 1071 1072 return ret == 0; 1073 } 1074 1075 static bool vfio_listener_log_global_start(MemoryListener *listener, 1076 Error **errp) 1077 { 1078 ERRP_GUARD(); 1079 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1080 listener); 1081 bool ret; 1082 1083 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1084 ret = vfio_devices_dma_logging_start(bcontainer, errp); 1085 } else { 1086 ret = vfio_container_set_dirty_page_tracking(bcontainer, true, errp) == 0; 1087 } 1088 1089 if (!ret) { 1090 error_prepend(errp, "vfio: Could not start dirty page tracking - "); 1091 } 1092 return ret; 1093 } 1094 1095 static void vfio_listener_log_global_stop(MemoryListener *listener) 1096 { 1097 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1098 listener); 1099 Error *local_err = NULL; 1100 int ret = 0; 1101 1102 if (vfio_devices_all_device_dirty_tracking(bcontainer)) { 1103 vfio_devices_dma_logging_stop(bcontainer); 1104 } else { 1105 ret = vfio_container_set_dirty_page_tracking(bcontainer, false, 1106 &local_err); 1107 } 1108 1109 if (ret) { 1110 error_prepend(&local_err, 1111 "vfio: Could not stop dirty page tracking - "); 1112 error_report_err(local_err); 1113 vfio_set_migration_error(ret); 1114 } 1115 } 1116 1117 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, 1118 hwaddr size, void *bitmap) 1119 { 1120 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 1121 sizeof(struct vfio_device_feature_dma_logging_report), 1122 sizeof(uint64_t))] = {}; 1123 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1124 struct vfio_device_feature_dma_logging_report *report = 1125 (struct vfio_device_feature_dma_logging_report *)feature->data; 1126 1127 report->iova = iova; 1128 report->length = size; 1129 report->page_size = qemu_real_host_page_size(); 1130 report->bitmap = (uintptr_t)bitmap; 1131 1132 feature->argsz = sizeof(buf); 1133 feature->flags = VFIO_DEVICE_FEATURE_GET | 1134 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; 1135 1136 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1137 return -errno; 1138 } 1139 1140 return 0; 1141 } 1142 1143 int vfio_devices_query_dirty_bitmap(const VFIOContainerBase *bcontainer, 1144 VFIOBitmap *vbmap, hwaddr iova, hwaddr size, Error **errp) 1145 { 1146 VFIODevice *vbasedev; 1147 int ret; 1148 1149 QLIST_FOREACH(vbasedev, &bcontainer->device_list, container_next) { 1150 ret = vfio_device_dma_logging_report(vbasedev, iova, size, 1151 vbmap->bitmap); 1152 if (ret) { 1153 error_setg_errno(errp, -ret, 1154 "%s: Failed to get DMA logging report, iova: " 1155 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx, 1156 vbasedev->name, iova, size); 1157 1158 return ret; 1159 } 1160 } 1161 1162 return 0; 1163 } 1164 1165 int vfio_get_dirty_bitmap(const VFIOContainerBase *bcontainer, uint64_t iova, 1166 uint64_t size, ram_addr_t ram_addr, Error **errp) 1167 { 1168 bool all_device_dirty_tracking = 1169 vfio_devices_all_device_dirty_tracking(bcontainer); 1170 uint64_t dirty_pages; 1171 VFIOBitmap vbmap; 1172 int ret; 1173 1174 if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { 1175 cpu_physical_memory_set_dirty_range(ram_addr, size, 1176 tcg_enabled() ? DIRTY_CLIENTS_ALL : 1177 DIRTY_CLIENTS_NOCODE); 1178 return 0; 1179 } 1180 1181 ret = vfio_bitmap_alloc(&vbmap, size); 1182 if (ret) { 1183 error_setg_errno(errp, -ret, 1184 "Failed to allocate dirty tracking bitmap"); 1185 return ret; 1186 } 1187 1188 if (all_device_dirty_tracking) { 1189 ret = vfio_devices_query_dirty_bitmap(bcontainer, &vbmap, iova, size, 1190 errp); 1191 } else { 1192 ret = vfio_container_query_dirty_bitmap(bcontainer, &vbmap, iova, size, 1193 errp); 1194 } 1195 1196 if (ret) { 1197 goto out; 1198 } 1199 1200 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, 1201 vbmap.pages); 1202 1203 trace_vfio_get_dirty_bitmap(iova, size, vbmap.size, ram_addr, dirty_pages); 1204 out: 1205 g_free(vbmap.bitmap); 1206 1207 return ret; 1208 } 1209 1210 typedef struct { 1211 IOMMUNotifier n; 1212 VFIOGuestIOMMU *giommu; 1213 } vfio_giommu_dirty_notifier; 1214 1215 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1216 { 1217 vfio_giommu_dirty_notifier *gdn = container_of(n, 1218 vfio_giommu_dirty_notifier, n); 1219 VFIOGuestIOMMU *giommu = gdn->giommu; 1220 VFIOContainerBase *bcontainer = giommu->bcontainer; 1221 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1222 ram_addr_t translated_addr; 1223 Error *local_err = NULL; 1224 int ret = -EINVAL; 1225 1226 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1227 1228 if (iotlb->target_as != &address_space_memory) { 1229 error_report("Wrong target AS \"%s\", only system memory is allowed", 1230 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1231 goto out; 1232 } 1233 1234 rcu_read_lock(); 1235 if (!vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL, &local_err)) { 1236 error_report_err(local_err); 1237 goto out_unlock; 1238 } 1239 1240 ret = vfio_get_dirty_bitmap(bcontainer, iova, iotlb->addr_mask + 1, 1241 translated_addr, &local_err); 1242 if (ret) { 1243 error_prepend(&local_err, 1244 "vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1245 "0x%"HWADDR_PRIx") failed - ", bcontainer, iova, 1246 iotlb->addr_mask + 1); 1247 error_report_err(local_err); 1248 } 1249 1250 out_unlock: 1251 rcu_read_unlock(); 1252 1253 out: 1254 if (ret) { 1255 vfio_set_migration_error(ret); 1256 } 1257 } 1258 1259 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1260 void *opaque) 1261 { 1262 const hwaddr size = int128_get64(section->size); 1263 const hwaddr iova = section->offset_within_address_space; 1264 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1265 section->offset_within_region; 1266 VFIORamDiscardListener *vrdl = opaque; 1267 Error *local_err = NULL; 1268 int ret; 1269 1270 /* 1271 * Sync the whole mapped region (spanning multiple individual mappings) 1272 * in one go. 1273 */ 1274 ret = vfio_get_dirty_bitmap(vrdl->bcontainer, iova, size, ram_addr, 1275 &local_err); 1276 if (ret) { 1277 error_report_err(local_err); 1278 } 1279 return ret; 1280 } 1281 1282 static int 1283 vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainerBase *bcontainer, 1284 MemoryRegionSection *section) 1285 { 1286 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1287 VFIORamDiscardListener *vrdl = NULL; 1288 1289 QLIST_FOREACH(vrdl, &bcontainer->vrdl_list, next) { 1290 if (vrdl->mr == section->mr && 1291 vrdl->offset_within_address_space == 1292 section->offset_within_address_space) { 1293 break; 1294 } 1295 } 1296 1297 if (!vrdl) { 1298 hw_error("vfio: Trying to sync missing RAM discard listener"); 1299 } 1300 1301 /* 1302 * We only want/can synchronize the bitmap for actually mapped parts - 1303 * which correspond to populated parts. Replay all populated parts. 1304 */ 1305 return ram_discard_manager_replay_populated(rdm, section, 1306 vfio_ram_discard_get_dirty_bitmap, 1307 &vrdl); 1308 } 1309 1310 static int vfio_sync_iommu_dirty_bitmap(VFIOContainerBase *bcontainer, 1311 MemoryRegionSection *section) 1312 { 1313 VFIOGuestIOMMU *giommu; 1314 bool found = false; 1315 Int128 llend; 1316 vfio_giommu_dirty_notifier gdn; 1317 int idx; 1318 1319 QLIST_FOREACH(giommu, &bcontainer->giommu_list, giommu_next) { 1320 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1321 giommu->n.start == section->offset_within_region) { 1322 found = true; 1323 break; 1324 } 1325 } 1326 1327 if (!found) { 1328 return 0; 1329 } 1330 1331 gdn.giommu = giommu; 1332 idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1333 MEMTXATTRS_UNSPECIFIED); 1334 1335 llend = int128_add(int128_make64(section->offset_within_region), 1336 section->size); 1337 llend = int128_sub(llend, int128_one()); 1338 1339 iommu_notifier_init(&gdn.n, vfio_iommu_map_dirty_notify, IOMMU_NOTIFIER_MAP, 1340 section->offset_within_region, int128_get64(llend), 1341 idx); 1342 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1343 1344 return 0; 1345 } 1346 1347 static int vfio_sync_dirty_bitmap(VFIOContainerBase *bcontainer, 1348 MemoryRegionSection *section, Error **errp) 1349 { 1350 ram_addr_t ram_addr; 1351 1352 if (memory_region_is_iommu(section->mr)) { 1353 return vfio_sync_iommu_dirty_bitmap(bcontainer, section); 1354 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1355 int ret; 1356 1357 ret = vfio_sync_ram_discard_listener_dirty_bitmap(bcontainer, section); 1358 if (ret) { 1359 error_setg(errp, 1360 "Failed to sync dirty bitmap with RAM discard listener"); 1361 } 1362 return ret; 1363 } 1364 1365 ram_addr = memory_region_get_ram_addr(section->mr) + 1366 section->offset_within_region; 1367 1368 return vfio_get_dirty_bitmap(bcontainer, 1369 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1370 int128_get64(section->size), ram_addr, errp); 1371 } 1372 1373 static void vfio_listener_log_sync(MemoryListener *listener, 1374 MemoryRegionSection *section) 1375 { 1376 VFIOContainerBase *bcontainer = container_of(listener, VFIOContainerBase, 1377 listener); 1378 int ret; 1379 Error *local_err = NULL; 1380 1381 if (vfio_listener_skipped_section(section)) { 1382 return; 1383 } 1384 1385 if (vfio_log_sync_needed(bcontainer)) { 1386 ret = vfio_sync_dirty_bitmap(bcontainer, section, &local_err); 1387 if (ret) { 1388 error_report_err(local_err); 1389 vfio_set_migration_error(ret); 1390 } 1391 } 1392 } 1393 1394 const MemoryListener vfio_memory_listener = { 1395 .name = "vfio", 1396 .region_add = vfio_listener_region_add, 1397 .region_del = vfio_listener_region_del, 1398 .log_global_start = vfio_listener_log_global_start, 1399 .log_global_stop = vfio_listener_log_global_stop, 1400 .log_sync = vfio_listener_log_sync, 1401 }; 1402 1403 void vfio_reset_handler(void *opaque) 1404 { 1405 VFIODevice *vbasedev; 1406 1407 trace_vfio_reset_handler(); 1408 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 1409 if (vbasedev->dev->realized) { 1410 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 1411 } 1412 } 1413 1414 QLIST_FOREACH(vbasedev, &vfio_device_list, global_next) { 1415 if (vbasedev->dev->realized && vbasedev->needs_reset) { 1416 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 1417 } 1418 } 1419 } 1420 1421 int vfio_kvm_device_add_fd(int fd, Error **errp) 1422 { 1423 #ifdef CONFIG_KVM 1424 struct kvm_device_attr attr = { 1425 .group = KVM_DEV_VFIO_FILE, 1426 .attr = KVM_DEV_VFIO_FILE_ADD, 1427 .addr = (uint64_t)(unsigned long)&fd, 1428 }; 1429 1430 if (!kvm_enabled()) { 1431 return 0; 1432 } 1433 1434 if (vfio_kvm_device_fd < 0) { 1435 struct kvm_create_device cd = { 1436 .type = KVM_DEV_TYPE_VFIO, 1437 }; 1438 1439 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 1440 error_setg_errno(errp, errno, "Failed to create KVM VFIO device"); 1441 return -errno; 1442 } 1443 1444 vfio_kvm_device_fd = cd.fd; 1445 } 1446 1447 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1448 error_setg_errno(errp, errno, "Failed to add fd %d to KVM VFIO device", 1449 fd); 1450 return -errno; 1451 } 1452 #endif 1453 return 0; 1454 } 1455 1456 int vfio_kvm_device_del_fd(int fd, Error **errp) 1457 { 1458 #ifdef CONFIG_KVM 1459 struct kvm_device_attr attr = { 1460 .group = KVM_DEV_VFIO_FILE, 1461 .attr = KVM_DEV_VFIO_FILE_DEL, 1462 .addr = (uint64_t)(unsigned long)&fd, 1463 }; 1464 1465 if (vfio_kvm_device_fd < 0) { 1466 error_setg(errp, "KVM VFIO device isn't created yet"); 1467 return -EINVAL; 1468 } 1469 1470 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1471 error_setg_errno(errp, errno, 1472 "Failed to remove fd %d from KVM VFIO device", fd); 1473 return -errno; 1474 } 1475 #endif 1476 return 0; 1477 } 1478 1479 VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1480 { 1481 VFIOAddressSpace *space; 1482 1483 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1484 if (space->as == as) { 1485 return space; 1486 } 1487 } 1488 1489 /* No suitable VFIOAddressSpace, create a new one */ 1490 space = g_malloc0(sizeof(*space)); 1491 space->as = as; 1492 QLIST_INIT(&space->containers); 1493 1494 if (QLIST_EMPTY(&vfio_address_spaces)) { 1495 qemu_register_reset(vfio_reset_handler, NULL); 1496 } 1497 1498 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1499 1500 return space; 1501 } 1502 1503 void vfio_put_address_space(VFIOAddressSpace *space) 1504 { 1505 if (!QLIST_EMPTY(&space->containers)) { 1506 return; 1507 } 1508 1509 QLIST_REMOVE(space, list); 1510 g_free(space); 1511 1512 if (QLIST_EMPTY(&vfio_address_spaces)) { 1513 qemu_unregister_reset(vfio_reset_handler, NULL); 1514 } 1515 } 1516 1517 void vfio_address_space_insert(VFIOAddressSpace *space, 1518 VFIOContainerBase *bcontainer) 1519 { 1520 QLIST_INSERT_HEAD(&space->containers, bcontainer, next); 1521 bcontainer->space = space; 1522 } 1523 1524 struct vfio_device_info *vfio_get_device_info(int fd) 1525 { 1526 struct vfio_device_info *info; 1527 uint32_t argsz = sizeof(*info); 1528 1529 info = g_malloc0(argsz); 1530 1531 retry: 1532 info->argsz = argsz; 1533 1534 if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) { 1535 g_free(info); 1536 return NULL; 1537 } 1538 1539 if (info->argsz > argsz) { 1540 argsz = info->argsz; 1541 info = g_realloc(info, argsz); 1542 goto retry; 1543 } 1544 1545 return info; 1546 } 1547 1548 bool vfio_attach_device(char *name, VFIODevice *vbasedev, 1549 AddressSpace *as, Error **errp) 1550 { 1551 const VFIOIOMMUClass *ops = 1552 VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_LEGACY)); 1553 HostIOMMUDevice *hiod = NULL; 1554 1555 if (vbasedev->iommufd) { 1556 ops = VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD)); 1557 } 1558 1559 assert(ops); 1560 1561 1562 if (!vbasedev->mdev) { 1563 hiod = HOST_IOMMU_DEVICE(object_new(ops->hiod_typename)); 1564 vbasedev->hiod = hiod; 1565 } 1566 1567 if (!ops->attach_device(name, vbasedev, as, errp)) { 1568 object_unref(hiod); 1569 vbasedev->hiod = NULL; 1570 return false; 1571 } 1572 1573 return true; 1574 } 1575 1576 void vfio_detach_device(VFIODevice *vbasedev) 1577 { 1578 if (!vbasedev->bcontainer) { 1579 return; 1580 } 1581 object_unref(vbasedev->hiod); 1582 VFIO_IOMMU_GET_CLASS(vbasedev->bcontainer)->detach_device(vbasedev); 1583 } 1584