1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/vfio.h" 30 #include "exec/address-spaces.h" 31 #include "exec/memory.h" 32 #include "exec/ram_addr.h" 33 #include "hw/hw.h" 34 #include "qemu/error-report.h" 35 #include "qemu/main-loop.h" 36 #include "qemu/range.h" 37 #include "sysemu/kvm.h" 38 #include "sysemu/reset.h" 39 #include "sysemu/runstate.h" 40 #include "trace.h" 41 #include "qapi/error.h" 42 #include "migration/migration.h" 43 #include "migration/misc.h" 44 #include "migration/blocker.h" 45 #include "migration/qemu-file.h" 46 #include "sysemu/tpm.h" 47 48 VFIOGroupList vfio_group_list = 49 QLIST_HEAD_INITIALIZER(vfio_group_list); 50 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 51 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 52 53 #ifdef CONFIG_KVM 54 /* 55 * We have a single VFIO pseudo device per KVM VM. Once created it lives 56 * for the life of the VM. Closing the file descriptor only drops our 57 * reference to it and the device's reference to kvm. Therefore once 58 * initialized, this file descriptor is only released on QEMU exit and 59 * we'll re-use it should another vfio device be attached before then. 60 */ 61 static int vfio_kvm_device_fd = -1; 62 #endif 63 64 /* 65 * Common VFIO interrupt disable 66 */ 67 void vfio_disable_irqindex(VFIODevice *vbasedev, int index) 68 { 69 struct vfio_irq_set irq_set = { 70 .argsz = sizeof(irq_set), 71 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 72 .index = index, 73 .start = 0, 74 .count = 0, 75 }; 76 77 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 78 } 79 80 void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index) 81 { 82 struct vfio_irq_set irq_set = { 83 .argsz = sizeof(irq_set), 84 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, 85 .index = index, 86 .start = 0, 87 .count = 1, 88 }; 89 90 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 91 } 92 93 void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index) 94 { 95 struct vfio_irq_set irq_set = { 96 .argsz = sizeof(irq_set), 97 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK, 98 .index = index, 99 .start = 0, 100 .count = 1, 101 }; 102 103 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 104 } 105 106 static inline const char *action_to_str(int action) 107 { 108 switch (action) { 109 case VFIO_IRQ_SET_ACTION_MASK: 110 return "MASK"; 111 case VFIO_IRQ_SET_ACTION_UNMASK: 112 return "UNMASK"; 113 case VFIO_IRQ_SET_ACTION_TRIGGER: 114 return "TRIGGER"; 115 default: 116 return "UNKNOWN ACTION"; 117 } 118 } 119 120 static const char *index_to_str(VFIODevice *vbasedev, int index) 121 { 122 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 123 return NULL; 124 } 125 126 switch (index) { 127 case VFIO_PCI_INTX_IRQ_INDEX: 128 return "INTX"; 129 case VFIO_PCI_MSI_IRQ_INDEX: 130 return "MSI"; 131 case VFIO_PCI_MSIX_IRQ_INDEX: 132 return "MSIX"; 133 case VFIO_PCI_ERR_IRQ_INDEX: 134 return "ERR"; 135 case VFIO_PCI_REQ_IRQ_INDEX: 136 return "REQ"; 137 default: 138 return NULL; 139 } 140 } 141 142 static int vfio_ram_block_discard_disable(VFIOContainer *container, bool state) 143 { 144 switch (container->iommu_type) { 145 case VFIO_TYPE1v2_IOMMU: 146 case VFIO_TYPE1_IOMMU: 147 /* 148 * We support coordinated discarding of RAM via the RamDiscardManager. 149 */ 150 return ram_block_uncoordinated_discard_disable(state); 151 default: 152 /* 153 * VFIO_SPAPR_TCE_IOMMU most probably works just fine with 154 * RamDiscardManager, however, it is completely untested. 155 * 156 * VFIO_SPAPR_TCE_v2_IOMMU with "DMA memory preregistering" does 157 * completely the opposite of managing mapping/pinning dynamically as 158 * required by RamDiscardManager. We would have to special-case sections 159 * with a RamDiscardManager. 160 */ 161 return ram_block_discard_disable(state); 162 } 163 } 164 165 int vfio_set_irq_signaling(VFIODevice *vbasedev, int index, int subindex, 166 int action, int fd, Error **errp) 167 { 168 struct vfio_irq_set *irq_set; 169 int argsz, ret = 0; 170 const char *name; 171 int32_t *pfd; 172 173 argsz = sizeof(*irq_set) + sizeof(*pfd); 174 175 irq_set = g_malloc0(argsz); 176 irq_set->argsz = argsz; 177 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | action; 178 irq_set->index = index; 179 irq_set->start = subindex; 180 irq_set->count = 1; 181 pfd = (int32_t *)&irq_set->data; 182 *pfd = fd; 183 184 if (ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set)) { 185 ret = -errno; 186 } 187 g_free(irq_set); 188 189 if (!ret) { 190 return 0; 191 } 192 193 error_setg_errno(errp, -ret, "VFIO_DEVICE_SET_IRQS failure"); 194 195 name = index_to_str(vbasedev, index); 196 if (name) { 197 error_prepend(errp, "%s-%d: ", name, subindex); 198 } else { 199 error_prepend(errp, "index %d-%d: ", index, subindex); 200 } 201 error_prepend(errp, 202 "Failed to %s %s eventfd signaling for interrupt ", 203 fd < 0 ? "tear down" : "set up", action_to_str(action)); 204 return ret; 205 } 206 207 /* 208 * IO Port/MMIO - Beware of the endians, VFIO is always little endian 209 */ 210 void vfio_region_write(void *opaque, hwaddr addr, 211 uint64_t data, unsigned size) 212 { 213 VFIORegion *region = opaque; 214 VFIODevice *vbasedev = region->vbasedev; 215 union { 216 uint8_t byte; 217 uint16_t word; 218 uint32_t dword; 219 uint64_t qword; 220 } buf; 221 222 switch (size) { 223 case 1: 224 buf.byte = data; 225 break; 226 case 2: 227 buf.word = cpu_to_le16(data); 228 break; 229 case 4: 230 buf.dword = cpu_to_le32(data); 231 break; 232 case 8: 233 buf.qword = cpu_to_le64(data); 234 break; 235 default: 236 hw_error("vfio: unsupported write size, %u bytes", size); 237 break; 238 } 239 240 if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { 241 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64 242 ",%d) failed: %m", 243 __func__, vbasedev->name, region->nr, 244 addr, data, size); 245 } 246 247 trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size); 248 249 /* 250 * A read or write to a BAR always signals an INTx EOI. This will 251 * do nothing if not pending (including not in INTx mode). We assume 252 * that a BAR access is in response to an interrupt and that BAR 253 * accesses will service the interrupt. Unfortunately, we don't know 254 * which access will service the interrupt, so we're potentially 255 * getting quite a few host interrupts per guest interrupt. 256 */ 257 vbasedev->ops->vfio_eoi(vbasedev); 258 } 259 260 uint64_t vfio_region_read(void *opaque, 261 hwaddr addr, unsigned size) 262 { 263 VFIORegion *region = opaque; 264 VFIODevice *vbasedev = region->vbasedev; 265 union { 266 uint8_t byte; 267 uint16_t word; 268 uint32_t dword; 269 uint64_t qword; 270 } buf; 271 uint64_t data = 0; 272 273 if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { 274 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m", 275 __func__, vbasedev->name, region->nr, 276 addr, size); 277 return (uint64_t)-1; 278 } 279 switch (size) { 280 case 1: 281 data = buf.byte; 282 break; 283 case 2: 284 data = le16_to_cpu(buf.word); 285 break; 286 case 4: 287 data = le32_to_cpu(buf.dword); 288 break; 289 case 8: 290 data = le64_to_cpu(buf.qword); 291 break; 292 default: 293 hw_error("vfio: unsupported read size, %u bytes", size); 294 break; 295 } 296 297 trace_vfio_region_read(vbasedev->name, region->nr, addr, size, data); 298 299 /* Same as write above */ 300 vbasedev->ops->vfio_eoi(vbasedev); 301 302 return data; 303 } 304 305 const MemoryRegionOps vfio_region_ops = { 306 .read = vfio_region_read, 307 .write = vfio_region_write, 308 .endianness = DEVICE_LITTLE_ENDIAN, 309 .valid = { 310 .min_access_size = 1, 311 .max_access_size = 8, 312 }, 313 .impl = { 314 .min_access_size = 1, 315 .max_access_size = 8, 316 }, 317 }; 318 319 /* 320 * Device state interfaces 321 */ 322 323 typedef struct { 324 unsigned long *bitmap; 325 hwaddr size; 326 hwaddr pages; 327 } VFIOBitmap; 328 329 static int vfio_bitmap_alloc(VFIOBitmap *vbmap, hwaddr size) 330 { 331 vbmap->pages = REAL_HOST_PAGE_ALIGN(size) / qemu_real_host_page_size(); 332 vbmap->size = ROUND_UP(vbmap->pages, sizeof(__u64) * BITS_PER_BYTE) / 333 BITS_PER_BYTE; 334 vbmap->bitmap = g_try_malloc0(vbmap->size); 335 if (!vbmap->bitmap) { 336 return -ENOMEM; 337 } 338 339 return 0; 340 } 341 342 static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova, 343 uint64_t size, ram_addr_t ram_addr); 344 345 bool vfio_mig_active(void) 346 { 347 VFIOGroup *group; 348 VFIODevice *vbasedev; 349 350 if (QLIST_EMPTY(&vfio_group_list)) { 351 return false; 352 } 353 354 QLIST_FOREACH(group, &vfio_group_list, next) { 355 QLIST_FOREACH(vbasedev, &group->device_list, next) { 356 if (vbasedev->migration_blocker) { 357 return false; 358 } 359 } 360 } 361 return true; 362 } 363 364 static Error *multiple_devices_migration_blocker; 365 static Error *giommu_migration_blocker; 366 367 static unsigned int vfio_migratable_device_num(void) 368 { 369 VFIOGroup *group; 370 VFIODevice *vbasedev; 371 unsigned int device_num = 0; 372 373 QLIST_FOREACH(group, &vfio_group_list, next) { 374 QLIST_FOREACH(vbasedev, &group->device_list, next) { 375 if (vbasedev->migration) { 376 device_num++; 377 } 378 } 379 } 380 381 return device_num; 382 } 383 384 int vfio_block_multiple_devices_migration(Error **errp) 385 { 386 int ret; 387 388 if (multiple_devices_migration_blocker || 389 vfio_migratable_device_num() <= 1) { 390 return 0; 391 } 392 393 error_setg(&multiple_devices_migration_blocker, 394 "Migration is currently not supported with multiple " 395 "VFIO devices"); 396 ret = migrate_add_blocker(multiple_devices_migration_blocker, errp); 397 if (ret < 0) { 398 error_free(multiple_devices_migration_blocker); 399 multiple_devices_migration_blocker = NULL; 400 } 401 402 return ret; 403 } 404 405 void vfio_unblock_multiple_devices_migration(void) 406 { 407 if (!multiple_devices_migration_blocker || 408 vfio_migratable_device_num() > 1) { 409 return; 410 } 411 412 migrate_del_blocker(multiple_devices_migration_blocker); 413 error_free(multiple_devices_migration_blocker); 414 multiple_devices_migration_blocker = NULL; 415 } 416 417 static bool vfio_viommu_preset(void) 418 { 419 VFIOAddressSpace *space; 420 421 QLIST_FOREACH(space, &vfio_address_spaces, list) { 422 if (space->as != &address_space_memory) { 423 return true; 424 } 425 } 426 427 return false; 428 } 429 430 int vfio_block_giommu_migration(Error **errp) 431 { 432 int ret; 433 434 if (giommu_migration_blocker || 435 !vfio_viommu_preset()) { 436 return 0; 437 } 438 439 error_setg(&giommu_migration_blocker, 440 "Migration is currently not supported with vIOMMU enabled"); 441 ret = migrate_add_blocker(giommu_migration_blocker, errp); 442 if (ret < 0) { 443 error_free(giommu_migration_blocker); 444 giommu_migration_blocker = NULL; 445 } 446 447 return ret; 448 } 449 450 void vfio_migration_finalize(void) 451 { 452 if (!giommu_migration_blocker || 453 vfio_viommu_preset()) { 454 return; 455 } 456 457 migrate_del_blocker(giommu_migration_blocker); 458 error_free(giommu_migration_blocker); 459 giommu_migration_blocker = NULL; 460 } 461 462 static void vfio_set_migration_error(int err) 463 { 464 MigrationState *ms = migrate_get_current(); 465 466 if (migration_is_setup_or_active(ms->state)) { 467 WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { 468 if (ms->to_dst_file) { 469 qemu_file_set_error(ms->to_dst_file, err); 470 } 471 } 472 } 473 } 474 475 static bool vfio_devices_all_dirty_tracking(VFIOContainer *container) 476 { 477 VFIOGroup *group; 478 VFIODevice *vbasedev; 479 MigrationState *ms = migrate_get_current(); 480 481 if (ms->state != MIGRATION_STATUS_ACTIVE && 482 ms->state != MIGRATION_STATUS_DEVICE) { 483 return false; 484 } 485 486 QLIST_FOREACH(group, &container->group_list, container_next) { 487 QLIST_FOREACH(vbasedev, &group->device_list, next) { 488 VFIOMigration *migration = vbasedev->migration; 489 490 if (!migration) { 491 return false; 492 } 493 494 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF && 495 migration->device_state == VFIO_DEVICE_STATE_RUNNING) { 496 return false; 497 } 498 } 499 } 500 return true; 501 } 502 503 static bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container) 504 { 505 VFIOGroup *group; 506 VFIODevice *vbasedev; 507 508 QLIST_FOREACH(group, &container->group_list, container_next) { 509 QLIST_FOREACH(vbasedev, &group->device_list, next) { 510 if (!vbasedev->dirty_pages_supported) { 511 return false; 512 } 513 } 514 } 515 516 return true; 517 } 518 519 /* 520 * Check if all VFIO devices are running and migration is active, which is 521 * essentially equivalent to the migration being in pre-copy phase. 522 */ 523 static bool vfio_devices_all_running_and_mig_active(VFIOContainer *container) 524 { 525 VFIOGroup *group; 526 VFIODevice *vbasedev; 527 528 if (!migration_is_active(migrate_get_current())) { 529 return false; 530 } 531 532 QLIST_FOREACH(group, &container->group_list, container_next) { 533 QLIST_FOREACH(vbasedev, &group->device_list, next) { 534 VFIOMigration *migration = vbasedev->migration; 535 536 if (!migration) { 537 return false; 538 } 539 540 if (migration->device_state == VFIO_DEVICE_STATE_RUNNING) { 541 continue; 542 } else { 543 return false; 544 } 545 } 546 } 547 return true; 548 } 549 550 static int vfio_dma_unmap_bitmap(VFIOContainer *container, 551 hwaddr iova, ram_addr_t size, 552 IOMMUTLBEntry *iotlb) 553 { 554 struct vfio_iommu_type1_dma_unmap *unmap; 555 struct vfio_bitmap *bitmap; 556 VFIOBitmap vbmap; 557 int ret; 558 559 ret = vfio_bitmap_alloc(&vbmap, size); 560 if (ret) { 561 return ret; 562 } 563 564 unmap = g_malloc0(sizeof(*unmap) + sizeof(*bitmap)); 565 566 unmap->argsz = sizeof(*unmap) + sizeof(*bitmap); 567 unmap->iova = iova; 568 unmap->size = size; 569 unmap->flags |= VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP; 570 bitmap = (struct vfio_bitmap *)&unmap->data; 571 572 /* 573 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of 574 * qemu_real_host_page_size to mark those dirty. Hence set bitmap_pgsize 575 * to qemu_real_host_page_size. 576 */ 577 bitmap->pgsize = qemu_real_host_page_size(); 578 bitmap->size = vbmap.size; 579 bitmap->data = (__u64 *)vbmap.bitmap; 580 581 if (vbmap.size > container->max_dirty_bitmap_size) { 582 error_report("UNMAP: Size of bitmap too big 0x%"PRIx64, vbmap.size); 583 ret = -E2BIG; 584 goto unmap_exit; 585 } 586 587 ret = ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, unmap); 588 if (!ret) { 589 cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, 590 iotlb->translated_addr, vbmap.pages); 591 } else { 592 error_report("VFIO_UNMAP_DMA with DIRTY_BITMAP : %m"); 593 } 594 595 unmap_exit: 596 g_free(unmap); 597 g_free(vbmap.bitmap); 598 599 return ret; 600 } 601 602 /* 603 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 604 */ 605 static int vfio_dma_unmap(VFIOContainer *container, 606 hwaddr iova, ram_addr_t size, 607 IOMMUTLBEntry *iotlb) 608 { 609 struct vfio_iommu_type1_dma_unmap unmap = { 610 .argsz = sizeof(unmap), 611 .flags = 0, 612 .iova = iova, 613 .size = size, 614 }; 615 bool need_dirty_sync = false; 616 int ret; 617 618 if (iotlb && vfio_devices_all_running_and_mig_active(container)) { 619 if (!vfio_devices_all_device_dirty_tracking(container) && 620 container->dirty_pages_supported) { 621 return vfio_dma_unmap_bitmap(container, iova, size, iotlb); 622 } 623 624 need_dirty_sync = true; 625 } 626 627 while (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) { 628 /* 629 * The type1 backend has an off-by-one bug in the kernel (71a7d3d78e3c 630 * v4.15) where an overflow in its wrap-around check prevents us from 631 * unmapping the last page of the address space. Test for the error 632 * condition and re-try the unmap excluding the last page. The 633 * expectation is that we've never mapped the last page anyway and this 634 * unmap request comes via vIOMMU support which also makes it unlikely 635 * that this page is used. This bug was introduced well after type1 v2 636 * support was introduced, so we shouldn't need to test for v1. A fix 637 * is queued for kernel v5.0 so this workaround can be removed once 638 * affected kernels are sufficiently deprecated. 639 */ 640 if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) && 641 container->iommu_type == VFIO_TYPE1v2_IOMMU) { 642 trace_vfio_dma_unmap_overflow_workaround(); 643 unmap.size -= 1ULL << ctz64(container->pgsizes); 644 continue; 645 } 646 error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno)); 647 return -errno; 648 } 649 650 if (need_dirty_sync) { 651 ret = vfio_get_dirty_bitmap(container, iova, size, 652 iotlb->translated_addr); 653 if (ret) { 654 return ret; 655 } 656 } 657 658 return 0; 659 } 660 661 static int vfio_dma_map(VFIOContainer *container, hwaddr iova, 662 ram_addr_t size, void *vaddr, bool readonly) 663 { 664 struct vfio_iommu_type1_dma_map map = { 665 .argsz = sizeof(map), 666 .flags = VFIO_DMA_MAP_FLAG_READ, 667 .vaddr = (__u64)(uintptr_t)vaddr, 668 .iova = iova, 669 .size = size, 670 }; 671 672 if (!readonly) { 673 map.flags |= VFIO_DMA_MAP_FLAG_WRITE; 674 } 675 676 /* 677 * Try the mapping, if it fails with EBUSY, unmap the region and try 678 * again. This shouldn't be necessary, but we sometimes see it in 679 * the VGA ROM space. 680 */ 681 if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 || 682 (errno == EBUSY && vfio_dma_unmap(container, iova, size, NULL) == 0 && 683 ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) { 684 return 0; 685 } 686 687 error_report("VFIO_MAP_DMA failed: %s", strerror(errno)); 688 return -errno; 689 } 690 691 static void vfio_host_win_add(VFIOContainer *container, 692 hwaddr min_iova, hwaddr max_iova, 693 uint64_t iova_pgsizes) 694 { 695 VFIOHostDMAWindow *hostwin; 696 697 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 698 if (ranges_overlap(hostwin->min_iova, 699 hostwin->max_iova - hostwin->min_iova + 1, 700 min_iova, 701 max_iova - min_iova + 1)) { 702 hw_error("%s: Overlapped IOMMU are not enabled", __func__); 703 } 704 } 705 706 hostwin = g_malloc0(sizeof(*hostwin)); 707 708 hostwin->min_iova = min_iova; 709 hostwin->max_iova = max_iova; 710 hostwin->iova_pgsizes = iova_pgsizes; 711 QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next); 712 } 713 714 static int vfio_host_win_del(VFIOContainer *container, hwaddr min_iova, 715 hwaddr max_iova) 716 { 717 VFIOHostDMAWindow *hostwin; 718 719 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 720 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) { 721 QLIST_REMOVE(hostwin, hostwin_next); 722 g_free(hostwin); 723 return 0; 724 } 725 } 726 727 return -1; 728 } 729 730 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 731 { 732 return (!memory_region_is_ram(section->mr) && 733 !memory_region_is_iommu(section->mr)) || 734 memory_region_is_protected(section->mr) || 735 /* 736 * Sizing an enabled 64-bit BAR can cause spurious mappings to 737 * addresses in the upper part of the 64-bit address space. These 738 * are never accessed by the CPU and beyond the address width of 739 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 740 */ 741 section->offset_within_address_space & (1ULL << 63); 742 } 743 744 /* Called with rcu_read_lock held. */ 745 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 746 ram_addr_t *ram_addr, bool *read_only) 747 { 748 bool ret, mr_has_discard_manager; 749 750 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, 751 &mr_has_discard_manager); 752 if (ret && mr_has_discard_manager) { 753 /* 754 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 755 * pages will remain pinned inside vfio until unmapped, resulting in a 756 * higher memory consumption than expected. If memory would get 757 * populated again later, there would be an inconsistency between pages 758 * pinned by vfio and pages seen by QEMU. This is the case until 759 * unmapped from the IOMMU (e.g., during device reset). 760 * 761 * With malicious guests, we really only care about pinning more memory 762 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 763 * exceeded and can be used to mitigate this problem. 764 */ 765 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 766 " RAM (e.g., virtio-mem) works, however, malicious" 767 " guests can trigger pinning of more memory than" 768 " intended via an IOMMU. It's possible to mitigate " 769 " by setting/adjusting RLIMIT_MEMLOCK."); 770 } 771 return ret; 772 } 773 774 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 775 { 776 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 777 VFIOContainer *container = giommu->container; 778 hwaddr iova = iotlb->iova + giommu->iommu_offset; 779 void *vaddr; 780 int ret; 781 782 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 783 iova, iova + iotlb->addr_mask); 784 785 if (iotlb->target_as != &address_space_memory) { 786 error_report("Wrong target AS \"%s\", only system memory is allowed", 787 iotlb->target_as->name ? iotlb->target_as->name : "none"); 788 vfio_set_migration_error(-EINVAL); 789 return; 790 } 791 792 rcu_read_lock(); 793 794 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 795 bool read_only; 796 797 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only)) { 798 goto out; 799 } 800 /* 801 * vaddr is only valid until rcu_read_unlock(). But after 802 * vfio_dma_map has set up the mapping the pages will be 803 * pinned by the kernel. This makes sure that the RAM backend 804 * of vaddr will always be there, even if the memory object is 805 * destroyed and its backing memory munmap-ed. 806 */ 807 ret = vfio_dma_map(container, iova, 808 iotlb->addr_mask + 1, vaddr, 809 read_only); 810 if (ret) { 811 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 812 "0x%"HWADDR_PRIx", %p) = %d (%s)", 813 container, iova, 814 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret)); 815 } 816 } else { 817 ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1, iotlb); 818 if (ret) { 819 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 820 "0x%"HWADDR_PRIx") = %d (%s)", 821 container, iova, 822 iotlb->addr_mask + 1, ret, strerror(-ret)); 823 vfio_set_migration_error(ret); 824 } 825 } 826 out: 827 rcu_read_unlock(); 828 } 829 830 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 831 MemoryRegionSection *section) 832 { 833 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 834 listener); 835 const hwaddr size = int128_get64(section->size); 836 const hwaddr iova = section->offset_within_address_space; 837 int ret; 838 839 /* Unmap with a single call. */ 840 ret = vfio_dma_unmap(vrdl->container, iova, size , NULL); 841 if (ret) { 842 error_report("%s: vfio_dma_unmap() failed: %s", __func__, 843 strerror(-ret)); 844 } 845 } 846 847 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 848 MemoryRegionSection *section) 849 { 850 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 851 listener); 852 const hwaddr end = section->offset_within_region + 853 int128_get64(section->size); 854 hwaddr start, next, iova; 855 void *vaddr; 856 int ret; 857 858 /* 859 * Map in (aligned within memory region) minimum granularity, so we can 860 * unmap in minimum granularity later. 861 */ 862 for (start = section->offset_within_region; start < end; start = next) { 863 next = ROUND_UP(start + 1, vrdl->granularity); 864 next = MIN(next, end); 865 866 iova = start - section->offset_within_region + 867 section->offset_within_address_space; 868 vaddr = memory_region_get_ram_ptr(section->mr) + start; 869 870 ret = vfio_dma_map(vrdl->container, iova, next - start, 871 vaddr, section->readonly); 872 if (ret) { 873 /* Rollback */ 874 vfio_ram_discard_notify_discard(rdl, section); 875 return ret; 876 } 877 } 878 return 0; 879 } 880 881 static void vfio_register_ram_discard_listener(VFIOContainer *container, 882 MemoryRegionSection *section) 883 { 884 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 885 VFIORamDiscardListener *vrdl; 886 887 /* Ignore some corner cases not relevant in practice. */ 888 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 889 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 890 TARGET_PAGE_SIZE)); 891 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 892 893 vrdl = g_new0(VFIORamDiscardListener, 1); 894 vrdl->container = container; 895 vrdl->mr = section->mr; 896 vrdl->offset_within_address_space = section->offset_within_address_space; 897 vrdl->size = int128_get64(section->size); 898 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 899 section->mr); 900 901 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 902 g_assert(container->pgsizes && 903 vrdl->granularity >= 1ULL << ctz64(container->pgsizes)); 904 905 ram_discard_listener_init(&vrdl->listener, 906 vfio_ram_discard_notify_populate, 907 vfio_ram_discard_notify_discard, true); 908 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 909 QLIST_INSERT_HEAD(&container->vrdl_list, vrdl, next); 910 911 /* 912 * Sanity-check if we have a theoretically problematic setup where we could 913 * exceed the maximum number of possible DMA mappings over time. We assume 914 * that each mapped section in the same address space as a RamDiscardManager 915 * section consumes exactly one DMA mapping, with the exception of 916 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 917 * in the same address space as RamDiscardManager sections. 918 * 919 * We assume that each section in the address space consumes one memslot. 920 * We take the number of KVM memory slots as a best guess for the maximum 921 * number of sections in the address space we could have over time, 922 * also consuming DMA mappings. 923 */ 924 if (container->dma_max_mappings) { 925 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 926 927 #ifdef CONFIG_KVM 928 if (kvm_enabled()) { 929 max_memslots = kvm_get_max_memslots(); 930 } 931 #endif 932 933 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 934 hwaddr start, end; 935 936 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 937 vrdl->granularity); 938 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 939 vrdl->granularity); 940 vrdl_mappings += (end - start) / vrdl->granularity; 941 vrdl_count++; 942 } 943 944 if (vrdl_mappings + max_memslots - vrdl_count > 945 container->dma_max_mappings) { 946 warn_report("%s: possibly running out of DMA mappings. E.g., try" 947 " increasing the 'block-size' of virtio-mem devies." 948 " Maximum possible DMA mappings: %d, Maximum possible" 949 " memslots: %d", __func__, container->dma_max_mappings, 950 max_memslots); 951 } 952 } 953 } 954 955 static void vfio_unregister_ram_discard_listener(VFIOContainer *container, 956 MemoryRegionSection *section) 957 { 958 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 959 VFIORamDiscardListener *vrdl = NULL; 960 961 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 962 if (vrdl->mr == section->mr && 963 vrdl->offset_within_address_space == 964 section->offset_within_address_space) { 965 break; 966 } 967 } 968 969 if (!vrdl) { 970 hw_error("vfio: Trying to unregister missing RAM discard listener"); 971 } 972 973 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 974 QLIST_REMOVE(vrdl, next); 975 g_free(vrdl); 976 } 977 978 static VFIOHostDMAWindow *vfio_find_hostwin(VFIOContainer *container, 979 hwaddr iova, hwaddr end) 980 { 981 VFIOHostDMAWindow *hostwin; 982 bool hostwin_found = false; 983 984 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 985 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 986 hostwin_found = true; 987 break; 988 } 989 } 990 991 return hostwin_found ? hostwin : NULL; 992 } 993 994 static bool vfio_known_safe_misalignment(MemoryRegionSection *section) 995 { 996 MemoryRegion *mr = section->mr; 997 998 if (!TPM_IS_CRB(mr->owner)) { 999 return false; 1000 } 1001 1002 /* this is a known safe misaligned region, just trace for debug purpose */ 1003 trace_vfio_known_safe_misalignment(memory_region_name(mr), 1004 section->offset_within_address_space, 1005 section->offset_within_region, 1006 qemu_real_host_page_size()); 1007 return true; 1008 } 1009 1010 static bool vfio_listener_valid_section(MemoryRegionSection *section, 1011 const char *name) 1012 { 1013 if (vfio_listener_skipped_section(section)) { 1014 trace_vfio_listener_region_skip(name, 1015 section->offset_within_address_space, 1016 section->offset_within_address_space + 1017 int128_get64(int128_sub(section->size, int128_one()))); 1018 return false; 1019 } 1020 1021 if (unlikely((section->offset_within_address_space & 1022 ~qemu_real_host_page_mask()) != 1023 (section->offset_within_region & ~qemu_real_host_page_mask()))) { 1024 if (!vfio_known_safe_misalignment(section)) { 1025 error_report("%s received unaligned region %s iova=0x%"PRIx64 1026 " offset_within_region=0x%"PRIx64 1027 " qemu_real_host_page_size=0x%"PRIxPTR, 1028 __func__, memory_region_name(section->mr), 1029 section->offset_within_address_space, 1030 section->offset_within_region, 1031 qemu_real_host_page_size()); 1032 } 1033 return false; 1034 } 1035 1036 return true; 1037 } 1038 1039 static bool vfio_get_section_iova_range(VFIOContainer *container, 1040 MemoryRegionSection *section, 1041 hwaddr *out_iova, hwaddr *out_end, 1042 Int128 *out_llend) 1043 { 1044 Int128 llend; 1045 hwaddr iova; 1046 1047 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 1048 llend = int128_make64(section->offset_within_address_space); 1049 llend = int128_add(llend, section->size); 1050 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask())); 1051 1052 if (int128_ge(int128_make64(iova), llend)) { 1053 return false; 1054 } 1055 1056 *out_iova = iova; 1057 *out_end = int128_get64(int128_sub(llend, int128_one())); 1058 if (out_llend) { 1059 *out_llend = llend; 1060 } 1061 return true; 1062 } 1063 1064 static void vfio_listener_region_add(MemoryListener *listener, 1065 MemoryRegionSection *section) 1066 { 1067 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1068 hwaddr iova, end; 1069 Int128 llend, llsize; 1070 void *vaddr; 1071 int ret; 1072 VFIOHostDMAWindow *hostwin; 1073 Error *err = NULL; 1074 1075 if (!vfio_listener_valid_section(section, "region_add")) { 1076 return; 1077 } 1078 1079 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) { 1080 if (memory_region_is_ram_device(section->mr)) { 1081 trace_vfio_listener_region_add_no_dma_map( 1082 memory_region_name(section->mr), 1083 section->offset_within_address_space, 1084 int128_getlo(section->size), 1085 qemu_real_host_page_size()); 1086 } 1087 return; 1088 } 1089 1090 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 1091 hwaddr pgsize = 0; 1092 1093 /* For now intersections are not allowed, we may relax this later */ 1094 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 1095 if (ranges_overlap(hostwin->min_iova, 1096 hostwin->max_iova - hostwin->min_iova + 1, 1097 section->offset_within_address_space, 1098 int128_get64(section->size))) { 1099 error_setg(&err, 1100 "region [0x%"PRIx64",0x%"PRIx64"] overlaps with existing" 1101 "host DMA window [0x%"PRIx64",0x%"PRIx64"]", 1102 section->offset_within_address_space, 1103 section->offset_within_address_space + 1104 int128_get64(section->size) - 1, 1105 hostwin->min_iova, hostwin->max_iova); 1106 goto fail; 1107 } 1108 } 1109 1110 ret = vfio_spapr_create_window(container, section, &pgsize); 1111 if (ret) { 1112 error_setg_errno(&err, -ret, "Failed to create SPAPR window"); 1113 goto fail; 1114 } 1115 1116 vfio_host_win_add(container, section->offset_within_address_space, 1117 section->offset_within_address_space + 1118 int128_get64(section->size) - 1, pgsize); 1119 #ifdef CONFIG_KVM 1120 if (kvm_enabled()) { 1121 VFIOGroup *group; 1122 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 1123 struct kvm_vfio_spapr_tce param; 1124 struct kvm_device_attr attr = { 1125 .group = KVM_DEV_VFIO_GROUP, 1126 .attr = KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE, 1127 .addr = (uint64_t)(unsigned long)¶m, 1128 }; 1129 1130 if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_TCE_FD, 1131 ¶m.tablefd)) { 1132 QLIST_FOREACH(group, &container->group_list, container_next) { 1133 param.groupfd = group->fd; 1134 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1135 error_report("vfio: failed to setup fd %d " 1136 "for a group with fd %d: %s", 1137 param.tablefd, param.groupfd, 1138 strerror(errno)); 1139 return; 1140 } 1141 trace_vfio_spapr_group_attach(param.groupfd, param.tablefd); 1142 } 1143 } 1144 } 1145 #endif 1146 } 1147 1148 hostwin = vfio_find_hostwin(container, iova, end); 1149 if (!hostwin) { 1150 error_setg(&err, "Container %p can't map guest IOVA region" 1151 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container, iova, end); 1152 goto fail; 1153 } 1154 1155 memory_region_ref(section->mr); 1156 1157 if (memory_region_is_iommu(section->mr)) { 1158 VFIOGuestIOMMU *giommu; 1159 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 1160 int iommu_idx; 1161 1162 trace_vfio_listener_region_add_iommu(iova, end); 1163 /* 1164 * FIXME: For VFIO iommu types which have KVM acceleration to 1165 * avoid bouncing all map/unmaps through qemu this way, this 1166 * would be the right place to wire that up (tell the KVM 1167 * device emulation the VFIO iommu handles to use). 1168 */ 1169 giommu = g_malloc0(sizeof(*giommu)); 1170 giommu->iommu_mr = iommu_mr; 1171 giommu->iommu_offset = section->offset_within_address_space - 1172 section->offset_within_region; 1173 giommu->container = container; 1174 llend = int128_add(int128_make64(section->offset_within_region), 1175 section->size); 1176 llend = int128_sub(llend, int128_one()); 1177 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 1178 MEMTXATTRS_UNSPECIFIED); 1179 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 1180 IOMMU_NOTIFIER_IOTLB_EVENTS, 1181 section->offset_within_region, 1182 int128_get64(llend), 1183 iommu_idx); 1184 1185 ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr, 1186 container->pgsizes, 1187 &err); 1188 if (ret) { 1189 g_free(giommu); 1190 goto fail; 1191 } 1192 1193 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 1194 &err); 1195 if (ret) { 1196 g_free(giommu); 1197 goto fail; 1198 } 1199 QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next); 1200 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); 1201 1202 return; 1203 } 1204 1205 /* Here we assume that memory_region_is_ram(section->mr)==true */ 1206 1207 /* 1208 * For RAM memory regions with a RamDiscardManager, we only want to map the 1209 * actually populated parts - and update the mapping whenever we're notified 1210 * about changes. 1211 */ 1212 if (memory_region_has_ram_discard_manager(section->mr)) { 1213 vfio_register_ram_discard_listener(container, section); 1214 return; 1215 } 1216 1217 vaddr = memory_region_get_ram_ptr(section->mr) + 1218 section->offset_within_region + 1219 (iova - section->offset_within_address_space); 1220 1221 trace_vfio_listener_region_add_ram(iova, end, vaddr); 1222 1223 llsize = int128_sub(llend, int128_make64(iova)); 1224 1225 if (memory_region_is_ram_device(section->mr)) { 1226 hwaddr pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 1227 1228 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 1229 trace_vfio_listener_region_add_no_dma_map( 1230 memory_region_name(section->mr), 1231 section->offset_within_address_space, 1232 int128_getlo(section->size), 1233 pgmask + 1); 1234 return; 1235 } 1236 } 1237 1238 ret = vfio_dma_map(container, iova, int128_get64(llsize), 1239 vaddr, section->readonly); 1240 if (ret) { 1241 error_setg(&err, "vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 1242 "0x%"HWADDR_PRIx", %p) = %d (%s)", 1243 container, iova, int128_get64(llsize), vaddr, ret, 1244 strerror(-ret)); 1245 if (memory_region_is_ram_device(section->mr)) { 1246 /* Allow unexpected mappings not to be fatal for RAM devices */ 1247 error_report_err(err); 1248 return; 1249 } 1250 goto fail; 1251 } 1252 1253 return; 1254 1255 fail: 1256 if (memory_region_is_ram_device(section->mr)) { 1257 error_report("failed to vfio_dma_map. pci p2p may not work"); 1258 return; 1259 } 1260 /* 1261 * On the initfn path, store the first error in the container so we 1262 * can gracefully fail. Runtime, there's not much we can do other 1263 * than throw a hardware error. 1264 */ 1265 if (!container->initialized) { 1266 if (!container->error) { 1267 error_propagate_prepend(&container->error, err, 1268 "Region %s: ", 1269 memory_region_name(section->mr)); 1270 } else { 1271 error_free(err); 1272 } 1273 } else { 1274 error_report_err(err); 1275 hw_error("vfio: DMA mapping failed, unable to continue"); 1276 } 1277 } 1278 1279 static void vfio_listener_region_del(MemoryListener *listener, 1280 MemoryRegionSection *section) 1281 { 1282 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1283 hwaddr iova, end; 1284 Int128 llend, llsize; 1285 int ret; 1286 bool try_unmap = true; 1287 1288 if (!vfio_listener_valid_section(section, "region_del")) { 1289 return; 1290 } 1291 1292 if (memory_region_is_iommu(section->mr)) { 1293 VFIOGuestIOMMU *giommu; 1294 1295 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 1296 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1297 giommu->n.start == section->offset_within_region) { 1298 memory_region_unregister_iommu_notifier(section->mr, 1299 &giommu->n); 1300 QLIST_REMOVE(giommu, giommu_next); 1301 g_free(giommu); 1302 break; 1303 } 1304 } 1305 1306 /* 1307 * FIXME: We assume the one big unmap below is adequate to 1308 * remove any individual page mappings in the IOMMU which 1309 * might have been copied into VFIO. This works for a page table 1310 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 1311 * That may not be true for all IOMMU types. 1312 */ 1313 } 1314 1315 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) { 1316 return; 1317 } 1318 1319 llsize = int128_sub(llend, int128_make64(iova)); 1320 1321 trace_vfio_listener_region_del(iova, end); 1322 1323 if (memory_region_is_ram_device(section->mr)) { 1324 hwaddr pgmask; 1325 VFIOHostDMAWindow *hostwin; 1326 1327 hostwin = vfio_find_hostwin(container, iova, end); 1328 assert(hostwin); /* or region_add() would have failed */ 1329 1330 pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 1331 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 1332 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1333 vfio_unregister_ram_discard_listener(container, section); 1334 /* Unregistering will trigger an unmap. */ 1335 try_unmap = false; 1336 } 1337 1338 if (try_unmap) { 1339 if (int128_eq(llsize, int128_2_64())) { 1340 /* The unmap ioctl doesn't accept a full 64-bit span. */ 1341 llsize = int128_rshift(llsize, 1); 1342 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 1343 if (ret) { 1344 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 1345 "0x%"HWADDR_PRIx") = %d (%s)", 1346 container, iova, int128_get64(llsize), ret, 1347 strerror(-ret)); 1348 } 1349 iova += int128_get64(llsize); 1350 } 1351 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 1352 if (ret) { 1353 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 1354 "0x%"HWADDR_PRIx") = %d (%s)", 1355 container, iova, int128_get64(llsize), ret, 1356 strerror(-ret)); 1357 } 1358 } 1359 1360 memory_region_unref(section->mr); 1361 1362 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 1363 vfio_spapr_remove_window(container, 1364 section->offset_within_address_space); 1365 if (vfio_host_win_del(container, 1366 section->offset_within_address_space, 1367 section->offset_within_address_space + 1368 int128_get64(section->size) - 1) < 0) { 1369 hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx, 1370 __func__, section->offset_within_address_space); 1371 } 1372 } 1373 } 1374 1375 static int vfio_set_dirty_page_tracking(VFIOContainer *container, bool start) 1376 { 1377 int ret; 1378 struct vfio_iommu_type1_dirty_bitmap dirty = { 1379 .argsz = sizeof(dirty), 1380 }; 1381 1382 if (!container->dirty_pages_supported) { 1383 return 0; 1384 } 1385 1386 if (start) { 1387 dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START; 1388 } else { 1389 dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP; 1390 } 1391 1392 ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, &dirty); 1393 if (ret) { 1394 ret = -errno; 1395 error_report("Failed to set dirty tracking flag 0x%x errno: %d", 1396 dirty.flags, errno); 1397 } 1398 1399 return ret; 1400 } 1401 1402 typedef struct VFIODirtyRanges { 1403 hwaddr min32; 1404 hwaddr max32; 1405 hwaddr min64; 1406 hwaddr max64; 1407 } VFIODirtyRanges; 1408 1409 typedef struct VFIODirtyRangesListener { 1410 VFIOContainer *container; 1411 VFIODirtyRanges ranges; 1412 MemoryListener listener; 1413 } VFIODirtyRangesListener; 1414 1415 static void vfio_dirty_tracking_update(MemoryListener *listener, 1416 MemoryRegionSection *section) 1417 { 1418 VFIODirtyRangesListener *dirty = container_of(listener, 1419 VFIODirtyRangesListener, 1420 listener); 1421 VFIODirtyRanges *range = &dirty->ranges; 1422 hwaddr iova, end, *min, *max; 1423 1424 if (!vfio_listener_valid_section(section, "tracking_update") || 1425 !vfio_get_section_iova_range(dirty->container, section, 1426 &iova, &end, NULL)) { 1427 return; 1428 } 1429 1430 /* 1431 * The address space passed to the dirty tracker is reduced to two ranges: 1432 * one for 32-bit DMA ranges, and another one for 64-bit DMA ranges. 1433 * The underlying reports of dirty will query a sub-interval of each of 1434 * these ranges. 1435 * 1436 * The purpose of the dual range handling is to handle known cases of big 1437 * holes in the address space, like the x86 AMD 1T hole. The alternative 1438 * would be an IOVATree but that has a much bigger runtime overhead and 1439 * unnecessary complexity. 1440 */ 1441 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64; 1442 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64; 1443 1444 if (*min > iova) { 1445 *min = iova; 1446 } 1447 if (*max < end) { 1448 *max = end; 1449 } 1450 1451 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max); 1452 return; 1453 } 1454 1455 static const MemoryListener vfio_dirty_tracking_listener = { 1456 .name = "vfio-tracking", 1457 .region_add = vfio_dirty_tracking_update, 1458 }; 1459 1460 static void vfio_dirty_tracking_init(VFIOContainer *container, 1461 VFIODirtyRanges *ranges) 1462 { 1463 VFIODirtyRangesListener dirty; 1464 1465 memset(&dirty, 0, sizeof(dirty)); 1466 dirty.ranges.min32 = UINT32_MAX; 1467 dirty.ranges.min64 = UINT64_MAX; 1468 dirty.listener = vfio_dirty_tracking_listener; 1469 dirty.container = container; 1470 1471 memory_listener_register(&dirty.listener, 1472 container->space->as); 1473 1474 *ranges = dirty.ranges; 1475 1476 /* 1477 * The memory listener is synchronous, and used to calculate the range 1478 * to dirty tracking. Unregister it after we are done as we are not 1479 * interested in any follow-up updates. 1480 */ 1481 memory_listener_unregister(&dirty.listener); 1482 } 1483 1484 static void vfio_devices_dma_logging_stop(VFIOContainer *container) 1485 { 1486 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), 1487 sizeof(uint64_t))] = {}; 1488 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1489 VFIODevice *vbasedev; 1490 VFIOGroup *group; 1491 1492 feature->argsz = sizeof(buf); 1493 feature->flags = VFIO_DEVICE_FEATURE_SET | 1494 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP; 1495 1496 QLIST_FOREACH(group, &container->group_list, container_next) { 1497 QLIST_FOREACH(vbasedev, &group->device_list, next) { 1498 if (!vbasedev->dirty_tracking) { 1499 continue; 1500 } 1501 1502 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1503 warn_report("%s: Failed to stop DMA logging, err %d (%s)", 1504 vbasedev->name, -errno, strerror(errno)); 1505 } 1506 vbasedev->dirty_tracking = false; 1507 } 1508 } 1509 } 1510 1511 static struct vfio_device_feature * 1512 vfio_device_feature_dma_logging_start_create(VFIOContainer *container, 1513 VFIODirtyRanges *tracking) 1514 { 1515 struct vfio_device_feature *feature; 1516 size_t feature_size; 1517 struct vfio_device_feature_dma_logging_control *control; 1518 struct vfio_device_feature_dma_logging_range *ranges; 1519 1520 feature_size = sizeof(struct vfio_device_feature) + 1521 sizeof(struct vfio_device_feature_dma_logging_control); 1522 feature = g_try_malloc0(feature_size); 1523 if (!feature) { 1524 errno = ENOMEM; 1525 return NULL; 1526 } 1527 feature->argsz = feature_size; 1528 feature->flags = VFIO_DEVICE_FEATURE_SET | 1529 VFIO_DEVICE_FEATURE_DMA_LOGGING_START; 1530 1531 control = (struct vfio_device_feature_dma_logging_control *)feature->data; 1532 control->page_size = qemu_real_host_page_size(); 1533 1534 /* 1535 * DMA logging uAPI guarantees to support at least a number of ranges that 1536 * fits into a single host kernel base page. 1537 */ 1538 control->num_ranges = !!tracking->max32 + !!tracking->max64; 1539 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range, 1540 control->num_ranges); 1541 if (!ranges) { 1542 g_free(feature); 1543 errno = ENOMEM; 1544 1545 return NULL; 1546 } 1547 1548 control->ranges = (__u64)(uintptr_t)ranges; 1549 if (tracking->max32) { 1550 ranges->iova = tracking->min32; 1551 ranges->length = (tracking->max32 - tracking->min32) + 1; 1552 ranges++; 1553 } 1554 if (tracking->max64) { 1555 ranges->iova = tracking->min64; 1556 ranges->length = (tracking->max64 - tracking->min64) + 1; 1557 } 1558 1559 trace_vfio_device_dirty_tracking_start(control->num_ranges, 1560 tracking->min32, tracking->max32, 1561 tracking->min64, tracking->max64); 1562 1563 return feature; 1564 } 1565 1566 static void vfio_device_feature_dma_logging_start_destroy( 1567 struct vfio_device_feature *feature) 1568 { 1569 struct vfio_device_feature_dma_logging_control *control = 1570 (struct vfio_device_feature_dma_logging_control *)feature->data; 1571 struct vfio_device_feature_dma_logging_range *ranges = 1572 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges; 1573 1574 g_free(ranges); 1575 g_free(feature); 1576 } 1577 1578 static int vfio_devices_dma_logging_start(VFIOContainer *container) 1579 { 1580 struct vfio_device_feature *feature; 1581 VFIODirtyRanges ranges; 1582 VFIODevice *vbasedev; 1583 VFIOGroup *group; 1584 int ret = 0; 1585 1586 vfio_dirty_tracking_init(container, &ranges); 1587 feature = vfio_device_feature_dma_logging_start_create(container, 1588 &ranges); 1589 if (!feature) { 1590 return -errno; 1591 } 1592 1593 QLIST_FOREACH(group, &container->group_list, container_next) { 1594 QLIST_FOREACH(vbasedev, &group->device_list, next) { 1595 if (vbasedev->dirty_tracking) { 1596 continue; 1597 } 1598 1599 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature); 1600 if (ret) { 1601 ret = -errno; 1602 error_report("%s: Failed to start DMA logging, err %d (%s)", 1603 vbasedev->name, ret, strerror(errno)); 1604 goto out; 1605 } 1606 vbasedev->dirty_tracking = true; 1607 } 1608 } 1609 1610 out: 1611 if (ret) { 1612 vfio_devices_dma_logging_stop(container); 1613 } 1614 1615 vfio_device_feature_dma_logging_start_destroy(feature); 1616 1617 return ret; 1618 } 1619 1620 static void vfio_listener_log_global_start(MemoryListener *listener) 1621 { 1622 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1623 int ret; 1624 1625 if (vfio_devices_all_device_dirty_tracking(container)) { 1626 ret = vfio_devices_dma_logging_start(container); 1627 } else { 1628 ret = vfio_set_dirty_page_tracking(container, true); 1629 } 1630 1631 if (ret) { 1632 error_report("vfio: Could not start dirty page tracking, err: %d (%s)", 1633 ret, strerror(-ret)); 1634 vfio_set_migration_error(ret); 1635 } 1636 } 1637 1638 static void vfio_listener_log_global_stop(MemoryListener *listener) 1639 { 1640 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1641 int ret = 0; 1642 1643 if (vfio_devices_all_device_dirty_tracking(container)) { 1644 vfio_devices_dma_logging_stop(container); 1645 } else { 1646 ret = vfio_set_dirty_page_tracking(container, false); 1647 } 1648 1649 if (ret) { 1650 error_report("vfio: Could not stop dirty page tracking, err: %d (%s)", 1651 ret, strerror(-ret)); 1652 vfio_set_migration_error(ret); 1653 } 1654 } 1655 1656 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova, 1657 hwaddr size, void *bitmap) 1658 { 1659 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) + 1660 sizeof(struct vfio_device_feature_dma_logging_report), 1661 sizeof(__u64))] = {}; 1662 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; 1663 struct vfio_device_feature_dma_logging_report *report = 1664 (struct vfio_device_feature_dma_logging_report *)feature->data; 1665 1666 report->iova = iova; 1667 report->length = size; 1668 report->page_size = qemu_real_host_page_size(); 1669 report->bitmap = (__u64)(uintptr_t)bitmap; 1670 1671 feature->argsz = sizeof(buf); 1672 feature->flags = VFIO_DEVICE_FEATURE_GET | 1673 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT; 1674 1675 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) { 1676 return -errno; 1677 } 1678 1679 return 0; 1680 } 1681 1682 static int vfio_devices_query_dirty_bitmap(VFIOContainer *container, 1683 VFIOBitmap *vbmap, hwaddr iova, 1684 hwaddr size) 1685 { 1686 VFIODevice *vbasedev; 1687 VFIOGroup *group; 1688 int ret; 1689 1690 QLIST_FOREACH(group, &container->group_list, container_next) { 1691 QLIST_FOREACH(vbasedev, &group->device_list, next) { 1692 ret = vfio_device_dma_logging_report(vbasedev, iova, size, 1693 vbmap->bitmap); 1694 if (ret) { 1695 error_report("%s: Failed to get DMA logging report, iova: " 1696 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx 1697 ", err: %d (%s)", 1698 vbasedev->name, iova, size, ret, strerror(-ret)); 1699 1700 return ret; 1701 } 1702 } 1703 } 1704 1705 return 0; 1706 } 1707 1708 static int vfio_query_dirty_bitmap(VFIOContainer *container, VFIOBitmap *vbmap, 1709 hwaddr iova, hwaddr size) 1710 { 1711 struct vfio_iommu_type1_dirty_bitmap *dbitmap; 1712 struct vfio_iommu_type1_dirty_bitmap_get *range; 1713 int ret; 1714 1715 dbitmap = g_malloc0(sizeof(*dbitmap) + sizeof(*range)); 1716 1717 dbitmap->argsz = sizeof(*dbitmap) + sizeof(*range); 1718 dbitmap->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; 1719 range = (struct vfio_iommu_type1_dirty_bitmap_get *)&dbitmap->data; 1720 range->iova = iova; 1721 range->size = size; 1722 1723 /* 1724 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of 1725 * qemu_real_host_page_size to mark those dirty. Hence set bitmap's pgsize 1726 * to qemu_real_host_page_size. 1727 */ 1728 range->bitmap.pgsize = qemu_real_host_page_size(); 1729 range->bitmap.size = vbmap->size; 1730 range->bitmap.data = (__u64 *)vbmap->bitmap; 1731 1732 ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, dbitmap); 1733 if (ret) { 1734 ret = -errno; 1735 error_report("Failed to get dirty bitmap for iova: 0x%"PRIx64 1736 " size: 0x%"PRIx64" err: %d", (uint64_t)range->iova, 1737 (uint64_t)range->size, errno); 1738 } 1739 1740 g_free(dbitmap); 1741 1742 return ret; 1743 } 1744 1745 static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova, 1746 uint64_t size, ram_addr_t ram_addr) 1747 { 1748 bool all_device_dirty_tracking = 1749 vfio_devices_all_device_dirty_tracking(container); 1750 uint64_t dirty_pages; 1751 VFIOBitmap vbmap; 1752 int ret; 1753 1754 if (!container->dirty_pages_supported && !all_device_dirty_tracking) { 1755 cpu_physical_memory_set_dirty_range(ram_addr, size, 1756 tcg_enabled() ? DIRTY_CLIENTS_ALL : 1757 DIRTY_CLIENTS_NOCODE); 1758 return 0; 1759 } 1760 1761 ret = vfio_bitmap_alloc(&vbmap, size); 1762 if (ret) { 1763 return ret; 1764 } 1765 1766 if (all_device_dirty_tracking) { 1767 ret = vfio_devices_query_dirty_bitmap(container, &vbmap, iova, size); 1768 } else { 1769 ret = vfio_query_dirty_bitmap(container, &vbmap, iova, size); 1770 } 1771 1772 if (ret) { 1773 goto out; 1774 } 1775 1776 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr, 1777 vbmap.pages); 1778 1779 trace_vfio_get_dirty_bitmap(container->fd, iova, size, vbmap.size, 1780 ram_addr, dirty_pages); 1781 out: 1782 g_free(vbmap.bitmap); 1783 1784 return ret; 1785 } 1786 1787 typedef struct { 1788 IOMMUNotifier n; 1789 VFIOGuestIOMMU *giommu; 1790 } vfio_giommu_dirty_notifier; 1791 1792 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1793 { 1794 vfio_giommu_dirty_notifier *gdn = container_of(n, 1795 vfio_giommu_dirty_notifier, n); 1796 VFIOGuestIOMMU *giommu = gdn->giommu; 1797 VFIOContainer *container = giommu->container; 1798 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1799 ram_addr_t translated_addr; 1800 int ret = -EINVAL; 1801 1802 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1803 1804 if (iotlb->target_as != &address_space_memory) { 1805 error_report("Wrong target AS \"%s\", only system memory is allowed", 1806 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1807 goto out; 1808 } 1809 1810 rcu_read_lock(); 1811 if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) { 1812 ret = vfio_get_dirty_bitmap(container, iova, iotlb->addr_mask + 1, 1813 translated_addr); 1814 if (ret) { 1815 error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1816 "0x%"HWADDR_PRIx") = %d (%s)", 1817 container, iova, iotlb->addr_mask + 1, ret, 1818 strerror(-ret)); 1819 } 1820 } 1821 rcu_read_unlock(); 1822 1823 out: 1824 if (ret) { 1825 vfio_set_migration_error(ret); 1826 } 1827 } 1828 1829 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1830 void *opaque) 1831 { 1832 const hwaddr size = int128_get64(section->size); 1833 const hwaddr iova = section->offset_within_address_space; 1834 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1835 section->offset_within_region; 1836 VFIORamDiscardListener *vrdl = opaque; 1837 1838 /* 1839 * Sync the whole mapped region (spanning multiple individual mappings) 1840 * in one go. 1841 */ 1842 return vfio_get_dirty_bitmap(vrdl->container, iova, size, ram_addr); 1843 } 1844 1845 static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container, 1846 MemoryRegionSection *section) 1847 { 1848 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1849 VFIORamDiscardListener *vrdl = NULL; 1850 1851 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 1852 if (vrdl->mr == section->mr && 1853 vrdl->offset_within_address_space == 1854 section->offset_within_address_space) { 1855 break; 1856 } 1857 } 1858 1859 if (!vrdl) { 1860 hw_error("vfio: Trying to sync missing RAM discard listener"); 1861 } 1862 1863 /* 1864 * We only want/can synchronize the bitmap for actually mapped parts - 1865 * which correspond to populated parts. Replay all populated parts. 1866 */ 1867 return ram_discard_manager_replay_populated(rdm, section, 1868 vfio_ram_discard_get_dirty_bitmap, 1869 &vrdl); 1870 } 1871 1872 static int vfio_sync_dirty_bitmap(VFIOContainer *container, 1873 MemoryRegionSection *section) 1874 { 1875 ram_addr_t ram_addr; 1876 1877 if (memory_region_is_iommu(section->mr)) { 1878 VFIOGuestIOMMU *giommu; 1879 1880 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 1881 if (MEMORY_REGION(giommu->iommu_mr) == section->mr && 1882 giommu->n.start == section->offset_within_region) { 1883 Int128 llend; 1884 vfio_giommu_dirty_notifier gdn = { .giommu = giommu }; 1885 int idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr, 1886 MEMTXATTRS_UNSPECIFIED); 1887 1888 llend = int128_add(int128_make64(section->offset_within_region), 1889 section->size); 1890 llend = int128_sub(llend, int128_one()); 1891 1892 iommu_notifier_init(&gdn.n, 1893 vfio_iommu_map_dirty_notify, 1894 IOMMU_NOTIFIER_MAP, 1895 section->offset_within_region, 1896 int128_get64(llend), 1897 idx); 1898 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n); 1899 break; 1900 } 1901 } 1902 return 0; 1903 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1904 return vfio_sync_ram_discard_listener_dirty_bitmap(container, section); 1905 } 1906 1907 ram_addr = memory_region_get_ram_addr(section->mr) + 1908 section->offset_within_region; 1909 1910 return vfio_get_dirty_bitmap(container, 1911 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1912 int128_get64(section->size), ram_addr); 1913 } 1914 1915 static void vfio_listener_log_sync(MemoryListener *listener, 1916 MemoryRegionSection *section) 1917 { 1918 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1919 int ret; 1920 1921 if (vfio_listener_skipped_section(section)) { 1922 return; 1923 } 1924 1925 if (vfio_devices_all_dirty_tracking(container)) { 1926 ret = vfio_sync_dirty_bitmap(container, section); 1927 if (ret) { 1928 error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret, 1929 strerror(-ret)); 1930 vfio_set_migration_error(ret); 1931 } 1932 } 1933 } 1934 1935 static const MemoryListener vfio_memory_listener = { 1936 .name = "vfio", 1937 .region_add = vfio_listener_region_add, 1938 .region_del = vfio_listener_region_del, 1939 .log_global_start = vfio_listener_log_global_start, 1940 .log_global_stop = vfio_listener_log_global_stop, 1941 .log_sync = vfio_listener_log_sync, 1942 }; 1943 1944 static void vfio_listener_release(VFIOContainer *container) 1945 { 1946 memory_listener_unregister(&container->listener); 1947 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 1948 memory_listener_unregister(&container->prereg_listener); 1949 } 1950 } 1951 1952 static struct vfio_info_cap_header * 1953 vfio_get_cap(void *ptr, uint32_t cap_offset, uint16_t id) 1954 { 1955 struct vfio_info_cap_header *hdr; 1956 1957 for (hdr = ptr + cap_offset; hdr != ptr; hdr = ptr + hdr->next) { 1958 if (hdr->id == id) { 1959 return hdr; 1960 } 1961 } 1962 1963 return NULL; 1964 } 1965 1966 struct vfio_info_cap_header * 1967 vfio_get_region_info_cap(struct vfio_region_info *info, uint16_t id) 1968 { 1969 if (!(info->flags & VFIO_REGION_INFO_FLAG_CAPS)) { 1970 return NULL; 1971 } 1972 1973 return vfio_get_cap((void *)info, info->cap_offset, id); 1974 } 1975 1976 static struct vfio_info_cap_header * 1977 vfio_get_iommu_type1_info_cap(struct vfio_iommu_type1_info *info, uint16_t id) 1978 { 1979 if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) { 1980 return NULL; 1981 } 1982 1983 return vfio_get_cap((void *)info, info->cap_offset, id); 1984 } 1985 1986 struct vfio_info_cap_header * 1987 vfio_get_device_info_cap(struct vfio_device_info *info, uint16_t id) 1988 { 1989 if (!(info->flags & VFIO_DEVICE_FLAGS_CAPS)) { 1990 return NULL; 1991 } 1992 1993 return vfio_get_cap((void *)info, info->cap_offset, id); 1994 } 1995 1996 bool vfio_get_info_dma_avail(struct vfio_iommu_type1_info *info, 1997 unsigned int *avail) 1998 { 1999 struct vfio_info_cap_header *hdr; 2000 struct vfio_iommu_type1_info_dma_avail *cap; 2001 2002 /* If the capability cannot be found, assume no DMA limiting */ 2003 hdr = vfio_get_iommu_type1_info_cap(info, 2004 VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL); 2005 if (hdr == NULL) { 2006 return false; 2007 } 2008 2009 if (avail != NULL) { 2010 cap = (void *) hdr; 2011 *avail = cap->avail; 2012 } 2013 2014 return true; 2015 } 2016 2017 static int vfio_setup_region_sparse_mmaps(VFIORegion *region, 2018 struct vfio_region_info *info) 2019 { 2020 struct vfio_info_cap_header *hdr; 2021 struct vfio_region_info_cap_sparse_mmap *sparse; 2022 int i, j; 2023 2024 hdr = vfio_get_region_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); 2025 if (!hdr) { 2026 return -ENODEV; 2027 } 2028 2029 sparse = container_of(hdr, struct vfio_region_info_cap_sparse_mmap, header); 2030 2031 trace_vfio_region_sparse_mmap_header(region->vbasedev->name, 2032 region->nr, sparse->nr_areas); 2033 2034 region->mmaps = g_new0(VFIOMmap, sparse->nr_areas); 2035 2036 for (i = 0, j = 0; i < sparse->nr_areas; i++) { 2037 if (sparse->areas[i].size) { 2038 trace_vfio_region_sparse_mmap_entry(i, sparse->areas[i].offset, 2039 sparse->areas[i].offset + 2040 sparse->areas[i].size - 1); 2041 region->mmaps[j].offset = sparse->areas[i].offset; 2042 region->mmaps[j].size = sparse->areas[i].size; 2043 j++; 2044 } 2045 } 2046 2047 region->nr_mmaps = j; 2048 region->mmaps = g_realloc(region->mmaps, j * sizeof(VFIOMmap)); 2049 2050 return 0; 2051 } 2052 2053 int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, 2054 int index, const char *name) 2055 { 2056 struct vfio_region_info *info; 2057 int ret; 2058 2059 ret = vfio_get_region_info(vbasedev, index, &info); 2060 if (ret) { 2061 return ret; 2062 } 2063 2064 region->vbasedev = vbasedev; 2065 region->flags = info->flags; 2066 region->size = info->size; 2067 region->fd_offset = info->offset; 2068 region->nr = index; 2069 2070 if (region->size) { 2071 region->mem = g_new0(MemoryRegion, 1); 2072 memory_region_init_io(region->mem, obj, &vfio_region_ops, 2073 region, name, region->size); 2074 2075 if (!vbasedev->no_mmap && 2076 region->flags & VFIO_REGION_INFO_FLAG_MMAP) { 2077 2078 ret = vfio_setup_region_sparse_mmaps(region, info); 2079 2080 if (ret) { 2081 region->nr_mmaps = 1; 2082 region->mmaps = g_new0(VFIOMmap, region->nr_mmaps); 2083 region->mmaps[0].offset = 0; 2084 region->mmaps[0].size = region->size; 2085 } 2086 } 2087 } 2088 2089 g_free(info); 2090 2091 trace_vfio_region_setup(vbasedev->name, index, name, 2092 region->flags, region->fd_offset, region->size); 2093 return 0; 2094 } 2095 2096 static void vfio_subregion_unmap(VFIORegion *region, int index) 2097 { 2098 trace_vfio_region_unmap(memory_region_name(®ion->mmaps[index].mem), 2099 region->mmaps[index].offset, 2100 region->mmaps[index].offset + 2101 region->mmaps[index].size - 1); 2102 memory_region_del_subregion(region->mem, ®ion->mmaps[index].mem); 2103 munmap(region->mmaps[index].mmap, region->mmaps[index].size); 2104 object_unparent(OBJECT(®ion->mmaps[index].mem)); 2105 region->mmaps[index].mmap = NULL; 2106 } 2107 2108 int vfio_region_mmap(VFIORegion *region) 2109 { 2110 int i, prot = 0; 2111 char *name; 2112 2113 if (!region->mem) { 2114 return 0; 2115 } 2116 2117 prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 2118 prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 2119 2120 for (i = 0; i < region->nr_mmaps; i++) { 2121 region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot, 2122 MAP_SHARED, region->vbasedev->fd, 2123 region->fd_offset + 2124 region->mmaps[i].offset); 2125 if (region->mmaps[i].mmap == MAP_FAILED) { 2126 int ret = -errno; 2127 2128 trace_vfio_region_mmap_fault(memory_region_name(region->mem), i, 2129 region->fd_offset + 2130 region->mmaps[i].offset, 2131 region->fd_offset + 2132 region->mmaps[i].offset + 2133 region->mmaps[i].size - 1, ret); 2134 2135 region->mmaps[i].mmap = NULL; 2136 2137 for (i--; i >= 0; i--) { 2138 vfio_subregion_unmap(region, i); 2139 } 2140 2141 return ret; 2142 } 2143 2144 name = g_strdup_printf("%s mmaps[%d]", 2145 memory_region_name(region->mem), i); 2146 memory_region_init_ram_device_ptr(®ion->mmaps[i].mem, 2147 memory_region_owner(region->mem), 2148 name, region->mmaps[i].size, 2149 region->mmaps[i].mmap); 2150 g_free(name); 2151 memory_region_add_subregion(region->mem, region->mmaps[i].offset, 2152 ®ion->mmaps[i].mem); 2153 2154 trace_vfio_region_mmap(memory_region_name(®ion->mmaps[i].mem), 2155 region->mmaps[i].offset, 2156 region->mmaps[i].offset + 2157 region->mmaps[i].size - 1); 2158 } 2159 2160 return 0; 2161 } 2162 2163 void vfio_region_unmap(VFIORegion *region) 2164 { 2165 int i; 2166 2167 if (!region->mem) { 2168 return; 2169 } 2170 2171 for (i = 0; i < region->nr_mmaps; i++) { 2172 if (region->mmaps[i].mmap) { 2173 vfio_subregion_unmap(region, i); 2174 } 2175 } 2176 } 2177 2178 void vfio_region_exit(VFIORegion *region) 2179 { 2180 int i; 2181 2182 if (!region->mem) { 2183 return; 2184 } 2185 2186 for (i = 0; i < region->nr_mmaps; i++) { 2187 if (region->mmaps[i].mmap) { 2188 memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); 2189 } 2190 } 2191 2192 trace_vfio_region_exit(region->vbasedev->name, region->nr); 2193 } 2194 2195 void vfio_region_finalize(VFIORegion *region) 2196 { 2197 int i; 2198 2199 if (!region->mem) { 2200 return; 2201 } 2202 2203 for (i = 0; i < region->nr_mmaps; i++) { 2204 if (region->mmaps[i].mmap) { 2205 munmap(region->mmaps[i].mmap, region->mmaps[i].size); 2206 object_unparent(OBJECT(®ion->mmaps[i].mem)); 2207 } 2208 } 2209 2210 object_unparent(OBJECT(region->mem)); 2211 2212 g_free(region->mem); 2213 g_free(region->mmaps); 2214 2215 trace_vfio_region_finalize(region->vbasedev->name, region->nr); 2216 2217 region->mem = NULL; 2218 region->mmaps = NULL; 2219 region->nr_mmaps = 0; 2220 region->size = 0; 2221 region->flags = 0; 2222 region->nr = 0; 2223 } 2224 2225 void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled) 2226 { 2227 int i; 2228 2229 if (!region->mem) { 2230 return; 2231 } 2232 2233 for (i = 0; i < region->nr_mmaps; i++) { 2234 if (region->mmaps[i].mmap) { 2235 memory_region_set_enabled(®ion->mmaps[i].mem, enabled); 2236 } 2237 } 2238 2239 trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem), 2240 enabled); 2241 } 2242 2243 void vfio_reset_handler(void *opaque) 2244 { 2245 VFIOGroup *group; 2246 VFIODevice *vbasedev; 2247 2248 QLIST_FOREACH(group, &vfio_group_list, next) { 2249 QLIST_FOREACH(vbasedev, &group->device_list, next) { 2250 if (vbasedev->dev->realized) { 2251 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 2252 } 2253 } 2254 } 2255 2256 QLIST_FOREACH(group, &vfio_group_list, next) { 2257 QLIST_FOREACH(vbasedev, &group->device_list, next) { 2258 if (vbasedev->dev->realized && vbasedev->needs_reset) { 2259 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 2260 } 2261 } 2262 } 2263 } 2264 2265 static void vfio_kvm_device_add_group(VFIOGroup *group) 2266 { 2267 #ifdef CONFIG_KVM 2268 struct kvm_device_attr attr = { 2269 .group = KVM_DEV_VFIO_GROUP, 2270 .attr = KVM_DEV_VFIO_GROUP_ADD, 2271 .addr = (uint64_t)(unsigned long)&group->fd, 2272 }; 2273 2274 if (!kvm_enabled()) { 2275 return; 2276 } 2277 2278 if (vfio_kvm_device_fd < 0) { 2279 struct kvm_create_device cd = { 2280 .type = KVM_DEV_TYPE_VFIO, 2281 }; 2282 2283 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 2284 error_report("Failed to create KVM VFIO device: %m"); 2285 return; 2286 } 2287 2288 vfio_kvm_device_fd = cd.fd; 2289 } 2290 2291 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 2292 error_report("Failed to add group %d to KVM VFIO device: %m", 2293 group->groupid); 2294 } 2295 #endif 2296 } 2297 2298 static void vfio_kvm_device_del_group(VFIOGroup *group) 2299 { 2300 #ifdef CONFIG_KVM 2301 struct kvm_device_attr attr = { 2302 .group = KVM_DEV_VFIO_GROUP, 2303 .attr = KVM_DEV_VFIO_GROUP_DEL, 2304 .addr = (uint64_t)(unsigned long)&group->fd, 2305 }; 2306 2307 if (vfio_kvm_device_fd < 0) { 2308 return; 2309 } 2310 2311 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 2312 error_report("Failed to remove group %d from KVM VFIO device: %m", 2313 group->groupid); 2314 } 2315 #endif 2316 } 2317 2318 static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 2319 { 2320 VFIOAddressSpace *space; 2321 2322 QLIST_FOREACH(space, &vfio_address_spaces, list) { 2323 if (space->as == as) { 2324 return space; 2325 } 2326 } 2327 2328 /* No suitable VFIOAddressSpace, create a new one */ 2329 space = g_malloc0(sizeof(*space)); 2330 space->as = as; 2331 QLIST_INIT(&space->containers); 2332 2333 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 2334 2335 return space; 2336 } 2337 2338 static void vfio_put_address_space(VFIOAddressSpace *space) 2339 { 2340 if (QLIST_EMPTY(&space->containers)) { 2341 QLIST_REMOVE(space, list); 2342 g_free(space); 2343 } 2344 } 2345 2346 /* 2347 * vfio_get_iommu_type - selects the richest iommu_type (v2 first) 2348 */ 2349 static int vfio_get_iommu_type(VFIOContainer *container, 2350 Error **errp) 2351 { 2352 int iommu_types[] = { VFIO_TYPE1v2_IOMMU, VFIO_TYPE1_IOMMU, 2353 VFIO_SPAPR_TCE_v2_IOMMU, VFIO_SPAPR_TCE_IOMMU }; 2354 int i; 2355 2356 for (i = 0; i < ARRAY_SIZE(iommu_types); i++) { 2357 if (ioctl(container->fd, VFIO_CHECK_EXTENSION, iommu_types[i])) { 2358 return iommu_types[i]; 2359 } 2360 } 2361 error_setg(errp, "No available IOMMU models"); 2362 return -EINVAL; 2363 } 2364 2365 static int vfio_init_container(VFIOContainer *container, int group_fd, 2366 Error **errp) 2367 { 2368 int iommu_type, ret; 2369 2370 iommu_type = vfio_get_iommu_type(container, errp); 2371 if (iommu_type < 0) { 2372 return iommu_type; 2373 } 2374 2375 ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container->fd); 2376 if (ret) { 2377 error_setg_errno(errp, errno, "Failed to set group container"); 2378 return -errno; 2379 } 2380 2381 while (ioctl(container->fd, VFIO_SET_IOMMU, iommu_type)) { 2382 if (iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 2383 /* 2384 * On sPAPR, despite the IOMMU subdriver always advertises v1 and 2385 * v2, the running platform may not support v2 and there is no 2386 * way to guess it until an IOMMU group gets added to the container. 2387 * So in case it fails with v2, try v1 as a fallback. 2388 */ 2389 iommu_type = VFIO_SPAPR_TCE_IOMMU; 2390 continue; 2391 } 2392 error_setg_errno(errp, errno, "Failed to set iommu for container"); 2393 return -errno; 2394 } 2395 2396 container->iommu_type = iommu_type; 2397 return 0; 2398 } 2399 2400 static int vfio_get_iommu_info(VFIOContainer *container, 2401 struct vfio_iommu_type1_info **info) 2402 { 2403 2404 size_t argsz = sizeof(struct vfio_iommu_type1_info); 2405 2406 *info = g_new0(struct vfio_iommu_type1_info, 1); 2407 again: 2408 (*info)->argsz = argsz; 2409 2410 if (ioctl(container->fd, VFIO_IOMMU_GET_INFO, *info)) { 2411 g_free(*info); 2412 *info = NULL; 2413 return -errno; 2414 } 2415 2416 if (((*info)->argsz > argsz)) { 2417 argsz = (*info)->argsz; 2418 *info = g_realloc(*info, argsz); 2419 goto again; 2420 } 2421 2422 return 0; 2423 } 2424 2425 static struct vfio_info_cap_header * 2426 vfio_get_iommu_info_cap(struct vfio_iommu_type1_info *info, uint16_t id) 2427 { 2428 struct vfio_info_cap_header *hdr; 2429 void *ptr = info; 2430 2431 if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) { 2432 return NULL; 2433 } 2434 2435 for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) { 2436 if (hdr->id == id) { 2437 return hdr; 2438 } 2439 } 2440 2441 return NULL; 2442 } 2443 2444 static void vfio_get_iommu_info_migration(VFIOContainer *container, 2445 struct vfio_iommu_type1_info *info) 2446 { 2447 struct vfio_info_cap_header *hdr; 2448 struct vfio_iommu_type1_info_cap_migration *cap_mig; 2449 2450 hdr = vfio_get_iommu_info_cap(info, VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION); 2451 if (!hdr) { 2452 return; 2453 } 2454 2455 cap_mig = container_of(hdr, struct vfio_iommu_type1_info_cap_migration, 2456 header); 2457 2458 /* 2459 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of 2460 * qemu_real_host_page_size to mark those dirty. 2461 */ 2462 if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) { 2463 container->dirty_pages_supported = true; 2464 container->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size; 2465 container->dirty_pgsizes = cap_mig->pgsize_bitmap; 2466 } 2467 } 2468 2469 static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, 2470 Error **errp) 2471 { 2472 VFIOContainer *container; 2473 int ret, fd; 2474 VFIOAddressSpace *space; 2475 2476 space = vfio_get_address_space(as); 2477 2478 /* 2479 * VFIO is currently incompatible with discarding of RAM insofar as the 2480 * madvise to purge (zap) the page from QEMU's address space does not 2481 * interact with the memory API and therefore leaves stale virtual to 2482 * physical mappings in the IOMMU if the page was previously pinned. We 2483 * therefore set discarding broken for each group added to a container, 2484 * whether the container is used individually or shared. This provides 2485 * us with options to allow devices within a group to opt-in and allow 2486 * discarding, so long as it is done consistently for a group (for instance 2487 * if the device is an mdev device where it is known that the host vendor 2488 * driver will never pin pages outside of the working set of the guest 2489 * driver, which would thus not be discarding candidates). 2490 * 2491 * The first opportunity to induce pinning occurs here where we attempt to 2492 * attach the group to existing containers within the AddressSpace. If any 2493 * pages are already zapped from the virtual address space, such as from 2494 * previous discards, new pinning will cause valid mappings to be 2495 * re-established. Likewise, when the overall MemoryListener for a new 2496 * container is registered, a replay of mappings within the AddressSpace 2497 * will occur, re-establishing any previously zapped pages as well. 2498 * 2499 * Especially virtio-balloon is currently only prevented from discarding 2500 * new memory, it will not yet set ram_block_discard_set_required() and 2501 * therefore, neither stops us here or deals with the sudden memory 2502 * consumption of inflated memory. 2503 * 2504 * We do support discarding of memory coordinated via the RamDiscardManager 2505 * with some IOMMU types. vfio_ram_block_discard_disable() handles the 2506 * details once we know which type of IOMMU we are using. 2507 */ 2508 2509 QLIST_FOREACH(container, &space->containers, next) { 2510 if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) { 2511 ret = vfio_ram_block_discard_disable(container, true); 2512 if (ret) { 2513 error_setg_errno(errp, -ret, 2514 "Cannot set discarding of RAM broken"); 2515 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, 2516 &container->fd)) { 2517 error_report("vfio: error disconnecting group %d from" 2518 " container", group->groupid); 2519 } 2520 return ret; 2521 } 2522 group->container = container; 2523 QLIST_INSERT_HEAD(&container->group_list, group, container_next); 2524 vfio_kvm_device_add_group(group); 2525 return 0; 2526 } 2527 } 2528 2529 fd = qemu_open_old("/dev/vfio/vfio", O_RDWR); 2530 if (fd < 0) { 2531 error_setg_errno(errp, errno, "failed to open /dev/vfio/vfio"); 2532 ret = -errno; 2533 goto put_space_exit; 2534 } 2535 2536 ret = ioctl(fd, VFIO_GET_API_VERSION); 2537 if (ret != VFIO_API_VERSION) { 2538 error_setg(errp, "supported vfio version: %d, " 2539 "reported version: %d", VFIO_API_VERSION, ret); 2540 ret = -EINVAL; 2541 goto close_fd_exit; 2542 } 2543 2544 container = g_malloc0(sizeof(*container)); 2545 container->space = space; 2546 container->fd = fd; 2547 container->error = NULL; 2548 container->dirty_pages_supported = false; 2549 container->dma_max_mappings = 0; 2550 QLIST_INIT(&container->giommu_list); 2551 QLIST_INIT(&container->hostwin_list); 2552 QLIST_INIT(&container->vrdl_list); 2553 2554 ret = vfio_init_container(container, group->fd, errp); 2555 if (ret) { 2556 goto free_container_exit; 2557 } 2558 2559 ret = vfio_ram_block_discard_disable(container, true); 2560 if (ret) { 2561 error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken"); 2562 goto free_container_exit; 2563 } 2564 2565 switch (container->iommu_type) { 2566 case VFIO_TYPE1v2_IOMMU: 2567 case VFIO_TYPE1_IOMMU: 2568 { 2569 struct vfio_iommu_type1_info *info; 2570 2571 ret = vfio_get_iommu_info(container, &info); 2572 if (ret) { 2573 error_setg_errno(errp, -ret, "Failed to get VFIO IOMMU info"); 2574 goto enable_discards_exit; 2575 } 2576 2577 if (info->flags & VFIO_IOMMU_INFO_PGSIZES) { 2578 container->pgsizes = info->iova_pgsizes; 2579 } else { 2580 container->pgsizes = qemu_real_host_page_size(); 2581 } 2582 2583 if (!vfio_get_info_dma_avail(info, &container->dma_max_mappings)) { 2584 container->dma_max_mappings = 65535; 2585 } 2586 vfio_get_iommu_info_migration(container, info); 2587 g_free(info); 2588 2589 /* 2590 * FIXME: We should parse VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE 2591 * information to get the actual window extent rather than assume 2592 * a 64-bit IOVA address space. 2593 */ 2594 vfio_host_win_add(container, 0, (hwaddr)-1, container->pgsizes); 2595 2596 break; 2597 } 2598 case VFIO_SPAPR_TCE_v2_IOMMU: 2599 case VFIO_SPAPR_TCE_IOMMU: 2600 { 2601 struct vfio_iommu_spapr_tce_info info; 2602 bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU; 2603 2604 /* 2605 * The host kernel code implementing VFIO_IOMMU_DISABLE is called 2606 * when container fd is closed so we do not call it explicitly 2607 * in this file. 2608 */ 2609 if (!v2) { 2610 ret = ioctl(fd, VFIO_IOMMU_ENABLE); 2611 if (ret) { 2612 error_setg_errno(errp, errno, "failed to enable container"); 2613 ret = -errno; 2614 goto enable_discards_exit; 2615 } 2616 } else { 2617 container->prereg_listener = vfio_prereg_listener; 2618 2619 memory_listener_register(&container->prereg_listener, 2620 &address_space_memory); 2621 if (container->error) { 2622 memory_listener_unregister(&container->prereg_listener); 2623 ret = -1; 2624 error_propagate_prepend(errp, container->error, 2625 "RAM memory listener initialization failed: "); 2626 goto enable_discards_exit; 2627 } 2628 } 2629 2630 info.argsz = sizeof(info); 2631 ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); 2632 if (ret) { 2633 error_setg_errno(errp, errno, 2634 "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed"); 2635 ret = -errno; 2636 if (v2) { 2637 memory_listener_unregister(&container->prereg_listener); 2638 } 2639 goto enable_discards_exit; 2640 } 2641 2642 if (v2) { 2643 container->pgsizes = info.ddw.pgsizes; 2644 /* 2645 * There is a default window in just created container. 2646 * To make region_add/del simpler, we better remove this 2647 * window now and let those iommu_listener callbacks 2648 * create/remove them when needed. 2649 */ 2650 ret = vfio_spapr_remove_window(container, info.dma32_window_start); 2651 if (ret) { 2652 error_setg_errno(errp, -ret, 2653 "failed to remove existing window"); 2654 goto enable_discards_exit; 2655 } 2656 } else { 2657 /* The default table uses 4K pages */ 2658 container->pgsizes = 0x1000; 2659 vfio_host_win_add(container, info.dma32_window_start, 2660 info.dma32_window_start + 2661 info.dma32_window_size - 1, 2662 0x1000); 2663 } 2664 } 2665 } 2666 2667 vfio_kvm_device_add_group(group); 2668 2669 QLIST_INIT(&container->group_list); 2670 QLIST_INSERT_HEAD(&space->containers, container, next); 2671 2672 group->container = container; 2673 QLIST_INSERT_HEAD(&container->group_list, group, container_next); 2674 2675 container->listener = vfio_memory_listener; 2676 2677 memory_listener_register(&container->listener, container->space->as); 2678 2679 if (container->error) { 2680 ret = -1; 2681 error_propagate_prepend(errp, container->error, 2682 "memory listener initialization failed: "); 2683 goto listener_release_exit; 2684 } 2685 2686 container->initialized = true; 2687 2688 return 0; 2689 listener_release_exit: 2690 QLIST_REMOVE(group, container_next); 2691 QLIST_REMOVE(container, next); 2692 vfio_kvm_device_del_group(group); 2693 vfio_listener_release(container); 2694 2695 enable_discards_exit: 2696 vfio_ram_block_discard_disable(container, false); 2697 2698 free_container_exit: 2699 g_free(container); 2700 2701 close_fd_exit: 2702 close(fd); 2703 2704 put_space_exit: 2705 vfio_put_address_space(space); 2706 2707 return ret; 2708 } 2709 2710 static void vfio_disconnect_container(VFIOGroup *group) 2711 { 2712 VFIOContainer *container = group->container; 2713 2714 QLIST_REMOVE(group, container_next); 2715 group->container = NULL; 2716 2717 /* 2718 * Explicitly release the listener first before unset container, 2719 * since unset may destroy the backend container if it's the last 2720 * group. 2721 */ 2722 if (QLIST_EMPTY(&container->group_list)) { 2723 vfio_listener_release(container); 2724 } 2725 2726 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) { 2727 error_report("vfio: error disconnecting group %d from container", 2728 group->groupid); 2729 } 2730 2731 if (QLIST_EMPTY(&container->group_list)) { 2732 VFIOAddressSpace *space = container->space; 2733 VFIOGuestIOMMU *giommu, *tmp; 2734 VFIOHostDMAWindow *hostwin, *next; 2735 2736 QLIST_REMOVE(container, next); 2737 2738 QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) { 2739 memory_region_unregister_iommu_notifier( 2740 MEMORY_REGION(giommu->iommu_mr), &giommu->n); 2741 QLIST_REMOVE(giommu, giommu_next); 2742 g_free(giommu); 2743 } 2744 2745 QLIST_FOREACH_SAFE(hostwin, &container->hostwin_list, hostwin_next, 2746 next) { 2747 QLIST_REMOVE(hostwin, hostwin_next); 2748 g_free(hostwin); 2749 } 2750 2751 trace_vfio_disconnect_container(container->fd); 2752 close(container->fd); 2753 g_free(container); 2754 2755 vfio_put_address_space(space); 2756 } 2757 } 2758 2759 VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) 2760 { 2761 VFIOGroup *group; 2762 char path[32]; 2763 struct vfio_group_status status = { .argsz = sizeof(status) }; 2764 2765 QLIST_FOREACH(group, &vfio_group_list, next) { 2766 if (group->groupid == groupid) { 2767 /* Found it. Now is it already in the right context? */ 2768 if (group->container->space->as == as) { 2769 return group; 2770 } else { 2771 error_setg(errp, "group %d used in multiple address spaces", 2772 group->groupid); 2773 return NULL; 2774 } 2775 } 2776 } 2777 2778 group = g_malloc0(sizeof(*group)); 2779 2780 snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); 2781 group->fd = qemu_open_old(path, O_RDWR); 2782 if (group->fd < 0) { 2783 error_setg_errno(errp, errno, "failed to open %s", path); 2784 goto free_group_exit; 2785 } 2786 2787 if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) { 2788 error_setg_errno(errp, errno, "failed to get group %d status", groupid); 2789 goto close_fd_exit; 2790 } 2791 2792 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { 2793 error_setg(errp, "group %d is not viable", groupid); 2794 error_append_hint(errp, 2795 "Please ensure all devices within the iommu_group " 2796 "are bound to their vfio bus driver.\n"); 2797 goto close_fd_exit; 2798 } 2799 2800 group->groupid = groupid; 2801 QLIST_INIT(&group->device_list); 2802 2803 if (vfio_connect_container(group, as, errp)) { 2804 error_prepend(errp, "failed to setup container for group %d: ", 2805 groupid); 2806 goto close_fd_exit; 2807 } 2808 2809 if (QLIST_EMPTY(&vfio_group_list)) { 2810 qemu_register_reset(vfio_reset_handler, NULL); 2811 } 2812 2813 QLIST_INSERT_HEAD(&vfio_group_list, group, next); 2814 2815 return group; 2816 2817 close_fd_exit: 2818 close(group->fd); 2819 2820 free_group_exit: 2821 g_free(group); 2822 2823 return NULL; 2824 } 2825 2826 void vfio_put_group(VFIOGroup *group) 2827 { 2828 if (!group || !QLIST_EMPTY(&group->device_list)) { 2829 return; 2830 } 2831 2832 if (!group->ram_block_discard_allowed) { 2833 vfio_ram_block_discard_disable(group->container, false); 2834 } 2835 vfio_kvm_device_del_group(group); 2836 vfio_disconnect_container(group); 2837 QLIST_REMOVE(group, next); 2838 trace_vfio_put_group(group->fd); 2839 close(group->fd); 2840 g_free(group); 2841 2842 if (QLIST_EMPTY(&vfio_group_list)) { 2843 qemu_unregister_reset(vfio_reset_handler, NULL); 2844 } 2845 } 2846 2847 int vfio_get_device(VFIOGroup *group, const char *name, 2848 VFIODevice *vbasedev, Error **errp) 2849 { 2850 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) }; 2851 int ret, fd; 2852 2853 fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name); 2854 if (fd < 0) { 2855 error_setg_errno(errp, errno, "error getting device from group %d", 2856 group->groupid); 2857 error_append_hint(errp, 2858 "Verify all devices in group %d are bound to vfio-<bus> " 2859 "or pci-stub and not already in use\n", group->groupid); 2860 return fd; 2861 } 2862 2863 ret = ioctl(fd, VFIO_DEVICE_GET_INFO, &dev_info); 2864 if (ret) { 2865 error_setg_errno(errp, errno, "error getting device info"); 2866 close(fd); 2867 return ret; 2868 } 2869 2870 /* 2871 * Set discarding of RAM as not broken for this group if the driver knows 2872 * the device operates compatibly with discarding. Setting must be 2873 * consistent per group, but since compatibility is really only possible 2874 * with mdev currently, we expect singleton groups. 2875 */ 2876 if (vbasedev->ram_block_discard_allowed != 2877 group->ram_block_discard_allowed) { 2878 if (!QLIST_EMPTY(&group->device_list)) { 2879 error_setg(errp, "Inconsistent setting of support for discarding " 2880 "RAM (e.g., balloon) within group"); 2881 close(fd); 2882 return -1; 2883 } 2884 2885 if (!group->ram_block_discard_allowed) { 2886 group->ram_block_discard_allowed = true; 2887 vfio_ram_block_discard_disable(group->container, false); 2888 } 2889 } 2890 2891 vbasedev->fd = fd; 2892 vbasedev->group = group; 2893 QLIST_INSERT_HEAD(&group->device_list, vbasedev, next); 2894 2895 vbasedev->num_irqs = dev_info.num_irqs; 2896 vbasedev->num_regions = dev_info.num_regions; 2897 vbasedev->flags = dev_info.flags; 2898 2899 trace_vfio_get_device(name, dev_info.flags, dev_info.num_regions, 2900 dev_info.num_irqs); 2901 2902 vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET); 2903 return 0; 2904 } 2905 2906 void vfio_put_base_device(VFIODevice *vbasedev) 2907 { 2908 if (!vbasedev->group) { 2909 return; 2910 } 2911 QLIST_REMOVE(vbasedev, next); 2912 vbasedev->group = NULL; 2913 trace_vfio_put_base_device(vbasedev->fd); 2914 close(vbasedev->fd); 2915 } 2916 2917 int vfio_get_region_info(VFIODevice *vbasedev, int index, 2918 struct vfio_region_info **info) 2919 { 2920 size_t argsz = sizeof(struct vfio_region_info); 2921 2922 *info = g_malloc0(argsz); 2923 2924 (*info)->index = index; 2925 retry: 2926 (*info)->argsz = argsz; 2927 2928 if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) { 2929 g_free(*info); 2930 *info = NULL; 2931 return -errno; 2932 } 2933 2934 if ((*info)->argsz > argsz) { 2935 argsz = (*info)->argsz; 2936 *info = g_realloc(*info, argsz); 2937 2938 goto retry; 2939 } 2940 2941 return 0; 2942 } 2943 2944 int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type, 2945 uint32_t subtype, struct vfio_region_info **info) 2946 { 2947 int i; 2948 2949 for (i = 0; i < vbasedev->num_regions; i++) { 2950 struct vfio_info_cap_header *hdr; 2951 struct vfio_region_info_cap_type *cap_type; 2952 2953 if (vfio_get_region_info(vbasedev, i, info)) { 2954 continue; 2955 } 2956 2957 hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE); 2958 if (!hdr) { 2959 g_free(*info); 2960 continue; 2961 } 2962 2963 cap_type = container_of(hdr, struct vfio_region_info_cap_type, header); 2964 2965 trace_vfio_get_dev_region(vbasedev->name, i, 2966 cap_type->type, cap_type->subtype); 2967 2968 if (cap_type->type == type && cap_type->subtype == subtype) { 2969 return 0; 2970 } 2971 2972 g_free(*info); 2973 } 2974 2975 *info = NULL; 2976 return -ENODEV; 2977 } 2978 2979 bool vfio_has_region_cap(VFIODevice *vbasedev, int region, uint16_t cap_type) 2980 { 2981 struct vfio_region_info *info = NULL; 2982 bool ret = false; 2983 2984 if (!vfio_get_region_info(vbasedev, region, &info)) { 2985 if (vfio_get_region_info_cap(info, cap_type)) { 2986 ret = true; 2987 } 2988 g_free(info); 2989 } 2990 2991 return ret; 2992 } 2993 2994 /* 2995 * Interfaces for IBM EEH (Enhanced Error Handling) 2996 */ 2997 static bool vfio_eeh_container_ok(VFIOContainer *container) 2998 { 2999 /* 3000 * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO 3001 * implementation is broken if there are multiple groups in a 3002 * container. The hardware works in units of Partitionable 3003 * Endpoints (== IOMMU groups) and the EEH operations naively 3004 * iterate across all groups in the container, without any logic 3005 * to make sure the groups have their state synchronized. For 3006 * certain operations (ENABLE) that might be ok, until an error 3007 * occurs, but for others (GET_STATE) it's clearly broken. 3008 */ 3009 3010 /* 3011 * XXX Once fixed kernels exist, test for them here 3012 */ 3013 3014 if (QLIST_EMPTY(&container->group_list)) { 3015 return false; 3016 } 3017 3018 if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) { 3019 return false; 3020 } 3021 3022 return true; 3023 } 3024 3025 static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op) 3026 { 3027 struct vfio_eeh_pe_op pe_op = { 3028 .argsz = sizeof(pe_op), 3029 .op = op, 3030 }; 3031 int ret; 3032 3033 if (!vfio_eeh_container_ok(container)) { 3034 error_report("vfio/eeh: EEH_PE_OP 0x%x: " 3035 "kernel requires a container with exactly one group", op); 3036 return -EPERM; 3037 } 3038 3039 ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op); 3040 if (ret < 0) { 3041 error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op); 3042 return -errno; 3043 } 3044 3045 return ret; 3046 } 3047 3048 static VFIOContainer *vfio_eeh_as_container(AddressSpace *as) 3049 { 3050 VFIOAddressSpace *space = vfio_get_address_space(as); 3051 VFIOContainer *container = NULL; 3052 3053 if (QLIST_EMPTY(&space->containers)) { 3054 /* No containers to act on */ 3055 goto out; 3056 } 3057 3058 container = QLIST_FIRST(&space->containers); 3059 3060 if (QLIST_NEXT(container, next)) { 3061 /* We don't yet have logic to synchronize EEH state across 3062 * multiple containers */ 3063 container = NULL; 3064 goto out; 3065 } 3066 3067 out: 3068 vfio_put_address_space(space); 3069 return container; 3070 } 3071 3072 bool vfio_eeh_as_ok(AddressSpace *as) 3073 { 3074 VFIOContainer *container = vfio_eeh_as_container(as); 3075 3076 return (container != NULL) && vfio_eeh_container_ok(container); 3077 } 3078 3079 int vfio_eeh_as_op(AddressSpace *as, uint32_t op) 3080 { 3081 VFIOContainer *container = vfio_eeh_as_container(as); 3082 3083 if (!container) { 3084 return -ENODEV; 3085 } 3086 return vfio_eeh_container_op(container, op); 3087 } 3088