1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/vfio.h" 30 #include "exec/address-spaces.h" 31 #include "exec/memory.h" 32 #include "exec/ram_addr.h" 33 #include "hw/hw.h" 34 #include "qemu/error-report.h" 35 #include "qemu/main-loop.h" 36 #include "qemu/range.h" 37 #include "sysemu/kvm.h" 38 #include "sysemu/reset.h" 39 #include "sysemu/runstate.h" 40 #include "trace.h" 41 #include "qapi/error.h" 42 #include "migration/migration.h" 43 44 VFIOGroupList vfio_group_list = 45 QLIST_HEAD_INITIALIZER(vfio_group_list); 46 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 47 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 48 49 #ifdef CONFIG_KVM 50 /* 51 * We have a single VFIO pseudo device per KVM VM. Once created it lives 52 * for the life of the VM. Closing the file descriptor only drops our 53 * reference to it and the device's reference to kvm. Therefore once 54 * initialized, this file descriptor is only released on QEMU exit and 55 * we'll re-use it should another vfio device be attached before then. 56 */ 57 static int vfio_kvm_device_fd = -1; 58 #endif 59 60 /* 61 * Common VFIO interrupt disable 62 */ 63 void vfio_disable_irqindex(VFIODevice *vbasedev, int index) 64 { 65 struct vfio_irq_set irq_set = { 66 .argsz = sizeof(irq_set), 67 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 68 .index = index, 69 .start = 0, 70 .count = 0, 71 }; 72 73 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 74 } 75 76 void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index) 77 { 78 struct vfio_irq_set irq_set = { 79 .argsz = sizeof(irq_set), 80 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, 81 .index = index, 82 .start = 0, 83 .count = 1, 84 }; 85 86 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 87 } 88 89 void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index) 90 { 91 struct vfio_irq_set irq_set = { 92 .argsz = sizeof(irq_set), 93 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK, 94 .index = index, 95 .start = 0, 96 .count = 1, 97 }; 98 99 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 100 } 101 102 static inline const char *action_to_str(int action) 103 { 104 switch (action) { 105 case VFIO_IRQ_SET_ACTION_MASK: 106 return "MASK"; 107 case VFIO_IRQ_SET_ACTION_UNMASK: 108 return "UNMASK"; 109 case VFIO_IRQ_SET_ACTION_TRIGGER: 110 return "TRIGGER"; 111 default: 112 return "UNKNOWN ACTION"; 113 } 114 } 115 116 static const char *index_to_str(VFIODevice *vbasedev, int index) 117 { 118 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) { 119 return NULL; 120 } 121 122 switch (index) { 123 case VFIO_PCI_INTX_IRQ_INDEX: 124 return "INTX"; 125 case VFIO_PCI_MSI_IRQ_INDEX: 126 return "MSI"; 127 case VFIO_PCI_MSIX_IRQ_INDEX: 128 return "MSIX"; 129 case VFIO_PCI_ERR_IRQ_INDEX: 130 return "ERR"; 131 case VFIO_PCI_REQ_IRQ_INDEX: 132 return "REQ"; 133 default: 134 return NULL; 135 } 136 } 137 138 static int vfio_ram_block_discard_disable(VFIOContainer *container, bool state) 139 { 140 switch (container->iommu_type) { 141 case VFIO_TYPE1v2_IOMMU: 142 case VFIO_TYPE1_IOMMU: 143 /* 144 * We support coordinated discarding of RAM via the RamDiscardManager. 145 */ 146 return ram_block_uncoordinated_discard_disable(state); 147 default: 148 /* 149 * VFIO_SPAPR_TCE_IOMMU most probably works just fine with 150 * RamDiscardManager, however, it is completely untested. 151 * 152 * VFIO_SPAPR_TCE_v2_IOMMU with "DMA memory preregistering" does 153 * completely the opposite of managing mapping/pinning dynamically as 154 * required by RamDiscardManager. We would have to special-case sections 155 * with a RamDiscardManager. 156 */ 157 return ram_block_discard_disable(state); 158 } 159 } 160 161 int vfio_set_irq_signaling(VFIODevice *vbasedev, int index, int subindex, 162 int action, int fd, Error **errp) 163 { 164 struct vfio_irq_set *irq_set; 165 int argsz, ret = 0; 166 const char *name; 167 int32_t *pfd; 168 169 argsz = sizeof(*irq_set) + sizeof(*pfd); 170 171 irq_set = g_malloc0(argsz); 172 irq_set->argsz = argsz; 173 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | action; 174 irq_set->index = index; 175 irq_set->start = subindex; 176 irq_set->count = 1; 177 pfd = (int32_t *)&irq_set->data; 178 *pfd = fd; 179 180 if (ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set)) { 181 ret = -errno; 182 } 183 g_free(irq_set); 184 185 if (!ret) { 186 return 0; 187 } 188 189 error_setg_errno(errp, -ret, "VFIO_DEVICE_SET_IRQS failure"); 190 191 name = index_to_str(vbasedev, index); 192 if (name) { 193 error_prepend(errp, "%s-%d: ", name, subindex); 194 } else { 195 error_prepend(errp, "index %d-%d: ", index, subindex); 196 } 197 error_prepend(errp, 198 "Failed to %s %s eventfd signaling for interrupt ", 199 fd < 0 ? "tear down" : "set up", action_to_str(action)); 200 return ret; 201 } 202 203 /* 204 * IO Port/MMIO - Beware of the endians, VFIO is always little endian 205 */ 206 void vfio_region_write(void *opaque, hwaddr addr, 207 uint64_t data, unsigned size) 208 { 209 VFIORegion *region = opaque; 210 VFIODevice *vbasedev = region->vbasedev; 211 union { 212 uint8_t byte; 213 uint16_t word; 214 uint32_t dword; 215 uint64_t qword; 216 } buf; 217 218 switch (size) { 219 case 1: 220 buf.byte = data; 221 break; 222 case 2: 223 buf.word = cpu_to_le16(data); 224 break; 225 case 4: 226 buf.dword = cpu_to_le32(data); 227 break; 228 case 8: 229 buf.qword = cpu_to_le64(data); 230 break; 231 default: 232 hw_error("vfio: unsupported write size, %u bytes", size); 233 break; 234 } 235 236 if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { 237 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64 238 ",%d) failed: %m", 239 __func__, vbasedev->name, region->nr, 240 addr, data, size); 241 } 242 243 trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size); 244 245 /* 246 * A read or write to a BAR always signals an INTx EOI. This will 247 * do nothing if not pending (including not in INTx mode). We assume 248 * that a BAR access is in response to an interrupt and that BAR 249 * accesses will service the interrupt. Unfortunately, we don't know 250 * which access will service the interrupt, so we're potentially 251 * getting quite a few host interrupts per guest interrupt. 252 */ 253 vbasedev->ops->vfio_eoi(vbasedev); 254 } 255 256 uint64_t vfio_region_read(void *opaque, 257 hwaddr addr, unsigned size) 258 { 259 VFIORegion *region = opaque; 260 VFIODevice *vbasedev = region->vbasedev; 261 union { 262 uint8_t byte; 263 uint16_t word; 264 uint32_t dword; 265 uint64_t qword; 266 } buf; 267 uint64_t data = 0; 268 269 if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { 270 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m", 271 __func__, vbasedev->name, region->nr, 272 addr, size); 273 return (uint64_t)-1; 274 } 275 switch (size) { 276 case 1: 277 data = buf.byte; 278 break; 279 case 2: 280 data = le16_to_cpu(buf.word); 281 break; 282 case 4: 283 data = le32_to_cpu(buf.dword); 284 break; 285 case 8: 286 data = le64_to_cpu(buf.qword); 287 break; 288 default: 289 hw_error("vfio: unsupported read size, %u bytes", size); 290 break; 291 } 292 293 trace_vfio_region_read(vbasedev->name, region->nr, addr, size, data); 294 295 /* Same as write above */ 296 vbasedev->ops->vfio_eoi(vbasedev); 297 298 return data; 299 } 300 301 const MemoryRegionOps vfio_region_ops = { 302 .read = vfio_region_read, 303 .write = vfio_region_write, 304 .endianness = DEVICE_LITTLE_ENDIAN, 305 .valid = { 306 .min_access_size = 1, 307 .max_access_size = 8, 308 }, 309 .impl = { 310 .min_access_size = 1, 311 .max_access_size = 8, 312 }, 313 }; 314 315 /* 316 * Device state interfaces 317 */ 318 319 bool vfio_mig_active(void) 320 { 321 VFIOGroup *group; 322 VFIODevice *vbasedev; 323 324 if (QLIST_EMPTY(&vfio_group_list)) { 325 return false; 326 } 327 328 QLIST_FOREACH(group, &vfio_group_list, next) { 329 QLIST_FOREACH(vbasedev, &group->device_list, next) { 330 if (vbasedev->migration_blocker) { 331 return false; 332 } 333 } 334 } 335 return true; 336 } 337 338 static bool vfio_devices_all_dirty_tracking(VFIOContainer *container) 339 { 340 VFIOGroup *group; 341 VFIODevice *vbasedev; 342 MigrationState *ms = migrate_get_current(); 343 344 if (!migration_is_setup_or_active(ms->state)) { 345 return false; 346 } 347 348 QLIST_FOREACH(group, &container->group_list, container_next) { 349 QLIST_FOREACH(vbasedev, &group->device_list, next) { 350 VFIOMigration *migration = vbasedev->migration; 351 352 if (!migration) { 353 return false; 354 } 355 356 if ((vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF) 357 && (migration->device_state & VFIO_DEVICE_STATE_RUNNING)) { 358 return false; 359 } 360 } 361 } 362 return true; 363 } 364 365 static bool vfio_devices_all_running_and_saving(VFIOContainer *container) 366 { 367 VFIOGroup *group; 368 VFIODevice *vbasedev; 369 MigrationState *ms = migrate_get_current(); 370 371 if (!migration_is_setup_or_active(ms->state)) { 372 return false; 373 } 374 375 QLIST_FOREACH(group, &container->group_list, container_next) { 376 QLIST_FOREACH(vbasedev, &group->device_list, next) { 377 VFIOMigration *migration = vbasedev->migration; 378 379 if (!migration) { 380 return false; 381 } 382 383 if ((migration->device_state & VFIO_DEVICE_STATE_SAVING) && 384 (migration->device_state & VFIO_DEVICE_STATE_RUNNING)) { 385 continue; 386 } else { 387 return false; 388 } 389 } 390 } 391 return true; 392 } 393 394 static int vfio_dma_unmap_bitmap(VFIOContainer *container, 395 hwaddr iova, ram_addr_t size, 396 IOMMUTLBEntry *iotlb) 397 { 398 struct vfio_iommu_type1_dma_unmap *unmap; 399 struct vfio_bitmap *bitmap; 400 uint64_t pages = REAL_HOST_PAGE_ALIGN(size) / qemu_real_host_page_size; 401 int ret; 402 403 unmap = g_malloc0(sizeof(*unmap) + sizeof(*bitmap)); 404 405 unmap->argsz = sizeof(*unmap) + sizeof(*bitmap); 406 unmap->iova = iova; 407 unmap->size = size; 408 unmap->flags |= VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP; 409 bitmap = (struct vfio_bitmap *)&unmap->data; 410 411 /* 412 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of 413 * qemu_real_host_page_size to mark those dirty. Hence set bitmap_pgsize 414 * to qemu_real_host_page_size. 415 */ 416 417 bitmap->pgsize = qemu_real_host_page_size; 418 bitmap->size = ROUND_UP(pages, sizeof(__u64) * BITS_PER_BYTE) / 419 BITS_PER_BYTE; 420 421 if (bitmap->size > container->max_dirty_bitmap_size) { 422 error_report("UNMAP: Size of bitmap too big 0x%"PRIx64, 423 (uint64_t)bitmap->size); 424 ret = -E2BIG; 425 goto unmap_exit; 426 } 427 428 bitmap->data = g_try_malloc0(bitmap->size); 429 if (!bitmap->data) { 430 ret = -ENOMEM; 431 goto unmap_exit; 432 } 433 434 ret = ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, unmap); 435 if (!ret) { 436 cpu_physical_memory_set_dirty_lebitmap((unsigned long *)bitmap->data, 437 iotlb->translated_addr, pages); 438 } else { 439 error_report("VFIO_UNMAP_DMA with DIRTY_BITMAP : %m"); 440 } 441 442 g_free(bitmap->data); 443 unmap_exit: 444 g_free(unmap); 445 return ret; 446 } 447 448 /* 449 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 450 */ 451 static int vfio_dma_unmap(VFIOContainer *container, 452 hwaddr iova, ram_addr_t size, 453 IOMMUTLBEntry *iotlb) 454 { 455 struct vfio_iommu_type1_dma_unmap unmap = { 456 .argsz = sizeof(unmap), 457 .flags = 0, 458 .iova = iova, 459 .size = size, 460 }; 461 462 if (iotlb && container->dirty_pages_supported && 463 vfio_devices_all_running_and_saving(container)) { 464 return vfio_dma_unmap_bitmap(container, iova, size, iotlb); 465 } 466 467 while (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) { 468 /* 469 * The type1 backend has an off-by-one bug in the kernel (71a7d3d78e3c 470 * v4.15) where an overflow in its wrap-around check prevents us from 471 * unmapping the last page of the address space. Test for the error 472 * condition and re-try the unmap excluding the last page. The 473 * expectation is that we've never mapped the last page anyway and this 474 * unmap request comes via vIOMMU support which also makes it unlikely 475 * that this page is used. This bug was introduced well after type1 v2 476 * support was introduced, so we shouldn't need to test for v1. A fix 477 * is queued for kernel v5.0 so this workaround can be removed once 478 * affected kernels are sufficiently deprecated. 479 */ 480 if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) && 481 container->iommu_type == VFIO_TYPE1v2_IOMMU) { 482 trace_vfio_dma_unmap_overflow_workaround(); 483 unmap.size -= 1ULL << ctz64(container->pgsizes); 484 continue; 485 } 486 error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno)); 487 return -errno; 488 } 489 490 return 0; 491 } 492 493 static int vfio_dma_map(VFIOContainer *container, hwaddr iova, 494 ram_addr_t size, void *vaddr, bool readonly) 495 { 496 struct vfio_iommu_type1_dma_map map = { 497 .argsz = sizeof(map), 498 .flags = VFIO_DMA_MAP_FLAG_READ, 499 .vaddr = (__u64)(uintptr_t)vaddr, 500 .iova = iova, 501 .size = size, 502 }; 503 504 if (!readonly) { 505 map.flags |= VFIO_DMA_MAP_FLAG_WRITE; 506 } 507 508 /* 509 * Try the mapping, if it fails with EBUSY, unmap the region and try 510 * again. This shouldn't be necessary, but we sometimes see it in 511 * the VGA ROM space. 512 */ 513 if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 || 514 (errno == EBUSY && vfio_dma_unmap(container, iova, size, NULL) == 0 && 515 ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) { 516 return 0; 517 } 518 519 error_report("VFIO_MAP_DMA failed: %s", strerror(errno)); 520 return -errno; 521 } 522 523 static void vfio_host_win_add(VFIOContainer *container, 524 hwaddr min_iova, hwaddr max_iova, 525 uint64_t iova_pgsizes) 526 { 527 VFIOHostDMAWindow *hostwin; 528 529 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 530 if (ranges_overlap(hostwin->min_iova, 531 hostwin->max_iova - hostwin->min_iova + 1, 532 min_iova, 533 max_iova - min_iova + 1)) { 534 hw_error("%s: Overlapped IOMMU are not enabled", __func__); 535 } 536 } 537 538 hostwin = g_malloc0(sizeof(*hostwin)); 539 540 hostwin->min_iova = min_iova; 541 hostwin->max_iova = max_iova; 542 hostwin->iova_pgsizes = iova_pgsizes; 543 QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next); 544 } 545 546 static int vfio_host_win_del(VFIOContainer *container, hwaddr min_iova, 547 hwaddr max_iova) 548 { 549 VFIOHostDMAWindow *hostwin; 550 551 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 552 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) { 553 QLIST_REMOVE(hostwin, hostwin_next); 554 return 0; 555 } 556 } 557 558 return -1; 559 } 560 561 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 562 { 563 return (!memory_region_is_ram(section->mr) && 564 !memory_region_is_iommu(section->mr)) || 565 memory_region_is_protected(section->mr) || 566 /* 567 * Sizing an enabled 64-bit BAR can cause spurious mappings to 568 * addresses in the upper part of the 64-bit address space. These 569 * are never accessed by the CPU and beyond the address width of 570 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 571 */ 572 section->offset_within_address_space & (1ULL << 63); 573 } 574 575 /* Called with rcu_read_lock held. */ 576 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, 577 ram_addr_t *ram_addr, bool *read_only) 578 { 579 MemoryRegion *mr; 580 hwaddr xlat; 581 hwaddr len = iotlb->addr_mask + 1; 582 bool writable = iotlb->perm & IOMMU_WO; 583 584 /* 585 * The IOMMU TLB entry we have just covers translation through 586 * this IOMMU to its immediate target. We need to translate 587 * it the rest of the way through to memory. 588 */ 589 mr = address_space_translate(&address_space_memory, 590 iotlb->translated_addr, 591 &xlat, &len, writable, 592 MEMTXATTRS_UNSPECIFIED); 593 if (!memory_region_is_ram(mr)) { 594 error_report("iommu map to non memory area %"HWADDR_PRIx"", 595 xlat); 596 return false; 597 } else if (memory_region_has_ram_discard_manager(mr)) { 598 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(mr); 599 MemoryRegionSection tmp = { 600 .mr = mr, 601 .offset_within_region = xlat, 602 .size = int128_make64(len), 603 }; 604 605 /* 606 * Malicious VMs can map memory into the IOMMU, which is expected 607 * to remain discarded. vfio will pin all pages, populating memory. 608 * Disallow that. vmstate priorities make sure any RamDiscardManager 609 * were already restored before IOMMUs are restored. 610 */ 611 if (!ram_discard_manager_is_populated(rdm, &tmp)) { 612 error_report("iommu map to discarded memory (e.g., unplugged via" 613 " virtio-mem): %"HWADDR_PRIx"", 614 iotlb->translated_addr); 615 return false; 616 } 617 618 /* 619 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The 620 * pages will remain pinned inside vfio until unmapped, resulting in a 621 * higher memory consumption than expected. If memory would get 622 * populated again later, there would be an inconsistency between pages 623 * pinned by vfio and pages seen by QEMU. This is the case until 624 * unmapped from the IOMMU (e.g., during device reset). 625 * 626 * With malicious guests, we really only care about pinning more memory 627 * than expected. RLIMIT_MEMLOCK set for the user/process can never be 628 * exceeded and can be used to mitigate this problem. 629 */ 630 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of" 631 " RAM (e.g., virtio-mem) works, however, malicious" 632 " guests can trigger pinning of more memory than" 633 " intended via an IOMMU. It's possible to mitigate " 634 " by setting/adjusting RLIMIT_MEMLOCK."); 635 } 636 637 /* 638 * Translation truncates length to the IOMMU page size, 639 * check that it did not truncate too much. 640 */ 641 if (len & iotlb->addr_mask) { 642 error_report("iommu has granularity incompatible with target AS"); 643 return false; 644 } 645 646 if (vaddr) { 647 *vaddr = memory_region_get_ram_ptr(mr) + xlat; 648 } 649 650 if (ram_addr) { 651 *ram_addr = memory_region_get_ram_addr(mr) + xlat; 652 } 653 654 if (read_only) { 655 *read_only = !writable || mr->readonly; 656 } 657 658 return true; 659 } 660 661 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 662 { 663 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 664 VFIOContainer *container = giommu->container; 665 hwaddr iova = iotlb->iova + giommu->iommu_offset; 666 void *vaddr; 667 int ret; 668 669 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 670 iova, iova + iotlb->addr_mask); 671 672 if (iotlb->target_as != &address_space_memory) { 673 error_report("Wrong target AS \"%s\", only system memory is allowed", 674 iotlb->target_as->name ? iotlb->target_as->name : "none"); 675 return; 676 } 677 678 rcu_read_lock(); 679 680 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 681 bool read_only; 682 683 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only)) { 684 goto out; 685 } 686 /* 687 * vaddr is only valid until rcu_read_unlock(). But after 688 * vfio_dma_map has set up the mapping the pages will be 689 * pinned by the kernel. This makes sure that the RAM backend 690 * of vaddr will always be there, even if the memory object is 691 * destroyed and its backing memory munmap-ed. 692 */ 693 ret = vfio_dma_map(container, iova, 694 iotlb->addr_mask + 1, vaddr, 695 read_only); 696 if (ret) { 697 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 698 "0x%"HWADDR_PRIx", %p) = %d (%m)", 699 container, iova, 700 iotlb->addr_mask + 1, vaddr, ret); 701 } 702 } else { 703 ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1, iotlb); 704 if (ret) { 705 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 706 "0x%"HWADDR_PRIx") = %d (%m)", 707 container, iova, 708 iotlb->addr_mask + 1, ret); 709 } 710 } 711 out: 712 rcu_read_unlock(); 713 } 714 715 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl, 716 MemoryRegionSection *section) 717 { 718 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 719 listener); 720 const hwaddr size = int128_get64(section->size); 721 const hwaddr iova = section->offset_within_address_space; 722 int ret; 723 724 /* Unmap with a single call. */ 725 ret = vfio_dma_unmap(vrdl->container, iova, size , NULL); 726 if (ret) { 727 error_report("%s: vfio_dma_unmap() failed: %s", __func__, 728 strerror(-ret)); 729 } 730 } 731 732 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl, 733 MemoryRegionSection *section) 734 { 735 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener, 736 listener); 737 const hwaddr end = section->offset_within_region + 738 int128_get64(section->size); 739 hwaddr start, next, iova; 740 void *vaddr; 741 int ret; 742 743 /* 744 * Map in (aligned within memory region) minimum granularity, so we can 745 * unmap in minimum granularity later. 746 */ 747 for (start = section->offset_within_region; start < end; start = next) { 748 next = ROUND_UP(start + 1, vrdl->granularity); 749 next = MIN(next, end); 750 751 iova = start - section->offset_within_region + 752 section->offset_within_address_space; 753 vaddr = memory_region_get_ram_ptr(section->mr) + start; 754 755 ret = vfio_dma_map(vrdl->container, iova, next - start, 756 vaddr, section->readonly); 757 if (ret) { 758 /* Rollback */ 759 vfio_ram_discard_notify_discard(rdl, section); 760 return ret; 761 } 762 } 763 return 0; 764 } 765 766 static void vfio_register_ram_discard_listener(VFIOContainer *container, 767 MemoryRegionSection *section) 768 { 769 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 770 VFIORamDiscardListener *vrdl; 771 772 /* Ignore some corner cases not relevant in practice. */ 773 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE)); 774 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space, 775 TARGET_PAGE_SIZE)); 776 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE)); 777 778 vrdl = g_new0(VFIORamDiscardListener, 1); 779 vrdl->container = container; 780 vrdl->mr = section->mr; 781 vrdl->offset_within_address_space = section->offset_within_address_space; 782 vrdl->size = int128_get64(section->size); 783 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm, 784 section->mr); 785 786 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity)); 787 g_assert(container->pgsizes && 788 vrdl->granularity >= 1ULL << ctz64(container->pgsizes)); 789 790 ram_discard_listener_init(&vrdl->listener, 791 vfio_ram_discard_notify_populate, 792 vfio_ram_discard_notify_discard, true); 793 ram_discard_manager_register_listener(rdm, &vrdl->listener, section); 794 QLIST_INSERT_HEAD(&container->vrdl_list, vrdl, next); 795 796 /* 797 * Sanity-check if we have a theoretically problematic setup where we could 798 * exceed the maximum number of possible DMA mappings over time. We assume 799 * that each mapped section in the same address space as a RamDiscardManager 800 * section consumes exactly one DMA mapping, with the exception of 801 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections 802 * in the same address space as RamDiscardManager sections. 803 * 804 * We assume that each section in the address space consumes one memslot. 805 * We take the number of KVM memory slots as a best guess for the maximum 806 * number of sections in the address space we could have over time, 807 * also consuming DMA mappings. 808 */ 809 if (container->dma_max_mappings) { 810 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512; 811 812 #ifdef CONFIG_KVM 813 if (kvm_enabled()) { 814 max_memslots = kvm_get_max_memslots(); 815 } 816 #endif 817 818 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 819 hwaddr start, end; 820 821 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space, 822 vrdl->granularity); 823 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size, 824 vrdl->granularity); 825 vrdl_mappings += (end - start) / vrdl->granularity; 826 vrdl_count++; 827 } 828 829 if (vrdl_mappings + max_memslots - vrdl_count > 830 container->dma_max_mappings) { 831 warn_report("%s: possibly running out of DMA mappings. E.g., try" 832 " increasing the 'block-size' of virtio-mem devies." 833 " Maximum possible DMA mappings: %d, Maximum possible" 834 " memslots: %d", __func__, container->dma_max_mappings, 835 max_memslots); 836 } 837 } 838 } 839 840 static void vfio_unregister_ram_discard_listener(VFIOContainer *container, 841 MemoryRegionSection *section) 842 { 843 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 844 VFIORamDiscardListener *vrdl = NULL; 845 846 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 847 if (vrdl->mr == section->mr && 848 vrdl->offset_within_address_space == 849 section->offset_within_address_space) { 850 break; 851 } 852 } 853 854 if (!vrdl) { 855 hw_error("vfio: Trying to unregister missing RAM discard listener"); 856 } 857 858 ram_discard_manager_unregister_listener(rdm, &vrdl->listener); 859 QLIST_REMOVE(vrdl, next); 860 g_free(vrdl); 861 } 862 863 static void vfio_listener_region_add(MemoryListener *listener, 864 MemoryRegionSection *section) 865 { 866 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 867 hwaddr iova, end; 868 Int128 llend, llsize; 869 void *vaddr; 870 int ret; 871 VFIOHostDMAWindow *hostwin; 872 bool hostwin_found; 873 Error *err = NULL; 874 875 if (vfio_listener_skipped_section(section)) { 876 trace_vfio_listener_region_add_skip( 877 section->offset_within_address_space, 878 section->offset_within_address_space + 879 int128_get64(int128_sub(section->size, int128_one()))); 880 return; 881 } 882 883 if (unlikely((section->offset_within_address_space & 884 ~qemu_real_host_page_mask) != 885 (section->offset_within_region & ~qemu_real_host_page_mask))) { 886 error_report("%s received unaligned region", __func__); 887 return; 888 } 889 890 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 891 llend = int128_make64(section->offset_within_address_space); 892 llend = int128_add(llend, section->size); 893 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask)); 894 895 if (int128_ge(int128_make64(iova), llend)) { 896 if (memory_region_is_ram_device(section->mr)) { 897 trace_vfio_listener_region_add_no_dma_map( 898 memory_region_name(section->mr), 899 section->offset_within_address_space, 900 int128_getlo(section->size), 901 qemu_real_host_page_size); 902 } 903 return; 904 } 905 end = int128_get64(int128_sub(llend, int128_one())); 906 907 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 908 hwaddr pgsize = 0; 909 910 /* For now intersections are not allowed, we may relax this later */ 911 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 912 if (ranges_overlap(hostwin->min_iova, 913 hostwin->max_iova - hostwin->min_iova + 1, 914 section->offset_within_address_space, 915 int128_get64(section->size))) { 916 error_setg(&err, 917 "region [0x%"PRIx64",0x%"PRIx64"] overlaps with existing" 918 "host DMA window [0x%"PRIx64",0x%"PRIx64"]", 919 section->offset_within_address_space, 920 section->offset_within_address_space + 921 int128_get64(section->size) - 1, 922 hostwin->min_iova, hostwin->max_iova); 923 goto fail; 924 } 925 } 926 927 ret = vfio_spapr_create_window(container, section, &pgsize); 928 if (ret) { 929 error_setg_errno(&err, -ret, "Failed to create SPAPR window"); 930 goto fail; 931 } 932 933 vfio_host_win_add(container, section->offset_within_address_space, 934 section->offset_within_address_space + 935 int128_get64(section->size) - 1, pgsize); 936 #ifdef CONFIG_KVM 937 if (kvm_enabled()) { 938 VFIOGroup *group; 939 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 940 struct kvm_vfio_spapr_tce param; 941 struct kvm_device_attr attr = { 942 .group = KVM_DEV_VFIO_GROUP, 943 .attr = KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE, 944 .addr = (uint64_t)(unsigned long)¶m, 945 }; 946 947 if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_TCE_FD, 948 ¶m.tablefd)) { 949 QLIST_FOREACH(group, &container->group_list, container_next) { 950 param.groupfd = group->fd; 951 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 952 error_report("vfio: failed to setup fd %d " 953 "for a group with fd %d: %s", 954 param.tablefd, param.groupfd, 955 strerror(errno)); 956 return; 957 } 958 trace_vfio_spapr_group_attach(param.groupfd, param.tablefd); 959 } 960 } 961 } 962 #endif 963 } 964 965 hostwin_found = false; 966 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 967 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 968 hostwin_found = true; 969 break; 970 } 971 } 972 973 if (!hostwin_found) { 974 error_setg(&err, "Container %p can't map guest IOVA region" 975 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container, iova, end); 976 goto fail; 977 } 978 979 memory_region_ref(section->mr); 980 981 if (memory_region_is_iommu(section->mr)) { 982 VFIOGuestIOMMU *giommu; 983 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 984 int iommu_idx; 985 986 trace_vfio_listener_region_add_iommu(iova, end); 987 /* 988 * FIXME: For VFIO iommu types which have KVM acceleration to 989 * avoid bouncing all map/unmaps through qemu this way, this 990 * would be the right place to wire that up (tell the KVM 991 * device emulation the VFIO iommu handles to use). 992 */ 993 giommu = g_malloc0(sizeof(*giommu)); 994 giommu->iommu = iommu_mr; 995 giommu->iommu_offset = section->offset_within_address_space - 996 section->offset_within_region; 997 giommu->container = container; 998 llend = int128_add(int128_make64(section->offset_within_region), 999 section->size); 1000 llend = int128_sub(llend, int128_one()); 1001 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 1002 MEMTXATTRS_UNSPECIFIED); 1003 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 1004 IOMMU_NOTIFIER_IOTLB_EVENTS, 1005 section->offset_within_region, 1006 int128_get64(llend), 1007 iommu_idx); 1008 1009 ret = memory_region_iommu_set_page_size_mask(giommu->iommu, 1010 container->pgsizes, 1011 &err); 1012 if (ret) { 1013 g_free(giommu); 1014 goto fail; 1015 } 1016 1017 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n, 1018 &err); 1019 if (ret) { 1020 g_free(giommu); 1021 goto fail; 1022 } 1023 QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next); 1024 memory_region_iommu_replay(giommu->iommu, &giommu->n); 1025 1026 return; 1027 } 1028 1029 /* Here we assume that memory_region_is_ram(section->mr)==true */ 1030 1031 /* 1032 * For RAM memory regions with a RamDiscardManager, we only want to map the 1033 * actually populated parts - and update the mapping whenever we're notified 1034 * about changes. 1035 */ 1036 if (memory_region_has_ram_discard_manager(section->mr)) { 1037 vfio_register_ram_discard_listener(container, section); 1038 return; 1039 } 1040 1041 vaddr = memory_region_get_ram_ptr(section->mr) + 1042 section->offset_within_region + 1043 (iova - section->offset_within_address_space); 1044 1045 trace_vfio_listener_region_add_ram(iova, end, vaddr); 1046 1047 llsize = int128_sub(llend, int128_make64(iova)); 1048 1049 if (memory_region_is_ram_device(section->mr)) { 1050 hwaddr pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 1051 1052 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 1053 trace_vfio_listener_region_add_no_dma_map( 1054 memory_region_name(section->mr), 1055 section->offset_within_address_space, 1056 int128_getlo(section->size), 1057 pgmask + 1); 1058 return; 1059 } 1060 } 1061 1062 ret = vfio_dma_map(container, iova, int128_get64(llsize), 1063 vaddr, section->readonly); 1064 if (ret) { 1065 error_setg(&err, "vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 1066 "0x%"HWADDR_PRIx", %p) = %d (%m)", 1067 container, iova, int128_get64(llsize), vaddr, ret); 1068 if (memory_region_is_ram_device(section->mr)) { 1069 /* Allow unexpected mappings not to be fatal for RAM devices */ 1070 error_report_err(err); 1071 return; 1072 } 1073 goto fail; 1074 } 1075 1076 return; 1077 1078 fail: 1079 if (memory_region_is_ram_device(section->mr)) { 1080 error_report("failed to vfio_dma_map. pci p2p may not work"); 1081 return; 1082 } 1083 /* 1084 * On the initfn path, store the first error in the container so we 1085 * can gracefully fail. Runtime, there's not much we can do other 1086 * than throw a hardware error. 1087 */ 1088 if (!container->initialized) { 1089 if (!container->error) { 1090 error_propagate_prepend(&container->error, err, 1091 "Region %s: ", 1092 memory_region_name(section->mr)); 1093 } else { 1094 error_free(err); 1095 } 1096 } else { 1097 error_report_err(err); 1098 hw_error("vfio: DMA mapping failed, unable to continue"); 1099 } 1100 } 1101 1102 static void vfio_listener_region_del(MemoryListener *listener, 1103 MemoryRegionSection *section) 1104 { 1105 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1106 hwaddr iova, end; 1107 Int128 llend, llsize; 1108 int ret; 1109 bool try_unmap = true; 1110 1111 if (vfio_listener_skipped_section(section)) { 1112 trace_vfio_listener_region_del_skip( 1113 section->offset_within_address_space, 1114 section->offset_within_address_space + 1115 int128_get64(int128_sub(section->size, int128_one()))); 1116 return; 1117 } 1118 1119 if (unlikely((section->offset_within_address_space & 1120 ~qemu_real_host_page_mask) != 1121 (section->offset_within_region & ~qemu_real_host_page_mask))) { 1122 error_report("%s received unaligned region", __func__); 1123 return; 1124 } 1125 1126 if (memory_region_is_iommu(section->mr)) { 1127 VFIOGuestIOMMU *giommu; 1128 1129 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 1130 if (MEMORY_REGION(giommu->iommu) == section->mr && 1131 giommu->n.start == section->offset_within_region) { 1132 memory_region_unregister_iommu_notifier(section->mr, 1133 &giommu->n); 1134 QLIST_REMOVE(giommu, giommu_next); 1135 g_free(giommu); 1136 break; 1137 } 1138 } 1139 1140 /* 1141 * FIXME: We assume the one big unmap below is adequate to 1142 * remove any individual page mappings in the IOMMU which 1143 * might have been copied into VFIO. This works for a page table 1144 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 1145 * That may not be true for all IOMMU types. 1146 */ 1147 } 1148 1149 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space); 1150 llend = int128_make64(section->offset_within_address_space); 1151 llend = int128_add(llend, section->size); 1152 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask)); 1153 1154 if (int128_ge(int128_make64(iova), llend)) { 1155 return; 1156 } 1157 end = int128_get64(int128_sub(llend, int128_one())); 1158 1159 llsize = int128_sub(llend, int128_make64(iova)); 1160 1161 trace_vfio_listener_region_del(iova, end); 1162 1163 if (memory_region_is_ram_device(section->mr)) { 1164 hwaddr pgmask; 1165 VFIOHostDMAWindow *hostwin; 1166 bool hostwin_found = false; 1167 1168 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 1169 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 1170 hostwin_found = true; 1171 break; 1172 } 1173 } 1174 assert(hostwin_found); /* or region_add() would have failed */ 1175 1176 pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 1177 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 1178 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1179 vfio_unregister_ram_discard_listener(container, section); 1180 /* Unregistering will trigger an unmap. */ 1181 try_unmap = false; 1182 } 1183 1184 if (try_unmap) { 1185 if (int128_eq(llsize, int128_2_64())) { 1186 /* The unmap ioctl doesn't accept a full 64-bit span. */ 1187 llsize = int128_rshift(llsize, 1); 1188 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 1189 if (ret) { 1190 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 1191 "0x%"HWADDR_PRIx") = %d (%m)", 1192 container, iova, int128_get64(llsize), ret); 1193 } 1194 iova += int128_get64(llsize); 1195 } 1196 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL); 1197 if (ret) { 1198 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 1199 "0x%"HWADDR_PRIx") = %d (%m)", 1200 container, iova, int128_get64(llsize), ret); 1201 } 1202 } 1203 1204 memory_region_unref(section->mr); 1205 1206 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 1207 vfio_spapr_remove_window(container, 1208 section->offset_within_address_space); 1209 if (vfio_host_win_del(container, 1210 section->offset_within_address_space, 1211 section->offset_within_address_space + 1212 int128_get64(section->size) - 1) < 0) { 1213 hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx, 1214 __func__, section->offset_within_address_space); 1215 } 1216 } 1217 } 1218 1219 static void vfio_set_dirty_page_tracking(VFIOContainer *container, bool start) 1220 { 1221 int ret; 1222 struct vfio_iommu_type1_dirty_bitmap dirty = { 1223 .argsz = sizeof(dirty), 1224 }; 1225 1226 if (start) { 1227 dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START; 1228 } else { 1229 dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP; 1230 } 1231 1232 ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, &dirty); 1233 if (ret) { 1234 error_report("Failed to set dirty tracking flag 0x%x errno: %d", 1235 dirty.flags, errno); 1236 } 1237 } 1238 1239 static void vfio_listener_log_global_start(MemoryListener *listener) 1240 { 1241 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1242 1243 vfio_set_dirty_page_tracking(container, true); 1244 } 1245 1246 static void vfio_listener_log_global_stop(MemoryListener *listener) 1247 { 1248 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1249 1250 vfio_set_dirty_page_tracking(container, false); 1251 } 1252 1253 static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova, 1254 uint64_t size, ram_addr_t ram_addr) 1255 { 1256 struct vfio_iommu_type1_dirty_bitmap *dbitmap; 1257 struct vfio_iommu_type1_dirty_bitmap_get *range; 1258 uint64_t pages; 1259 int ret; 1260 1261 dbitmap = g_malloc0(sizeof(*dbitmap) + sizeof(*range)); 1262 1263 dbitmap->argsz = sizeof(*dbitmap) + sizeof(*range); 1264 dbitmap->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; 1265 range = (struct vfio_iommu_type1_dirty_bitmap_get *)&dbitmap->data; 1266 range->iova = iova; 1267 range->size = size; 1268 1269 /* 1270 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of 1271 * qemu_real_host_page_size to mark those dirty. Hence set bitmap's pgsize 1272 * to qemu_real_host_page_size. 1273 */ 1274 range->bitmap.pgsize = qemu_real_host_page_size; 1275 1276 pages = REAL_HOST_PAGE_ALIGN(range->size) / qemu_real_host_page_size; 1277 range->bitmap.size = ROUND_UP(pages, sizeof(__u64) * BITS_PER_BYTE) / 1278 BITS_PER_BYTE; 1279 range->bitmap.data = g_try_malloc0(range->bitmap.size); 1280 if (!range->bitmap.data) { 1281 ret = -ENOMEM; 1282 goto err_out; 1283 } 1284 1285 ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, dbitmap); 1286 if (ret) { 1287 error_report("Failed to get dirty bitmap for iova: 0x%"PRIx64 1288 " size: 0x%"PRIx64" err: %d", (uint64_t)range->iova, 1289 (uint64_t)range->size, errno); 1290 goto err_out; 1291 } 1292 1293 cpu_physical_memory_set_dirty_lebitmap((unsigned long *)range->bitmap.data, 1294 ram_addr, pages); 1295 1296 trace_vfio_get_dirty_bitmap(container->fd, range->iova, range->size, 1297 range->bitmap.size, ram_addr); 1298 err_out: 1299 g_free(range->bitmap.data); 1300 g_free(dbitmap); 1301 1302 return ret; 1303 } 1304 1305 typedef struct { 1306 IOMMUNotifier n; 1307 VFIOGuestIOMMU *giommu; 1308 } vfio_giommu_dirty_notifier; 1309 1310 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 1311 { 1312 vfio_giommu_dirty_notifier *gdn = container_of(n, 1313 vfio_giommu_dirty_notifier, n); 1314 VFIOGuestIOMMU *giommu = gdn->giommu; 1315 VFIOContainer *container = giommu->container; 1316 hwaddr iova = iotlb->iova + giommu->iommu_offset; 1317 ram_addr_t translated_addr; 1318 1319 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask); 1320 1321 if (iotlb->target_as != &address_space_memory) { 1322 error_report("Wrong target AS \"%s\", only system memory is allowed", 1323 iotlb->target_as->name ? iotlb->target_as->name : "none"); 1324 return; 1325 } 1326 1327 rcu_read_lock(); 1328 if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) { 1329 int ret; 1330 1331 ret = vfio_get_dirty_bitmap(container, iova, iotlb->addr_mask + 1, 1332 translated_addr); 1333 if (ret) { 1334 error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", " 1335 "0x%"HWADDR_PRIx") = %d (%m)", 1336 container, iova, 1337 iotlb->addr_mask + 1, ret); 1338 } 1339 } 1340 rcu_read_unlock(); 1341 } 1342 1343 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section, 1344 void *opaque) 1345 { 1346 const hwaddr size = int128_get64(section->size); 1347 const hwaddr iova = section->offset_within_address_space; 1348 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) + 1349 section->offset_within_region; 1350 VFIORamDiscardListener *vrdl = opaque; 1351 1352 /* 1353 * Sync the whole mapped region (spanning multiple individual mappings) 1354 * in one go. 1355 */ 1356 return vfio_get_dirty_bitmap(vrdl->container, iova, size, ram_addr); 1357 } 1358 1359 static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container, 1360 MemoryRegionSection *section) 1361 { 1362 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); 1363 VFIORamDiscardListener *vrdl = NULL; 1364 1365 QLIST_FOREACH(vrdl, &container->vrdl_list, next) { 1366 if (vrdl->mr == section->mr && 1367 vrdl->offset_within_address_space == 1368 section->offset_within_address_space) { 1369 break; 1370 } 1371 } 1372 1373 if (!vrdl) { 1374 hw_error("vfio: Trying to sync missing RAM discard listener"); 1375 } 1376 1377 /* 1378 * We only want/can synchronize the bitmap for actually mapped parts - 1379 * which correspond to populated parts. Replay all populated parts. 1380 */ 1381 return ram_discard_manager_replay_populated(rdm, section, 1382 vfio_ram_discard_get_dirty_bitmap, 1383 &vrdl); 1384 } 1385 1386 static int vfio_sync_dirty_bitmap(VFIOContainer *container, 1387 MemoryRegionSection *section) 1388 { 1389 ram_addr_t ram_addr; 1390 1391 if (memory_region_is_iommu(section->mr)) { 1392 VFIOGuestIOMMU *giommu; 1393 1394 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 1395 if (MEMORY_REGION(giommu->iommu) == section->mr && 1396 giommu->n.start == section->offset_within_region) { 1397 Int128 llend; 1398 vfio_giommu_dirty_notifier gdn = { .giommu = giommu }; 1399 int idx = memory_region_iommu_attrs_to_index(giommu->iommu, 1400 MEMTXATTRS_UNSPECIFIED); 1401 1402 llend = int128_add(int128_make64(section->offset_within_region), 1403 section->size); 1404 llend = int128_sub(llend, int128_one()); 1405 1406 iommu_notifier_init(&gdn.n, 1407 vfio_iommu_map_dirty_notify, 1408 IOMMU_NOTIFIER_MAP, 1409 section->offset_within_region, 1410 int128_get64(llend), 1411 idx); 1412 memory_region_iommu_replay(giommu->iommu, &gdn.n); 1413 break; 1414 } 1415 } 1416 return 0; 1417 } else if (memory_region_has_ram_discard_manager(section->mr)) { 1418 return vfio_sync_ram_discard_listener_dirty_bitmap(container, section); 1419 } 1420 1421 ram_addr = memory_region_get_ram_addr(section->mr) + 1422 section->offset_within_region; 1423 1424 return vfio_get_dirty_bitmap(container, 1425 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space), 1426 int128_get64(section->size), ram_addr); 1427 } 1428 1429 static void vfio_listener_log_sync(MemoryListener *listener, 1430 MemoryRegionSection *section) 1431 { 1432 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 1433 1434 if (vfio_listener_skipped_section(section) || 1435 !container->dirty_pages_supported) { 1436 return; 1437 } 1438 1439 if (vfio_devices_all_dirty_tracking(container)) { 1440 vfio_sync_dirty_bitmap(container, section); 1441 } 1442 } 1443 1444 static const MemoryListener vfio_memory_listener = { 1445 .name = "vfio", 1446 .region_add = vfio_listener_region_add, 1447 .region_del = vfio_listener_region_del, 1448 .log_global_start = vfio_listener_log_global_start, 1449 .log_global_stop = vfio_listener_log_global_stop, 1450 .log_sync = vfio_listener_log_sync, 1451 }; 1452 1453 static void vfio_listener_release(VFIOContainer *container) 1454 { 1455 memory_listener_unregister(&container->listener); 1456 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 1457 memory_listener_unregister(&container->prereg_listener); 1458 } 1459 } 1460 1461 static struct vfio_info_cap_header * 1462 vfio_get_cap(void *ptr, uint32_t cap_offset, uint16_t id) 1463 { 1464 struct vfio_info_cap_header *hdr; 1465 1466 for (hdr = ptr + cap_offset; hdr != ptr; hdr = ptr + hdr->next) { 1467 if (hdr->id == id) { 1468 return hdr; 1469 } 1470 } 1471 1472 return NULL; 1473 } 1474 1475 struct vfio_info_cap_header * 1476 vfio_get_region_info_cap(struct vfio_region_info *info, uint16_t id) 1477 { 1478 if (!(info->flags & VFIO_REGION_INFO_FLAG_CAPS)) { 1479 return NULL; 1480 } 1481 1482 return vfio_get_cap((void *)info, info->cap_offset, id); 1483 } 1484 1485 static struct vfio_info_cap_header * 1486 vfio_get_iommu_type1_info_cap(struct vfio_iommu_type1_info *info, uint16_t id) 1487 { 1488 if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) { 1489 return NULL; 1490 } 1491 1492 return vfio_get_cap((void *)info, info->cap_offset, id); 1493 } 1494 1495 struct vfio_info_cap_header * 1496 vfio_get_device_info_cap(struct vfio_device_info *info, uint16_t id) 1497 { 1498 if (!(info->flags & VFIO_DEVICE_FLAGS_CAPS)) { 1499 return NULL; 1500 } 1501 1502 return vfio_get_cap((void *)info, info->cap_offset, id); 1503 } 1504 1505 bool vfio_get_info_dma_avail(struct vfio_iommu_type1_info *info, 1506 unsigned int *avail) 1507 { 1508 struct vfio_info_cap_header *hdr; 1509 struct vfio_iommu_type1_info_dma_avail *cap; 1510 1511 /* If the capability cannot be found, assume no DMA limiting */ 1512 hdr = vfio_get_iommu_type1_info_cap(info, 1513 VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL); 1514 if (hdr == NULL) { 1515 return false; 1516 } 1517 1518 if (avail != NULL) { 1519 cap = (void *) hdr; 1520 *avail = cap->avail; 1521 } 1522 1523 return true; 1524 } 1525 1526 static int vfio_setup_region_sparse_mmaps(VFIORegion *region, 1527 struct vfio_region_info *info) 1528 { 1529 struct vfio_info_cap_header *hdr; 1530 struct vfio_region_info_cap_sparse_mmap *sparse; 1531 int i, j; 1532 1533 hdr = vfio_get_region_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); 1534 if (!hdr) { 1535 return -ENODEV; 1536 } 1537 1538 sparse = container_of(hdr, struct vfio_region_info_cap_sparse_mmap, header); 1539 1540 trace_vfio_region_sparse_mmap_header(region->vbasedev->name, 1541 region->nr, sparse->nr_areas); 1542 1543 region->mmaps = g_new0(VFIOMmap, sparse->nr_areas); 1544 1545 for (i = 0, j = 0; i < sparse->nr_areas; i++) { 1546 trace_vfio_region_sparse_mmap_entry(i, sparse->areas[i].offset, 1547 sparse->areas[i].offset + 1548 sparse->areas[i].size); 1549 1550 if (sparse->areas[i].size) { 1551 region->mmaps[j].offset = sparse->areas[i].offset; 1552 region->mmaps[j].size = sparse->areas[i].size; 1553 j++; 1554 } 1555 } 1556 1557 region->nr_mmaps = j; 1558 region->mmaps = g_realloc(region->mmaps, j * sizeof(VFIOMmap)); 1559 1560 return 0; 1561 } 1562 1563 int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, 1564 int index, const char *name) 1565 { 1566 struct vfio_region_info *info; 1567 int ret; 1568 1569 ret = vfio_get_region_info(vbasedev, index, &info); 1570 if (ret) { 1571 return ret; 1572 } 1573 1574 region->vbasedev = vbasedev; 1575 region->flags = info->flags; 1576 region->size = info->size; 1577 region->fd_offset = info->offset; 1578 region->nr = index; 1579 1580 if (region->size) { 1581 region->mem = g_new0(MemoryRegion, 1); 1582 memory_region_init_io(region->mem, obj, &vfio_region_ops, 1583 region, name, region->size); 1584 1585 if (!vbasedev->no_mmap && 1586 region->flags & VFIO_REGION_INFO_FLAG_MMAP) { 1587 1588 ret = vfio_setup_region_sparse_mmaps(region, info); 1589 1590 if (ret) { 1591 region->nr_mmaps = 1; 1592 region->mmaps = g_new0(VFIOMmap, region->nr_mmaps); 1593 region->mmaps[0].offset = 0; 1594 region->mmaps[0].size = region->size; 1595 } 1596 } 1597 } 1598 1599 g_free(info); 1600 1601 trace_vfio_region_setup(vbasedev->name, index, name, 1602 region->flags, region->fd_offset, region->size); 1603 return 0; 1604 } 1605 1606 static void vfio_subregion_unmap(VFIORegion *region, int index) 1607 { 1608 trace_vfio_region_unmap(memory_region_name(®ion->mmaps[index].mem), 1609 region->mmaps[index].offset, 1610 region->mmaps[index].offset + 1611 region->mmaps[index].size - 1); 1612 memory_region_del_subregion(region->mem, ®ion->mmaps[index].mem); 1613 munmap(region->mmaps[index].mmap, region->mmaps[index].size); 1614 object_unparent(OBJECT(®ion->mmaps[index].mem)); 1615 region->mmaps[index].mmap = NULL; 1616 } 1617 1618 int vfio_region_mmap(VFIORegion *region) 1619 { 1620 int i, prot = 0; 1621 char *name; 1622 1623 if (!region->mem) { 1624 return 0; 1625 } 1626 1627 prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 1628 prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 1629 1630 for (i = 0; i < region->nr_mmaps; i++) { 1631 region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot, 1632 MAP_SHARED, region->vbasedev->fd, 1633 region->fd_offset + 1634 region->mmaps[i].offset); 1635 if (region->mmaps[i].mmap == MAP_FAILED) { 1636 int ret = -errno; 1637 1638 trace_vfio_region_mmap_fault(memory_region_name(region->mem), i, 1639 region->fd_offset + 1640 region->mmaps[i].offset, 1641 region->fd_offset + 1642 region->mmaps[i].offset + 1643 region->mmaps[i].size - 1, ret); 1644 1645 region->mmaps[i].mmap = NULL; 1646 1647 for (i--; i >= 0; i--) { 1648 vfio_subregion_unmap(region, i); 1649 } 1650 1651 return ret; 1652 } 1653 1654 name = g_strdup_printf("%s mmaps[%d]", 1655 memory_region_name(region->mem), i); 1656 memory_region_init_ram_device_ptr(®ion->mmaps[i].mem, 1657 memory_region_owner(region->mem), 1658 name, region->mmaps[i].size, 1659 region->mmaps[i].mmap); 1660 g_free(name); 1661 memory_region_add_subregion(region->mem, region->mmaps[i].offset, 1662 ®ion->mmaps[i].mem); 1663 1664 trace_vfio_region_mmap(memory_region_name(®ion->mmaps[i].mem), 1665 region->mmaps[i].offset, 1666 region->mmaps[i].offset + 1667 region->mmaps[i].size - 1); 1668 } 1669 1670 return 0; 1671 } 1672 1673 void vfio_region_unmap(VFIORegion *region) 1674 { 1675 int i; 1676 1677 if (!region->mem) { 1678 return; 1679 } 1680 1681 for (i = 0; i < region->nr_mmaps; i++) { 1682 if (region->mmaps[i].mmap) { 1683 vfio_subregion_unmap(region, i); 1684 } 1685 } 1686 } 1687 1688 void vfio_region_exit(VFIORegion *region) 1689 { 1690 int i; 1691 1692 if (!region->mem) { 1693 return; 1694 } 1695 1696 for (i = 0; i < region->nr_mmaps; i++) { 1697 if (region->mmaps[i].mmap) { 1698 memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); 1699 } 1700 } 1701 1702 trace_vfio_region_exit(region->vbasedev->name, region->nr); 1703 } 1704 1705 void vfio_region_finalize(VFIORegion *region) 1706 { 1707 int i; 1708 1709 if (!region->mem) { 1710 return; 1711 } 1712 1713 for (i = 0; i < region->nr_mmaps; i++) { 1714 if (region->mmaps[i].mmap) { 1715 munmap(region->mmaps[i].mmap, region->mmaps[i].size); 1716 object_unparent(OBJECT(®ion->mmaps[i].mem)); 1717 } 1718 } 1719 1720 object_unparent(OBJECT(region->mem)); 1721 1722 g_free(region->mem); 1723 g_free(region->mmaps); 1724 1725 trace_vfio_region_finalize(region->vbasedev->name, region->nr); 1726 1727 region->mem = NULL; 1728 region->mmaps = NULL; 1729 region->nr_mmaps = 0; 1730 region->size = 0; 1731 region->flags = 0; 1732 region->nr = 0; 1733 } 1734 1735 void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled) 1736 { 1737 int i; 1738 1739 if (!region->mem) { 1740 return; 1741 } 1742 1743 for (i = 0; i < region->nr_mmaps; i++) { 1744 if (region->mmaps[i].mmap) { 1745 memory_region_set_enabled(®ion->mmaps[i].mem, enabled); 1746 } 1747 } 1748 1749 trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem), 1750 enabled); 1751 } 1752 1753 void vfio_reset_handler(void *opaque) 1754 { 1755 VFIOGroup *group; 1756 VFIODevice *vbasedev; 1757 1758 QLIST_FOREACH(group, &vfio_group_list, next) { 1759 QLIST_FOREACH(vbasedev, &group->device_list, next) { 1760 if (vbasedev->dev->realized) { 1761 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 1762 } 1763 } 1764 } 1765 1766 QLIST_FOREACH(group, &vfio_group_list, next) { 1767 QLIST_FOREACH(vbasedev, &group->device_list, next) { 1768 if (vbasedev->dev->realized && vbasedev->needs_reset) { 1769 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 1770 } 1771 } 1772 } 1773 } 1774 1775 static void vfio_kvm_device_add_group(VFIOGroup *group) 1776 { 1777 #ifdef CONFIG_KVM 1778 struct kvm_device_attr attr = { 1779 .group = KVM_DEV_VFIO_GROUP, 1780 .attr = KVM_DEV_VFIO_GROUP_ADD, 1781 .addr = (uint64_t)(unsigned long)&group->fd, 1782 }; 1783 1784 if (!kvm_enabled()) { 1785 return; 1786 } 1787 1788 if (vfio_kvm_device_fd < 0) { 1789 struct kvm_create_device cd = { 1790 .type = KVM_DEV_TYPE_VFIO, 1791 }; 1792 1793 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 1794 error_report("Failed to create KVM VFIO device: %m"); 1795 return; 1796 } 1797 1798 vfio_kvm_device_fd = cd.fd; 1799 } 1800 1801 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1802 error_report("Failed to add group %d to KVM VFIO device: %m", 1803 group->groupid); 1804 } 1805 #endif 1806 } 1807 1808 static void vfio_kvm_device_del_group(VFIOGroup *group) 1809 { 1810 #ifdef CONFIG_KVM 1811 struct kvm_device_attr attr = { 1812 .group = KVM_DEV_VFIO_GROUP, 1813 .attr = KVM_DEV_VFIO_GROUP_DEL, 1814 .addr = (uint64_t)(unsigned long)&group->fd, 1815 }; 1816 1817 if (vfio_kvm_device_fd < 0) { 1818 return; 1819 } 1820 1821 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1822 error_report("Failed to remove group %d from KVM VFIO device: %m", 1823 group->groupid); 1824 } 1825 #endif 1826 } 1827 1828 static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1829 { 1830 VFIOAddressSpace *space; 1831 1832 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1833 if (space->as == as) { 1834 return space; 1835 } 1836 } 1837 1838 /* No suitable VFIOAddressSpace, create a new one */ 1839 space = g_malloc0(sizeof(*space)); 1840 space->as = as; 1841 QLIST_INIT(&space->containers); 1842 1843 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1844 1845 return space; 1846 } 1847 1848 static void vfio_put_address_space(VFIOAddressSpace *space) 1849 { 1850 if (QLIST_EMPTY(&space->containers)) { 1851 QLIST_REMOVE(space, list); 1852 g_free(space); 1853 } 1854 } 1855 1856 /* 1857 * vfio_get_iommu_type - selects the richest iommu_type (v2 first) 1858 */ 1859 static int vfio_get_iommu_type(VFIOContainer *container, 1860 Error **errp) 1861 { 1862 int iommu_types[] = { VFIO_TYPE1v2_IOMMU, VFIO_TYPE1_IOMMU, 1863 VFIO_SPAPR_TCE_v2_IOMMU, VFIO_SPAPR_TCE_IOMMU }; 1864 int i; 1865 1866 for (i = 0; i < ARRAY_SIZE(iommu_types); i++) { 1867 if (ioctl(container->fd, VFIO_CHECK_EXTENSION, iommu_types[i])) { 1868 return iommu_types[i]; 1869 } 1870 } 1871 error_setg(errp, "No available IOMMU models"); 1872 return -EINVAL; 1873 } 1874 1875 static int vfio_init_container(VFIOContainer *container, int group_fd, 1876 Error **errp) 1877 { 1878 int iommu_type, ret; 1879 1880 iommu_type = vfio_get_iommu_type(container, errp); 1881 if (iommu_type < 0) { 1882 return iommu_type; 1883 } 1884 1885 ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container->fd); 1886 if (ret) { 1887 error_setg_errno(errp, errno, "Failed to set group container"); 1888 return -errno; 1889 } 1890 1891 while (ioctl(container->fd, VFIO_SET_IOMMU, iommu_type)) { 1892 if (iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 1893 /* 1894 * On sPAPR, despite the IOMMU subdriver always advertises v1 and 1895 * v2, the running platform may not support v2 and there is no 1896 * way to guess it until an IOMMU group gets added to the container. 1897 * So in case it fails with v2, try v1 as a fallback. 1898 */ 1899 iommu_type = VFIO_SPAPR_TCE_IOMMU; 1900 continue; 1901 } 1902 error_setg_errno(errp, errno, "Failed to set iommu for container"); 1903 return -errno; 1904 } 1905 1906 container->iommu_type = iommu_type; 1907 return 0; 1908 } 1909 1910 static int vfio_get_iommu_info(VFIOContainer *container, 1911 struct vfio_iommu_type1_info **info) 1912 { 1913 1914 size_t argsz = sizeof(struct vfio_iommu_type1_info); 1915 1916 *info = g_new0(struct vfio_iommu_type1_info, 1); 1917 again: 1918 (*info)->argsz = argsz; 1919 1920 if (ioctl(container->fd, VFIO_IOMMU_GET_INFO, *info)) { 1921 g_free(*info); 1922 *info = NULL; 1923 return -errno; 1924 } 1925 1926 if (((*info)->argsz > argsz)) { 1927 argsz = (*info)->argsz; 1928 *info = g_realloc(*info, argsz); 1929 goto again; 1930 } 1931 1932 return 0; 1933 } 1934 1935 static struct vfio_info_cap_header * 1936 vfio_get_iommu_info_cap(struct vfio_iommu_type1_info *info, uint16_t id) 1937 { 1938 struct vfio_info_cap_header *hdr; 1939 void *ptr = info; 1940 1941 if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) { 1942 return NULL; 1943 } 1944 1945 for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) { 1946 if (hdr->id == id) { 1947 return hdr; 1948 } 1949 } 1950 1951 return NULL; 1952 } 1953 1954 static void vfio_get_iommu_info_migration(VFIOContainer *container, 1955 struct vfio_iommu_type1_info *info) 1956 { 1957 struct vfio_info_cap_header *hdr; 1958 struct vfio_iommu_type1_info_cap_migration *cap_mig; 1959 1960 hdr = vfio_get_iommu_info_cap(info, VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION); 1961 if (!hdr) { 1962 return; 1963 } 1964 1965 cap_mig = container_of(hdr, struct vfio_iommu_type1_info_cap_migration, 1966 header); 1967 1968 /* 1969 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of 1970 * qemu_real_host_page_size to mark those dirty. 1971 */ 1972 if (cap_mig->pgsize_bitmap & qemu_real_host_page_size) { 1973 container->dirty_pages_supported = true; 1974 container->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size; 1975 container->dirty_pgsizes = cap_mig->pgsize_bitmap; 1976 } 1977 } 1978 1979 static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, 1980 Error **errp) 1981 { 1982 VFIOContainer *container; 1983 int ret, fd; 1984 VFIOAddressSpace *space; 1985 1986 space = vfio_get_address_space(as); 1987 1988 /* 1989 * VFIO is currently incompatible with discarding of RAM insofar as the 1990 * madvise to purge (zap) the page from QEMU's address space does not 1991 * interact with the memory API and therefore leaves stale virtual to 1992 * physical mappings in the IOMMU if the page was previously pinned. We 1993 * therefore set discarding broken for each group added to a container, 1994 * whether the container is used individually or shared. This provides 1995 * us with options to allow devices within a group to opt-in and allow 1996 * discarding, so long as it is done consistently for a group (for instance 1997 * if the device is an mdev device where it is known that the host vendor 1998 * driver will never pin pages outside of the working set of the guest 1999 * driver, which would thus not be discarding candidates). 2000 * 2001 * The first opportunity to induce pinning occurs here where we attempt to 2002 * attach the group to existing containers within the AddressSpace. If any 2003 * pages are already zapped from the virtual address space, such as from 2004 * previous discards, new pinning will cause valid mappings to be 2005 * re-established. Likewise, when the overall MemoryListener for a new 2006 * container is registered, a replay of mappings within the AddressSpace 2007 * will occur, re-establishing any previously zapped pages as well. 2008 * 2009 * Especially virtio-balloon is currently only prevented from discarding 2010 * new memory, it will not yet set ram_block_discard_set_required() and 2011 * therefore, neither stops us here or deals with the sudden memory 2012 * consumption of inflated memory. 2013 * 2014 * We do support discarding of memory coordinated via the RamDiscardManager 2015 * with some IOMMU types. vfio_ram_block_discard_disable() handles the 2016 * details once we know which type of IOMMU we are using. 2017 */ 2018 2019 QLIST_FOREACH(container, &space->containers, next) { 2020 if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) { 2021 ret = vfio_ram_block_discard_disable(container, true); 2022 if (ret) { 2023 error_setg_errno(errp, -ret, 2024 "Cannot set discarding of RAM broken"); 2025 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, 2026 &container->fd)) { 2027 error_report("vfio: error disconnecting group %d from" 2028 " container", group->groupid); 2029 } 2030 return ret; 2031 } 2032 group->container = container; 2033 QLIST_INSERT_HEAD(&container->group_list, group, container_next); 2034 vfio_kvm_device_add_group(group); 2035 return 0; 2036 } 2037 } 2038 2039 fd = qemu_open_old("/dev/vfio/vfio", O_RDWR); 2040 if (fd < 0) { 2041 error_setg_errno(errp, errno, "failed to open /dev/vfio/vfio"); 2042 ret = -errno; 2043 goto put_space_exit; 2044 } 2045 2046 ret = ioctl(fd, VFIO_GET_API_VERSION); 2047 if (ret != VFIO_API_VERSION) { 2048 error_setg(errp, "supported vfio version: %d, " 2049 "reported version: %d", VFIO_API_VERSION, ret); 2050 ret = -EINVAL; 2051 goto close_fd_exit; 2052 } 2053 2054 container = g_malloc0(sizeof(*container)); 2055 container->space = space; 2056 container->fd = fd; 2057 container->error = NULL; 2058 container->dirty_pages_supported = false; 2059 container->dma_max_mappings = 0; 2060 QLIST_INIT(&container->giommu_list); 2061 QLIST_INIT(&container->hostwin_list); 2062 QLIST_INIT(&container->vrdl_list); 2063 2064 ret = vfio_init_container(container, group->fd, errp); 2065 if (ret) { 2066 goto free_container_exit; 2067 } 2068 2069 ret = vfio_ram_block_discard_disable(container, true); 2070 if (ret) { 2071 error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken"); 2072 goto free_container_exit; 2073 } 2074 2075 switch (container->iommu_type) { 2076 case VFIO_TYPE1v2_IOMMU: 2077 case VFIO_TYPE1_IOMMU: 2078 { 2079 struct vfio_iommu_type1_info *info; 2080 2081 /* 2082 * FIXME: This assumes that a Type1 IOMMU can map any 64-bit 2083 * IOVA whatsoever. That's not actually true, but the current 2084 * kernel interface doesn't tell us what it can map, and the 2085 * existing Type1 IOMMUs generally support any IOVA we're 2086 * going to actually try in practice. 2087 */ 2088 ret = vfio_get_iommu_info(container, &info); 2089 2090 if (ret || !(info->flags & VFIO_IOMMU_INFO_PGSIZES)) { 2091 /* Assume 4k IOVA page size */ 2092 info->iova_pgsizes = 4096; 2093 } 2094 vfio_host_win_add(container, 0, (hwaddr)-1, info->iova_pgsizes); 2095 container->pgsizes = info->iova_pgsizes; 2096 2097 /* The default in the kernel ("dma_entry_limit") is 65535. */ 2098 container->dma_max_mappings = 65535; 2099 if (!ret) { 2100 vfio_get_info_dma_avail(info, &container->dma_max_mappings); 2101 vfio_get_iommu_info_migration(container, info); 2102 } 2103 g_free(info); 2104 break; 2105 } 2106 case VFIO_SPAPR_TCE_v2_IOMMU: 2107 case VFIO_SPAPR_TCE_IOMMU: 2108 { 2109 struct vfio_iommu_spapr_tce_info info; 2110 bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU; 2111 2112 /* 2113 * The host kernel code implementing VFIO_IOMMU_DISABLE is called 2114 * when container fd is closed so we do not call it explicitly 2115 * in this file. 2116 */ 2117 if (!v2) { 2118 ret = ioctl(fd, VFIO_IOMMU_ENABLE); 2119 if (ret) { 2120 error_setg_errno(errp, errno, "failed to enable container"); 2121 ret = -errno; 2122 goto enable_discards_exit; 2123 } 2124 } else { 2125 container->prereg_listener = vfio_prereg_listener; 2126 2127 memory_listener_register(&container->prereg_listener, 2128 &address_space_memory); 2129 if (container->error) { 2130 memory_listener_unregister(&container->prereg_listener); 2131 ret = -1; 2132 error_propagate_prepend(errp, container->error, 2133 "RAM memory listener initialization failed: "); 2134 goto enable_discards_exit; 2135 } 2136 } 2137 2138 info.argsz = sizeof(info); 2139 ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); 2140 if (ret) { 2141 error_setg_errno(errp, errno, 2142 "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed"); 2143 ret = -errno; 2144 if (v2) { 2145 memory_listener_unregister(&container->prereg_listener); 2146 } 2147 goto enable_discards_exit; 2148 } 2149 2150 if (v2) { 2151 container->pgsizes = info.ddw.pgsizes; 2152 /* 2153 * There is a default window in just created container. 2154 * To make region_add/del simpler, we better remove this 2155 * window now and let those iommu_listener callbacks 2156 * create/remove them when needed. 2157 */ 2158 ret = vfio_spapr_remove_window(container, info.dma32_window_start); 2159 if (ret) { 2160 error_setg_errno(errp, -ret, 2161 "failed to remove existing window"); 2162 goto enable_discards_exit; 2163 } 2164 } else { 2165 /* The default table uses 4K pages */ 2166 container->pgsizes = 0x1000; 2167 vfio_host_win_add(container, info.dma32_window_start, 2168 info.dma32_window_start + 2169 info.dma32_window_size - 1, 2170 0x1000); 2171 } 2172 } 2173 } 2174 2175 vfio_kvm_device_add_group(group); 2176 2177 QLIST_INIT(&container->group_list); 2178 QLIST_INSERT_HEAD(&space->containers, container, next); 2179 2180 group->container = container; 2181 QLIST_INSERT_HEAD(&container->group_list, group, container_next); 2182 2183 container->listener = vfio_memory_listener; 2184 2185 memory_listener_register(&container->listener, container->space->as); 2186 2187 if (container->error) { 2188 ret = -1; 2189 error_propagate_prepend(errp, container->error, 2190 "memory listener initialization failed: "); 2191 goto listener_release_exit; 2192 } 2193 2194 container->initialized = true; 2195 2196 return 0; 2197 listener_release_exit: 2198 QLIST_REMOVE(group, container_next); 2199 QLIST_REMOVE(container, next); 2200 vfio_kvm_device_del_group(group); 2201 vfio_listener_release(container); 2202 2203 enable_discards_exit: 2204 vfio_ram_block_discard_disable(container, false); 2205 2206 free_container_exit: 2207 g_free(container); 2208 2209 close_fd_exit: 2210 close(fd); 2211 2212 put_space_exit: 2213 vfio_put_address_space(space); 2214 2215 return ret; 2216 } 2217 2218 static void vfio_disconnect_container(VFIOGroup *group) 2219 { 2220 VFIOContainer *container = group->container; 2221 2222 QLIST_REMOVE(group, container_next); 2223 group->container = NULL; 2224 2225 /* 2226 * Explicitly release the listener first before unset container, 2227 * since unset may destroy the backend container if it's the last 2228 * group. 2229 */ 2230 if (QLIST_EMPTY(&container->group_list)) { 2231 vfio_listener_release(container); 2232 } 2233 2234 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) { 2235 error_report("vfio: error disconnecting group %d from container", 2236 group->groupid); 2237 } 2238 2239 if (QLIST_EMPTY(&container->group_list)) { 2240 VFIOAddressSpace *space = container->space; 2241 VFIOGuestIOMMU *giommu, *tmp; 2242 2243 QLIST_REMOVE(container, next); 2244 2245 QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) { 2246 memory_region_unregister_iommu_notifier( 2247 MEMORY_REGION(giommu->iommu), &giommu->n); 2248 QLIST_REMOVE(giommu, giommu_next); 2249 g_free(giommu); 2250 } 2251 2252 trace_vfio_disconnect_container(container->fd); 2253 close(container->fd); 2254 g_free(container); 2255 2256 vfio_put_address_space(space); 2257 } 2258 } 2259 2260 VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) 2261 { 2262 VFIOGroup *group; 2263 char path[32]; 2264 struct vfio_group_status status = { .argsz = sizeof(status) }; 2265 2266 QLIST_FOREACH(group, &vfio_group_list, next) { 2267 if (group->groupid == groupid) { 2268 /* Found it. Now is it already in the right context? */ 2269 if (group->container->space->as == as) { 2270 return group; 2271 } else { 2272 error_setg(errp, "group %d used in multiple address spaces", 2273 group->groupid); 2274 return NULL; 2275 } 2276 } 2277 } 2278 2279 group = g_malloc0(sizeof(*group)); 2280 2281 snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); 2282 group->fd = qemu_open_old(path, O_RDWR); 2283 if (group->fd < 0) { 2284 error_setg_errno(errp, errno, "failed to open %s", path); 2285 goto free_group_exit; 2286 } 2287 2288 if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) { 2289 error_setg_errno(errp, errno, "failed to get group %d status", groupid); 2290 goto close_fd_exit; 2291 } 2292 2293 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { 2294 error_setg(errp, "group %d is not viable", groupid); 2295 error_append_hint(errp, 2296 "Please ensure all devices within the iommu_group " 2297 "are bound to their vfio bus driver.\n"); 2298 goto close_fd_exit; 2299 } 2300 2301 group->groupid = groupid; 2302 QLIST_INIT(&group->device_list); 2303 2304 if (vfio_connect_container(group, as, errp)) { 2305 error_prepend(errp, "failed to setup container for group %d: ", 2306 groupid); 2307 goto close_fd_exit; 2308 } 2309 2310 if (QLIST_EMPTY(&vfio_group_list)) { 2311 qemu_register_reset(vfio_reset_handler, NULL); 2312 } 2313 2314 QLIST_INSERT_HEAD(&vfio_group_list, group, next); 2315 2316 return group; 2317 2318 close_fd_exit: 2319 close(group->fd); 2320 2321 free_group_exit: 2322 g_free(group); 2323 2324 return NULL; 2325 } 2326 2327 void vfio_put_group(VFIOGroup *group) 2328 { 2329 if (!group || !QLIST_EMPTY(&group->device_list)) { 2330 return; 2331 } 2332 2333 if (!group->ram_block_discard_allowed) { 2334 vfio_ram_block_discard_disable(group->container, false); 2335 } 2336 vfio_kvm_device_del_group(group); 2337 vfio_disconnect_container(group); 2338 QLIST_REMOVE(group, next); 2339 trace_vfio_put_group(group->fd); 2340 close(group->fd); 2341 g_free(group); 2342 2343 if (QLIST_EMPTY(&vfio_group_list)) { 2344 qemu_unregister_reset(vfio_reset_handler, NULL); 2345 } 2346 } 2347 2348 int vfio_get_device(VFIOGroup *group, const char *name, 2349 VFIODevice *vbasedev, Error **errp) 2350 { 2351 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) }; 2352 int ret, fd; 2353 2354 fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name); 2355 if (fd < 0) { 2356 error_setg_errno(errp, errno, "error getting device from group %d", 2357 group->groupid); 2358 error_append_hint(errp, 2359 "Verify all devices in group %d are bound to vfio-<bus> " 2360 "or pci-stub and not already in use\n", group->groupid); 2361 return fd; 2362 } 2363 2364 ret = ioctl(fd, VFIO_DEVICE_GET_INFO, &dev_info); 2365 if (ret) { 2366 error_setg_errno(errp, errno, "error getting device info"); 2367 close(fd); 2368 return ret; 2369 } 2370 2371 /* 2372 * Set discarding of RAM as not broken for this group if the driver knows 2373 * the device operates compatibly with discarding. Setting must be 2374 * consistent per group, but since compatibility is really only possible 2375 * with mdev currently, we expect singleton groups. 2376 */ 2377 if (vbasedev->ram_block_discard_allowed != 2378 group->ram_block_discard_allowed) { 2379 if (!QLIST_EMPTY(&group->device_list)) { 2380 error_setg(errp, "Inconsistent setting of support for discarding " 2381 "RAM (e.g., balloon) within group"); 2382 close(fd); 2383 return -1; 2384 } 2385 2386 if (!group->ram_block_discard_allowed) { 2387 group->ram_block_discard_allowed = true; 2388 vfio_ram_block_discard_disable(group->container, false); 2389 } 2390 } 2391 2392 vbasedev->fd = fd; 2393 vbasedev->group = group; 2394 QLIST_INSERT_HEAD(&group->device_list, vbasedev, next); 2395 2396 vbasedev->num_irqs = dev_info.num_irqs; 2397 vbasedev->num_regions = dev_info.num_regions; 2398 vbasedev->flags = dev_info.flags; 2399 2400 trace_vfio_get_device(name, dev_info.flags, dev_info.num_regions, 2401 dev_info.num_irqs); 2402 2403 vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET); 2404 return 0; 2405 } 2406 2407 void vfio_put_base_device(VFIODevice *vbasedev) 2408 { 2409 if (!vbasedev->group) { 2410 return; 2411 } 2412 QLIST_REMOVE(vbasedev, next); 2413 vbasedev->group = NULL; 2414 trace_vfio_put_base_device(vbasedev->fd); 2415 close(vbasedev->fd); 2416 } 2417 2418 int vfio_get_region_info(VFIODevice *vbasedev, int index, 2419 struct vfio_region_info **info) 2420 { 2421 size_t argsz = sizeof(struct vfio_region_info); 2422 2423 *info = g_malloc0(argsz); 2424 2425 (*info)->index = index; 2426 retry: 2427 (*info)->argsz = argsz; 2428 2429 if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) { 2430 g_free(*info); 2431 *info = NULL; 2432 return -errno; 2433 } 2434 2435 if ((*info)->argsz > argsz) { 2436 argsz = (*info)->argsz; 2437 *info = g_realloc(*info, argsz); 2438 2439 goto retry; 2440 } 2441 2442 return 0; 2443 } 2444 2445 int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type, 2446 uint32_t subtype, struct vfio_region_info **info) 2447 { 2448 int i; 2449 2450 for (i = 0; i < vbasedev->num_regions; i++) { 2451 struct vfio_info_cap_header *hdr; 2452 struct vfio_region_info_cap_type *cap_type; 2453 2454 if (vfio_get_region_info(vbasedev, i, info)) { 2455 continue; 2456 } 2457 2458 hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE); 2459 if (!hdr) { 2460 g_free(*info); 2461 continue; 2462 } 2463 2464 cap_type = container_of(hdr, struct vfio_region_info_cap_type, header); 2465 2466 trace_vfio_get_dev_region(vbasedev->name, i, 2467 cap_type->type, cap_type->subtype); 2468 2469 if (cap_type->type == type && cap_type->subtype == subtype) { 2470 return 0; 2471 } 2472 2473 g_free(*info); 2474 } 2475 2476 *info = NULL; 2477 return -ENODEV; 2478 } 2479 2480 bool vfio_has_region_cap(VFIODevice *vbasedev, int region, uint16_t cap_type) 2481 { 2482 struct vfio_region_info *info = NULL; 2483 bool ret = false; 2484 2485 if (!vfio_get_region_info(vbasedev, region, &info)) { 2486 if (vfio_get_region_info_cap(info, cap_type)) { 2487 ret = true; 2488 } 2489 g_free(info); 2490 } 2491 2492 return ret; 2493 } 2494 2495 /* 2496 * Interfaces for IBM EEH (Enhanced Error Handling) 2497 */ 2498 static bool vfio_eeh_container_ok(VFIOContainer *container) 2499 { 2500 /* 2501 * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO 2502 * implementation is broken if there are multiple groups in a 2503 * container. The hardware works in units of Partitionable 2504 * Endpoints (== IOMMU groups) and the EEH operations naively 2505 * iterate across all groups in the container, without any logic 2506 * to make sure the groups have their state synchronized. For 2507 * certain operations (ENABLE) that might be ok, until an error 2508 * occurs, but for others (GET_STATE) it's clearly broken. 2509 */ 2510 2511 /* 2512 * XXX Once fixed kernels exist, test for them here 2513 */ 2514 2515 if (QLIST_EMPTY(&container->group_list)) { 2516 return false; 2517 } 2518 2519 if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) { 2520 return false; 2521 } 2522 2523 return true; 2524 } 2525 2526 static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op) 2527 { 2528 struct vfio_eeh_pe_op pe_op = { 2529 .argsz = sizeof(pe_op), 2530 .op = op, 2531 }; 2532 int ret; 2533 2534 if (!vfio_eeh_container_ok(container)) { 2535 error_report("vfio/eeh: EEH_PE_OP 0x%x: " 2536 "kernel requires a container with exactly one group", op); 2537 return -EPERM; 2538 } 2539 2540 ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op); 2541 if (ret < 0) { 2542 error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op); 2543 return -errno; 2544 } 2545 2546 return ret; 2547 } 2548 2549 static VFIOContainer *vfio_eeh_as_container(AddressSpace *as) 2550 { 2551 VFIOAddressSpace *space = vfio_get_address_space(as); 2552 VFIOContainer *container = NULL; 2553 2554 if (QLIST_EMPTY(&space->containers)) { 2555 /* No containers to act on */ 2556 goto out; 2557 } 2558 2559 container = QLIST_FIRST(&space->containers); 2560 2561 if (QLIST_NEXT(container, next)) { 2562 /* We don't yet have logic to synchronize EEH state across 2563 * multiple containers */ 2564 container = NULL; 2565 goto out; 2566 } 2567 2568 out: 2569 vfio_put_address_space(space); 2570 return container; 2571 } 2572 2573 bool vfio_eeh_as_ok(AddressSpace *as) 2574 { 2575 VFIOContainer *container = vfio_eeh_as_container(as); 2576 2577 return (container != NULL) && vfio_eeh_container_ok(container); 2578 } 2579 2580 int vfio_eeh_as_op(AddressSpace *as, uint32_t op) 2581 { 2582 VFIOContainer *container = vfio_eeh_as_container(as); 2583 2584 if (!container) { 2585 return -ENODEV; 2586 } 2587 return vfio_eeh_container_op(container, op); 2588 } 2589