1 /* 2 * generic functions used by VFIO devices 3 * 4 * Copyright Red Hat, Inc. 2012 5 * 6 * Authors: 7 * Alex Williamson <alex.williamson@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 * Based on qemu-kvm device-assignment: 13 * Adapted for KVM by Qumranet. 14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com) 15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com) 16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com) 17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com) 18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com) 19 */ 20 21 #include "qemu/osdep.h" 22 #include <sys/ioctl.h> 23 #ifdef CONFIG_KVM 24 #include <linux/kvm.h> 25 #endif 26 #include <linux/vfio.h> 27 28 #include "hw/vfio/vfio-common.h" 29 #include "hw/vfio/vfio.h" 30 #include "exec/address-spaces.h" 31 #include "exec/memory.h" 32 #include "hw/hw.h" 33 #include "qemu/error-report.h" 34 #include "qemu/range.h" 35 #include "sysemu/balloon.h" 36 #include "sysemu/kvm.h" 37 #include "trace.h" 38 #include "qapi/error.h" 39 40 VFIOGroupList vfio_group_list = 41 QLIST_HEAD_INITIALIZER(vfio_group_list); 42 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces = 43 QLIST_HEAD_INITIALIZER(vfio_address_spaces); 44 45 #ifdef CONFIG_KVM 46 /* 47 * We have a single VFIO pseudo device per KVM VM. Once created it lives 48 * for the life of the VM. Closing the file descriptor only drops our 49 * reference to it and the device's reference to kvm. Therefore once 50 * initialized, this file descriptor is only released on QEMU exit and 51 * we'll re-use it should another vfio device be attached before then. 52 */ 53 static int vfio_kvm_device_fd = -1; 54 #endif 55 56 /* 57 * Common VFIO interrupt disable 58 */ 59 void vfio_disable_irqindex(VFIODevice *vbasedev, int index) 60 { 61 struct vfio_irq_set irq_set = { 62 .argsz = sizeof(irq_set), 63 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 64 .index = index, 65 .start = 0, 66 .count = 0, 67 }; 68 69 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 70 } 71 72 void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index) 73 { 74 struct vfio_irq_set irq_set = { 75 .argsz = sizeof(irq_set), 76 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, 77 .index = index, 78 .start = 0, 79 .count = 1, 80 }; 81 82 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 83 } 84 85 void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index) 86 { 87 struct vfio_irq_set irq_set = { 88 .argsz = sizeof(irq_set), 89 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK, 90 .index = index, 91 .start = 0, 92 .count = 1, 93 }; 94 95 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); 96 } 97 98 /* 99 * IO Port/MMIO - Beware of the endians, VFIO is always little endian 100 */ 101 void vfio_region_write(void *opaque, hwaddr addr, 102 uint64_t data, unsigned size) 103 { 104 VFIORegion *region = opaque; 105 VFIODevice *vbasedev = region->vbasedev; 106 union { 107 uint8_t byte; 108 uint16_t word; 109 uint32_t dword; 110 uint64_t qword; 111 } buf; 112 113 switch (size) { 114 case 1: 115 buf.byte = data; 116 break; 117 case 2: 118 buf.word = cpu_to_le16(data); 119 break; 120 case 4: 121 buf.dword = cpu_to_le32(data); 122 break; 123 case 8: 124 buf.qword = cpu_to_le64(data); 125 break; 126 default: 127 hw_error("vfio: unsupported write size, %d bytes", size); 128 break; 129 } 130 131 if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { 132 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64 133 ",%d) failed: %m", 134 __func__, vbasedev->name, region->nr, 135 addr, data, size); 136 } 137 138 trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size); 139 140 /* 141 * A read or write to a BAR always signals an INTx EOI. This will 142 * do nothing if not pending (including not in INTx mode). We assume 143 * that a BAR access is in response to an interrupt and that BAR 144 * accesses will service the interrupt. Unfortunately, we don't know 145 * which access will service the interrupt, so we're potentially 146 * getting quite a few host interrupts per guest interrupt. 147 */ 148 vbasedev->ops->vfio_eoi(vbasedev); 149 } 150 151 uint64_t vfio_region_read(void *opaque, 152 hwaddr addr, unsigned size) 153 { 154 VFIORegion *region = opaque; 155 VFIODevice *vbasedev = region->vbasedev; 156 union { 157 uint8_t byte; 158 uint16_t word; 159 uint32_t dword; 160 uint64_t qword; 161 } buf; 162 uint64_t data = 0; 163 164 if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) { 165 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m", 166 __func__, vbasedev->name, region->nr, 167 addr, size); 168 return (uint64_t)-1; 169 } 170 switch (size) { 171 case 1: 172 data = buf.byte; 173 break; 174 case 2: 175 data = le16_to_cpu(buf.word); 176 break; 177 case 4: 178 data = le32_to_cpu(buf.dword); 179 break; 180 case 8: 181 data = le64_to_cpu(buf.qword); 182 break; 183 default: 184 hw_error("vfio: unsupported read size, %d bytes", size); 185 break; 186 } 187 188 trace_vfio_region_read(vbasedev->name, region->nr, addr, size, data); 189 190 /* Same as write above */ 191 vbasedev->ops->vfio_eoi(vbasedev); 192 193 return data; 194 } 195 196 const MemoryRegionOps vfio_region_ops = { 197 .read = vfio_region_read, 198 .write = vfio_region_write, 199 .endianness = DEVICE_LITTLE_ENDIAN, 200 .valid = { 201 .min_access_size = 1, 202 .max_access_size = 8, 203 }, 204 .impl = { 205 .min_access_size = 1, 206 .max_access_size = 8, 207 }, 208 }; 209 210 /* 211 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86 212 */ 213 static int vfio_dma_unmap(VFIOContainer *container, 214 hwaddr iova, ram_addr_t size) 215 { 216 struct vfio_iommu_type1_dma_unmap unmap = { 217 .argsz = sizeof(unmap), 218 .flags = 0, 219 .iova = iova, 220 .size = size, 221 }; 222 223 if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) { 224 error_report("VFIO_UNMAP_DMA: %d", -errno); 225 return -errno; 226 } 227 228 return 0; 229 } 230 231 static int vfio_dma_map(VFIOContainer *container, hwaddr iova, 232 ram_addr_t size, void *vaddr, bool readonly) 233 { 234 struct vfio_iommu_type1_dma_map map = { 235 .argsz = sizeof(map), 236 .flags = VFIO_DMA_MAP_FLAG_READ, 237 .vaddr = (__u64)(uintptr_t)vaddr, 238 .iova = iova, 239 .size = size, 240 }; 241 242 if (!readonly) { 243 map.flags |= VFIO_DMA_MAP_FLAG_WRITE; 244 } 245 246 /* 247 * Try the mapping, if it fails with EBUSY, unmap the region and try 248 * again. This shouldn't be necessary, but we sometimes see it in 249 * the VGA ROM space. 250 */ 251 if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 || 252 (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 && 253 ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) { 254 return 0; 255 } 256 257 error_report("VFIO_MAP_DMA: %d", -errno); 258 return -errno; 259 } 260 261 static void vfio_host_win_add(VFIOContainer *container, 262 hwaddr min_iova, hwaddr max_iova, 263 uint64_t iova_pgsizes) 264 { 265 VFIOHostDMAWindow *hostwin; 266 267 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 268 if (ranges_overlap(hostwin->min_iova, 269 hostwin->max_iova - hostwin->min_iova + 1, 270 min_iova, 271 max_iova - min_iova + 1)) { 272 hw_error("%s: Overlapped IOMMU are not enabled", __func__); 273 } 274 } 275 276 hostwin = g_malloc0(sizeof(*hostwin)); 277 278 hostwin->min_iova = min_iova; 279 hostwin->max_iova = max_iova; 280 hostwin->iova_pgsizes = iova_pgsizes; 281 QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next); 282 } 283 284 static int vfio_host_win_del(VFIOContainer *container, hwaddr min_iova, 285 hwaddr max_iova) 286 { 287 VFIOHostDMAWindow *hostwin; 288 289 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 290 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) { 291 QLIST_REMOVE(hostwin, hostwin_next); 292 return 0; 293 } 294 } 295 296 return -1; 297 } 298 299 static bool vfio_listener_skipped_section(MemoryRegionSection *section) 300 { 301 return (!memory_region_is_ram(section->mr) && 302 !memory_region_is_iommu(section->mr)) || 303 /* 304 * Sizing an enabled 64-bit BAR can cause spurious mappings to 305 * addresses in the upper part of the 64-bit address space. These 306 * are never accessed by the CPU and beyond the address width of 307 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width. 308 */ 309 section->offset_within_address_space & (1ULL << 63); 310 } 311 312 /* Called with rcu_read_lock held. */ 313 static bool vfio_get_vaddr(IOMMUTLBEntry *iotlb, void **vaddr, 314 bool *read_only) 315 { 316 MemoryRegion *mr; 317 hwaddr xlat; 318 hwaddr len = iotlb->addr_mask + 1; 319 bool writable = iotlb->perm & IOMMU_WO; 320 321 /* 322 * The IOMMU TLB entry we have just covers translation through 323 * this IOMMU to its immediate target. We need to translate 324 * it the rest of the way through to memory. 325 */ 326 mr = address_space_translate(&address_space_memory, 327 iotlb->translated_addr, 328 &xlat, &len, writable, 329 MEMTXATTRS_UNSPECIFIED); 330 if (!memory_region_is_ram(mr)) { 331 error_report("iommu map to non memory area %"HWADDR_PRIx"", 332 xlat); 333 return false; 334 } 335 336 /* 337 * Translation truncates length to the IOMMU page size, 338 * check that it did not truncate too much. 339 */ 340 if (len & iotlb->addr_mask) { 341 error_report("iommu has granularity incompatible with target AS"); 342 return false; 343 } 344 345 *vaddr = memory_region_get_ram_ptr(mr) + xlat; 346 *read_only = !writable || mr->readonly; 347 348 return true; 349 } 350 351 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) 352 { 353 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n); 354 VFIOContainer *container = giommu->container; 355 hwaddr iova = iotlb->iova + giommu->iommu_offset; 356 bool read_only; 357 void *vaddr; 358 int ret; 359 360 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP", 361 iova, iova + iotlb->addr_mask); 362 363 if (iotlb->target_as != &address_space_memory) { 364 error_report("Wrong target AS \"%s\", only system memory is allowed", 365 iotlb->target_as->name ? iotlb->target_as->name : "none"); 366 return; 367 } 368 369 rcu_read_lock(); 370 371 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) { 372 if (!vfio_get_vaddr(iotlb, &vaddr, &read_only)) { 373 goto out; 374 } 375 /* 376 * vaddr is only valid until rcu_read_unlock(). But after 377 * vfio_dma_map has set up the mapping the pages will be 378 * pinned by the kernel. This makes sure that the RAM backend 379 * of vaddr will always be there, even if the memory object is 380 * destroyed and its backing memory munmap-ed. 381 */ 382 ret = vfio_dma_map(container, iova, 383 iotlb->addr_mask + 1, vaddr, 384 read_only); 385 if (ret) { 386 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 387 "0x%"HWADDR_PRIx", %p) = %d (%m)", 388 container, iova, 389 iotlb->addr_mask + 1, vaddr, ret); 390 } 391 } else { 392 ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1); 393 if (ret) { 394 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 395 "0x%"HWADDR_PRIx") = %d (%m)", 396 container, iova, 397 iotlb->addr_mask + 1, ret); 398 } 399 } 400 out: 401 rcu_read_unlock(); 402 } 403 404 static void vfio_listener_region_add(MemoryListener *listener, 405 MemoryRegionSection *section) 406 { 407 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 408 hwaddr iova, end; 409 Int128 llend, llsize; 410 void *vaddr; 411 int ret; 412 VFIOHostDMAWindow *hostwin; 413 bool hostwin_found; 414 415 if (vfio_listener_skipped_section(section)) { 416 trace_vfio_listener_region_add_skip( 417 section->offset_within_address_space, 418 section->offset_within_address_space + 419 int128_get64(int128_sub(section->size, int128_one()))); 420 return; 421 } 422 423 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != 424 (section->offset_within_region & ~TARGET_PAGE_MASK))) { 425 error_report("%s received unaligned region", __func__); 426 return; 427 } 428 429 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); 430 llend = int128_make64(section->offset_within_address_space); 431 llend = int128_add(llend, section->size); 432 llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); 433 434 if (int128_ge(int128_make64(iova), llend)) { 435 return; 436 } 437 end = int128_get64(int128_sub(llend, int128_one())); 438 439 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 440 hwaddr pgsize = 0; 441 442 /* For now intersections are not allowed, we may relax this later */ 443 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 444 if (ranges_overlap(hostwin->min_iova, 445 hostwin->max_iova - hostwin->min_iova + 1, 446 section->offset_within_address_space, 447 int128_get64(section->size))) { 448 ret = -1; 449 goto fail; 450 } 451 } 452 453 ret = vfio_spapr_create_window(container, section, &pgsize); 454 if (ret) { 455 goto fail; 456 } 457 458 vfio_host_win_add(container, section->offset_within_address_space, 459 section->offset_within_address_space + 460 int128_get64(section->size) - 1, pgsize); 461 #ifdef CONFIG_KVM 462 if (kvm_enabled()) { 463 VFIOGroup *group; 464 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 465 struct kvm_vfio_spapr_tce param; 466 struct kvm_device_attr attr = { 467 .group = KVM_DEV_VFIO_GROUP, 468 .attr = KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE, 469 .addr = (uint64_t)(unsigned long)¶m, 470 }; 471 472 if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_TCE_FD, 473 ¶m.tablefd)) { 474 QLIST_FOREACH(group, &container->group_list, container_next) { 475 param.groupfd = group->fd; 476 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 477 error_report("vfio: failed to setup fd %d " 478 "for a group with fd %d: %s", 479 param.tablefd, param.groupfd, 480 strerror(errno)); 481 return; 482 } 483 trace_vfio_spapr_group_attach(param.groupfd, param.tablefd); 484 } 485 } 486 } 487 #endif 488 } 489 490 hostwin_found = false; 491 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 492 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 493 hostwin_found = true; 494 break; 495 } 496 } 497 498 if (!hostwin_found) { 499 error_report("vfio: IOMMU container %p can't map guest IOVA region" 500 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, 501 container, iova, end); 502 ret = -EFAULT; 503 goto fail; 504 } 505 506 memory_region_ref(section->mr); 507 508 if (memory_region_is_iommu(section->mr)) { 509 VFIOGuestIOMMU *giommu; 510 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); 511 int iommu_idx; 512 513 trace_vfio_listener_region_add_iommu(iova, end); 514 /* 515 * FIXME: For VFIO iommu types which have KVM acceleration to 516 * avoid bouncing all map/unmaps through qemu this way, this 517 * would be the right place to wire that up (tell the KVM 518 * device emulation the VFIO iommu handles to use). 519 */ 520 giommu = g_malloc0(sizeof(*giommu)); 521 giommu->iommu = iommu_mr; 522 giommu->iommu_offset = section->offset_within_address_space - 523 section->offset_within_region; 524 giommu->container = container; 525 llend = int128_add(int128_make64(section->offset_within_region), 526 section->size); 527 llend = int128_sub(llend, int128_one()); 528 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr, 529 MEMTXATTRS_UNSPECIFIED); 530 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify, 531 IOMMU_NOTIFIER_ALL, 532 section->offset_within_region, 533 int128_get64(llend), 534 iommu_idx); 535 QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next); 536 537 memory_region_register_iommu_notifier(section->mr, &giommu->n); 538 memory_region_iommu_replay(giommu->iommu, &giommu->n); 539 540 return; 541 } 542 543 /* Here we assume that memory_region_is_ram(section->mr)==true */ 544 545 vaddr = memory_region_get_ram_ptr(section->mr) + 546 section->offset_within_region + 547 (iova - section->offset_within_address_space); 548 549 trace_vfio_listener_region_add_ram(iova, end, vaddr); 550 551 llsize = int128_sub(llend, int128_make64(iova)); 552 553 if (memory_region_is_ram_device(section->mr)) { 554 hwaddr pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 555 556 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) { 557 trace_vfio_listener_region_add_no_dma_map( 558 memory_region_name(section->mr), 559 section->offset_within_address_space, 560 int128_getlo(section->size), 561 pgmask + 1); 562 return; 563 } 564 } 565 566 ret = vfio_dma_map(container, iova, int128_get64(llsize), 567 vaddr, section->readonly); 568 if (ret) { 569 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", " 570 "0x%"HWADDR_PRIx", %p) = %d (%m)", 571 container, iova, int128_get64(llsize), vaddr, ret); 572 if (memory_region_is_ram_device(section->mr)) { 573 /* Allow unexpected mappings not to be fatal for RAM devices */ 574 return; 575 } 576 goto fail; 577 } 578 579 return; 580 581 fail: 582 if (memory_region_is_ram_device(section->mr)) { 583 error_report("failed to vfio_dma_map. pci p2p may not work"); 584 return; 585 } 586 /* 587 * On the initfn path, store the first error in the container so we 588 * can gracefully fail. Runtime, there's not much we can do other 589 * than throw a hardware error. 590 */ 591 if (!container->initialized) { 592 if (!container->error) { 593 container->error = ret; 594 } 595 } else { 596 hw_error("vfio: DMA mapping failed, unable to continue"); 597 } 598 } 599 600 static void vfio_listener_region_del(MemoryListener *listener, 601 MemoryRegionSection *section) 602 { 603 VFIOContainer *container = container_of(listener, VFIOContainer, listener); 604 hwaddr iova, end; 605 Int128 llend, llsize; 606 int ret; 607 bool try_unmap = true; 608 609 if (vfio_listener_skipped_section(section)) { 610 trace_vfio_listener_region_del_skip( 611 section->offset_within_address_space, 612 section->offset_within_address_space + 613 int128_get64(int128_sub(section->size, int128_one()))); 614 return; 615 } 616 617 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) != 618 (section->offset_within_region & ~TARGET_PAGE_MASK))) { 619 error_report("%s received unaligned region", __func__); 620 return; 621 } 622 623 if (memory_region_is_iommu(section->mr)) { 624 VFIOGuestIOMMU *giommu; 625 626 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) { 627 if (MEMORY_REGION(giommu->iommu) == section->mr && 628 giommu->n.start == section->offset_within_region) { 629 memory_region_unregister_iommu_notifier(section->mr, 630 &giommu->n); 631 QLIST_REMOVE(giommu, giommu_next); 632 g_free(giommu); 633 break; 634 } 635 } 636 637 /* 638 * FIXME: We assume the one big unmap below is adequate to 639 * remove any individual page mappings in the IOMMU which 640 * might have been copied into VFIO. This works for a page table 641 * based IOMMU where a big unmap flattens a large range of IO-PTEs. 642 * That may not be true for all IOMMU types. 643 */ 644 } 645 646 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space); 647 llend = int128_make64(section->offset_within_address_space); 648 llend = int128_add(llend, section->size); 649 llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK)); 650 651 if (int128_ge(int128_make64(iova), llend)) { 652 return; 653 } 654 end = int128_get64(int128_sub(llend, int128_one())); 655 656 llsize = int128_sub(llend, int128_make64(iova)); 657 658 trace_vfio_listener_region_del(iova, end); 659 660 if (memory_region_is_ram_device(section->mr)) { 661 hwaddr pgmask; 662 VFIOHostDMAWindow *hostwin; 663 bool hostwin_found = false; 664 665 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { 666 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { 667 hostwin_found = true; 668 break; 669 } 670 } 671 assert(hostwin_found); /* or region_add() would have failed */ 672 673 pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1; 674 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask)); 675 } 676 677 if (try_unmap) { 678 ret = vfio_dma_unmap(container, iova, int128_get64(llsize)); 679 if (ret) { 680 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", " 681 "0x%"HWADDR_PRIx") = %d (%m)", 682 container, iova, int128_get64(llsize), ret); 683 } 684 } 685 686 memory_region_unref(section->mr); 687 688 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 689 vfio_spapr_remove_window(container, 690 section->offset_within_address_space); 691 if (vfio_host_win_del(container, 692 section->offset_within_address_space, 693 section->offset_within_address_space + 694 int128_get64(section->size) - 1) < 0) { 695 hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx, 696 __func__, section->offset_within_address_space); 697 } 698 } 699 } 700 701 static const MemoryListener vfio_memory_listener = { 702 .region_add = vfio_listener_region_add, 703 .region_del = vfio_listener_region_del, 704 }; 705 706 static void vfio_listener_release(VFIOContainer *container) 707 { 708 memory_listener_unregister(&container->listener); 709 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { 710 memory_listener_unregister(&container->prereg_listener); 711 } 712 } 713 714 static struct vfio_info_cap_header * 715 vfio_get_region_info_cap(struct vfio_region_info *info, uint16_t id) 716 { 717 struct vfio_info_cap_header *hdr; 718 void *ptr = info; 719 720 if (!(info->flags & VFIO_REGION_INFO_FLAG_CAPS)) { 721 return NULL; 722 } 723 724 for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) { 725 if (hdr->id == id) { 726 return hdr; 727 } 728 } 729 730 return NULL; 731 } 732 733 static int vfio_setup_region_sparse_mmaps(VFIORegion *region, 734 struct vfio_region_info *info) 735 { 736 struct vfio_info_cap_header *hdr; 737 struct vfio_region_info_cap_sparse_mmap *sparse; 738 int i, j; 739 740 hdr = vfio_get_region_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); 741 if (!hdr) { 742 return -ENODEV; 743 } 744 745 sparse = container_of(hdr, struct vfio_region_info_cap_sparse_mmap, header); 746 747 trace_vfio_region_sparse_mmap_header(region->vbasedev->name, 748 region->nr, sparse->nr_areas); 749 750 region->mmaps = g_new0(VFIOMmap, sparse->nr_areas); 751 752 for (i = 0, j = 0; i < sparse->nr_areas; i++) { 753 trace_vfio_region_sparse_mmap_entry(i, sparse->areas[i].offset, 754 sparse->areas[i].offset + 755 sparse->areas[i].size); 756 757 if (sparse->areas[i].size) { 758 region->mmaps[j].offset = sparse->areas[i].offset; 759 region->mmaps[j].size = sparse->areas[i].size; 760 j++; 761 } 762 } 763 764 region->nr_mmaps = j; 765 region->mmaps = g_realloc(region->mmaps, j * sizeof(VFIOMmap)); 766 767 return 0; 768 } 769 770 int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region, 771 int index, const char *name) 772 { 773 struct vfio_region_info *info; 774 int ret; 775 776 ret = vfio_get_region_info(vbasedev, index, &info); 777 if (ret) { 778 return ret; 779 } 780 781 region->vbasedev = vbasedev; 782 region->flags = info->flags; 783 region->size = info->size; 784 region->fd_offset = info->offset; 785 region->nr = index; 786 787 if (region->size) { 788 region->mem = g_new0(MemoryRegion, 1); 789 memory_region_init_io(region->mem, obj, &vfio_region_ops, 790 region, name, region->size); 791 792 if (!vbasedev->no_mmap && 793 region->flags & VFIO_REGION_INFO_FLAG_MMAP) { 794 795 ret = vfio_setup_region_sparse_mmaps(region, info); 796 797 if (ret) { 798 region->nr_mmaps = 1; 799 region->mmaps = g_new0(VFIOMmap, region->nr_mmaps); 800 region->mmaps[0].offset = 0; 801 region->mmaps[0].size = region->size; 802 } 803 } 804 } 805 806 g_free(info); 807 808 trace_vfio_region_setup(vbasedev->name, index, name, 809 region->flags, region->fd_offset, region->size); 810 return 0; 811 } 812 813 int vfio_region_mmap(VFIORegion *region) 814 { 815 int i, prot = 0; 816 char *name; 817 818 if (!region->mem) { 819 return 0; 820 } 821 822 prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 823 prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 824 825 for (i = 0; i < region->nr_mmaps; i++) { 826 region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot, 827 MAP_SHARED, region->vbasedev->fd, 828 region->fd_offset + 829 region->mmaps[i].offset); 830 if (region->mmaps[i].mmap == MAP_FAILED) { 831 int ret = -errno; 832 833 trace_vfio_region_mmap_fault(memory_region_name(region->mem), i, 834 region->fd_offset + 835 region->mmaps[i].offset, 836 region->fd_offset + 837 region->mmaps[i].offset + 838 region->mmaps[i].size - 1, ret); 839 840 region->mmaps[i].mmap = NULL; 841 842 for (i--; i >= 0; i--) { 843 memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); 844 munmap(region->mmaps[i].mmap, region->mmaps[i].size); 845 object_unparent(OBJECT(®ion->mmaps[i].mem)); 846 region->mmaps[i].mmap = NULL; 847 } 848 849 return ret; 850 } 851 852 name = g_strdup_printf("%s mmaps[%d]", 853 memory_region_name(region->mem), i); 854 memory_region_init_ram_device_ptr(®ion->mmaps[i].mem, 855 memory_region_owner(region->mem), 856 name, region->mmaps[i].size, 857 region->mmaps[i].mmap); 858 g_free(name); 859 memory_region_add_subregion(region->mem, region->mmaps[i].offset, 860 ®ion->mmaps[i].mem); 861 862 trace_vfio_region_mmap(memory_region_name(®ion->mmaps[i].mem), 863 region->mmaps[i].offset, 864 region->mmaps[i].offset + 865 region->mmaps[i].size - 1); 866 } 867 868 return 0; 869 } 870 871 void vfio_region_exit(VFIORegion *region) 872 { 873 int i; 874 875 if (!region->mem) { 876 return; 877 } 878 879 for (i = 0; i < region->nr_mmaps; i++) { 880 if (region->mmaps[i].mmap) { 881 memory_region_del_subregion(region->mem, ®ion->mmaps[i].mem); 882 } 883 } 884 885 trace_vfio_region_exit(region->vbasedev->name, region->nr); 886 } 887 888 void vfio_region_finalize(VFIORegion *region) 889 { 890 int i; 891 892 if (!region->mem) { 893 return; 894 } 895 896 for (i = 0; i < region->nr_mmaps; i++) { 897 if (region->mmaps[i].mmap) { 898 munmap(region->mmaps[i].mmap, region->mmaps[i].size); 899 object_unparent(OBJECT(®ion->mmaps[i].mem)); 900 } 901 } 902 903 object_unparent(OBJECT(region->mem)); 904 905 g_free(region->mem); 906 g_free(region->mmaps); 907 908 trace_vfio_region_finalize(region->vbasedev->name, region->nr); 909 910 region->mem = NULL; 911 region->mmaps = NULL; 912 region->nr_mmaps = 0; 913 region->size = 0; 914 region->flags = 0; 915 region->nr = 0; 916 } 917 918 void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled) 919 { 920 int i; 921 922 if (!region->mem) { 923 return; 924 } 925 926 for (i = 0; i < region->nr_mmaps; i++) { 927 if (region->mmaps[i].mmap) { 928 memory_region_set_enabled(®ion->mmaps[i].mem, enabled); 929 } 930 } 931 932 trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem), 933 enabled); 934 } 935 936 void vfio_reset_handler(void *opaque) 937 { 938 VFIOGroup *group; 939 VFIODevice *vbasedev; 940 941 QLIST_FOREACH(group, &vfio_group_list, next) { 942 QLIST_FOREACH(vbasedev, &group->device_list, next) { 943 if (vbasedev->dev->realized) { 944 vbasedev->ops->vfio_compute_needs_reset(vbasedev); 945 } 946 } 947 } 948 949 QLIST_FOREACH(group, &vfio_group_list, next) { 950 QLIST_FOREACH(vbasedev, &group->device_list, next) { 951 if (vbasedev->dev->realized && vbasedev->needs_reset) { 952 vbasedev->ops->vfio_hot_reset_multi(vbasedev); 953 } 954 } 955 } 956 } 957 958 static void vfio_kvm_device_add_group(VFIOGroup *group) 959 { 960 #ifdef CONFIG_KVM 961 struct kvm_device_attr attr = { 962 .group = KVM_DEV_VFIO_GROUP, 963 .attr = KVM_DEV_VFIO_GROUP_ADD, 964 .addr = (uint64_t)(unsigned long)&group->fd, 965 }; 966 967 if (!kvm_enabled()) { 968 return; 969 } 970 971 if (vfio_kvm_device_fd < 0) { 972 struct kvm_create_device cd = { 973 .type = KVM_DEV_TYPE_VFIO, 974 }; 975 976 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) { 977 error_report("Failed to create KVM VFIO device: %m"); 978 return; 979 } 980 981 vfio_kvm_device_fd = cd.fd; 982 } 983 984 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 985 error_report("Failed to add group %d to KVM VFIO device: %m", 986 group->groupid); 987 } 988 #endif 989 } 990 991 static void vfio_kvm_device_del_group(VFIOGroup *group) 992 { 993 #ifdef CONFIG_KVM 994 struct kvm_device_attr attr = { 995 .group = KVM_DEV_VFIO_GROUP, 996 .attr = KVM_DEV_VFIO_GROUP_DEL, 997 .addr = (uint64_t)(unsigned long)&group->fd, 998 }; 999 1000 if (vfio_kvm_device_fd < 0) { 1001 return; 1002 } 1003 1004 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { 1005 error_report("Failed to remove group %d from KVM VFIO device: %m", 1006 group->groupid); 1007 } 1008 #endif 1009 } 1010 1011 static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as) 1012 { 1013 VFIOAddressSpace *space; 1014 1015 QLIST_FOREACH(space, &vfio_address_spaces, list) { 1016 if (space->as == as) { 1017 return space; 1018 } 1019 } 1020 1021 /* No suitable VFIOAddressSpace, create a new one */ 1022 space = g_malloc0(sizeof(*space)); 1023 space->as = as; 1024 QLIST_INIT(&space->containers); 1025 1026 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list); 1027 1028 return space; 1029 } 1030 1031 static void vfio_put_address_space(VFIOAddressSpace *space) 1032 { 1033 if (QLIST_EMPTY(&space->containers)) { 1034 QLIST_REMOVE(space, list); 1035 g_free(space); 1036 } 1037 } 1038 1039 static int vfio_connect_container(VFIOGroup *group, AddressSpace *as, 1040 Error **errp) 1041 { 1042 VFIOContainer *container; 1043 int ret, fd; 1044 VFIOAddressSpace *space; 1045 1046 space = vfio_get_address_space(as); 1047 1048 /* 1049 * VFIO is currently incompatible with memory ballooning insofar as the 1050 * madvise to purge (zap) the page from QEMU's address space does not 1051 * interact with the memory API and therefore leaves stale virtual to 1052 * physical mappings in the IOMMU if the page was previously pinned. We 1053 * therefore add a balloon inhibit for each group added to a container, 1054 * whether the container is used individually or shared. This provides 1055 * us with options to allow devices within a group to opt-in and allow 1056 * ballooning, so long as it is done consistently for a group (for instance 1057 * if the device is an mdev device where it is known that the host vendor 1058 * driver will never pin pages outside of the working set of the guest 1059 * driver, which would thus not be ballooning candidates). 1060 * 1061 * The first opportunity to induce pinning occurs here where we attempt to 1062 * attach the group to existing containers within the AddressSpace. If any 1063 * pages are already zapped from the virtual address space, such as from a 1064 * previous ballooning opt-in, new pinning will cause valid mappings to be 1065 * re-established. Likewise, when the overall MemoryListener for a new 1066 * container is registered, a replay of mappings within the AddressSpace 1067 * will occur, re-establishing any previously zapped pages as well. 1068 * 1069 * NB. Balloon inhibiting does not currently block operation of the 1070 * balloon driver or revoke previously pinned pages, it only prevents 1071 * calling madvise to modify the virtual mapping of ballooned pages. 1072 */ 1073 qemu_balloon_inhibit(true); 1074 1075 QLIST_FOREACH(container, &space->containers, next) { 1076 if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) { 1077 group->container = container; 1078 QLIST_INSERT_HEAD(&container->group_list, group, container_next); 1079 vfio_kvm_device_add_group(group); 1080 return 0; 1081 } 1082 } 1083 1084 fd = qemu_open("/dev/vfio/vfio", O_RDWR); 1085 if (fd < 0) { 1086 error_setg_errno(errp, errno, "failed to open /dev/vfio/vfio"); 1087 ret = -errno; 1088 goto put_space_exit; 1089 } 1090 1091 ret = ioctl(fd, VFIO_GET_API_VERSION); 1092 if (ret != VFIO_API_VERSION) { 1093 error_setg(errp, "supported vfio version: %d, " 1094 "reported version: %d", VFIO_API_VERSION, ret); 1095 ret = -EINVAL; 1096 goto close_fd_exit; 1097 } 1098 1099 container = g_malloc0(sizeof(*container)); 1100 container->space = space; 1101 container->fd = fd; 1102 QLIST_INIT(&container->giommu_list); 1103 QLIST_INIT(&container->hostwin_list); 1104 if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) || 1105 ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)) { 1106 bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU); 1107 struct vfio_iommu_type1_info info; 1108 1109 ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd); 1110 if (ret) { 1111 error_setg_errno(errp, errno, "failed to set group container"); 1112 ret = -errno; 1113 goto free_container_exit; 1114 } 1115 1116 container->iommu_type = v2 ? VFIO_TYPE1v2_IOMMU : VFIO_TYPE1_IOMMU; 1117 ret = ioctl(fd, VFIO_SET_IOMMU, container->iommu_type); 1118 if (ret) { 1119 error_setg_errno(errp, errno, "failed to set iommu for container"); 1120 ret = -errno; 1121 goto free_container_exit; 1122 } 1123 1124 /* 1125 * FIXME: This assumes that a Type1 IOMMU can map any 64-bit 1126 * IOVA whatsoever. That's not actually true, but the current 1127 * kernel interface doesn't tell us what it can map, and the 1128 * existing Type1 IOMMUs generally support any IOVA we're 1129 * going to actually try in practice. 1130 */ 1131 info.argsz = sizeof(info); 1132 ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info); 1133 /* Ignore errors */ 1134 if (ret || !(info.flags & VFIO_IOMMU_INFO_PGSIZES)) { 1135 /* Assume 4k IOVA page size */ 1136 info.iova_pgsizes = 4096; 1137 } 1138 vfio_host_win_add(container, 0, (hwaddr)-1, info.iova_pgsizes); 1139 container->pgsizes = info.iova_pgsizes; 1140 } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU) || 1141 ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU)) { 1142 struct vfio_iommu_spapr_tce_info info; 1143 bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU); 1144 1145 ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd); 1146 if (ret) { 1147 error_setg_errno(errp, errno, "failed to set group container"); 1148 ret = -errno; 1149 goto free_container_exit; 1150 } 1151 container->iommu_type = 1152 v2 ? VFIO_SPAPR_TCE_v2_IOMMU : VFIO_SPAPR_TCE_IOMMU; 1153 ret = ioctl(fd, VFIO_SET_IOMMU, container->iommu_type); 1154 if (ret) { 1155 container->iommu_type = VFIO_SPAPR_TCE_IOMMU; 1156 v2 = false; 1157 ret = ioctl(fd, VFIO_SET_IOMMU, container->iommu_type); 1158 } 1159 if (ret) { 1160 error_setg_errno(errp, errno, "failed to set iommu for container"); 1161 ret = -errno; 1162 goto free_container_exit; 1163 } 1164 1165 /* 1166 * The host kernel code implementing VFIO_IOMMU_DISABLE is called 1167 * when container fd is closed so we do not call it explicitly 1168 * in this file. 1169 */ 1170 if (!v2) { 1171 ret = ioctl(fd, VFIO_IOMMU_ENABLE); 1172 if (ret) { 1173 error_setg_errno(errp, errno, "failed to enable container"); 1174 ret = -errno; 1175 goto free_container_exit; 1176 } 1177 } else { 1178 container->prereg_listener = vfio_prereg_listener; 1179 1180 memory_listener_register(&container->prereg_listener, 1181 &address_space_memory); 1182 if (container->error) { 1183 memory_listener_unregister(&container->prereg_listener); 1184 ret = container->error; 1185 error_setg(errp, 1186 "RAM memory listener initialization failed for container"); 1187 goto free_container_exit; 1188 } 1189 } 1190 1191 info.argsz = sizeof(info); 1192 ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); 1193 if (ret) { 1194 error_setg_errno(errp, errno, 1195 "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed"); 1196 ret = -errno; 1197 if (v2) { 1198 memory_listener_unregister(&container->prereg_listener); 1199 } 1200 goto free_container_exit; 1201 } 1202 1203 if (v2) { 1204 container->pgsizes = info.ddw.pgsizes; 1205 /* 1206 * There is a default window in just created container. 1207 * To make region_add/del simpler, we better remove this 1208 * window now and let those iommu_listener callbacks 1209 * create/remove them when needed. 1210 */ 1211 ret = vfio_spapr_remove_window(container, info.dma32_window_start); 1212 if (ret) { 1213 error_setg_errno(errp, -ret, 1214 "failed to remove existing window"); 1215 goto free_container_exit; 1216 } 1217 } else { 1218 /* The default table uses 4K pages */ 1219 container->pgsizes = 0x1000; 1220 vfio_host_win_add(container, info.dma32_window_start, 1221 info.dma32_window_start + 1222 info.dma32_window_size - 1, 1223 0x1000); 1224 } 1225 } else { 1226 error_setg(errp, "No available IOMMU models"); 1227 ret = -EINVAL; 1228 goto free_container_exit; 1229 } 1230 1231 vfio_kvm_device_add_group(group); 1232 1233 QLIST_INIT(&container->group_list); 1234 QLIST_INSERT_HEAD(&space->containers, container, next); 1235 1236 group->container = container; 1237 QLIST_INSERT_HEAD(&container->group_list, group, container_next); 1238 1239 container->listener = vfio_memory_listener; 1240 1241 memory_listener_register(&container->listener, container->space->as); 1242 1243 if (container->error) { 1244 ret = container->error; 1245 error_setg_errno(errp, -ret, 1246 "memory listener initialization failed for container"); 1247 goto listener_release_exit; 1248 } 1249 1250 container->initialized = true; 1251 1252 return 0; 1253 listener_release_exit: 1254 QLIST_REMOVE(group, container_next); 1255 QLIST_REMOVE(container, next); 1256 vfio_kvm_device_del_group(group); 1257 vfio_listener_release(container); 1258 1259 free_container_exit: 1260 g_free(container); 1261 1262 close_fd_exit: 1263 close(fd); 1264 1265 put_space_exit: 1266 qemu_balloon_inhibit(false); 1267 vfio_put_address_space(space); 1268 1269 return ret; 1270 } 1271 1272 static void vfio_disconnect_container(VFIOGroup *group) 1273 { 1274 VFIOContainer *container = group->container; 1275 1276 QLIST_REMOVE(group, container_next); 1277 group->container = NULL; 1278 1279 /* 1280 * Explicitly release the listener first before unset container, 1281 * since unset may destroy the backend container if it's the last 1282 * group. 1283 */ 1284 if (QLIST_EMPTY(&container->group_list)) { 1285 vfio_listener_release(container); 1286 } 1287 1288 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) { 1289 error_report("vfio: error disconnecting group %d from container", 1290 group->groupid); 1291 } 1292 1293 if (QLIST_EMPTY(&container->group_list)) { 1294 VFIOAddressSpace *space = container->space; 1295 VFIOGuestIOMMU *giommu, *tmp; 1296 1297 QLIST_REMOVE(container, next); 1298 1299 QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) { 1300 memory_region_unregister_iommu_notifier( 1301 MEMORY_REGION(giommu->iommu), &giommu->n); 1302 QLIST_REMOVE(giommu, giommu_next); 1303 g_free(giommu); 1304 } 1305 1306 trace_vfio_disconnect_container(container->fd); 1307 close(container->fd); 1308 g_free(container); 1309 1310 vfio_put_address_space(space); 1311 } 1312 } 1313 1314 VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp) 1315 { 1316 VFIOGroup *group; 1317 char path[32]; 1318 struct vfio_group_status status = { .argsz = sizeof(status) }; 1319 1320 QLIST_FOREACH(group, &vfio_group_list, next) { 1321 if (group->groupid == groupid) { 1322 /* Found it. Now is it already in the right context? */ 1323 if (group->container->space->as == as) { 1324 return group; 1325 } else { 1326 error_setg(errp, "group %d used in multiple address spaces", 1327 group->groupid); 1328 return NULL; 1329 } 1330 } 1331 } 1332 1333 group = g_malloc0(sizeof(*group)); 1334 1335 snprintf(path, sizeof(path), "/dev/vfio/%d", groupid); 1336 group->fd = qemu_open(path, O_RDWR); 1337 if (group->fd < 0) { 1338 error_setg_errno(errp, errno, "failed to open %s", path); 1339 goto free_group_exit; 1340 } 1341 1342 if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) { 1343 error_setg_errno(errp, errno, "failed to get group %d status", groupid); 1344 goto close_fd_exit; 1345 } 1346 1347 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) { 1348 error_setg(errp, "group %d is not viable", groupid); 1349 error_append_hint(errp, 1350 "Please ensure all devices within the iommu_group " 1351 "are bound to their vfio bus driver.\n"); 1352 goto close_fd_exit; 1353 } 1354 1355 group->groupid = groupid; 1356 QLIST_INIT(&group->device_list); 1357 1358 if (vfio_connect_container(group, as, errp)) { 1359 error_prepend(errp, "failed to setup container for group %d: ", 1360 groupid); 1361 goto close_fd_exit; 1362 } 1363 1364 if (QLIST_EMPTY(&vfio_group_list)) { 1365 qemu_register_reset(vfio_reset_handler, NULL); 1366 } 1367 1368 QLIST_INSERT_HEAD(&vfio_group_list, group, next); 1369 1370 return group; 1371 1372 close_fd_exit: 1373 close(group->fd); 1374 1375 free_group_exit: 1376 g_free(group); 1377 1378 return NULL; 1379 } 1380 1381 void vfio_put_group(VFIOGroup *group) 1382 { 1383 if (!group || !QLIST_EMPTY(&group->device_list)) { 1384 return; 1385 } 1386 1387 if (!group->balloon_allowed) { 1388 qemu_balloon_inhibit(false); 1389 } 1390 vfio_kvm_device_del_group(group); 1391 vfio_disconnect_container(group); 1392 QLIST_REMOVE(group, next); 1393 trace_vfio_put_group(group->fd); 1394 close(group->fd); 1395 g_free(group); 1396 1397 if (QLIST_EMPTY(&vfio_group_list)) { 1398 qemu_unregister_reset(vfio_reset_handler, NULL); 1399 } 1400 } 1401 1402 int vfio_get_device(VFIOGroup *group, const char *name, 1403 VFIODevice *vbasedev, Error **errp) 1404 { 1405 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) }; 1406 int ret, fd; 1407 1408 fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name); 1409 if (fd < 0) { 1410 error_setg_errno(errp, errno, "error getting device from group %d", 1411 group->groupid); 1412 error_append_hint(errp, 1413 "Verify all devices in group %d are bound to vfio-<bus> " 1414 "or pci-stub and not already in use\n", group->groupid); 1415 return fd; 1416 } 1417 1418 ret = ioctl(fd, VFIO_DEVICE_GET_INFO, &dev_info); 1419 if (ret) { 1420 error_setg_errno(errp, errno, "error getting device info"); 1421 close(fd); 1422 return ret; 1423 } 1424 1425 /* 1426 * Clear the balloon inhibitor for this group if the driver knows the 1427 * device operates compatibly with ballooning. Setting must be consistent 1428 * per group, but since compatibility is really only possible with mdev 1429 * currently, we expect singleton groups. 1430 */ 1431 if (vbasedev->balloon_allowed != group->balloon_allowed) { 1432 if (!QLIST_EMPTY(&group->device_list)) { 1433 error_setg(errp, 1434 "Inconsistent device balloon setting within group"); 1435 close(fd); 1436 return -1; 1437 } 1438 1439 if (!group->balloon_allowed) { 1440 group->balloon_allowed = true; 1441 qemu_balloon_inhibit(false); 1442 } 1443 } 1444 1445 vbasedev->fd = fd; 1446 vbasedev->group = group; 1447 QLIST_INSERT_HEAD(&group->device_list, vbasedev, next); 1448 1449 vbasedev->num_irqs = dev_info.num_irqs; 1450 vbasedev->num_regions = dev_info.num_regions; 1451 vbasedev->flags = dev_info.flags; 1452 1453 trace_vfio_get_device(name, dev_info.flags, dev_info.num_regions, 1454 dev_info.num_irqs); 1455 1456 vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET); 1457 return 0; 1458 } 1459 1460 void vfio_put_base_device(VFIODevice *vbasedev) 1461 { 1462 if (!vbasedev->group) { 1463 return; 1464 } 1465 QLIST_REMOVE(vbasedev, next); 1466 vbasedev->group = NULL; 1467 trace_vfio_put_base_device(vbasedev->fd); 1468 close(vbasedev->fd); 1469 } 1470 1471 int vfio_get_region_info(VFIODevice *vbasedev, int index, 1472 struct vfio_region_info **info) 1473 { 1474 size_t argsz = sizeof(struct vfio_region_info); 1475 1476 *info = g_malloc0(argsz); 1477 1478 (*info)->index = index; 1479 retry: 1480 (*info)->argsz = argsz; 1481 1482 if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) { 1483 g_free(*info); 1484 *info = NULL; 1485 return -errno; 1486 } 1487 1488 if ((*info)->argsz > argsz) { 1489 argsz = (*info)->argsz; 1490 *info = g_realloc(*info, argsz); 1491 1492 goto retry; 1493 } 1494 1495 return 0; 1496 } 1497 1498 int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type, 1499 uint32_t subtype, struct vfio_region_info **info) 1500 { 1501 int i; 1502 1503 for (i = 0; i < vbasedev->num_regions; i++) { 1504 struct vfio_info_cap_header *hdr; 1505 struct vfio_region_info_cap_type *cap_type; 1506 1507 if (vfio_get_region_info(vbasedev, i, info)) { 1508 continue; 1509 } 1510 1511 hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE); 1512 if (!hdr) { 1513 g_free(*info); 1514 continue; 1515 } 1516 1517 cap_type = container_of(hdr, struct vfio_region_info_cap_type, header); 1518 1519 trace_vfio_get_dev_region(vbasedev->name, i, 1520 cap_type->type, cap_type->subtype); 1521 1522 if (cap_type->type == type && cap_type->subtype == subtype) { 1523 return 0; 1524 } 1525 1526 g_free(*info); 1527 } 1528 1529 *info = NULL; 1530 return -ENODEV; 1531 } 1532 1533 bool vfio_has_region_cap(VFIODevice *vbasedev, int region, uint16_t cap_type) 1534 { 1535 struct vfio_region_info *info = NULL; 1536 bool ret = false; 1537 1538 if (!vfio_get_region_info(vbasedev, region, &info)) { 1539 if (vfio_get_region_info_cap(info, cap_type)) { 1540 ret = true; 1541 } 1542 g_free(info); 1543 } 1544 1545 return ret; 1546 } 1547 1548 /* 1549 * Interfaces for IBM EEH (Enhanced Error Handling) 1550 */ 1551 static bool vfio_eeh_container_ok(VFIOContainer *container) 1552 { 1553 /* 1554 * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO 1555 * implementation is broken if there are multiple groups in a 1556 * container. The hardware works in units of Partitionable 1557 * Endpoints (== IOMMU groups) and the EEH operations naively 1558 * iterate across all groups in the container, without any logic 1559 * to make sure the groups have their state synchronized. For 1560 * certain operations (ENABLE) that might be ok, until an error 1561 * occurs, but for others (GET_STATE) it's clearly broken. 1562 */ 1563 1564 /* 1565 * XXX Once fixed kernels exist, test for them here 1566 */ 1567 1568 if (QLIST_EMPTY(&container->group_list)) { 1569 return false; 1570 } 1571 1572 if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) { 1573 return false; 1574 } 1575 1576 return true; 1577 } 1578 1579 static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op) 1580 { 1581 struct vfio_eeh_pe_op pe_op = { 1582 .argsz = sizeof(pe_op), 1583 .op = op, 1584 }; 1585 int ret; 1586 1587 if (!vfio_eeh_container_ok(container)) { 1588 error_report("vfio/eeh: EEH_PE_OP 0x%x: " 1589 "kernel requires a container with exactly one group", op); 1590 return -EPERM; 1591 } 1592 1593 ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op); 1594 if (ret < 0) { 1595 error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op); 1596 return -errno; 1597 } 1598 1599 return ret; 1600 } 1601 1602 static VFIOContainer *vfio_eeh_as_container(AddressSpace *as) 1603 { 1604 VFIOAddressSpace *space = vfio_get_address_space(as); 1605 VFIOContainer *container = NULL; 1606 1607 if (QLIST_EMPTY(&space->containers)) { 1608 /* No containers to act on */ 1609 goto out; 1610 } 1611 1612 container = QLIST_FIRST(&space->containers); 1613 1614 if (QLIST_NEXT(container, next)) { 1615 /* We don't yet have logic to synchronize EEH state across 1616 * multiple containers */ 1617 container = NULL; 1618 goto out; 1619 } 1620 1621 out: 1622 vfio_put_address_space(space); 1623 return container; 1624 } 1625 1626 bool vfio_eeh_as_ok(AddressSpace *as) 1627 { 1628 VFIOContainer *container = vfio_eeh_as_container(as); 1629 1630 return (container != NULL) && vfio_eeh_container_ok(container); 1631 } 1632 1633 int vfio_eeh_as_op(AddressSpace *as, uint32_t op) 1634 { 1635 VFIOContainer *container = vfio_eeh_as_container(as); 1636 1637 if (!container) { 1638 return -ENODEV; 1639 } 1640 return vfio_eeh_container_op(container, op); 1641 } 1642