1 /* 2 * VFIO utility 3 * 4 * Copyright 2016 - 2018 Red Hat, Inc. 5 * 6 * Authors: 7 * Fam Zheng <famz@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include <sys/ioctl.h> 15 #include <linux/vfio.h> 16 #include "qapi/error.h" 17 #include "exec/ramlist.h" 18 #include "exec/cpu-common.h" 19 #include "exec/memory.h" 20 #include "trace.h" 21 #include "qemu/error-report.h" 22 #include "standard-headers/linux/pci_regs.h" 23 #include "qemu/event_notifier.h" 24 #include "qemu/vfio-helpers.h" 25 #include "qemu/lockable.h" 26 #include "trace.h" 27 28 #define QEMU_VFIO_DEBUG 0 29 30 #define QEMU_VFIO_IOVA_MIN 0x10000ULL 31 /* XXX: Once VFIO exposes the iova bit width in the IOMMU capability interface, 32 * we can use a runtime limit; alternatively it's also possible to do platform 33 * specific detection by reading sysfs entries. Until then, 39 is a safe bet. 34 **/ 35 #define QEMU_VFIO_IOVA_MAX (1ULL << 39) 36 37 typedef struct { 38 /* Page aligned addr. */ 39 void *host; 40 size_t size; 41 uint64_t iova; 42 } IOVAMapping; 43 44 struct IOVARange { 45 uint64_t start; 46 uint64_t end; 47 }; 48 49 struct QEMUVFIOState { 50 QemuMutex lock; 51 52 /* These fields are protected by BQL */ 53 int container; 54 int group; 55 int device; 56 RAMBlockNotifier ram_notifier; 57 struct vfio_region_info config_region_info, bar_region_info[6]; 58 struct IOVARange *usable_iova_ranges; 59 uint8_t nb_iova_ranges; 60 61 /* These fields are protected by @lock */ 62 /* VFIO's IO virtual address space is managed by splitting into a few 63 * sections: 64 * 65 * --------------- <= 0 66 * |xxxxxxxxxxxxx| 67 * |-------------| <= QEMU_VFIO_IOVA_MIN 68 * | | 69 * | Fixed | 70 * | | 71 * |-------------| <= low_water_mark 72 * | | 73 * | Free | 74 * | | 75 * |-------------| <= high_water_mark 76 * | | 77 * | Temp | 78 * | | 79 * |-------------| <= QEMU_VFIO_IOVA_MAX 80 * |xxxxxxxxxxxxx| 81 * |xxxxxxxxxxxxx| 82 * --------------- 83 * 84 * - Addresses lower than QEMU_VFIO_IOVA_MIN are reserved as invalid; 85 * 86 * - Fixed mappings of HVAs are assigned "low" IOVAs in the range of 87 * [QEMU_VFIO_IOVA_MIN, low_water_mark). Once allocated they will not be 88 * reclaimed - low_water_mark never shrinks; 89 * 90 * - IOVAs in range [low_water_mark, high_water_mark) are free; 91 * 92 * - IOVAs in range [high_water_mark, QEMU_VFIO_IOVA_MAX) are volatile 93 * mappings. At each qemu_vfio_dma_reset_temporary() call, the whole area 94 * is recycled. The caller should make sure I/O's depending on these 95 * mappings are completed before calling. 96 **/ 97 uint64_t low_water_mark; 98 uint64_t high_water_mark; 99 IOVAMapping *mappings; 100 int nr_mappings; 101 }; 102 103 /** 104 * Find group file by PCI device address as specified @device, and return the 105 * path. The returned string is owned by caller and should be g_free'ed later. 106 */ 107 static char *sysfs_find_group_file(const char *device, Error **errp) 108 { 109 char *sysfs_link; 110 char *sysfs_group; 111 char *p; 112 char *path = NULL; 113 114 sysfs_link = g_strdup_printf("/sys/bus/pci/devices/%s/iommu_group", device); 115 sysfs_group = g_malloc0(PATH_MAX); 116 if (readlink(sysfs_link, sysfs_group, PATH_MAX - 1) == -1) { 117 error_setg_errno(errp, errno, "Failed to find iommu group sysfs path"); 118 goto out; 119 } 120 p = strrchr(sysfs_group, '/'); 121 if (!p) { 122 error_setg(errp, "Failed to find iommu group number"); 123 goto out; 124 } 125 126 path = g_strdup_printf("/dev/vfio/%s", p + 1); 127 out: 128 g_free(sysfs_link); 129 g_free(sysfs_group); 130 return path; 131 } 132 133 static inline void assert_bar_index_valid(QEMUVFIOState *s, int index) 134 { 135 assert(index >= 0 && index < ARRAY_SIZE(s->bar_region_info)); 136 } 137 138 static int qemu_vfio_pci_init_bar(QEMUVFIOState *s, int index, Error **errp) 139 { 140 g_autofree char *barname = NULL; 141 assert_bar_index_valid(s, index); 142 s->bar_region_info[index] = (struct vfio_region_info) { 143 .index = VFIO_PCI_BAR0_REGION_INDEX + index, 144 .argsz = sizeof(struct vfio_region_info), 145 }; 146 if (ioctl(s->device, VFIO_DEVICE_GET_REGION_INFO, &s->bar_region_info[index])) { 147 error_setg_errno(errp, errno, "Failed to get BAR region info"); 148 return -errno; 149 } 150 barname = g_strdup_printf("bar[%d]", index); 151 trace_qemu_vfio_region_info(barname, s->bar_region_info[index].offset, 152 s->bar_region_info[index].size, 153 s->bar_region_info[index].cap_offset); 154 155 return 0; 156 } 157 158 /** 159 * Map a PCI bar area. 160 */ 161 void *qemu_vfio_pci_map_bar(QEMUVFIOState *s, int index, 162 uint64_t offset, uint64_t size, int prot, 163 Error **errp) 164 { 165 void *p; 166 assert(QEMU_IS_ALIGNED(offset, qemu_real_host_page_size())); 167 assert_bar_index_valid(s, index); 168 p = mmap(NULL, MIN(size, s->bar_region_info[index].size - offset), 169 prot, MAP_SHARED, 170 s->device, s->bar_region_info[index].offset + offset); 171 trace_qemu_vfio_pci_map_bar(index, s->bar_region_info[index].offset , 172 size, offset, p); 173 if (p == MAP_FAILED) { 174 error_setg_errno(errp, errno, "Failed to map BAR region"); 175 p = NULL; 176 } 177 return p; 178 } 179 180 /** 181 * Unmap a PCI bar area. 182 */ 183 void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar, 184 uint64_t offset, uint64_t size) 185 { 186 if (bar) { 187 munmap(bar, MIN(size, s->bar_region_info[index].size - offset)); 188 } 189 } 190 191 /** 192 * Initialize device IRQ with @irq_type and register an event notifier. 193 */ 194 int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, 195 int irq_type, Error **errp) 196 { 197 int r; 198 struct vfio_irq_set *irq_set; 199 size_t irq_set_size; 200 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) }; 201 202 irq_info.index = irq_type; 203 if (ioctl(s->device, VFIO_DEVICE_GET_IRQ_INFO, &irq_info)) { 204 error_setg_errno(errp, errno, "Failed to get device interrupt info"); 205 return -errno; 206 } 207 if (!(irq_info.flags & VFIO_IRQ_INFO_EVENTFD)) { 208 error_setg(errp, "Device interrupt doesn't support eventfd"); 209 return -EINVAL; 210 } 211 212 irq_set_size = sizeof(*irq_set) + sizeof(int); 213 irq_set = g_malloc0(irq_set_size); 214 215 /* Get to a known IRQ state */ 216 *irq_set = (struct vfio_irq_set) { 217 .argsz = irq_set_size, 218 .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, 219 .index = irq_info.index, 220 .start = 0, 221 .count = 1, 222 }; 223 224 *(int *)&irq_set->data = event_notifier_get_fd(e); 225 r = ioctl(s->device, VFIO_DEVICE_SET_IRQS, irq_set); 226 g_free(irq_set); 227 if (r) { 228 error_setg_errno(errp, errno, "Failed to setup device interrupt"); 229 return -errno; 230 } 231 return 0; 232 } 233 234 static int qemu_vfio_pci_read_config(QEMUVFIOState *s, void *buf, 235 int size, int ofs) 236 { 237 int ret; 238 239 trace_qemu_vfio_pci_read_config(buf, ofs, size, 240 s->config_region_info.offset, 241 s->config_region_info.size); 242 assert(QEMU_IS_ALIGNED(s->config_region_info.offset + ofs, size)); 243 ret = RETRY_ON_EINTR( 244 pread(s->device, buf, size, s->config_region_info.offset + ofs) 245 ); 246 return ret == size ? 0 : -errno; 247 } 248 249 static int qemu_vfio_pci_write_config(QEMUVFIOState *s, void *buf, int size, int ofs) 250 { 251 int ret; 252 253 trace_qemu_vfio_pci_write_config(buf, ofs, size, 254 s->config_region_info.offset, 255 s->config_region_info.size); 256 assert(QEMU_IS_ALIGNED(s->config_region_info.offset + ofs, size)); 257 ret = RETRY_ON_EINTR( 258 pwrite(s->device, buf, size, s->config_region_info.offset + ofs) 259 ); 260 return ret == size ? 0 : -errno; 261 } 262 263 static void collect_usable_iova_ranges(QEMUVFIOState *s, void *buf) 264 { 265 struct vfio_iommu_type1_info *info = (struct vfio_iommu_type1_info *)buf; 266 struct vfio_info_cap_header *cap = (void *)buf + info->cap_offset; 267 struct vfio_iommu_type1_info_cap_iova_range *cap_iova_range; 268 int i; 269 270 while (cap->id != VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE) { 271 if (!cap->next) { 272 return; 273 } 274 cap = buf + cap->next; 275 } 276 277 cap_iova_range = (struct vfio_iommu_type1_info_cap_iova_range *)cap; 278 279 s->nb_iova_ranges = cap_iova_range->nr_iovas; 280 if (s->nb_iova_ranges > 1) { 281 s->usable_iova_ranges = 282 g_renew(struct IOVARange, s->usable_iova_ranges, 283 s->nb_iova_ranges); 284 } 285 286 for (i = 0; i < s->nb_iova_ranges; i++) { 287 s->usable_iova_ranges[i].start = cap_iova_range->iova_ranges[i].start; 288 s->usable_iova_ranges[i].end = cap_iova_range->iova_ranges[i].end; 289 } 290 } 291 292 static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device, 293 Error **errp) 294 { 295 int ret; 296 int i; 297 uint16_t pci_cmd; 298 struct vfio_group_status group_status = { .argsz = sizeof(group_status) }; 299 struct vfio_iommu_type1_info *iommu_info = NULL; 300 size_t iommu_info_size = sizeof(*iommu_info); 301 struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; 302 char *group_file = NULL; 303 304 s->usable_iova_ranges = NULL; 305 306 /* Create a new container */ 307 s->container = open("/dev/vfio/vfio", O_RDWR); 308 309 if (s->container == -1) { 310 error_setg_errno(errp, errno, "Failed to open /dev/vfio/vfio"); 311 return -errno; 312 } 313 if (ioctl(s->container, VFIO_GET_API_VERSION) != VFIO_API_VERSION) { 314 error_setg(errp, "Invalid VFIO version"); 315 ret = -EINVAL; 316 goto fail_container; 317 } 318 319 if (!ioctl(s->container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) { 320 error_setg_errno(errp, errno, "VFIO IOMMU Type1 is not supported"); 321 ret = -EINVAL; 322 goto fail_container; 323 } 324 325 /* Open the group */ 326 group_file = sysfs_find_group_file(device, errp); 327 if (!group_file) { 328 ret = -EINVAL; 329 goto fail_container; 330 } 331 332 s->group = open(group_file, O_RDWR); 333 if (s->group == -1) { 334 error_setg_errno(errp, errno, "Failed to open VFIO group file: %s", 335 group_file); 336 g_free(group_file); 337 ret = -errno; 338 goto fail_container; 339 } 340 g_free(group_file); 341 342 /* Test the group is viable and available */ 343 if (ioctl(s->group, VFIO_GROUP_GET_STATUS, &group_status)) { 344 error_setg_errno(errp, errno, "Failed to get VFIO group status"); 345 ret = -errno; 346 goto fail; 347 } 348 349 if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { 350 error_setg(errp, "VFIO group is not viable"); 351 ret = -EINVAL; 352 goto fail; 353 } 354 355 /* Add the group to the container */ 356 if (ioctl(s->group, VFIO_GROUP_SET_CONTAINER, &s->container)) { 357 error_setg_errno(errp, errno, "Failed to add group to VFIO container"); 358 ret = -errno; 359 goto fail; 360 } 361 362 /* Enable the IOMMU model we want */ 363 if (ioctl(s->container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)) { 364 error_setg_errno(errp, errno, "Failed to set VFIO IOMMU type"); 365 ret = -errno; 366 goto fail; 367 } 368 369 iommu_info = g_malloc0(iommu_info_size); 370 iommu_info->argsz = iommu_info_size; 371 372 /* Get additional IOMMU info */ 373 if (ioctl(s->container, VFIO_IOMMU_GET_INFO, iommu_info)) { 374 error_setg_errno(errp, errno, "Failed to get IOMMU info"); 375 ret = -errno; 376 goto fail; 377 } 378 379 /* 380 * if the kernel does not report usable IOVA regions, choose 381 * the legacy [QEMU_VFIO_IOVA_MIN, QEMU_VFIO_IOVA_MAX -1] region 382 */ 383 s->nb_iova_ranges = 1; 384 s->usable_iova_ranges = g_new0(struct IOVARange, 1); 385 s->usable_iova_ranges[0].start = QEMU_VFIO_IOVA_MIN; 386 s->usable_iova_ranges[0].end = QEMU_VFIO_IOVA_MAX - 1; 387 388 if (iommu_info->argsz > iommu_info_size) { 389 iommu_info_size = iommu_info->argsz; 390 iommu_info = g_realloc(iommu_info, iommu_info_size); 391 if (ioctl(s->container, VFIO_IOMMU_GET_INFO, iommu_info)) { 392 ret = -errno; 393 goto fail; 394 } 395 collect_usable_iova_ranges(s, iommu_info); 396 } 397 398 s->device = ioctl(s->group, VFIO_GROUP_GET_DEVICE_FD, device); 399 400 if (s->device < 0) { 401 error_setg_errno(errp, errno, "Failed to get device fd"); 402 ret = -errno; 403 goto fail; 404 } 405 406 /* Test and setup the device */ 407 if (ioctl(s->device, VFIO_DEVICE_GET_INFO, &device_info)) { 408 error_setg_errno(errp, errno, "Failed to get device info"); 409 ret = -errno; 410 goto fail; 411 } 412 413 if (device_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX) { 414 error_setg(errp, "Invalid device regions"); 415 ret = -EINVAL; 416 goto fail; 417 } 418 419 s->config_region_info = (struct vfio_region_info) { 420 .index = VFIO_PCI_CONFIG_REGION_INDEX, 421 .argsz = sizeof(struct vfio_region_info), 422 }; 423 if (ioctl(s->device, VFIO_DEVICE_GET_REGION_INFO, &s->config_region_info)) { 424 error_setg_errno(errp, errno, "Failed to get config region info"); 425 ret = -errno; 426 goto fail; 427 } 428 trace_qemu_vfio_region_info("config", s->config_region_info.offset, 429 s->config_region_info.size, 430 s->config_region_info.cap_offset); 431 432 for (i = 0; i < ARRAY_SIZE(s->bar_region_info); i++) { 433 ret = qemu_vfio_pci_init_bar(s, i, errp); 434 if (ret) { 435 goto fail; 436 } 437 } 438 439 /* Enable bus master */ 440 ret = qemu_vfio_pci_read_config(s, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND); 441 if (ret) { 442 goto fail; 443 } 444 pci_cmd |= PCI_COMMAND_MASTER; 445 ret = qemu_vfio_pci_write_config(s, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND); 446 if (ret) { 447 goto fail; 448 } 449 g_free(iommu_info); 450 return 0; 451 fail: 452 g_free(s->usable_iova_ranges); 453 s->usable_iova_ranges = NULL; 454 s->nb_iova_ranges = 0; 455 g_free(iommu_info); 456 close(s->group); 457 fail_container: 458 close(s->container); 459 return ret; 460 } 461 462 static void qemu_vfio_ram_block_added(RAMBlockNotifier *n, void *host, 463 size_t size, size_t max_size) 464 { 465 QEMUVFIOState *s = container_of(n, QEMUVFIOState, ram_notifier); 466 Error *local_err = NULL; 467 int ret; 468 469 trace_qemu_vfio_ram_block_added(s, host, max_size); 470 ret = qemu_vfio_dma_map(s, host, max_size, false, NULL, &local_err); 471 if (ret) { 472 error_reportf_err(local_err, 473 "qemu_vfio_dma_map(%p, %zu) failed: ", 474 host, max_size); 475 } 476 } 477 478 static void qemu_vfio_ram_block_removed(RAMBlockNotifier *n, void *host, 479 size_t size, size_t max_size) 480 { 481 QEMUVFIOState *s = container_of(n, QEMUVFIOState, ram_notifier); 482 if (host) { 483 trace_qemu_vfio_ram_block_removed(s, host, max_size); 484 qemu_vfio_dma_unmap(s, host); 485 } 486 } 487 488 static void qemu_vfio_open_common(QEMUVFIOState *s) 489 { 490 qemu_mutex_init(&s->lock); 491 s->ram_notifier.ram_block_added = qemu_vfio_ram_block_added; 492 s->ram_notifier.ram_block_removed = qemu_vfio_ram_block_removed; 493 s->low_water_mark = QEMU_VFIO_IOVA_MIN; 494 s->high_water_mark = QEMU_VFIO_IOVA_MAX; 495 ram_block_notifier_add(&s->ram_notifier); 496 } 497 498 /** 499 * Open a PCI device, e.g. "0000:00:01.0". 500 */ 501 QEMUVFIOState *qemu_vfio_open_pci(const char *device, Error **errp) 502 { 503 int r; 504 QEMUVFIOState *s = g_new0(QEMUVFIOState, 1); 505 506 /* 507 * VFIO may pin all memory inside mappings, resulting it in pinning 508 * all memory inside RAM blocks unconditionally. 509 */ 510 r = ram_block_discard_disable(true); 511 if (r) { 512 error_setg_errno(errp, -r, "Cannot set discarding of RAM broken"); 513 g_free(s); 514 return NULL; 515 } 516 517 r = qemu_vfio_init_pci(s, device, errp); 518 if (r) { 519 ram_block_discard_disable(false); 520 g_free(s); 521 return NULL; 522 } 523 qemu_vfio_open_common(s); 524 return s; 525 } 526 527 static void qemu_vfio_dump_mappings(QEMUVFIOState *s) 528 { 529 for (int i = 0; i < s->nr_mappings; ++i) { 530 trace_qemu_vfio_dump_mapping(s->mappings[i].host, 531 s->mappings[i].iova, 532 s->mappings[i].size); 533 } 534 } 535 536 /** 537 * Find the mapping entry that contains [host, host + size) and set @index to 538 * the position. If no entry contains it, @index is the position _after_ which 539 * to insert the new mapping. IOW, it is the index of the largest element that 540 * is smaller than @host, or -1 if no entry is. 541 */ 542 static IOVAMapping *qemu_vfio_find_mapping(QEMUVFIOState *s, void *host, 543 int *index) 544 { 545 IOVAMapping *p = s->mappings; 546 IOVAMapping *q = p ? p + s->nr_mappings - 1 : NULL; 547 IOVAMapping *mid; 548 trace_qemu_vfio_find_mapping(s, host); 549 if (!p) { 550 *index = -1; 551 return NULL; 552 } 553 while (true) { 554 mid = p + (q - p) / 2; 555 if (mid == p) { 556 break; 557 } 558 if (mid->host > host) { 559 q = mid; 560 } else if (mid->host < host) { 561 p = mid; 562 } else { 563 break; 564 } 565 } 566 if (mid->host > host) { 567 mid--; 568 } else if (mid < &s->mappings[s->nr_mappings - 1] 569 && (mid + 1)->host <= host) { 570 mid++; 571 } 572 *index = mid - &s->mappings[0]; 573 if (mid >= &s->mappings[0] && 574 mid->host <= host && mid->host + mid->size > host) { 575 assert(mid < &s->mappings[s->nr_mappings]); 576 return mid; 577 } 578 /* At this point *index + 1 is the right position to insert the new 579 * mapping.*/ 580 return NULL; 581 } 582 583 /** 584 * Allocate IOVA and create a new mapping record and insert it in @s. 585 */ 586 static IOVAMapping *qemu_vfio_add_mapping(QEMUVFIOState *s, 587 void *host, size_t size, 588 int index, uint64_t iova) 589 { 590 int shift; 591 IOVAMapping m = {.host = host, .size = size, .iova = iova}; 592 IOVAMapping *insert; 593 594 assert(QEMU_IS_ALIGNED(size, qemu_real_host_page_size())); 595 assert(QEMU_IS_ALIGNED(s->low_water_mark, qemu_real_host_page_size())); 596 assert(QEMU_IS_ALIGNED(s->high_water_mark, qemu_real_host_page_size())); 597 trace_qemu_vfio_new_mapping(s, host, size, index, iova); 598 599 assert(index >= 0); 600 s->nr_mappings++; 601 s->mappings = g_renew(IOVAMapping, s->mappings, s->nr_mappings); 602 insert = &s->mappings[index]; 603 shift = s->nr_mappings - index - 1; 604 if (shift) { 605 memmove(insert + 1, insert, shift * sizeof(s->mappings[0])); 606 } 607 *insert = m; 608 return insert; 609 } 610 611 /* Do the DMA mapping with VFIO. */ 612 static int qemu_vfio_do_mapping(QEMUVFIOState *s, void *host, size_t size, 613 uint64_t iova, Error **errp) 614 { 615 struct vfio_iommu_type1_dma_map dma_map = { 616 .argsz = sizeof(dma_map), 617 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, 618 .iova = iova, 619 .vaddr = (uintptr_t)host, 620 .size = size, 621 }; 622 trace_qemu_vfio_do_mapping(s, host, iova, size); 623 624 if (ioctl(s->container, VFIO_IOMMU_MAP_DMA, &dma_map)) { 625 error_setg_errno(errp, errno, "VFIO_MAP_DMA failed"); 626 return -errno; 627 } 628 return 0; 629 } 630 631 /** 632 * Undo the DMA mapping from @s with VFIO, and remove from mapping list. 633 */ 634 static void qemu_vfio_undo_mapping(QEMUVFIOState *s, IOVAMapping *mapping, 635 Error **errp) 636 { 637 int index; 638 struct vfio_iommu_type1_dma_unmap unmap = { 639 .argsz = sizeof(unmap), 640 .flags = 0, 641 .iova = mapping->iova, 642 .size = mapping->size, 643 }; 644 645 index = mapping - s->mappings; 646 assert(mapping->size > 0); 647 assert(QEMU_IS_ALIGNED(mapping->size, qemu_real_host_page_size())); 648 assert(index >= 0 && index < s->nr_mappings); 649 if (ioctl(s->container, VFIO_IOMMU_UNMAP_DMA, &unmap)) { 650 error_setg_errno(errp, errno, "VFIO_UNMAP_DMA failed"); 651 } 652 memmove(mapping, &s->mappings[index + 1], 653 sizeof(s->mappings[0]) * (s->nr_mappings - index - 1)); 654 s->nr_mappings--; 655 s->mappings = g_renew(IOVAMapping, s->mappings, s->nr_mappings); 656 } 657 658 /* Check if the mapping list is (ascending) ordered. */ 659 static bool qemu_vfio_verify_mappings(QEMUVFIOState *s) 660 { 661 int i; 662 if (QEMU_VFIO_DEBUG) { 663 for (i = 0; i < s->nr_mappings - 1; ++i) { 664 if (!(s->mappings[i].host < s->mappings[i + 1].host)) { 665 error_report("item %d not sorted!", i); 666 qemu_vfio_dump_mappings(s); 667 return false; 668 } 669 if (!(s->mappings[i].host + s->mappings[i].size <= 670 s->mappings[i + 1].host)) { 671 error_report("item %d overlap with next!", i); 672 qemu_vfio_dump_mappings(s); 673 return false; 674 } 675 } 676 } 677 return true; 678 } 679 680 static bool qemu_vfio_find_fixed_iova(QEMUVFIOState *s, size_t size, 681 uint64_t *iova, Error **errp) 682 { 683 int i; 684 685 for (i = 0; i < s->nb_iova_ranges; i++) { 686 if (s->usable_iova_ranges[i].end < s->low_water_mark) { 687 continue; 688 } 689 s->low_water_mark = 690 MAX(s->low_water_mark, s->usable_iova_ranges[i].start); 691 692 if (s->usable_iova_ranges[i].end - s->low_water_mark + 1 >= size || 693 s->usable_iova_ranges[i].end - s->low_water_mark + 1 == 0) { 694 *iova = s->low_water_mark; 695 s->low_water_mark += size; 696 return true; 697 } 698 } 699 error_setg(errp, "fixed iova range not found"); 700 701 return false; 702 } 703 704 static bool qemu_vfio_find_temp_iova(QEMUVFIOState *s, size_t size, 705 uint64_t *iova, Error **errp) 706 { 707 int i; 708 709 for (i = s->nb_iova_ranges - 1; i >= 0; i--) { 710 if (s->usable_iova_ranges[i].start > s->high_water_mark) { 711 continue; 712 } 713 s->high_water_mark = 714 MIN(s->high_water_mark, s->usable_iova_ranges[i].end + 1); 715 716 if (s->high_water_mark - s->usable_iova_ranges[i].start + 1 >= size || 717 s->high_water_mark - s->usable_iova_ranges[i].start + 1 == 0) { 718 *iova = s->high_water_mark - size; 719 s->high_water_mark = *iova; 720 return true; 721 } 722 } 723 error_setg(errp, "temporary iova range not found"); 724 725 return false; 726 } 727 728 /** 729 * qemu_vfio_water_mark_reached: 730 * 731 * Returns %true if high watermark has been reached, %false otherwise. 732 */ 733 static bool qemu_vfio_water_mark_reached(QEMUVFIOState *s, size_t size, 734 Error **errp) 735 { 736 if (s->high_water_mark - s->low_water_mark + 1 < size) { 737 error_setg(errp, "iova exhausted (water mark reached)"); 738 return true; 739 } 740 return false; 741 } 742 743 /* Map [host, host + size) area into a contiguous IOVA address space, and store 744 * the result in @iova if not NULL. The caller need to make sure the area is 745 * aligned to page size, and mustn't overlap with existing mapping areas (split 746 * mapping status within this area is not allowed). 747 */ 748 int qemu_vfio_dma_map(QEMUVFIOState *s, void *host, size_t size, 749 bool temporary, uint64_t *iova, Error **errp) 750 { 751 int index; 752 IOVAMapping *mapping; 753 uint64_t iova0; 754 755 assert(QEMU_PTR_IS_ALIGNED(host, qemu_real_host_page_size())); 756 assert(QEMU_IS_ALIGNED(size, qemu_real_host_page_size())); 757 trace_qemu_vfio_dma_map(s, host, size, temporary, iova); 758 QEMU_LOCK_GUARD(&s->lock); 759 mapping = qemu_vfio_find_mapping(s, host, &index); 760 if (mapping) { 761 iova0 = mapping->iova + ((uint8_t *)host - (uint8_t *)mapping->host); 762 } else { 763 int ret; 764 765 if (qemu_vfio_water_mark_reached(s, size, errp)) { 766 return -ENOMEM; 767 } 768 if (!temporary) { 769 if (!qemu_vfio_find_fixed_iova(s, size, &iova0, errp)) { 770 return -ENOMEM; 771 } 772 773 mapping = qemu_vfio_add_mapping(s, host, size, index + 1, iova0); 774 assert(qemu_vfio_verify_mappings(s)); 775 ret = qemu_vfio_do_mapping(s, host, size, iova0, errp); 776 if (ret < 0) { 777 qemu_vfio_undo_mapping(s, mapping, NULL); 778 return ret; 779 } 780 qemu_vfio_dump_mappings(s); 781 } else { 782 if (!qemu_vfio_find_temp_iova(s, size, &iova0, errp)) { 783 return -ENOMEM; 784 } 785 ret = qemu_vfio_do_mapping(s, host, size, iova0, errp); 786 if (ret < 0) { 787 return ret; 788 } 789 } 790 } 791 trace_qemu_vfio_dma_mapped(s, host, iova0, size); 792 if (iova) { 793 *iova = iova0; 794 } 795 return 0; 796 } 797 798 /* Reset the high watermark and free all "temporary" mappings. */ 799 int qemu_vfio_dma_reset_temporary(QEMUVFIOState *s) 800 { 801 struct vfio_iommu_type1_dma_unmap unmap = { 802 .argsz = sizeof(unmap), 803 .flags = 0, 804 .iova = s->high_water_mark, 805 .size = QEMU_VFIO_IOVA_MAX - s->high_water_mark, 806 }; 807 trace_qemu_vfio_dma_reset_temporary(s); 808 QEMU_LOCK_GUARD(&s->lock); 809 if (ioctl(s->container, VFIO_IOMMU_UNMAP_DMA, &unmap)) { 810 error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno)); 811 return -errno; 812 } 813 s->high_water_mark = QEMU_VFIO_IOVA_MAX; 814 return 0; 815 } 816 817 /* Unmapping the whole area that was previously mapped with 818 * qemu_vfio_dma_map(). */ 819 void qemu_vfio_dma_unmap(QEMUVFIOState *s, void *host) 820 { 821 int index = 0; 822 IOVAMapping *m; 823 824 if (!host) { 825 return; 826 } 827 828 trace_qemu_vfio_dma_unmap(s, host); 829 QEMU_LOCK_GUARD(&s->lock); 830 m = qemu_vfio_find_mapping(s, host, &index); 831 if (!m) { 832 return; 833 } 834 qemu_vfio_undo_mapping(s, m, NULL); 835 } 836 837 static void qemu_vfio_reset(QEMUVFIOState *s) 838 { 839 ioctl(s->device, VFIO_DEVICE_RESET); 840 } 841 842 /* Close and free the VFIO resources. */ 843 void qemu_vfio_close(QEMUVFIOState *s) 844 { 845 int i; 846 847 if (!s) { 848 return; 849 } 850 851 ram_block_notifier_remove(&s->ram_notifier); 852 853 for (i = 0; i < s->nr_mappings; ++i) { 854 qemu_vfio_undo_mapping(s, &s->mappings[i], NULL); 855 } 856 857 g_free(s->usable_iova_ranges); 858 s->nb_iova_ranges = 0; 859 qemu_vfio_reset(s); 860 close(s->device); 861 close(s->group); 862 close(s->container); 863 ram_block_discard_disable(false); 864 } 865