1 /* 2 * virtio-iommu device 3 * 4 * Copyright (c) 2020 Red Hat, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2 or later, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along with 16 * this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/log.h" 22 #include "qemu/iov.h" 23 #include "qemu/range.h" 24 #include "qemu/reserved-region.h" 25 #include "exec/target_page.h" 26 #include "hw/qdev-properties.h" 27 #include "hw/virtio/virtio.h" 28 #include "sysemu/kvm.h" 29 #include "sysemu/reset.h" 30 #include "sysemu/sysemu.h" 31 #include "qemu/reserved-region.h" 32 #include "qemu/units.h" 33 #include "qapi/error.h" 34 #include "qemu/error-report.h" 35 #include "trace.h" 36 37 #include "standard-headers/linux/virtio_ids.h" 38 39 #include "hw/virtio/virtio-bus.h" 40 #include "hw/virtio/virtio-iommu.h" 41 #include "hw/pci/pci_bus.h" 42 #include "hw/pci/pci.h" 43 44 /* Max size */ 45 #define VIOMMU_DEFAULT_QUEUE_SIZE 256 46 #define VIOMMU_PROBE_SIZE 512 47 48 typedef struct VirtIOIOMMUDomain { 49 uint32_t id; 50 bool bypass; 51 GTree *mappings; 52 QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list; 53 } VirtIOIOMMUDomain; 54 55 typedef struct VirtIOIOMMUEndpoint { 56 uint32_t id; 57 VirtIOIOMMUDomain *domain; 58 IOMMUMemoryRegion *iommu_mr; 59 QLIST_ENTRY(VirtIOIOMMUEndpoint) next; 60 } VirtIOIOMMUEndpoint; 61 62 typedef struct VirtIOIOMMUInterval { 63 uint64_t low; 64 uint64_t high; 65 } VirtIOIOMMUInterval; 66 67 typedef struct VirtIOIOMMUMapping { 68 uint64_t phys_addr; 69 uint32_t flags; 70 } VirtIOIOMMUMapping; 71 72 struct hiod_key { 73 PCIBus *bus; 74 uint8_t devfn; 75 }; 76 77 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev) 78 { 79 return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn); 80 } 81 82 static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev) 83 { 84 uint32_t sid; 85 bool bypassed; 86 VirtIOIOMMU *s = sdev->viommu; 87 VirtIOIOMMUEndpoint *ep; 88 89 sid = virtio_iommu_get_bdf(sdev); 90 91 qemu_rec_mutex_lock(&s->mutex); 92 /* need to check bypass before system reset */ 93 if (!s->endpoints) { 94 bypassed = s->config.bypass; 95 goto unlock; 96 } 97 98 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 99 if (!ep || !ep->domain) { 100 bypassed = s->config.bypass; 101 } else { 102 bypassed = ep->domain->bypass; 103 } 104 105 unlock: 106 qemu_rec_mutex_unlock(&s->mutex); 107 return bypassed; 108 } 109 110 /* Return whether the device is using IOMMU translation. */ 111 static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev) 112 { 113 bool use_remapping; 114 115 assert(sdev); 116 117 use_remapping = !virtio_iommu_device_bypassed(sdev); 118 119 trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus), 120 PCI_SLOT(sdev->devfn), 121 PCI_FUNC(sdev->devfn), 122 use_remapping); 123 124 /* Turn off first then on the other */ 125 if (use_remapping) { 126 memory_region_set_enabled(&sdev->bypass_mr, false); 127 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true); 128 } else { 129 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false); 130 memory_region_set_enabled(&sdev->bypass_mr, true); 131 } 132 133 return use_remapping; 134 } 135 136 static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s) 137 { 138 GHashTableIter iter; 139 IOMMUPciBus *iommu_pci_bus; 140 int i; 141 142 g_hash_table_iter_init(&iter, s->as_by_busptr); 143 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 144 for (i = 0; i < PCI_DEVFN_MAX; i++) { 145 if (!iommu_pci_bus->pbdev[i]) { 146 continue; 147 } 148 virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]); 149 } 150 } 151 } 152 153 /** 154 * The bus number is used for lookup when SID based operations occur. 155 * In that case we lazily populate the IOMMUPciBus array from the bus hash 156 * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus 157 * numbers may not be always initialized yet. 158 */ 159 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num) 160 { 161 IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num]; 162 163 if (!iommu_pci_bus) { 164 GHashTableIter iter; 165 166 g_hash_table_iter_init(&iter, s->as_by_busptr); 167 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 168 if (pci_bus_num(iommu_pci_bus->bus) == bus_num) { 169 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus; 170 return iommu_pci_bus; 171 } 172 } 173 return NULL; 174 } 175 return iommu_pci_bus; 176 } 177 178 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid) 179 { 180 uint8_t bus_n, devfn; 181 IOMMUPciBus *iommu_pci_bus; 182 IOMMUDevice *dev; 183 184 bus_n = PCI_BUS_NUM(sid); 185 iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n); 186 if (iommu_pci_bus) { 187 devfn = sid & (PCI_DEVFN_MAX - 1); 188 dev = iommu_pci_bus->pbdev[devfn]; 189 if (dev) { 190 return &dev->iommu_mr; 191 } 192 } 193 return NULL; 194 } 195 196 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 197 { 198 VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a; 199 VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b; 200 201 if (inta->high < intb->low) { 202 return -1; 203 } else if (intb->high < inta->low) { 204 return 1; 205 } else { 206 return 0; 207 } 208 } 209 210 static void virtio_iommu_notify_map_unmap(IOMMUMemoryRegion *mr, 211 IOMMUTLBEvent *event, 212 hwaddr virt_start, hwaddr virt_end) 213 { 214 uint64_t delta = virt_end - virt_start; 215 216 event->entry.iova = virt_start; 217 event->entry.addr_mask = delta; 218 219 if (delta == UINT64_MAX) { 220 memory_region_notify_iommu(mr, 0, *event); 221 } 222 223 while (virt_start != virt_end + 1) { 224 uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64); 225 226 event->entry.addr_mask = mask; 227 event->entry.iova = virt_start; 228 memory_region_notify_iommu(mr, 0, *event); 229 virt_start += mask + 1; 230 if (event->entry.perm != IOMMU_NONE) { 231 event->entry.translated_addr += mask + 1; 232 } 233 } 234 } 235 236 static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start, 237 hwaddr virt_end, hwaddr paddr, 238 uint32_t flags) 239 { 240 IOMMUTLBEvent event; 241 IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ, 242 flags & VIRTIO_IOMMU_MAP_F_WRITE); 243 244 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) || 245 (flags & VIRTIO_IOMMU_MAP_F_MMIO) || !perm) { 246 return; 247 } 248 249 trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end, 250 paddr, perm); 251 252 event.type = IOMMU_NOTIFIER_MAP; 253 event.entry.target_as = &address_space_memory; 254 event.entry.perm = perm; 255 event.entry.translated_addr = paddr; 256 257 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end); 258 } 259 260 static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start, 261 hwaddr virt_end) 262 { 263 IOMMUTLBEvent event; 264 265 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) { 266 return; 267 } 268 269 trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end); 270 271 event.type = IOMMU_NOTIFIER_UNMAP; 272 event.entry.target_as = &address_space_memory; 273 event.entry.perm = IOMMU_NONE; 274 event.entry.translated_addr = 0; 275 276 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end); 277 } 278 279 static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value, 280 gpointer data) 281 { 282 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 283 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 284 285 virtio_iommu_notify_unmap(mr, interval->low, interval->high); 286 287 return false; 288 } 289 290 static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value, 291 gpointer data) 292 { 293 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 294 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 295 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 296 297 virtio_iommu_notify_map(mr, interval->low, interval->high, 298 mapping->phys_addr, mapping->flags); 299 300 return false; 301 } 302 303 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep) 304 { 305 VirtIOIOMMUDomain *domain = ep->domain; 306 IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 307 308 if (!ep->domain) { 309 return; 310 } 311 g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb, 312 ep->iommu_mr); 313 QLIST_REMOVE(ep, next); 314 ep->domain = NULL; 315 virtio_iommu_switch_address_space(sdev); 316 } 317 318 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s, 319 uint32_t ep_id) 320 { 321 VirtIOIOMMUEndpoint *ep; 322 IOMMUMemoryRegion *mr; 323 324 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 325 if (ep) { 326 return ep; 327 } 328 mr = virtio_iommu_mr(s, ep_id); 329 if (!mr) { 330 return NULL; 331 } 332 ep = g_malloc0(sizeof(*ep)); 333 ep->id = ep_id; 334 ep->iommu_mr = mr; 335 trace_virtio_iommu_get_endpoint(ep_id); 336 g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep); 337 return ep; 338 } 339 340 static void virtio_iommu_put_endpoint(gpointer data) 341 { 342 VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data; 343 344 if (ep->domain) { 345 virtio_iommu_detach_endpoint_from_domain(ep); 346 } 347 348 trace_virtio_iommu_put_endpoint(ep->id); 349 g_free(ep); 350 } 351 352 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s, 353 uint32_t domain_id, 354 bool bypass) 355 { 356 VirtIOIOMMUDomain *domain; 357 358 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 359 if (domain) { 360 if (domain->bypass != bypass) { 361 return NULL; 362 } 363 return domain; 364 } 365 domain = g_malloc0(sizeof(*domain)); 366 domain->id = domain_id; 367 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 368 NULL, (GDestroyNotify)g_free, 369 (GDestroyNotify)g_free); 370 domain->bypass = bypass; 371 g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain); 372 QLIST_INIT(&domain->endpoint_list); 373 trace_virtio_iommu_get_domain(domain_id); 374 return domain; 375 } 376 377 static void virtio_iommu_put_domain(gpointer data) 378 { 379 VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data; 380 VirtIOIOMMUEndpoint *iter, *tmp; 381 382 QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) { 383 virtio_iommu_detach_endpoint_from_domain(iter); 384 } 385 g_tree_destroy(domain->mappings); 386 trace_virtio_iommu_put_domain(domain->id); 387 g_free(domain); 388 } 389 390 static void add_prop_resv_regions(IOMMUDevice *sdev) 391 { 392 VirtIOIOMMU *s = sdev->viommu; 393 int i; 394 395 for (i = 0; i < s->nr_prop_resv_regions; i++) { 396 ReservedRegion *reg = g_new0(ReservedRegion, 1); 397 398 *reg = s->prop_resv_regions[i]; 399 sdev->resv_regions = resv_region_list_insert(sdev->resv_regions, reg); 400 } 401 } 402 403 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque, 404 int devfn) 405 { 406 VirtIOIOMMU *s = opaque; 407 IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus); 408 static uint32_t mr_index; 409 IOMMUDevice *sdev; 410 411 if (!sbus) { 412 sbus = g_malloc0(sizeof(IOMMUPciBus) + 413 sizeof(IOMMUDevice *) * PCI_DEVFN_MAX); 414 sbus->bus = bus; 415 g_hash_table_insert(s->as_by_busptr, bus, sbus); 416 } 417 418 sdev = sbus->pbdev[devfn]; 419 if (!sdev) { 420 char *name = g_strdup_printf("%s-%d-%d", 421 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 422 mr_index++, devfn); 423 sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1); 424 425 sdev->viommu = s; 426 sdev->bus = bus; 427 sdev->devfn = devfn; 428 429 trace_virtio_iommu_init_iommu_mr(name); 430 431 memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX); 432 address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU); 433 add_prop_resv_regions(sdev); 434 435 /* 436 * Build the IOMMU disabled container with aliases to the 437 * shared MRs. Note that aliasing to a shared memory region 438 * could help the memory API to detect same FlatViews so we 439 * can have devices to share the same FlatView when in bypass 440 * mode. (either by not configuring virtio-iommu driver or with 441 * "iommu=pt"). It will greatly reduce the total number of 442 * FlatViews of the system hence VM runs faster. 443 */ 444 memory_region_init_alias(&sdev->bypass_mr, OBJECT(s), 445 "system", get_system_memory(), 0, 446 memory_region_size(get_system_memory())); 447 448 memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr), 449 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 450 OBJECT(s), name, 451 UINT64_MAX); 452 453 /* 454 * Hook both the containers under the root container, we 455 * switch between iommu & bypass MRs by enable/disable 456 * corresponding sub-containers 457 */ 458 memory_region_add_subregion_overlap(&sdev->root, 0, 459 MEMORY_REGION(&sdev->iommu_mr), 460 0); 461 memory_region_add_subregion_overlap(&sdev->root, 0, 462 &sdev->bypass_mr, 0); 463 464 virtio_iommu_switch_address_space(sdev); 465 g_free(name); 466 } 467 return &sdev->as; 468 } 469 470 static void virtio_iommu_device_clear(VirtIOIOMMU *s, PCIBus *bus, int devfn) 471 { 472 IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus); 473 IOMMUDevice *sdev; 474 475 if (!sbus) { 476 return; 477 } 478 479 sdev = sbus->pbdev[devfn]; 480 if (!sdev) { 481 return; 482 } 483 484 g_list_free_full(sdev->resv_regions, g_free); 485 sdev->resv_regions = NULL; 486 g_free(sdev); 487 sbus->pbdev[devfn] = NULL; 488 } 489 490 static gboolean hiod_equal(gconstpointer v1, gconstpointer v2) 491 { 492 const struct hiod_key *key1 = v1; 493 const struct hiod_key *key2 = v2; 494 495 return (key1->bus == key2->bus) && (key1->devfn == key2->devfn); 496 } 497 498 static guint hiod_hash(gconstpointer v) 499 { 500 const struct hiod_key *key = v; 501 guint value = (guint)(uintptr_t)key->bus; 502 503 return (guint)(value << 8 | key->devfn); 504 } 505 506 static void hiod_destroy(gpointer v) 507 { 508 object_unref(v); 509 } 510 511 static HostIOMMUDevice * 512 get_host_iommu_device(VirtIOIOMMU *viommu, PCIBus *bus, int devfn) { 513 struct hiod_key key = { 514 .bus = bus, 515 .devfn = devfn, 516 }; 517 518 return g_hash_table_lookup(viommu->host_iommu_devices, &key); 519 } 520 521 /** 522 * rebuild_resv_regions: rebuild resv regions with both the 523 * info of host resv ranges and property set resv ranges 524 */ 525 static int rebuild_resv_regions(IOMMUDevice *sdev) 526 { 527 GList *l; 528 int i = 0; 529 530 /* free the existing list and rebuild it from scratch */ 531 g_list_free_full(sdev->resv_regions, g_free); 532 sdev->resv_regions = NULL; 533 534 /* First add host reserved regions if any, all tagged as RESERVED */ 535 for (l = sdev->host_resv_ranges; l; l = l->next) { 536 ReservedRegion *reg = g_new0(ReservedRegion, 1); 537 Range *r = (Range *)l->data; 538 539 reg->type = VIRTIO_IOMMU_RESV_MEM_T_RESERVED; 540 range_set_bounds(®->range, range_lob(r), range_upb(r)); 541 sdev->resv_regions = resv_region_list_insert(sdev->resv_regions, reg); 542 trace_virtio_iommu_host_resv_regions(sdev->iommu_mr.parent_obj.name, i, 543 range_lob(®->range), 544 range_upb(®->range)); 545 i++; 546 } 547 /* 548 * then add higher priority reserved regions set by the machine 549 * through properties 550 */ 551 add_prop_resv_regions(sdev); 552 return 0; 553 } 554 555 static int virtio_iommu_set_host_iova_ranges(VirtIOIOMMU *s, PCIBus *bus, 556 int devfn, GList *iova_ranges, 557 Error **errp) 558 { 559 IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus); 560 IOMMUDevice *sdev; 561 GList *current_ranges; 562 GList *l, *tmp, *new_ranges = NULL; 563 int ret = -EINVAL; 564 565 if (!sbus) { 566 error_report("%s no sbus", __func__); 567 } 568 569 sdev = sbus->pbdev[devfn]; 570 571 current_ranges = sdev->host_resv_ranges; 572 573 g_assert(!sdev->probe_done); 574 575 /* check that each new resv region is included in an existing one */ 576 if (sdev->host_resv_ranges) { 577 range_inverse_array(iova_ranges, 578 &new_ranges, 579 0, UINT64_MAX); 580 581 for (tmp = new_ranges; tmp; tmp = tmp->next) { 582 Range *newr = (Range *)tmp->data; 583 bool included = false; 584 585 for (l = current_ranges; l; l = l->next) { 586 Range * r = (Range *)l->data; 587 588 if (range_contains_range(r, newr)) { 589 included = true; 590 break; 591 } 592 } 593 if (!included) { 594 goto error; 595 } 596 } 597 /* all new reserved ranges are included in existing ones */ 598 ret = 0; 599 goto out; 600 } 601 602 range_inverse_array(iova_ranges, 603 &sdev->host_resv_ranges, 604 0, UINT64_MAX); 605 rebuild_resv_regions(sdev); 606 607 return 0; 608 error: 609 error_setg(errp, "%s Conflicting host reserved ranges set!", 610 __func__); 611 out: 612 g_list_free_full(new_ranges, g_free); 613 return ret; 614 } 615 616 static bool virtio_iommu_set_iommu_device(PCIBus *bus, void *opaque, int devfn, 617 HostIOMMUDevice *hiod, Error **errp) 618 { 619 VirtIOIOMMU *viommu = opaque; 620 HostIOMMUDeviceClass *hiodc = HOST_IOMMU_DEVICE_GET_CLASS(hiod); 621 struct hiod_key *new_key; 622 GList *host_iova_ranges = NULL; 623 624 assert(hiod); 625 626 if (get_host_iommu_device(viommu, bus, devfn)) { 627 error_setg(errp, "Host IOMMU device already exists"); 628 return false; 629 } 630 631 if (hiodc->get_iova_ranges) { 632 int ret; 633 host_iova_ranges = hiodc->get_iova_ranges(hiod, errp); 634 if (!host_iova_ranges) { 635 return true; /* some old kernels may not support that capability */ 636 } 637 ret = virtio_iommu_set_host_iova_ranges(viommu, hiod->aliased_bus, 638 hiod->aliased_devfn, 639 host_iova_ranges, errp); 640 if (ret) { 641 g_list_free_full(host_iova_ranges, g_free); 642 return false; 643 } 644 } 645 646 new_key = g_malloc(sizeof(*new_key)); 647 new_key->bus = bus; 648 new_key->devfn = devfn; 649 650 object_ref(hiod); 651 g_hash_table_insert(viommu->host_iommu_devices, new_key, hiod); 652 g_list_free_full(host_iova_ranges, g_free); 653 654 return true; 655 } 656 657 static void 658 virtio_iommu_unset_iommu_device(PCIBus *bus, void *opaque, int devfn) 659 { 660 VirtIOIOMMU *viommu = opaque; 661 HostIOMMUDevice *hiod; 662 struct hiod_key key = { 663 .bus = bus, 664 .devfn = devfn, 665 }; 666 667 hiod = g_hash_table_lookup(viommu->host_iommu_devices, &key); 668 if (!hiod) { 669 return; 670 } 671 672 g_hash_table_remove(viommu->host_iommu_devices, &key); 673 virtio_iommu_device_clear(viommu, bus, devfn); 674 } 675 676 static const PCIIOMMUOps virtio_iommu_ops = { 677 .get_address_space = virtio_iommu_find_add_as, 678 .set_iommu_device = virtio_iommu_set_iommu_device, 679 .unset_iommu_device = virtio_iommu_unset_iommu_device, 680 }; 681 682 static int virtio_iommu_attach(VirtIOIOMMU *s, 683 struct virtio_iommu_req_attach *req) 684 { 685 uint32_t domain_id = le32_to_cpu(req->domain); 686 uint32_t ep_id = le32_to_cpu(req->endpoint); 687 uint32_t flags = le32_to_cpu(req->flags); 688 VirtIOIOMMUDomain *domain; 689 VirtIOIOMMUEndpoint *ep; 690 IOMMUDevice *sdev; 691 692 trace_virtio_iommu_attach(domain_id, ep_id); 693 694 if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) { 695 return VIRTIO_IOMMU_S_INVAL; 696 } 697 698 ep = virtio_iommu_get_endpoint(s, ep_id); 699 if (!ep) { 700 return VIRTIO_IOMMU_S_NOENT; 701 } 702 703 if (ep->domain) { 704 VirtIOIOMMUDomain *previous_domain = ep->domain; 705 /* 706 * the device is already attached to a domain, 707 * detach it first 708 */ 709 virtio_iommu_detach_endpoint_from_domain(ep); 710 if (QLIST_EMPTY(&previous_domain->endpoint_list)) { 711 g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id)); 712 } 713 } 714 715 domain = virtio_iommu_get_domain(s, domain_id, 716 flags & VIRTIO_IOMMU_ATTACH_F_BYPASS); 717 if (!domain) { 718 /* Incompatible bypass flag */ 719 return VIRTIO_IOMMU_S_INVAL; 720 } 721 QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next); 722 723 ep->domain = domain; 724 sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 725 virtio_iommu_switch_address_space(sdev); 726 727 /* Replay domain mappings on the associated memory region */ 728 g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb, 729 ep->iommu_mr); 730 731 return VIRTIO_IOMMU_S_OK; 732 } 733 734 static int virtio_iommu_detach(VirtIOIOMMU *s, 735 struct virtio_iommu_req_detach *req) 736 { 737 uint32_t domain_id = le32_to_cpu(req->domain); 738 uint32_t ep_id = le32_to_cpu(req->endpoint); 739 VirtIOIOMMUDomain *domain; 740 VirtIOIOMMUEndpoint *ep; 741 742 trace_virtio_iommu_detach(domain_id, ep_id); 743 744 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 745 if (!ep) { 746 return VIRTIO_IOMMU_S_NOENT; 747 } 748 749 domain = ep->domain; 750 751 if (!domain || domain->id != domain_id) { 752 return VIRTIO_IOMMU_S_INVAL; 753 } 754 755 virtio_iommu_detach_endpoint_from_domain(ep); 756 757 if (QLIST_EMPTY(&domain->endpoint_list)) { 758 g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id)); 759 } 760 return VIRTIO_IOMMU_S_OK; 761 } 762 763 static int virtio_iommu_map(VirtIOIOMMU *s, 764 struct virtio_iommu_req_map *req) 765 { 766 uint32_t domain_id = le32_to_cpu(req->domain); 767 uint64_t phys_start = le64_to_cpu(req->phys_start); 768 uint64_t virt_start = le64_to_cpu(req->virt_start); 769 uint64_t virt_end = le64_to_cpu(req->virt_end); 770 uint32_t flags = le32_to_cpu(req->flags); 771 VirtIOIOMMUDomain *domain; 772 VirtIOIOMMUInterval *interval; 773 VirtIOIOMMUMapping *mapping; 774 VirtIOIOMMUEndpoint *ep; 775 776 if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) { 777 return VIRTIO_IOMMU_S_INVAL; 778 } 779 780 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 781 if (!domain) { 782 return VIRTIO_IOMMU_S_NOENT; 783 } 784 785 if (domain->bypass) { 786 return VIRTIO_IOMMU_S_INVAL; 787 } 788 789 interval = g_malloc0(sizeof(*interval)); 790 791 interval->low = virt_start; 792 interval->high = virt_end; 793 794 mapping = g_tree_lookup(domain->mappings, (gpointer)interval); 795 if (mapping) { 796 g_free(interval); 797 return VIRTIO_IOMMU_S_INVAL; 798 } 799 800 trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags); 801 802 mapping = g_malloc0(sizeof(*mapping)); 803 mapping->phys_addr = phys_start; 804 mapping->flags = flags; 805 806 g_tree_insert(domain->mappings, interval, mapping); 807 808 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 809 virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start, 810 flags); 811 } 812 813 return VIRTIO_IOMMU_S_OK; 814 } 815 816 static int virtio_iommu_unmap(VirtIOIOMMU *s, 817 struct virtio_iommu_req_unmap *req) 818 { 819 uint32_t domain_id = le32_to_cpu(req->domain); 820 uint64_t virt_start = le64_to_cpu(req->virt_start); 821 uint64_t virt_end = le64_to_cpu(req->virt_end); 822 VirtIOIOMMUMapping *iter_val; 823 VirtIOIOMMUInterval interval, *iter_key; 824 VirtIOIOMMUDomain *domain; 825 VirtIOIOMMUEndpoint *ep; 826 int ret = VIRTIO_IOMMU_S_OK; 827 828 trace_virtio_iommu_unmap(domain_id, virt_start, virt_end); 829 830 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 831 if (!domain) { 832 return VIRTIO_IOMMU_S_NOENT; 833 } 834 835 if (domain->bypass) { 836 return VIRTIO_IOMMU_S_INVAL; 837 } 838 839 interval.low = virt_start; 840 interval.high = virt_end; 841 842 while (g_tree_lookup_extended(domain->mappings, &interval, 843 (void **)&iter_key, (void**)&iter_val)) { 844 uint64_t current_low = iter_key->low; 845 uint64_t current_high = iter_key->high; 846 847 if (interval.low <= current_low && interval.high >= current_high) { 848 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 849 virtio_iommu_notify_unmap(ep->iommu_mr, current_low, 850 current_high); 851 } 852 g_tree_remove(domain->mappings, iter_key); 853 trace_virtio_iommu_unmap_done(domain_id, current_low, current_high); 854 } else { 855 ret = VIRTIO_IOMMU_S_RANGE; 856 break; 857 } 858 } 859 return ret; 860 } 861 862 static ssize_t virtio_iommu_fill_resv_mem_prop(IOMMUDevice *sdev, uint32_t ep, 863 uint8_t *buf, size_t free) 864 { 865 struct virtio_iommu_probe_resv_mem prop = {}; 866 size_t size = sizeof(prop), length = size - sizeof(prop.head), total; 867 GList *l; 868 869 total = size * g_list_length(sdev->resv_regions); 870 if (total > free) { 871 return -ENOSPC; 872 } 873 874 for (l = sdev->resv_regions; l; l = l->next) { 875 ReservedRegion *reg = l->data; 876 unsigned subtype = reg->type; 877 Range *range = ®->range; 878 879 assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED || 880 subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI); 881 prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM); 882 prop.head.length = cpu_to_le16(length); 883 prop.subtype = subtype; 884 prop.start = cpu_to_le64(range_lob(range)); 885 prop.end = cpu_to_le64(range_upb(range)); 886 887 memcpy(buf, &prop, size); 888 889 trace_virtio_iommu_fill_resv_property(ep, prop.subtype, 890 prop.start, prop.end); 891 buf += size; 892 } 893 return total; 894 } 895 896 /** 897 * virtio_iommu_probe - Fill the probe request buffer with 898 * the properties the device is able to return 899 */ 900 static int virtio_iommu_probe(VirtIOIOMMU *s, 901 struct virtio_iommu_req_probe *req, 902 uint8_t *buf) 903 { 904 uint32_t ep_id = le32_to_cpu(req->endpoint); 905 IOMMUMemoryRegion *iommu_mr = virtio_iommu_mr(s, ep_id); 906 size_t free = VIOMMU_PROBE_SIZE; 907 IOMMUDevice *sdev; 908 ssize_t count; 909 910 if (!iommu_mr) { 911 return VIRTIO_IOMMU_S_NOENT; 912 } 913 914 sdev = container_of(iommu_mr, IOMMUDevice, iommu_mr); 915 916 count = virtio_iommu_fill_resv_mem_prop(sdev, ep_id, buf, free); 917 if (count < 0) { 918 return VIRTIO_IOMMU_S_INVAL; 919 } 920 buf += count; 921 free -= count; 922 sdev->probe_done = true; 923 924 return VIRTIO_IOMMU_S_OK; 925 } 926 927 static int virtio_iommu_iov_to_req(struct iovec *iov, 928 unsigned int iov_cnt, 929 void *req, size_t payload_sz) 930 { 931 size_t sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz); 932 933 if (unlikely(sz != payload_sz)) { 934 return VIRTIO_IOMMU_S_INVAL; 935 } 936 return 0; 937 } 938 939 #define virtio_iommu_handle_req(__req) \ 940 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \ 941 struct iovec *iov, \ 942 unsigned int iov_cnt) \ 943 { \ 944 struct virtio_iommu_req_ ## __req req; \ 945 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, \ 946 sizeof(req) - sizeof(struct virtio_iommu_req_tail));\ 947 \ 948 return ret ? ret : virtio_iommu_ ## __req(s, &req); \ 949 } 950 951 virtio_iommu_handle_req(attach) 952 virtio_iommu_handle_req(detach) 953 virtio_iommu_handle_req(map) 954 virtio_iommu_handle_req(unmap) 955 956 static int virtio_iommu_handle_probe(VirtIOIOMMU *s, 957 struct iovec *iov, 958 unsigned int iov_cnt, 959 uint8_t *buf) 960 { 961 struct virtio_iommu_req_probe req; 962 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); 963 964 return ret ? ret : virtio_iommu_probe(s, &req, buf); 965 } 966 967 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq) 968 { 969 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 970 struct virtio_iommu_req_head head; 971 struct virtio_iommu_req_tail tail = {}; 972 VirtQueueElement *elem; 973 unsigned int iov_cnt; 974 struct iovec *iov; 975 void *buf = NULL; 976 size_t sz; 977 978 for (;;) { 979 size_t output_size = sizeof(tail); 980 981 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 982 if (!elem) { 983 return; 984 } 985 986 if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) || 987 iov_size(elem->out_sg, elem->out_num) < sizeof(head)) { 988 virtio_error(vdev, "virtio-iommu bad head/tail size"); 989 virtqueue_detach_element(vq, elem, 0); 990 g_free(elem); 991 break; 992 } 993 994 iov_cnt = elem->out_num; 995 iov = elem->out_sg; 996 sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head)); 997 if (unlikely(sz != sizeof(head))) { 998 qemu_log_mask(LOG_GUEST_ERROR, 999 "%s: read %zu bytes from command head" 1000 "but expected %zu\n", __func__, sz, sizeof(head)); 1001 tail.status = VIRTIO_IOMMU_S_DEVERR; 1002 goto out; 1003 } 1004 qemu_rec_mutex_lock(&s->mutex); 1005 switch (head.type) { 1006 case VIRTIO_IOMMU_T_ATTACH: 1007 tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt); 1008 break; 1009 case VIRTIO_IOMMU_T_DETACH: 1010 tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt); 1011 break; 1012 case VIRTIO_IOMMU_T_MAP: 1013 tail.status = virtio_iommu_handle_map(s, iov, iov_cnt); 1014 break; 1015 case VIRTIO_IOMMU_T_UNMAP: 1016 tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt); 1017 break; 1018 case VIRTIO_IOMMU_T_PROBE: 1019 { 1020 struct virtio_iommu_req_tail *ptail; 1021 1022 output_size = s->config.probe_size + sizeof(tail); 1023 buf = g_malloc0(output_size); 1024 1025 ptail = buf + s->config.probe_size; 1026 ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf); 1027 break; 1028 } 1029 default: 1030 tail.status = VIRTIO_IOMMU_S_UNSUPP; 1031 } 1032 qemu_rec_mutex_unlock(&s->mutex); 1033 1034 out: 1035 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 1036 buf ? buf : &tail, output_size); 1037 if (unlikely(sz != output_size)) { 1038 qemu_log_mask(LOG_GUEST_ERROR, 1039 "%s: wrote %zu bytes to command response" 1040 "but response size is %zu\n", 1041 __func__, sz, output_size); 1042 tail.status = VIRTIO_IOMMU_S_DEVERR; 1043 /* 1044 * We checked that sizeof(tail) can fit to elem->in_sg at the 1045 * beginning of the loop 1046 */ 1047 output_size = sizeof(tail); 1048 g_free(buf); 1049 buf = NULL; 1050 sz = iov_from_buf(elem->in_sg, 1051 elem->in_num, 1052 0, 1053 &tail, 1054 output_size); 1055 } 1056 assert(sz == output_size); 1057 1058 virtqueue_push(vq, elem, sz); 1059 virtio_notify(vdev, vq); 1060 g_free(elem); 1061 g_free(buf); 1062 buf = NULL; 1063 } 1064 } 1065 1066 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason, 1067 int flags, uint32_t endpoint, 1068 uint64_t address) 1069 { 1070 VirtIODevice *vdev = &viommu->parent_obj; 1071 VirtQueue *vq = viommu->event_vq; 1072 struct virtio_iommu_fault fault; 1073 VirtQueueElement *elem; 1074 size_t sz; 1075 1076 memset(&fault, 0, sizeof(fault)); 1077 fault.reason = reason; 1078 fault.flags = cpu_to_le32(flags); 1079 fault.endpoint = cpu_to_le32(endpoint); 1080 fault.address = cpu_to_le64(address); 1081 1082 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 1083 1084 if (!elem) { 1085 error_report_once( 1086 "no buffer available in event queue to report event"); 1087 return; 1088 } 1089 1090 if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) { 1091 virtio_error(vdev, "error buffer of wrong size"); 1092 virtqueue_detach_element(vq, elem, 0); 1093 g_free(elem); 1094 return; 1095 } 1096 1097 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 1098 &fault, sizeof(fault)); 1099 assert(sz == sizeof(fault)); 1100 1101 trace_virtio_iommu_report_fault(reason, flags, endpoint, address); 1102 virtqueue_push(vq, elem, sz); 1103 virtio_notify(vdev, vq); 1104 g_free(elem); 1105 1106 } 1107 1108 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr, 1109 IOMMUAccessFlags flag, 1110 int iommu_idx) 1111 { 1112 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1113 VirtIOIOMMUInterval interval, *mapping_key; 1114 VirtIOIOMMUMapping *mapping_value; 1115 VirtIOIOMMU *s = sdev->viommu; 1116 bool read_fault, write_fault; 1117 VirtIOIOMMUEndpoint *ep; 1118 uint32_t sid, flags; 1119 bool bypass_allowed; 1120 int granule; 1121 bool found; 1122 GList *l; 1123 1124 interval.low = addr; 1125 interval.high = addr + 1; 1126 granule = ctz64(s->config.page_size_mask); 1127 1128 IOMMUTLBEntry entry = { 1129 .target_as = &address_space_memory, 1130 .iova = addr, 1131 .translated_addr = addr, 1132 .addr_mask = BIT_ULL(granule) - 1, 1133 .perm = IOMMU_NONE, 1134 }; 1135 1136 bypass_allowed = s->config.bypass; 1137 1138 sid = virtio_iommu_get_bdf(sdev); 1139 1140 trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag); 1141 qemu_rec_mutex_lock(&s->mutex); 1142 1143 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 1144 1145 if (bypass_allowed) 1146 assert(ep && ep->domain && !ep->domain->bypass); 1147 1148 if (!ep) { 1149 if (!bypass_allowed) { 1150 error_report_once("%s sid=%d is not known!!", __func__, sid); 1151 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN, 1152 VIRTIO_IOMMU_FAULT_F_ADDRESS, 1153 sid, addr); 1154 } else { 1155 entry.perm = flag; 1156 } 1157 goto unlock; 1158 } 1159 1160 for (l = sdev->resv_regions; l; l = l->next) { 1161 ReservedRegion *reg = l->data; 1162 1163 if (range_contains(®->range, addr)) { 1164 switch (reg->type) { 1165 case VIRTIO_IOMMU_RESV_MEM_T_MSI: 1166 entry.perm = flag; 1167 break; 1168 case VIRTIO_IOMMU_RESV_MEM_T_RESERVED: 1169 default: 1170 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 1171 VIRTIO_IOMMU_FAULT_F_ADDRESS, 1172 sid, addr); 1173 break; 1174 } 1175 goto unlock; 1176 } 1177 } 1178 1179 if (!ep->domain) { 1180 if (!bypass_allowed) { 1181 error_report_once("%s %02x:%02x.%01x not attached to any domain", 1182 __func__, PCI_BUS_NUM(sid), 1183 PCI_SLOT(sid), PCI_FUNC(sid)); 1184 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN, 1185 VIRTIO_IOMMU_FAULT_F_ADDRESS, 1186 sid, addr); 1187 } else { 1188 entry.perm = flag; 1189 } 1190 goto unlock; 1191 } else if (ep->domain->bypass) { 1192 entry.perm = flag; 1193 goto unlock; 1194 } 1195 1196 found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval), 1197 (void **)&mapping_key, 1198 (void **)&mapping_value); 1199 if (!found) { 1200 error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d", 1201 __func__, addr, sid); 1202 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 1203 VIRTIO_IOMMU_FAULT_F_ADDRESS, 1204 sid, addr); 1205 goto unlock; 1206 } 1207 1208 read_fault = (flag & IOMMU_RO) && 1209 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ); 1210 write_fault = (flag & IOMMU_WO) && 1211 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE); 1212 1213 flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0; 1214 flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0; 1215 if (flags) { 1216 error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d", 1217 __func__, addr, flag, mapping_value->flags); 1218 flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS; 1219 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 1220 flags | VIRTIO_IOMMU_FAULT_F_ADDRESS, 1221 sid, addr); 1222 goto unlock; 1223 } 1224 entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr; 1225 entry.perm = flag; 1226 trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid); 1227 1228 unlock: 1229 qemu_rec_mutex_unlock(&s->mutex); 1230 return entry; 1231 } 1232 1233 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data) 1234 { 1235 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1236 struct virtio_iommu_config *dev_config = &dev->config; 1237 struct virtio_iommu_config *out_config = (void *)config_data; 1238 1239 out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask); 1240 out_config->input_range.start = cpu_to_le64(dev_config->input_range.start); 1241 out_config->input_range.end = cpu_to_le64(dev_config->input_range.end); 1242 out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start); 1243 out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end); 1244 out_config->probe_size = cpu_to_le32(dev_config->probe_size); 1245 out_config->bypass = dev_config->bypass; 1246 1247 trace_virtio_iommu_get_config(dev_config->page_size_mask, 1248 dev_config->input_range.start, 1249 dev_config->input_range.end, 1250 dev_config->domain_range.start, 1251 dev_config->domain_range.end, 1252 dev_config->probe_size, 1253 dev_config->bypass); 1254 } 1255 1256 static void virtio_iommu_set_config(VirtIODevice *vdev, 1257 const uint8_t *config_data) 1258 { 1259 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1260 struct virtio_iommu_config *dev_config = &dev->config; 1261 const struct virtio_iommu_config *in_config = (void *)config_data; 1262 1263 if (in_config->bypass != dev_config->bypass) { 1264 if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) { 1265 virtio_error(vdev, "cannot set config.bypass"); 1266 return; 1267 } else if (in_config->bypass != 0 && in_config->bypass != 1) { 1268 virtio_error(vdev, "invalid config.bypass value '%u'", 1269 in_config->bypass); 1270 return; 1271 } 1272 dev_config->bypass = in_config->bypass; 1273 virtio_iommu_switch_address_space_all(dev); 1274 } 1275 1276 trace_virtio_iommu_set_config(in_config->bypass); 1277 } 1278 1279 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f, 1280 Error **errp) 1281 { 1282 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1283 1284 f |= dev->features; 1285 trace_virtio_iommu_get_features(f); 1286 return f; 1287 } 1288 1289 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 1290 { 1291 guint ua = GPOINTER_TO_UINT(a); 1292 guint ub = GPOINTER_TO_UINT(b); 1293 return (ua > ub) - (ua < ub); 1294 } 1295 1296 static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data) 1297 { 1298 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 1299 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 1300 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 1301 1302 trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high, 1303 mapping->phys_addr); 1304 virtio_iommu_notify_map(mr, interval->low, interval->high, 1305 mapping->phys_addr, mapping->flags); 1306 return false; 1307 } 1308 1309 static void virtio_iommu_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n) 1310 { 1311 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1312 VirtIOIOMMU *s = sdev->viommu; 1313 uint32_t sid; 1314 VirtIOIOMMUEndpoint *ep; 1315 1316 sid = virtio_iommu_get_bdf(sdev); 1317 1318 qemu_rec_mutex_lock(&s->mutex); 1319 1320 if (!s->endpoints) { 1321 goto unlock; 1322 } 1323 1324 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 1325 if (!ep || !ep->domain) { 1326 goto unlock; 1327 } 1328 1329 g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr); 1330 1331 unlock: 1332 qemu_rec_mutex_unlock(&s->mutex); 1333 } 1334 1335 static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, 1336 IOMMUNotifierFlag old, 1337 IOMMUNotifierFlag new, 1338 Error **errp) 1339 { 1340 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { 1341 error_setg(errp, "Virtio-iommu does not support dev-iotlb yet"); 1342 return -EINVAL; 1343 } 1344 1345 if (old == IOMMU_NOTIFIER_NONE) { 1346 trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name); 1347 } else if (new == IOMMU_NOTIFIER_NONE) { 1348 trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name); 1349 } 1350 return 0; 1351 } 1352 1353 /* 1354 * The default mask depends on the "granule" property. For example, with 1355 * 4k granule, it is -(4 * KiB). When an assigned device has page size 1356 * restrictions due to the hardware IOMMU configuration, apply this restriction 1357 * to the mask. 1358 */ 1359 static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, 1360 uint64_t new_mask, 1361 Error **errp) 1362 { 1363 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1364 VirtIOIOMMU *s = sdev->viommu; 1365 uint64_t cur_mask = s->config.page_size_mask; 1366 1367 trace_virtio_iommu_set_page_size_mask(mr->parent_obj.name, cur_mask, 1368 new_mask); 1369 1370 if ((cur_mask & new_mask) == 0) { 1371 error_setg(errp, "virtio-iommu %s reports a page size mask 0x%"PRIx64 1372 " incompatible with currently supported mask 0x%"PRIx64, 1373 mr->parent_obj.name, new_mask, cur_mask); 1374 return -1; 1375 } 1376 1377 /* 1378 * Once the granule is frozen we can't change the mask anymore. If by 1379 * chance the hotplugged device supports the same granule, we can still 1380 * accept it. 1381 */ 1382 if (s->granule_frozen) { 1383 int cur_granule = ctz64(cur_mask); 1384 1385 if (!(BIT_ULL(cur_granule) & new_mask)) { 1386 error_setg(errp, "virtio-iommu %s does not support frozen granule 0x%llx", 1387 mr->parent_obj.name, BIT_ULL(cur_granule)); 1388 return -1; 1389 } 1390 return 0; 1391 } 1392 1393 s->config.page_size_mask &= new_mask; 1394 return 0; 1395 } 1396 1397 static void virtio_iommu_system_reset(void *opaque) 1398 { 1399 VirtIOIOMMU *s = opaque; 1400 1401 trace_virtio_iommu_system_reset(); 1402 1403 memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num)); 1404 1405 /* 1406 * config.bypass is sticky across device reset, but should be restored on 1407 * system reset 1408 */ 1409 s->config.bypass = s->boot_bypass; 1410 virtio_iommu_switch_address_space_all(s); 1411 1412 } 1413 1414 static void virtio_iommu_freeze_granule(Notifier *notifier, void *data) 1415 { 1416 VirtIOIOMMU *s = container_of(notifier, VirtIOIOMMU, machine_done); 1417 int granule; 1418 1419 if (likely(s->config.bypass)) { 1420 /* 1421 * Transient IOMMU MR enable to collect page_size_mask requirements 1422 * through memory_region_iommu_set_page_size_mask() called by 1423 * VFIO region_add() callback 1424 */ 1425 s->config.bypass = false; 1426 virtio_iommu_switch_address_space_all(s); 1427 /* restore default */ 1428 s->config.bypass = true; 1429 virtio_iommu_switch_address_space_all(s); 1430 } 1431 s->granule_frozen = true; 1432 granule = ctz64(s->config.page_size_mask); 1433 trace_virtio_iommu_freeze_granule(BIT_ULL(granule)); 1434 } 1435 1436 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) 1437 { 1438 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1439 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1440 1441 virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config)); 1442 1443 s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, 1444 virtio_iommu_handle_command); 1445 s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL); 1446 1447 /* 1448 * config.bypass is needed to get initial address space early, such as 1449 * in vfio realize 1450 */ 1451 s->config.bypass = s->boot_bypass; 1452 if (s->aw_bits < 32 || s->aw_bits > 64) { 1453 error_setg(errp, "aw-bits must be within [32,64]"); 1454 return; 1455 } 1456 s->config.input_range.end = 1457 s->aw_bits == 64 ? UINT64_MAX : BIT_ULL(s->aw_bits) - 1; 1458 1459 switch (s->granule_mode) { 1460 case GRANULE_MODE_4K: 1461 s->config.page_size_mask = -(4 * KiB); 1462 break; 1463 case GRANULE_MODE_8K: 1464 s->config.page_size_mask = -(8 * KiB); 1465 break; 1466 case GRANULE_MODE_16K: 1467 s->config.page_size_mask = -(16 * KiB); 1468 break; 1469 case GRANULE_MODE_64K: 1470 s->config.page_size_mask = -(64 * KiB); 1471 break; 1472 case GRANULE_MODE_HOST: 1473 s->config.page_size_mask = qemu_real_host_page_mask(); 1474 break; 1475 default: 1476 error_setg(errp, "Unsupported granule mode"); 1477 } 1478 s->config.domain_range.end = UINT32_MAX; 1479 s->config.probe_size = VIOMMU_PROBE_SIZE; 1480 1481 virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX); 1482 virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC); 1483 virtio_add_feature(&s->features, VIRTIO_F_VERSION_1); 1484 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE); 1485 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE); 1486 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP); 1487 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO); 1488 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE); 1489 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG); 1490 1491 qemu_rec_mutex_init(&s->mutex); 1492 1493 s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free); 1494 1495 s->host_iommu_devices = g_hash_table_new_full(hiod_hash, hiod_equal, 1496 g_free, hiod_destroy); 1497 1498 if (s->primary_bus) { 1499 pci_setup_iommu(s->primary_bus, &virtio_iommu_ops, s); 1500 } else { 1501 error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!"); 1502 } 1503 1504 s->machine_done.notify = virtio_iommu_freeze_granule; 1505 qemu_add_machine_init_done_notifier(&s->machine_done); 1506 1507 qemu_register_reset(virtio_iommu_system_reset, s); 1508 } 1509 1510 static void virtio_iommu_device_unrealize(DeviceState *dev) 1511 { 1512 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1513 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1514 1515 qemu_unregister_reset(virtio_iommu_system_reset, s); 1516 qemu_remove_machine_init_done_notifier(&s->machine_done); 1517 1518 g_hash_table_destroy(s->as_by_busptr); 1519 if (s->domains) { 1520 g_tree_destroy(s->domains); 1521 } 1522 if (s->endpoints) { 1523 g_tree_destroy(s->endpoints); 1524 } 1525 1526 qemu_rec_mutex_destroy(&s->mutex); 1527 1528 virtio_delete_queue(s->req_vq); 1529 virtio_delete_queue(s->event_vq); 1530 virtio_cleanup(vdev); 1531 } 1532 1533 static void virtio_iommu_device_reset(VirtIODevice *vdev) 1534 { 1535 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 1536 1537 trace_virtio_iommu_device_reset(); 1538 1539 if (s->domains) { 1540 g_tree_destroy(s->domains); 1541 } 1542 if (s->endpoints) { 1543 g_tree_destroy(s->endpoints); 1544 } 1545 s->domains = g_tree_new_full((GCompareDataFunc)int_cmp, 1546 NULL, NULL, virtio_iommu_put_domain); 1547 s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp, 1548 NULL, NULL, virtio_iommu_put_endpoint); 1549 } 1550 1551 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status) 1552 { 1553 trace_virtio_iommu_device_status(status); 1554 } 1555 1556 static void virtio_iommu_instance_init(Object *obj) 1557 { 1558 } 1559 1560 #define VMSTATE_INTERVAL \ 1561 { \ 1562 .name = "interval", \ 1563 .version_id = 1, \ 1564 .minimum_version_id = 1, \ 1565 .fields = (const VMStateField[]) { \ 1566 VMSTATE_UINT64(low, VirtIOIOMMUInterval), \ 1567 VMSTATE_UINT64(high, VirtIOIOMMUInterval), \ 1568 VMSTATE_END_OF_LIST() \ 1569 } \ 1570 } 1571 1572 #define VMSTATE_MAPPING \ 1573 { \ 1574 .name = "mapping", \ 1575 .version_id = 1, \ 1576 .minimum_version_id = 1, \ 1577 .fields = (const VMStateField[]) { \ 1578 VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\ 1579 VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \ 1580 VMSTATE_END_OF_LIST() \ 1581 }, \ 1582 } 1583 1584 static const VMStateDescription vmstate_interval_mapping[2] = { 1585 VMSTATE_MAPPING, /* value */ 1586 VMSTATE_INTERVAL /* key */ 1587 }; 1588 1589 static int domain_preload(void *opaque) 1590 { 1591 VirtIOIOMMUDomain *domain = opaque; 1592 1593 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 1594 NULL, g_free, g_free); 1595 return 0; 1596 } 1597 1598 static const VMStateDescription vmstate_endpoint = { 1599 .name = "endpoint", 1600 .version_id = 1, 1601 .minimum_version_id = 1, 1602 .fields = (const VMStateField[]) { 1603 VMSTATE_UINT32(id, VirtIOIOMMUEndpoint), 1604 VMSTATE_END_OF_LIST() 1605 } 1606 }; 1607 1608 static const VMStateDescription vmstate_domain = { 1609 .name = "domain", 1610 .version_id = 2, 1611 .minimum_version_id = 2, 1612 .pre_load = domain_preload, 1613 .fields = (const VMStateField[]) { 1614 VMSTATE_UINT32(id, VirtIOIOMMUDomain), 1615 VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1, 1616 vmstate_interval_mapping, 1617 VirtIOIOMMUInterval, VirtIOIOMMUMapping), 1618 VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1, 1619 vmstate_endpoint, VirtIOIOMMUEndpoint, next), 1620 VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2), 1621 VMSTATE_END_OF_LIST() 1622 } 1623 }; 1624 1625 static gboolean reconstruct_endpoints(gpointer key, gpointer value, 1626 gpointer data) 1627 { 1628 VirtIOIOMMU *s = (VirtIOIOMMU *)data; 1629 VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value; 1630 VirtIOIOMMUEndpoint *iter; 1631 IOMMUMemoryRegion *mr; 1632 1633 QLIST_FOREACH(iter, &d->endpoint_list, next) { 1634 mr = virtio_iommu_mr(s, iter->id); 1635 assert(mr); 1636 1637 iter->domain = d; 1638 iter->iommu_mr = mr; 1639 g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter); 1640 } 1641 return false; /* continue the domain traversal */ 1642 } 1643 1644 static int iommu_post_load(void *opaque, int version_id) 1645 { 1646 VirtIOIOMMU *s = opaque; 1647 1648 g_tree_foreach(s->domains, reconstruct_endpoints, s); 1649 1650 /* 1651 * Memory regions are dynamically turned on/off depending on 1652 * 'config.bypass' and attached domain type if there is. After 1653 * migration, we need to make sure the memory regions are 1654 * still correct. 1655 */ 1656 virtio_iommu_switch_address_space_all(s); 1657 return 0; 1658 } 1659 1660 static const VMStateDescription vmstate_virtio_iommu_device = { 1661 .name = "virtio-iommu-device", 1662 .minimum_version_id = 2, 1663 .version_id = 2, 1664 .post_load = iommu_post_load, 1665 .fields = (const VMStateField[]) { 1666 VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2, 1667 &vmstate_domain, VirtIOIOMMUDomain), 1668 VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2), 1669 VMSTATE_END_OF_LIST() 1670 }, 1671 }; 1672 1673 static const VMStateDescription vmstate_virtio_iommu = { 1674 .name = "virtio-iommu", 1675 .minimum_version_id = 2, 1676 .priority = MIG_PRI_IOMMU, 1677 .version_id = 2, 1678 .fields = (const VMStateField[]) { 1679 VMSTATE_VIRTIO_DEVICE, 1680 VMSTATE_END_OF_LIST() 1681 }, 1682 }; 1683 1684 static Property virtio_iommu_properties[] = { 1685 DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, 1686 TYPE_PCI_BUS, PCIBus *), 1687 DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true), 1688 DEFINE_PROP_GRANULE_MODE("granule", VirtIOIOMMU, granule_mode, 1689 GRANULE_MODE_HOST), 1690 DEFINE_PROP_UINT8("aw-bits", VirtIOIOMMU, aw_bits, 64), 1691 DEFINE_PROP_END_OF_LIST(), 1692 }; 1693 1694 static void virtio_iommu_class_init(ObjectClass *klass, void *data) 1695 { 1696 DeviceClass *dc = DEVICE_CLASS(klass); 1697 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 1698 1699 device_class_set_props(dc, virtio_iommu_properties); 1700 dc->vmsd = &vmstate_virtio_iommu; 1701 1702 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1703 vdc->realize = virtio_iommu_device_realize; 1704 vdc->unrealize = virtio_iommu_device_unrealize; 1705 vdc->reset = virtio_iommu_device_reset; 1706 vdc->get_config = virtio_iommu_get_config; 1707 vdc->set_config = virtio_iommu_set_config; 1708 vdc->get_features = virtio_iommu_get_features; 1709 vdc->set_status = virtio_iommu_set_status; 1710 vdc->vmsd = &vmstate_virtio_iommu_device; 1711 } 1712 1713 static void virtio_iommu_memory_region_class_init(ObjectClass *klass, 1714 void *data) 1715 { 1716 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 1717 1718 imrc->translate = virtio_iommu_translate; 1719 imrc->replay = virtio_iommu_replay; 1720 imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; 1721 imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; 1722 } 1723 1724 static const TypeInfo virtio_iommu_info = { 1725 .name = TYPE_VIRTIO_IOMMU, 1726 .parent = TYPE_VIRTIO_DEVICE, 1727 .instance_size = sizeof(VirtIOIOMMU), 1728 .instance_init = virtio_iommu_instance_init, 1729 .class_init = virtio_iommu_class_init, 1730 }; 1731 1732 static const TypeInfo virtio_iommu_memory_region_info = { 1733 .parent = TYPE_IOMMU_MEMORY_REGION, 1734 .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION, 1735 .class_init = virtio_iommu_memory_region_class_init, 1736 }; 1737 1738 static void virtio_register_types(void) 1739 { 1740 type_register_static(&virtio_iommu_info); 1741 type_register_static(&virtio_iommu_memory_region_info); 1742 } 1743 1744 type_init(virtio_register_types) 1745