1 /* 2 * virtio-iommu device 3 * 4 * Copyright (c) 2020 Red Hat, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2 or later, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along with 16 * this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/log.h" 22 #include "qemu/iov.h" 23 #include "qemu/range.h" 24 #include "qemu/reserved-region.h" 25 #include "exec/target_page.h" 26 #include "hw/qdev-properties.h" 27 #include "hw/virtio/virtio.h" 28 #include "sysemu/kvm.h" 29 #include "sysemu/reset.h" 30 #include "sysemu/sysemu.h" 31 #include "qemu/reserved-region.h" 32 #include "qapi/error.h" 33 #include "qemu/error-report.h" 34 #include "trace.h" 35 36 #include "standard-headers/linux/virtio_ids.h" 37 38 #include "hw/virtio/virtio-bus.h" 39 #include "hw/virtio/virtio-iommu.h" 40 #include "hw/pci/pci_bus.h" 41 #include "hw/pci/pci.h" 42 43 /* Max size */ 44 #define VIOMMU_DEFAULT_QUEUE_SIZE 256 45 #define VIOMMU_PROBE_SIZE 512 46 47 typedef struct VirtIOIOMMUDomain { 48 uint32_t id; 49 bool bypass; 50 GTree *mappings; 51 QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list; 52 } VirtIOIOMMUDomain; 53 54 typedef struct VirtIOIOMMUEndpoint { 55 uint32_t id; 56 VirtIOIOMMUDomain *domain; 57 IOMMUMemoryRegion *iommu_mr; 58 QLIST_ENTRY(VirtIOIOMMUEndpoint) next; 59 } VirtIOIOMMUEndpoint; 60 61 typedef struct VirtIOIOMMUInterval { 62 uint64_t low; 63 uint64_t high; 64 } VirtIOIOMMUInterval; 65 66 typedef struct VirtIOIOMMUMapping { 67 uint64_t phys_addr; 68 uint32_t flags; 69 } VirtIOIOMMUMapping; 70 71 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev) 72 { 73 return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn); 74 } 75 76 static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev) 77 { 78 uint32_t sid; 79 bool bypassed; 80 VirtIOIOMMU *s = sdev->viommu; 81 VirtIOIOMMUEndpoint *ep; 82 83 sid = virtio_iommu_get_bdf(sdev); 84 85 qemu_rec_mutex_lock(&s->mutex); 86 /* need to check bypass before system reset */ 87 if (!s->endpoints) { 88 bypassed = s->config.bypass; 89 goto unlock; 90 } 91 92 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 93 if (!ep || !ep->domain) { 94 bypassed = s->config.bypass; 95 } else { 96 bypassed = ep->domain->bypass; 97 } 98 99 unlock: 100 qemu_rec_mutex_unlock(&s->mutex); 101 return bypassed; 102 } 103 104 /* Return whether the device is using IOMMU translation. */ 105 static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev) 106 { 107 bool use_remapping; 108 109 assert(sdev); 110 111 use_remapping = !virtio_iommu_device_bypassed(sdev); 112 113 trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus), 114 PCI_SLOT(sdev->devfn), 115 PCI_FUNC(sdev->devfn), 116 use_remapping); 117 118 /* Turn off first then on the other */ 119 if (use_remapping) { 120 memory_region_set_enabled(&sdev->bypass_mr, false); 121 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true); 122 } else { 123 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false); 124 memory_region_set_enabled(&sdev->bypass_mr, true); 125 } 126 127 return use_remapping; 128 } 129 130 static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s) 131 { 132 GHashTableIter iter; 133 IOMMUPciBus *iommu_pci_bus; 134 int i; 135 136 g_hash_table_iter_init(&iter, s->as_by_busptr); 137 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 138 for (i = 0; i < PCI_DEVFN_MAX; i++) { 139 if (!iommu_pci_bus->pbdev[i]) { 140 continue; 141 } 142 virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]); 143 } 144 } 145 } 146 147 /** 148 * The bus number is used for lookup when SID based operations occur. 149 * In that case we lazily populate the IOMMUPciBus array from the bus hash 150 * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus 151 * numbers may not be always initialized yet. 152 */ 153 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num) 154 { 155 IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num]; 156 157 if (!iommu_pci_bus) { 158 GHashTableIter iter; 159 160 g_hash_table_iter_init(&iter, s->as_by_busptr); 161 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 162 if (pci_bus_num(iommu_pci_bus->bus) == bus_num) { 163 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus; 164 return iommu_pci_bus; 165 } 166 } 167 return NULL; 168 } 169 return iommu_pci_bus; 170 } 171 172 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid) 173 { 174 uint8_t bus_n, devfn; 175 IOMMUPciBus *iommu_pci_bus; 176 IOMMUDevice *dev; 177 178 bus_n = PCI_BUS_NUM(sid); 179 iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n); 180 if (iommu_pci_bus) { 181 devfn = sid & (PCI_DEVFN_MAX - 1); 182 dev = iommu_pci_bus->pbdev[devfn]; 183 if (dev) { 184 return &dev->iommu_mr; 185 } 186 } 187 return NULL; 188 } 189 190 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 191 { 192 VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a; 193 VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b; 194 195 if (inta->high < intb->low) { 196 return -1; 197 } else if (intb->high < inta->low) { 198 return 1; 199 } else { 200 return 0; 201 } 202 } 203 204 static void virtio_iommu_notify_map_unmap(IOMMUMemoryRegion *mr, 205 IOMMUTLBEvent *event, 206 hwaddr virt_start, hwaddr virt_end) 207 { 208 uint64_t delta = virt_end - virt_start; 209 210 event->entry.iova = virt_start; 211 event->entry.addr_mask = delta; 212 213 if (delta == UINT64_MAX) { 214 memory_region_notify_iommu(mr, 0, *event); 215 } 216 217 while (virt_start != virt_end + 1) { 218 uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64); 219 220 event->entry.addr_mask = mask; 221 event->entry.iova = virt_start; 222 memory_region_notify_iommu(mr, 0, *event); 223 virt_start += mask + 1; 224 if (event->entry.perm != IOMMU_NONE) { 225 event->entry.translated_addr += mask + 1; 226 } 227 } 228 } 229 230 static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start, 231 hwaddr virt_end, hwaddr paddr, 232 uint32_t flags) 233 { 234 IOMMUTLBEvent event; 235 IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ, 236 flags & VIRTIO_IOMMU_MAP_F_WRITE); 237 238 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) || 239 (flags & VIRTIO_IOMMU_MAP_F_MMIO) || !perm) { 240 return; 241 } 242 243 trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end, 244 paddr, perm); 245 246 event.type = IOMMU_NOTIFIER_MAP; 247 event.entry.target_as = &address_space_memory; 248 event.entry.perm = perm; 249 event.entry.translated_addr = paddr; 250 251 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end); 252 } 253 254 static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start, 255 hwaddr virt_end) 256 { 257 IOMMUTLBEvent event; 258 259 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) { 260 return; 261 } 262 263 trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end); 264 265 event.type = IOMMU_NOTIFIER_UNMAP; 266 event.entry.target_as = &address_space_memory; 267 event.entry.perm = IOMMU_NONE; 268 event.entry.translated_addr = 0; 269 270 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end); 271 } 272 273 static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value, 274 gpointer data) 275 { 276 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 277 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 278 279 virtio_iommu_notify_unmap(mr, interval->low, interval->high); 280 281 return false; 282 } 283 284 static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value, 285 gpointer data) 286 { 287 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 288 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 289 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 290 291 virtio_iommu_notify_map(mr, interval->low, interval->high, 292 mapping->phys_addr, mapping->flags); 293 294 return false; 295 } 296 297 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep) 298 { 299 VirtIOIOMMUDomain *domain = ep->domain; 300 IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 301 302 if (!ep->domain) { 303 return; 304 } 305 g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb, 306 ep->iommu_mr); 307 QLIST_REMOVE(ep, next); 308 ep->domain = NULL; 309 virtio_iommu_switch_address_space(sdev); 310 } 311 312 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s, 313 uint32_t ep_id) 314 { 315 VirtIOIOMMUEndpoint *ep; 316 IOMMUMemoryRegion *mr; 317 318 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 319 if (ep) { 320 return ep; 321 } 322 mr = virtio_iommu_mr(s, ep_id); 323 if (!mr) { 324 return NULL; 325 } 326 ep = g_malloc0(sizeof(*ep)); 327 ep->id = ep_id; 328 ep->iommu_mr = mr; 329 trace_virtio_iommu_get_endpoint(ep_id); 330 g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep); 331 return ep; 332 } 333 334 static void virtio_iommu_put_endpoint(gpointer data) 335 { 336 VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data; 337 338 if (ep->domain) { 339 virtio_iommu_detach_endpoint_from_domain(ep); 340 } 341 342 trace_virtio_iommu_put_endpoint(ep->id); 343 g_free(ep); 344 } 345 346 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s, 347 uint32_t domain_id, 348 bool bypass) 349 { 350 VirtIOIOMMUDomain *domain; 351 352 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 353 if (domain) { 354 if (domain->bypass != bypass) { 355 return NULL; 356 } 357 return domain; 358 } 359 domain = g_malloc0(sizeof(*domain)); 360 domain->id = domain_id; 361 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 362 NULL, (GDestroyNotify)g_free, 363 (GDestroyNotify)g_free); 364 domain->bypass = bypass; 365 g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain); 366 QLIST_INIT(&domain->endpoint_list); 367 trace_virtio_iommu_get_domain(domain_id); 368 return domain; 369 } 370 371 static void virtio_iommu_put_domain(gpointer data) 372 { 373 VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data; 374 VirtIOIOMMUEndpoint *iter, *tmp; 375 376 QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) { 377 virtio_iommu_detach_endpoint_from_domain(iter); 378 } 379 g_tree_destroy(domain->mappings); 380 trace_virtio_iommu_put_domain(domain->id); 381 g_free(domain); 382 } 383 384 static void add_prop_resv_regions(IOMMUDevice *sdev) 385 { 386 VirtIOIOMMU *s = sdev->viommu; 387 int i; 388 389 for (i = 0; i < s->nr_prop_resv_regions; i++) { 390 ReservedRegion *reg = g_new0(ReservedRegion, 1); 391 392 *reg = s->prop_resv_regions[i]; 393 sdev->resv_regions = resv_region_list_insert(sdev->resv_regions, reg); 394 } 395 } 396 397 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque, 398 int devfn) 399 { 400 VirtIOIOMMU *s = opaque; 401 IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus); 402 static uint32_t mr_index; 403 IOMMUDevice *sdev; 404 405 if (!sbus) { 406 sbus = g_malloc0(sizeof(IOMMUPciBus) + 407 sizeof(IOMMUDevice *) * PCI_DEVFN_MAX); 408 sbus->bus = bus; 409 g_hash_table_insert(s->as_by_busptr, bus, sbus); 410 } 411 412 sdev = sbus->pbdev[devfn]; 413 if (!sdev) { 414 char *name = g_strdup_printf("%s-%d-%d", 415 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 416 mr_index++, devfn); 417 sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1); 418 419 sdev->viommu = s; 420 sdev->bus = bus; 421 sdev->devfn = devfn; 422 423 trace_virtio_iommu_init_iommu_mr(name); 424 425 memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX); 426 address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU); 427 add_prop_resv_regions(sdev); 428 429 /* 430 * Build the IOMMU disabled container with aliases to the 431 * shared MRs. Note that aliasing to a shared memory region 432 * could help the memory API to detect same FlatViews so we 433 * can have devices to share the same FlatView when in bypass 434 * mode. (either by not configuring virtio-iommu driver or with 435 * "iommu=pt"). It will greatly reduce the total number of 436 * FlatViews of the system hence VM runs faster. 437 */ 438 memory_region_init_alias(&sdev->bypass_mr, OBJECT(s), 439 "system", get_system_memory(), 0, 440 memory_region_size(get_system_memory())); 441 442 memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr), 443 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 444 OBJECT(s), name, 445 UINT64_MAX); 446 447 /* 448 * Hook both the containers under the root container, we 449 * switch between iommu & bypass MRs by enable/disable 450 * corresponding sub-containers 451 */ 452 memory_region_add_subregion_overlap(&sdev->root, 0, 453 MEMORY_REGION(&sdev->iommu_mr), 454 0); 455 memory_region_add_subregion_overlap(&sdev->root, 0, 456 &sdev->bypass_mr, 0); 457 458 virtio_iommu_switch_address_space(sdev); 459 g_free(name); 460 } 461 return &sdev->as; 462 } 463 464 static const PCIIOMMUOps virtio_iommu_ops = { 465 .get_address_space = virtio_iommu_find_add_as, 466 }; 467 468 static int virtio_iommu_attach(VirtIOIOMMU *s, 469 struct virtio_iommu_req_attach *req) 470 { 471 uint32_t domain_id = le32_to_cpu(req->domain); 472 uint32_t ep_id = le32_to_cpu(req->endpoint); 473 uint32_t flags = le32_to_cpu(req->flags); 474 VirtIOIOMMUDomain *domain; 475 VirtIOIOMMUEndpoint *ep; 476 IOMMUDevice *sdev; 477 478 trace_virtio_iommu_attach(domain_id, ep_id); 479 480 if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) { 481 return VIRTIO_IOMMU_S_INVAL; 482 } 483 484 ep = virtio_iommu_get_endpoint(s, ep_id); 485 if (!ep) { 486 return VIRTIO_IOMMU_S_NOENT; 487 } 488 489 if (ep->domain) { 490 VirtIOIOMMUDomain *previous_domain = ep->domain; 491 /* 492 * the device is already attached to a domain, 493 * detach it first 494 */ 495 virtio_iommu_detach_endpoint_from_domain(ep); 496 if (QLIST_EMPTY(&previous_domain->endpoint_list)) { 497 g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id)); 498 } 499 } 500 501 domain = virtio_iommu_get_domain(s, domain_id, 502 flags & VIRTIO_IOMMU_ATTACH_F_BYPASS); 503 if (!domain) { 504 /* Incompatible bypass flag */ 505 return VIRTIO_IOMMU_S_INVAL; 506 } 507 QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next); 508 509 ep->domain = domain; 510 sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 511 virtio_iommu_switch_address_space(sdev); 512 513 /* Replay domain mappings on the associated memory region */ 514 g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb, 515 ep->iommu_mr); 516 517 return VIRTIO_IOMMU_S_OK; 518 } 519 520 static int virtio_iommu_detach(VirtIOIOMMU *s, 521 struct virtio_iommu_req_detach *req) 522 { 523 uint32_t domain_id = le32_to_cpu(req->domain); 524 uint32_t ep_id = le32_to_cpu(req->endpoint); 525 VirtIOIOMMUDomain *domain; 526 VirtIOIOMMUEndpoint *ep; 527 528 trace_virtio_iommu_detach(domain_id, ep_id); 529 530 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 531 if (!ep) { 532 return VIRTIO_IOMMU_S_NOENT; 533 } 534 535 domain = ep->domain; 536 537 if (!domain || domain->id != domain_id) { 538 return VIRTIO_IOMMU_S_INVAL; 539 } 540 541 virtio_iommu_detach_endpoint_from_domain(ep); 542 543 if (QLIST_EMPTY(&domain->endpoint_list)) { 544 g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id)); 545 } 546 return VIRTIO_IOMMU_S_OK; 547 } 548 549 static int virtio_iommu_map(VirtIOIOMMU *s, 550 struct virtio_iommu_req_map *req) 551 { 552 uint32_t domain_id = le32_to_cpu(req->domain); 553 uint64_t phys_start = le64_to_cpu(req->phys_start); 554 uint64_t virt_start = le64_to_cpu(req->virt_start); 555 uint64_t virt_end = le64_to_cpu(req->virt_end); 556 uint32_t flags = le32_to_cpu(req->flags); 557 VirtIOIOMMUDomain *domain; 558 VirtIOIOMMUInterval *interval; 559 VirtIOIOMMUMapping *mapping; 560 VirtIOIOMMUEndpoint *ep; 561 562 if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) { 563 return VIRTIO_IOMMU_S_INVAL; 564 } 565 566 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 567 if (!domain) { 568 return VIRTIO_IOMMU_S_NOENT; 569 } 570 571 if (domain->bypass) { 572 return VIRTIO_IOMMU_S_INVAL; 573 } 574 575 interval = g_malloc0(sizeof(*interval)); 576 577 interval->low = virt_start; 578 interval->high = virt_end; 579 580 mapping = g_tree_lookup(domain->mappings, (gpointer)interval); 581 if (mapping) { 582 g_free(interval); 583 return VIRTIO_IOMMU_S_INVAL; 584 } 585 586 trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags); 587 588 mapping = g_malloc0(sizeof(*mapping)); 589 mapping->phys_addr = phys_start; 590 mapping->flags = flags; 591 592 g_tree_insert(domain->mappings, interval, mapping); 593 594 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 595 virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start, 596 flags); 597 } 598 599 return VIRTIO_IOMMU_S_OK; 600 } 601 602 static int virtio_iommu_unmap(VirtIOIOMMU *s, 603 struct virtio_iommu_req_unmap *req) 604 { 605 uint32_t domain_id = le32_to_cpu(req->domain); 606 uint64_t virt_start = le64_to_cpu(req->virt_start); 607 uint64_t virt_end = le64_to_cpu(req->virt_end); 608 VirtIOIOMMUMapping *iter_val; 609 VirtIOIOMMUInterval interval, *iter_key; 610 VirtIOIOMMUDomain *domain; 611 VirtIOIOMMUEndpoint *ep; 612 int ret = VIRTIO_IOMMU_S_OK; 613 614 trace_virtio_iommu_unmap(domain_id, virt_start, virt_end); 615 616 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 617 if (!domain) { 618 return VIRTIO_IOMMU_S_NOENT; 619 } 620 621 if (domain->bypass) { 622 return VIRTIO_IOMMU_S_INVAL; 623 } 624 625 interval.low = virt_start; 626 interval.high = virt_end; 627 628 while (g_tree_lookup_extended(domain->mappings, &interval, 629 (void **)&iter_key, (void**)&iter_val)) { 630 uint64_t current_low = iter_key->low; 631 uint64_t current_high = iter_key->high; 632 633 if (interval.low <= current_low && interval.high >= current_high) { 634 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 635 virtio_iommu_notify_unmap(ep->iommu_mr, current_low, 636 current_high); 637 } 638 g_tree_remove(domain->mappings, iter_key); 639 trace_virtio_iommu_unmap_done(domain_id, current_low, current_high); 640 } else { 641 ret = VIRTIO_IOMMU_S_RANGE; 642 break; 643 } 644 } 645 return ret; 646 } 647 648 static ssize_t virtio_iommu_fill_resv_mem_prop(IOMMUDevice *sdev, uint32_t ep, 649 uint8_t *buf, size_t free) 650 { 651 struct virtio_iommu_probe_resv_mem prop = {}; 652 size_t size = sizeof(prop), length = size - sizeof(prop.head), total; 653 GList *l; 654 655 total = size * g_list_length(sdev->resv_regions); 656 if (total > free) { 657 return -ENOSPC; 658 } 659 660 for (l = sdev->resv_regions; l; l = l->next) { 661 ReservedRegion *reg = l->data; 662 unsigned subtype = reg->type; 663 Range *range = ®->range; 664 665 assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED || 666 subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI); 667 prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM); 668 prop.head.length = cpu_to_le16(length); 669 prop.subtype = subtype; 670 prop.start = cpu_to_le64(range_lob(range)); 671 prop.end = cpu_to_le64(range_upb(range)); 672 673 memcpy(buf, &prop, size); 674 675 trace_virtio_iommu_fill_resv_property(ep, prop.subtype, 676 prop.start, prop.end); 677 buf += size; 678 } 679 return total; 680 } 681 682 /** 683 * virtio_iommu_probe - Fill the probe request buffer with 684 * the properties the device is able to return 685 */ 686 static int virtio_iommu_probe(VirtIOIOMMU *s, 687 struct virtio_iommu_req_probe *req, 688 uint8_t *buf) 689 { 690 uint32_t ep_id = le32_to_cpu(req->endpoint); 691 IOMMUMemoryRegion *iommu_mr = virtio_iommu_mr(s, ep_id); 692 size_t free = VIOMMU_PROBE_SIZE; 693 IOMMUDevice *sdev; 694 ssize_t count; 695 696 if (!iommu_mr) { 697 return VIRTIO_IOMMU_S_NOENT; 698 } 699 700 sdev = container_of(iommu_mr, IOMMUDevice, iommu_mr); 701 if (!sdev) { 702 return -EINVAL; 703 } 704 705 count = virtio_iommu_fill_resv_mem_prop(sdev, ep_id, buf, free); 706 if (count < 0) { 707 return VIRTIO_IOMMU_S_INVAL; 708 } 709 buf += count; 710 free -= count; 711 sdev->probe_done = true; 712 713 return VIRTIO_IOMMU_S_OK; 714 } 715 716 static int virtio_iommu_iov_to_req(struct iovec *iov, 717 unsigned int iov_cnt, 718 void *req, size_t payload_sz) 719 { 720 size_t sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz); 721 722 if (unlikely(sz != payload_sz)) { 723 return VIRTIO_IOMMU_S_INVAL; 724 } 725 return 0; 726 } 727 728 #define virtio_iommu_handle_req(__req) \ 729 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \ 730 struct iovec *iov, \ 731 unsigned int iov_cnt) \ 732 { \ 733 struct virtio_iommu_req_ ## __req req; \ 734 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, \ 735 sizeof(req) - sizeof(struct virtio_iommu_req_tail));\ 736 \ 737 return ret ? ret : virtio_iommu_ ## __req(s, &req); \ 738 } 739 740 virtio_iommu_handle_req(attach) 741 virtio_iommu_handle_req(detach) 742 virtio_iommu_handle_req(map) 743 virtio_iommu_handle_req(unmap) 744 745 static int virtio_iommu_handle_probe(VirtIOIOMMU *s, 746 struct iovec *iov, 747 unsigned int iov_cnt, 748 uint8_t *buf) 749 { 750 struct virtio_iommu_req_probe req; 751 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); 752 753 return ret ? ret : virtio_iommu_probe(s, &req, buf); 754 } 755 756 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq) 757 { 758 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 759 struct virtio_iommu_req_head head; 760 struct virtio_iommu_req_tail tail = {}; 761 VirtQueueElement *elem; 762 unsigned int iov_cnt; 763 struct iovec *iov; 764 void *buf = NULL; 765 size_t sz; 766 767 for (;;) { 768 size_t output_size = sizeof(tail); 769 770 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 771 if (!elem) { 772 return; 773 } 774 775 if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) || 776 iov_size(elem->out_sg, elem->out_num) < sizeof(head)) { 777 virtio_error(vdev, "virtio-iommu bad head/tail size"); 778 virtqueue_detach_element(vq, elem, 0); 779 g_free(elem); 780 break; 781 } 782 783 iov_cnt = elem->out_num; 784 iov = elem->out_sg; 785 sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head)); 786 if (unlikely(sz != sizeof(head))) { 787 tail.status = VIRTIO_IOMMU_S_DEVERR; 788 goto out; 789 } 790 qemu_rec_mutex_lock(&s->mutex); 791 switch (head.type) { 792 case VIRTIO_IOMMU_T_ATTACH: 793 tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt); 794 break; 795 case VIRTIO_IOMMU_T_DETACH: 796 tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt); 797 break; 798 case VIRTIO_IOMMU_T_MAP: 799 tail.status = virtio_iommu_handle_map(s, iov, iov_cnt); 800 break; 801 case VIRTIO_IOMMU_T_UNMAP: 802 tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt); 803 break; 804 case VIRTIO_IOMMU_T_PROBE: 805 { 806 struct virtio_iommu_req_tail *ptail; 807 808 output_size = s->config.probe_size + sizeof(tail); 809 buf = g_malloc0(output_size); 810 811 ptail = buf + s->config.probe_size; 812 ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf); 813 break; 814 } 815 default: 816 tail.status = VIRTIO_IOMMU_S_UNSUPP; 817 } 818 qemu_rec_mutex_unlock(&s->mutex); 819 820 out: 821 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 822 buf ? buf : &tail, output_size); 823 assert(sz == output_size); 824 825 virtqueue_push(vq, elem, sz); 826 virtio_notify(vdev, vq); 827 g_free(elem); 828 g_free(buf); 829 buf = NULL; 830 } 831 } 832 833 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason, 834 int flags, uint32_t endpoint, 835 uint64_t address) 836 { 837 VirtIODevice *vdev = &viommu->parent_obj; 838 VirtQueue *vq = viommu->event_vq; 839 struct virtio_iommu_fault fault; 840 VirtQueueElement *elem; 841 size_t sz; 842 843 memset(&fault, 0, sizeof(fault)); 844 fault.reason = reason; 845 fault.flags = cpu_to_le32(flags); 846 fault.endpoint = cpu_to_le32(endpoint); 847 fault.address = cpu_to_le64(address); 848 849 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 850 851 if (!elem) { 852 error_report_once( 853 "no buffer available in event queue to report event"); 854 return; 855 } 856 857 if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) { 858 virtio_error(vdev, "error buffer of wrong size"); 859 virtqueue_detach_element(vq, elem, 0); 860 g_free(elem); 861 return; 862 } 863 864 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 865 &fault, sizeof(fault)); 866 assert(sz == sizeof(fault)); 867 868 trace_virtio_iommu_report_fault(reason, flags, endpoint, address); 869 virtqueue_push(vq, elem, sz); 870 virtio_notify(vdev, vq); 871 g_free(elem); 872 873 } 874 875 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr, 876 IOMMUAccessFlags flag, 877 int iommu_idx) 878 { 879 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 880 VirtIOIOMMUInterval interval, *mapping_key; 881 VirtIOIOMMUMapping *mapping_value; 882 VirtIOIOMMU *s = sdev->viommu; 883 bool read_fault, write_fault; 884 VirtIOIOMMUEndpoint *ep; 885 uint32_t sid, flags; 886 bool bypass_allowed; 887 int granule; 888 bool found; 889 GList *l; 890 891 interval.low = addr; 892 interval.high = addr + 1; 893 granule = ctz64(s->config.page_size_mask); 894 895 IOMMUTLBEntry entry = { 896 .target_as = &address_space_memory, 897 .iova = addr, 898 .translated_addr = addr, 899 .addr_mask = BIT_ULL(granule) - 1, 900 .perm = IOMMU_NONE, 901 }; 902 903 bypass_allowed = s->config.bypass; 904 905 sid = virtio_iommu_get_bdf(sdev); 906 907 trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag); 908 qemu_rec_mutex_lock(&s->mutex); 909 910 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 911 912 if (bypass_allowed) 913 assert(ep && ep->domain && !ep->domain->bypass); 914 915 if (!ep) { 916 if (!bypass_allowed) { 917 error_report_once("%s sid=%d is not known!!", __func__, sid); 918 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN, 919 VIRTIO_IOMMU_FAULT_F_ADDRESS, 920 sid, addr); 921 } else { 922 entry.perm = flag; 923 } 924 goto unlock; 925 } 926 927 for (l = sdev->resv_regions; l; l = l->next) { 928 ReservedRegion *reg = l->data; 929 930 if (range_contains(®->range, addr)) { 931 switch (reg->type) { 932 case VIRTIO_IOMMU_RESV_MEM_T_MSI: 933 entry.perm = flag; 934 break; 935 case VIRTIO_IOMMU_RESV_MEM_T_RESERVED: 936 default: 937 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 938 VIRTIO_IOMMU_FAULT_F_ADDRESS, 939 sid, addr); 940 break; 941 } 942 goto unlock; 943 } 944 } 945 946 if (!ep->domain) { 947 if (!bypass_allowed) { 948 error_report_once("%s %02x:%02x.%01x not attached to any domain", 949 __func__, PCI_BUS_NUM(sid), 950 PCI_SLOT(sid), PCI_FUNC(sid)); 951 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN, 952 VIRTIO_IOMMU_FAULT_F_ADDRESS, 953 sid, addr); 954 } else { 955 entry.perm = flag; 956 } 957 goto unlock; 958 } else if (ep->domain->bypass) { 959 entry.perm = flag; 960 goto unlock; 961 } 962 963 found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval), 964 (void **)&mapping_key, 965 (void **)&mapping_value); 966 if (!found) { 967 error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d", 968 __func__, addr, sid); 969 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 970 VIRTIO_IOMMU_FAULT_F_ADDRESS, 971 sid, addr); 972 goto unlock; 973 } 974 975 read_fault = (flag & IOMMU_RO) && 976 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ); 977 write_fault = (flag & IOMMU_WO) && 978 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE); 979 980 flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0; 981 flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0; 982 if (flags) { 983 error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d", 984 __func__, addr, flag, mapping_value->flags); 985 flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS; 986 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 987 flags | VIRTIO_IOMMU_FAULT_F_ADDRESS, 988 sid, addr); 989 goto unlock; 990 } 991 entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr; 992 entry.perm = flag; 993 trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid); 994 995 unlock: 996 qemu_rec_mutex_unlock(&s->mutex); 997 return entry; 998 } 999 1000 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data) 1001 { 1002 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1003 struct virtio_iommu_config *dev_config = &dev->config; 1004 struct virtio_iommu_config *out_config = (void *)config_data; 1005 1006 out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask); 1007 out_config->input_range.start = cpu_to_le64(dev_config->input_range.start); 1008 out_config->input_range.end = cpu_to_le64(dev_config->input_range.end); 1009 out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start); 1010 out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end); 1011 out_config->probe_size = cpu_to_le32(dev_config->probe_size); 1012 out_config->bypass = dev_config->bypass; 1013 1014 trace_virtio_iommu_get_config(dev_config->page_size_mask, 1015 dev_config->input_range.start, 1016 dev_config->input_range.end, 1017 dev_config->domain_range.start, 1018 dev_config->domain_range.end, 1019 dev_config->probe_size, 1020 dev_config->bypass); 1021 } 1022 1023 static void virtio_iommu_set_config(VirtIODevice *vdev, 1024 const uint8_t *config_data) 1025 { 1026 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1027 struct virtio_iommu_config *dev_config = &dev->config; 1028 const struct virtio_iommu_config *in_config = (void *)config_data; 1029 1030 if (in_config->bypass != dev_config->bypass) { 1031 if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) { 1032 virtio_error(vdev, "cannot set config.bypass"); 1033 return; 1034 } else if (in_config->bypass != 0 && in_config->bypass != 1) { 1035 virtio_error(vdev, "invalid config.bypass value '%u'", 1036 in_config->bypass); 1037 return; 1038 } 1039 dev_config->bypass = in_config->bypass; 1040 virtio_iommu_switch_address_space_all(dev); 1041 } 1042 1043 trace_virtio_iommu_set_config(in_config->bypass); 1044 } 1045 1046 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f, 1047 Error **errp) 1048 { 1049 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1050 1051 f |= dev->features; 1052 trace_virtio_iommu_get_features(f); 1053 return f; 1054 } 1055 1056 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 1057 { 1058 guint ua = GPOINTER_TO_UINT(a); 1059 guint ub = GPOINTER_TO_UINT(b); 1060 return (ua > ub) - (ua < ub); 1061 } 1062 1063 static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data) 1064 { 1065 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 1066 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 1067 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 1068 1069 trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high, 1070 mapping->phys_addr); 1071 virtio_iommu_notify_map(mr, interval->low, interval->high, 1072 mapping->phys_addr, mapping->flags); 1073 return false; 1074 } 1075 1076 static void virtio_iommu_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n) 1077 { 1078 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1079 VirtIOIOMMU *s = sdev->viommu; 1080 uint32_t sid; 1081 VirtIOIOMMUEndpoint *ep; 1082 1083 sid = virtio_iommu_get_bdf(sdev); 1084 1085 qemu_rec_mutex_lock(&s->mutex); 1086 1087 if (!s->endpoints) { 1088 goto unlock; 1089 } 1090 1091 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 1092 if (!ep || !ep->domain) { 1093 goto unlock; 1094 } 1095 1096 g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr); 1097 1098 unlock: 1099 qemu_rec_mutex_unlock(&s->mutex); 1100 } 1101 1102 static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, 1103 IOMMUNotifierFlag old, 1104 IOMMUNotifierFlag new, 1105 Error **errp) 1106 { 1107 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { 1108 error_setg(errp, "Virtio-iommu does not support dev-iotlb yet"); 1109 return -EINVAL; 1110 } 1111 1112 if (old == IOMMU_NOTIFIER_NONE) { 1113 trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name); 1114 } else if (new == IOMMU_NOTIFIER_NONE) { 1115 trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name); 1116 } 1117 return 0; 1118 } 1119 1120 /* 1121 * The default mask (TARGET_PAGE_MASK) is the smallest supported guest granule, 1122 * for example 0xfffffffffffff000. When an assigned device has page size 1123 * restrictions due to the hardware IOMMU configuration, apply this restriction 1124 * to the mask. 1125 */ 1126 static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, 1127 uint64_t new_mask, 1128 Error **errp) 1129 { 1130 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1131 VirtIOIOMMU *s = sdev->viommu; 1132 uint64_t cur_mask = s->config.page_size_mask; 1133 1134 trace_virtio_iommu_set_page_size_mask(mr->parent_obj.name, cur_mask, 1135 new_mask); 1136 1137 if ((cur_mask & new_mask) == 0) { 1138 error_setg(errp, "virtio-iommu %s reports a page size mask 0x%"PRIx64 1139 " incompatible with currently supported mask 0x%"PRIx64, 1140 mr->parent_obj.name, new_mask, cur_mask); 1141 return -1; 1142 } 1143 1144 /* 1145 * Once the granule is frozen we can't change the mask anymore. If by 1146 * chance the hotplugged device supports the same granule, we can still 1147 * accept it. 1148 */ 1149 if (s->granule_frozen) { 1150 int cur_granule = ctz64(cur_mask); 1151 1152 if (!(BIT_ULL(cur_granule) & new_mask)) { 1153 error_setg(errp, "virtio-iommu %s does not support frozen granule 0x%llx", 1154 mr->parent_obj.name, BIT_ULL(cur_granule)); 1155 return -1; 1156 } 1157 return 0; 1158 } 1159 1160 s->config.page_size_mask &= new_mask; 1161 return 0; 1162 } 1163 1164 /** 1165 * rebuild_resv_regions: rebuild resv regions with both the 1166 * info of host resv ranges and property set resv ranges 1167 */ 1168 static int rebuild_resv_regions(IOMMUDevice *sdev) 1169 { 1170 GList *l; 1171 int i = 0; 1172 1173 /* free the existing list and rebuild it from scratch */ 1174 g_list_free_full(sdev->resv_regions, g_free); 1175 sdev->resv_regions = NULL; 1176 1177 /* First add host reserved regions if any, all tagged as RESERVED */ 1178 for (l = sdev->host_resv_ranges; l; l = l->next) { 1179 ReservedRegion *reg = g_new0(ReservedRegion, 1); 1180 Range *r = (Range *)l->data; 1181 1182 reg->type = VIRTIO_IOMMU_RESV_MEM_T_RESERVED; 1183 range_set_bounds(®->range, range_lob(r), range_upb(r)); 1184 sdev->resv_regions = resv_region_list_insert(sdev->resv_regions, reg); 1185 trace_virtio_iommu_host_resv_regions(sdev->iommu_mr.parent_obj.name, i, 1186 range_lob(®->range), 1187 range_upb(®->range)); 1188 i++; 1189 } 1190 /* 1191 * then add higher priority reserved regions set by the machine 1192 * through properties 1193 */ 1194 add_prop_resv_regions(sdev); 1195 return 0; 1196 } 1197 1198 /** 1199 * virtio_iommu_set_iova_ranges: Conveys the usable IOVA ranges 1200 * 1201 * The function turns those into reserved ranges. Once some 1202 * reserved ranges have been set, new reserved regions cannot be 1203 * added outside of the original ones. 1204 * 1205 * @mr: IOMMU MR 1206 * @iova_ranges: list of usable IOVA ranges 1207 * @errp: error handle 1208 */ 1209 static int virtio_iommu_set_iova_ranges(IOMMUMemoryRegion *mr, 1210 GList *iova_ranges, 1211 Error **errp) 1212 { 1213 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1214 GList *current_ranges = sdev->host_resv_ranges; 1215 GList *l, *tmp, *new_ranges = NULL; 1216 int ret = -EINVAL; 1217 1218 /* check that each new resv region is included in an existing one */ 1219 if (sdev->host_resv_ranges) { 1220 range_inverse_array(iova_ranges, 1221 &new_ranges, 1222 0, UINT64_MAX); 1223 1224 for (tmp = new_ranges; tmp; tmp = tmp->next) { 1225 Range *newr = (Range *)tmp->data; 1226 bool included = false; 1227 1228 for (l = current_ranges; l; l = l->next) { 1229 Range * r = (Range *)l->data; 1230 1231 if (range_contains_range(r, newr)) { 1232 included = true; 1233 break; 1234 } 1235 } 1236 if (!included) { 1237 goto error; 1238 } 1239 } 1240 /* all new reserved ranges are included in existing ones */ 1241 ret = 0; 1242 goto out; 1243 } 1244 1245 if (sdev->probe_done) { 1246 warn_report("%s: Notified about new host reserved regions after probe", 1247 mr->parent_obj.name); 1248 } 1249 1250 range_inverse_array(iova_ranges, 1251 &sdev->host_resv_ranges, 1252 0, UINT64_MAX); 1253 rebuild_resv_regions(sdev); 1254 1255 return 0; 1256 error: 1257 error_setg(errp, "IOMMU mr=%s Conflicting host reserved ranges set!", 1258 mr->parent_obj.name); 1259 out: 1260 g_list_free_full(new_ranges, g_free); 1261 return ret; 1262 } 1263 1264 static void virtio_iommu_system_reset(void *opaque) 1265 { 1266 VirtIOIOMMU *s = opaque; 1267 1268 trace_virtio_iommu_system_reset(); 1269 1270 /* 1271 * config.bypass is sticky across device reset, but should be restored on 1272 * system reset 1273 */ 1274 s->config.bypass = s->boot_bypass; 1275 virtio_iommu_switch_address_space_all(s); 1276 1277 } 1278 1279 static void virtio_iommu_freeze_granule(Notifier *notifier, void *data) 1280 { 1281 VirtIOIOMMU *s = container_of(notifier, VirtIOIOMMU, machine_done); 1282 int granule; 1283 1284 if (likely(s->config.bypass)) { 1285 /* 1286 * Transient IOMMU MR enable to collect page_size_mask requirements 1287 * through memory_region_iommu_set_page_size_mask() called by 1288 * VFIO region_add() callback 1289 */ 1290 s->config.bypass = false; 1291 virtio_iommu_switch_address_space_all(s); 1292 /* restore default */ 1293 s->config.bypass = true; 1294 virtio_iommu_switch_address_space_all(s); 1295 } 1296 s->granule_frozen = true; 1297 granule = ctz64(s->config.page_size_mask); 1298 trace_virtio_iommu_freeze_granule(BIT_ULL(granule)); 1299 } 1300 1301 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) 1302 { 1303 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1304 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1305 1306 virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config)); 1307 1308 memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num)); 1309 1310 s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, 1311 virtio_iommu_handle_command); 1312 s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL); 1313 1314 /* 1315 * config.bypass is needed to get initial address space early, such as 1316 * in vfio realize 1317 */ 1318 s->config.bypass = s->boot_bypass; 1319 s->config.page_size_mask = qemu_target_page_mask(); 1320 s->config.input_range.end = UINT64_MAX; 1321 s->config.domain_range.end = UINT32_MAX; 1322 s->config.probe_size = VIOMMU_PROBE_SIZE; 1323 1324 virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX); 1325 virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC); 1326 virtio_add_feature(&s->features, VIRTIO_F_VERSION_1); 1327 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE); 1328 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE); 1329 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP); 1330 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO); 1331 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE); 1332 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG); 1333 1334 qemu_rec_mutex_init(&s->mutex); 1335 1336 s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free); 1337 1338 if (s->primary_bus) { 1339 pci_setup_iommu(s->primary_bus, &virtio_iommu_ops, s); 1340 } else { 1341 error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!"); 1342 } 1343 1344 s->machine_done.notify = virtio_iommu_freeze_granule; 1345 qemu_add_machine_init_done_notifier(&s->machine_done); 1346 1347 qemu_register_reset(virtio_iommu_system_reset, s); 1348 } 1349 1350 static void virtio_iommu_device_unrealize(DeviceState *dev) 1351 { 1352 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1353 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1354 1355 qemu_unregister_reset(virtio_iommu_system_reset, s); 1356 qemu_remove_machine_init_done_notifier(&s->machine_done); 1357 1358 g_hash_table_destroy(s->as_by_busptr); 1359 if (s->domains) { 1360 g_tree_destroy(s->domains); 1361 } 1362 if (s->endpoints) { 1363 g_tree_destroy(s->endpoints); 1364 } 1365 1366 qemu_rec_mutex_destroy(&s->mutex); 1367 1368 virtio_delete_queue(s->req_vq); 1369 virtio_delete_queue(s->event_vq); 1370 virtio_cleanup(vdev); 1371 } 1372 1373 static void virtio_iommu_device_reset(VirtIODevice *vdev) 1374 { 1375 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 1376 1377 trace_virtio_iommu_device_reset(); 1378 1379 if (s->domains) { 1380 g_tree_destroy(s->domains); 1381 } 1382 if (s->endpoints) { 1383 g_tree_destroy(s->endpoints); 1384 } 1385 s->domains = g_tree_new_full((GCompareDataFunc)int_cmp, 1386 NULL, NULL, virtio_iommu_put_domain); 1387 s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp, 1388 NULL, NULL, virtio_iommu_put_endpoint); 1389 } 1390 1391 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status) 1392 { 1393 trace_virtio_iommu_device_status(status); 1394 } 1395 1396 static void virtio_iommu_instance_init(Object *obj) 1397 { 1398 } 1399 1400 #define VMSTATE_INTERVAL \ 1401 { \ 1402 .name = "interval", \ 1403 .version_id = 1, \ 1404 .minimum_version_id = 1, \ 1405 .fields = (VMStateField[]) { \ 1406 VMSTATE_UINT64(low, VirtIOIOMMUInterval), \ 1407 VMSTATE_UINT64(high, VirtIOIOMMUInterval), \ 1408 VMSTATE_END_OF_LIST() \ 1409 } \ 1410 } 1411 1412 #define VMSTATE_MAPPING \ 1413 { \ 1414 .name = "mapping", \ 1415 .version_id = 1, \ 1416 .minimum_version_id = 1, \ 1417 .fields = (VMStateField[]) { \ 1418 VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\ 1419 VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \ 1420 VMSTATE_END_OF_LIST() \ 1421 }, \ 1422 } 1423 1424 static const VMStateDescription vmstate_interval_mapping[2] = { 1425 VMSTATE_MAPPING, /* value */ 1426 VMSTATE_INTERVAL /* key */ 1427 }; 1428 1429 static int domain_preload(void *opaque) 1430 { 1431 VirtIOIOMMUDomain *domain = opaque; 1432 1433 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 1434 NULL, g_free, g_free); 1435 return 0; 1436 } 1437 1438 static const VMStateDescription vmstate_endpoint = { 1439 .name = "endpoint", 1440 .version_id = 1, 1441 .minimum_version_id = 1, 1442 .fields = (VMStateField[]) { 1443 VMSTATE_UINT32(id, VirtIOIOMMUEndpoint), 1444 VMSTATE_END_OF_LIST() 1445 } 1446 }; 1447 1448 static const VMStateDescription vmstate_domain = { 1449 .name = "domain", 1450 .version_id = 2, 1451 .minimum_version_id = 2, 1452 .pre_load = domain_preload, 1453 .fields = (VMStateField[]) { 1454 VMSTATE_UINT32(id, VirtIOIOMMUDomain), 1455 VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1, 1456 vmstate_interval_mapping, 1457 VirtIOIOMMUInterval, VirtIOIOMMUMapping), 1458 VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1, 1459 vmstate_endpoint, VirtIOIOMMUEndpoint, next), 1460 VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2), 1461 VMSTATE_END_OF_LIST() 1462 } 1463 }; 1464 1465 static gboolean reconstruct_endpoints(gpointer key, gpointer value, 1466 gpointer data) 1467 { 1468 VirtIOIOMMU *s = (VirtIOIOMMU *)data; 1469 VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value; 1470 VirtIOIOMMUEndpoint *iter; 1471 IOMMUMemoryRegion *mr; 1472 1473 QLIST_FOREACH(iter, &d->endpoint_list, next) { 1474 mr = virtio_iommu_mr(s, iter->id); 1475 assert(mr); 1476 1477 iter->domain = d; 1478 iter->iommu_mr = mr; 1479 g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter); 1480 } 1481 return false; /* continue the domain traversal */ 1482 } 1483 1484 static int iommu_post_load(void *opaque, int version_id) 1485 { 1486 VirtIOIOMMU *s = opaque; 1487 1488 g_tree_foreach(s->domains, reconstruct_endpoints, s); 1489 1490 /* 1491 * Memory regions are dynamically turned on/off depending on 1492 * 'config.bypass' and attached domain type if there is. After 1493 * migration, we need to make sure the memory regions are 1494 * still correct. 1495 */ 1496 virtio_iommu_switch_address_space_all(s); 1497 return 0; 1498 } 1499 1500 static const VMStateDescription vmstate_virtio_iommu_device = { 1501 .name = "virtio-iommu-device", 1502 .minimum_version_id = 2, 1503 .version_id = 2, 1504 .post_load = iommu_post_load, 1505 .fields = (VMStateField[]) { 1506 VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2, 1507 &vmstate_domain, VirtIOIOMMUDomain), 1508 VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2), 1509 VMSTATE_END_OF_LIST() 1510 }, 1511 }; 1512 1513 static const VMStateDescription vmstate_virtio_iommu = { 1514 .name = "virtio-iommu", 1515 .minimum_version_id = 2, 1516 .priority = MIG_PRI_IOMMU, 1517 .version_id = 2, 1518 .fields = (VMStateField[]) { 1519 VMSTATE_VIRTIO_DEVICE, 1520 VMSTATE_END_OF_LIST() 1521 }, 1522 }; 1523 1524 static Property virtio_iommu_properties[] = { 1525 DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, 1526 TYPE_PCI_BUS, PCIBus *), 1527 DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true), 1528 DEFINE_PROP_END_OF_LIST(), 1529 }; 1530 1531 static void virtio_iommu_class_init(ObjectClass *klass, void *data) 1532 { 1533 DeviceClass *dc = DEVICE_CLASS(klass); 1534 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 1535 1536 device_class_set_props(dc, virtio_iommu_properties); 1537 dc->vmsd = &vmstate_virtio_iommu; 1538 1539 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1540 vdc->realize = virtio_iommu_device_realize; 1541 vdc->unrealize = virtio_iommu_device_unrealize; 1542 vdc->reset = virtio_iommu_device_reset; 1543 vdc->get_config = virtio_iommu_get_config; 1544 vdc->set_config = virtio_iommu_set_config; 1545 vdc->get_features = virtio_iommu_get_features; 1546 vdc->set_status = virtio_iommu_set_status; 1547 vdc->vmsd = &vmstate_virtio_iommu_device; 1548 } 1549 1550 static void virtio_iommu_memory_region_class_init(ObjectClass *klass, 1551 void *data) 1552 { 1553 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 1554 1555 imrc->translate = virtio_iommu_translate; 1556 imrc->replay = virtio_iommu_replay; 1557 imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; 1558 imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; 1559 imrc->iommu_set_iova_ranges = virtio_iommu_set_iova_ranges; 1560 } 1561 1562 static const TypeInfo virtio_iommu_info = { 1563 .name = TYPE_VIRTIO_IOMMU, 1564 .parent = TYPE_VIRTIO_DEVICE, 1565 .instance_size = sizeof(VirtIOIOMMU), 1566 .instance_init = virtio_iommu_instance_init, 1567 .class_init = virtio_iommu_class_init, 1568 }; 1569 1570 static const TypeInfo virtio_iommu_memory_region_info = { 1571 .parent = TYPE_IOMMU_MEMORY_REGION, 1572 .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION, 1573 .class_init = virtio_iommu_memory_region_class_init, 1574 }; 1575 1576 static void virtio_register_types(void) 1577 { 1578 type_register_static(&virtio_iommu_info); 1579 type_register_static(&virtio_iommu_memory_region_info); 1580 } 1581 1582 type_init(virtio_register_types) 1583