1 /* 2 * virtio-iommu device 3 * 4 * Copyright (c) 2020 Red Hat, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2 or later, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along with 16 * this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/log.h" 22 #include "qemu/iov.h" 23 #include "exec/target_page.h" 24 #include "hw/qdev-properties.h" 25 #include "hw/virtio/virtio.h" 26 #include "sysemu/kvm.h" 27 #include "sysemu/reset.h" 28 #include "sysemu/sysemu.h" 29 #include "qapi/error.h" 30 #include "qemu/error-report.h" 31 #include "trace.h" 32 33 #include "standard-headers/linux/virtio_ids.h" 34 35 #include "hw/virtio/virtio-bus.h" 36 #include "hw/virtio/virtio-iommu.h" 37 #include "hw/pci/pci_bus.h" 38 #include "hw/pci/pci.h" 39 40 /* Max size */ 41 #define VIOMMU_DEFAULT_QUEUE_SIZE 256 42 #define VIOMMU_PROBE_SIZE 512 43 44 typedef struct VirtIOIOMMUDomain { 45 uint32_t id; 46 bool bypass; 47 GTree *mappings; 48 QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list; 49 } VirtIOIOMMUDomain; 50 51 typedef struct VirtIOIOMMUEndpoint { 52 uint32_t id; 53 VirtIOIOMMUDomain *domain; 54 IOMMUMemoryRegion *iommu_mr; 55 QLIST_ENTRY(VirtIOIOMMUEndpoint) next; 56 } VirtIOIOMMUEndpoint; 57 58 typedef struct VirtIOIOMMUInterval { 59 uint64_t low; 60 uint64_t high; 61 } VirtIOIOMMUInterval; 62 63 typedef struct VirtIOIOMMUMapping { 64 uint64_t phys_addr; 65 uint32_t flags; 66 } VirtIOIOMMUMapping; 67 68 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev) 69 { 70 return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn); 71 } 72 73 static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev) 74 { 75 uint32_t sid; 76 bool bypassed; 77 VirtIOIOMMU *s = sdev->viommu; 78 VirtIOIOMMUEndpoint *ep; 79 80 sid = virtio_iommu_get_bdf(sdev); 81 82 qemu_rec_mutex_lock(&s->mutex); 83 /* need to check bypass before system reset */ 84 if (!s->endpoints) { 85 bypassed = s->config.bypass; 86 goto unlock; 87 } 88 89 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 90 if (!ep || !ep->domain) { 91 bypassed = s->config.bypass; 92 } else { 93 bypassed = ep->domain->bypass; 94 } 95 96 unlock: 97 qemu_rec_mutex_unlock(&s->mutex); 98 return bypassed; 99 } 100 101 /* Return whether the device is using IOMMU translation. */ 102 static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev) 103 { 104 bool use_remapping; 105 106 assert(sdev); 107 108 use_remapping = !virtio_iommu_device_bypassed(sdev); 109 110 trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus), 111 PCI_SLOT(sdev->devfn), 112 PCI_FUNC(sdev->devfn), 113 use_remapping); 114 115 /* Turn off first then on the other */ 116 if (use_remapping) { 117 memory_region_set_enabled(&sdev->bypass_mr, false); 118 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true); 119 } else { 120 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false); 121 memory_region_set_enabled(&sdev->bypass_mr, true); 122 } 123 124 return use_remapping; 125 } 126 127 static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s) 128 { 129 GHashTableIter iter; 130 IOMMUPciBus *iommu_pci_bus; 131 int i; 132 133 g_hash_table_iter_init(&iter, s->as_by_busptr); 134 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 135 for (i = 0; i < PCI_DEVFN_MAX; i++) { 136 if (!iommu_pci_bus->pbdev[i]) { 137 continue; 138 } 139 virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]); 140 } 141 } 142 } 143 144 /** 145 * The bus number is used for lookup when SID based operations occur. 146 * In that case we lazily populate the IOMMUPciBus array from the bus hash 147 * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus 148 * numbers may not be always initialized yet. 149 */ 150 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num) 151 { 152 IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num]; 153 154 if (!iommu_pci_bus) { 155 GHashTableIter iter; 156 157 g_hash_table_iter_init(&iter, s->as_by_busptr); 158 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { 159 if (pci_bus_num(iommu_pci_bus->bus) == bus_num) { 160 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus; 161 return iommu_pci_bus; 162 } 163 } 164 return NULL; 165 } 166 return iommu_pci_bus; 167 } 168 169 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid) 170 { 171 uint8_t bus_n, devfn; 172 IOMMUPciBus *iommu_pci_bus; 173 IOMMUDevice *dev; 174 175 bus_n = PCI_BUS_NUM(sid); 176 iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n); 177 if (iommu_pci_bus) { 178 devfn = sid & (PCI_DEVFN_MAX - 1); 179 dev = iommu_pci_bus->pbdev[devfn]; 180 if (dev) { 181 return &dev->iommu_mr; 182 } 183 } 184 return NULL; 185 } 186 187 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 188 { 189 VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a; 190 VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b; 191 192 if (inta->high < intb->low) { 193 return -1; 194 } else if (intb->high < inta->low) { 195 return 1; 196 } else { 197 return 0; 198 } 199 } 200 201 static void virtio_iommu_notify_map_unmap(IOMMUMemoryRegion *mr, 202 IOMMUTLBEvent *event, 203 hwaddr virt_start, hwaddr virt_end) 204 { 205 uint64_t delta = virt_end - virt_start; 206 207 event->entry.iova = virt_start; 208 event->entry.addr_mask = delta; 209 210 if (delta == UINT64_MAX) { 211 memory_region_notify_iommu(mr, 0, *event); 212 } 213 214 while (virt_start != virt_end + 1) { 215 uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64); 216 217 event->entry.addr_mask = mask; 218 event->entry.iova = virt_start; 219 memory_region_notify_iommu(mr, 0, *event); 220 virt_start += mask + 1; 221 if (event->entry.perm != IOMMU_NONE) { 222 event->entry.translated_addr += mask + 1; 223 } 224 } 225 } 226 227 static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start, 228 hwaddr virt_end, hwaddr paddr, 229 uint32_t flags) 230 { 231 IOMMUTLBEvent event; 232 IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ, 233 flags & VIRTIO_IOMMU_MAP_F_WRITE); 234 235 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) || 236 (flags & VIRTIO_IOMMU_MAP_F_MMIO) || !perm) { 237 return; 238 } 239 240 trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end, 241 paddr, perm); 242 243 event.type = IOMMU_NOTIFIER_MAP; 244 event.entry.target_as = &address_space_memory; 245 event.entry.perm = perm; 246 event.entry.translated_addr = paddr; 247 248 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end); 249 } 250 251 static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start, 252 hwaddr virt_end) 253 { 254 IOMMUTLBEvent event; 255 256 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) { 257 return; 258 } 259 260 trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end); 261 262 event.type = IOMMU_NOTIFIER_UNMAP; 263 event.entry.target_as = &address_space_memory; 264 event.entry.perm = IOMMU_NONE; 265 event.entry.translated_addr = 0; 266 267 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end); 268 } 269 270 static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value, 271 gpointer data) 272 { 273 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 274 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 275 276 virtio_iommu_notify_unmap(mr, interval->low, interval->high); 277 278 return false; 279 } 280 281 static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value, 282 gpointer data) 283 { 284 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 285 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 286 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 287 288 virtio_iommu_notify_map(mr, interval->low, interval->high, 289 mapping->phys_addr, mapping->flags); 290 291 return false; 292 } 293 294 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep) 295 { 296 VirtIOIOMMUDomain *domain = ep->domain; 297 IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 298 299 if (!ep->domain) { 300 return; 301 } 302 g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb, 303 ep->iommu_mr); 304 QLIST_REMOVE(ep, next); 305 ep->domain = NULL; 306 virtio_iommu_switch_address_space(sdev); 307 } 308 309 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s, 310 uint32_t ep_id) 311 { 312 VirtIOIOMMUEndpoint *ep; 313 IOMMUMemoryRegion *mr; 314 315 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 316 if (ep) { 317 return ep; 318 } 319 mr = virtio_iommu_mr(s, ep_id); 320 if (!mr) { 321 return NULL; 322 } 323 ep = g_malloc0(sizeof(*ep)); 324 ep->id = ep_id; 325 ep->iommu_mr = mr; 326 trace_virtio_iommu_get_endpoint(ep_id); 327 g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep); 328 return ep; 329 } 330 331 static void virtio_iommu_put_endpoint(gpointer data) 332 { 333 VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data; 334 335 if (ep->domain) { 336 virtio_iommu_detach_endpoint_from_domain(ep); 337 } 338 339 trace_virtio_iommu_put_endpoint(ep->id); 340 g_free(ep); 341 } 342 343 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s, 344 uint32_t domain_id, 345 bool bypass) 346 { 347 VirtIOIOMMUDomain *domain; 348 349 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 350 if (domain) { 351 if (domain->bypass != bypass) { 352 return NULL; 353 } 354 return domain; 355 } 356 domain = g_malloc0(sizeof(*domain)); 357 domain->id = domain_id; 358 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 359 NULL, (GDestroyNotify)g_free, 360 (GDestroyNotify)g_free); 361 domain->bypass = bypass; 362 g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain); 363 QLIST_INIT(&domain->endpoint_list); 364 trace_virtio_iommu_get_domain(domain_id); 365 return domain; 366 } 367 368 static void virtio_iommu_put_domain(gpointer data) 369 { 370 VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data; 371 VirtIOIOMMUEndpoint *iter, *tmp; 372 373 QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) { 374 virtio_iommu_detach_endpoint_from_domain(iter); 375 } 376 g_tree_destroy(domain->mappings); 377 trace_virtio_iommu_put_domain(domain->id); 378 g_free(domain); 379 } 380 381 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque, 382 int devfn) 383 { 384 VirtIOIOMMU *s = opaque; 385 IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus); 386 static uint32_t mr_index; 387 IOMMUDevice *sdev; 388 389 if (!sbus) { 390 sbus = g_malloc0(sizeof(IOMMUPciBus) + 391 sizeof(IOMMUDevice *) * PCI_DEVFN_MAX); 392 sbus->bus = bus; 393 g_hash_table_insert(s->as_by_busptr, bus, sbus); 394 } 395 396 sdev = sbus->pbdev[devfn]; 397 if (!sdev) { 398 char *name = g_strdup_printf("%s-%d-%d", 399 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 400 mr_index++, devfn); 401 sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1); 402 403 sdev->viommu = s; 404 sdev->bus = bus; 405 sdev->devfn = devfn; 406 407 trace_virtio_iommu_init_iommu_mr(name); 408 409 memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX); 410 address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU); 411 412 /* 413 * Build the IOMMU disabled container with aliases to the 414 * shared MRs. Note that aliasing to a shared memory region 415 * could help the memory API to detect same FlatViews so we 416 * can have devices to share the same FlatView when in bypass 417 * mode. (either by not configuring virtio-iommu driver or with 418 * "iommu=pt"). It will greatly reduce the total number of 419 * FlatViews of the system hence VM runs faster. 420 */ 421 memory_region_init_alias(&sdev->bypass_mr, OBJECT(s), 422 "system", get_system_memory(), 0, 423 memory_region_size(get_system_memory())); 424 425 memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr), 426 TYPE_VIRTIO_IOMMU_MEMORY_REGION, 427 OBJECT(s), name, 428 UINT64_MAX); 429 430 /* 431 * Hook both the containers under the root container, we 432 * switch between iommu & bypass MRs by enable/disable 433 * corresponding sub-containers 434 */ 435 memory_region_add_subregion_overlap(&sdev->root, 0, 436 MEMORY_REGION(&sdev->iommu_mr), 437 0); 438 memory_region_add_subregion_overlap(&sdev->root, 0, 439 &sdev->bypass_mr, 0); 440 441 virtio_iommu_switch_address_space(sdev); 442 g_free(name); 443 } 444 return &sdev->as; 445 } 446 447 static int virtio_iommu_attach(VirtIOIOMMU *s, 448 struct virtio_iommu_req_attach *req) 449 { 450 uint32_t domain_id = le32_to_cpu(req->domain); 451 uint32_t ep_id = le32_to_cpu(req->endpoint); 452 uint32_t flags = le32_to_cpu(req->flags); 453 VirtIOIOMMUDomain *domain; 454 VirtIOIOMMUEndpoint *ep; 455 IOMMUDevice *sdev; 456 457 trace_virtio_iommu_attach(domain_id, ep_id); 458 459 if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) { 460 return VIRTIO_IOMMU_S_INVAL; 461 } 462 463 ep = virtio_iommu_get_endpoint(s, ep_id); 464 if (!ep) { 465 return VIRTIO_IOMMU_S_NOENT; 466 } 467 468 if (ep->domain) { 469 VirtIOIOMMUDomain *previous_domain = ep->domain; 470 /* 471 * the device is already attached to a domain, 472 * detach it first 473 */ 474 virtio_iommu_detach_endpoint_from_domain(ep); 475 if (QLIST_EMPTY(&previous_domain->endpoint_list)) { 476 g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id)); 477 } 478 } 479 480 domain = virtio_iommu_get_domain(s, domain_id, 481 flags & VIRTIO_IOMMU_ATTACH_F_BYPASS); 482 if (!domain) { 483 /* Incompatible bypass flag */ 484 return VIRTIO_IOMMU_S_INVAL; 485 } 486 QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next); 487 488 ep->domain = domain; 489 sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); 490 virtio_iommu_switch_address_space(sdev); 491 492 /* Replay domain mappings on the associated memory region */ 493 g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb, 494 ep->iommu_mr); 495 496 return VIRTIO_IOMMU_S_OK; 497 } 498 499 static int virtio_iommu_detach(VirtIOIOMMU *s, 500 struct virtio_iommu_req_detach *req) 501 { 502 uint32_t domain_id = le32_to_cpu(req->domain); 503 uint32_t ep_id = le32_to_cpu(req->endpoint); 504 VirtIOIOMMUDomain *domain; 505 VirtIOIOMMUEndpoint *ep; 506 507 trace_virtio_iommu_detach(domain_id, ep_id); 508 509 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); 510 if (!ep) { 511 return VIRTIO_IOMMU_S_NOENT; 512 } 513 514 domain = ep->domain; 515 516 if (!domain || domain->id != domain_id) { 517 return VIRTIO_IOMMU_S_INVAL; 518 } 519 520 virtio_iommu_detach_endpoint_from_domain(ep); 521 522 if (QLIST_EMPTY(&domain->endpoint_list)) { 523 g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id)); 524 } 525 return VIRTIO_IOMMU_S_OK; 526 } 527 528 static int virtio_iommu_map(VirtIOIOMMU *s, 529 struct virtio_iommu_req_map *req) 530 { 531 uint32_t domain_id = le32_to_cpu(req->domain); 532 uint64_t phys_start = le64_to_cpu(req->phys_start); 533 uint64_t virt_start = le64_to_cpu(req->virt_start); 534 uint64_t virt_end = le64_to_cpu(req->virt_end); 535 uint32_t flags = le32_to_cpu(req->flags); 536 VirtIOIOMMUDomain *domain; 537 VirtIOIOMMUInterval *interval; 538 VirtIOIOMMUMapping *mapping; 539 VirtIOIOMMUEndpoint *ep; 540 541 if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) { 542 return VIRTIO_IOMMU_S_INVAL; 543 } 544 545 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 546 if (!domain) { 547 return VIRTIO_IOMMU_S_NOENT; 548 } 549 550 if (domain->bypass) { 551 return VIRTIO_IOMMU_S_INVAL; 552 } 553 554 interval = g_malloc0(sizeof(*interval)); 555 556 interval->low = virt_start; 557 interval->high = virt_end; 558 559 mapping = g_tree_lookup(domain->mappings, (gpointer)interval); 560 if (mapping) { 561 g_free(interval); 562 return VIRTIO_IOMMU_S_INVAL; 563 } 564 565 trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags); 566 567 mapping = g_malloc0(sizeof(*mapping)); 568 mapping->phys_addr = phys_start; 569 mapping->flags = flags; 570 571 g_tree_insert(domain->mappings, interval, mapping); 572 573 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 574 virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start, 575 flags); 576 } 577 578 return VIRTIO_IOMMU_S_OK; 579 } 580 581 static int virtio_iommu_unmap(VirtIOIOMMU *s, 582 struct virtio_iommu_req_unmap *req) 583 { 584 uint32_t domain_id = le32_to_cpu(req->domain); 585 uint64_t virt_start = le64_to_cpu(req->virt_start); 586 uint64_t virt_end = le64_to_cpu(req->virt_end); 587 VirtIOIOMMUMapping *iter_val; 588 VirtIOIOMMUInterval interval, *iter_key; 589 VirtIOIOMMUDomain *domain; 590 VirtIOIOMMUEndpoint *ep; 591 int ret = VIRTIO_IOMMU_S_OK; 592 593 trace_virtio_iommu_unmap(domain_id, virt_start, virt_end); 594 595 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); 596 if (!domain) { 597 return VIRTIO_IOMMU_S_NOENT; 598 } 599 600 if (domain->bypass) { 601 return VIRTIO_IOMMU_S_INVAL; 602 } 603 604 interval.low = virt_start; 605 interval.high = virt_end; 606 607 while (g_tree_lookup_extended(domain->mappings, &interval, 608 (void **)&iter_key, (void**)&iter_val)) { 609 uint64_t current_low = iter_key->low; 610 uint64_t current_high = iter_key->high; 611 612 if (interval.low <= current_low && interval.high >= current_high) { 613 QLIST_FOREACH(ep, &domain->endpoint_list, next) { 614 virtio_iommu_notify_unmap(ep->iommu_mr, current_low, 615 current_high); 616 } 617 g_tree_remove(domain->mappings, iter_key); 618 trace_virtio_iommu_unmap_done(domain_id, current_low, current_high); 619 } else { 620 ret = VIRTIO_IOMMU_S_RANGE; 621 break; 622 } 623 } 624 return ret; 625 } 626 627 static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep, 628 uint8_t *buf, size_t free) 629 { 630 struct virtio_iommu_probe_resv_mem prop = {}; 631 size_t size = sizeof(prop), length = size - sizeof(prop.head), total; 632 int i; 633 634 total = size * s->nr_prop_resv_regions; 635 636 if (total > free) { 637 return -ENOSPC; 638 } 639 640 for (i = 0; i < s->nr_prop_resv_regions; i++) { 641 unsigned subtype = s->prop_resv_regions[i].type; 642 Range *range = &s->prop_resv_regions[i].range; 643 644 assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED || 645 subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI); 646 prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM); 647 prop.head.length = cpu_to_le16(length); 648 prop.subtype = subtype; 649 prop.start = cpu_to_le64(range_lob(range)); 650 prop.end = cpu_to_le64(range_upb(range)); 651 652 memcpy(buf, &prop, size); 653 654 trace_virtio_iommu_fill_resv_property(ep, prop.subtype, 655 prop.start, prop.end); 656 buf += size; 657 } 658 return total; 659 } 660 661 /** 662 * virtio_iommu_probe - Fill the probe request buffer with 663 * the properties the device is able to return 664 */ 665 static int virtio_iommu_probe(VirtIOIOMMU *s, 666 struct virtio_iommu_req_probe *req, 667 uint8_t *buf) 668 { 669 uint32_t ep_id = le32_to_cpu(req->endpoint); 670 size_t free = VIOMMU_PROBE_SIZE; 671 ssize_t count; 672 673 if (!virtio_iommu_mr(s, ep_id)) { 674 return VIRTIO_IOMMU_S_NOENT; 675 } 676 677 count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free); 678 if (count < 0) { 679 return VIRTIO_IOMMU_S_INVAL; 680 } 681 buf += count; 682 free -= count; 683 684 return VIRTIO_IOMMU_S_OK; 685 } 686 687 static int virtio_iommu_iov_to_req(struct iovec *iov, 688 unsigned int iov_cnt, 689 void *req, size_t payload_sz) 690 { 691 size_t sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz); 692 693 if (unlikely(sz != payload_sz)) { 694 return VIRTIO_IOMMU_S_INVAL; 695 } 696 return 0; 697 } 698 699 #define virtio_iommu_handle_req(__req) \ 700 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \ 701 struct iovec *iov, \ 702 unsigned int iov_cnt) \ 703 { \ 704 struct virtio_iommu_req_ ## __req req; \ 705 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, \ 706 sizeof(req) - sizeof(struct virtio_iommu_req_tail));\ 707 \ 708 return ret ? ret : virtio_iommu_ ## __req(s, &req); \ 709 } 710 711 virtio_iommu_handle_req(attach) 712 virtio_iommu_handle_req(detach) 713 virtio_iommu_handle_req(map) 714 virtio_iommu_handle_req(unmap) 715 716 static int virtio_iommu_handle_probe(VirtIOIOMMU *s, 717 struct iovec *iov, 718 unsigned int iov_cnt, 719 uint8_t *buf) 720 { 721 struct virtio_iommu_req_probe req; 722 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); 723 724 return ret ? ret : virtio_iommu_probe(s, &req, buf); 725 } 726 727 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq) 728 { 729 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 730 struct virtio_iommu_req_head head; 731 struct virtio_iommu_req_tail tail = {}; 732 VirtQueueElement *elem; 733 unsigned int iov_cnt; 734 struct iovec *iov; 735 void *buf = NULL; 736 size_t sz; 737 738 for (;;) { 739 size_t output_size = sizeof(tail); 740 741 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 742 if (!elem) { 743 return; 744 } 745 746 if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) || 747 iov_size(elem->out_sg, elem->out_num) < sizeof(head)) { 748 virtio_error(vdev, "virtio-iommu bad head/tail size"); 749 virtqueue_detach_element(vq, elem, 0); 750 g_free(elem); 751 break; 752 } 753 754 iov_cnt = elem->out_num; 755 iov = elem->out_sg; 756 sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head)); 757 if (unlikely(sz != sizeof(head))) { 758 tail.status = VIRTIO_IOMMU_S_DEVERR; 759 goto out; 760 } 761 qemu_rec_mutex_lock(&s->mutex); 762 switch (head.type) { 763 case VIRTIO_IOMMU_T_ATTACH: 764 tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt); 765 break; 766 case VIRTIO_IOMMU_T_DETACH: 767 tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt); 768 break; 769 case VIRTIO_IOMMU_T_MAP: 770 tail.status = virtio_iommu_handle_map(s, iov, iov_cnt); 771 break; 772 case VIRTIO_IOMMU_T_UNMAP: 773 tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt); 774 break; 775 case VIRTIO_IOMMU_T_PROBE: 776 { 777 struct virtio_iommu_req_tail *ptail; 778 779 output_size = s->config.probe_size + sizeof(tail); 780 buf = g_malloc0(output_size); 781 782 ptail = buf + s->config.probe_size; 783 ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf); 784 break; 785 } 786 default: 787 tail.status = VIRTIO_IOMMU_S_UNSUPP; 788 } 789 qemu_rec_mutex_unlock(&s->mutex); 790 791 out: 792 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 793 buf ? buf : &tail, output_size); 794 assert(sz == output_size); 795 796 virtqueue_push(vq, elem, sz); 797 virtio_notify(vdev, vq); 798 g_free(elem); 799 g_free(buf); 800 buf = NULL; 801 } 802 } 803 804 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason, 805 int flags, uint32_t endpoint, 806 uint64_t address) 807 { 808 VirtIODevice *vdev = &viommu->parent_obj; 809 VirtQueue *vq = viommu->event_vq; 810 struct virtio_iommu_fault fault; 811 VirtQueueElement *elem; 812 size_t sz; 813 814 memset(&fault, 0, sizeof(fault)); 815 fault.reason = reason; 816 fault.flags = cpu_to_le32(flags); 817 fault.endpoint = cpu_to_le32(endpoint); 818 fault.address = cpu_to_le64(address); 819 820 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 821 822 if (!elem) { 823 error_report_once( 824 "no buffer available in event queue to report event"); 825 return; 826 } 827 828 if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) { 829 virtio_error(vdev, "error buffer of wrong size"); 830 virtqueue_detach_element(vq, elem, 0); 831 g_free(elem); 832 return; 833 } 834 835 sz = iov_from_buf(elem->in_sg, elem->in_num, 0, 836 &fault, sizeof(fault)); 837 assert(sz == sizeof(fault)); 838 839 trace_virtio_iommu_report_fault(reason, flags, endpoint, address); 840 virtqueue_push(vq, elem, sz); 841 virtio_notify(vdev, vq); 842 g_free(elem); 843 844 } 845 846 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr, 847 IOMMUAccessFlags flag, 848 int iommu_idx) 849 { 850 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 851 VirtIOIOMMUInterval interval, *mapping_key; 852 VirtIOIOMMUMapping *mapping_value; 853 VirtIOIOMMU *s = sdev->viommu; 854 bool read_fault, write_fault; 855 VirtIOIOMMUEndpoint *ep; 856 uint32_t sid, flags; 857 bool bypass_allowed; 858 int granule; 859 bool found; 860 int i; 861 862 interval.low = addr; 863 interval.high = addr + 1; 864 granule = ctz64(s->config.page_size_mask); 865 866 IOMMUTLBEntry entry = { 867 .target_as = &address_space_memory, 868 .iova = addr, 869 .translated_addr = addr, 870 .addr_mask = BIT_ULL(granule) - 1, 871 .perm = IOMMU_NONE, 872 }; 873 874 bypass_allowed = s->config.bypass; 875 876 sid = virtio_iommu_get_bdf(sdev); 877 878 trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag); 879 qemu_rec_mutex_lock(&s->mutex); 880 881 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 882 883 if (bypass_allowed) 884 assert(ep && ep->domain && !ep->domain->bypass); 885 886 if (!ep) { 887 if (!bypass_allowed) { 888 error_report_once("%s sid=%d is not known!!", __func__, sid); 889 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN, 890 VIRTIO_IOMMU_FAULT_F_ADDRESS, 891 sid, addr); 892 } else { 893 entry.perm = flag; 894 } 895 goto unlock; 896 } 897 898 for (i = 0; i < s->nr_prop_resv_regions; i++) { 899 ReservedRegion *reg = &s->prop_resv_regions[i]; 900 901 if (range_contains(®->range, addr)) { 902 switch (reg->type) { 903 case VIRTIO_IOMMU_RESV_MEM_T_MSI: 904 entry.perm = flag; 905 break; 906 case VIRTIO_IOMMU_RESV_MEM_T_RESERVED: 907 default: 908 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 909 VIRTIO_IOMMU_FAULT_F_ADDRESS, 910 sid, addr); 911 break; 912 } 913 goto unlock; 914 } 915 } 916 917 if (!ep->domain) { 918 if (!bypass_allowed) { 919 error_report_once("%s %02x:%02x.%01x not attached to any domain", 920 __func__, PCI_BUS_NUM(sid), 921 PCI_SLOT(sid), PCI_FUNC(sid)); 922 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN, 923 VIRTIO_IOMMU_FAULT_F_ADDRESS, 924 sid, addr); 925 } else { 926 entry.perm = flag; 927 } 928 goto unlock; 929 } else if (ep->domain->bypass) { 930 entry.perm = flag; 931 goto unlock; 932 } 933 934 found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval), 935 (void **)&mapping_key, 936 (void **)&mapping_value); 937 if (!found) { 938 error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d", 939 __func__, addr, sid); 940 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 941 VIRTIO_IOMMU_FAULT_F_ADDRESS, 942 sid, addr); 943 goto unlock; 944 } 945 946 read_fault = (flag & IOMMU_RO) && 947 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ); 948 write_fault = (flag & IOMMU_WO) && 949 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE); 950 951 flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0; 952 flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0; 953 if (flags) { 954 error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d", 955 __func__, addr, flag, mapping_value->flags); 956 flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS; 957 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, 958 flags | VIRTIO_IOMMU_FAULT_F_ADDRESS, 959 sid, addr); 960 goto unlock; 961 } 962 entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr; 963 entry.perm = flag; 964 trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid); 965 966 unlock: 967 qemu_rec_mutex_unlock(&s->mutex); 968 return entry; 969 } 970 971 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data) 972 { 973 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 974 struct virtio_iommu_config *dev_config = &dev->config; 975 struct virtio_iommu_config *out_config = (void *)config_data; 976 977 out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask); 978 out_config->input_range.start = cpu_to_le64(dev_config->input_range.start); 979 out_config->input_range.end = cpu_to_le64(dev_config->input_range.end); 980 out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start); 981 out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end); 982 out_config->probe_size = cpu_to_le32(dev_config->probe_size); 983 out_config->bypass = dev_config->bypass; 984 985 trace_virtio_iommu_get_config(dev_config->page_size_mask, 986 dev_config->input_range.start, 987 dev_config->input_range.end, 988 dev_config->domain_range.start, 989 dev_config->domain_range.end, 990 dev_config->probe_size, 991 dev_config->bypass); 992 } 993 994 static void virtio_iommu_set_config(VirtIODevice *vdev, 995 const uint8_t *config_data) 996 { 997 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 998 struct virtio_iommu_config *dev_config = &dev->config; 999 const struct virtio_iommu_config *in_config = (void *)config_data; 1000 1001 if (in_config->bypass != dev_config->bypass) { 1002 if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) { 1003 virtio_error(vdev, "cannot set config.bypass"); 1004 return; 1005 } else if (in_config->bypass != 0 && in_config->bypass != 1) { 1006 virtio_error(vdev, "invalid config.bypass value '%u'", 1007 in_config->bypass); 1008 return; 1009 } 1010 dev_config->bypass = in_config->bypass; 1011 virtio_iommu_switch_address_space_all(dev); 1012 } 1013 1014 trace_virtio_iommu_set_config(in_config->bypass); 1015 } 1016 1017 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f, 1018 Error **errp) 1019 { 1020 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); 1021 1022 f |= dev->features; 1023 trace_virtio_iommu_get_features(f); 1024 return f; 1025 } 1026 1027 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data) 1028 { 1029 guint ua = GPOINTER_TO_UINT(a); 1030 guint ub = GPOINTER_TO_UINT(b); 1031 return (ua > ub) - (ua < ub); 1032 } 1033 1034 static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data) 1035 { 1036 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; 1037 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; 1038 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; 1039 1040 trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high, 1041 mapping->phys_addr); 1042 virtio_iommu_notify_map(mr, interval->low, interval->high, 1043 mapping->phys_addr, mapping->flags); 1044 return false; 1045 } 1046 1047 static void virtio_iommu_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n) 1048 { 1049 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1050 VirtIOIOMMU *s = sdev->viommu; 1051 uint32_t sid; 1052 VirtIOIOMMUEndpoint *ep; 1053 1054 sid = virtio_iommu_get_bdf(sdev); 1055 1056 qemu_rec_mutex_lock(&s->mutex); 1057 1058 if (!s->endpoints) { 1059 goto unlock; 1060 } 1061 1062 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); 1063 if (!ep || !ep->domain) { 1064 goto unlock; 1065 } 1066 1067 g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr); 1068 1069 unlock: 1070 qemu_rec_mutex_unlock(&s->mutex); 1071 } 1072 1073 static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, 1074 IOMMUNotifierFlag old, 1075 IOMMUNotifierFlag new, 1076 Error **errp) 1077 { 1078 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { 1079 error_setg(errp, "Virtio-iommu does not support dev-iotlb yet"); 1080 return -EINVAL; 1081 } 1082 1083 if (old == IOMMU_NOTIFIER_NONE) { 1084 trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name); 1085 } else if (new == IOMMU_NOTIFIER_NONE) { 1086 trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name); 1087 } 1088 return 0; 1089 } 1090 1091 /* 1092 * The default mask (TARGET_PAGE_MASK) is the smallest supported guest granule, 1093 * for example 0xfffffffffffff000. When an assigned device has page size 1094 * restrictions due to the hardware IOMMU configuration, apply this restriction 1095 * to the mask. 1096 */ 1097 static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, 1098 uint64_t new_mask, 1099 Error **errp) 1100 { 1101 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); 1102 VirtIOIOMMU *s = sdev->viommu; 1103 uint64_t cur_mask = s->config.page_size_mask; 1104 1105 trace_virtio_iommu_set_page_size_mask(mr->parent_obj.name, cur_mask, 1106 new_mask); 1107 1108 if ((cur_mask & new_mask) == 0) { 1109 error_setg(errp, "virtio-iommu %s reports a page size mask 0x%"PRIx64 1110 " incompatible with currently supported mask 0x%"PRIx64, 1111 mr->parent_obj.name, new_mask, cur_mask); 1112 return -1; 1113 } 1114 1115 /* 1116 * Once the granule is frozen we can't change the mask anymore. If by 1117 * chance the hotplugged device supports the same granule, we can still 1118 * accept it. 1119 */ 1120 if (s->granule_frozen) { 1121 int cur_granule = ctz64(cur_mask); 1122 1123 if (!(BIT_ULL(cur_granule) & new_mask)) { 1124 error_setg(errp, "virtio-iommu %s does not support frozen granule 0x%llx", 1125 mr->parent_obj.name, BIT_ULL(cur_granule)); 1126 return -1; 1127 } 1128 return 0; 1129 } 1130 1131 s->config.page_size_mask &= new_mask; 1132 return 0; 1133 } 1134 1135 static void virtio_iommu_system_reset(void *opaque) 1136 { 1137 VirtIOIOMMU *s = opaque; 1138 1139 trace_virtio_iommu_system_reset(); 1140 1141 /* 1142 * config.bypass is sticky across device reset, but should be restored on 1143 * system reset 1144 */ 1145 s->config.bypass = s->boot_bypass; 1146 virtio_iommu_switch_address_space_all(s); 1147 1148 } 1149 1150 static void virtio_iommu_freeze_granule(Notifier *notifier, void *data) 1151 { 1152 VirtIOIOMMU *s = container_of(notifier, VirtIOIOMMU, machine_done); 1153 int granule; 1154 1155 if (likely(s->config.bypass)) { 1156 /* 1157 * Transient IOMMU MR enable to collect page_size_mask requirements 1158 * through memory_region_iommu_set_page_size_mask() called by 1159 * VFIO region_add() callback 1160 */ 1161 s->config.bypass = false; 1162 virtio_iommu_switch_address_space_all(s); 1163 /* restore default */ 1164 s->config.bypass = true; 1165 virtio_iommu_switch_address_space_all(s); 1166 } 1167 s->granule_frozen = true; 1168 granule = ctz64(s->config.page_size_mask); 1169 trace_virtio_iommu_freeze_granule(BIT_ULL(granule)); 1170 } 1171 1172 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) 1173 { 1174 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1175 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1176 1177 virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config)); 1178 1179 memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num)); 1180 1181 s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, 1182 virtio_iommu_handle_command); 1183 s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL); 1184 1185 /* 1186 * config.bypass is needed to get initial address space early, such as 1187 * in vfio realize 1188 */ 1189 s->config.bypass = s->boot_bypass; 1190 s->config.page_size_mask = qemu_target_page_mask(); 1191 s->config.input_range.end = UINT64_MAX; 1192 s->config.domain_range.end = UINT32_MAX; 1193 s->config.probe_size = VIOMMU_PROBE_SIZE; 1194 1195 virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX); 1196 virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC); 1197 virtio_add_feature(&s->features, VIRTIO_F_VERSION_1); 1198 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE); 1199 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE); 1200 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP); 1201 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO); 1202 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE); 1203 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG); 1204 1205 qemu_rec_mutex_init(&s->mutex); 1206 1207 s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free); 1208 1209 if (s->primary_bus) { 1210 pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s); 1211 } else { 1212 error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!"); 1213 } 1214 1215 s->machine_done.notify = virtio_iommu_freeze_granule; 1216 qemu_add_machine_init_done_notifier(&s->machine_done); 1217 1218 qemu_register_reset(virtio_iommu_system_reset, s); 1219 } 1220 1221 static void virtio_iommu_device_unrealize(DeviceState *dev) 1222 { 1223 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 1224 VirtIOIOMMU *s = VIRTIO_IOMMU(dev); 1225 1226 qemu_unregister_reset(virtio_iommu_system_reset, s); 1227 qemu_remove_machine_init_done_notifier(&s->machine_done); 1228 1229 g_hash_table_destroy(s->as_by_busptr); 1230 if (s->domains) { 1231 g_tree_destroy(s->domains); 1232 } 1233 if (s->endpoints) { 1234 g_tree_destroy(s->endpoints); 1235 } 1236 1237 qemu_rec_mutex_destroy(&s->mutex); 1238 1239 virtio_delete_queue(s->req_vq); 1240 virtio_delete_queue(s->event_vq); 1241 virtio_cleanup(vdev); 1242 } 1243 1244 static void virtio_iommu_device_reset(VirtIODevice *vdev) 1245 { 1246 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); 1247 1248 trace_virtio_iommu_device_reset(); 1249 1250 if (s->domains) { 1251 g_tree_destroy(s->domains); 1252 } 1253 if (s->endpoints) { 1254 g_tree_destroy(s->endpoints); 1255 } 1256 s->domains = g_tree_new_full((GCompareDataFunc)int_cmp, 1257 NULL, NULL, virtio_iommu_put_domain); 1258 s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp, 1259 NULL, NULL, virtio_iommu_put_endpoint); 1260 } 1261 1262 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status) 1263 { 1264 trace_virtio_iommu_device_status(status); 1265 } 1266 1267 static void virtio_iommu_instance_init(Object *obj) 1268 { 1269 } 1270 1271 #define VMSTATE_INTERVAL \ 1272 { \ 1273 .name = "interval", \ 1274 .version_id = 1, \ 1275 .minimum_version_id = 1, \ 1276 .fields = (VMStateField[]) { \ 1277 VMSTATE_UINT64(low, VirtIOIOMMUInterval), \ 1278 VMSTATE_UINT64(high, VirtIOIOMMUInterval), \ 1279 VMSTATE_END_OF_LIST() \ 1280 } \ 1281 } 1282 1283 #define VMSTATE_MAPPING \ 1284 { \ 1285 .name = "mapping", \ 1286 .version_id = 1, \ 1287 .minimum_version_id = 1, \ 1288 .fields = (VMStateField[]) { \ 1289 VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\ 1290 VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \ 1291 VMSTATE_END_OF_LIST() \ 1292 }, \ 1293 } 1294 1295 static const VMStateDescription vmstate_interval_mapping[2] = { 1296 VMSTATE_MAPPING, /* value */ 1297 VMSTATE_INTERVAL /* key */ 1298 }; 1299 1300 static int domain_preload(void *opaque) 1301 { 1302 VirtIOIOMMUDomain *domain = opaque; 1303 1304 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, 1305 NULL, g_free, g_free); 1306 return 0; 1307 } 1308 1309 static const VMStateDescription vmstate_endpoint = { 1310 .name = "endpoint", 1311 .version_id = 1, 1312 .minimum_version_id = 1, 1313 .fields = (VMStateField[]) { 1314 VMSTATE_UINT32(id, VirtIOIOMMUEndpoint), 1315 VMSTATE_END_OF_LIST() 1316 } 1317 }; 1318 1319 static const VMStateDescription vmstate_domain = { 1320 .name = "domain", 1321 .version_id = 2, 1322 .minimum_version_id = 2, 1323 .pre_load = domain_preload, 1324 .fields = (VMStateField[]) { 1325 VMSTATE_UINT32(id, VirtIOIOMMUDomain), 1326 VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1, 1327 vmstate_interval_mapping, 1328 VirtIOIOMMUInterval, VirtIOIOMMUMapping), 1329 VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1, 1330 vmstate_endpoint, VirtIOIOMMUEndpoint, next), 1331 VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2), 1332 VMSTATE_END_OF_LIST() 1333 } 1334 }; 1335 1336 static gboolean reconstruct_endpoints(gpointer key, gpointer value, 1337 gpointer data) 1338 { 1339 VirtIOIOMMU *s = (VirtIOIOMMU *)data; 1340 VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value; 1341 VirtIOIOMMUEndpoint *iter; 1342 IOMMUMemoryRegion *mr; 1343 1344 QLIST_FOREACH(iter, &d->endpoint_list, next) { 1345 mr = virtio_iommu_mr(s, iter->id); 1346 assert(mr); 1347 1348 iter->domain = d; 1349 iter->iommu_mr = mr; 1350 g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter); 1351 } 1352 return false; /* continue the domain traversal */ 1353 } 1354 1355 static int iommu_post_load(void *opaque, int version_id) 1356 { 1357 VirtIOIOMMU *s = opaque; 1358 1359 g_tree_foreach(s->domains, reconstruct_endpoints, s); 1360 1361 /* 1362 * Memory regions are dynamically turned on/off depending on 1363 * 'config.bypass' and attached domain type if there is. After 1364 * migration, we need to make sure the memory regions are 1365 * still correct. 1366 */ 1367 virtio_iommu_switch_address_space_all(s); 1368 return 0; 1369 } 1370 1371 static const VMStateDescription vmstate_virtio_iommu_device = { 1372 .name = "virtio-iommu-device", 1373 .minimum_version_id = 2, 1374 .version_id = 2, 1375 .post_load = iommu_post_load, 1376 .fields = (VMStateField[]) { 1377 VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2, 1378 &vmstate_domain, VirtIOIOMMUDomain), 1379 VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2), 1380 VMSTATE_END_OF_LIST() 1381 }, 1382 }; 1383 1384 static const VMStateDescription vmstate_virtio_iommu = { 1385 .name = "virtio-iommu", 1386 .minimum_version_id = 2, 1387 .priority = MIG_PRI_IOMMU, 1388 .version_id = 2, 1389 .fields = (VMStateField[]) { 1390 VMSTATE_VIRTIO_DEVICE, 1391 VMSTATE_END_OF_LIST() 1392 }, 1393 }; 1394 1395 static Property virtio_iommu_properties[] = { 1396 DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, 1397 TYPE_PCI_BUS, PCIBus *), 1398 DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true), 1399 DEFINE_PROP_END_OF_LIST(), 1400 }; 1401 1402 static void virtio_iommu_class_init(ObjectClass *klass, void *data) 1403 { 1404 DeviceClass *dc = DEVICE_CLASS(klass); 1405 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 1406 1407 device_class_set_props(dc, virtio_iommu_properties); 1408 dc->vmsd = &vmstate_virtio_iommu; 1409 1410 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1411 vdc->realize = virtio_iommu_device_realize; 1412 vdc->unrealize = virtio_iommu_device_unrealize; 1413 vdc->reset = virtio_iommu_device_reset; 1414 vdc->get_config = virtio_iommu_get_config; 1415 vdc->set_config = virtio_iommu_set_config; 1416 vdc->get_features = virtio_iommu_get_features; 1417 vdc->set_status = virtio_iommu_set_status; 1418 vdc->vmsd = &vmstate_virtio_iommu_device; 1419 } 1420 1421 static void virtio_iommu_memory_region_class_init(ObjectClass *klass, 1422 void *data) 1423 { 1424 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 1425 1426 imrc->translate = virtio_iommu_translate; 1427 imrc->replay = virtio_iommu_replay; 1428 imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; 1429 imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; 1430 } 1431 1432 static const TypeInfo virtio_iommu_info = { 1433 .name = TYPE_VIRTIO_IOMMU, 1434 .parent = TYPE_VIRTIO_DEVICE, 1435 .instance_size = sizeof(VirtIOIOMMU), 1436 .instance_init = virtio_iommu_instance_init, 1437 .class_init = virtio_iommu_class_init, 1438 }; 1439 1440 static const TypeInfo virtio_iommu_memory_region_info = { 1441 .parent = TYPE_IOMMU_MEMORY_REGION, 1442 .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION, 1443 .class_init = virtio_iommu_memory_region_class_init, 1444 }; 1445 1446 static void virtio_register_types(void) 1447 { 1448 type_register_static(&virtio_iommu_info); 1449 type_register_static(&virtio_iommu_memory_region_info); 1450 } 1451 1452 type_init(virtio_register_types) 1453