xref: /openbmc/qemu/hw/virtio/virtio-iommu.c (revision 43f04cbeff863ae68b6ead432af5e771b92b934c)
1 /*
2  * virtio-iommu device
3  *
4  * Copyright (c) 2020 Red Hat, Inc.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2 or later, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License along with
16  * this program.  If not, see <http://www.gnu.org/licenses/>.
17  *
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/iov.h"
23 #include "exec/target_page.h"
24 #include "hw/qdev-properties.h"
25 #include "hw/virtio/virtio.h"
26 #include "sysemu/kvm.h"
27 #include "sysemu/reset.h"
28 #include "sysemu/sysemu.h"
29 #include "qapi/error.h"
30 #include "qemu/error-report.h"
31 #include "trace.h"
32 
33 #include "standard-headers/linux/virtio_ids.h"
34 
35 #include "hw/virtio/virtio-bus.h"
36 #include "hw/virtio/virtio-iommu.h"
37 #include "hw/pci/pci_bus.h"
38 #include "hw/pci/pci.h"
39 
40 /* Max size */
41 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
42 #define VIOMMU_PROBE_SIZE 512
43 
44 typedef struct VirtIOIOMMUDomain {
45     uint32_t id;
46     bool bypass;
47     GTree *mappings;
48     QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list;
49 } VirtIOIOMMUDomain;
50 
51 typedef struct VirtIOIOMMUEndpoint {
52     uint32_t id;
53     VirtIOIOMMUDomain *domain;
54     IOMMUMemoryRegion *iommu_mr;
55     QLIST_ENTRY(VirtIOIOMMUEndpoint) next;
56 } VirtIOIOMMUEndpoint;
57 
58 typedef struct VirtIOIOMMUInterval {
59     uint64_t low;
60     uint64_t high;
61 } VirtIOIOMMUInterval;
62 
63 typedef struct VirtIOIOMMUMapping {
64     uint64_t phys_addr;
65     uint32_t flags;
66 } VirtIOIOMMUMapping;
67 
68 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev)
69 {
70     return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn);
71 }
72 
73 static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev)
74 {
75     uint32_t sid;
76     bool bypassed;
77     VirtIOIOMMU *s = sdev->viommu;
78     VirtIOIOMMUEndpoint *ep;
79 
80     sid = virtio_iommu_get_bdf(sdev);
81 
82     qemu_rec_mutex_lock(&s->mutex);
83     /* need to check bypass before system reset */
84     if (!s->endpoints) {
85         bypassed = s->config.bypass;
86         goto unlock;
87     }
88 
89     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
90     if (!ep || !ep->domain) {
91         bypassed = s->config.bypass;
92     } else {
93         bypassed = ep->domain->bypass;
94     }
95 
96 unlock:
97     qemu_rec_mutex_unlock(&s->mutex);
98     return bypassed;
99 }
100 
101 /* Return whether the device is using IOMMU translation. */
102 static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev)
103 {
104     bool use_remapping;
105 
106     assert(sdev);
107 
108     use_remapping = !virtio_iommu_device_bypassed(sdev);
109 
110     trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus),
111                                             PCI_SLOT(sdev->devfn),
112                                             PCI_FUNC(sdev->devfn),
113                                             use_remapping);
114 
115     /* Turn off first then on the other */
116     if (use_remapping) {
117         memory_region_set_enabled(&sdev->bypass_mr, false);
118         memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true);
119     } else {
120         memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false);
121         memory_region_set_enabled(&sdev->bypass_mr, true);
122     }
123 
124     return use_remapping;
125 }
126 
127 static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s)
128 {
129     GHashTableIter iter;
130     IOMMUPciBus *iommu_pci_bus;
131     int i;
132 
133     g_hash_table_iter_init(&iter, s->as_by_busptr);
134     while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
135         for (i = 0; i < PCI_DEVFN_MAX; i++) {
136             if (!iommu_pci_bus->pbdev[i]) {
137                 continue;
138             }
139             virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]);
140         }
141     }
142 }
143 
144 /**
145  * The bus number is used for lookup when SID based operations occur.
146  * In that case we lazily populate the IOMMUPciBus array from the bus hash
147  * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus
148  * numbers may not be always initialized yet.
149  */
150 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num)
151 {
152     IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num];
153 
154     if (!iommu_pci_bus) {
155         GHashTableIter iter;
156 
157         g_hash_table_iter_init(&iter, s->as_by_busptr);
158         while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
159             if (pci_bus_num(iommu_pci_bus->bus) == bus_num) {
160                 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus;
161                 return iommu_pci_bus;
162             }
163         }
164         return NULL;
165     }
166     return iommu_pci_bus;
167 }
168 
169 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid)
170 {
171     uint8_t bus_n, devfn;
172     IOMMUPciBus *iommu_pci_bus;
173     IOMMUDevice *dev;
174 
175     bus_n = PCI_BUS_NUM(sid);
176     iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n);
177     if (iommu_pci_bus) {
178         devfn = sid & (PCI_DEVFN_MAX - 1);
179         dev = iommu_pci_bus->pbdev[devfn];
180         if (dev) {
181             return &dev->iommu_mr;
182         }
183     }
184     return NULL;
185 }
186 
187 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
188 {
189     VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a;
190     VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b;
191 
192     if (inta->high < intb->low) {
193         return -1;
194     } else if (intb->high < inta->low) {
195         return 1;
196     } else {
197         return 0;
198     }
199 }
200 
201 static void virtio_iommu_notify_map_unmap(IOMMUMemoryRegion *mr,
202                                           IOMMUTLBEvent *event,
203                                           hwaddr virt_start, hwaddr virt_end)
204 {
205     uint64_t delta = virt_end - virt_start;
206 
207     event->entry.iova = virt_start;
208     event->entry.addr_mask = delta;
209 
210     if (delta == UINT64_MAX) {
211         memory_region_notify_iommu(mr, 0, *event);
212     }
213 
214     while (virt_start != virt_end + 1) {
215         uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64);
216 
217         event->entry.addr_mask = mask;
218         event->entry.iova = virt_start;
219         memory_region_notify_iommu(mr, 0, *event);
220         virt_start += mask + 1;
221         if (event->entry.perm != IOMMU_NONE) {
222             event->entry.translated_addr += mask + 1;
223         }
224     }
225 }
226 
227 static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start,
228                                     hwaddr virt_end, hwaddr paddr,
229                                     uint32_t flags)
230 {
231     IOMMUTLBEvent event;
232     IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ,
233                                               flags & VIRTIO_IOMMU_MAP_F_WRITE);
234 
235     if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) ||
236         (flags & VIRTIO_IOMMU_MAP_F_MMIO) || !perm) {
237         return;
238     }
239 
240     trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end,
241                                   paddr, perm);
242 
243     event.type = IOMMU_NOTIFIER_MAP;
244     event.entry.target_as = &address_space_memory;
245     event.entry.perm = perm;
246     event.entry.translated_addr = paddr;
247 
248     virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end);
249 }
250 
251 static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start,
252                                       hwaddr virt_end)
253 {
254     IOMMUTLBEvent event;
255 
256     if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) {
257         return;
258     }
259 
260     trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end);
261 
262     event.type = IOMMU_NOTIFIER_UNMAP;
263     event.entry.target_as = &address_space_memory;
264     event.entry.perm = IOMMU_NONE;
265     event.entry.translated_addr = 0;
266 
267     virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end);
268 }
269 
270 static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value,
271                                              gpointer data)
272 {
273     VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key;
274     IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data;
275 
276     virtio_iommu_notify_unmap(mr, interval->low, interval->high);
277 
278     return false;
279 }
280 
281 static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value,
282                                            gpointer data)
283 {
284     VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value;
285     VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key;
286     IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data;
287 
288     virtio_iommu_notify_map(mr, interval->low, interval->high,
289                             mapping->phys_addr, mapping->flags);
290 
291     return false;
292 }
293 
294 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep)
295 {
296     VirtIOIOMMUDomain *domain = ep->domain;
297     IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr);
298 
299     if (!ep->domain) {
300         return;
301     }
302     g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb,
303                    ep->iommu_mr);
304     QLIST_REMOVE(ep, next);
305     ep->domain = NULL;
306     virtio_iommu_switch_address_space(sdev);
307 }
308 
309 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s,
310                                                       uint32_t ep_id)
311 {
312     VirtIOIOMMUEndpoint *ep;
313     IOMMUMemoryRegion *mr;
314 
315     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
316     if (ep) {
317         return ep;
318     }
319     mr = virtio_iommu_mr(s, ep_id);
320     if (!mr) {
321         return NULL;
322     }
323     ep = g_malloc0(sizeof(*ep));
324     ep->id = ep_id;
325     ep->iommu_mr = mr;
326     trace_virtio_iommu_get_endpoint(ep_id);
327     g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep);
328     return ep;
329 }
330 
331 static void virtio_iommu_put_endpoint(gpointer data)
332 {
333     VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data;
334 
335     if (ep->domain) {
336         virtio_iommu_detach_endpoint_from_domain(ep);
337     }
338 
339     trace_virtio_iommu_put_endpoint(ep->id);
340     g_free(ep);
341 }
342 
343 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s,
344                                                   uint32_t domain_id,
345                                                   bool bypass)
346 {
347     VirtIOIOMMUDomain *domain;
348 
349     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
350     if (domain) {
351         if (domain->bypass != bypass) {
352             return NULL;
353         }
354         return domain;
355     }
356     domain = g_malloc0(sizeof(*domain));
357     domain->id = domain_id;
358     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
359                                    NULL, (GDestroyNotify)g_free,
360                                    (GDestroyNotify)g_free);
361     domain->bypass = bypass;
362     g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain);
363     QLIST_INIT(&domain->endpoint_list);
364     trace_virtio_iommu_get_domain(domain_id);
365     return domain;
366 }
367 
368 static void virtio_iommu_put_domain(gpointer data)
369 {
370     VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data;
371     VirtIOIOMMUEndpoint *iter, *tmp;
372 
373     QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) {
374         virtio_iommu_detach_endpoint_from_domain(iter);
375     }
376     g_tree_destroy(domain->mappings);
377     trace_virtio_iommu_put_domain(domain->id);
378     g_free(domain);
379 }
380 
381 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque,
382                                               int devfn)
383 {
384     VirtIOIOMMU *s = opaque;
385     IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
386     static uint32_t mr_index;
387     IOMMUDevice *sdev;
388 
389     if (!sbus) {
390         sbus = g_malloc0(sizeof(IOMMUPciBus) +
391                          sizeof(IOMMUDevice *) * PCI_DEVFN_MAX);
392         sbus->bus = bus;
393         g_hash_table_insert(s->as_by_busptr, bus, sbus);
394     }
395 
396     sdev = sbus->pbdev[devfn];
397     if (!sdev) {
398         char *name = g_strdup_printf("%s-%d-%d",
399                                      TYPE_VIRTIO_IOMMU_MEMORY_REGION,
400                                      mr_index++, devfn);
401         sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1);
402 
403         sdev->viommu = s;
404         sdev->bus = bus;
405         sdev->devfn = devfn;
406 
407         trace_virtio_iommu_init_iommu_mr(name);
408 
409         memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX);
410         address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU);
411 
412         /*
413          * Build the IOMMU disabled container with aliases to the
414          * shared MRs.  Note that aliasing to a shared memory region
415          * could help the memory API to detect same FlatViews so we
416          * can have devices to share the same FlatView when in bypass
417          * mode. (either by not configuring virtio-iommu driver or with
418          * "iommu=pt").  It will greatly reduce the total number of
419          * FlatViews of the system hence VM runs faster.
420          */
421         memory_region_init_alias(&sdev->bypass_mr, OBJECT(s),
422                                  "system", get_system_memory(), 0,
423                                  memory_region_size(get_system_memory()));
424 
425         memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr),
426                                  TYPE_VIRTIO_IOMMU_MEMORY_REGION,
427                                  OBJECT(s), name,
428                                  UINT64_MAX);
429 
430         /*
431          * Hook both the containers under the root container, we
432          * switch between iommu & bypass MRs by enable/disable
433          * corresponding sub-containers
434          */
435         memory_region_add_subregion_overlap(&sdev->root, 0,
436                                             MEMORY_REGION(&sdev->iommu_mr),
437                                             0);
438         memory_region_add_subregion_overlap(&sdev->root, 0,
439                                             &sdev->bypass_mr, 0);
440 
441         virtio_iommu_switch_address_space(sdev);
442         g_free(name);
443     }
444     return &sdev->as;
445 }
446 
447 static int virtio_iommu_attach(VirtIOIOMMU *s,
448                                struct virtio_iommu_req_attach *req)
449 {
450     uint32_t domain_id = le32_to_cpu(req->domain);
451     uint32_t ep_id = le32_to_cpu(req->endpoint);
452     uint32_t flags = le32_to_cpu(req->flags);
453     VirtIOIOMMUDomain *domain;
454     VirtIOIOMMUEndpoint *ep;
455     IOMMUDevice *sdev;
456 
457     trace_virtio_iommu_attach(domain_id, ep_id);
458 
459     if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) {
460         return VIRTIO_IOMMU_S_INVAL;
461     }
462 
463     ep = virtio_iommu_get_endpoint(s, ep_id);
464     if (!ep) {
465         return VIRTIO_IOMMU_S_NOENT;
466     }
467 
468     if (ep->domain) {
469         VirtIOIOMMUDomain *previous_domain = ep->domain;
470         /*
471          * the device is already attached to a domain,
472          * detach it first
473          */
474         virtio_iommu_detach_endpoint_from_domain(ep);
475         if (QLIST_EMPTY(&previous_domain->endpoint_list)) {
476             g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id));
477         }
478     }
479 
480     domain = virtio_iommu_get_domain(s, domain_id,
481                                      flags & VIRTIO_IOMMU_ATTACH_F_BYPASS);
482     if (!domain) {
483         /* Incompatible bypass flag */
484         return VIRTIO_IOMMU_S_INVAL;
485     }
486     QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next);
487 
488     ep->domain = domain;
489     sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr);
490     virtio_iommu_switch_address_space(sdev);
491 
492     /* Replay domain mappings on the associated memory region */
493     g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb,
494                    ep->iommu_mr);
495 
496     return VIRTIO_IOMMU_S_OK;
497 }
498 
499 static int virtio_iommu_detach(VirtIOIOMMU *s,
500                                struct virtio_iommu_req_detach *req)
501 {
502     uint32_t domain_id = le32_to_cpu(req->domain);
503     uint32_t ep_id = le32_to_cpu(req->endpoint);
504     VirtIOIOMMUDomain *domain;
505     VirtIOIOMMUEndpoint *ep;
506 
507     trace_virtio_iommu_detach(domain_id, ep_id);
508 
509     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
510     if (!ep) {
511         return VIRTIO_IOMMU_S_NOENT;
512     }
513 
514     domain = ep->domain;
515 
516     if (!domain || domain->id != domain_id) {
517         return VIRTIO_IOMMU_S_INVAL;
518     }
519 
520     virtio_iommu_detach_endpoint_from_domain(ep);
521 
522     if (QLIST_EMPTY(&domain->endpoint_list)) {
523         g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id));
524     }
525     return VIRTIO_IOMMU_S_OK;
526 }
527 
528 static int virtio_iommu_map(VirtIOIOMMU *s,
529                             struct virtio_iommu_req_map *req)
530 {
531     uint32_t domain_id = le32_to_cpu(req->domain);
532     uint64_t phys_start = le64_to_cpu(req->phys_start);
533     uint64_t virt_start = le64_to_cpu(req->virt_start);
534     uint64_t virt_end = le64_to_cpu(req->virt_end);
535     uint32_t flags = le32_to_cpu(req->flags);
536     VirtIOIOMMUDomain *domain;
537     VirtIOIOMMUInterval *interval;
538     VirtIOIOMMUMapping *mapping;
539     VirtIOIOMMUEndpoint *ep;
540 
541     if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) {
542         return VIRTIO_IOMMU_S_INVAL;
543     }
544 
545     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
546     if (!domain) {
547         return VIRTIO_IOMMU_S_NOENT;
548     }
549 
550     if (domain->bypass) {
551         return VIRTIO_IOMMU_S_INVAL;
552     }
553 
554     interval = g_malloc0(sizeof(*interval));
555 
556     interval->low = virt_start;
557     interval->high = virt_end;
558 
559     mapping = g_tree_lookup(domain->mappings, (gpointer)interval);
560     if (mapping) {
561         g_free(interval);
562         return VIRTIO_IOMMU_S_INVAL;
563     }
564 
565     trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags);
566 
567     mapping = g_malloc0(sizeof(*mapping));
568     mapping->phys_addr = phys_start;
569     mapping->flags = flags;
570 
571     g_tree_insert(domain->mappings, interval, mapping);
572 
573     QLIST_FOREACH(ep, &domain->endpoint_list, next) {
574         virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start,
575                                 flags);
576     }
577 
578     return VIRTIO_IOMMU_S_OK;
579 }
580 
581 static int virtio_iommu_unmap(VirtIOIOMMU *s,
582                               struct virtio_iommu_req_unmap *req)
583 {
584     uint32_t domain_id = le32_to_cpu(req->domain);
585     uint64_t virt_start = le64_to_cpu(req->virt_start);
586     uint64_t virt_end = le64_to_cpu(req->virt_end);
587     VirtIOIOMMUMapping *iter_val;
588     VirtIOIOMMUInterval interval, *iter_key;
589     VirtIOIOMMUDomain *domain;
590     VirtIOIOMMUEndpoint *ep;
591     int ret = VIRTIO_IOMMU_S_OK;
592 
593     trace_virtio_iommu_unmap(domain_id, virt_start, virt_end);
594 
595     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
596     if (!domain) {
597         return VIRTIO_IOMMU_S_NOENT;
598     }
599 
600     if (domain->bypass) {
601         return VIRTIO_IOMMU_S_INVAL;
602     }
603 
604     interval.low = virt_start;
605     interval.high = virt_end;
606 
607     while (g_tree_lookup_extended(domain->mappings, &interval,
608                                   (void **)&iter_key, (void**)&iter_val)) {
609         uint64_t current_low = iter_key->low;
610         uint64_t current_high = iter_key->high;
611 
612         if (interval.low <= current_low && interval.high >= current_high) {
613             QLIST_FOREACH(ep, &domain->endpoint_list, next) {
614                 virtio_iommu_notify_unmap(ep->iommu_mr, current_low,
615                                           current_high);
616             }
617             g_tree_remove(domain->mappings, iter_key);
618             trace_virtio_iommu_unmap_done(domain_id, current_low, current_high);
619         } else {
620             ret = VIRTIO_IOMMU_S_RANGE;
621             break;
622         }
623     }
624     return ret;
625 }
626 
627 static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep,
628                                                uint8_t *buf, size_t free)
629 {
630     struct virtio_iommu_probe_resv_mem prop = {};
631     size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
632     int i;
633 
634     total = size * s->nr_prop_resv_regions;
635 
636     if (total > free) {
637         return -ENOSPC;
638     }
639 
640     for (i = 0; i < s->nr_prop_resv_regions; i++) {
641         unsigned subtype = s->prop_resv_regions[i].type;
642         Range *range = &s->prop_resv_regions[i].range;
643 
644         assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED ||
645                subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI);
646         prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
647         prop.head.length = cpu_to_le16(length);
648         prop.subtype = subtype;
649         prop.start = cpu_to_le64(range_lob(range));
650         prop.end = cpu_to_le64(range_upb(range));
651 
652         memcpy(buf, &prop, size);
653 
654         trace_virtio_iommu_fill_resv_property(ep, prop.subtype,
655                                               prop.start, prop.end);
656         buf += size;
657     }
658     return total;
659 }
660 
661 /**
662  * virtio_iommu_probe - Fill the probe request buffer with
663  * the properties the device is able to return
664  */
665 static int virtio_iommu_probe(VirtIOIOMMU *s,
666                               struct virtio_iommu_req_probe *req,
667                               uint8_t *buf)
668 {
669     uint32_t ep_id = le32_to_cpu(req->endpoint);
670     size_t free = VIOMMU_PROBE_SIZE;
671     ssize_t count;
672 
673     if (!virtio_iommu_mr(s, ep_id)) {
674         return VIRTIO_IOMMU_S_NOENT;
675     }
676 
677     count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free);
678     if (count < 0) {
679         return VIRTIO_IOMMU_S_INVAL;
680     }
681     buf += count;
682     free -= count;
683 
684     return VIRTIO_IOMMU_S_OK;
685 }
686 
687 static int virtio_iommu_iov_to_req(struct iovec *iov,
688                                    unsigned int iov_cnt,
689                                    void *req, size_t payload_sz)
690 {
691     size_t sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz);
692 
693     if (unlikely(sz != payload_sz)) {
694         return VIRTIO_IOMMU_S_INVAL;
695     }
696     return 0;
697 }
698 
699 #define virtio_iommu_handle_req(__req)                                  \
700 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s,                \
701                                          struct iovec *iov,             \
702                                          unsigned int iov_cnt)          \
703 {                                                                       \
704     struct virtio_iommu_req_ ## __req req;                              \
705     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req,               \
706                     sizeof(req) - sizeof(struct virtio_iommu_req_tail));\
707                                                                         \
708     return ret ? ret : virtio_iommu_ ## __req(s, &req);                 \
709 }
710 
711 virtio_iommu_handle_req(attach)
712 virtio_iommu_handle_req(detach)
713 virtio_iommu_handle_req(map)
714 virtio_iommu_handle_req(unmap)
715 
716 static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
717                                      struct iovec *iov,
718                                      unsigned int iov_cnt,
719                                      uint8_t *buf)
720 {
721     struct virtio_iommu_req_probe req;
722     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
723 
724     return ret ? ret : virtio_iommu_probe(s, &req, buf);
725 }
726 
727 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
728 {
729     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
730     struct virtio_iommu_req_head head;
731     struct virtio_iommu_req_tail tail = {};
732     VirtQueueElement *elem;
733     unsigned int iov_cnt;
734     struct iovec *iov;
735     void *buf = NULL;
736     size_t sz;
737 
738     for (;;) {
739         size_t output_size = sizeof(tail);
740 
741         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
742         if (!elem) {
743             return;
744         }
745 
746         if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) ||
747             iov_size(elem->out_sg, elem->out_num) < sizeof(head)) {
748             virtio_error(vdev, "virtio-iommu bad head/tail size");
749             virtqueue_detach_element(vq, elem, 0);
750             g_free(elem);
751             break;
752         }
753 
754         iov_cnt = elem->out_num;
755         iov = elem->out_sg;
756         sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head));
757         if (unlikely(sz != sizeof(head))) {
758             tail.status = VIRTIO_IOMMU_S_DEVERR;
759             goto out;
760         }
761         qemu_rec_mutex_lock(&s->mutex);
762         switch (head.type) {
763         case VIRTIO_IOMMU_T_ATTACH:
764             tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt);
765             break;
766         case VIRTIO_IOMMU_T_DETACH:
767             tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt);
768             break;
769         case VIRTIO_IOMMU_T_MAP:
770             tail.status = virtio_iommu_handle_map(s, iov, iov_cnt);
771             break;
772         case VIRTIO_IOMMU_T_UNMAP:
773             tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
774             break;
775         case VIRTIO_IOMMU_T_PROBE:
776         {
777             struct virtio_iommu_req_tail *ptail;
778 
779             output_size = s->config.probe_size + sizeof(tail);
780             buf = g_malloc0(output_size);
781 
782             ptail = buf + s->config.probe_size;
783             ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
784             break;
785         }
786         default:
787             tail.status = VIRTIO_IOMMU_S_UNSUPP;
788         }
789         qemu_rec_mutex_unlock(&s->mutex);
790 
791 out:
792         sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
793                           buf ? buf : &tail, output_size);
794         assert(sz == output_size);
795 
796         virtqueue_push(vq, elem, sz);
797         virtio_notify(vdev, vq);
798         g_free(elem);
799         g_free(buf);
800         buf = NULL;
801     }
802 }
803 
804 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason,
805                                       int flags, uint32_t endpoint,
806                                       uint64_t address)
807 {
808     VirtIODevice *vdev = &viommu->parent_obj;
809     VirtQueue *vq = viommu->event_vq;
810     struct virtio_iommu_fault fault;
811     VirtQueueElement *elem;
812     size_t sz;
813 
814     memset(&fault, 0, sizeof(fault));
815     fault.reason = reason;
816     fault.flags = cpu_to_le32(flags);
817     fault.endpoint = cpu_to_le32(endpoint);
818     fault.address = cpu_to_le64(address);
819 
820     elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
821 
822     if (!elem) {
823         error_report_once(
824             "no buffer available in event queue to report event");
825         return;
826     }
827 
828     if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) {
829         virtio_error(vdev, "error buffer of wrong size");
830         virtqueue_detach_element(vq, elem, 0);
831         g_free(elem);
832         return;
833     }
834 
835     sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
836                       &fault, sizeof(fault));
837     assert(sz == sizeof(fault));
838 
839     trace_virtio_iommu_report_fault(reason, flags, endpoint, address);
840     virtqueue_push(vq, elem, sz);
841     virtio_notify(vdev, vq);
842     g_free(elem);
843 
844 }
845 
846 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
847                                             IOMMUAccessFlags flag,
848                                             int iommu_idx)
849 {
850     IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
851     VirtIOIOMMUInterval interval, *mapping_key;
852     VirtIOIOMMUMapping *mapping_value;
853     VirtIOIOMMU *s = sdev->viommu;
854     bool read_fault, write_fault;
855     VirtIOIOMMUEndpoint *ep;
856     uint32_t sid, flags;
857     bool bypass_allowed;
858     int granule;
859     bool found;
860     int i;
861 
862     interval.low = addr;
863     interval.high = addr + 1;
864     granule = ctz64(s->config.page_size_mask);
865 
866     IOMMUTLBEntry entry = {
867         .target_as = &address_space_memory,
868         .iova = addr,
869         .translated_addr = addr,
870         .addr_mask = BIT_ULL(granule) - 1,
871         .perm = IOMMU_NONE,
872     };
873 
874     bypass_allowed = s->config.bypass;
875 
876     sid = virtio_iommu_get_bdf(sdev);
877 
878     trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag);
879     qemu_rec_mutex_lock(&s->mutex);
880 
881     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
882 
883     if (bypass_allowed)
884         assert(ep && ep->domain && !ep->domain->bypass);
885 
886     if (!ep) {
887         if (!bypass_allowed) {
888             error_report_once("%s sid=%d is not known!!", __func__, sid);
889             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN,
890                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
891                                       sid, addr);
892         } else {
893             entry.perm = flag;
894         }
895         goto unlock;
896     }
897 
898     for (i = 0; i < s->nr_prop_resv_regions; i++) {
899         ReservedRegion *reg = &s->prop_resv_regions[i];
900 
901         if (range_contains(&reg->range, addr)) {
902             switch (reg->type) {
903             case VIRTIO_IOMMU_RESV_MEM_T_MSI:
904                 entry.perm = flag;
905                 break;
906             case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
907             default:
908                 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
909                                           VIRTIO_IOMMU_FAULT_F_ADDRESS,
910                                           sid, addr);
911                 break;
912             }
913             goto unlock;
914         }
915     }
916 
917     if (!ep->domain) {
918         if (!bypass_allowed) {
919             error_report_once("%s %02x:%02x.%01x not attached to any domain",
920                               __func__, PCI_BUS_NUM(sid),
921                               PCI_SLOT(sid), PCI_FUNC(sid));
922             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN,
923                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
924                                       sid, addr);
925         } else {
926             entry.perm = flag;
927         }
928         goto unlock;
929     } else if (ep->domain->bypass) {
930         entry.perm = flag;
931         goto unlock;
932     }
933 
934     found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval),
935                                    (void **)&mapping_key,
936                                    (void **)&mapping_value);
937     if (!found) {
938         error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d",
939                           __func__, addr, sid);
940         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
941                                   VIRTIO_IOMMU_FAULT_F_ADDRESS,
942                                   sid, addr);
943         goto unlock;
944     }
945 
946     read_fault = (flag & IOMMU_RO) &&
947                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ);
948     write_fault = (flag & IOMMU_WO) &&
949                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE);
950 
951     flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0;
952     flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0;
953     if (flags) {
954         error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d",
955                           __func__, addr, flag, mapping_value->flags);
956         flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS;
957         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
958                                   flags | VIRTIO_IOMMU_FAULT_F_ADDRESS,
959                                   sid, addr);
960         goto unlock;
961     }
962     entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr;
963     entry.perm = flag;
964     trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid);
965 
966 unlock:
967     qemu_rec_mutex_unlock(&s->mutex);
968     return entry;
969 }
970 
971 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data)
972 {
973     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
974     struct virtio_iommu_config *dev_config = &dev->config;
975     struct virtio_iommu_config *out_config = (void *)config_data;
976 
977     out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask);
978     out_config->input_range.start = cpu_to_le64(dev_config->input_range.start);
979     out_config->input_range.end = cpu_to_le64(dev_config->input_range.end);
980     out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start);
981     out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end);
982     out_config->probe_size = cpu_to_le32(dev_config->probe_size);
983     out_config->bypass = dev_config->bypass;
984 
985     trace_virtio_iommu_get_config(dev_config->page_size_mask,
986                                   dev_config->input_range.start,
987                                   dev_config->input_range.end,
988                                   dev_config->domain_range.start,
989                                   dev_config->domain_range.end,
990                                   dev_config->probe_size,
991                                   dev_config->bypass);
992 }
993 
994 static void virtio_iommu_set_config(VirtIODevice *vdev,
995                                     const uint8_t *config_data)
996 {
997     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
998     struct virtio_iommu_config *dev_config = &dev->config;
999     const struct virtio_iommu_config *in_config = (void *)config_data;
1000 
1001     if (in_config->bypass != dev_config->bypass) {
1002         if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) {
1003             virtio_error(vdev, "cannot set config.bypass");
1004             return;
1005         } else if (in_config->bypass != 0 && in_config->bypass != 1) {
1006             virtio_error(vdev, "invalid config.bypass value '%u'",
1007                          in_config->bypass);
1008             return;
1009         }
1010         dev_config->bypass = in_config->bypass;
1011         virtio_iommu_switch_address_space_all(dev);
1012     }
1013 
1014     trace_virtio_iommu_set_config(in_config->bypass);
1015 }
1016 
1017 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f,
1018                                           Error **errp)
1019 {
1020     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
1021 
1022     f |= dev->features;
1023     trace_virtio_iommu_get_features(f);
1024     return f;
1025 }
1026 
1027 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
1028 {
1029     guint ua = GPOINTER_TO_UINT(a);
1030     guint ub = GPOINTER_TO_UINT(b);
1031     return (ua > ub) - (ua < ub);
1032 }
1033 
1034 static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data)
1035 {
1036     VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value;
1037     VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key;
1038     IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data;
1039 
1040     trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high,
1041                              mapping->phys_addr);
1042     virtio_iommu_notify_map(mr, interval->low, interval->high,
1043                             mapping->phys_addr, mapping->flags);
1044     return false;
1045 }
1046 
1047 static void virtio_iommu_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n)
1048 {
1049     IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
1050     VirtIOIOMMU *s = sdev->viommu;
1051     uint32_t sid;
1052     VirtIOIOMMUEndpoint *ep;
1053 
1054     sid = virtio_iommu_get_bdf(sdev);
1055 
1056     qemu_rec_mutex_lock(&s->mutex);
1057 
1058     if (!s->endpoints) {
1059         goto unlock;
1060     }
1061 
1062     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
1063     if (!ep || !ep->domain) {
1064         goto unlock;
1065     }
1066 
1067     g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr);
1068 
1069 unlock:
1070     qemu_rec_mutex_unlock(&s->mutex);
1071 }
1072 
1073 static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr,
1074                                             IOMMUNotifierFlag old,
1075                                             IOMMUNotifierFlag new,
1076                                             Error **errp)
1077 {
1078     if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
1079         error_setg(errp, "Virtio-iommu does not support dev-iotlb yet");
1080         return -EINVAL;
1081     }
1082 
1083     if (old == IOMMU_NOTIFIER_NONE) {
1084         trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name);
1085     } else if (new == IOMMU_NOTIFIER_NONE) {
1086         trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name);
1087     }
1088     return 0;
1089 }
1090 
1091 /*
1092  * The default mask (TARGET_PAGE_MASK) is the smallest supported guest granule,
1093  * for example 0xfffffffffffff000. When an assigned device has page size
1094  * restrictions due to the hardware IOMMU configuration, apply this restriction
1095  * to the mask.
1096  */
1097 static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr,
1098                                            uint64_t new_mask,
1099                                            Error **errp)
1100 {
1101     IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
1102     VirtIOIOMMU *s = sdev->viommu;
1103     uint64_t cur_mask = s->config.page_size_mask;
1104 
1105     trace_virtio_iommu_set_page_size_mask(mr->parent_obj.name, cur_mask,
1106                                           new_mask);
1107 
1108     if ((cur_mask & new_mask) == 0) {
1109         error_setg(errp, "virtio-iommu %s reports a page size mask 0x%"PRIx64
1110                    " incompatible with currently supported mask 0x%"PRIx64,
1111                    mr->parent_obj.name, new_mask, cur_mask);
1112         return -1;
1113     }
1114 
1115     /*
1116      * Once the granule is frozen we can't change the mask anymore. If by
1117      * chance the hotplugged device supports the same granule, we can still
1118      * accept it.
1119      */
1120     if (s->granule_frozen) {
1121         int cur_granule = ctz64(cur_mask);
1122 
1123         if (!(BIT_ULL(cur_granule) & new_mask)) {
1124             error_setg(errp, "virtio-iommu %s does not support frozen granule 0x%llx",
1125                        mr->parent_obj.name, BIT_ULL(cur_granule));
1126             return -1;
1127         }
1128         return 0;
1129     }
1130 
1131     s->config.page_size_mask &= new_mask;
1132     return 0;
1133 }
1134 
1135 static void virtio_iommu_system_reset(void *opaque)
1136 {
1137     VirtIOIOMMU *s = opaque;
1138 
1139     trace_virtio_iommu_system_reset();
1140 
1141     /*
1142      * config.bypass is sticky across device reset, but should be restored on
1143      * system reset
1144      */
1145     s->config.bypass = s->boot_bypass;
1146     virtio_iommu_switch_address_space_all(s);
1147 
1148 }
1149 
1150 static void virtio_iommu_freeze_granule(Notifier *notifier, void *data)
1151 {
1152     VirtIOIOMMU *s = container_of(notifier, VirtIOIOMMU, machine_done);
1153     int granule;
1154 
1155     if (likely(s->config.bypass)) {
1156         /*
1157          * Transient IOMMU MR enable to collect page_size_mask requirements
1158          * through memory_region_iommu_set_page_size_mask() called by
1159          * VFIO region_add() callback
1160          */
1161          s->config.bypass = false;
1162          virtio_iommu_switch_address_space_all(s);
1163          /* restore default */
1164          s->config.bypass = true;
1165          virtio_iommu_switch_address_space_all(s);
1166     }
1167     s->granule_frozen = true;
1168     granule = ctz64(s->config.page_size_mask);
1169     trace_virtio_iommu_freeze_granule(BIT_ULL(granule));
1170 }
1171 
1172 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
1173 {
1174     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1175     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
1176 
1177     virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config));
1178 
1179     memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num));
1180 
1181     s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE,
1182                              virtio_iommu_handle_command);
1183     s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL);
1184 
1185     /*
1186      * config.bypass is needed to get initial address space early, such as
1187      * in vfio realize
1188      */
1189     s->config.bypass = s->boot_bypass;
1190     s->config.page_size_mask = qemu_target_page_mask();
1191     s->config.input_range.end = UINT64_MAX;
1192     s->config.domain_range.end = UINT32_MAX;
1193     s->config.probe_size = VIOMMU_PROBE_SIZE;
1194 
1195     virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
1196     virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
1197     virtio_add_feature(&s->features, VIRTIO_F_VERSION_1);
1198     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE);
1199     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE);
1200     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
1201     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
1202     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
1203     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG);
1204 
1205     qemu_rec_mutex_init(&s->mutex);
1206 
1207     s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free);
1208 
1209     if (s->primary_bus) {
1210         pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s);
1211     } else {
1212         error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!");
1213     }
1214 
1215     s->machine_done.notify = virtio_iommu_freeze_granule;
1216     qemu_add_machine_init_done_notifier(&s->machine_done);
1217 
1218     qemu_register_reset(virtio_iommu_system_reset, s);
1219 }
1220 
1221 static void virtio_iommu_device_unrealize(DeviceState *dev)
1222 {
1223     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1224     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
1225 
1226     qemu_unregister_reset(virtio_iommu_system_reset, s);
1227     qemu_remove_machine_init_done_notifier(&s->machine_done);
1228 
1229     g_hash_table_destroy(s->as_by_busptr);
1230     if (s->domains) {
1231         g_tree_destroy(s->domains);
1232     }
1233     if (s->endpoints) {
1234         g_tree_destroy(s->endpoints);
1235     }
1236 
1237     qemu_rec_mutex_destroy(&s->mutex);
1238 
1239     virtio_delete_queue(s->req_vq);
1240     virtio_delete_queue(s->event_vq);
1241     virtio_cleanup(vdev);
1242 }
1243 
1244 static void virtio_iommu_device_reset(VirtIODevice *vdev)
1245 {
1246     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
1247 
1248     trace_virtio_iommu_device_reset();
1249 
1250     if (s->domains) {
1251         g_tree_destroy(s->domains);
1252     }
1253     if (s->endpoints) {
1254         g_tree_destroy(s->endpoints);
1255     }
1256     s->domains = g_tree_new_full((GCompareDataFunc)int_cmp,
1257                                  NULL, NULL, virtio_iommu_put_domain);
1258     s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp,
1259                                    NULL, NULL, virtio_iommu_put_endpoint);
1260 }
1261 
1262 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status)
1263 {
1264     trace_virtio_iommu_device_status(status);
1265 }
1266 
1267 static void virtio_iommu_instance_init(Object *obj)
1268 {
1269 }
1270 
1271 #define VMSTATE_INTERVAL                               \
1272 {                                                      \
1273     .name = "interval",                                \
1274     .version_id = 1,                                   \
1275     .minimum_version_id = 1,                           \
1276     .fields = (VMStateField[]) {                       \
1277         VMSTATE_UINT64(low, VirtIOIOMMUInterval),      \
1278         VMSTATE_UINT64(high, VirtIOIOMMUInterval),     \
1279         VMSTATE_END_OF_LIST()                          \
1280     }                                                  \
1281 }
1282 
1283 #define VMSTATE_MAPPING                               \
1284 {                                                     \
1285     .name = "mapping",                                \
1286     .version_id = 1,                                  \
1287     .minimum_version_id = 1,                          \
1288     .fields = (VMStateField[]) {                      \
1289         VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\
1290         VMSTATE_UINT32(flags, VirtIOIOMMUMapping),    \
1291         VMSTATE_END_OF_LIST()                         \
1292     },                                                \
1293 }
1294 
1295 static const VMStateDescription vmstate_interval_mapping[2] = {
1296     VMSTATE_MAPPING,   /* value */
1297     VMSTATE_INTERVAL   /* key   */
1298 };
1299 
1300 static int domain_preload(void *opaque)
1301 {
1302     VirtIOIOMMUDomain *domain = opaque;
1303 
1304     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
1305                                        NULL, g_free, g_free);
1306     return 0;
1307 }
1308 
1309 static const VMStateDescription vmstate_endpoint = {
1310     .name = "endpoint",
1311     .version_id = 1,
1312     .minimum_version_id = 1,
1313     .fields = (VMStateField[]) {
1314         VMSTATE_UINT32(id, VirtIOIOMMUEndpoint),
1315         VMSTATE_END_OF_LIST()
1316     }
1317 };
1318 
1319 static const VMStateDescription vmstate_domain = {
1320     .name = "domain",
1321     .version_id = 2,
1322     .minimum_version_id = 2,
1323     .pre_load = domain_preload,
1324     .fields = (VMStateField[]) {
1325         VMSTATE_UINT32(id, VirtIOIOMMUDomain),
1326         VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1,
1327                         vmstate_interval_mapping,
1328                         VirtIOIOMMUInterval, VirtIOIOMMUMapping),
1329         VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1,
1330                         vmstate_endpoint, VirtIOIOMMUEndpoint, next),
1331         VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2),
1332         VMSTATE_END_OF_LIST()
1333     }
1334 };
1335 
1336 static gboolean reconstruct_endpoints(gpointer key, gpointer value,
1337                                       gpointer data)
1338 {
1339     VirtIOIOMMU *s = (VirtIOIOMMU *)data;
1340     VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value;
1341     VirtIOIOMMUEndpoint *iter;
1342     IOMMUMemoryRegion *mr;
1343 
1344     QLIST_FOREACH(iter, &d->endpoint_list, next) {
1345         mr = virtio_iommu_mr(s, iter->id);
1346         assert(mr);
1347 
1348         iter->domain = d;
1349         iter->iommu_mr = mr;
1350         g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter);
1351     }
1352     return false; /* continue the domain traversal */
1353 }
1354 
1355 static int iommu_post_load(void *opaque, int version_id)
1356 {
1357     VirtIOIOMMU *s = opaque;
1358 
1359     g_tree_foreach(s->domains, reconstruct_endpoints, s);
1360 
1361     /*
1362      * Memory regions are dynamically turned on/off depending on
1363      * 'config.bypass' and attached domain type if there is. After
1364      * migration, we need to make sure the memory regions are
1365      * still correct.
1366      */
1367     virtio_iommu_switch_address_space_all(s);
1368     return 0;
1369 }
1370 
1371 static const VMStateDescription vmstate_virtio_iommu_device = {
1372     .name = "virtio-iommu-device",
1373     .minimum_version_id = 2,
1374     .version_id = 2,
1375     .post_load = iommu_post_load,
1376     .fields = (VMStateField[]) {
1377         VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2,
1378                                    &vmstate_domain, VirtIOIOMMUDomain),
1379         VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2),
1380         VMSTATE_END_OF_LIST()
1381     },
1382 };
1383 
1384 static const VMStateDescription vmstate_virtio_iommu = {
1385     .name = "virtio-iommu",
1386     .minimum_version_id = 2,
1387     .priority = MIG_PRI_IOMMU,
1388     .version_id = 2,
1389     .fields = (VMStateField[]) {
1390         VMSTATE_VIRTIO_DEVICE,
1391         VMSTATE_END_OF_LIST()
1392     },
1393 };
1394 
1395 static Property virtio_iommu_properties[] = {
1396     DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus,
1397                      TYPE_PCI_BUS, PCIBus *),
1398     DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true),
1399     DEFINE_PROP_END_OF_LIST(),
1400 };
1401 
1402 static void virtio_iommu_class_init(ObjectClass *klass, void *data)
1403 {
1404     DeviceClass *dc = DEVICE_CLASS(klass);
1405     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
1406 
1407     device_class_set_props(dc, virtio_iommu_properties);
1408     dc->vmsd = &vmstate_virtio_iommu;
1409 
1410     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
1411     vdc->realize = virtio_iommu_device_realize;
1412     vdc->unrealize = virtio_iommu_device_unrealize;
1413     vdc->reset = virtio_iommu_device_reset;
1414     vdc->get_config = virtio_iommu_get_config;
1415     vdc->set_config = virtio_iommu_set_config;
1416     vdc->get_features = virtio_iommu_get_features;
1417     vdc->set_status = virtio_iommu_set_status;
1418     vdc->vmsd = &vmstate_virtio_iommu_device;
1419 }
1420 
1421 static void virtio_iommu_memory_region_class_init(ObjectClass *klass,
1422                                                   void *data)
1423 {
1424     IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1425 
1426     imrc->translate = virtio_iommu_translate;
1427     imrc->replay = virtio_iommu_replay;
1428     imrc->notify_flag_changed = virtio_iommu_notify_flag_changed;
1429     imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask;
1430 }
1431 
1432 static const TypeInfo virtio_iommu_info = {
1433     .name = TYPE_VIRTIO_IOMMU,
1434     .parent = TYPE_VIRTIO_DEVICE,
1435     .instance_size = sizeof(VirtIOIOMMU),
1436     .instance_init = virtio_iommu_instance_init,
1437     .class_init = virtio_iommu_class_init,
1438 };
1439 
1440 static const TypeInfo virtio_iommu_memory_region_info = {
1441     .parent = TYPE_IOMMU_MEMORY_REGION,
1442     .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION,
1443     .class_init = virtio_iommu_memory_region_class_init,
1444 };
1445 
1446 static void virtio_register_types(void)
1447 {
1448     type_register_static(&virtio_iommu_info);
1449     type_register_static(&virtio_iommu_memory_region_info);
1450 }
1451 
1452 type_init(virtio_register_types)
1453