xref: /openbmc/qemu/hw/virtio/virtio-iommu.c (revision 740b1759)
1 /*
2  * virtio-iommu device
3  *
4  * Copyright (c) 2020 Red Hat, Inc.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2 or later, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License along with
16  * this program.  If not, see <http://www.gnu.org/licenses/>.
17  *
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/iov.h"
23 #include "qemu-common.h"
24 #include "hw/qdev-properties.h"
25 #include "hw/virtio/virtio.h"
26 #include "sysemu/kvm.h"
27 #include "qapi/error.h"
28 #include "qemu/error-report.h"
29 #include "trace.h"
30 
31 #include "standard-headers/linux/virtio_ids.h"
32 
33 #include "hw/virtio/virtio-bus.h"
34 #include "hw/virtio/virtio-access.h"
35 #include "hw/virtio/virtio-iommu.h"
36 #include "hw/pci/pci_bus.h"
37 #include "hw/pci/pci.h"
38 
39 /* Max size */
40 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
41 #define VIOMMU_PROBE_SIZE 512
42 
43 typedef struct VirtIOIOMMUDomain {
44     uint32_t id;
45     GTree *mappings;
46     QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list;
47 } VirtIOIOMMUDomain;
48 
49 typedef struct VirtIOIOMMUEndpoint {
50     uint32_t id;
51     VirtIOIOMMUDomain *domain;
52     QLIST_ENTRY(VirtIOIOMMUEndpoint) next;
53 } VirtIOIOMMUEndpoint;
54 
55 typedef struct VirtIOIOMMUInterval {
56     uint64_t low;
57     uint64_t high;
58 } VirtIOIOMMUInterval;
59 
60 typedef struct VirtIOIOMMUMapping {
61     uint64_t phys_addr;
62     uint32_t flags;
63 } VirtIOIOMMUMapping;
64 
65 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev)
66 {
67     return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn);
68 }
69 
70 /**
71  * The bus number is used for lookup when SID based operations occur.
72  * In that case we lazily populate the IOMMUPciBus array from the bus hash
73  * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus
74  * numbers may not be always initialized yet.
75  */
76 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num)
77 {
78     IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num];
79 
80     if (!iommu_pci_bus) {
81         GHashTableIter iter;
82 
83         g_hash_table_iter_init(&iter, s->as_by_busptr);
84         while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
85             if (pci_bus_num(iommu_pci_bus->bus) == bus_num) {
86                 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus;
87                 return iommu_pci_bus;
88             }
89         }
90         return NULL;
91     }
92     return iommu_pci_bus;
93 }
94 
95 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid)
96 {
97     uint8_t bus_n, devfn;
98     IOMMUPciBus *iommu_pci_bus;
99     IOMMUDevice *dev;
100 
101     bus_n = PCI_BUS_NUM(sid);
102     iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n);
103     if (iommu_pci_bus) {
104         devfn = sid & PCI_DEVFN_MAX;
105         dev = iommu_pci_bus->pbdev[devfn];
106         if (dev) {
107             return &dev->iommu_mr;
108         }
109     }
110     return NULL;
111 }
112 
113 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
114 {
115     VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a;
116     VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b;
117 
118     if (inta->high < intb->low) {
119         return -1;
120     } else if (intb->high < inta->low) {
121         return 1;
122     } else {
123         return 0;
124     }
125 }
126 
127 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep)
128 {
129     if (!ep->domain) {
130         return;
131     }
132     QLIST_REMOVE(ep, next);
133     ep->domain = NULL;
134 }
135 
136 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s,
137                                                       uint32_t ep_id)
138 {
139     VirtIOIOMMUEndpoint *ep;
140 
141     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
142     if (ep) {
143         return ep;
144     }
145     if (!virtio_iommu_mr(s, ep_id)) {
146         return NULL;
147     }
148     ep = g_malloc0(sizeof(*ep));
149     ep->id = ep_id;
150     trace_virtio_iommu_get_endpoint(ep_id);
151     g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep);
152     return ep;
153 }
154 
155 static void virtio_iommu_put_endpoint(gpointer data)
156 {
157     VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data;
158 
159     if (ep->domain) {
160         virtio_iommu_detach_endpoint_from_domain(ep);
161     }
162 
163     trace_virtio_iommu_put_endpoint(ep->id);
164     g_free(ep);
165 }
166 
167 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s,
168                                                   uint32_t domain_id)
169 {
170     VirtIOIOMMUDomain *domain;
171 
172     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
173     if (domain) {
174         return domain;
175     }
176     domain = g_malloc0(sizeof(*domain));
177     domain->id = domain_id;
178     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
179                                    NULL, (GDestroyNotify)g_free,
180                                    (GDestroyNotify)g_free);
181     g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain);
182     QLIST_INIT(&domain->endpoint_list);
183     trace_virtio_iommu_get_domain(domain_id);
184     return domain;
185 }
186 
187 static void virtio_iommu_put_domain(gpointer data)
188 {
189     VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data;
190     VirtIOIOMMUEndpoint *iter, *tmp;
191 
192     QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) {
193         virtio_iommu_detach_endpoint_from_domain(iter);
194     }
195     g_tree_destroy(domain->mappings);
196     trace_virtio_iommu_put_domain(domain->id);
197     g_free(domain);
198 }
199 
200 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque,
201                                               int devfn)
202 {
203     VirtIOIOMMU *s = opaque;
204     IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
205     static uint32_t mr_index;
206     IOMMUDevice *sdev;
207 
208     if (!sbus) {
209         sbus = g_malloc0(sizeof(IOMMUPciBus) +
210                          sizeof(IOMMUDevice *) * PCI_DEVFN_MAX);
211         sbus->bus = bus;
212         g_hash_table_insert(s->as_by_busptr, bus, sbus);
213     }
214 
215     sdev = sbus->pbdev[devfn];
216     if (!sdev) {
217         char *name = g_strdup_printf("%s-%d-%d",
218                                      TYPE_VIRTIO_IOMMU_MEMORY_REGION,
219                                      mr_index++, devfn);
220         sdev = sbus->pbdev[devfn] = g_malloc0(sizeof(IOMMUDevice));
221 
222         sdev->viommu = s;
223         sdev->bus = bus;
224         sdev->devfn = devfn;
225 
226         trace_virtio_iommu_init_iommu_mr(name);
227 
228         memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr),
229                                  TYPE_VIRTIO_IOMMU_MEMORY_REGION,
230                                  OBJECT(s), name,
231                                  UINT64_MAX);
232         address_space_init(&sdev->as,
233                            MEMORY_REGION(&sdev->iommu_mr), TYPE_VIRTIO_IOMMU);
234         g_free(name);
235     }
236     return &sdev->as;
237 }
238 
239 static int virtio_iommu_attach(VirtIOIOMMU *s,
240                                struct virtio_iommu_req_attach *req)
241 {
242     uint32_t domain_id = le32_to_cpu(req->domain);
243     uint32_t ep_id = le32_to_cpu(req->endpoint);
244     VirtIOIOMMUDomain *domain;
245     VirtIOIOMMUEndpoint *ep;
246 
247     trace_virtio_iommu_attach(domain_id, ep_id);
248 
249     ep = virtio_iommu_get_endpoint(s, ep_id);
250     if (!ep) {
251         return VIRTIO_IOMMU_S_NOENT;
252     }
253 
254     if (ep->domain) {
255         VirtIOIOMMUDomain *previous_domain = ep->domain;
256         /*
257          * the device is already attached to a domain,
258          * detach it first
259          */
260         virtio_iommu_detach_endpoint_from_domain(ep);
261         if (QLIST_EMPTY(&previous_domain->endpoint_list)) {
262             g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id));
263         }
264     }
265 
266     domain = virtio_iommu_get_domain(s, domain_id);
267     QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next);
268 
269     ep->domain = domain;
270 
271     return VIRTIO_IOMMU_S_OK;
272 }
273 
274 static int virtio_iommu_detach(VirtIOIOMMU *s,
275                                struct virtio_iommu_req_detach *req)
276 {
277     uint32_t domain_id = le32_to_cpu(req->domain);
278     uint32_t ep_id = le32_to_cpu(req->endpoint);
279     VirtIOIOMMUDomain *domain;
280     VirtIOIOMMUEndpoint *ep;
281 
282     trace_virtio_iommu_detach(domain_id, ep_id);
283 
284     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
285     if (!ep) {
286         return VIRTIO_IOMMU_S_NOENT;
287     }
288 
289     domain = ep->domain;
290 
291     if (!domain || domain->id != domain_id) {
292         return VIRTIO_IOMMU_S_INVAL;
293     }
294 
295     virtio_iommu_detach_endpoint_from_domain(ep);
296 
297     if (QLIST_EMPTY(&domain->endpoint_list)) {
298         g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id));
299     }
300     return VIRTIO_IOMMU_S_OK;
301 }
302 
303 static int virtio_iommu_map(VirtIOIOMMU *s,
304                             struct virtio_iommu_req_map *req)
305 {
306     uint32_t domain_id = le32_to_cpu(req->domain);
307     uint64_t phys_start = le64_to_cpu(req->phys_start);
308     uint64_t virt_start = le64_to_cpu(req->virt_start);
309     uint64_t virt_end = le64_to_cpu(req->virt_end);
310     uint32_t flags = le32_to_cpu(req->flags);
311     VirtIOIOMMUDomain *domain;
312     VirtIOIOMMUInterval *interval;
313     VirtIOIOMMUMapping *mapping;
314 
315     if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) {
316         return VIRTIO_IOMMU_S_INVAL;
317     }
318 
319     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
320     if (!domain) {
321         return VIRTIO_IOMMU_S_NOENT;
322     }
323 
324     interval = g_malloc0(sizeof(*interval));
325 
326     interval->low = virt_start;
327     interval->high = virt_end;
328 
329     mapping = g_tree_lookup(domain->mappings, (gpointer)interval);
330     if (mapping) {
331         g_free(interval);
332         return VIRTIO_IOMMU_S_INVAL;
333     }
334 
335     trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags);
336 
337     mapping = g_malloc0(sizeof(*mapping));
338     mapping->phys_addr = phys_start;
339     mapping->flags = flags;
340 
341     g_tree_insert(domain->mappings, interval, mapping);
342 
343     return VIRTIO_IOMMU_S_OK;
344 }
345 
346 static int virtio_iommu_unmap(VirtIOIOMMU *s,
347                               struct virtio_iommu_req_unmap *req)
348 {
349     uint32_t domain_id = le32_to_cpu(req->domain);
350     uint64_t virt_start = le64_to_cpu(req->virt_start);
351     uint64_t virt_end = le64_to_cpu(req->virt_end);
352     VirtIOIOMMUMapping *iter_val;
353     VirtIOIOMMUInterval interval, *iter_key;
354     VirtIOIOMMUDomain *domain;
355     int ret = VIRTIO_IOMMU_S_OK;
356 
357     trace_virtio_iommu_unmap(domain_id, virt_start, virt_end);
358 
359     domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
360     if (!domain) {
361         return VIRTIO_IOMMU_S_NOENT;
362     }
363     interval.low = virt_start;
364     interval.high = virt_end;
365 
366     while (g_tree_lookup_extended(domain->mappings, &interval,
367                                   (void **)&iter_key, (void**)&iter_val)) {
368         uint64_t current_low = iter_key->low;
369         uint64_t current_high = iter_key->high;
370 
371         if (interval.low <= current_low && interval.high >= current_high) {
372             g_tree_remove(domain->mappings, iter_key);
373             trace_virtio_iommu_unmap_done(domain_id, current_low, current_high);
374         } else {
375             ret = VIRTIO_IOMMU_S_RANGE;
376             break;
377         }
378     }
379     return ret;
380 }
381 
382 static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep,
383                                                uint8_t *buf, size_t free)
384 {
385     struct virtio_iommu_probe_resv_mem prop = {};
386     size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
387     int i;
388 
389     total = size * s->nb_reserved_regions;
390 
391     if (total > free) {
392         return -ENOSPC;
393     }
394 
395     for (i = 0; i < s->nb_reserved_regions; i++) {
396         unsigned subtype = s->reserved_regions[i].type;
397 
398         assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED ||
399                subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI);
400         prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
401         prop.head.length = cpu_to_le16(length);
402         prop.subtype = subtype;
403         prop.start = cpu_to_le64(s->reserved_regions[i].low);
404         prop.end = cpu_to_le64(s->reserved_regions[i].high);
405 
406         memcpy(buf, &prop, size);
407 
408         trace_virtio_iommu_fill_resv_property(ep, prop.subtype,
409                                               prop.start, prop.end);
410         buf += size;
411     }
412     return total;
413 }
414 
415 /**
416  * virtio_iommu_probe - Fill the probe request buffer with
417  * the properties the device is able to return
418  */
419 static int virtio_iommu_probe(VirtIOIOMMU *s,
420                               struct virtio_iommu_req_probe *req,
421                               uint8_t *buf)
422 {
423     uint32_t ep_id = le32_to_cpu(req->endpoint);
424     size_t free = VIOMMU_PROBE_SIZE;
425     ssize_t count;
426 
427     if (!virtio_iommu_mr(s, ep_id)) {
428         return VIRTIO_IOMMU_S_NOENT;
429     }
430 
431     count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free);
432     if (count < 0) {
433         return VIRTIO_IOMMU_S_INVAL;
434     }
435     buf += count;
436     free -= count;
437 
438     return VIRTIO_IOMMU_S_OK;
439 }
440 
441 static int virtio_iommu_iov_to_req(struct iovec *iov,
442                                    unsigned int iov_cnt,
443                                    void *req, size_t req_sz)
444 {
445     size_t sz, payload_sz = req_sz - sizeof(struct virtio_iommu_req_tail);
446 
447     sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz);
448     if (unlikely(sz != payload_sz)) {
449         return VIRTIO_IOMMU_S_INVAL;
450     }
451     return 0;
452 }
453 
454 #define virtio_iommu_handle_req(__req)                                  \
455 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s,                \
456                                          struct iovec *iov,             \
457                                          unsigned int iov_cnt)          \
458 {                                                                       \
459     struct virtio_iommu_req_ ## __req req;                              \
460     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); \
461                                                                         \
462     return ret ? ret : virtio_iommu_ ## __req(s, &req);                 \
463 }
464 
465 virtio_iommu_handle_req(attach)
466 virtio_iommu_handle_req(detach)
467 virtio_iommu_handle_req(map)
468 virtio_iommu_handle_req(unmap)
469 
470 static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
471                                      struct iovec *iov,
472                                      unsigned int iov_cnt,
473                                      uint8_t *buf)
474 {
475     struct virtio_iommu_req_probe req;
476     int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
477 
478     return ret ? ret : virtio_iommu_probe(s, &req, buf);
479 }
480 
481 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
482 {
483     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
484     struct virtio_iommu_req_head head;
485     struct virtio_iommu_req_tail tail = {};
486     size_t output_size = sizeof(tail), sz;
487     VirtQueueElement *elem;
488     unsigned int iov_cnt;
489     struct iovec *iov;
490     void *buf = NULL;
491 
492     for (;;) {
493         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
494         if (!elem) {
495             return;
496         }
497 
498         if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) ||
499             iov_size(elem->out_sg, elem->out_num) < sizeof(head)) {
500             virtio_error(vdev, "virtio-iommu bad head/tail size");
501             virtqueue_detach_element(vq, elem, 0);
502             g_free(elem);
503             break;
504         }
505 
506         iov_cnt = elem->out_num;
507         iov = elem->out_sg;
508         sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head));
509         if (unlikely(sz != sizeof(head))) {
510             tail.status = VIRTIO_IOMMU_S_DEVERR;
511             goto out;
512         }
513         qemu_mutex_lock(&s->mutex);
514         switch (head.type) {
515         case VIRTIO_IOMMU_T_ATTACH:
516             tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt);
517             break;
518         case VIRTIO_IOMMU_T_DETACH:
519             tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt);
520             break;
521         case VIRTIO_IOMMU_T_MAP:
522             tail.status = virtio_iommu_handle_map(s, iov, iov_cnt);
523             break;
524         case VIRTIO_IOMMU_T_UNMAP:
525             tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
526             break;
527         case VIRTIO_IOMMU_T_PROBE:
528         {
529             struct virtio_iommu_req_tail *ptail;
530 
531             output_size = s->config.probe_size + sizeof(tail);
532             buf = g_malloc0(output_size);
533 
534             ptail = (struct virtio_iommu_req_tail *)
535                         (buf + s->config.probe_size);
536             ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
537             break;
538         }
539         default:
540             tail.status = VIRTIO_IOMMU_S_UNSUPP;
541         }
542         qemu_mutex_unlock(&s->mutex);
543 
544 out:
545         sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
546                           buf ? buf : &tail, output_size);
547         assert(sz == output_size);
548 
549         virtqueue_push(vq, elem, sz);
550         virtio_notify(vdev, vq);
551         g_free(elem);
552         g_free(buf);
553     }
554 }
555 
556 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason,
557                                       int flags, uint32_t endpoint,
558                                       uint64_t address)
559 {
560     VirtIODevice *vdev = &viommu->parent_obj;
561     VirtQueue *vq = viommu->event_vq;
562     struct virtio_iommu_fault fault;
563     VirtQueueElement *elem;
564     size_t sz;
565 
566     memset(&fault, 0, sizeof(fault));
567     fault.reason = reason;
568     fault.flags = cpu_to_le32(flags);
569     fault.endpoint = cpu_to_le32(endpoint);
570     fault.address = cpu_to_le64(address);
571 
572     elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
573 
574     if (!elem) {
575         error_report_once(
576             "no buffer available in event queue to report event");
577         return;
578     }
579 
580     if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) {
581         virtio_error(vdev, "error buffer of wrong size");
582         virtqueue_detach_element(vq, elem, 0);
583         g_free(elem);
584         return;
585     }
586 
587     sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
588                       &fault, sizeof(fault));
589     assert(sz == sizeof(fault));
590 
591     trace_virtio_iommu_report_fault(reason, flags, endpoint, address);
592     virtqueue_push(vq, elem, sz);
593     virtio_notify(vdev, vq);
594     g_free(elem);
595 
596 }
597 
598 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
599                                             IOMMUAccessFlags flag,
600                                             int iommu_idx)
601 {
602     IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
603     VirtIOIOMMUInterval interval, *mapping_key;
604     VirtIOIOMMUMapping *mapping_value;
605     VirtIOIOMMU *s = sdev->viommu;
606     bool read_fault, write_fault;
607     VirtIOIOMMUEndpoint *ep;
608     uint32_t sid, flags;
609     bool bypass_allowed;
610     bool found;
611     int i;
612 
613     interval.low = addr;
614     interval.high = addr + 1;
615 
616     IOMMUTLBEntry entry = {
617         .target_as = &address_space_memory,
618         .iova = addr,
619         .translated_addr = addr,
620         .addr_mask = (1 << ctz32(s->config.page_size_mask)) - 1,
621         .perm = IOMMU_NONE,
622     };
623 
624     bypass_allowed = virtio_vdev_has_feature(&s->parent_obj,
625                                              VIRTIO_IOMMU_F_BYPASS);
626 
627     sid = virtio_iommu_get_bdf(sdev);
628 
629     trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag);
630     qemu_mutex_lock(&s->mutex);
631 
632     ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
633     if (!ep) {
634         if (!bypass_allowed) {
635             error_report_once("%s sid=%d is not known!!", __func__, sid);
636             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN,
637                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
638                                       sid, addr);
639         } else {
640             entry.perm = flag;
641         }
642         goto unlock;
643     }
644 
645     for (i = 0; i < s->nb_reserved_regions; i++) {
646         ReservedRegion *reg = &s->reserved_regions[i];
647 
648         if (addr >= reg->low && addr <= reg->high) {
649             switch (reg->type) {
650             case VIRTIO_IOMMU_RESV_MEM_T_MSI:
651                 entry.perm = flag;
652                 break;
653             case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
654             default:
655                 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
656                                           VIRTIO_IOMMU_FAULT_F_ADDRESS,
657                                           sid, addr);
658                 break;
659             }
660             goto unlock;
661         }
662     }
663 
664     if (!ep->domain) {
665         if (!bypass_allowed) {
666             error_report_once("%s %02x:%02x.%01x not attached to any domain",
667                               __func__, PCI_BUS_NUM(sid),
668                               PCI_SLOT(sid), PCI_FUNC(sid));
669             virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN,
670                                       VIRTIO_IOMMU_FAULT_F_ADDRESS,
671                                       sid, addr);
672         } else {
673             entry.perm = flag;
674         }
675         goto unlock;
676     }
677 
678     found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval),
679                                    (void **)&mapping_key,
680                                    (void **)&mapping_value);
681     if (!found) {
682         error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d",
683                           __func__, addr, sid);
684         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
685                                   VIRTIO_IOMMU_FAULT_F_ADDRESS,
686                                   sid, addr);
687         goto unlock;
688     }
689 
690     read_fault = (flag & IOMMU_RO) &&
691                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ);
692     write_fault = (flag & IOMMU_WO) &&
693                     !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE);
694 
695     flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0;
696     flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0;
697     if (flags) {
698         error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d",
699                           __func__, addr, flag, mapping_value->flags);
700         flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS;
701         virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
702                                   flags | VIRTIO_IOMMU_FAULT_F_ADDRESS,
703                                   sid, addr);
704         goto unlock;
705     }
706     entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr;
707     entry.perm = flag;
708     trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid);
709 
710 unlock:
711     qemu_mutex_unlock(&s->mutex);
712     return entry;
713 }
714 
715 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data)
716 {
717     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
718     struct virtio_iommu_config *config = &dev->config;
719 
720     trace_virtio_iommu_get_config(config->page_size_mask,
721                                   config->input_range.start,
722                                   config->input_range.end,
723                                   config->domain_range.end,
724                                   config->probe_size);
725     memcpy(config_data, &dev->config, sizeof(struct virtio_iommu_config));
726 }
727 
728 static void virtio_iommu_set_config(VirtIODevice *vdev,
729                                       const uint8_t *config_data)
730 {
731     struct virtio_iommu_config config;
732 
733     memcpy(&config, config_data, sizeof(struct virtio_iommu_config));
734     trace_virtio_iommu_set_config(config.page_size_mask,
735                                   config.input_range.start,
736                                   config.input_range.end,
737                                   config.domain_range.end,
738                                   config.probe_size);
739 }
740 
741 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f,
742                                           Error **errp)
743 {
744     VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
745 
746     f |= dev->features;
747     trace_virtio_iommu_get_features(f);
748     return f;
749 }
750 
751 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
752 {
753     guint ua = GPOINTER_TO_UINT(a);
754     guint ub = GPOINTER_TO_UINT(b);
755     return (ua > ub) - (ua < ub);
756 }
757 
758 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
759 {
760     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
761     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
762 
763     virtio_init(vdev, "virtio-iommu", VIRTIO_ID_IOMMU,
764                 sizeof(struct virtio_iommu_config));
765 
766     memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num));
767 
768     s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE,
769                              virtio_iommu_handle_command);
770     s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL);
771 
772     s->config.page_size_mask = TARGET_PAGE_MASK;
773     s->config.input_range.end = -1UL;
774     s->config.domain_range.end = 32;
775     s->config.probe_size = VIOMMU_PROBE_SIZE;
776 
777     virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
778     virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
779     virtio_add_feature(&s->features, VIRTIO_F_VERSION_1);
780     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE);
781     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE);
782     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
783     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS);
784     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
785     virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
786 
787     qemu_mutex_init(&s->mutex);
788 
789     s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free);
790 
791     if (s->primary_bus) {
792         pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s);
793     } else {
794         error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!");
795     }
796 }
797 
798 static void virtio_iommu_device_unrealize(DeviceState *dev)
799 {
800     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
801     VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
802 
803     g_hash_table_destroy(s->as_by_busptr);
804     if (s->domains) {
805         g_tree_destroy(s->domains);
806     }
807     if (s->endpoints) {
808         g_tree_destroy(s->endpoints);
809     }
810 
811     virtio_delete_queue(s->req_vq);
812     virtio_delete_queue(s->event_vq);
813     virtio_cleanup(vdev);
814 }
815 
816 static void virtio_iommu_device_reset(VirtIODevice *vdev)
817 {
818     VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
819 
820     trace_virtio_iommu_device_reset();
821 
822     if (s->domains) {
823         g_tree_destroy(s->domains);
824     }
825     if (s->endpoints) {
826         g_tree_destroy(s->endpoints);
827     }
828     s->domains = g_tree_new_full((GCompareDataFunc)int_cmp,
829                                  NULL, NULL, virtio_iommu_put_domain);
830     s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp,
831                                    NULL, NULL, virtio_iommu_put_endpoint);
832 }
833 
834 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status)
835 {
836     trace_virtio_iommu_device_status(status);
837 }
838 
839 static void virtio_iommu_instance_init(Object *obj)
840 {
841 }
842 
843 #define VMSTATE_INTERVAL                               \
844 {                                                      \
845     .name = "interval",                                \
846     .version_id = 1,                                   \
847     .minimum_version_id = 1,                           \
848     .fields = (VMStateField[]) {                       \
849         VMSTATE_UINT64(low, VirtIOIOMMUInterval),      \
850         VMSTATE_UINT64(high, VirtIOIOMMUInterval),     \
851         VMSTATE_END_OF_LIST()                          \
852     }                                                  \
853 }
854 
855 #define VMSTATE_MAPPING                               \
856 {                                                     \
857     .name = "mapping",                                \
858     .version_id = 1,                                  \
859     .minimum_version_id = 1,                          \
860     .fields = (VMStateField[]) {                      \
861         VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\
862         VMSTATE_UINT32(flags, VirtIOIOMMUMapping),    \
863         VMSTATE_END_OF_LIST()                         \
864     },                                                \
865 }
866 
867 static const VMStateDescription vmstate_interval_mapping[2] = {
868     VMSTATE_MAPPING,   /* value */
869     VMSTATE_INTERVAL   /* key   */
870 };
871 
872 static int domain_preload(void *opaque)
873 {
874     VirtIOIOMMUDomain *domain = opaque;
875 
876     domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
877                                        NULL, g_free, g_free);
878     return 0;
879 }
880 
881 static const VMStateDescription vmstate_endpoint = {
882     .name = "endpoint",
883     .version_id = 1,
884     .minimum_version_id = 1,
885     .fields = (VMStateField[]) {
886         VMSTATE_UINT32(id, VirtIOIOMMUEndpoint),
887         VMSTATE_END_OF_LIST()
888     }
889 };
890 
891 static const VMStateDescription vmstate_domain = {
892     .name = "domain",
893     .version_id = 1,
894     .minimum_version_id = 1,
895     .pre_load = domain_preload,
896     .fields = (VMStateField[]) {
897         VMSTATE_UINT32(id, VirtIOIOMMUDomain),
898         VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1,
899                         vmstate_interval_mapping,
900                         VirtIOIOMMUInterval, VirtIOIOMMUMapping),
901         VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1,
902                         vmstate_endpoint, VirtIOIOMMUEndpoint, next),
903         VMSTATE_END_OF_LIST()
904     }
905 };
906 
907 static gboolean reconstruct_endpoints(gpointer key, gpointer value,
908                                       gpointer data)
909 {
910     VirtIOIOMMU *s = (VirtIOIOMMU *)data;
911     VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value;
912     VirtIOIOMMUEndpoint *iter;
913 
914     QLIST_FOREACH(iter, &d->endpoint_list, next) {
915         iter->domain = d;
916         g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter);
917     }
918     return false; /* continue the domain traversal */
919 }
920 
921 static int iommu_post_load(void *opaque, int version_id)
922 {
923     VirtIOIOMMU *s = opaque;
924 
925     g_tree_foreach(s->domains, reconstruct_endpoints, s);
926     return 0;
927 }
928 
929 static const VMStateDescription vmstate_virtio_iommu_device = {
930     .name = "virtio-iommu-device",
931     .minimum_version_id = 1,
932     .version_id = 1,
933     .post_load = iommu_post_load,
934     .fields = (VMStateField[]) {
935         VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 1,
936                                    &vmstate_domain, VirtIOIOMMUDomain),
937         VMSTATE_END_OF_LIST()
938     },
939 };
940 
941 static const VMStateDescription vmstate_virtio_iommu = {
942     .name = "virtio-iommu",
943     .minimum_version_id = 1,
944     .priority = MIG_PRI_IOMMU,
945     .version_id = 1,
946     .fields = (VMStateField[]) {
947         VMSTATE_VIRTIO_DEVICE,
948         VMSTATE_END_OF_LIST()
949     },
950 };
951 
952 static Property virtio_iommu_properties[] = {
953     DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, "PCI", PCIBus *),
954     DEFINE_PROP_END_OF_LIST(),
955 };
956 
957 static void virtio_iommu_class_init(ObjectClass *klass, void *data)
958 {
959     DeviceClass *dc = DEVICE_CLASS(klass);
960     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
961 
962     device_class_set_props(dc, virtio_iommu_properties);
963     dc->vmsd = &vmstate_virtio_iommu;
964 
965     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
966     vdc->realize = virtio_iommu_device_realize;
967     vdc->unrealize = virtio_iommu_device_unrealize;
968     vdc->reset = virtio_iommu_device_reset;
969     vdc->get_config = virtio_iommu_get_config;
970     vdc->set_config = virtio_iommu_set_config;
971     vdc->get_features = virtio_iommu_get_features;
972     vdc->set_status = virtio_iommu_set_status;
973     vdc->vmsd = &vmstate_virtio_iommu_device;
974 }
975 
976 static void virtio_iommu_memory_region_class_init(ObjectClass *klass,
977                                                   void *data)
978 {
979     IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
980 
981     imrc->translate = virtio_iommu_translate;
982 }
983 
984 static const TypeInfo virtio_iommu_info = {
985     .name = TYPE_VIRTIO_IOMMU,
986     .parent = TYPE_VIRTIO_DEVICE,
987     .instance_size = sizeof(VirtIOIOMMU),
988     .instance_init = virtio_iommu_instance_init,
989     .class_init = virtio_iommu_class_init,
990 };
991 
992 static const TypeInfo virtio_iommu_memory_region_info = {
993     .parent = TYPE_IOMMU_MEMORY_REGION,
994     .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION,
995     .class_init = virtio_iommu_memory_region_class_init,
996 };
997 
998 static void virtio_register_types(void)
999 {
1000     type_register_static(&virtio_iommu_info);
1001     type_register_static(&virtio_iommu_memory_region_info);
1002 }
1003 
1004 type_init(virtio_register_types)
1005