xref: /openbmc/qemu/util/vfio-helpers.c (revision dbdea0db)
1 /*
2  * VFIO utility
3  *
4  * Copyright 2016 - 2018 Red Hat, Inc.
5  *
6  * Authors:
7  *   Fam Zheng <famz@redhat.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2 or later.
10  * See the COPYING file in the top-level directory.
11  */
12 
13 #include "qemu/osdep.h"
14 #include <sys/ioctl.h>
15 #include <linux/vfio.h>
16 #include "qapi/error.h"
17 #include "exec/ramlist.h"
18 #include "exec/cpu-common.h"
19 #include "exec/memory.h"
20 #include "trace.h"
21 #include "qemu/error-report.h"
22 #include "standard-headers/linux/pci_regs.h"
23 #include "qemu/event_notifier.h"
24 #include "qemu/vfio-helpers.h"
25 #include "qemu/lockable.h"
26 #include "trace.h"
27 
28 #define QEMU_VFIO_DEBUG 0
29 
30 #define QEMU_VFIO_IOVA_MIN 0x10000ULL
31 /* XXX: Once VFIO exposes the iova bit width in the IOMMU capability interface,
32  * we can use a runtime limit; alternatively it's also possible to do platform
33  * specific detection by reading sysfs entries. Until then, 39 is a safe bet.
34  **/
35 #define QEMU_VFIO_IOVA_MAX (1ULL << 39)
36 
37 typedef struct {
38     /* Page aligned addr. */
39     void *host;
40     size_t size;
41     uint64_t iova;
42 } IOVAMapping;
43 
44 struct IOVARange {
45     uint64_t start;
46     uint64_t end;
47 };
48 
49 struct QEMUVFIOState {
50     QemuMutex lock;
51 
52     /* These fields are protected by BQL */
53     int container;
54     int group;
55     int device;
56     RAMBlockNotifier ram_notifier;
57     struct vfio_region_info config_region_info, bar_region_info[6];
58     struct IOVARange *usable_iova_ranges;
59     uint8_t nb_iova_ranges;
60 
61     /* These fields are protected by @lock */
62     /* VFIO's IO virtual address space is managed by splitting into a few
63      * sections:
64      *
65      * ---------------       <= 0
66      * |xxxxxxxxxxxxx|
67      * |-------------|       <= QEMU_VFIO_IOVA_MIN
68      * |             |
69      * |    Fixed    |
70      * |             |
71      * |-------------|       <= low_water_mark
72      * |             |
73      * |    Free     |
74      * |             |
75      * |-------------|       <= high_water_mark
76      * |             |
77      * |    Temp     |
78      * |             |
79      * |-------------|       <= QEMU_VFIO_IOVA_MAX
80      * |xxxxxxxxxxxxx|
81      * |xxxxxxxxxxxxx|
82      * ---------------
83      *
84      * - Addresses lower than QEMU_VFIO_IOVA_MIN are reserved as invalid;
85      *
86      * - Fixed mappings of HVAs are assigned "low" IOVAs in the range of
87      *   [QEMU_VFIO_IOVA_MIN, low_water_mark).  Once allocated they will not be
88      *   reclaimed - low_water_mark never shrinks;
89      *
90      * - IOVAs in range [low_water_mark, high_water_mark) are free;
91      *
92      * - IOVAs in range [high_water_mark, QEMU_VFIO_IOVA_MAX) are volatile
93      *   mappings. At each qemu_vfio_dma_reset_temporary() call, the whole area
94      *   is recycled. The caller should make sure I/O's depending on these
95      *   mappings are completed before calling.
96      **/
97     uint64_t low_water_mark;
98     uint64_t high_water_mark;
99     IOVAMapping *mappings;
100     int nr_mappings;
101 };
102 
103 /**
104  * Find group file by PCI device address as specified @device, and return the
105  * path. The returned string is owned by caller and should be g_free'ed later.
106  */
sysfs_find_group_file(const char * device,Error ** errp)107 static char *sysfs_find_group_file(const char *device, Error **errp)
108 {
109     g_autoptr(GError) gerr = NULL;
110     char *sysfs_link;
111     char *sysfs_group;
112     char *p;
113     char *path = NULL;
114 
115     sysfs_link = g_strdup_printf("/sys/bus/pci/devices/%s/iommu_group", device);
116     sysfs_group = g_file_read_link(sysfs_link, &gerr);
117     if (gerr) {
118         error_setg(errp, "Failed to find iommu group sysfs path: %s",
119                    gerr->message);
120         goto out;
121     }
122     p = strrchr(sysfs_group, '/');
123     if (!p) {
124         error_setg(errp, "Failed to find iommu group number");
125         goto out;
126     }
127 
128     path = g_strdup_printf("/dev/vfio/%s", p + 1);
129 out:
130     g_free(sysfs_link);
131     g_free(sysfs_group);
132     return path;
133 }
134 
assert_bar_index_valid(QEMUVFIOState * s,int index)135 static inline void assert_bar_index_valid(QEMUVFIOState *s, int index)
136 {
137     assert(index >= 0 && index < ARRAY_SIZE(s->bar_region_info));
138 }
139 
qemu_vfio_pci_init_bar(QEMUVFIOState * s,int index,Error ** errp)140 static int qemu_vfio_pci_init_bar(QEMUVFIOState *s, int index, Error **errp)
141 {
142     g_autofree char *barname = NULL;
143     assert_bar_index_valid(s, index);
144     s->bar_region_info[index] = (struct vfio_region_info) {
145         .index = VFIO_PCI_BAR0_REGION_INDEX + index,
146         .argsz = sizeof(struct vfio_region_info),
147     };
148     if (ioctl(s->device, VFIO_DEVICE_GET_REGION_INFO, &s->bar_region_info[index])) {
149         error_setg_errno(errp, errno, "Failed to get BAR region info");
150         return -errno;
151     }
152     barname = g_strdup_printf("bar[%d]", index);
153     trace_qemu_vfio_region_info(barname, s->bar_region_info[index].offset,
154                                 s->bar_region_info[index].size,
155                                 s->bar_region_info[index].cap_offset);
156 
157     return 0;
158 }
159 
160 /**
161  * Map a PCI bar area.
162  */
qemu_vfio_pci_map_bar(QEMUVFIOState * s,int index,uint64_t offset,uint64_t size,int prot,Error ** errp)163 void *qemu_vfio_pci_map_bar(QEMUVFIOState *s, int index,
164                             uint64_t offset, uint64_t size, int prot,
165                             Error **errp)
166 {
167     void *p;
168     assert(QEMU_IS_ALIGNED(offset, qemu_real_host_page_size()));
169     assert_bar_index_valid(s, index);
170     p = mmap(NULL, MIN(size, s->bar_region_info[index].size - offset),
171              prot, MAP_SHARED,
172              s->device, s->bar_region_info[index].offset + offset);
173     trace_qemu_vfio_pci_map_bar(index, s->bar_region_info[index].offset ,
174                                 size, offset, p);
175     if (p == MAP_FAILED) {
176         error_setg_errno(errp, errno, "Failed to map BAR region");
177         p = NULL;
178     }
179     return p;
180 }
181 
182 /**
183  * Unmap a PCI bar area.
184  */
qemu_vfio_pci_unmap_bar(QEMUVFIOState * s,int index,void * bar,uint64_t offset,uint64_t size)185 void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar,
186                              uint64_t offset, uint64_t size)
187 {
188     if (bar) {
189         munmap(bar, MIN(size, s->bar_region_info[index].size - offset));
190     }
191 }
192 
193 /**
194  * Initialize device IRQ with @irq_type and register an event notifier.
195  */
qemu_vfio_pci_init_irq(QEMUVFIOState * s,EventNotifier * e,int irq_type,Error ** errp)196 int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e,
197                            int irq_type, Error **errp)
198 {
199     int r;
200     struct vfio_irq_set *irq_set;
201     size_t irq_set_size;
202     struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
203 
204     irq_info.index = irq_type;
205     if (ioctl(s->device, VFIO_DEVICE_GET_IRQ_INFO, &irq_info)) {
206         error_setg_errno(errp, errno, "Failed to get device interrupt info");
207         return -errno;
208     }
209     if (!(irq_info.flags & VFIO_IRQ_INFO_EVENTFD)) {
210         error_setg(errp, "Device interrupt doesn't support eventfd");
211         return -EINVAL;
212     }
213 
214     irq_set_size = sizeof(*irq_set) + sizeof(int);
215     irq_set = g_malloc0(irq_set_size);
216 
217     /* Get to a known IRQ state */
218     *irq_set = (struct vfio_irq_set) {
219         .argsz = irq_set_size,
220         .flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER,
221         .index = irq_info.index,
222         .start = 0,
223         .count = 1,
224     };
225 
226     *(int *)&irq_set->data = event_notifier_get_fd(e);
227     r = ioctl(s->device, VFIO_DEVICE_SET_IRQS, irq_set);
228     g_free(irq_set);
229     if (r) {
230         error_setg_errno(errp, errno, "Failed to setup device interrupt");
231         return -errno;
232     }
233     return 0;
234 }
235 
qemu_vfio_pci_read_config(QEMUVFIOState * s,void * buf,int size,int ofs)236 static int qemu_vfio_pci_read_config(QEMUVFIOState *s, void *buf,
237                                      int size, int ofs)
238 {
239     int ret;
240 
241     trace_qemu_vfio_pci_read_config(buf, ofs, size,
242                                     s->config_region_info.offset,
243                                     s->config_region_info.size);
244     assert(QEMU_IS_ALIGNED(s->config_region_info.offset + ofs, size));
245     ret = RETRY_ON_EINTR(
246         pread(s->device, buf, size, s->config_region_info.offset + ofs)
247     );
248     return ret == size ? 0 : -errno;
249 }
250 
qemu_vfio_pci_write_config(QEMUVFIOState * s,void * buf,int size,int ofs)251 static int qemu_vfio_pci_write_config(QEMUVFIOState *s, void *buf, int size, int ofs)
252 {
253     int ret;
254 
255     trace_qemu_vfio_pci_write_config(buf, ofs, size,
256                                      s->config_region_info.offset,
257                                      s->config_region_info.size);
258     assert(QEMU_IS_ALIGNED(s->config_region_info.offset + ofs, size));
259     ret = RETRY_ON_EINTR(
260         pwrite(s->device, buf, size, s->config_region_info.offset + ofs)
261     );
262     return ret == size ? 0 : -errno;
263 }
264 
collect_usable_iova_ranges(QEMUVFIOState * s,void * buf)265 static void collect_usable_iova_ranges(QEMUVFIOState *s, void *buf)
266 {
267     struct vfio_iommu_type1_info *info = (struct vfio_iommu_type1_info *)buf;
268     struct vfio_info_cap_header *cap = (void *)buf + info->cap_offset;
269     struct vfio_iommu_type1_info_cap_iova_range *cap_iova_range;
270     int i;
271 
272     while (cap->id != VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE) {
273         if (!cap->next) {
274             return;
275         }
276         cap = buf + cap->next;
277     }
278 
279     cap_iova_range = (struct vfio_iommu_type1_info_cap_iova_range *)cap;
280 
281     s->nb_iova_ranges = cap_iova_range->nr_iovas;
282     if (s->nb_iova_ranges > 1) {
283         s->usable_iova_ranges =
284             g_renew(struct IOVARange, s->usable_iova_ranges,
285                     s->nb_iova_ranges);
286     }
287 
288     for (i = 0; i < s->nb_iova_ranges; i++) {
289         s->usable_iova_ranges[i].start = cap_iova_range->iova_ranges[i].start;
290         s->usable_iova_ranges[i].end = cap_iova_range->iova_ranges[i].end;
291     }
292 }
293 
qemu_vfio_init_pci(QEMUVFIOState * s,const char * device,Error ** errp)294 static int qemu_vfio_init_pci(QEMUVFIOState *s, const char *device,
295                               Error **errp)
296 {
297     int ret;
298     int i;
299     uint16_t pci_cmd;
300     struct vfio_group_status group_status = { .argsz = sizeof(group_status) };
301     struct vfio_iommu_type1_info *iommu_info = NULL;
302     size_t iommu_info_size = sizeof(*iommu_info);
303     struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
304     char *group_file = NULL;
305 
306     s->usable_iova_ranges = NULL;
307 
308     /* Create a new container */
309     s->container = open("/dev/vfio/vfio", O_RDWR);
310 
311     if (s->container == -1) {
312         error_setg_errno(errp, errno, "Failed to open /dev/vfio/vfio");
313         return -errno;
314     }
315     if (ioctl(s->container, VFIO_GET_API_VERSION) != VFIO_API_VERSION) {
316         error_setg(errp, "Invalid VFIO version");
317         ret = -EINVAL;
318         goto fail_container;
319     }
320 
321     if (!ioctl(s->container, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
322         error_setg_errno(errp, errno, "VFIO IOMMU Type1 is not supported");
323         ret = -EINVAL;
324         goto fail_container;
325     }
326 
327     /* Open the group */
328     group_file = sysfs_find_group_file(device, errp);
329     if (!group_file) {
330         ret = -EINVAL;
331         goto fail_container;
332     }
333 
334     s->group = open(group_file, O_RDWR);
335     if (s->group == -1) {
336         error_setg_errno(errp, errno, "Failed to open VFIO group file: %s",
337                          group_file);
338         g_free(group_file);
339         ret = -errno;
340         goto fail_container;
341     }
342     g_free(group_file);
343 
344     /* Test the group is viable and available */
345     if (ioctl(s->group, VFIO_GROUP_GET_STATUS, &group_status)) {
346         error_setg_errno(errp, errno, "Failed to get VFIO group status");
347         ret = -errno;
348         goto fail;
349     }
350 
351     if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
352         error_setg(errp, "VFIO group is not viable");
353         ret = -EINVAL;
354         goto fail;
355     }
356 
357     /* Add the group to the container */
358     if (ioctl(s->group, VFIO_GROUP_SET_CONTAINER, &s->container)) {
359         error_setg_errno(errp, errno, "Failed to add group to VFIO container");
360         ret = -errno;
361         goto fail;
362     }
363 
364     /* Enable the IOMMU model we want */
365     if (ioctl(s->container, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)) {
366         error_setg_errno(errp, errno, "Failed to set VFIO IOMMU type");
367         ret = -errno;
368         goto fail;
369     }
370 
371     iommu_info = g_malloc0(iommu_info_size);
372     iommu_info->argsz = iommu_info_size;
373 
374     /* Get additional IOMMU info */
375     if (ioctl(s->container, VFIO_IOMMU_GET_INFO, iommu_info)) {
376         error_setg_errno(errp, errno, "Failed to get IOMMU info");
377         ret = -errno;
378         goto fail;
379     }
380 
381     /*
382      * if the kernel does not report usable IOVA regions, choose
383      * the legacy [QEMU_VFIO_IOVA_MIN, QEMU_VFIO_IOVA_MAX -1] region
384      */
385     s->nb_iova_ranges = 1;
386     s->usable_iova_ranges = g_new0(struct IOVARange, 1);
387     s->usable_iova_ranges[0].start = QEMU_VFIO_IOVA_MIN;
388     s->usable_iova_ranges[0].end = QEMU_VFIO_IOVA_MAX - 1;
389 
390     if (iommu_info->argsz > iommu_info_size) {
391         iommu_info_size = iommu_info->argsz;
392         iommu_info = g_realloc(iommu_info, iommu_info_size);
393         if (ioctl(s->container, VFIO_IOMMU_GET_INFO, iommu_info)) {
394             ret = -errno;
395             goto fail;
396         }
397         collect_usable_iova_ranges(s, iommu_info);
398     }
399 
400     s->device = ioctl(s->group, VFIO_GROUP_GET_DEVICE_FD, device);
401 
402     if (s->device < 0) {
403         error_setg_errno(errp, errno, "Failed to get device fd");
404         ret = -errno;
405         goto fail;
406     }
407 
408     /* Test and setup the device */
409     if (ioctl(s->device, VFIO_DEVICE_GET_INFO, &device_info)) {
410         error_setg_errno(errp, errno, "Failed to get device info");
411         ret = -errno;
412         goto fail;
413     }
414 
415     if (device_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX) {
416         error_setg(errp, "Invalid device regions");
417         ret = -EINVAL;
418         goto fail;
419     }
420 
421     s->config_region_info = (struct vfio_region_info) {
422         .index = VFIO_PCI_CONFIG_REGION_INDEX,
423         .argsz = sizeof(struct vfio_region_info),
424     };
425     if (ioctl(s->device, VFIO_DEVICE_GET_REGION_INFO, &s->config_region_info)) {
426         error_setg_errno(errp, errno, "Failed to get config region info");
427         ret = -errno;
428         goto fail;
429     }
430     trace_qemu_vfio_region_info("config", s->config_region_info.offset,
431                                 s->config_region_info.size,
432                                 s->config_region_info.cap_offset);
433 
434     for (i = 0; i < ARRAY_SIZE(s->bar_region_info); i++) {
435         ret = qemu_vfio_pci_init_bar(s, i, errp);
436         if (ret) {
437             goto fail;
438         }
439     }
440 
441     /* Enable bus master */
442     ret = qemu_vfio_pci_read_config(s, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND);
443     if (ret) {
444         goto fail;
445     }
446     pci_cmd |= PCI_COMMAND_MASTER;
447     ret = qemu_vfio_pci_write_config(s, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND);
448     if (ret) {
449         goto fail;
450     }
451     g_free(iommu_info);
452     return 0;
453 fail:
454     g_free(s->usable_iova_ranges);
455     s->usable_iova_ranges = NULL;
456     s->nb_iova_ranges = 0;
457     g_free(iommu_info);
458     close(s->group);
459 fail_container:
460     close(s->container);
461     return ret;
462 }
463 
qemu_vfio_ram_block_added(RAMBlockNotifier * n,void * host,size_t size,size_t max_size)464 static void qemu_vfio_ram_block_added(RAMBlockNotifier *n, void *host,
465                                       size_t size, size_t max_size)
466 {
467     QEMUVFIOState *s = container_of(n, QEMUVFIOState, ram_notifier);
468     Error *local_err = NULL;
469     int ret;
470 
471     trace_qemu_vfio_ram_block_added(s, host, max_size);
472     ret = qemu_vfio_dma_map(s, host, max_size, false, NULL, &local_err);
473     if (ret) {
474         error_reportf_err(local_err,
475                           "qemu_vfio_dma_map(%p, %zu) failed: ",
476                           host, max_size);
477     }
478 }
479 
qemu_vfio_ram_block_removed(RAMBlockNotifier * n,void * host,size_t size,size_t max_size)480 static void qemu_vfio_ram_block_removed(RAMBlockNotifier *n, void *host,
481                                         size_t size, size_t max_size)
482 {
483     QEMUVFIOState *s = container_of(n, QEMUVFIOState, ram_notifier);
484     if (host) {
485         trace_qemu_vfio_ram_block_removed(s, host, max_size);
486         qemu_vfio_dma_unmap(s, host);
487     }
488 }
489 
qemu_vfio_open_common(QEMUVFIOState * s)490 static void qemu_vfio_open_common(QEMUVFIOState *s)
491 {
492     qemu_mutex_init(&s->lock);
493     s->ram_notifier.ram_block_added = qemu_vfio_ram_block_added;
494     s->ram_notifier.ram_block_removed = qemu_vfio_ram_block_removed;
495     s->low_water_mark = QEMU_VFIO_IOVA_MIN;
496     s->high_water_mark = QEMU_VFIO_IOVA_MAX;
497     ram_block_notifier_add(&s->ram_notifier);
498 }
499 
500 /**
501  * Open a PCI device, e.g. "0000:00:01.0".
502  */
qemu_vfio_open_pci(const char * device,Error ** errp)503 QEMUVFIOState *qemu_vfio_open_pci(const char *device, Error **errp)
504 {
505     int r;
506     QEMUVFIOState *s = g_new0(QEMUVFIOState, 1);
507 
508     /*
509      * VFIO may pin all memory inside mappings, resulting it in pinning
510      * all memory inside RAM blocks unconditionally.
511      */
512     r = ram_block_discard_disable(true);
513     if (r) {
514         error_setg_errno(errp, -r, "Cannot set discarding of RAM broken");
515         g_free(s);
516         return NULL;
517     }
518 
519     r = qemu_vfio_init_pci(s, device, errp);
520     if (r) {
521         ram_block_discard_disable(false);
522         g_free(s);
523         return NULL;
524     }
525     qemu_vfio_open_common(s);
526     return s;
527 }
528 
qemu_vfio_dump_mappings(QEMUVFIOState * s)529 static void qemu_vfio_dump_mappings(QEMUVFIOState *s)
530 {
531     for (int i = 0; i < s->nr_mappings; ++i) {
532         trace_qemu_vfio_dump_mapping(s->mappings[i].host,
533                                      s->mappings[i].iova,
534                                      s->mappings[i].size);
535     }
536 }
537 
538 /**
539  * Find the mapping entry that contains [host, host + size) and set @index to
540  * the position. If no entry contains it, @index is the position _after_ which
541  * to insert the new mapping. IOW, it is the index of the largest element that
542  * is smaller than @host, or -1 if no entry is.
543  */
qemu_vfio_find_mapping(QEMUVFIOState * s,void * host,int * index)544 static IOVAMapping *qemu_vfio_find_mapping(QEMUVFIOState *s, void *host,
545                                            int *index)
546 {
547     IOVAMapping *p = s->mappings;
548     IOVAMapping *q = p ? p + s->nr_mappings - 1 : NULL;
549     IOVAMapping *mid;
550     trace_qemu_vfio_find_mapping(s, host);
551     if (!p) {
552         *index = -1;
553         return NULL;
554     }
555     while (true) {
556         mid = p + (q - p) / 2;
557         if (mid == p) {
558             break;
559         }
560         if (mid->host > host) {
561             q = mid;
562         } else if (mid->host < host) {
563             p = mid;
564         } else {
565             break;
566         }
567     }
568     if (mid->host > host) {
569         mid--;
570     } else if (mid < &s->mappings[s->nr_mappings - 1]
571                && (mid + 1)->host <= host) {
572         mid++;
573     }
574     *index = mid - &s->mappings[0];
575     if (mid >= &s->mappings[0] &&
576         mid->host <= host && mid->host + mid->size > host) {
577         assert(mid < &s->mappings[s->nr_mappings]);
578         return mid;
579     }
580     /* At this point *index + 1 is the right position to insert the new
581      * mapping.*/
582     return NULL;
583 }
584 
585 /**
586  * Allocate IOVA and create a new mapping record and insert it in @s.
587  */
qemu_vfio_add_mapping(QEMUVFIOState * s,void * host,size_t size,int index,uint64_t iova)588 static IOVAMapping *qemu_vfio_add_mapping(QEMUVFIOState *s,
589                                           void *host, size_t size,
590                                           int index, uint64_t iova)
591 {
592     int shift;
593     IOVAMapping m = {.host = host, .size = size, .iova = iova};
594     IOVAMapping *insert;
595 
596     assert(QEMU_IS_ALIGNED(size, qemu_real_host_page_size()));
597     assert(QEMU_IS_ALIGNED(s->low_water_mark, qemu_real_host_page_size()));
598     assert(QEMU_IS_ALIGNED(s->high_water_mark, qemu_real_host_page_size()));
599     trace_qemu_vfio_new_mapping(s, host, size, index, iova);
600 
601     assert(index >= 0);
602     s->nr_mappings++;
603     s->mappings = g_renew(IOVAMapping, s->mappings, s->nr_mappings);
604     insert = &s->mappings[index];
605     shift = s->nr_mappings - index - 1;
606     if (shift) {
607         memmove(insert + 1, insert, shift * sizeof(s->mappings[0]));
608     }
609     *insert = m;
610     return insert;
611 }
612 
613 /* Do the DMA mapping with VFIO. */
qemu_vfio_do_mapping(QEMUVFIOState * s,void * host,size_t size,uint64_t iova,Error ** errp)614 static int qemu_vfio_do_mapping(QEMUVFIOState *s, void *host, size_t size,
615                                 uint64_t iova, Error **errp)
616 {
617     struct vfio_iommu_type1_dma_map dma_map = {
618         .argsz = sizeof(dma_map),
619         .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
620         .iova = iova,
621         .vaddr = (uintptr_t)host,
622         .size = size,
623     };
624     trace_qemu_vfio_do_mapping(s, host, iova, size);
625 
626     if (ioctl(s->container, VFIO_IOMMU_MAP_DMA, &dma_map)) {
627         error_setg_errno(errp, errno, "VFIO_MAP_DMA failed");
628         return -errno;
629     }
630     return 0;
631 }
632 
633 /**
634  * Undo the DMA mapping from @s with VFIO, and remove from mapping list.
635  */
qemu_vfio_undo_mapping(QEMUVFIOState * s,IOVAMapping * mapping,Error ** errp)636 static void qemu_vfio_undo_mapping(QEMUVFIOState *s, IOVAMapping *mapping,
637                                    Error **errp)
638 {
639     int index;
640     struct vfio_iommu_type1_dma_unmap unmap = {
641         .argsz = sizeof(unmap),
642         .flags = 0,
643         .iova = mapping->iova,
644         .size = mapping->size,
645     };
646 
647     index = mapping - s->mappings;
648     assert(mapping->size > 0);
649     assert(QEMU_IS_ALIGNED(mapping->size, qemu_real_host_page_size()));
650     assert(index >= 0 && index < s->nr_mappings);
651     if (ioctl(s->container, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
652         error_setg_errno(errp, errno, "VFIO_UNMAP_DMA failed");
653     }
654     memmove(mapping, &s->mappings[index + 1],
655             sizeof(s->mappings[0]) * (s->nr_mappings - index - 1));
656     s->nr_mappings--;
657     s->mappings = g_renew(IOVAMapping, s->mappings, s->nr_mappings);
658 }
659 
660 /* Check if the mapping list is (ascending) ordered. */
qemu_vfio_verify_mappings(QEMUVFIOState * s)661 static bool qemu_vfio_verify_mappings(QEMUVFIOState *s)
662 {
663     int i;
664     if (QEMU_VFIO_DEBUG) {
665         for (i = 0; i < s->nr_mappings - 1; ++i) {
666             if (!(s->mappings[i].host < s->mappings[i + 1].host)) {
667                 error_report("item %d not sorted!", i);
668                 qemu_vfio_dump_mappings(s);
669                 return false;
670             }
671             if (!(s->mappings[i].host + s->mappings[i].size <=
672                   s->mappings[i + 1].host)) {
673                 error_report("item %d overlap with next!", i);
674                 qemu_vfio_dump_mappings(s);
675                 return false;
676             }
677         }
678     }
679     return true;
680 }
681 
qemu_vfio_find_fixed_iova(QEMUVFIOState * s,size_t size,uint64_t * iova,Error ** errp)682 static bool qemu_vfio_find_fixed_iova(QEMUVFIOState *s, size_t size,
683                                       uint64_t *iova, Error **errp)
684 {
685     int i;
686 
687     for (i = 0; i < s->nb_iova_ranges; i++) {
688         if (s->usable_iova_ranges[i].end < s->low_water_mark) {
689             continue;
690         }
691         s->low_water_mark =
692             MAX(s->low_water_mark, s->usable_iova_ranges[i].start);
693 
694         if (s->usable_iova_ranges[i].end - s->low_water_mark + 1 >= size ||
695             s->usable_iova_ranges[i].end - s->low_water_mark + 1 == 0) {
696             *iova = s->low_water_mark;
697             s->low_water_mark += size;
698             return true;
699         }
700     }
701     error_setg(errp, "fixed iova range not found");
702 
703     return false;
704 }
705 
qemu_vfio_find_temp_iova(QEMUVFIOState * s,size_t size,uint64_t * iova,Error ** errp)706 static bool qemu_vfio_find_temp_iova(QEMUVFIOState *s, size_t size,
707                                      uint64_t *iova, Error **errp)
708 {
709     int i;
710 
711     for (i = s->nb_iova_ranges - 1; i >= 0; i--) {
712         if (s->usable_iova_ranges[i].start > s->high_water_mark) {
713             continue;
714         }
715         s->high_water_mark =
716             MIN(s->high_water_mark, s->usable_iova_ranges[i].end + 1);
717 
718         if (s->high_water_mark - s->usable_iova_ranges[i].start + 1 >= size ||
719             s->high_water_mark - s->usable_iova_ranges[i].start + 1 == 0) {
720             *iova = s->high_water_mark - size;
721             s->high_water_mark = *iova;
722             return true;
723         }
724     }
725     error_setg(errp, "temporary iova range not found");
726 
727     return false;
728 }
729 
730 /**
731  * qemu_vfio_water_mark_reached:
732  *
733  * Returns %true if high watermark has been reached, %false otherwise.
734  */
qemu_vfio_water_mark_reached(QEMUVFIOState * s,size_t size,Error ** errp)735 static bool qemu_vfio_water_mark_reached(QEMUVFIOState *s, size_t size,
736                                          Error **errp)
737 {
738     if (s->high_water_mark - s->low_water_mark + 1 < size) {
739         error_setg(errp, "iova exhausted (water mark reached)");
740         return true;
741     }
742     return false;
743 }
744 
745 /* Map [host, host + size) area into a contiguous IOVA address space, and store
746  * the result in @iova if not NULL. The caller need to make sure the area is
747  * aligned to page size, and mustn't overlap with existing mapping areas (split
748  * mapping status within this area is not allowed).
749  */
qemu_vfio_dma_map(QEMUVFIOState * s,void * host,size_t size,bool temporary,uint64_t * iova,Error ** errp)750 int qemu_vfio_dma_map(QEMUVFIOState *s, void *host, size_t size,
751                       bool temporary, uint64_t *iova, Error **errp)
752 {
753     int index;
754     IOVAMapping *mapping;
755     uint64_t iova0;
756 
757     assert(QEMU_PTR_IS_ALIGNED(host, qemu_real_host_page_size()));
758     assert(QEMU_IS_ALIGNED(size, qemu_real_host_page_size()));
759     trace_qemu_vfio_dma_map(s, host, size, temporary, iova);
760     QEMU_LOCK_GUARD(&s->lock);
761     mapping = qemu_vfio_find_mapping(s, host, &index);
762     if (mapping) {
763         iova0 = mapping->iova + ((uint8_t *)host - (uint8_t *)mapping->host);
764     } else {
765         int ret;
766 
767         if (qemu_vfio_water_mark_reached(s, size, errp)) {
768             return -ENOMEM;
769         }
770         if (!temporary) {
771             if (!qemu_vfio_find_fixed_iova(s, size, &iova0, errp)) {
772                 return -ENOMEM;
773             }
774 
775             mapping = qemu_vfio_add_mapping(s, host, size, index + 1, iova0);
776             assert(qemu_vfio_verify_mappings(s));
777             ret = qemu_vfio_do_mapping(s, host, size, iova0, errp);
778             if (ret < 0) {
779                 qemu_vfio_undo_mapping(s, mapping, NULL);
780                 return ret;
781             }
782             qemu_vfio_dump_mappings(s);
783         } else {
784             if (!qemu_vfio_find_temp_iova(s, size, &iova0, errp)) {
785                 return -ENOMEM;
786             }
787             ret = qemu_vfio_do_mapping(s, host, size, iova0, errp);
788             if (ret < 0) {
789                 return ret;
790             }
791         }
792     }
793     trace_qemu_vfio_dma_mapped(s, host, iova0, size);
794     if (iova) {
795         *iova = iova0;
796     }
797     return 0;
798 }
799 
800 /* Reset the high watermark and free all "temporary" mappings. */
qemu_vfio_dma_reset_temporary(QEMUVFIOState * s)801 int qemu_vfio_dma_reset_temporary(QEMUVFIOState *s)
802 {
803     struct vfio_iommu_type1_dma_unmap unmap = {
804         .argsz = sizeof(unmap),
805         .flags = 0,
806         .iova = s->high_water_mark,
807         .size = QEMU_VFIO_IOVA_MAX - s->high_water_mark,
808     };
809     trace_qemu_vfio_dma_reset_temporary(s);
810     QEMU_LOCK_GUARD(&s->lock);
811     if (ioctl(s->container, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
812         error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno));
813         return -errno;
814     }
815     s->high_water_mark = QEMU_VFIO_IOVA_MAX;
816     return 0;
817 }
818 
819 /* Unmapping the whole area that was previously mapped with
820  * qemu_vfio_dma_map(). */
qemu_vfio_dma_unmap(QEMUVFIOState * s,void * host)821 void qemu_vfio_dma_unmap(QEMUVFIOState *s, void *host)
822 {
823     int index = 0;
824     IOVAMapping *m;
825 
826     if (!host) {
827         return;
828     }
829 
830     trace_qemu_vfio_dma_unmap(s, host);
831     QEMU_LOCK_GUARD(&s->lock);
832     m = qemu_vfio_find_mapping(s, host, &index);
833     if (!m) {
834         return;
835     }
836     qemu_vfio_undo_mapping(s, m, NULL);
837 }
838 
qemu_vfio_reset(QEMUVFIOState * s)839 static void qemu_vfio_reset(QEMUVFIOState *s)
840 {
841     ioctl(s->device, VFIO_DEVICE_RESET);
842 }
843 
844 /* Close and free the VFIO resources. */
qemu_vfio_close(QEMUVFIOState * s)845 void qemu_vfio_close(QEMUVFIOState *s)
846 {
847     int i;
848 
849     if (!s) {
850         return;
851     }
852 
853     ram_block_notifier_remove(&s->ram_notifier);
854 
855     for (i = 0; i < s->nr_mappings; ++i) {
856         qemu_vfio_undo_mapping(s, &s->mappings[i], NULL);
857     }
858 
859     g_free(s->usable_iova_ranges);
860     s->nb_iova_ranges = 0;
861     qemu_vfio_reset(s);
862     close(s->device);
863     close(s->group);
864     close(s->container);
865     ram_block_discard_disable(false);
866 }
867