xref: /openbmc/qemu/hw/vfio/iommufd.c (revision d76795ea) 
1 /*
2  * iommufd container backend
3  *
4  * Copyright (C) 2023 Intel Corporation.
5  * Copyright Red Hat, Inc. 2023
6  *
7  * Authors: Yi Liu <yi.l.liu@intel.com>
8  *          Eric Auger <eric.auger@redhat.com>
9  *
10  * SPDX-License-Identifier: GPL-2.0-or-later
11  */
12 
13 #include "qemu/osdep.h"
14 #include <sys/ioctl.h>
15 #include <linux/vfio.h>
16 #include <linux/iommufd.h>
17 
18 #include "hw/vfio/vfio-common.h"
19 #include "qemu/error-report.h"
20 #include "trace.h"
21 #include "qapi/error.h"
22 #include "sysemu/iommufd.h"
23 #include "hw/qdev-core.h"
24 #include "sysemu/reset.h"
25 #include "qemu/cutils.h"
26 #include "qemu/chardev_open.h"
27 #include "pci.h"
28 
29 static int iommufd_cdev_map(const VFIOContainerBase *bcontainer, hwaddr iova,
30                             ram_addr_t size, void *vaddr, bool readonly)
31 {
32     const VFIOIOMMUFDContainer *container =
33         container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
34 
35     return iommufd_backend_map_dma(container->be,
36                                    container->ioas_id,
37                                    iova, size, vaddr, readonly);
38 }
39 
40 static int iommufd_cdev_unmap(const VFIOContainerBase *bcontainer,
41                               hwaddr iova, ram_addr_t size,
42                               IOMMUTLBEntry *iotlb)
43 {
44     const VFIOIOMMUFDContainer *container =
45         container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
46 
47     /* TODO: Handle dma_unmap_bitmap with iotlb args (migration) */
48     return iommufd_backend_unmap_dma(container->be,
49                                      container->ioas_id, iova, size);
50 }
51 
52 static bool iommufd_cdev_kvm_device_add(VFIODevice *vbasedev, Error **errp)
53 {
54     return !vfio_kvm_device_add_fd(vbasedev->fd, errp);
55 }
56 
57 static void iommufd_cdev_kvm_device_del(VFIODevice *vbasedev)
58 {
59     Error *err = NULL;
60 
61     if (vfio_kvm_device_del_fd(vbasedev->fd, &err)) {
62         error_report_err(err);
63     }
64 }
65 
66 static bool iommufd_cdev_connect_and_bind(VFIODevice *vbasedev, Error **errp)
67 {
68     IOMMUFDBackend *iommufd = vbasedev->iommufd;
69     struct vfio_device_bind_iommufd bind = {
70         .argsz = sizeof(bind),
71         .flags = 0,
72     };
73 
74     if (!iommufd_backend_connect(iommufd, errp)) {
75         return false;
76     }
77 
78     /*
79      * Add device to kvm-vfio to be prepared for the tracking
80      * in KVM. Especially for some emulated devices, it requires
81      * to have kvm information in the device open.
82      */
83     if (!iommufd_cdev_kvm_device_add(vbasedev, errp)) {
84         goto err_kvm_device_add;
85     }
86 
87     /* Bind device to iommufd */
88     bind.iommufd = iommufd->fd;
89     if (ioctl(vbasedev->fd, VFIO_DEVICE_BIND_IOMMUFD, &bind)) {
90         error_setg_errno(errp, errno, "error bind device fd=%d to iommufd=%d",
91                          vbasedev->fd, bind.iommufd);
92         goto err_bind;
93     }
94 
95     vbasedev->devid = bind.out_devid;
96     trace_iommufd_cdev_connect_and_bind(bind.iommufd, vbasedev->name,
97                                         vbasedev->fd, vbasedev->devid);
98     return true;
99 err_bind:
100     iommufd_cdev_kvm_device_del(vbasedev);
101 err_kvm_device_add:
102     iommufd_backend_disconnect(iommufd);
103     return false;
104 }
105 
106 static void iommufd_cdev_unbind_and_disconnect(VFIODevice *vbasedev)
107 {
108     /* Unbind is automatically conducted when device fd is closed */
109     iommufd_cdev_kvm_device_del(vbasedev);
110     iommufd_backend_disconnect(vbasedev->iommufd);
111 }
112 
113 static int iommufd_cdev_getfd(const char *sysfs_path, Error **errp)
114 {
115     ERRP_GUARD();
116     long int ret = -ENOTTY;
117     g_autofree char *path = NULL;
118     g_autofree char *vfio_dev_path = NULL;
119     g_autofree char *vfio_path = NULL;
120     DIR *dir = NULL;
121     struct dirent *dent;
122     g_autofree gchar *contents = NULL;
123     gsize length;
124     int major, minor;
125     dev_t vfio_devt;
126 
127     path = g_strdup_printf("%s/vfio-dev", sysfs_path);
128     dir = opendir(path);
129     if (!dir) {
130         error_setg_errno(errp, errno, "couldn't open directory %s", path);
131         goto out;
132     }
133 
134     while ((dent = readdir(dir))) {
135         if (!strncmp(dent->d_name, "vfio", 4)) {
136             vfio_dev_path = g_strdup_printf("%s/%s/dev", path, dent->d_name);
137             break;
138         }
139     }
140 
141     if (!vfio_dev_path) {
142         error_setg(errp, "failed to find vfio-dev/vfioX/dev");
143         goto out_close_dir;
144     }
145 
146     if (!g_file_get_contents(vfio_dev_path, &contents, &length, NULL)) {
147         error_setg(errp, "failed to load \"%s\"", vfio_dev_path);
148         goto out_close_dir;
149     }
150 
151     if (sscanf(contents, "%d:%d", &major, &minor) != 2) {
152         error_setg(errp, "failed to get major:minor for \"%s\"", vfio_dev_path);
153         goto out_close_dir;
154     }
155     vfio_devt = makedev(major, minor);
156 
157     vfio_path = g_strdup_printf("/dev/vfio/devices/%s", dent->d_name);
158     ret = open_cdev(vfio_path, vfio_devt);
159     if (ret < 0) {
160         error_setg(errp, "Failed to open %s", vfio_path);
161     }
162 
163     trace_iommufd_cdev_getfd(vfio_path, ret);
164 
165 out_close_dir:
166     closedir(dir);
167 out:
168     if (*errp) {
169         error_prepend(errp, VFIO_MSG_PREFIX, path);
170     }
171 
172     return ret;
173 }
174 
175 static bool iommufd_cdev_attach_ioas_hwpt(VFIODevice *vbasedev, uint32_t id,
176                                          Error **errp)
177 {
178     int iommufd = vbasedev->iommufd->fd;
179     struct vfio_device_attach_iommufd_pt attach_data = {
180         .argsz = sizeof(attach_data),
181         .flags = 0,
182         .pt_id = id,
183     };
184 
185     /* Attach device to an IOAS or hwpt within iommufd */
186     if (ioctl(vbasedev->fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data)) {
187         error_setg_errno(errp, errno,
188                          "[iommufd=%d] error attach %s (%d) to id=%d",
189                          iommufd, vbasedev->name, vbasedev->fd, id);
190         return false;
191     }
192 
193     trace_iommufd_cdev_attach_ioas_hwpt(iommufd, vbasedev->name,
194                                         vbasedev->fd, id);
195     return true;
196 }
197 
198 static bool iommufd_cdev_detach_ioas_hwpt(VFIODevice *vbasedev, Error **errp)
199 {
200     int iommufd = vbasedev->iommufd->fd;
201     struct vfio_device_detach_iommufd_pt detach_data = {
202         .argsz = sizeof(detach_data),
203         .flags = 0,
204     };
205 
206     if (ioctl(vbasedev->fd, VFIO_DEVICE_DETACH_IOMMUFD_PT, &detach_data)) {
207         error_setg_errno(errp, errno, "detach %s failed", vbasedev->name);
208         return false;
209     }
210 
211     trace_iommufd_cdev_detach_ioas_hwpt(iommufd, vbasedev->name);
212     return true;
213 }
214 
215 static bool iommufd_cdev_attach_container(VFIODevice *vbasedev,
216                                           VFIOIOMMUFDContainer *container,
217                                           Error **errp)
218 {
219     return iommufd_cdev_attach_ioas_hwpt(vbasedev, container->ioas_id, errp);
220 }
221 
222 static void iommufd_cdev_detach_container(VFIODevice *vbasedev,
223                                           VFIOIOMMUFDContainer *container)
224 {
225     Error *err = NULL;
226 
227     if (!iommufd_cdev_detach_ioas_hwpt(vbasedev, &err)) {
228         error_report_err(err);
229     }
230 }
231 
232 static void iommufd_cdev_container_destroy(VFIOIOMMUFDContainer *container)
233 {
234     VFIOContainerBase *bcontainer = &container->bcontainer;
235 
236     if (!QLIST_EMPTY(&bcontainer->device_list)) {
237         return;
238     }
239     memory_listener_unregister(&bcontainer->listener);
240     vfio_container_destroy(bcontainer);
241     iommufd_backend_free_id(container->be, container->ioas_id);
242     g_free(container);
243 }
244 
245 static int iommufd_cdev_ram_block_discard_disable(bool state)
246 {
247     /*
248      * We support coordinated discarding of RAM via the RamDiscardManager.
249      */
250     return ram_block_uncoordinated_discard_disable(state);
251 }
252 
253 static bool iommufd_cdev_get_info_iova_range(VFIOIOMMUFDContainer *container,
254                                              uint32_t ioas_id, Error **errp)
255 {
256     VFIOContainerBase *bcontainer = &container->bcontainer;
257     g_autofree struct iommu_ioas_iova_ranges *info = NULL;
258     struct iommu_iova_range *iova_ranges;
259     int sz, fd = container->be->fd;
260 
261     info = g_malloc0(sizeof(*info));
262     info->size = sizeof(*info);
263     info->ioas_id = ioas_id;
264 
265     if (ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info) && errno != EMSGSIZE) {
266         goto error;
267     }
268 
269     sz = info->num_iovas * sizeof(struct iommu_iova_range);
270     info = g_realloc(info, sizeof(*info) + sz);
271     info->allowed_iovas = (uintptr_t)(info + 1);
272 
273     if (ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info)) {
274         goto error;
275     }
276 
277     iova_ranges = (struct iommu_iova_range *)(uintptr_t)info->allowed_iovas;
278 
279     for (int i = 0; i < info->num_iovas; i++) {
280         Range *range = g_new(Range, 1);
281 
282         range_set_bounds(range, iova_ranges[i].start, iova_ranges[i].last);
283         bcontainer->iova_ranges =
284             range_list_insert(bcontainer->iova_ranges, range);
285     }
286     bcontainer->pgsizes = info->out_iova_alignment;
287 
288     return true;
289 
290 error:
291     error_setg_errno(errp, errno, "Cannot get IOVA ranges");
292     return false;
293 }
294 
295 static bool iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
296                                 AddressSpace *as, Error **errp)
297 {
298     VFIOContainerBase *bcontainer;
299     VFIOIOMMUFDContainer *container;
300     VFIOAddressSpace *space;
301     struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
302     int ret, devfd;
303     uint32_t ioas_id;
304     Error *err = NULL;
305     const VFIOIOMMUClass *iommufd_vioc =
306         VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD));
307 
308     if (vbasedev->fd < 0) {
309         devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
310         if (devfd < 0) {
311             return false;
312         }
313         vbasedev->fd = devfd;
314     } else {
315         devfd = vbasedev->fd;
316     }
317 
318     if (!iommufd_cdev_connect_and_bind(vbasedev, errp)) {
319         goto err_connect_bind;
320     }
321 
322     space = vfio_get_address_space(as);
323 
324     /* try to attach to an existing container in this space */
325     QLIST_FOREACH(bcontainer, &space->containers, next) {
326         container = container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
327         if (bcontainer->ops != iommufd_vioc ||
328             vbasedev->iommufd != container->be) {
329             continue;
330         }
331         if (!iommufd_cdev_attach_container(vbasedev, container, &err)) {
332             const char *msg = error_get_pretty(err);
333 
334             trace_iommufd_cdev_fail_attach_existing_container(msg);
335             error_free(err);
336             err = NULL;
337         } else {
338             ret = iommufd_cdev_ram_block_discard_disable(true);
339             if (ret) {
340                 error_setg(errp,
341                               "Cannot set discarding of RAM broken (%d)", ret);
342                 goto err_discard_disable;
343             }
344             goto found_container;
345         }
346     }
347 
348     /* Need to allocate a new dedicated container */
349     if (!iommufd_backend_alloc_ioas(vbasedev->iommufd, &ioas_id, errp)) {
350         goto err_alloc_ioas;
351     }
352 
353     trace_iommufd_cdev_alloc_ioas(vbasedev->iommufd->fd, ioas_id);
354 
355     container = g_malloc0(sizeof(*container));
356     container->be = vbasedev->iommufd;
357     container->ioas_id = ioas_id;
358 
359     bcontainer = &container->bcontainer;
360     vfio_container_init(bcontainer, space, iommufd_vioc);
361     QLIST_INSERT_HEAD(&space->containers, bcontainer, next);
362 
363     if (!iommufd_cdev_attach_container(vbasedev, container, errp)) {
364         goto err_attach_container;
365     }
366 
367     ret = iommufd_cdev_ram_block_discard_disable(true);
368     if (ret) {
369         goto err_discard_disable;
370     }
371 
372     if (!iommufd_cdev_get_info_iova_range(container, ioas_id, &err)) {
373         error_append_hint(&err,
374                    "Fallback to default 64bit IOVA range and 4K page size\n");
375         warn_report_err(err);
376         err = NULL;
377         bcontainer->pgsizes = qemu_real_host_page_size();
378     }
379 
380     bcontainer->listener = vfio_memory_listener;
381     memory_listener_register(&bcontainer->listener, bcontainer->space->as);
382 
383     if (bcontainer->error) {
384         error_propagate_prepend(errp, bcontainer->error,
385                                 "memory listener initialization failed: ");
386         goto err_listener_register;
387     }
388 
389     bcontainer->initialized = true;
390 
391 found_container:
392     ret = ioctl(devfd, VFIO_DEVICE_GET_INFO, &dev_info);
393     if (ret) {
394         error_setg_errno(errp, errno, "error getting device info");
395         goto err_listener_register;
396     }
397 
398     if (!vfio_cpr_register_container(bcontainer, errp)) {
399         goto err_listener_register;
400     }
401 
402     /*
403      * TODO: examine RAM_BLOCK_DISCARD stuff, should we do group level
404      * for discarding incompatibility check as well?
405      */
406     if (vbasedev->ram_block_discard_allowed) {
407         iommufd_cdev_ram_block_discard_disable(false);
408     }
409 
410     vbasedev->group = 0;
411     vbasedev->num_irqs = dev_info.num_irqs;
412     vbasedev->num_regions = dev_info.num_regions;
413     vbasedev->flags = dev_info.flags;
414     vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
415     vbasedev->bcontainer = bcontainer;
416     QLIST_INSERT_HEAD(&bcontainer->device_list, vbasedev, container_next);
417     QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next);
418 
419     trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev->num_irqs,
420                                    vbasedev->num_regions, vbasedev->flags);
421     return true;
422 
423 err_listener_register:
424     iommufd_cdev_ram_block_discard_disable(false);
425 err_discard_disable:
426     iommufd_cdev_detach_container(vbasedev, container);
427 err_attach_container:
428     iommufd_cdev_container_destroy(container);
429 err_alloc_ioas:
430     vfio_put_address_space(space);
431     iommufd_cdev_unbind_and_disconnect(vbasedev);
432 err_connect_bind:
433     close(vbasedev->fd);
434     return false;
435 }
436 
437 static void iommufd_cdev_detach(VFIODevice *vbasedev)
438 {
439     VFIOContainerBase *bcontainer = vbasedev->bcontainer;
440     VFIOAddressSpace *space = bcontainer->space;
441     VFIOIOMMUFDContainer *container = container_of(bcontainer,
442                                                    VFIOIOMMUFDContainer,
443                                                    bcontainer);
444     QLIST_REMOVE(vbasedev, global_next);
445     QLIST_REMOVE(vbasedev, container_next);
446     vbasedev->bcontainer = NULL;
447 
448     if (!vbasedev->ram_block_discard_allowed) {
449         iommufd_cdev_ram_block_discard_disable(false);
450     }
451 
452     vfio_cpr_unregister_container(bcontainer);
453     iommufd_cdev_detach_container(vbasedev, container);
454     iommufd_cdev_container_destroy(container);
455     vfio_put_address_space(space);
456 
457     iommufd_cdev_unbind_and_disconnect(vbasedev);
458     close(vbasedev->fd);
459 }
460 
461 static VFIODevice *iommufd_cdev_pci_find_by_devid(__u32 devid)
462 {
463     VFIODevice *vbasedev_iter;
464     const VFIOIOMMUClass *iommufd_vioc =
465         VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD));
466 
467     QLIST_FOREACH(vbasedev_iter, &vfio_device_list, global_next) {
468         if (vbasedev_iter->bcontainer->ops != iommufd_vioc) {
469             continue;
470         }
471         if (devid == vbasedev_iter->devid) {
472             return vbasedev_iter;
473         }
474     }
475     return NULL;
476 }
477 
478 static VFIOPCIDevice *
479 iommufd_cdev_dep_get_realized_vpdev(struct vfio_pci_dependent_device *dep_dev,
480                                     VFIODevice *reset_dev)
481 {
482     VFIODevice *vbasedev_tmp;
483 
484     if (dep_dev->devid == reset_dev->devid ||
485         dep_dev->devid == VFIO_PCI_DEVID_OWNED) {
486         return NULL;
487     }
488 
489     vbasedev_tmp = iommufd_cdev_pci_find_by_devid(dep_dev->devid);
490     if (!vbasedev_tmp || !vbasedev_tmp->dev->realized ||
491         vbasedev_tmp->type != VFIO_DEVICE_TYPE_PCI) {
492         return NULL;
493     }
494 
495     return container_of(vbasedev_tmp, VFIOPCIDevice, vbasedev);
496 }
497 
498 static int iommufd_cdev_pci_hot_reset(VFIODevice *vbasedev, bool single)
499 {
500     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
501     struct vfio_pci_hot_reset_info *info = NULL;
502     struct vfio_pci_dependent_device *devices;
503     struct vfio_pci_hot_reset *reset;
504     int ret, i;
505     bool multi = false;
506 
507     trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
508 
509     if (!single) {
510         vfio_pci_pre_reset(vdev);
511     }
512     vdev->vbasedev.needs_reset = false;
513 
514     ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
515 
516     if (ret) {
517         goto out_single;
518     }
519 
520     assert(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID);
521 
522     devices = &info->devices[0];
523 
524     if (!(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED)) {
525         if (!vdev->has_pm_reset) {
526             for (i = 0; i < info->count; i++) {
527                 if (devices[i].devid == VFIO_PCI_DEVID_NOT_OWNED) {
528                     error_report("vfio: Cannot reset device %s, "
529                                  "depends on device %04x:%02x:%02x.%x "
530                                  "which is not owned.",
531                                  vdev->vbasedev.name, devices[i].segment,
532                                  devices[i].bus, PCI_SLOT(devices[i].devfn),
533                                  PCI_FUNC(devices[i].devfn));
534                 }
535             }
536         }
537         ret = -EPERM;
538         goto out_single;
539     }
540 
541     trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
542 
543     for (i = 0; i < info->count; i++) {
544         VFIOPCIDevice *tmp;
545 
546         trace_iommufd_cdev_pci_hot_reset_dep_devices(devices[i].segment,
547                                                      devices[i].bus,
548                                                      PCI_SLOT(devices[i].devfn),
549                                                      PCI_FUNC(devices[i].devfn),
550                                                      devices[i].devid);
551 
552         /*
553          * If a VFIO cdev device is resettable, all the dependent devices
554          * are either bound to same iommufd or within same iommu_groups as
555          * one of the iommufd bound devices.
556          */
557         assert(devices[i].devid != VFIO_PCI_DEVID_NOT_OWNED);
558 
559         tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
560         if (!tmp) {
561             continue;
562         }
563 
564         if (single) {
565             ret = -EINVAL;
566             goto out_single;
567         }
568         vfio_pci_pre_reset(tmp);
569         tmp->vbasedev.needs_reset = false;
570         multi = true;
571     }
572 
573     if (!single && !multi) {
574         ret = -EINVAL;
575         goto out_single;
576     }
577 
578     /* Use zero length array for hot reset with iommufd backend */
579     reset = g_malloc0(sizeof(*reset));
580     reset->argsz = sizeof(*reset);
581 
582      /* Bus reset! */
583     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
584     g_free(reset);
585     if (ret) {
586         ret = -errno;
587     }
588 
589     trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
590                                     ret ? strerror(errno) : "Success");
591 
592     /* Re-enable INTx on affected devices */
593     for (i = 0; i < info->count; i++) {
594         VFIOPCIDevice *tmp;
595 
596         tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
597         if (!tmp) {
598             continue;
599         }
600         vfio_pci_post_reset(tmp);
601     }
602 out_single:
603     if (!single) {
604         vfio_pci_post_reset(vdev);
605     }
606     g_free(info);
607 
608     return ret;
609 }
610 
611 static void vfio_iommu_iommufd_class_init(ObjectClass *klass, void *data)
612 {
613     VFIOIOMMUClass *vioc = VFIO_IOMMU_CLASS(klass);
614 
615     vioc->dma_map = iommufd_cdev_map;
616     vioc->dma_unmap = iommufd_cdev_unmap;
617     vioc->attach_device = iommufd_cdev_attach;
618     vioc->detach_device = iommufd_cdev_detach;
619     vioc->pci_hot_reset = iommufd_cdev_pci_hot_reset;
620 };
621 
622 static const TypeInfo types[] = {
623     {
624         .name = TYPE_VFIO_IOMMU_IOMMUFD,
625         .parent = TYPE_VFIO_IOMMU,
626         .class_init = vfio_iommu_iommufd_class_init,
627     },
628 };
629 
630 DEFINE_TYPES(types)
631