xref: /openbmc/qemu/hw/vfio/pci.c (revision ea363626)
1 /*
2  * vfio based device assignment support
3  *
4  * Copyright Red Hat, Inc. 2012
5  *
6  * Authors:
7  *  Alex Williamson <alex.williamson@redhat.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  * Based on qemu-kvm device-assignment:
13  *  Adapted for KVM by Qumranet.
14  *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15  *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16  *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17  *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19  */
20 
21 #include "qemu/osdep.h"
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
24 
25 #include "hw/hw.h"
26 #include "hw/pci/msi.h"
27 #include "hw/pci/msix.h"
28 #include "hw/pci/pci_bridge.h"
29 #include "hw/qdev-properties.h"
30 #include "hw/qdev-properties-system.h"
31 #include "migration/vmstate.h"
32 #include "qapi/qmp/qdict.h"
33 #include "qemu/error-report.h"
34 #include "qemu/main-loop.h"
35 #include "qemu/module.h"
36 #include "qemu/range.h"
37 #include "qemu/units.h"
38 #include "sysemu/kvm.h"
39 #include "sysemu/runstate.h"
40 #include "pci.h"
41 #include "trace.h"
42 #include "qapi/error.h"
43 #include "migration/blocker.h"
44 #include "migration/qemu-file.h"
45 
46 #define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
47 
48 /* Protected by BQL */
49 static KVMRouteChange vfio_route_change;
50 
51 static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
52 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
53 static void vfio_msi_disable_common(VFIOPCIDevice *vdev);
54 
55 /*
56  * Disabling BAR mmaping can be slow, but toggling it around INTx can
57  * also be a huge overhead.  We try to get the best of both worlds by
58  * waiting until an interrupt to disable mmaps (subsequent transitions
59  * to the same state are effectively no overhead).  If the interrupt has
60  * been serviced and the time gap is long enough, we re-enable mmaps for
61  * performance.  This works well for things like graphics cards, which
62  * may not use their interrupt at all and are penalized to an unusable
63  * level by read/write BAR traps.  Other devices, like NICs, have more
64  * regular interrupts and see much better latency by staying in non-mmap
65  * mode.  We therefore set the default mmap_timeout such that a ping
66  * is just enough to keep the mmap disabled.  Users can experiment with
67  * other options with the x-intx-mmap-timeout-ms parameter (a value of
68  * zero disables the timer).
69  */
70 static void vfio_intx_mmap_enable(void *opaque)
71 {
72     VFIOPCIDevice *vdev = opaque;
73 
74     if (vdev->intx.pending) {
75         timer_mod(vdev->intx.mmap_timer,
76                        qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
77         return;
78     }
79 
80     vfio_mmap_set_enabled(vdev, true);
81 }
82 
83 static void vfio_intx_interrupt(void *opaque)
84 {
85     VFIOPCIDevice *vdev = opaque;
86 
87     if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
88         return;
89     }
90 
91     trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
92 
93     vdev->intx.pending = true;
94     pci_irq_assert(&vdev->pdev);
95     vfio_mmap_set_enabled(vdev, false);
96     if (vdev->intx.mmap_timeout) {
97         timer_mod(vdev->intx.mmap_timer,
98                        qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
99     }
100 }
101 
102 static void vfio_intx_eoi(VFIODevice *vbasedev)
103 {
104     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
105 
106     if (!vdev->intx.pending) {
107         return;
108     }
109 
110     trace_vfio_intx_eoi(vbasedev->name);
111 
112     vdev->intx.pending = false;
113     pci_irq_deassert(&vdev->pdev);
114     vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
115 }
116 
117 static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp)
118 {
119 #ifdef CONFIG_KVM
120     int irq_fd = event_notifier_get_fd(&vdev->intx.interrupt);
121 
122     if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
123         vdev->intx.route.mode != PCI_INTX_ENABLED ||
124         !kvm_resamplefds_enabled()) {
125         return;
126     }
127 
128     /* Get to a known interrupt state */
129     qemu_set_fd_handler(irq_fd, NULL, NULL, vdev);
130     vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
131     vdev->intx.pending = false;
132     pci_irq_deassert(&vdev->pdev);
133 
134     /* Get an eventfd for resample/unmask */
135     if (event_notifier_init(&vdev->intx.unmask, 0)) {
136         error_setg(errp, "event_notifier_init failed eoi");
137         goto fail;
138     }
139 
140     if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
141                                            &vdev->intx.interrupt,
142                                            &vdev->intx.unmask,
143                                            vdev->intx.route.irq)) {
144         error_setg_errno(errp, errno, "failed to setup resample irqfd");
145         goto fail_irqfd;
146     }
147 
148     if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
149                                VFIO_IRQ_SET_ACTION_UNMASK,
150                                event_notifier_get_fd(&vdev->intx.unmask),
151                                errp)) {
152         goto fail_vfio;
153     }
154 
155     /* Let'em rip */
156     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
157 
158     vdev->intx.kvm_accel = true;
159 
160     trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
161 
162     return;
163 
164 fail_vfio:
165     kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
166                                           vdev->intx.route.irq);
167 fail_irqfd:
168     event_notifier_cleanup(&vdev->intx.unmask);
169 fail:
170     qemu_set_fd_handler(irq_fd, vfio_intx_interrupt, NULL, vdev);
171     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
172 #endif
173 }
174 
175 static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
176 {
177 #ifdef CONFIG_KVM
178     if (!vdev->intx.kvm_accel) {
179         return;
180     }
181 
182     /*
183      * Get to a known state, hardware masked, QEMU ready to accept new
184      * interrupts, QEMU IRQ de-asserted.
185      */
186     vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
187     vdev->intx.pending = false;
188     pci_irq_deassert(&vdev->pdev);
189 
190     /* Tell KVM to stop listening for an INTx irqfd */
191     if (kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vdev->intx.interrupt,
192                                               vdev->intx.route.irq)) {
193         error_report("vfio: Error: Failed to disable INTx irqfd: %m");
194     }
195 
196     /* We only need to close the eventfd for VFIO to cleanup the kernel side */
197     event_notifier_cleanup(&vdev->intx.unmask);
198 
199     /* QEMU starts listening for interrupt events. */
200     qemu_set_fd_handler(event_notifier_get_fd(&vdev->intx.interrupt),
201                         vfio_intx_interrupt, NULL, vdev);
202 
203     vdev->intx.kvm_accel = false;
204 
205     /* If we've missed an event, let it re-fire through QEMU */
206     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
207 
208     trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
209 #endif
210 }
211 
212 static void vfio_intx_update(VFIOPCIDevice *vdev, PCIINTxRoute *route)
213 {
214     Error *err = NULL;
215 
216     trace_vfio_intx_update(vdev->vbasedev.name,
217                            vdev->intx.route.irq, route->irq);
218 
219     vfio_intx_disable_kvm(vdev);
220 
221     vdev->intx.route = *route;
222 
223     if (route->mode != PCI_INTX_ENABLED) {
224         return;
225     }
226 
227     vfio_intx_enable_kvm(vdev, &err);
228     if (err) {
229         warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
230     }
231 
232     /* Re-enable the interrupt in cased we missed an EOI */
233     vfio_intx_eoi(&vdev->vbasedev);
234 }
235 
236 static void vfio_intx_routing_notifier(PCIDevice *pdev)
237 {
238     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
239     PCIINTxRoute route;
240 
241     if (vdev->interrupt != VFIO_INT_INTx) {
242         return;
243     }
244 
245     route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
246 
247     if (pci_intx_route_changed(&vdev->intx.route, &route)) {
248         vfio_intx_update(vdev, &route);
249     }
250 }
251 
252 static void vfio_irqchip_change(Notifier *notify, void *data)
253 {
254     VFIOPCIDevice *vdev = container_of(notify, VFIOPCIDevice,
255                                        irqchip_change_notifier);
256 
257     vfio_intx_update(vdev, &vdev->intx.route);
258 }
259 
260 static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp)
261 {
262     uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
263     Error *err = NULL;
264     int32_t fd;
265     int ret;
266 
267 
268     if (!pin) {
269         return 0;
270     }
271 
272     vfio_disable_interrupts(vdev);
273 
274     vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
275     pci_config_set_interrupt_pin(vdev->pdev.config, pin);
276 
277 #ifdef CONFIG_KVM
278     /*
279      * Only conditional to avoid generating error messages on platforms
280      * where we won't actually use the result anyway.
281      */
282     if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
283         vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
284                                                         vdev->intx.pin);
285     }
286 #endif
287 
288     ret = event_notifier_init(&vdev->intx.interrupt, 0);
289     if (ret) {
290         error_setg_errno(errp, -ret, "event_notifier_init failed");
291         return ret;
292     }
293     fd = event_notifier_get_fd(&vdev->intx.interrupt);
294     qemu_set_fd_handler(fd, vfio_intx_interrupt, NULL, vdev);
295 
296     if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
297                                VFIO_IRQ_SET_ACTION_TRIGGER, fd, errp)) {
298         qemu_set_fd_handler(fd, NULL, NULL, vdev);
299         event_notifier_cleanup(&vdev->intx.interrupt);
300         return -errno;
301     }
302 
303     vfio_intx_enable_kvm(vdev, &err);
304     if (err) {
305         warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
306     }
307 
308     vdev->interrupt = VFIO_INT_INTx;
309 
310     trace_vfio_intx_enable(vdev->vbasedev.name);
311     return 0;
312 }
313 
314 static void vfio_intx_disable(VFIOPCIDevice *vdev)
315 {
316     int fd;
317 
318     timer_del(vdev->intx.mmap_timer);
319     vfio_intx_disable_kvm(vdev);
320     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
321     vdev->intx.pending = false;
322     pci_irq_deassert(&vdev->pdev);
323     vfio_mmap_set_enabled(vdev, true);
324 
325     fd = event_notifier_get_fd(&vdev->intx.interrupt);
326     qemu_set_fd_handler(fd, NULL, NULL, vdev);
327     event_notifier_cleanup(&vdev->intx.interrupt);
328 
329     vdev->interrupt = VFIO_INT_NONE;
330 
331     trace_vfio_intx_disable(vdev->vbasedev.name);
332 }
333 
334 /*
335  * MSI/X
336  */
337 static void vfio_msi_interrupt(void *opaque)
338 {
339     VFIOMSIVector *vector = opaque;
340     VFIOPCIDevice *vdev = vector->vdev;
341     MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
342     void (*notify)(PCIDevice *dev, unsigned vector);
343     MSIMessage msg;
344     int nr = vector - vdev->msi_vectors;
345 
346     if (!event_notifier_test_and_clear(&vector->interrupt)) {
347         return;
348     }
349 
350     if (vdev->interrupt == VFIO_INT_MSIX) {
351         get_msg = msix_get_message;
352         notify = msix_notify;
353 
354         /* A masked vector firing needs to use the PBA, enable it */
355         if (msix_is_masked(&vdev->pdev, nr)) {
356             set_bit(nr, vdev->msix->pending);
357             memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
358             trace_vfio_msix_pba_enable(vdev->vbasedev.name);
359         }
360     } else if (vdev->interrupt == VFIO_INT_MSI) {
361         get_msg = msi_get_message;
362         notify = msi_notify;
363     } else {
364         abort();
365     }
366 
367     msg = get_msg(&vdev->pdev, nr);
368     trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
369     notify(&vdev->pdev, nr);
370 }
371 
372 /*
373  * Get MSI-X enabled, but no vector enabled, by setting vector 0 with an invalid
374  * fd to kernel.
375  */
376 static int vfio_enable_msix_no_vec(VFIOPCIDevice *vdev)
377 {
378     g_autofree struct vfio_irq_set *irq_set = NULL;
379     int ret = 0, argsz;
380     int32_t *fd;
381 
382     argsz = sizeof(*irq_set) + sizeof(*fd);
383 
384     irq_set = g_malloc0(argsz);
385     irq_set->argsz = argsz;
386     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
387                      VFIO_IRQ_SET_ACTION_TRIGGER;
388     irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
389     irq_set->start = 0;
390     irq_set->count = 1;
391     fd = (int32_t *)&irq_set->data;
392     *fd = -1;
393 
394     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
395 
396     return ret;
397 }
398 
399 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
400 {
401     struct vfio_irq_set *irq_set;
402     int ret = 0, i, argsz;
403     int32_t *fds;
404 
405     /*
406      * If dynamic MSI-X allocation is supported, the vectors to be allocated
407      * and enabled can be scattered. Before kernel enabling MSI-X, setting
408      * nr_vectors causes all these vectors to be allocated on host.
409      *
410      * To keep allocation as needed, use vector 0 with an invalid fd to get
411      * MSI-X enabled first, then set vectors with a potentially sparse set of
412      * eventfds to enable interrupts only when enabled in guest.
413      */
414     if (msix && !vdev->msix->noresize) {
415         ret = vfio_enable_msix_no_vec(vdev);
416 
417         if (ret) {
418             return ret;
419         }
420     }
421 
422     argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
423 
424     irq_set = g_malloc0(argsz);
425     irq_set->argsz = argsz;
426     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
427     irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
428     irq_set->start = 0;
429     irq_set->count = vdev->nr_vectors;
430     fds = (int32_t *)&irq_set->data;
431 
432     for (i = 0; i < vdev->nr_vectors; i++) {
433         int fd = -1;
434 
435         /*
436          * MSI vs MSI-X - The guest has direct access to MSI mask and pending
437          * bits, therefore we always use the KVM signaling path when setup.
438          * MSI-X mask and pending bits are emulated, so we want to use the
439          * KVM signaling path only when configured and unmasked.
440          */
441         if (vdev->msi_vectors[i].use) {
442             if (vdev->msi_vectors[i].virq < 0 ||
443                 (msix && msix_is_masked(&vdev->pdev, i))) {
444                 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
445             } else {
446                 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
447             }
448         }
449 
450         fds[i] = fd;
451     }
452 
453     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
454 
455     g_free(irq_set);
456 
457     return ret;
458 }
459 
460 static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
461                                   int vector_n, bool msix)
462 {
463     if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
464         return;
465     }
466 
467     vector->virq = kvm_irqchip_add_msi_route(&vfio_route_change,
468                                              vector_n, &vdev->pdev);
469 }
470 
471 static void vfio_connect_kvm_msi_virq(VFIOMSIVector *vector)
472 {
473     if (vector->virq < 0) {
474         return;
475     }
476 
477     if (event_notifier_init(&vector->kvm_interrupt, 0)) {
478         goto fail_notifier;
479     }
480 
481     if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
482                                            NULL, vector->virq) < 0) {
483         goto fail_kvm;
484     }
485 
486     return;
487 
488 fail_kvm:
489     event_notifier_cleanup(&vector->kvm_interrupt);
490 fail_notifier:
491     kvm_irqchip_release_virq(kvm_state, vector->virq);
492     vector->virq = -1;
493 }
494 
495 static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
496 {
497     kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
498                                           vector->virq);
499     kvm_irqchip_release_virq(kvm_state, vector->virq);
500     vector->virq = -1;
501     event_notifier_cleanup(&vector->kvm_interrupt);
502 }
503 
504 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
505                                      PCIDevice *pdev)
506 {
507     kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
508     kvm_irqchip_commit_routes(kvm_state);
509 }
510 
511 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
512                                    MSIMessage *msg, IOHandler *handler)
513 {
514     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
515     VFIOMSIVector *vector;
516     int ret;
517     bool resizing = !!(vdev->nr_vectors < nr + 1);
518 
519     trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
520 
521     vector = &vdev->msi_vectors[nr];
522 
523     if (!vector->use) {
524         vector->vdev = vdev;
525         vector->virq = -1;
526         if (event_notifier_init(&vector->interrupt, 0)) {
527             error_report("vfio: Error: event_notifier_init failed");
528         }
529         vector->use = true;
530         msix_vector_use(pdev, nr);
531     }
532 
533     qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
534                         handler, NULL, vector);
535 
536     /*
537      * Attempt to enable route through KVM irqchip,
538      * default to userspace handling if unavailable.
539      */
540     if (vector->virq >= 0) {
541         if (!msg) {
542             vfio_remove_kvm_msi_virq(vector);
543         } else {
544             vfio_update_kvm_msi_virq(vector, *msg, pdev);
545         }
546     } else {
547         if (msg) {
548             if (vdev->defer_kvm_irq_routing) {
549                 vfio_add_kvm_msi_virq(vdev, vector, nr, true);
550             } else {
551                 vfio_route_change = kvm_irqchip_begin_route_changes(kvm_state);
552                 vfio_add_kvm_msi_virq(vdev, vector, nr, true);
553                 kvm_irqchip_commit_route_changes(&vfio_route_change);
554                 vfio_connect_kvm_msi_virq(vector);
555             }
556         }
557     }
558 
559     /*
560      * When dynamic allocation is not supported, we don't want to have the
561      * host allocate all possible MSI vectors for a device if they're not
562      * in use, so we shutdown and incrementally increase them as needed.
563      * nr_vectors represents the total number of vectors allocated.
564      *
565      * When dynamic allocation is supported, let the host only allocate
566      * and enable a vector when it is in use in guest. nr_vectors represents
567      * the upper bound of vectors being enabled (but not all of the ranges
568      * is allocated or enabled).
569      */
570     if (resizing) {
571         vdev->nr_vectors = nr + 1;
572     }
573 
574     if (!vdev->defer_kvm_irq_routing) {
575         if (vdev->msix->noresize && resizing) {
576             vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
577             ret = vfio_enable_vectors(vdev, true);
578             if (ret) {
579                 error_report("vfio: failed to enable vectors, %d", ret);
580             }
581         } else {
582             Error *err = NULL;
583             int32_t fd;
584 
585             if (vector->virq >= 0) {
586                 fd = event_notifier_get_fd(&vector->kvm_interrupt);
587             } else {
588                 fd = event_notifier_get_fd(&vector->interrupt);
589             }
590 
591             if (vfio_set_irq_signaling(&vdev->vbasedev,
592                                        VFIO_PCI_MSIX_IRQ_INDEX, nr,
593                                        VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
594                 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
595             }
596         }
597     }
598 
599     /* Disable PBA emulation when nothing more is pending. */
600     clear_bit(nr, vdev->msix->pending);
601     if (find_first_bit(vdev->msix->pending,
602                        vdev->nr_vectors) == vdev->nr_vectors) {
603         memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
604         trace_vfio_msix_pba_disable(vdev->vbasedev.name);
605     }
606 
607     return 0;
608 }
609 
610 static int vfio_msix_vector_use(PCIDevice *pdev,
611                                 unsigned int nr, MSIMessage msg)
612 {
613     return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
614 }
615 
616 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
617 {
618     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
619     VFIOMSIVector *vector = &vdev->msi_vectors[nr];
620 
621     trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
622 
623     /*
624      * There are still old guests that mask and unmask vectors on every
625      * interrupt.  If we're using QEMU bypass with a KVM irqfd, leave all of
626      * the KVM setup in place, simply switch VFIO to use the non-bypass
627      * eventfd.  We'll then fire the interrupt through QEMU and the MSI-X
628      * core will mask the interrupt and set pending bits, allowing it to
629      * be re-asserted on unmask.  Nothing to do if already using QEMU mode.
630      */
631     if (vector->virq >= 0) {
632         int32_t fd = event_notifier_get_fd(&vector->interrupt);
633         Error *err = NULL;
634 
635         if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, nr,
636                                    VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
637             error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
638         }
639     }
640 }
641 
642 static void vfio_prepare_kvm_msi_virq_batch(VFIOPCIDevice *vdev)
643 {
644     assert(!vdev->defer_kvm_irq_routing);
645     vdev->defer_kvm_irq_routing = true;
646     vfio_route_change = kvm_irqchip_begin_route_changes(kvm_state);
647 }
648 
649 static void vfio_commit_kvm_msi_virq_batch(VFIOPCIDevice *vdev)
650 {
651     int i;
652 
653     assert(vdev->defer_kvm_irq_routing);
654     vdev->defer_kvm_irq_routing = false;
655 
656     kvm_irqchip_commit_route_changes(&vfio_route_change);
657 
658     for (i = 0; i < vdev->nr_vectors; i++) {
659         vfio_connect_kvm_msi_virq(&vdev->msi_vectors[i]);
660     }
661 }
662 
663 static void vfio_msix_enable(VFIOPCIDevice *vdev)
664 {
665     int ret;
666 
667     vfio_disable_interrupts(vdev);
668 
669     vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
670 
671     vdev->interrupt = VFIO_INT_MSIX;
672 
673     /*
674      * Setting vector notifiers triggers synchronous vector-use
675      * callbacks for each active vector.  Deferring to commit the KVM
676      * routes once rather than per vector provides a substantial
677      * performance improvement.
678      */
679     vfio_prepare_kvm_msi_virq_batch(vdev);
680 
681     if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
682                                   vfio_msix_vector_release, NULL)) {
683         error_report("vfio: msix_set_vector_notifiers failed");
684     }
685 
686     vfio_commit_kvm_msi_virq_batch(vdev);
687 
688     if (vdev->nr_vectors) {
689         ret = vfio_enable_vectors(vdev, true);
690         if (ret) {
691             error_report("vfio: failed to enable vectors, %d", ret);
692         }
693     } else {
694         /*
695          * Some communication channels between VF & PF or PF & fw rely on the
696          * physical state of the device and expect that enabling MSI-X from the
697          * guest enables the same on the host.  When our guest is Linux, the
698          * guest driver call to pci_enable_msix() sets the enabling bit in the
699          * MSI-X capability, but leaves the vector table masked.  We therefore
700          * can't rely on a vector_use callback (from request_irq() in the guest)
701          * to switch the physical device into MSI-X mode because that may come a
702          * long time after pci_enable_msix().  This code sets vector 0 with an
703          * invalid fd to make the physical device MSI-X enabled, but with no
704          * vectors enabled, just like the guest view.
705          */
706         ret = vfio_enable_msix_no_vec(vdev);
707         if (ret) {
708             error_report("vfio: failed to enable MSI-X, %d", ret);
709         }
710     }
711 
712     trace_vfio_msix_enable(vdev->vbasedev.name);
713 }
714 
715 static void vfio_msi_enable(VFIOPCIDevice *vdev)
716 {
717     int ret, i;
718 
719     vfio_disable_interrupts(vdev);
720 
721     vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
722 retry:
723     /*
724      * Setting vector notifiers needs to enable route for each vector.
725      * Deferring to commit the KVM routes once rather than per vector
726      * provides a substantial performance improvement.
727      */
728     vfio_prepare_kvm_msi_virq_batch(vdev);
729 
730     vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
731 
732     for (i = 0; i < vdev->nr_vectors; i++) {
733         VFIOMSIVector *vector = &vdev->msi_vectors[i];
734 
735         vector->vdev = vdev;
736         vector->virq = -1;
737         vector->use = true;
738 
739         if (event_notifier_init(&vector->interrupt, 0)) {
740             error_report("vfio: Error: event_notifier_init failed");
741         }
742 
743         qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
744                             vfio_msi_interrupt, NULL, vector);
745 
746         /*
747          * Attempt to enable route through KVM irqchip,
748          * default to userspace handling if unavailable.
749          */
750         vfio_add_kvm_msi_virq(vdev, vector, i, false);
751     }
752 
753     vfio_commit_kvm_msi_virq_batch(vdev);
754 
755     /* Set interrupt type prior to possible interrupts */
756     vdev->interrupt = VFIO_INT_MSI;
757 
758     ret = vfio_enable_vectors(vdev, false);
759     if (ret) {
760         if (ret < 0) {
761             error_report("vfio: Error: Failed to setup MSI fds: %m");
762         } else {
763             error_report("vfio: Error: Failed to enable %d "
764                          "MSI vectors, retry with %d", vdev->nr_vectors, ret);
765         }
766 
767         vfio_msi_disable_common(vdev);
768 
769         if (ret > 0) {
770             vdev->nr_vectors = ret;
771             goto retry;
772         }
773 
774         /*
775          * Failing to setup MSI doesn't really fall within any specification.
776          * Let's try leaving interrupts disabled and hope the guest figures
777          * out to fall back to INTx for this device.
778          */
779         error_report("vfio: Error: Failed to enable MSI");
780 
781         return;
782     }
783 
784     trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
785 }
786 
787 static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
788 {
789     int i;
790 
791     for (i = 0; i < vdev->nr_vectors; i++) {
792         VFIOMSIVector *vector = &vdev->msi_vectors[i];
793         if (vdev->msi_vectors[i].use) {
794             if (vector->virq >= 0) {
795                 vfio_remove_kvm_msi_virq(vector);
796             }
797             qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
798                                 NULL, NULL, NULL);
799             event_notifier_cleanup(&vector->interrupt);
800         }
801     }
802 
803     g_free(vdev->msi_vectors);
804     vdev->msi_vectors = NULL;
805     vdev->nr_vectors = 0;
806     vdev->interrupt = VFIO_INT_NONE;
807 }
808 
809 static void vfio_msix_disable(VFIOPCIDevice *vdev)
810 {
811     Error *err = NULL;
812     int i;
813 
814     msix_unset_vector_notifiers(&vdev->pdev);
815 
816     /*
817      * MSI-X will only release vectors if MSI-X is still enabled on the
818      * device, check through the rest and release it ourselves if necessary.
819      */
820     for (i = 0; i < vdev->nr_vectors; i++) {
821         if (vdev->msi_vectors[i].use) {
822             vfio_msix_vector_release(&vdev->pdev, i);
823             msix_vector_unuse(&vdev->pdev, i);
824         }
825     }
826 
827     if (vdev->nr_vectors) {
828         vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
829     }
830 
831     vfio_msi_disable_common(vdev);
832     vfio_intx_enable(vdev, &err);
833     if (err) {
834         error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
835     }
836 
837     memset(vdev->msix->pending, 0,
838            BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
839 
840     trace_vfio_msix_disable(vdev->vbasedev.name);
841 }
842 
843 static void vfio_msi_disable(VFIOPCIDevice *vdev)
844 {
845     Error *err = NULL;
846 
847     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
848     vfio_msi_disable_common(vdev);
849     vfio_intx_enable(vdev, &err);
850     if (err) {
851         error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
852     }
853 
854     trace_vfio_msi_disable(vdev->vbasedev.name);
855 }
856 
857 static void vfio_update_msi(VFIOPCIDevice *vdev)
858 {
859     int i;
860 
861     for (i = 0; i < vdev->nr_vectors; i++) {
862         VFIOMSIVector *vector = &vdev->msi_vectors[i];
863         MSIMessage msg;
864 
865         if (!vector->use || vector->virq < 0) {
866             continue;
867         }
868 
869         msg = msi_get_message(&vdev->pdev, i);
870         vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
871     }
872 }
873 
874 static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
875 {
876     struct vfio_region_info *reg_info;
877     uint64_t size;
878     off_t off = 0;
879     ssize_t bytes;
880 
881     if (vfio_get_region_info(&vdev->vbasedev,
882                              VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
883         error_report("vfio: Error getting ROM info: %m");
884         return;
885     }
886 
887     trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
888                             (unsigned long)reg_info->offset,
889                             (unsigned long)reg_info->flags);
890 
891     vdev->rom_size = size = reg_info->size;
892     vdev->rom_offset = reg_info->offset;
893 
894     g_free(reg_info);
895 
896     if (!vdev->rom_size) {
897         vdev->rom_read_failed = true;
898         error_report("vfio-pci: Cannot read device rom at "
899                     "%s", vdev->vbasedev.name);
900         error_printf("Device option ROM contents are probably invalid "
901                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
902                     "or load from file with romfile=\n");
903         return;
904     }
905 
906     vdev->rom = g_malloc(size);
907     memset(vdev->rom, 0xff, size);
908 
909     while (size) {
910         bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
911                       size, vdev->rom_offset + off);
912         if (bytes == 0) {
913             break;
914         } else if (bytes > 0) {
915             off += bytes;
916             size -= bytes;
917         } else {
918             if (errno == EINTR || errno == EAGAIN) {
919                 continue;
920             }
921             error_report("vfio: Error reading device ROM: %m");
922             break;
923         }
924     }
925 
926     /*
927      * Test the ROM signature against our device, if the vendor is correct
928      * but the device ID doesn't match, store the correct device ID and
929      * recompute the checksum.  Intel IGD devices need this and are known
930      * to have bogus checksums so we can't simply adjust the checksum.
931      */
932     if (pci_get_word(vdev->rom) == 0xaa55 &&
933         pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
934         !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
935         uint16_t vid, did;
936 
937         vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
938         did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
939 
940         if (vid == vdev->vendor_id && did != vdev->device_id) {
941             int i;
942             uint8_t csum, *data = vdev->rom;
943 
944             pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
945                          vdev->device_id);
946             data[6] = 0;
947 
948             for (csum = 0, i = 0; i < vdev->rom_size; i++) {
949                 csum += data[i];
950             }
951 
952             data[6] = -csum;
953         }
954     }
955 }
956 
957 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
958 {
959     VFIOPCIDevice *vdev = opaque;
960     union {
961         uint8_t byte;
962         uint16_t word;
963         uint32_t dword;
964         uint64_t qword;
965     } val;
966     uint64_t data = 0;
967 
968     /* Load the ROM lazily when the guest tries to read it */
969     if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
970         vfio_pci_load_rom(vdev);
971     }
972 
973     memcpy(&val, vdev->rom + addr,
974            (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
975 
976     switch (size) {
977     case 1:
978         data = val.byte;
979         break;
980     case 2:
981         data = le16_to_cpu(val.word);
982         break;
983     case 4:
984         data = le32_to_cpu(val.dword);
985         break;
986     default:
987         hw_error("vfio: unsupported read size, %d bytes\n", size);
988         break;
989     }
990 
991     trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
992 
993     return data;
994 }
995 
996 static void vfio_rom_write(void *opaque, hwaddr addr,
997                            uint64_t data, unsigned size)
998 {
999 }
1000 
1001 static const MemoryRegionOps vfio_rom_ops = {
1002     .read = vfio_rom_read,
1003     .write = vfio_rom_write,
1004     .endianness = DEVICE_LITTLE_ENDIAN,
1005 };
1006 
1007 static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
1008 {
1009     uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
1010     off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
1011     DeviceState *dev = DEVICE(vdev);
1012     char *name;
1013     int fd = vdev->vbasedev.fd;
1014 
1015     if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
1016         /* Since pci handles romfile, just print a message and return */
1017         if (vfio_opt_rom_in_denylist(vdev) && vdev->pdev.romfile) {
1018             warn_report("Device at %s is known to cause system instability"
1019                         " issues during option rom execution",
1020                         vdev->vbasedev.name);
1021             error_printf("Proceeding anyway since user specified romfile\n");
1022         }
1023         return;
1024     }
1025 
1026     /*
1027      * Use the same size ROM BAR as the physical device.  The contents
1028      * will get filled in later when the guest tries to read it.
1029      */
1030     if (pread(fd, &orig, 4, offset) != 4 ||
1031         pwrite(fd, &size, 4, offset) != 4 ||
1032         pread(fd, &size, 4, offset) != 4 ||
1033         pwrite(fd, &orig, 4, offset) != 4) {
1034         error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
1035         return;
1036     }
1037 
1038     size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
1039 
1040     if (!size) {
1041         return;
1042     }
1043 
1044     if (vfio_opt_rom_in_denylist(vdev)) {
1045         if (dev->opts && qdict_haskey(dev->opts, "rombar")) {
1046             warn_report("Device at %s is known to cause system instability"
1047                         " issues during option rom execution",
1048                         vdev->vbasedev.name);
1049             error_printf("Proceeding anyway since user specified"
1050                          " non zero value for rombar\n");
1051         } else {
1052             warn_report("Rom loading for device at %s has been disabled"
1053                         " due to system instability issues",
1054                         vdev->vbasedev.name);
1055             error_printf("Specify rombar=1 or romfile to force\n");
1056             return;
1057         }
1058     }
1059 
1060     trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
1061 
1062     name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
1063 
1064     memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
1065                           &vfio_rom_ops, vdev, name, size);
1066     g_free(name);
1067 
1068     pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
1069                      PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
1070 
1071     vdev->rom_read_failed = false;
1072 }
1073 
1074 void vfio_vga_write(void *opaque, hwaddr addr,
1075                            uint64_t data, unsigned size)
1076 {
1077     VFIOVGARegion *region = opaque;
1078     VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1079     union {
1080         uint8_t byte;
1081         uint16_t word;
1082         uint32_t dword;
1083         uint64_t qword;
1084     } buf;
1085     off_t offset = vga->fd_offset + region->offset + addr;
1086 
1087     switch (size) {
1088     case 1:
1089         buf.byte = data;
1090         break;
1091     case 2:
1092         buf.word = cpu_to_le16(data);
1093         break;
1094     case 4:
1095         buf.dword = cpu_to_le32(data);
1096         break;
1097     default:
1098         hw_error("vfio: unsupported write size, %d bytes", size);
1099         break;
1100     }
1101 
1102     if (pwrite(vga->fd, &buf, size, offset) != size) {
1103         error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1104                      __func__, region->offset + addr, data, size);
1105     }
1106 
1107     trace_vfio_vga_write(region->offset + addr, data, size);
1108 }
1109 
1110 uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
1111 {
1112     VFIOVGARegion *region = opaque;
1113     VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1114     union {
1115         uint8_t byte;
1116         uint16_t word;
1117         uint32_t dword;
1118         uint64_t qword;
1119     } buf;
1120     uint64_t data = 0;
1121     off_t offset = vga->fd_offset + region->offset + addr;
1122 
1123     if (pread(vga->fd, &buf, size, offset) != size) {
1124         error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1125                      __func__, region->offset + addr, size);
1126         return (uint64_t)-1;
1127     }
1128 
1129     switch (size) {
1130     case 1:
1131         data = buf.byte;
1132         break;
1133     case 2:
1134         data = le16_to_cpu(buf.word);
1135         break;
1136     case 4:
1137         data = le32_to_cpu(buf.dword);
1138         break;
1139     default:
1140         hw_error("vfio: unsupported read size, %d bytes", size);
1141         break;
1142     }
1143 
1144     trace_vfio_vga_read(region->offset + addr, size, data);
1145 
1146     return data;
1147 }
1148 
1149 static const MemoryRegionOps vfio_vga_ops = {
1150     .read = vfio_vga_read,
1151     .write = vfio_vga_write,
1152     .endianness = DEVICE_LITTLE_ENDIAN,
1153 };
1154 
1155 /*
1156  * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1157  * size if the BAR is in an exclusive page in host so that we could map
1158  * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1159  * page in guest. So we should set the priority of the expanded memory
1160  * region to zero in case of overlap with BARs which share the same page
1161  * with the sub-page BAR in guest. Besides, we should also recover the
1162  * size of this sub-page BAR when its base address is changed in guest
1163  * and not page aligned any more.
1164  */
1165 static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1166 {
1167     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1168     VFIORegion *region = &vdev->bars[bar].region;
1169     MemoryRegion *mmap_mr, *region_mr, *base_mr;
1170     PCIIORegion *r;
1171     pcibus_t bar_addr;
1172     uint64_t size = region->size;
1173 
1174     /* Make sure that the whole region is allowed to be mmapped */
1175     if (region->nr_mmaps != 1 || !region->mmaps[0].mmap ||
1176         region->mmaps[0].size != region->size) {
1177         return;
1178     }
1179 
1180     r = &pdev->io_regions[bar];
1181     bar_addr = r->addr;
1182     base_mr = vdev->bars[bar].mr;
1183     region_mr = region->mem;
1184     mmap_mr = &region->mmaps[0].mem;
1185 
1186     /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1187     if (bar_addr != PCI_BAR_UNMAPPED &&
1188         !(bar_addr & ~qemu_real_host_page_mask())) {
1189         size = qemu_real_host_page_size();
1190     }
1191 
1192     memory_region_transaction_begin();
1193 
1194     if (vdev->bars[bar].size < size) {
1195         memory_region_set_size(base_mr, size);
1196     }
1197     memory_region_set_size(region_mr, size);
1198     memory_region_set_size(mmap_mr, size);
1199     if (size != vdev->bars[bar].size && memory_region_is_mapped(base_mr)) {
1200         memory_region_del_subregion(r->address_space, base_mr);
1201         memory_region_add_subregion_overlap(r->address_space,
1202                                             bar_addr, base_mr, 0);
1203     }
1204 
1205     memory_region_transaction_commit();
1206 }
1207 
1208 /*
1209  * PCI config space
1210  */
1211 uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1212 {
1213     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1214     uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1215 
1216     memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1217     emu_bits = le32_to_cpu(emu_bits);
1218 
1219     if (emu_bits) {
1220         emu_val = pci_default_read_config(pdev, addr, len);
1221     }
1222 
1223     if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1224         ssize_t ret;
1225 
1226         ret = pread(vdev->vbasedev.fd, &phys_val, len,
1227                     vdev->config_offset + addr);
1228         if (ret != len) {
1229             error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1230                          __func__, vdev->vbasedev.name, addr, len);
1231             return -errno;
1232         }
1233         phys_val = le32_to_cpu(phys_val);
1234     }
1235 
1236     val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
1237 
1238     trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1239 
1240     return val;
1241 }
1242 
1243 void vfio_pci_write_config(PCIDevice *pdev,
1244                            uint32_t addr, uint32_t val, int len)
1245 {
1246     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
1247     uint32_t val_le = cpu_to_le32(val);
1248 
1249     trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1250 
1251     /* Write everything to VFIO, let it filter out what we can't write */
1252     if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
1253                 != len) {
1254         error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1255                      __func__, vdev->vbasedev.name, addr, val, len);
1256     }
1257 
1258     /* MSI/MSI-X Enabling/Disabling */
1259     if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1260         ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1261         int is_enabled, was_enabled = msi_enabled(pdev);
1262 
1263         pci_default_write_config(pdev, addr, val, len);
1264 
1265         is_enabled = msi_enabled(pdev);
1266 
1267         if (!was_enabled) {
1268             if (is_enabled) {
1269                 vfio_msi_enable(vdev);
1270             }
1271         } else {
1272             if (!is_enabled) {
1273                 vfio_msi_disable(vdev);
1274             } else {
1275                 vfio_update_msi(vdev);
1276             }
1277         }
1278     } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1279         ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1280         int is_enabled, was_enabled = msix_enabled(pdev);
1281 
1282         pci_default_write_config(pdev, addr, val, len);
1283 
1284         is_enabled = msix_enabled(pdev);
1285 
1286         if (!was_enabled && is_enabled) {
1287             vfio_msix_enable(vdev);
1288         } else if (was_enabled && !is_enabled) {
1289             vfio_msix_disable(vdev);
1290         }
1291     } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
1292         range_covers_byte(addr, len, PCI_COMMAND)) {
1293         pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1294         int bar;
1295 
1296         for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1297             old_addr[bar] = pdev->io_regions[bar].addr;
1298         }
1299 
1300         pci_default_write_config(pdev, addr, val, len);
1301 
1302         for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1303             if (old_addr[bar] != pdev->io_regions[bar].addr &&
1304                 vdev->bars[bar].region.size > 0 &&
1305                 vdev->bars[bar].region.size < qemu_real_host_page_size()) {
1306                 vfio_sub_page_bar_update_mapping(pdev, bar);
1307             }
1308         }
1309     } else {
1310         /* Write everything to QEMU to keep emulated bits correct */
1311         pci_default_write_config(pdev, addr, val, len);
1312     }
1313 }
1314 
1315 /*
1316  * Interrupt setup
1317  */
1318 static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1319 {
1320     /*
1321      * More complicated than it looks.  Disabling MSI/X transitions the
1322      * device to INTx mode (if supported).  Therefore we need to first
1323      * disable MSI/X and then cleanup by disabling INTx.
1324      */
1325     if (vdev->interrupt == VFIO_INT_MSIX) {
1326         vfio_msix_disable(vdev);
1327     } else if (vdev->interrupt == VFIO_INT_MSI) {
1328         vfio_msi_disable(vdev);
1329     }
1330 
1331     if (vdev->interrupt == VFIO_INT_INTx) {
1332         vfio_intx_disable(vdev);
1333     }
1334 }
1335 
1336 static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1337 {
1338     uint16_t ctrl;
1339     bool msi_64bit, msi_maskbit;
1340     int ret, entries;
1341     Error *err = NULL;
1342 
1343     if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
1344               vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
1345         error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS");
1346         return -errno;
1347     }
1348     ctrl = le16_to_cpu(ctrl);
1349 
1350     msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1351     msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1352     entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1353 
1354     trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1355 
1356     ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1357     if (ret < 0) {
1358         if (ret == -ENOTSUP) {
1359             return 0;
1360         }
1361         error_propagate_prepend(errp, err, "msi_init failed: ");
1362         return ret;
1363     }
1364     vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1365 
1366     return 0;
1367 }
1368 
1369 static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1370 {
1371     off_t start, end;
1372     VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1373 
1374     /*
1375      * If the host driver allows mapping of a MSIX data, we are going to
1376      * do map the entire BAR and emulate MSIX table on top of that.
1377      */
1378     if (vfio_has_region_cap(&vdev->vbasedev, region->nr,
1379                             VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)) {
1380         return;
1381     }
1382 
1383     /*
1384      * We expect to find a single mmap covering the whole BAR, anything else
1385      * means it's either unsupported or already setup.
1386      */
1387     if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
1388         region->size != region->mmaps[0].size) {
1389         return;
1390     }
1391 
1392     /* MSI-X table start and end aligned to host page size */
1393     start = vdev->msix->table_offset & qemu_real_host_page_mask();
1394     end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1395                                (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1396 
1397     /*
1398      * Does the MSI-X table cover the beginning of the BAR?  The whole BAR?
1399      * NB - Host page size is necessarily a power of two and so is the PCI
1400      * BAR (not counting EA yet), therefore if we have host page aligned
1401      * @start and @end, then any remainder of the BAR before or after those
1402      * must be at least host page sized and therefore mmap'able.
1403      */
1404     if (!start) {
1405         if (end >= region->size) {
1406             region->nr_mmaps = 0;
1407             g_free(region->mmaps);
1408             region->mmaps = NULL;
1409             trace_vfio_msix_fixup(vdev->vbasedev.name,
1410                                   vdev->msix->table_bar, 0, 0);
1411         } else {
1412             region->mmaps[0].offset = end;
1413             region->mmaps[0].size = region->size - end;
1414             trace_vfio_msix_fixup(vdev->vbasedev.name,
1415                               vdev->msix->table_bar, region->mmaps[0].offset,
1416                               region->mmaps[0].offset + region->mmaps[0].size);
1417         }
1418 
1419     /* Maybe it's aligned at the end of the BAR */
1420     } else if (end >= region->size) {
1421         region->mmaps[0].size = start;
1422         trace_vfio_msix_fixup(vdev->vbasedev.name,
1423                               vdev->msix->table_bar, region->mmaps[0].offset,
1424                               region->mmaps[0].offset + region->mmaps[0].size);
1425 
1426     /* Otherwise it must split the BAR */
1427     } else {
1428         region->nr_mmaps = 2;
1429         region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1430 
1431         memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
1432 
1433         region->mmaps[0].size = start;
1434         trace_vfio_msix_fixup(vdev->vbasedev.name,
1435                               vdev->msix->table_bar, region->mmaps[0].offset,
1436                               region->mmaps[0].offset + region->mmaps[0].size);
1437 
1438         region->mmaps[1].offset = end;
1439         region->mmaps[1].size = region->size - end;
1440         trace_vfio_msix_fixup(vdev->vbasedev.name,
1441                               vdev->msix->table_bar, region->mmaps[1].offset,
1442                               region->mmaps[1].offset + region->mmaps[1].size);
1443     }
1444 }
1445 
1446 static void vfio_pci_relocate_msix(VFIOPCIDevice *vdev, Error **errp)
1447 {
1448     int target_bar = -1;
1449     size_t msix_sz;
1450 
1451     if (!vdev->msix || vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1452         return;
1453     }
1454 
1455     /* The actual minimum size of MSI-X structures */
1456     msix_sz = (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE) +
1457               (QEMU_ALIGN_UP(vdev->msix->entries, 64) / 8);
1458     /* Round up to host pages, we don't want to share a page */
1459     msix_sz = REAL_HOST_PAGE_ALIGN(msix_sz);
1460     /* PCI BARs must be a power of 2 */
1461     msix_sz = pow2ceil(msix_sz);
1462 
1463     if (vdev->msix_relo == OFF_AUTOPCIBAR_AUTO) {
1464         /*
1465          * TODO: Lookup table for known devices.
1466          *
1467          * Logically we might use an algorithm here to select the BAR adding
1468          * the least additional MMIO space, but we cannot programmatically
1469          * predict the driver dependency on BAR ordering or sizing, therefore
1470          * 'auto' becomes a lookup for combinations reported to work.
1471          */
1472         if (target_bar < 0) {
1473             error_setg(errp, "No automatic MSI-X relocation available for "
1474                        "device %04x:%04x", vdev->vendor_id, vdev->device_id);
1475             return;
1476         }
1477     } else {
1478         target_bar = (int)(vdev->msix_relo - OFF_AUTOPCIBAR_BAR0);
1479     }
1480 
1481     /* I/O port BARs cannot host MSI-X structures */
1482     if (vdev->bars[target_bar].ioport) {
1483         error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1484                    "I/O port BAR", target_bar);
1485         return;
1486     }
1487 
1488     /* Cannot use a BAR in the "shadow" of a 64-bit BAR */
1489     if (!vdev->bars[target_bar].size &&
1490          target_bar > 0 && vdev->bars[target_bar - 1].mem64) {
1491         error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1492                    "consumed by 64-bit BAR %d", target_bar, target_bar - 1);
1493         return;
1494     }
1495 
1496     /* 2GB max size for 32-bit BARs, cannot double if already > 1G */
1497     if (vdev->bars[target_bar].size > 1 * GiB &&
1498         !vdev->bars[target_bar].mem64) {
1499         error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1500                    "no space to extend 32-bit BAR", target_bar);
1501         return;
1502     }
1503 
1504     /*
1505      * If adding a new BAR, test if we can make it 64bit.  We make it
1506      * prefetchable since QEMU MSI-X emulation has no read side effects
1507      * and doing so makes mapping more flexible.
1508      */
1509     if (!vdev->bars[target_bar].size) {
1510         if (target_bar < (PCI_ROM_SLOT - 1) &&
1511             !vdev->bars[target_bar + 1].size) {
1512             vdev->bars[target_bar].mem64 = true;
1513             vdev->bars[target_bar].type = PCI_BASE_ADDRESS_MEM_TYPE_64;
1514         }
1515         vdev->bars[target_bar].type |= PCI_BASE_ADDRESS_MEM_PREFETCH;
1516         vdev->bars[target_bar].size = msix_sz;
1517         vdev->msix->table_offset = 0;
1518     } else {
1519         vdev->bars[target_bar].size = MAX(vdev->bars[target_bar].size * 2,
1520                                           msix_sz * 2);
1521         /*
1522          * Due to above size calc, MSI-X always starts halfway into the BAR,
1523          * which will always be a separate host page.
1524          */
1525         vdev->msix->table_offset = vdev->bars[target_bar].size / 2;
1526     }
1527 
1528     vdev->msix->table_bar = target_bar;
1529     vdev->msix->pba_bar = target_bar;
1530     /* Requires 8-byte alignment, but PCI_MSIX_ENTRY_SIZE guarantees that */
1531     vdev->msix->pba_offset = vdev->msix->table_offset +
1532                                   (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE);
1533 
1534     trace_vfio_msix_relo(vdev->vbasedev.name,
1535                          vdev->msix->table_bar, vdev->msix->table_offset);
1536 }
1537 
1538 /*
1539  * We don't have any control over how pci_add_capability() inserts
1540  * capabilities into the chain.  In order to setup MSI-X we need a
1541  * MemoryRegion for the BAR.  In order to setup the BAR and not
1542  * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1543  * need to first look for where the MSI-X table lives.  So we
1544  * unfortunately split MSI-X setup across two functions.
1545  */
1546 static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
1547 {
1548     uint8_t pos;
1549     uint16_t ctrl;
1550     uint32_t table, pba;
1551     int ret, fd = vdev->vbasedev.fd;
1552     struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
1553                                       .index = VFIO_PCI_MSIX_IRQ_INDEX };
1554     VFIOMSIXInfo *msix;
1555 
1556     pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
1557     if (!pos) {
1558         return;
1559     }
1560 
1561     if (pread(fd, &ctrl, sizeof(ctrl),
1562               vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
1563         error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS");
1564         return;
1565     }
1566 
1567     if (pread(fd, &table, sizeof(table),
1568               vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
1569         error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE");
1570         return;
1571     }
1572 
1573     if (pread(fd, &pba, sizeof(pba),
1574               vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
1575         error_setg_errno(errp, errno, "failed to read PCI MSIX PBA");
1576         return;
1577     }
1578 
1579     ctrl = le16_to_cpu(ctrl);
1580     table = le32_to_cpu(table);
1581     pba = le32_to_cpu(pba);
1582 
1583     msix = g_malloc0(sizeof(*msix));
1584     msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1585     msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1586     msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1587     msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1588     msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1589 
1590     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
1591     if (ret < 0) {
1592         error_setg_errno(errp, -ret, "failed to get MSI-X irq info");
1593         g_free(msix);
1594         return;
1595     }
1596 
1597     msix->noresize = !!(irq_info.flags & VFIO_IRQ_INFO_NORESIZE);
1598 
1599     /*
1600      * Test the size of the pba_offset variable and catch if it extends outside
1601      * of the specified BAR. If it is the case, we need to apply a hardware
1602      * specific quirk if the device is known or we have a broken configuration.
1603      */
1604     if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1605         /*
1606          * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1607          * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1608          * the VF PBA offset while the BAR itself is only 8k. The correct value
1609          * is 0x1000, so we hard code that here.
1610          */
1611         if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1612             (vdev->device_id & 0xff00) == 0x5800) {
1613             msix->pba_offset = 0x1000;
1614         /*
1615          * BAIDU KUNLUN Virtual Function devices for KUNLUN AI processor
1616          * return an incorrect value of 0x460000 for the VF PBA offset while
1617          * the BAR itself is only 0x10000.  The correct value is 0xb400.
1618          */
1619         } else if (vfio_pci_is(vdev, PCI_VENDOR_ID_BAIDU,
1620                                PCI_DEVICE_ID_KUNLUN_VF)) {
1621             msix->pba_offset = 0xb400;
1622         } else if (vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1623             error_setg(errp, "hardware reports invalid configuration, "
1624                        "MSIX PBA outside of specified BAR");
1625             g_free(msix);
1626             return;
1627         }
1628     }
1629 
1630     trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1631                                 msix->table_offset, msix->entries,
1632                                 msix->noresize);
1633     vdev->msix = msix;
1634 
1635     vfio_pci_fixup_msix_region(vdev);
1636 
1637     vfio_pci_relocate_msix(vdev, errp);
1638 }
1639 
1640 static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1641 {
1642     int ret;
1643     Error *err = NULL;
1644 
1645     vdev->msix->pending = g_new0(unsigned long,
1646                                  BITS_TO_LONGS(vdev->msix->entries));
1647     ret = msix_init(&vdev->pdev, vdev->msix->entries,
1648                     vdev->bars[vdev->msix->table_bar].mr,
1649                     vdev->msix->table_bar, vdev->msix->table_offset,
1650                     vdev->bars[vdev->msix->pba_bar].mr,
1651                     vdev->msix->pba_bar, vdev->msix->pba_offset, pos,
1652                     &err);
1653     if (ret < 0) {
1654         if (ret == -ENOTSUP) {
1655             warn_report_err(err);
1656             return 0;
1657         }
1658 
1659         error_propagate(errp, err);
1660         return ret;
1661     }
1662 
1663     /*
1664      * The PCI spec suggests that devices provide additional alignment for
1665      * MSI-X structures and avoid overlapping non-MSI-X related registers.
1666      * For an assigned device, this hopefully means that emulation of MSI-X
1667      * structures does not affect the performance of the device.  If devices
1668      * fail to provide that alignment, a significant performance penalty may
1669      * result, for instance Mellanox MT27500 VFs:
1670      * http://www.spinics.net/lists/kvm/msg125881.html
1671      *
1672      * The PBA is simply not that important for such a serious regression and
1673      * most drivers do not appear to look at it.  The solution for this is to
1674      * disable the PBA MemoryRegion unless it's being used.  We disable it
1675      * here and only enable it if a masked vector fires through QEMU.  As the
1676      * vector-use notifier is called, which occurs on unmask, we test whether
1677      * PBA emulation is needed and again disable if not.
1678      */
1679     memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
1680 
1681     /*
1682      * The emulated machine may provide a paravirt interface for MSIX setup
1683      * so it is not strictly necessary to emulate MSIX here. This becomes
1684      * helpful when frequently accessed MMIO registers are located in
1685      * subpages adjacent to the MSIX table but the MSIX data containing page
1686      * cannot be mapped because of a host page size bigger than the MSIX table
1687      * alignment.
1688      */
1689     if (object_property_get_bool(OBJECT(qdev_get_machine()),
1690                                  "vfio-no-msix-emulation", NULL)) {
1691         memory_region_set_enabled(&vdev->pdev.msix_table_mmio, false);
1692     }
1693 
1694     return 0;
1695 }
1696 
1697 static void vfio_teardown_msi(VFIOPCIDevice *vdev)
1698 {
1699     msi_uninit(&vdev->pdev);
1700 
1701     if (vdev->msix) {
1702         msix_uninit(&vdev->pdev,
1703                     vdev->bars[vdev->msix->table_bar].mr,
1704                     vdev->bars[vdev->msix->pba_bar].mr);
1705         g_free(vdev->msix->pending);
1706     }
1707 }
1708 
1709 /*
1710  * Resource setup
1711  */
1712 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1713 {
1714     int i;
1715 
1716     for (i = 0; i < PCI_ROM_SLOT; i++) {
1717         vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1718     }
1719 }
1720 
1721 static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
1722 {
1723     VFIOBAR *bar = &vdev->bars[nr];
1724 
1725     uint32_t pci_bar;
1726     int ret;
1727 
1728     /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1729     if (!bar->region.size) {
1730         return;
1731     }
1732 
1733     /* Determine what type of BAR this is for registration */
1734     ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
1735                 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
1736     if (ret != sizeof(pci_bar)) {
1737         error_report("vfio: Failed to read BAR %d (%m)", nr);
1738         return;
1739     }
1740 
1741     pci_bar = le32_to_cpu(pci_bar);
1742     bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1743     bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1744     bar->type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1745                                          ~PCI_BASE_ADDRESS_MEM_MASK);
1746     bar->size = bar->region.size;
1747 }
1748 
1749 static void vfio_bars_prepare(VFIOPCIDevice *vdev)
1750 {
1751     int i;
1752 
1753     for (i = 0; i < PCI_ROM_SLOT; i++) {
1754         vfio_bar_prepare(vdev, i);
1755     }
1756 }
1757 
1758 static void vfio_bar_register(VFIOPCIDevice *vdev, int nr)
1759 {
1760     VFIOBAR *bar = &vdev->bars[nr];
1761     char *name;
1762 
1763     if (!bar->size) {
1764         return;
1765     }
1766 
1767     bar->mr = g_new0(MemoryRegion, 1);
1768     name = g_strdup_printf("%s base BAR %d", vdev->vbasedev.name, nr);
1769     memory_region_init_io(bar->mr, OBJECT(vdev), NULL, NULL, name, bar->size);
1770     g_free(name);
1771 
1772     if (bar->region.size) {
1773         memory_region_add_subregion(bar->mr, 0, bar->region.mem);
1774 
1775         if (vfio_region_mmap(&bar->region)) {
1776             error_report("Failed to mmap %s BAR %d. Performance may be slow",
1777                          vdev->vbasedev.name, nr);
1778         }
1779     }
1780 
1781     pci_register_bar(&vdev->pdev, nr, bar->type, bar->mr);
1782 }
1783 
1784 static void vfio_bars_register(VFIOPCIDevice *vdev)
1785 {
1786     int i;
1787 
1788     for (i = 0; i < PCI_ROM_SLOT; i++) {
1789         vfio_bar_register(vdev, i);
1790     }
1791 }
1792 
1793 static void vfio_bars_exit(VFIOPCIDevice *vdev)
1794 {
1795     int i;
1796 
1797     for (i = 0; i < PCI_ROM_SLOT; i++) {
1798         VFIOBAR *bar = &vdev->bars[i];
1799 
1800         vfio_bar_quirk_exit(vdev, i);
1801         vfio_region_exit(&bar->region);
1802         if (bar->region.size) {
1803             memory_region_del_subregion(bar->mr, bar->region.mem);
1804         }
1805     }
1806 
1807     if (vdev->vga) {
1808         pci_unregister_vga(&vdev->pdev);
1809         vfio_vga_quirk_exit(vdev);
1810     }
1811 }
1812 
1813 static void vfio_bars_finalize(VFIOPCIDevice *vdev)
1814 {
1815     int i;
1816 
1817     for (i = 0; i < PCI_ROM_SLOT; i++) {
1818         VFIOBAR *bar = &vdev->bars[i];
1819 
1820         vfio_bar_quirk_finalize(vdev, i);
1821         vfio_region_finalize(&bar->region);
1822         if (bar->mr) {
1823             assert(bar->size);
1824             object_unparent(OBJECT(bar->mr));
1825             g_free(bar->mr);
1826             bar->mr = NULL;
1827         }
1828     }
1829 
1830     if (vdev->vga) {
1831         vfio_vga_quirk_finalize(vdev);
1832         for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
1833             object_unparent(OBJECT(&vdev->vga->region[i].mem));
1834         }
1835         g_free(vdev->vga);
1836     }
1837 }
1838 
1839 /*
1840  * General setup
1841  */
1842 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
1843 {
1844     uint8_t tmp;
1845     uint16_t next = PCI_CONFIG_SPACE_SIZE;
1846 
1847     for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
1848          tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
1849         if (tmp > pos && tmp < next) {
1850             next = tmp;
1851         }
1852     }
1853 
1854     return next - pos;
1855 }
1856 
1857 
1858 static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
1859 {
1860     uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
1861 
1862     for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
1863         tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
1864         if (tmp > pos && tmp < next) {
1865             next = tmp;
1866         }
1867     }
1868 
1869     return next - pos;
1870 }
1871 
1872 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
1873 {
1874     pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
1875 }
1876 
1877 static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
1878                                    uint16_t val, uint16_t mask)
1879 {
1880     vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
1881     vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
1882     vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
1883 }
1884 
1885 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
1886 {
1887     pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
1888 }
1889 
1890 static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
1891                                    uint32_t val, uint32_t mask)
1892 {
1893     vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
1894     vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
1895     vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
1896 }
1897 
1898 static void vfio_pci_enable_rp_atomics(VFIOPCIDevice *vdev)
1899 {
1900     struct vfio_device_info_cap_pci_atomic_comp *cap;
1901     g_autofree struct vfio_device_info *info = NULL;
1902     PCIBus *bus = pci_get_bus(&vdev->pdev);
1903     PCIDevice *parent = bus->parent_dev;
1904     struct vfio_info_cap_header *hdr;
1905     uint32_t mask = 0;
1906     uint8_t *pos;
1907 
1908     /*
1909      * PCIe Atomic Ops completer support is only added automatically for single
1910      * function devices downstream of a root port supporting DEVCAP2.  Support
1911      * is added during realize and, if added, removed during device exit.  The
1912      * single function requirement avoids conflicting requirements should a
1913      * slot be composed of multiple devices with differing capabilities.
1914      */
1915     if (pci_bus_is_root(bus) || !parent || !parent->exp.exp_cap ||
1916         pcie_cap_get_type(parent) != PCI_EXP_TYPE_ROOT_PORT ||
1917         pcie_cap_get_version(parent) != PCI_EXP_FLAGS_VER2 ||
1918         vdev->pdev.devfn ||
1919         vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
1920         return;
1921     }
1922 
1923     pos = parent->config + parent->exp.exp_cap + PCI_EXP_DEVCAP2;
1924 
1925     /* Abort if there'a already an Atomic Ops configuration on the root port */
1926     if (pci_get_long(pos) & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
1927                              PCI_EXP_DEVCAP2_ATOMIC_COMP64 |
1928                              PCI_EXP_DEVCAP2_ATOMIC_COMP128)) {
1929         return;
1930     }
1931 
1932     info = vfio_get_device_info(vdev->vbasedev.fd);
1933     if (!info) {
1934         return;
1935     }
1936 
1937     hdr = vfio_get_device_info_cap(info, VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP);
1938     if (!hdr) {
1939         return;
1940     }
1941 
1942     cap = (void *)hdr;
1943     if (cap->flags & VFIO_PCI_ATOMIC_COMP32) {
1944         mask |= PCI_EXP_DEVCAP2_ATOMIC_COMP32;
1945     }
1946     if (cap->flags & VFIO_PCI_ATOMIC_COMP64) {
1947         mask |= PCI_EXP_DEVCAP2_ATOMIC_COMP64;
1948     }
1949     if (cap->flags & VFIO_PCI_ATOMIC_COMP128) {
1950         mask |= PCI_EXP_DEVCAP2_ATOMIC_COMP128;
1951     }
1952 
1953     if (!mask) {
1954         return;
1955     }
1956 
1957     pci_long_test_and_set_mask(pos, mask);
1958     vdev->clear_parent_atomics_on_exit = true;
1959 }
1960 
1961 static void vfio_pci_disable_rp_atomics(VFIOPCIDevice *vdev)
1962 {
1963     if (vdev->clear_parent_atomics_on_exit) {
1964         PCIDevice *parent = pci_get_bus(&vdev->pdev)->parent_dev;
1965         uint8_t *pos = parent->config + parent->exp.exp_cap + PCI_EXP_DEVCAP2;
1966 
1967         pci_long_test_and_clear_mask(pos, PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
1968                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64 |
1969                                           PCI_EXP_DEVCAP2_ATOMIC_COMP128);
1970     }
1971 }
1972 
1973 static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
1974                                Error **errp)
1975 {
1976     uint16_t flags;
1977     uint8_t type;
1978 
1979     flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
1980     type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
1981 
1982     if (type != PCI_EXP_TYPE_ENDPOINT &&
1983         type != PCI_EXP_TYPE_LEG_END &&
1984         type != PCI_EXP_TYPE_RC_END) {
1985 
1986         error_setg(errp, "assignment of PCIe type 0x%x "
1987                    "devices is not currently supported", type);
1988         return -EINVAL;
1989     }
1990 
1991     if (!pci_bus_is_express(pci_get_bus(&vdev->pdev))) {
1992         PCIBus *bus = pci_get_bus(&vdev->pdev);
1993         PCIDevice *bridge;
1994 
1995         /*
1996          * Traditionally PCI device assignment exposes the PCIe capability
1997          * as-is on non-express buses.  The reason being that some drivers
1998          * simply assume that it's there, for example tg3.  However when
1999          * we're running on a native PCIe machine type, like Q35, we need
2000          * to hide the PCIe capability.  The reason for this is twofold;
2001          * first Windows guests get a Code 10 error when the PCIe capability
2002          * is exposed in this configuration.  Therefore express devices won't
2003          * work at all unless they're attached to express buses in the VM.
2004          * Second, a native PCIe machine introduces the possibility of fine
2005          * granularity IOMMUs supporting both translation and isolation.
2006          * Guest code to discover the IOMMU visibility of a device, such as
2007          * IOMMU grouping code on Linux, is very aware of device types and
2008          * valid transitions between bus types.  An express device on a non-
2009          * express bus is not a valid combination on bare metal systems.
2010          *
2011          * Drivers that require a PCIe capability to make the device
2012          * functional are simply going to need to have their devices placed
2013          * on a PCIe bus in the VM.
2014          */
2015         while (!pci_bus_is_root(bus)) {
2016             bridge = pci_bridge_get_device(bus);
2017             bus = pci_get_bus(bridge);
2018         }
2019 
2020         if (pci_bus_is_express(bus)) {
2021             return 0;
2022         }
2023 
2024     } else if (pci_bus_is_root(pci_get_bus(&vdev->pdev))) {
2025         /*
2026          * On a Root Complex bus Endpoints become Root Complex Integrated
2027          * Endpoints, which changes the type and clears the LNK & LNK2 fields.
2028          */
2029         if (type == PCI_EXP_TYPE_ENDPOINT) {
2030             vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2031                                    PCI_EXP_TYPE_RC_END << 4,
2032                                    PCI_EXP_FLAGS_TYPE);
2033 
2034             /* Link Capabilities, Status, and Control goes away */
2035             if (size > PCI_EXP_LNKCTL) {
2036                 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
2037                 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2038                 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
2039 
2040 #ifndef PCI_EXP_LNKCAP2
2041 #define PCI_EXP_LNKCAP2 44
2042 #endif
2043 #ifndef PCI_EXP_LNKSTA2
2044 #define PCI_EXP_LNKSTA2 50
2045 #endif
2046                 /* Link 2 Capabilities, Status, and Control goes away */
2047                 if (size > PCI_EXP_LNKCAP2) {
2048                     vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
2049                     vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
2050                     vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
2051                 }
2052             }
2053 
2054         } else if (type == PCI_EXP_TYPE_LEG_END) {
2055             /*
2056              * Legacy endpoints don't belong on the root complex.  Windows
2057              * seems to be happier with devices if we skip the capability.
2058              */
2059             return 0;
2060         }
2061 
2062     } else {
2063         /*
2064          * Convert Root Complex Integrated Endpoints to regular endpoints.
2065          * These devices don't support LNK/LNK2 capabilities, so make them up.
2066          */
2067         if (type == PCI_EXP_TYPE_RC_END) {
2068             vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2069                                    PCI_EXP_TYPE_ENDPOINT << 4,
2070                                    PCI_EXP_FLAGS_TYPE);
2071             vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
2072                            QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1) |
2073                            QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT), ~0);
2074             vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2075         }
2076 
2077         vfio_pci_enable_rp_atomics(vdev);
2078     }
2079 
2080     /*
2081      * Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0
2082      * (Niantic errate #35) causing Windows to error with a Code 10 for the
2083      * device on Q35.  Fixup any such devices to report version 1.  If we
2084      * were to remove the capability entirely the guest would lose extended
2085      * config space.
2086      */
2087     if ((flags & PCI_EXP_FLAGS_VERS) == 0) {
2088         vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2089                                1, PCI_EXP_FLAGS_VERS);
2090     }
2091 
2092     pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size,
2093                              errp);
2094     if (pos < 0) {
2095         return pos;
2096     }
2097 
2098     vdev->pdev.exp.exp_cap = pos;
2099 
2100     return pos;
2101 }
2102 
2103 static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
2104 {
2105     uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
2106 
2107     if (cap & PCI_EXP_DEVCAP_FLR) {
2108         trace_vfio_check_pcie_flr(vdev->vbasedev.name);
2109         vdev->has_flr = true;
2110     }
2111 }
2112 
2113 static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
2114 {
2115     uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
2116 
2117     if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
2118         trace_vfio_check_pm_reset(vdev->vbasedev.name);
2119         vdev->has_pm_reset = true;
2120     }
2121 }
2122 
2123 static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
2124 {
2125     uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
2126 
2127     if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
2128         trace_vfio_check_af_flr(vdev->vbasedev.name);
2129         vdev->has_flr = true;
2130     }
2131 }
2132 
2133 static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp)
2134 {
2135     PCIDevice *pdev = &vdev->pdev;
2136     uint8_t cap_id, next, size;
2137     int ret;
2138 
2139     cap_id = pdev->config[pos];
2140     next = pdev->config[pos + PCI_CAP_LIST_NEXT];
2141 
2142     /*
2143      * If it becomes important to configure capabilities to their actual
2144      * size, use this as the default when it's something we don't recognize.
2145      * Since QEMU doesn't actually handle many of the config accesses,
2146      * exact size doesn't seem worthwhile.
2147      */
2148     size = vfio_std_cap_max_size(pdev, pos);
2149 
2150     /*
2151      * pci_add_capability always inserts the new capability at the head
2152      * of the chain.  Therefore to end up with a chain that matches the
2153      * physical device, we insert from the end by making this recursive.
2154      * This is also why we pre-calculate size above as cached config space
2155      * will be changed as we unwind the stack.
2156      */
2157     if (next) {
2158         ret = vfio_add_std_cap(vdev, next, errp);
2159         if (ret) {
2160             return ret;
2161         }
2162     } else {
2163         /* Begin the rebuild, use QEMU emulated list bits */
2164         pdev->config[PCI_CAPABILITY_LIST] = 0;
2165         vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
2166         vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
2167 
2168         ret = vfio_add_virt_caps(vdev, errp);
2169         if (ret) {
2170             return ret;
2171         }
2172     }
2173 
2174     /* Scale down size, esp in case virt caps were added above */
2175     size = MIN(size, vfio_std_cap_max_size(pdev, pos));
2176 
2177     /* Use emulated next pointer to allow dropping caps */
2178     pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
2179 
2180     switch (cap_id) {
2181     case PCI_CAP_ID_MSI:
2182         ret = vfio_msi_setup(vdev, pos, errp);
2183         break;
2184     case PCI_CAP_ID_EXP:
2185         vfio_check_pcie_flr(vdev, pos);
2186         ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
2187         break;
2188     case PCI_CAP_ID_MSIX:
2189         ret = vfio_msix_setup(vdev, pos, errp);
2190         break;
2191     case PCI_CAP_ID_PM:
2192         vfio_check_pm_reset(vdev, pos);
2193         vdev->pm_cap = pos;
2194         ret = pci_add_capability(pdev, cap_id, pos, size, errp);
2195         break;
2196     case PCI_CAP_ID_AF:
2197         vfio_check_af_flr(vdev, pos);
2198         ret = pci_add_capability(pdev, cap_id, pos, size, errp);
2199         break;
2200     default:
2201         ret = pci_add_capability(pdev, cap_id, pos, size, errp);
2202         break;
2203     }
2204 
2205     if (ret < 0) {
2206         error_prepend(errp,
2207                       "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
2208                       cap_id, size, pos);
2209         return ret;
2210     }
2211 
2212     return 0;
2213 }
2214 
2215 static int vfio_setup_rebar_ecap(VFIOPCIDevice *vdev, uint16_t pos)
2216 {
2217     uint32_t ctrl;
2218     int i, nbar;
2219 
2220     ctrl = pci_get_long(vdev->pdev.config + pos + PCI_REBAR_CTRL);
2221     nbar = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> PCI_REBAR_CTRL_NBAR_SHIFT;
2222 
2223     for (i = 0; i < nbar; i++) {
2224         uint32_t cap;
2225         int size;
2226 
2227         ctrl = pci_get_long(vdev->pdev.config + pos + PCI_REBAR_CTRL + (i * 8));
2228         size = (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT;
2229 
2230         /* The cap register reports sizes 1MB to 128TB, with 4 reserved bits */
2231         cap = size <= 27 ? 1U << (size + 4) : 0;
2232 
2233         /*
2234          * The PCIe spec (v6.0.1, 7.8.6) requires HW to support at least one
2235          * size in the range 1MB to 512GB.  We intend to mask all sizes except
2236          * the one currently enabled in the size field, therefore if it's
2237          * outside the range, hide the whole capability as this virtualization
2238          * trick won't work.  If >512GB resizable BARs start to appear, we
2239          * might need an opt-in or reservation scheme in the kernel.
2240          */
2241         if (!(cap & PCI_REBAR_CAP_SIZES)) {
2242             return -EINVAL;
2243         }
2244 
2245         /* Hide all sizes reported in the ctrl reg per above requirement. */
2246         ctrl &= (PCI_REBAR_CTRL_BAR_SIZE |
2247                  PCI_REBAR_CTRL_NBAR_MASK |
2248                  PCI_REBAR_CTRL_BAR_IDX);
2249 
2250         /*
2251          * The BAR size field is RW, however we've mangled the capability
2252          * register such that we only report a single size, ie. the current
2253          * BAR size.  A write of an unsupported value is undefined, therefore
2254          * the register field is essentially RO.
2255          */
2256         vfio_add_emulated_long(vdev, pos + PCI_REBAR_CAP + (i * 8), cap, ~0);
2257         vfio_add_emulated_long(vdev, pos + PCI_REBAR_CTRL + (i * 8), ctrl, ~0);
2258     }
2259 
2260     return 0;
2261 }
2262 
2263 static void vfio_add_ext_cap(VFIOPCIDevice *vdev)
2264 {
2265     PCIDevice *pdev = &vdev->pdev;
2266     uint32_t header;
2267     uint16_t cap_id, next, size;
2268     uint8_t cap_ver;
2269     uint8_t *config;
2270 
2271     /* Only add extended caps if we have them and the guest can see them */
2272     if (!pci_is_express(pdev) || !pci_bus_is_express(pci_get_bus(pdev)) ||
2273         !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
2274         return;
2275     }
2276 
2277     /*
2278      * pcie_add_capability always inserts the new capability at the tail
2279      * of the chain.  Therefore to end up with a chain that matches the
2280      * physical device, we cache the config space to avoid overwriting
2281      * the original config space when we parse the extended capabilities.
2282      */
2283     config = g_memdup(pdev->config, vdev->config_size);
2284 
2285     /*
2286      * Extended capabilities are chained with each pointing to the next, so we
2287      * can drop anything other than the head of the chain simply by modifying
2288      * the previous next pointer.  Seed the head of the chain here such that
2289      * we can simply skip any capabilities we want to drop below, regardless
2290      * of their position in the chain.  If this stub capability still exists
2291      * after we add the capabilities we want to expose, update the capability
2292      * ID to zero.  Note that we cannot seed with the capability header being
2293      * zero as this conflicts with definition of an absent capability chain
2294      * and prevents capabilities beyond the head of the list from being added.
2295      * By replacing the dummy capability ID with zero after walking the device
2296      * chain, we also transparently mark extended capabilities as absent if
2297      * no capabilities were added.  Note that the PCIe spec defines an absence
2298      * of extended capabilities to be determined by a value of zero for the
2299      * capability ID, version, AND next pointer.  A non-zero next pointer
2300      * should be sufficient to indicate additional capabilities are present,
2301      * which will occur if we call pcie_add_capability() below.  The entire
2302      * first dword is emulated to support this.
2303      *
2304      * NB. The kernel side does similar masking, so be prepared that our
2305      * view of the device may also contain a capability ID zero in the head
2306      * of the chain.  Skip it for the same reason that we cannot seed the
2307      * chain with a zero capability.
2308      */
2309     pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
2310                  PCI_EXT_CAP(0xFFFF, 0, 0));
2311     pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
2312     pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
2313 
2314     for (next = PCI_CONFIG_SPACE_SIZE; next;
2315          next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
2316         header = pci_get_long(config + next);
2317         cap_id = PCI_EXT_CAP_ID(header);
2318         cap_ver = PCI_EXT_CAP_VER(header);
2319 
2320         /*
2321          * If it becomes important to configure extended capabilities to their
2322          * actual size, use this as the default when it's something we don't
2323          * recognize. Since QEMU doesn't actually handle many of the config
2324          * accesses, exact size doesn't seem worthwhile.
2325          */
2326         size = vfio_ext_cap_max_size(config, next);
2327 
2328         /* Use emulated next pointer to allow dropping extended caps */
2329         pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
2330                                    PCI_EXT_CAP_NEXT_MASK);
2331 
2332         switch (cap_id) {
2333         case 0: /* kernel masked capability */
2334         case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
2335         case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
2336             trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
2337             break;
2338         case PCI_EXT_CAP_ID_REBAR:
2339             if (!vfio_setup_rebar_ecap(vdev, next)) {
2340                 pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2341             }
2342             break;
2343         default:
2344             pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2345         }
2346 
2347     }
2348 
2349     /* Cleanup chain head ID if necessary */
2350     if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
2351         pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
2352     }
2353 
2354     g_free(config);
2355     return;
2356 }
2357 
2358 static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
2359 {
2360     PCIDevice *pdev = &vdev->pdev;
2361     int ret;
2362 
2363     if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
2364         !pdev->config[PCI_CAPABILITY_LIST]) {
2365         return 0; /* Nothing to add */
2366     }
2367 
2368     ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp);
2369     if (ret) {
2370         return ret;
2371     }
2372 
2373     vfio_add_ext_cap(vdev);
2374     return 0;
2375 }
2376 
2377 static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
2378 {
2379     PCIDevice *pdev = &vdev->pdev;
2380     uint16_t cmd;
2381 
2382     vfio_disable_interrupts(vdev);
2383 
2384     /* Make sure the device is in D0 */
2385     if (vdev->pm_cap) {
2386         uint16_t pmcsr;
2387         uint8_t state;
2388 
2389         pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2390         state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2391         if (state) {
2392             pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2393             vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
2394             /* vfio handles the necessary delay here */
2395             pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2396             state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2397             if (state) {
2398                 error_report("vfio: Unable to power on device, stuck in D%d",
2399                              state);
2400             }
2401         }
2402     }
2403 
2404     /*
2405      * Stop any ongoing DMA by disconnecting I/O, MMIO, and bus master.
2406      * Also put INTx Disable in known state.
2407      */
2408     cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
2409     cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
2410              PCI_COMMAND_INTX_DISABLE);
2411     vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2412 }
2413 
2414 static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
2415 {
2416     Error *err = NULL;
2417     int nr;
2418 
2419     vfio_intx_enable(vdev, &err);
2420     if (err) {
2421         error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2422     }
2423 
2424     for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2425         off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr);
2426         uint32_t val = 0;
2427         uint32_t len = sizeof(val);
2428 
2429         if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) {
2430             error_report("%s(%s) reset bar %d failed: %m", __func__,
2431                          vdev->vbasedev.name, nr);
2432         }
2433     }
2434 
2435     vfio_quirk_reset(vdev);
2436 }
2437 
2438 static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
2439 {
2440     char tmp[13];
2441 
2442     sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2443             addr->bus, addr->slot, addr->function);
2444 
2445     return (strcmp(tmp, name) == 0);
2446 }
2447 
2448 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2449 {
2450     VFIOGroup *group;
2451     struct vfio_pci_hot_reset_info *info;
2452     struct vfio_pci_dependent_device *devices;
2453     struct vfio_pci_hot_reset *reset;
2454     int32_t *fds;
2455     int ret, i, count;
2456     bool multi = false;
2457 
2458     trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
2459 
2460     if (!single) {
2461         vfio_pci_pre_reset(vdev);
2462     }
2463     vdev->vbasedev.needs_reset = false;
2464 
2465     info = g_malloc0(sizeof(*info));
2466     info->argsz = sizeof(*info);
2467 
2468     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2469     if (ret && errno != ENOSPC) {
2470         ret = -errno;
2471         if (!vdev->has_pm_reset) {
2472             error_report("vfio: Cannot reset device %s, "
2473                          "no available reset mechanism.", vdev->vbasedev.name);
2474         }
2475         goto out_single;
2476     }
2477 
2478     count = info->count;
2479     info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2480     info->argsz = sizeof(*info) + (count * sizeof(*devices));
2481     devices = &info->devices[0];
2482 
2483     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2484     if (ret) {
2485         ret = -errno;
2486         error_report("vfio: hot reset info failed: %m");
2487         goto out_single;
2488     }
2489 
2490     trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
2491 
2492     /* Verify that we have all the groups required */
2493     for (i = 0; i < info->count; i++) {
2494         PCIHostDeviceAddress host;
2495         VFIOPCIDevice *tmp;
2496         VFIODevice *vbasedev_iter;
2497 
2498         host.domain = devices[i].segment;
2499         host.bus = devices[i].bus;
2500         host.slot = PCI_SLOT(devices[i].devfn);
2501         host.function = PCI_FUNC(devices[i].devfn);
2502 
2503         trace_vfio_pci_hot_reset_dep_devices(host.domain,
2504                 host.bus, host.slot, host.function, devices[i].group_id);
2505 
2506         if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2507             continue;
2508         }
2509 
2510         QLIST_FOREACH(group, &vfio_group_list, next) {
2511             if (group->groupid == devices[i].group_id) {
2512                 break;
2513             }
2514         }
2515 
2516         if (!group) {
2517             if (!vdev->has_pm_reset) {
2518                 error_report("vfio: Cannot reset device %s, "
2519                              "depends on group %d which is not owned.",
2520                              vdev->vbasedev.name, devices[i].group_id);
2521             }
2522             ret = -EPERM;
2523             goto out;
2524         }
2525 
2526         /* Prep dependent devices for reset and clear our marker. */
2527         QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2528             if (!vbasedev_iter->dev->realized ||
2529                 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2530                 continue;
2531             }
2532             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2533             if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2534                 if (single) {
2535                     ret = -EINVAL;
2536                     goto out_single;
2537                 }
2538                 vfio_pci_pre_reset(tmp);
2539                 tmp->vbasedev.needs_reset = false;
2540                 multi = true;
2541                 break;
2542             }
2543         }
2544     }
2545 
2546     if (!single && !multi) {
2547         ret = -EINVAL;
2548         goto out_single;
2549     }
2550 
2551     /* Determine how many group fds need to be passed */
2552     count = 0;
2553     QLIST_FOREACH(group, &vfio_group_list, next) {
2554         for (i = 0; i < info->count; i++) {
2555             if (group->groupid == devices[i].group_id) {
2556                 count++;
2557                 break;
2558             }
2559         }
2560     }
2561 
2562     reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
2563     reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
2564     fds = &reset->group_fds[0];
2565 
2566     /* Fill in group fds */
2567     QLIST_FOREACH(group, &vfio_group_list, next) {
2568         for (i = 0; i < info->count; i++) {
2569             if (group->groupid == devices[i].group_id) {
2570                 fds[reset->count++] = group->fd;
2571                 break;
2572             }
2573         }
2574     }
2575 
2576     /* Bus reset! */
2577     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
2578     g_free(reset);
2579 
2580     trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
2581                                     ret ? strerror(errno) : "Success");
2582 
2583 out:
2584     /* Re-enable INTx on affected devices */
2585     for (i = 0; i < info->count; i++) {
2586         PCIHostDeviceAddress host;
2587         VFIOPCIDevice *tmp;
2588         VFIODevice *vbasedev_iter;
2589 
2590         host.domain = devices[i].segment;
2591         host.bus = devices[i].bus;
2592         host.slot = PCI_SLOT(devices[i].devfn);
2593         host.function = PCI_FUNC(devices[i].devfn);
2594 
2595         if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2596             continue;
2597         }
2598 
2599         QLIST_FOREACH(group, &vfio_group_list, next) {
2600             if (group->groupid == devices[i].group_id) {
2601                 break;
2602             }
2603         }
2604 
2605         if (!group) {
2606             break;
2607         }
2608 
2609         QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2610             if (!vbasedev_iter->dev->realized ||
2611                 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2612                 continue;
2613             }
2614             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2615             if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2616                 vfio_pci_post_reset(tmp);
2617                 break;
2618             }
2619         }
2620     }
2621 out_single:
2622     if (!single) {
2623         vfio_pci_post_reset(vdev);
2624     }
2625     g_free(info);
2626 
2627     return ret;
2628 }
2629 
2630 /*
2631  * We want to differentiate hot reset of multiple in-use devices vs hot reset
2632  * of a single in-use device.  VFIO_DEVICE_RESET will already handle the case
2633  * of doing hot resets when there is only a single device per bus.  The in-use
2634  * here refers to how many VFIODevices are affected.  A hot reset that affects
2635  * multiple devices, but only a single in-use device, means that we can call
2636  * it from our bus ->reset() callback since the extent is effectively a single
2637  * device.  This allows us to make use of it in the hotplug path.  When there
2638  * are multiple in-use devices, we can only trigger the hot reset during a
2639  * system reset and thus from our reset handler.  We separate _one vs _multi
2640  * here so that we don't overlap and do a double reset on the system reset
2641  * path where both our reset handler and ->reset() callback are used.  Calling
2642  * _one() will only do a hot reset for the one in-use devices case, calling
2643  * _multi() will do nothing if a _one() would have been sufficient.
2644  */
2645 static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2646 {
2647     return vfio_pci_hot_reset(vdev, true);
2648 }
2649 
2650 static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2651 {
2652     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2653     return vfio_pci_hot_reset(vdev, false);
2654 }
2655 
2656 static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2657 {
2658     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2659     if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
2660         vbasedev->needs_reset = true;
2661     }
2662 }
2663 
2664 static Object *vfio_pci_get_object(VFIODevice *vbasedev)
2665 {
2666     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2667 
2668     return OBJECT(vdev);
2669 }
2670 
2671 static bool vfio_msix_present(void *opaque, int version_id)
2672 {
2673     PCIDevice *pdev = opaque;
2674 
2675     return msix_present(pdev);
2676 }
2677 
2678 static bool vfio_display_migration_needed(void *opaque)
2679 {
2680     VFIOPCIDevice *vdev = opaque;
2681 
2682     /*
2683      * We need to migrate the VFIODisplay object if ramfb *migration* was
2684      * explicitly requested (in which case we enforced both ramfb=on and
2685      * display=on), or ramfb migration was left at the default "auto"
2686      * setting, and *ramfb* was explicitly requested (in which case we
2687      * enforced display=on).
2688      */
2689     return vdev->ramfb_migrate == ON_OFF_AUTO_ON ||
2690         (vdev->ramfb_migrate == ON_OFF_AUTO_AUTO && vdev->enable_ramfb);
2691 }
2692 
2693 const VMStateDescription vmstate_vfio_display = {
2694     .name = "VFIOPCIDevice/VFIODisplay",
2695     .version_id = 1,
2696     .minimum_version_id = 1,
2697     .needed = vfio_display_migration_needed,
2698     .fields = (VMStateField[]){
2699         VMSTATE_STRUCT_POINTER(dpy, VFIOPCIDevice, vfio_display_vmstate,
2700                                VFIODisplay),
2701         VMSTATE_END_OF_LIST()
2702     }
2703 };
2704 
2705 const VMStateDescription vmstate_vfio_pci_config = {
2706     .name = "VFIOPCIDevice",
2707     .version_id = 1,
2708     .minimum_version_id = 1,
2709     .fields = (VMStateField[]) {
2710         VMSTATE_PCI_DEVICE(pdev, VFIOPCIDevice),
2711         VMSTATE_MSIX_TEST(pdev, VFIOPCIDevice, vfio_msix_present),
2712         VMSTATE_END_OF_LIST()
2713     },
2714     .subsections = (const VMStateDescription * []) {
2715         &vmstate_vfio_display,
2716         NULL
2717     }
2718 };
2719 
2720 static void vfio_pci_save_config(VFIODevice *vbasedev, QEMUFile *f)
2721 {
2722     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2723 
2724     vmstate_save_state(f, &vmstate_vfio_pci_config, vdev, NULL);
2725 }
2726 
2727 static int vfio_pci_load_config(VFIODevice *vbasedev, QEMUFile *f)
2728 {
2729     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2730     PCIDevice *pdev = &vdev->pdev;
2731     pcibus_t old_addr[PCI_NUM_REGIONS - 1];
2732     int bar, ret;
2733 
2734     for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
2735         old_addr[bar] = pdev->io_regions[bar].addr;
2736     }
2737 
2738     ret = vmstate_load_state(f, &vmstate_vfio_pci_config, vdev, 1);
2739     if (ret) {
2740         return ret;
2741     }
2742 
2743     vfio_pci_write_config(pdev, PCI_COMMAND,
2744                           pci_get_word(pdev->config + PCI_COMMAND), 2);
2745 
2746     for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
2747         /*
2748          * The address may not be changed in some scenarios
2749          * (e.g. the VF driver isn't loaded in VM).
2750          */
2751         if (old_addr[bar] != pdev->io_regions[bar].addr &&
2752             vdev->bars[bar].region.size > 0 &&
2753             vdev->bars[bar].region.size < qemu_real_host_page_size()) {
2754             vfio_sub_page_bar_update_mapping(pdev, bar);
2755         }
2756     }
2757 
2758     if (msi_enabled(pdev)) {
2759         vfio_msi_enable(vdev);
2760     } else if (msix_enabled(pdev)) {
2761         vfio_msix_enable(vdev);
2762     }
2763 
2764     return ret;
2765 }
2766 
2767 static VFIODeviceOps vfio_pci_ops = {
2768     .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
2769     .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
2770     .vfio_eoi = vfio_intx_eoi,
2771     .vfio_get_object = vfio_pci_get_object,
2772     .vfio_save_config = vfio_pci_save_config,
2773     .vfio_load_config = vfio_pci_load_config,
2774 };
2775 
2776 int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp)
2777 {
2778     VFIODevice *vbasedev = &vdev->vbasedev;
2779     struct vfio_region_info *reg_info;
2780     int ret;
2781 
2782     ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
2783     if (ret) {
2784         error_setg_errno(errp, -ret,
2785                          "failed getting region info for VGA region index %d",
2786                          VFIO_PCI_VGA_REGION_INDEX);
2787         return ret;
2788     }
2789 
2790     if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
2791         !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
2792         reg_info->size < 0xbffff + 1) {
2793         error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2794                    (unsigned long)reg_info->flags,
2795                    (unsigned long)reg_info->size);
2796         g_free(reg_info);
2797         return -EINVAL;
2798     }
2799 
2800     vdev->vga = g_new0(VFIOVGA, 1);
2801 
2802     vdev->vga->fd_offset = reg_info->offset;
2803     vdev->vga->fd = vdev->vbasedev.fd;
2804 
2805     g_free(reg_info);
2806 
2807     vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
2808     vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
2809     QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
2810 
2811     memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2812                           OBJECT(vdev), &vfio_vga_ops,
2813                           &vdev->vga->region[QEMU_PCI_VGA_MEM],
2814                           "vfio-vga-mmio@0xa0000",
2815                           QEMU_PCI_VGA_MEM_SIZE);
2816 
2817     vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
2818     vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
2819     QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
2820 
2821     memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2822                           OBJECT(vdev), &vfio_vga_ops,
2823                           &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
2824                           "vfio-vga-io@0x3b0",
2825                           QEMU_PCI_VGA_IO_LO_SIZE);
2826 
2827     vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
2828     vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
2829     QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
2830 
2831     memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
2832                           OBJECT(vdev), &vfio_vga_ops,
2833                           &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
2834                           "vfio-vga-io@0x3c0",
2835                           QEMU_PCI_VGA_IO_HI_SIZE);
2836 
2837     pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2838                      &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2839                      &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
2840 
2841     return 0;
2842 }
2843 
2844 static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
2845 {
2846     VFIODevice *vbasedev = &vdev->vbasedev;
2847     struct vfio_region_info *reg_info;
2848     struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
2849     int i, ret = -1;
2850 
2851     /* Sanity check device */
2852     if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
2853         error_setg(errp, "this isn't a PCI device");
2854         return;
2855     }
2856 
2857     if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
2858         error_setg(errp, "unexpected number of io regions %u",
2859                    vbasedev->num_regions);
2860         return;
2861     }
2862 
2863     if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
2864         error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
2865         return;
2866     }
2867 
2868     for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2869         char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
2870 
2871         ret = vfio_region_setup(OBJECT(vdev), vbasedev,
2872                                 &vdev->bars[i].region, i, name);
2873         g_free(name);
2874 
2875         if (ret) {
2876             error_setg_errno(errp, -ret, "failed to get region %d info", i);
2877             return;
2878         }
2879 
2880         QLIST_INIT(&vdev->bars[i].quirks);
2881     }
2882 
2883     ret = vfio_get_region_info(vbasedev,
2884                                VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
2885     if (ret) {
2886         error_setg_errno(errp, -ret, "failed to get config info");
2887         return;
2888     }
2889 
2890     trace_vfio_populate_device_config(vdev->vbasedev.name,
2891                                       (unsigned long)reg_info->size,
2892                                       (unsigned long)reg_info->offset,
2893                                       (unsigned long)reg_info->flags);
2894 
2895     vdev->config_size = reg_info->size;
2896     if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
2897         vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2898     }
2899     vdev->config_offset = reg_info->offset;
2900 
2901     g_free(reg_info);
2902 
2903     if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
2904         ret = vfio_populate_vga(vdev, errp);
2905         if (ret) {
2906             error_append_hint(errp, "device does not support "
2907                               "requested feature x-vga\n");
2908             return;
2909         }
2910     }
2911 
2912     irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
2913 
2914     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
2915     if (ret) {
2916         /* This can fail for an old kernel or legacy PCI dev */
2917         trace_vfio_populate_device_get_irq_info_failure(strerror(errno));
2918     } else if (irq_info.count == 1) {
2919         vdev->pci_aer = true;
2920     } else {
2921         warn_report(VFIO_MSG_PREFIX
2922                     "Could not enable error recovery for the device",
2923                     vbasedev->name);
2924     }
2925 }
2926 
2927 static void vfio_pci_put_device(VFIOPCIDevice *vdev)
2928 {
2929     vfio_detach_device(&vdev->vbasedev);
2930 
2931     g_free(vdev->vbasedev.name);
2932     g_free(vdev->msix);
2933 }
2934 
2935 static void vfio_err_notifier_handler(void *opaque)
2936 {
2937     VFIOPCIDevice *vdev = opaque;
2938 
2939     if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
2940         return;
2941     }
2942 
2943     /*
2944      * TBD. Retrieve the error details and decide what action
2945      * needs to be taken. One of the actions could be to pass
2946      * the error to the guest and have the guest driver recover
2947      * from the error. This requires that PCIe capabilities be
2948      * exposed to the guest. For now, we just terminate the
2949      * guest to contain the error.
2950      */
2951 
2952     error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
2953 
2954     vm_stop(RUN_STATE_INTERNAL_ERROR);
2955 }
2956 
2957 /*
2958  * Registers error notifier for devices supporting error recovery.
2959  * If we encounter a failure in this function, we report an error
2960  * and continue after disabling error recovery support for the
2961  * device.
2962  */
2963 static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
2964 {
2965     Error *err = NULL;
2966     int32_t fd;
2967 
2968     if (!vdev->pci_aer) {
2969         return;
2970     }
2971 
2972     if (event_notifier_init(&vdev->err_notifier, 0)) {
2973         error_report("vfio: Unable to init event notifier for error detection");
2974         vdev->pci_aer = false;
2975         return;
2976     }
2977 
2978     fd = event_notifier_get_fd(&vdev->err_notifier);
2979     qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
2980 
2981     if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2982                                VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2983         error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2984         qemu_set_fd_handler(fd, NULL, NULL, vdev);
2985         event_notifier_cleanup(&vdev->err_notifier);
2986         vdev->pci_aer = false;
2987     }
2988 }
2989 
2990 static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
2991 {
2992     Error *err = NULL;
2993 
2994     if (!vdev->pci_aer) {
2995         return;
2996     }
2997 
2998     if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2999                                VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
3000         error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3001     }
3002     qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
3003                         NULL, NULL, vdev);
3004     event_notifier_cleanup(&vdev->err_notifier);
3005 }
3006 
3007 static void vfio_req_notifier_handler(void *opaque)
3008 {
3009     VFIOPCIDevice *vdev = opaque;
3010     Error *err = NULL;
3011 
3012     if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
3013         return;
3014     }
3015 
3016     qdev_unplug(DEVICE(vdev), &err);
3017     if (err) {
3018         warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3019     }
3020 }
3021 
3022 static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
3023 {
3024     struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
3025                                       .index = VFIO_PCI_REQ_IRQ_INDEX };
3026     Error *err = NULL;
3027     int32_t fd;
3028 
3029     if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
3030         return;
3031     }
3032 
3033     if (ioctl(vdev->vbasedev.fd,
3034               VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
3035         return;
3036     }
3037 
3038     if (event_notifier_init(&vdev->req_notifier, 0)) {
3039         error_report("vfio: Unable to init event notifier for device request");
3040         return;
3041     }
3042 
3043     fd = event_notifier_get_fd(&vdev->req_notifier);
3044     qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
3045 
3046     if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
3047                            VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
3048         error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3049         qemu_set_fd_handler(fd, NULL, NULL, vdev);
3050         event_notifier_cleanup(&vdev->req_notifier);
3051     } else {
3052         vdev->req_enabled = true;
3053     }
3054 }
3055 
3056 static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
3057 {
3058     Error *err = NULL;
3059 
3060     if (!vdev->req_enabled) {
3061         return;
3062     }
3063 
3064     if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
3065                                VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
3066         error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3067     }
3068     qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
3069                         NULL, NULL, vdev);
3070     event_notifier_cleanup(&vdev->req_notifier);
3071 
3072     vdev->req_enabled = false;
3073 }
3074 
3075 static void vfio_realize(PCIDevice *pdev, Error **errp)
3076 {
3077     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
3078     VFIODevice *vbasedev = &vdev->vbasedev;
3079     char *tmp, *subsys;
3080     Error *err = NULL;
3081     struct stat st;
3082     int i, ret;
3083     bool is_mdev;
3084     char uuid[UUID_FMT_LEN];
3085     char *name;
3086 
3087     if (!vbasedev->sysfsdev) {
3088         if (!(~vdev->host.domain || ~vdev->host.bus ||
3089               ~vdev->host.slot || ~vdev->host.function)) {
3090             error_setg(errp, "No provided host device");
3091             error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
3092                               "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
3093             return;
3094         }
3095         vbasedev->sysfsdev =
3096             g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
3097                             vdev->host.domain, vdev->host.bus,
3098                             vdev->host.slot, vdev->host.function);
3099     }
3100 
3101     if (stat(vbasedev->sysfsdev, &st) < 0) {
3102         error_setg_errno(errp, errno, "no such host device");
3103         error_prepend(errp, VFIO_MSG_PREFIX, vbasedev->sysfsdev);
3104         return;
3105     }
3106 
3107     vbasedev->name = g_path_get_basename(vbasedev->sysfsdev);
3108     vbasedev->ops = &vfio_pci_ops;
3109     vbasedev->type = VFIO_DEVICE_TYPE_PCI;
3110     vbasedev->dev = DEVICE(vdev);
3111 
3112     /*
3113      * Mediated devices *might* operate compatibly with discarding of RAM, but
3114      * we cannot know for certain, it depends on whether the mdev vendor driver
3115      * stays in sync with the active working set of the guest driver.  Prevent
3116      * the x-balloon-allowed option unless this is minimally an mdev device.
3117      */
3118     tmp = g_strdup_printf("%s/subsystem", vbasedev->sysfsdev);
3119     subsys = realpath(tmp, NULL);
3120     g_free(tmp);
3121     is_mdev = subsys && (strcmp(subsys, "/sys/bus/mdev") == 0);
3122     free(subsys);
3123 
3124     trace_vfio_mdev(vbasedev->name, is_mdev);
3125 
3126     if (vbasedev->ram_block_discard_allowed && !is_mdev) {
3127         error_setg(errp, "x-balloon-allowed only potentially compatible "
3128                    "with mdev devices");
3129         goto error;
3130     }
3131 
3132     if (!qemu_uuid_is_null(&vdev->vf_token)) {
3133         qemu_uuid_unparse(&vdev->vf_token, uuid);
3134         name = g_strdup_printf("%s vf_token=%s", vbasedev->name, uuid);
3135     } else {
3136         name = g_strdup(vbasedev->name);
3137     }
3138 
3139     ret = vfio_attach_device(name, vbasedev,
3140                              pci_device_iommu_address_space(pdev), errp);
3141     g_free(name);
3142     if (ret) {
3143         goto error;
3144     }
3145 
3146     vfio_populate_device(vdev, &err);
3147     if (err) {
3148         error_propagate(errp, err);
3149         goto error;
3150     }
3151 
3152     /* Get a copy of config space */
3153     ret = pread(vbasedev->fd, vdev->pdev.config,
3154                 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
3155                 vdev->config_offset);
3156     if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
3157         ret = ret < 0 ? -errno : -EFAULT;
3158         error_setg_errno(errp, -ret, "failed to read device config space");
3159         goto error;
3160     }
3161 
3162     /* vfio emulates a lot for us, but some bits need extra love */
3163     vdev->emulated_config_bits = g_malloc0(vdev->config_size);
3164 
3165     /* QEMU can choose to expose the ROM or not */
3166     memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
3167     /* QEMU can also add or extend BARs */
3168     memset(vdev->emulated_config_bits + PCI_BASE_ADDRESS_0, 0xff, 6 * 4);
3169 
3170     /*
3171      * The PCI spec reserves vendor ID 0xffff as an invalid value.  The
3172      * device ID is managed by the vendor and need only be a 16-bit value.
3173      * Allow any 16-bit value for subsystem so they can be hidden or changed.
3174      */
3175     if (vdev->vendor_id != PCI_ANY_ID) {
3176         if (vdev->vendor_id >= 0xffff) {
3177             error_setg(errp, "invalid PCI vendor ID provided");
3178             goto error;
3179         }
3180         vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
3181         trace_vfio_pci_emulated_vendor_id(vbasedev->name, vdev->vendor_id);
3182     } else {
3183         vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
3184     }
3185 
3186     if (vdev->device_id != PCI_ANY_ID) {
3187         if (vdev->device_id > 0xffff) {
3188             error_setg(errp, "invalid PCI device ID provided");
3189             goto error;
3190         }
3191         vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
3192         trace_vfio_pci_emulated_device_id(vbasedev->name, vdev->device_id);
3193     } else {
3194         vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
3195     }
3196 
3197     if (vdev->sub_vendor_id != PCI_ANY_ID) {
3198         if (vdev->sub_vendor_id > 0xffff) {
3199             error_setg(errp, "invalid PCI subsystem vendor ID provided");
3200             goto error;
3201         }
3202         vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
3203                                vdev->sub_vendor_id, ~0);
3204         trace_vfio_pci_emulated_sub_vendor_id(vbasedev->name,
3205                                               vdev->sub_vendor_id);
3206     }
3207 
3208     if (vdev->sub_device_id != PCI_ANY_ID) {
3209         if (vdev->sub_device_id > 0xffff) {
3210             error_setg(errp, "invalid PCI subsystem device ID provided");
3211             goto error;
3212         }
3213         vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
3214         trace_vfio_pci_emulated_sub_device_id(vbasedev->name,
3215                                               vdev->sub_device_id);
3216     }
3217 
3218     /* QEMU can change multi-function devices to single function, or reverse */
3219     vdev->emulated_config_bits[PCI_HEADER_TYPE] =
3220                                               PCI_HEADER_TYPE_MULTI_FUNCTION;
3221 
3222     /* Restore or clear multifunction, this is always controlled by QEMU */
3223     if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
3224         vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
3225     } else {
3226         vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
3227     }
3228 
3229     /*
3230      * Clear host resource mapping info.  If we choose not to register a
3231      * BAR, such as might be the case with the option ROM, we can get
3232      * confusing, unwritable, residual addresses from the host here.
3233      */
3234     memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
3235     memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
3236 
3237     vfio_pci_size_rom(vdev);
3238 
3239     vfio_bars_prepare(vdev);
3240 
3241     vfio_msix_early_setup(vdev, &err);
3242     if (err) {
3243         error_propagate(errp, err);
3244         goto error;
3245     }
3246 
3247     vfio_bars_register(vdev);
3248 
3249     ret = vfio_add_capabilities(vdev, errp);
3250     if (ret) {
3251         goto out_teardown;
3252     }
3253 
3254     if (vdev->vga) {
3255         vfio_vga_quirk_setup(vdev);
3256     }
3257 
3258     for (i = 0; i < PCI_ROM_SLOT; i++) {
3259         vfio_bar_quirk_setup(vdev, i);
3260     }
3261 
3262     if (!vdev->igd_opregion &&
3263         vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
3264         struct vfio_region_info *opregion;
3265 
3266         if (vdev->pdev.qdev.hotplugged) {
3267             error_setg(errp,
3268                        "cannot support IGD OpRegion feature on hotplugged "
3269                        "device");
3270             goto out_teardown;
3271         }
3272 
3273         ret = vfio_get_dev_region_info(vbasedev,
3274                         VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
3275                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
3276         if (ret) {
3277             error_setg_errno(errp, -ret,
3278                              "does not support requested IGD OpRegion feature");
3279             goto out_teardown;
3280         }
3281 
3282         ret = vfio_pci_igd_opregion_init(vdev, opregion, errp);
3283         g_free(opregion);
3284         if (ret) {
3285             goto out_teardown;
3286         }
3287     }
3288 
3289     /* QEMU emulates all of MSI & MSIX */
3290     if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
3291         memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
3292                MSIX_CAP_LENGTH);
3293     }
3294 
3295     if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
3296         memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
3297                vdev->msi_cap_size);
3298     }
3299 
3300     if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
3301         vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
3302                                                   vfio_intx_mmap_enable, vdev);
3303         pci_device_set_intx_routing_notifier(&vdev->pdev,
3304                                              vfio_intx_routing_notifier);
3305         vdev->irqchip_change_notifier.notify = vfio_irqchip_change;
3306         kvm_irqchip_add_change_notifier(&vdev->irqchip_change_notifier);
3307         ret = vfio_intx_enable(vdev, errp);
3308         if (ret) {
3309             goto out_deregister;
3310         }
3311     }
3312 
3313     if (vdev->display != ON_OFF_AUTO_OFF) {
3314         ret = vfio_display_probe(vdev, errp);
3315         if (ret) {
3316             goto out_deregister;
3317         }
3318     }
3319     if (vdev->enable_ramfb && vdev->dpy == NULL) {
3320         error_setg(errp, "ramfb=on requires display=on");
3321         goto out_deregister;
3322     }
3323     if (vdev->display_xres || vdev->display_yres) {
3324         if (vdev->dpy == NULL) {
3325             error_setg(errp, "xres and yres properties require display=on");
3326             goto out_deregister;
3327         }
3328         if (vdev->dpy->edid_regs == NULL) {
3329             error_setg(errp, "xres and yres properties need edid support");
3330             goto out_deregister;
3331         }
3332     }
3333 
3334     if (vdev->ramfb_migrate == ON_OFF_AUTO_ON && !vdev->enable_ramfb) {
3335         warn_report("x-ramfb-migrate=on but ramfb=off. "
3336                     "Forcing x-ramfb-migrate to off.");
3337         vdev->ramfb_migrate = ON_OFF_AUTO_OFF;
3338     }
3339     if (vbasedev->enable_migration == ON_OFF_AUTO_OFF) {
3340         if (vdev->ramfb_migrate == ON_OFF_AUTO_AUTO) {
3341             vdev->ramfb_migrate = ON_OFF_AUTO_OFF;
3342         } else if (vdev->ramfb_migrate == ON_OFF_AUTO_ON) {
3343             error_setg(errp, "x-ramfb-migrate requires enable-migration");
3344             goto out_deregister;
3345         }
3346     }
3347 
3348     if (!pdev->failover_pair_id) {
3349         if (!vfio_migration_realize(vbasedev, errp)) {
3350             goto out_deregister;
3351         }
3352     }
3353 
3354     vfio_register_err_notifier(vdev);
3355     vfio_register_req_notifier(vdev);
3356     vfio_setup_resetfn_quirk(vdev);
3357 
3358     return;
3359 
3360 out_deregister:
3361     if (vdev->interrupt == VFIO_INT_INTx) {
3362         vfio_intx_disable(vdev);
3363     }
3364     pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3365     if (vdev->irqchip_change_notifier.notify) {
3366         kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3367     }
3368     if (vdev->intx.mmap_timer) {
3369         timer_free(vdev->intx.mmap_timer);
3370     }
3371 out_teardown:
3372     vfio_teardown_msi(vdev);
3373     vfio_bars_exit(vdev);
3374 error:
3375     error_prepend(errp, VFIO_MSG_PREFIX, vbasedev->name);
3376 }
3377 
3378 static void vfio_instance_finalize(Object *obj)
3379 {
3380     VFIOPCIDevice *vdev = VFIO_PCI(obj);
3381 
3382     vfio_display_finalize(vdev);
3383     vfio_bars_finalize(vdev);
3384     g_free(vdev->emulated_config_bits);
3385     g_free(vdev->rom);
3386     /*
3387      * XXX Leaking igd_opregion is not an oversight, we can't remove the
3388      * fw_cfg entry therefore leaking this allocation seems like the safest
3389      * option.
3390      *
3391      * g_free(vdev->igd_opregion);
3392      */
3393     vfio_pci_put_device(vdev);
3394 }
3395 
3396 static void vfio_exitfn(PCIDevice *pdev)
3397 {
3398     VFIOPCIDevice *vdev = VFIO_PCI(pdev);
3399 
3400     vfio_unregister_req_notifier(vdev);
3401     vfio_unregister_err_notifier(vdev);
3402     pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3403     if (vdev->irqchip_change_notifier.notify) {
3404         kvm_irqchip_remove_change_notifier(&vdev->irqchip_change_notifier);
3405     }
3406     vfio_disable_interrupts(vdev);
3407     if (vdev->intx.mmap_timer) {
3408         timer_free(vdev->intx.mmap_timer);
3409     }
3410     vfio_teardown_msi(vdev);
3411     vfio_pci_disable_rp_atomics(vdev);
3412     vfio_bars_exit(vdev);
3413     vfio_migration_exit(&vdev->vbasedev);
3414 }
3415 
3416 static void vfio_pci_reset(DeviceState *dev)
3417 {
3418     VFIOPCIDevice *vdev = VFIO_PCI(dev);
3419 
3420     trace_vfio_pci_reset(vdev->vbasedev.name);
3421 
3422     vfio_pci_pre_reset(vdev);
3423 
3424     if (vdev->display != ON_OFF_AUTO_OFF) {
3425         vfio_display_reset(vdev);
3426     }
3427 
3428     if (vdev->resetfn && !vdev->resetfn(vdev)) {
3429         goto post_reset;
3430     }
3431 
3432     if (vdev->vbasedev.reset_works &&
3433         (vdev->has_flr || !vdev->has_pm_reset) &&
3434         !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3435         trace_vfio_pci_reset_flr(vdev->vbasedev.name);
3436         goto post_reset;
3437     }
3438 
3439     /* See if we can do our own bus reset */
3440     if (!vfio_pci_hot_reset_one(vdev)) {
3441         goto post_reset;
3442     }
3443 
3444     /* If nothing else works and the device supports PM reset, use it */
3445     if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
3446         !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3447         trace_vfio_pci_reset_pm(vdev->vbasedev.name);
3448         goto post_reset;
3449     }
3450 
3451 post_reset:
3452     vfio_pci_post_reset(vdev);
3453 }
3454 
3455 static void vfio_instance_init(Object *obj)
3456 {
3457     PCIDevice *pci_dev = PCI_DEVICE(obj);
3458     VFIOPCIDevice *vdev = VFIO_PCI(obj);
3459 
3460     device_add_bootindex_property(obj, &vdev->bootindex,
3461                                   "bootindex", NULL,
3462                                   &pci_dev->qdev);
3463     vdev->host.domain = ~0U;
3464     vdev->host.bus = ~0U;
3465     vdev->host.slot = ~0U;
3466     vdev->host.function = ~0U;
3467 
3468     vdev->nv_gpudirect_clique = 0xFF;
3469 
3470     /* QEMU_PCI_CAP_EXPRESS initialization does not depend on QEMU command
3471      * line, therefore, no need to wait to realize like other devices */
3472     pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
3473 }
3474 
3475 static Property vfio_pci_dev_properties[] = {
3476     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
3477     DEFINE_PROP_UUID_NODEFAULT("vf-token", VFIOPCIDevice, vf_token),
3478     DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
3479     DEFINE_PROP_ON_OFF_AUTO("x-pre-copy-dirty-page-tracking", VFIOPCIDevice,
3480                             vbasedev.pre_copy_dirty_page_tracking,
3481                             ON_OFF_AUTO_ON),
3482     DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
3483                             display, ON_OFF_AUTO_OFF),
3484     DEFINE_PROP_UINT32("xres", VFIOPCIDevice, display_xres, 0),
3485     DEFINE_PROP_UINT32("yres", VFIOPCIDevice, display_yres, 0),
3486     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
3487                        intx.mmap_timeout, 1100),
3488     DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
3489                     VFIO_FEATURE_ENABLE_VGA_BIT, false),
3490     DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
3491                     VFIO_FEATURE_ENABLE_REQ_BIT, true),
3492     DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
3493                     VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
3494     DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOPCIDevice,
3495                             vbasedev.enable_migration, ON_OFF_AUTO_AUTO),
3496     DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
3497     DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice,
3498                      vbasedev.ram_block_discard_allowed, false),
3499     DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
3500     DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
3501     DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
3502     DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice,
3503                      no_geforce_quirks, false),
3504     DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice, no_kvm_ioeventfd,
3505                      false),
3506     DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice, no_vfio_ioeventfd,
3507                      false),
3508     DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
3509     DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
3510     DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
3511                        sub_vendor_id, PCI_ANY_ID),
3512     DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
3513                        sub_device_id, PCI_ANY_ID),
3514     DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
3515     DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice,
3516                                    nv_gpudirect_clique,
3517                                    qdev_prop_nv_gpudirect_clique, uint8_t),
3518     DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo,
3519                                 OFF_AUTOPCIBAR_OFF),
3520     /*
3521      * TODO - support passed fds... is this necessary?
3522      * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
3523      * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
3524      */
3525     DEFINE_PROP_END_OF_LIST(),
3526 };
3527 
3528 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
3529 {
3530     DeviceClass *dc = DEVICE_CLASS(klass);
3531     PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3532 
3533     dc->reset = vfio_pci_reset;
3534     device_class_set_props(dc, vfio_pci_dev_properties);
3535     dc->desc = "VFIO-based PCI device assignment";
3536     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3537     pdc->realize = vfio_realize;
3538     pdc->exit = vfio_exitfn;
3539     pdc->config_read = vfio_pci_read_config;
3540     pdc->config_write = vfio_pci_write_config;
3541 }
3542 
3543 static const TypeInfo vfio_pci_dev_info = {
3544     .name = TYPE_VFIO_PCI,
3545     .parent = TYPE_PCI_DEVICE,
3546     .instance_size = sizeof(VFIOPCIDevice),
3547     .class_init = vfio_pci_dev_class_init,
3548     .instance_init = vfio_instance_init,
3549     .instance_finalize = vfio_instance_finalize,
3550     .interfaces = (InterfaceInfo[]) {
3551         { INTERFACE_PCIE_DEVICE },
3552         { INTERFACE_CONVENTIONAL_PCI_DEVICE },
3553         { }
3554     },
3555 };
3556 
3557 static Property vfio_pci_dev_nohotplug_properties[] = {
3558     DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice, enable_ramfb, false),
3559     DEFINE_PROP_ON_OFF_AUTO("x-ramfb-migrate", VFIOPCIDevice, ramfb_migrate,
3560                             ON_OFF_AUTO_AUTO),
3561     DEFINE_PROP_END_OF_LIST(),
3562 };
3563 
3564 static void vfio_pci_nohotplug_dev_class_init(ObjectClass *klass, void *data)
3565 {
3566     DeviceClass *dc = DEVICE_CLASS(klass);
3567 
3568     device_class_set_props(dc, vfio_pci_dev_nohotplug_properties);
3569     dc->hotpluggable = false;
3570 }
3571 
3572 static const TypeInfo vfio_pci_nohotplug_dev_info = {
3573     .name = TYPE_VFIO_PCI_NOHOTPLUG,
3574     .parent = TYPE_VFIO_PCI,
3575     .instance_size = sizeof(VFIOPCIDevice),
3576     .class_init = vfio_pci_nohotplug_dev_class_init,
3577 };
3578 
3579 static void register_vfio_pci_dev_type(void)
3580 {
3581     type_register_static(&vfio_pci_dev_info);
3582     type_register_static(&vfio_pci_nohotplug_dev_info);
3583 }
3584 
3585 type_init(register_vfio_pci_dev_type)
3586