xref: /openbmc/qemu/hw/vfio/pci.c (revision 795c40b8)
1 /*
2  * vfio based device assignment support
3  *
4  * Copyright Red Hat, Inc. 2012
5  *
6  * Authors:
7  *  Alex Williamson <alex.williamson@redhat.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  * Based on qemu-kvm device-assignment:
13  *  Adapted for KVM by Qumranet.
14  *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15  *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16  *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17  *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19  */
20 
21 #include "qemu/osdep.h"
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
24 
25 #include "hw/pci/msi.h"
26 #include "hw/pci/msix.h"
27 #include "hw/pci/pci_bridge.h"
28 #include "qemu/error-report.h"
29 #include "qemu/range.h"
30 #include "sysemu/kvm.h"
31 #include "sysemu/sysemu.h"
32 #include "pci.h"
33 #include "trace.h"
34 #include "qapi/error.h"
35 
36 #define MSIX_CAP_LENGTH 12
37 
38 static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
39 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
40 
41 /*
42  * Disabling BAR mmaping can be slow, but toggling it around INTx can
43  * also be a huge overhead.  We try to get the best of both worlds by
44  * waiting until an interrupt to disable mmaps (subsequent transitions
45  * to the same state are effectively no overhead).  If the interrupt has
46  * been serviced and the time gap is long enough, we re-enable mmaps for
47  * performance.  This works well for things like graphics cards, which
48  * may not use their interrupt at all and are penalized to an unusable
49  * level by read/write BAR traps.  Other devices, like NICs, have more
50  * regular interrupts and see much better latency by staying in non-mmap
51  * mode.  We therefore set the default mmap_timeout such that a ping
52  * is just enough to keep the mmap disabled.  Users can experiment with
53  * other options with the x-intx-mmap-timeout-ms parameter (a value of
54  * zero disables the timer).
55  */
56 static void vfio_intx_mmap_enable(void *opaque)
57 {
58     VFIOPCIDevice *vdev = opaque;
59 
60     if (vdev->intx.pending) {
61         timer_mod(vdev->intx.mmap_timer,
62                        qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
63         return;
64     }
65 
66     vfio_mmap_set_enabled(vdev, true);
67 }
68 
69 static void vfio_intx_interrupt(void *opaque)
70 {
71     VFIOPCIDevice *vdev = opaque;
72 
73     if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
74         return;
75     }
76 
77     trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
78 
79     vdev->intx.pending = true;
80     pci_irq_assert(&vdev->pdev);
81     vfio_mmap_set_enabled(vdev, false);
82     if (vdev->intx.mmap_timeout) {
83         timer_mod(vdev->intx.mmap_timer,
84                        qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
85     }
86 }
87 
88 static void vfio_intx_eoi(VFIODevice *vbasedev)
89 {
90     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
91 
92     if (!vdev->intx.pending) {
93         return;
94     }
95 
96     trace_vfio_intx_eoi(vbasedev->name);
97 
98     vdev->intx.pending = false;
99     pci_irq_deassert(&vdev->pdev);
100     vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
101 }
102 
103 static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp)
104 {
105 #ifdef CONFIG_KVM
106     struct kvm_irqfd irqfd = {
107         .fd = event_notifier_get_fd(&vdev->intx.interrupt),
108         .gsi = vdev->intx.route.irq,
109         .flags = KVM_IRQFD_FLAG_RESAMPLE,
110     };
111     struct vfio_irq_set *irq_set;
112     int ret, argsz;
113     int32_t *pfd;
114 
115     if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
116         vdev->intx.route.mode != PCI_INTX_ENABLED ||
117         !kvm_resamplefds_enabled()) {
118         return;
119     }
120 
121     /* Get to a known interrupt state */
122     qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
123     vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
124     vdev->intx.pending = false;
125     pci_irq_deassert(&vdev->pdev);
126 
127     /* Get an eventfd for resample/unmask */
128     if (event_notifier_init(&vdev->intx.unmask, 0)) {
129         error_setg(errp, "event_notifier_init failed eoi");
130         goto fail;
131     }
132 
133     /* KVM triggers it, VFIO listens for it */
134     irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
135 
136     if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
137         error_setg_errno(errp, errno, "failed to setup resample irqfd");
138         goto fail_irqfd;
139     }
140 
141     argsz = sizeof(*irq_set) + sizeof(*pfd);
142 
143     irq_set = g_malloc0(argsz);
144     irq_set->argsz = argsz;
145     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
146     irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
147     irq_set->start = 0;
148     irq_set->count = 1;
149     pfd = (int32_t *)&irq_set->data;
150 
151     *pfd = irqfd.resamplefd;
152 
153     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
154     g_free(irq_set);
155     if (ret) {
156         error_setg_errno(errp, -ret, "failed to setup INTx unmask fd");
157         goto fail_vfio;
158     }
159 
160     /* Let'em rip */
161     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
162 
163     vdev->intx.kvm_accel = true;
164 
165     trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
166 
167     return;
168 
169 fail_vfio:
170     irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
171     kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
172 fail_irqfd:
173     event_notifier_cleanup(&vdev->intx.unmask);
174 fail:
175     qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
176     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
177 #endif
178 }
179 
180 static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
181 {
182 #ifdef CONFIG_KVM
183     struct kvm_irqfd irqfd = {
184         .fd = event_notifier_get_fd(&vdev->intx.interrupt),
185         .gsi = vdev->intx.route.irq,
186         .flags = KVM_IRQFD_FLAG_DEASSIGN,
187     };
188 
189     if (!vdev->intx.kvm_accel) {
190         return;
191     }
192 
193     /*
194      * Get to a known state, hardware masked, QEMU ready to accept new
195      * interrupts, QEMU IRQ de-asserted.
196      */
197     vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
198     vdev->intx.pending = false;
199     pci_irq_deassert(&vdev->pdev);
200 
201     /* Tell KVM to stop listening for an INTx irqfd */
202     if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
203         error_report("vfio: Error: Failed to disable INTx irqfd: %m");
204     }
205 
206     /* We only need to close the eventfd for VFIO to cleanup the kernel side */
207     event_notifier_cleanup(&vdev->intx.unmask);
208 
209     /* QEMU starts listening for interrupt events. */
210     qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
211 
212     vdev->intx.kvm_accel = false;
213 
214     /* If we've missed an event, let it re-fire through QEMU */
215     vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
216 
217     trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
218 #endif
219 }
220 
221 static void vfio_intx_update(PCIDevice *pdev)
222 {
223     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
224     PCIINTxRoute route;
225     Error *err = NULL;
226 
227     if (vdev->interrupt != VFIO_INT_INTx) {
228         return;
229     }
230 
231     route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
232 
233     if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
234         return; /* Nothing changed */
235     }
236 
237     trace_vfio_intx_update(vdev->vbasedev.name,
238                            vdev->intx.route.irq, route.irq);
239 
240     vfio_intx_disable_kvm(vdev);
241 
242     vdev->intx.route = route;
243 
244     if (route.mode != PCI_INTX_ENABLED) {
245         return;
246     }
247 
248     vfio_intx_enable_kvm(vdev, &err);
249     if (err) {
250         error_reportf_err(err, WARN_PREFIX, vdev->vbasedev.name);
251     }
252 
253     /* Re-enable the interrupt in cased we missed an EOI */
254     vfio_intx_eoi(&vdev->vbasedev);
255 }
256 
257 static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp)
258 {
259     uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
260     int ret, argsz;
261     struct vfio_irq_set *irq_set;
262     int32_t *pfd;
263     Error *err = NULL;
264 
265     if (!pin) {
266         return 0;
267     }
268 
269     vfio_disable_interrupts(vdev);
270 
271     vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
272     pci_config_set_interrupt_pin(vdev->pdev.config, pin);
273 
274 #ifdef CONFIG_KVM
275     /*
276      * Only conditional to avoid generating error messages on platforms
277      * where we won't actually use the result anyway.
278      */
279     if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
280         vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
281                                                         vdev->intx.pin);
282     }
283 #endif
284 
285     ret = event_notifier_init(&vdev->intx.interrupt, 0);
286     if (ret) {
287         error_setg_errno(errp, -ret, "event_notifier_init failed");
288         return ret;
289     }
290 
291     argsz = sizeof(*irq_set) + sizeof(*pfd);
292 
293     irq_set = g_malloc0(argsz);
294     irq_set->argsz = argsz;
295     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
296     irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
297     irq_set->start = 0;
298     irq_set->count = 1;
299     pfd = (int32_t *)&irq_set->data;
300 
301     *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
302     qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
303 
304     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
305     g_free(irq_set);
306     if (ret) {
307         error_setg_errno(errp, -ret, "failed to setup INTx fd");
308         qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
309         event_notifier_cleanup(&vdev->intx.interrupt);
310         return -errno;
311     }
312 
313     vfio_intx_enable_kvm(vdev, &err);
314     if (err) {
315         error_reportf_err(err, WARN_PREFIX, vdev->vbasedev.name);
316     }
317 
318     vdev->interrupt = VFIO_INT_INTx;
319 
320     trace_vfio_intx_enable(vdev->vbasedev.name);
321 
322     return 0;
323 }
324 
325 static void vfio_intx_disable(VFIOPCIDevice *vdev)
326 {
327     int fd;
328 
329     timer_del(vdev->intx.mmap_timer);
330     vfio_intx_disable_kvm(vdev);
331     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
332     vdev->intx.pending = false;
333     pci_irq_deassert(&vdev->pdev);
334     vfio_mmap_set_enabled(vdev, true);
335 
336     fd = event_notifier_get_fd(&vdev->intx.interrupt);
337     qemu_set_fd_handler(fd, NULL, NULL, vdev);
338     event_notifier_cleanup(&vdev->intx.interrupt);
339 
340     vdev->interrupt = VFIO_INT_NONE;
341 
342     trace_vfio_intx_disable(vdev->vbasedev.name);
343 }
344 
345 /*
346  * MSI/X
347  */
348 static void vfio_msi_interrupt(void *opaque)
349 {
350     VFIOMSIVector *vector = opaque;
351     VFIOPCIDevice *vdev = vector->vdev;
352     MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
353     void (*notify)(PCIDevice *dev, unsigned vector);
354     MSIMessage msg;
355     int nr = vector - vdev->msi_vectors;
356 
357     if (!event_notifier_test_and_clear(&vector->interrupt)) {
358         return;
359     }
360 
361     if (vdev->interrupt == VFIO_INT_MSIX) {
362         get_msg = msix_get_message;
363         notify = msix_notify;
364 
365         /* A masked vector firing needs to use the PBA, enable it */
366         if (msix_is_masked(&vdev->pdev, nr)) {
367             set_bit(nr, vdev->msix->pending);
368             memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
369             trace_vfio_msix_pba_enable(vdev->vbasedev.name);
370         }
371     } else if (vdev->interrupt == VFIO_INT_MSI) {
372         get_msg = msi_get_message;
373         notify = msi_notify;
374     } else {
375         abort();
376     }
377 
378     msg = get_msg(&vdev->pdev, nr);
379     trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
380     notify(&vdev->pdev, nr);
381 }
382 
383 static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
384 {
385     struct vfio_irq_set *irq_set;
386     int ret = 0, i, argsz;
387     int32_t *fds;
388 
389     argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
390 
391     irq_set = g_malloc0(argsz);
392     irq_set->argsz = argsz;
393     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
394     irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
395     irq_set->start = 0;
396     irq_set->count = vdev->nr_vectors;
397     fds = (int32_t *)&irq_set->data;
398 
399     for (i = 0; i < vdev->nr_vectors; i++) {
400         int fd = -1;
401 
402         /*
403          * MSI vs MSI-X - The guest has direct access to MSI mask and pending
404          * bits, therefore we always use the KVM signaling path when setup.
405          * MSI-X mask and pending bits are emulated, so we want to use the
406          * KVM signaling path only when configured and unmasked.
407          */
408         if (vdev->msi_vectors[i].use) {
409             if (vdev->msi_vectors[i].virq < 0 ||
410                 (msix && msix_is_masked(&vdev->pdev, i))) {
411                 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
412             } else {
413                 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
414             }
415         }
416 
417         fds[i] = fd;
418     }
419 
420     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
421 
422     g_free(irq_set);
423 
424     return ret;
425 }
426 
427 static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
428                                   int vector_n, bool msix)
429 {
430     int virq;
431 
432     if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
433         return;
434     }
435 
436     if (event_notifier_init(&vector->kvm_interrupt, 0)) {
437         return;
438     }
439 
440     virq = kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev);
441     if (virq < 0) {
442         event_notifier_cleanup(&vector->kvm_interrupt);
443         return;
444     }
445 
446     if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
447                                        NULL, virq) < 0) {
448         kvm_irqchip_release_virq(kvm_state, virq);
449         event_notifier_cleanup(&vector->kvm_interrupt);
450         return;
451     }
452 
453     vector->virq = virq;
454 }
455 
456 static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
457 {
458     kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
459                                           vector->virq);
460     kvm_irqchip_release_virq(kvm_state, vector->virq);
461     vector->virq = -1;
462     event_notifier_cleanup(&vector->kvm_interrupt);
463 }
464 
465 static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
466                                      PCIDevice *pdev)
467 {
468     kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
469     kvm_irqchip_commit_routes(kvm_state);
470 }
471 
472 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
473                                    MSIMessage *msg, IOHandler *handler)
474 {
475     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
476     VFIOMSIVector *vector;
477     int ret;
478 
479     trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
480 
481     vector = &vdev->msi_vectors[nr];
482 
483     if (!vector->use) {
484         vector->vdev = vdev;
485         vector->virq = -1;
486         if (event_notifier_init(&vector->interrupt, 0)) {
487             error_report("vfio: Error: event_notifier_init failed");
488         }
489         vector->use = true;
490         msix_vector_use(pdev, nr);
491     }
492 
493     qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
494                         handler, NULL, vector);
495 
496     /*
497      * Attempt to enable route through KVM irqchip,
498      * default to userspace handling if unavailable.
499      */
500     if (vector->virq >= 0) {
501         if (!msg) {
502             vfio_remove_kvm_msi_virq(vector);
503         } else {
504             vfio_update_kvm_msi_virq(vector, *msg, pdev);
505         }
506     } else {
507         if (msg) {
508             vfio_add_kvm_msi_virq(vdev, vector, nr, true);
509         }
510     }
511 
512     /*
513      * We don't want to have the host allocate all possible MSI vectors
514      * for a device if they're not in use, so we shutdown and incrementally
515      * increase them as needed.
516      */
517     if (vdev->nr_vectors < nr + 1) {
518         vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
519         vdev->nr_vectors = nr + 1;
520         ret = vfio_enable_vectors(vdev, true);
521         if (ret) {
522             error_report("vfio: failed to enable vectors, %d", ret);
523         }
524     } else {
525         int argsz;
526         struct vfio_irq_set *irq_set;
527         int32_t *pfd;
528 
529         argsz = sizeof(*irq_set) + sizeof(*pfd);
530 
531         irq_set = g_malloc0(argsz);
532         irq_set->argsz = argsz;
533         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
534                          VFIO_IRQ_SET_ACTION_TRIGGER;
535         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
536         irq_set->start = nr;
537         irq_set->count = 1;
538         pfd = (int32_t *)&irq_set->data;
539 
540         if (vector->virq >= 0) {
541             *pfd = event_notifier_get_fd(&vector->kvm_interrupt);
542         } else {
543             *pfd = event_notifier_get_fd(&vector->interrupt);
544         }
545 
546         ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
547         g_free(irq_set);
548         if (ret) {
549             error_report("vfio: failed to modify vector, %d", ret);
550         }
551     }
552 
553     /* Disable PBA emulation when nothing more is pending. */
554     clear_bit(nr, vdev->msix->pending);
555     if (find_first_bit(vdev->msix->pending,
556                        vdev->nr_vectors) == vdev->nr_vectors) {
557         memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
558         trace_vfio_msix_pba_disable(vdev->vbasedev.name);
559     }
560 
561     return 0;
562 }
563 
564 static int vfio_msix_vector_use(PCIDevice *pdev,
565                                 unsigned int nr, MSIMessage msg)
566 {
567     return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
568 }
569 
570 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
571 {
572     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
573     VFIOMSIVector *vector = &vdev->msi_vectors[nr];
574 
575     trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
576 
577     /*
578      * There are still old guests that mask and unmask vectors on every
579      * interrupt.  If we're using QEMU bypass with a KVM irqfd, leave all of
580      * the KVM setup in place, simply switch VFIO to use the non-bypass
581      * eventfd.  We'll then fire the interrupt through QEMU and the MSI-X
582      * core will mask the interrupt and set pending bits, allowing it to
583      * be re-asserted on unmask.  Nothing to do if already using QEMU mode.
584      */
585     if (vector->virq >= 0) {
586         int argsz;
587         struct vfio_irq_set *irq_set;
588         int32_t *pfd;
589 
590         argsz = sizeof(*irq_set) + sizeof(*pfd);
591 
592         irq_set = g_malloc0(argsz);
593         irq_set->argsz = argsz;
594         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
595                          VFIO_IRQ_SET_ACTION_TRIGGER;
596         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
597         irq_set->start = nr;
598         irq_set->count = 1;
599         pfd = (int32_t *)&irq_set->data;
600 
601         *pfd = event_notifier_get_fd(&vector->interrupt);
602 
603         ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
604 
605         g_free(irq_set);
606     }
607 }
608 
609 static void vfio_msix_enable(VFIOPCIDevice *vdev)
610 {
611     vfio_disable_interrupts(vdev);
612 
613     vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
614 
615     vdev->interrupt = VFIO_INT_MSIX;
616 
617     /*
618      * Some communication channels between VF & PF or PF & fw rely on the
619      * physical state of the device and expect that enabling MSI-X from the
620      * guest enables the same on the host.  When our guest is Linux, the
621      * guest driver call to pci_enable_msix() sets the enabling bit in the
622      * MSI-X capability, but leaves the vector table masked.  We therefore
623      * can't rely on a vector_use callback (from request_irq() in the guest)
624      * to switch the physical device into MSI-X mode because that may come a
625      * long time after pci_enable_msix().  This code enables vector 0 with
626      * triggering to userspace, then immediately release the vector, leaving
627      * the physical device with no vectors enabled, but MSI-X enabled, just
628      * like the guest view.
629      */
630     vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
631     vfio_msix_vector_release(&vdev->pdev, 0);
632 
633     if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
634                                   vfio_msix_vector_release, NULL)) {
635         error_report("vfio: msix_set_vector_notifiers failed");
636     }
637 
638     trace_vfio_msix_enable(vdev->vbasedev.name);
639 }
640 
641 static void vfio_msi_enable(VFIOPCIDevice *vdev)
642 {
643     int ret, i;
644 
645     vfio_disable_interrupts(vdev);
646 
647     vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
648 retry:
649     vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
650 
651     for (i = 0; i < vdev->nr_vectors; i++) {
652         VFIOMSIVector *vector = &vdev->msi_vectors[i];
653 
654         vector->vdev = vdev;
655         vector->virq = -1;
656         vector->use = true;
657 
658         if (event_notifier_init(&vector->interrupt, 0)) {
659             error_report("vfio: Error: event_notifier_init failed");
660         }
661 
662         qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
663                             vfio_msi_interrupt, NULL, vector);
664 
665         /*
666          * Attempt to enable route through KVM irqchip,
667          * default to userspace handling if unavailable.
668          */
669         vfio_add_kvm_msi_virq(vdev, vector, i, false);
670     }
671 
672     /* Set interrupt type prior to possible interrupts */
673     vdev->interrupt = VFIO_INT_MSI;
674 
675     ret = vfio_enable_vectors(vdev, false);
676     if (ret) {
677         if (ret < 0) {
678             error_report("vfio: Error: Failed to setup MSI fds: %m");
679         } else if (ret != vdev->nr_vectors) {
680             error_report("vfio: Error: Failed to enable %d "
681                          "MSI vectors, retry with %d", vdev->nr_vectors, ret);
682         }
683 
684         for (i = 0; i < vdev->nr_vectors; i++) {
685             VFIOMSIVector *vector = &vdev->msi_vectors[i];
686             if (vector->virq >= 0) {
687                 vfio_remove_kvm_msi_virq(vector);
688             }
689             qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
690                                 NULL, NULL, NULL);
691             event_notifier_cleanup(&vector->interrupt);
692         }
693 
694         g_free(vdev->msi_vectors);
695 
696         if (ret > 0 && ret != vdev->nr_vectors) {
697             vdev->nr_vectors = ret;
698             goto retry;
699         }
700         vdev->nr_vectors = 0;
701 
702         /*
703          * Failing to setup MSI doesn't really fall within any specification.
704          * Let's try leaving interrupts disabled and hope the guest figures
705          * out to fall back to INTx for this device.
706          */
707         error_report("vfio: Error: Failed to enable MSI");
708         vdev->interrupt = VFIO_INT_NONE;
709 
710         return;
711     }
712 
713     trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
714 }
715 
716 static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
717 {
718     Error *err = NULL;
719     int i;
720 
721     for (i = 0; i < vdev->nr_vectors; i++) {
722         VFIOMSIVector *vector = &vdev->msi_vectors[i];
723         if (vdev->msi_vectors[i].use) {
724             if (vector->virq >= 0) {
725                 vfio_remove_kvm_msi_virq(vector);
726             }
727             qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
728                                 NULL, NULL, NULL);
729             event_notifier_cleanup(&vector->interrupt);
730         }
731     }
732 
733     g_free(vdev->msi_vectors);
734     vdev->msi_vectors = NULL;
735     vdev->nr_vectors = 0;
736     vdev->interrupt = VFIO_INT_NONE;
737 
738     vfio_intx_enable(vdev, &err);
739     if (err) {
740         error_reportf_err(err, ERR_PREFIX, vdev->vbasedev.name);
741     }
742 }
743 
744 static void vfio_msix_disable(VFIOPCIDevice *vdev)
745 {
746     int i;
747 
748     msix_unset_vector_notifiers(&vdev->pdev);
749 
750     /*
751      * MSI-X will only release vectors if MSI-X is still enabled on the
752      * device, check through the rest and release it ourselves if necessary.
753      */
754     for (i = 0; i < vdev->nr_vectors; i++) {
755         if (vdev->msi_vectors[i].use) {
756             vfio_msix_vector_release(&vdev->pdev, i);
757             msix_vector_unuse(&vdev->pdev, i);
758         }
759     }
760 
761     if (vdev->nr_vectors) {
762         vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
763     }
764 
765     vfio_msi_disable_common(vdev);
766 
767     memset(vdev->msix->pending, 0,
768            BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
769 
770     trace_vfio_msix_disable(vdev->vbasedev.name);
771 }
772 
773 static void vfio_msi_disable(VFIOPCIDevice *vdev)
774 {
775     vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
776     vfio_msi_disable_common(vdev);
777 
778     trace_vfio_msi_disable(vdev->vbasedev.name);
779 }
780 
781 static void vfio_update_msi(VFIOPCIDevice *vdev)
782 {
783     int i;
784 
785     for (i = 0; i < vdev->nr_vectors; i++) {
786         VFIOMSIVector *vector = &vdev->msi_vectors[i];
787         MSIMessage msg;
788 
789         if (!vector->use || vector->virq < 0) {
790             continue;
791         }
792 
793         msg = msi_get_message(&vdev->pdev, i);
794         vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
795     }
796 }
797 
798 static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
799 {
800     struct vfio_region_info *reg_info;
801     uint64_t size;
802     off_t off = 0;
803     ssize_t bytes;
804 
805     if (vfio_get_region_info(&vdev->vbasedev,
806                              VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
807         error_report("vfio: Error getting ROM info: %m");
808         return;
809     }
810 
811     trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
812                             (unsigned long)reg_info->offset,
813                             (unsigned long)reg_info->flags);
814 
815     vdev->rom_size = size = reg_info->size;
816     vdev->rom_offset = reg_info->offset;
817 
818     g_free(reg_info);
819 
820     if (!vdev->rom_size) {
821         vdev->rom_read_failed = true;
822         error_report("vfio-pci: Cannot read device rom at "
823                     "%s", vdev->vbasedev.name);
824         error_printf("Device option ROM contents are probably invalid "
825                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
826                     "or load from file with romfile=\n");
827         return;
828     }
829 
830     vdev->rom = g_malloc(size);
831     memset(vdev->rom, 0xff, size);
832 
833     while (size) {
834         bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
835                       size, vdev->rom_offset + off);
836         if (bytes == 0) {
837             break;
838         } else if (bytes > 0) {
839             off += bytes;
840             size -= bytes;
841         } else {
842             if (errno == EINTR || errno == EAGAIN) {
843                 continue;
844             }
845             error_report("vfio: Error reading device ROM: %m");
846             break;
847         }
848     }
849 
850     /*
851      * Test the ROM signature against our device, if the vendor is correct
852      * but the device ID doesn't match, store the correct device ID and
853      * recompute the checksum.  Intel IGD devices need this and are known
854      * to have bogus checksums so we can't simply adjust the checksum.
855      */
856     if (pci_get_word(vdev->rom) == 0xaa55 &&
857         pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
858         !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
859         uint16_t vid, did;
860 
861         vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
862         did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
863 
864         if (vid == vdev->vendor_id && did != vdev->device_id) {
865             int i;
866             uint8_t csum, *data = vdev->rom;
867 
868             pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
869                          vdev->device_id);
870             data[6] = 0;
871 
872             for (csum = 0, i = 0; i < vdev->rom_size; i++) {
873                 csum += data[i];
874             }
875 
876             data[6] = -csum;
877         }
878     }
879 }
880 
881 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
882 {
883     VFIOPCIDevice *vdev = opaque;
884     union {
885         uint8_t byte;
886         uint16_t word;
887         uint32_t dword;
888         uint64_t qword;
889     } val;
890     uint64_t data = 0;
891 
892     /* Load the ROM lazily when the guest tries to read it */
893     if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
894         vfio_pci_load_rom(vdev);
895     }
896 
897     memcpy(&val, vdev->rom + addr,
898            (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
899 
900     switch (size) {
901     case 1:
902         data = val.byte;
903         break;
904     case 2:
905         data = le16_to_cpu(val.word);
906         break;
907     case 4:
908         data = le32_to_cpu(val.dword);
909         break;
910     default:
911         hw_error("vfio: unsupported read size, %d bytes\n", size);
912         break;
913     }
914 
915     trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
916 
917     return data;
918 }
919 
920 static void vfio_rom_write(void *opaque, hwaddr addr,
921                            uint64_t data, unsigned size)
922 {
923 }
924 
925 static const MemoryRegionOps vfio_rom_ops = {
926     .read = vfio_rom_read,
927     .write = vfio_rom_write,
928     .endianness = DEVICE_LITTLE_ENDIAN,
929 };
930 
931 static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
932 {
933     uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
934     off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
935     DeviceState *dev = DEVICE(vdev);
936     char *name;
937     int fd = vdev->vbasedev.fd;
938 
939     if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
940         /* Since pci handles romfile, just print a message and return */
941         if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
942             error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified romfile\n",
943                          vdev->vbasedev.name);
944         }
945         return;
946     }
947 
948     /*
949      * Use the same size ROM BAR as the physical device.  The contents
950      * will get filled in later when the guest tries to read it.
951      */
952     if (pread(fd, &orig, 4, offset) != 4 ||
953         pwrite(fd, &size, 4, offset) != 4 ||
954         pread(fd, &size, 4, offset) != 4 ||
955         pwrite(fd, &orig, 4, offset) != 4) {
956         error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
957         return;
958     }
959 
960     size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
961 
962     if (!size) {
963         return;
964     }
965 
966     if (vfio_blacklist_opt_rom(vdev)) {
967         if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
968             error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified non zero value for rombar\n",
969                          vdev->vbasedev.name);
970         } else {
971             error_printf("Warning : Rom loading for device at %s has been disabled due to system instability issues. Specify rombar=1 or romfile to force\n",
972                          vdev->vbasedev.name);
973             return;
974         }
975     }
976 
977     trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
978 
979     name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
980 
981     memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
982                           &vfio_rom_ops, vdev, name, size);
983     g_free(name);
984 
985     pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
986                      PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
987 
988     vdev->pdev.has_rom = true;
989     vdev->rom_read_failed = false;
990 }
991 
992 void vfio_vga_write(void *opaque, hwaddr addr,
993                            uint64_t data, unsigned size)
994 {
995     VFIOVGARegion *region = opaque;
996     VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
997     union {
998         uint8_t byte;
999         uint16_t word;
1000         uint32_t dword;
1001         uint64_t qword;
1002     } buf;
1003     off_t offset = vga->fd_offset + region->offset + addr;
1004 
1005     switch (size) {
1006     case 1:
1007         buf.byte = data;
1008         break;
1009     case 2:
1010         buf.word = cpu_to_le16(data);
1011         break;
1012     case 4:
1013         buf.dword = cpu_to_le32(data);
1014         break;
1015     default:
1016         hw_error("vfio: unsupported write size, %d bytes", size);
1017         break;
1018     }
1019 
1020     if (pwrite(vga->fd, &buf, size, offset) != size) {
1021         error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1022                      __func__, region->offset + addr, data, size);
1023     }
1024 
1025     trace_vfio_vga_write(region->offset + addr, data, size);
1026 }
1027 
1028 uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
1029 {
1030     VFIOVGARegion *region = opaque;
1031     VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1032     union {
1033         uint8_t byte;
1034         uint16_t word;
1035         uint32_t dword;
1036         uint64_t qword;
1037     } buf;
1038     uint64_t data = 0;
1039     off_t offset = vga->fd_offset + region->offset + addr;
1040 
1041     if (pread(vga->fd, &buf, size, offset) != size) {
1042         error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1043                      __func__, region->offset + addr, size);
1044         return (uint64_t)-1;
1045     }
1046 
1047     switch (size) {
1048     case 1:
1049         data = buf.byte;
1050         break;
1051     case 2:
1052         data = le16_to_cpu(buf.word);
1053         break;
1054     case 4:
1055         data = le32_to_cpu(buf.dword);
1056         break;
1057     default:
1058         hw_error("vfio: unsupported read size, %d bytes", size);
1059         break;
1060     }
1061 
1062     trace_vfio_vga_read(region->offset + addr, size, data);
1063 
1064     return data;
1065 }
1066 
1067 static const MemoryRegionOps vfio_vga_ops = {
1068     .read = vfio_vga_read,
1069     .write = vfio_vga_write,
1070     .endianness = DEVICE_LITTLE_ENDIAN,
1071 };
1072 
1073 /*
1074  * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1075  * size if the BAR is in an exclusive page in host so that we could map
1076  * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1077  * page in guest. So we should set the priority of the expanded memory
1078  * region to zero in case of overlap with BARs which share the same page
1079  * with the sub-page BAR in guest. Besides, we should also recover the
1080  * size of this sub-page BAR when its base address is changed in guest
1081  * and not page aligned any more.
1082  */
1083 static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1084 {
1085     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
1086     VFIORegion *region = &vdev->bars[bar].region;
1087     MemoryRegion *mmap_mr, *mr;
1088     PCIIORegion *r;
1089     pcibus_t bar_addr;
1090     uint64_t size = region->size;
1091 
1092     /* Make sure that the whole region is allowed to be mmapped */
1093     if (region->nr_mmaps != 1 || !region->mmaps[0].mmap ||
1094         region->mmaps[0].size != region->size) {
1095         return;
1096     }
1097 
1098     r = &pdev->io_regions[bar];
1099     bar_addr = r->addr;
1100     mr = region->mem;
1101     mmap_mr = &region->mmaps[0].mem;
1102 
1103     /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1104     if (bar_addr != PCI_BAR_UNMAPPED &&
1105         !(bar_addr & ~qemu_real_host_page_mask)) {
1106         size = qemu_real_host_page_size;
1107     }
1108 
1109     memory_region_transaction_begin();
1110 
1111     memory_region_set_size(mr, size);
1112     memory_region_set_size(mmap_mr, size);
1113     if (size != region->size && memory_region_is_mapped(mr)) {
1114         memory_region_del_subregion(r->address_space, mr);
1115         memory_region_add_subregion_overlap(r->address_space,
1116                                             bar_addr, mr, 0);
1117     }
1118 
1119     memory_region_transaction_commit();
1120 }
1121 
1122 /*
1123  * PCI config space
1124  */
1125 uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1126 {
1127     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
1128     uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1129 
1130     memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1131     emu_bits = le32_to_cpu(emu_bits);
1132 
1133     if (emu_bits) {
1134         emu_val = pci_default_read_config(pdev, addr, len);
1135     }
1136 
1137     if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1138         ssize_t ret;
1139 
1140         ret = pread(vdev->vbasedev.fd, &phys_val, len,
1141                     vdev->config_offset + addr);
1142         if (ret != len) {
1143             error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1144                          __func__, vdev->vbasedev.name, addr, len);
1145             return -errno;
1146         }
1147         phys_val = le32_to_cpu(phys_val);
1148     }
1149 
1150     val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
1151 
1152     trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1153 
1154     return val;
1155 }
1156 
1157 void vfio_pci_write_config(PCIDevice *pdev,
1158                            uint32_t addr, uint32_t val, int len)
1159 {
1160     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
1161     uint32_t val_le = cpu_to_le32(val);
1162 
1163     trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1164 
1165     /* Write everything to VFIO, let it filter out what we can't write */
1166     if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
1167                 != len) {
1168         error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1169                      __func__, vdev->vbasedev.name, addr, val, len);
1170     }
1171 
1172     /* MSI/MSI-X Enabling/Disabling */
1173     if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1174         ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1175         int is_enabled, was_enabled = msi_enabled(pdev);
1176 
1177         pci_default_write_config(pdev, addr, val, len);
1178 
1179         is_enabled = msi_enabled(pdev);
1180 
1181         if (!was_enabled) {
1182             if (is_enabled) {
1183                 vfio_msi_enable(vdev);
1184             }
1185         } else {
1186             if (!is_enabled) {
1187                 vfio_msi_disable(vdev);
1188             } else {
1189                 vfio_update_msi(vdev);
1190             }
1191         }
1192     } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1193         ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1194         int is_enabled, was_enabled = msix_enabled(pdev);
1195 
1196         pci_default_write_config(pdev, addr, val, len);
1197 
1198         is_enabled = msix_enabled(pdev);
1199 
1200         if (!was_enabled && is_enabled) {
1201             vfio_msix_enable(vdev);
1202         } else if (was_enabled && !is_enabled) {
1203             vfio_msix_disable(vdev);
1204         }
1205     } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
1206         range_covers_byte(addr, len, PCI_COMMAND)) {
1207         pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1208         int bar;
1209 
1210         for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1211             old_addr[bar] = pdev->io_regions[bar].addr;
1212         }
1213 
1214         pci_default_write_config(pdev, addr, val, len);
1215 
1216         for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1217             if (old_addr[bar] != pdev->io_regions[bar].addr &&
1218                 pdev->io_regions[bar].size > 0 &&
1219                 pdev->io_regions[bar].size < qemu_real_host_page_size) {
1220                 vfio_sub_page_bar_update_mapping(pdev, bar);
1221             }
1222         }
1223     } else {
1224         /* Write everything to QEMU to keep emulated bits correct */
1225         pci_default_write_config(pdev, addr, val, len);
1226     }
1227 }
1228 
1229 /*
1230  * Interrupt setup
1231  */
1232 static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1233 {
1234     /*
1235      * More complicated than it looks.  Disabling MSI/X transitions the
1236      * device to INTx mode (if supported).  Therefore we need to first
1237      * disable MSI/X and then cleanup by disabling INTx.
1238      */
1239     if (vdev->interrupt == VFIO_INT_MSIX) {
1240         vfio_msix_disable(vdev);
1241     } else if (vdev->interrupt == VFIO_INT_MSI) {
1242         vfio_msi_disable(vdev);
1243     }
1244 
1245     if (vdev->interrupt == VFIO_INT_INTx) {
1246         vfio_intx_disable(vdev);
1247     }
1248 }
1249 
1250 static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1251 {
1252     uint16_t ctrl;
1253     bool msi_64bit, msi_maskbit;
1254     int ret, entries;
1255     Error *err = NULL;
1256 
1257     if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
1258               vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
1259         error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS");
1260         return -errno;
1261     }
1262     ctrl = le16_to_cpu(ctrl);
1263 
1264     msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1265     msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1266     entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1267 
1268     trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1269 
1270     ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1271     if (ret < 0) {
1272         if (ret == -ENOTSUP) {
1273             return 0;
1274         }
1275         error_prepend(&err, "msi_init failed: ");
1276         error_propagate(errp, err);
1277         return ret;
1278     }
1279     vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1280 
1281     return 0;
1282 }
1283 
1284 static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1285 {
1286     off_t start, end;
1287     VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1288 
1289     /*
1290      * We expect to find a single mmap covering the whole BAR, anything else
1291      * means it's either unsupported or already setup.
1292      */
1293     if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
1294         region->size != region->mmaps[0].size) {
1295         return;
1296     }
1297 
1298     /* MSI-X table start and end aligned to host page size */
1299     start = vdev->msix->table_offset & qemu_real_host_page_mask;
1300     end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1301                                (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1302 
1303     /*
1304      * Does the MSI-X table cover the beginning of the BAR?  The whole BAR?
1305      * NB - Host page size is necessarily a power of two and so is the PCI
1306      * BAR (not counting EA yet), therefore if we have host page aligned
1307      * @start and @end, then any remainder of the BAR before or after those
1308      * must be at least host page sized and therefore mmap'able.
1309      */
1310     if (!start) {
1311         if (end >= region->size) {
1312             region->nr_mmaps = 0;
1313             g_free(region->mmaps);
1314             region->mmaps = NULL;
1315             trace_vfio_msix_fixup(vdev->vbasedev.name,
1316                                   vdev->msix->table_bar, 0, 0);
1317         } else {
1318             region->mmaps[0].offset = end;
1319             region->mmaps[0].size = region->size - end;
1320             trace_vfio_msix_fixup(vdev->vbasedev.name,
1321                               vdev->msix->table_bar, region->mmaps[0].offset,
1322                               region->mmaps[0].offset + region->mmaps[0].size);
1323         }
1324 
1325     /* Maybe it's aligned at the end of the BAR */
1326     } else if (end >= region->size) {
1327         region->mmaps[0].size = start;
1328         trace_vfio_msix_fixup(vdev->vbasedev.name,
1329                               vdev->msix->table_bar, region->mmaps[0].offset,
1330                               region->mmaps[0].offset + region->mmaps[0].size);
1331 
1332     /* Otherwise it must split the BAR */
1333     } else {
1334         region->nr_mmaps = 2;
1335         region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1336 
1337         memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
1338 
1339         region->mmaps[0].size = start;
1340         trace_vfio_msix_fixup(vdev->vbasedev.name,
1341                               vdev->msix->table_bar, region->mmaps[0].offset,
1342                               region->mmaps[0].offset + region->mmaps[0].size);
1343 
1344         region->mmaps[1].offset = end;
1345         region->mmaps[1].size = region->size - end;
1346         trace_vfio_msix_fixup(vdev->vbasedev.name,
1347                               vdev->msix->table_bar, region->mmaps[1].offset,
1348                               region->mmaps[1].offset + region->mmaps[1].size);
1349     }
1350 }
1351 
1352 /*
1353  * We don't have any control over how pci_add_capability() inserts
1354  * capabilities into the chain.  In order to setup MSI-X we need a
1355  * MemoryRegion for the BAR.  In order to setup the BAR and not
1356  * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1357  * need to first look for where the MSI-X table lives.  So we
1358  * unfortunately split MSI-X setup across two functions.
1359  */
1360 static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
1361 {
1362     uint8_t pos;
1363     uint16_t ctrl;
1364     uint32_t table, pba;
1365     int fd = vdev->vbasedev.fd;
1366     VFIOMSIXInfo *msix;
1367 
1368     pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
1369     if (!pos) {
1370         return;
1371     }
1372 
1373     if (pread(fd, &ctrl, sizeof(ctrl),
1374               vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
1375         error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS");
1376         return;
1377     }
1378 
1379     if (pread(fd, &table, sizeof(table),
1380               vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
1381         error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE");
1382         return;
1383     }
1384 
1385     if (pread(fd, &pba, sizeof(pba),
1386               vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
1387         error_setg_errno(errp, errno, "failed to read PCI MSIX PBA");
1388         return;
1389     }
1390 
1391     ctrl = le16_to_cpu(ctrl);
1392     table = le32_to_cpu(table);
1393     pba = le32_to_cpu(pba);
1394 
1395     msix = g_malloc0(sizeof(*msix));
1396     msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1397     msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1398     msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1399     msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1400     msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1401 
1402     /*
1403      * Test the size of the pba_offset variable and catch if it extends outside
1404      * of the specified BAR. If it is the case, we need to apply a hardware
1405      * specific quirk if the device is known or we have a broken configuration.
1406      */
1407     if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1408         /*
1409          * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1410          * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1411          * the VF PBA offset while the BAR itself is only 8k. The correct value
1412          * is 0x1000, so we hard code that here.
1413          */
1414         if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1415             (vdev->device_id & 0xff00) == 0x5800) {
1416             msix->pba_offset = 0x1000;
1417         } else {
1418             error_setg(errp, "hardware reports invalid configuration, "
1419                        "MSIX PBA outside of specified BAR");
1420             g_free(msix);
1421             return;
1422         }
1423     }
1424 
1425     trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1426                                 msix->table_offset, msix->entries);
1427     vdev->msix = msix;
1428 
1429     vfio_pci_fixup_msix_region(vdev);
1430 }
1431 
1432 static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1433 {
1434     int ret;
1435     Error *err = NULL;
1436 
1437     vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) *
1438                                     sizeof(unsigned long));
1439     ret = msix_init(&vdev->pdev, vdev->msix->entries,
1440                     vdev->bars[vdev->msix->table_bar].region.mem,
1441                     vdev->msix->table_bar, vdev->msix->table_offset,
1442                     vdev->bars[vdev->msix->pba_bar].region.mem,
1443                     vdev->msix->pba_bar, vdev->msix->pba_offset, pos,
1444                     &err);
1445     if (ret < 0) {
1446         if (ret == -ENOTSUP) {
1447             error_report_err(err);
1448             return 0;
1449         }
1450 
1451         error_propagate(errp, err);
1452         return ret;
1453     }
1454 
1455     /*
1456      * The PCI spec suggests that devices provide additional alignment for
1457      * MSI-X structures and avoid overlapping non-MSI-X related registers.
1458      * For an assigned device, this hopefully means that emulation of MSI-X
1459      * structures does not affect the performance of the device.  If devices
1460      * fail to provide that alignment, a significant performance penalty may
1461      * result, for instance Mellanox MT27500 VFs:
1462      * http://www.spinics.net/lists/kvm/msg125881.html
1463      *
1464      * The PBA is simply not that important for such a serious regression and
1465      * most drivers do not appear to look at it.  The solution for this is to
1466      * disable the PBA MemoryRegion unless it's being used.  We disable it
1467      * here and only enable it if a masked vector fires through QEMU.  As the
1468      * vector-use notifier is called, which occurs on unmask, we test whether
1469      * PBA emulation is needed and again disable if not.
1470      */
1471     memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
1472 
1473     return 0;
1474 }
1475 
1476 static void vfio_teardown_msi(VFIOPCIDevice *vdev)
1477 {
1478     msi_uninit(&vdev->pdev);
1479 
1480     if (vdev->msix) {
1481         msix_uninit(&vdev->pdev,
1482                     vdev->bars[vdev->msix->table_bar].region.mem,
1483                     vdev->bars[vdev->msix->pba_bar].region.mem);
1484         g_free(vdev->msix->pending);
1485     }
1486 }
1487 
1488 /*
1489  * Resource setup
1490  */
1491 static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1492 {
1493     int i;
1494 
1495     for (i = 0; i < PCI_ROM_SLOT; i++) {
1496         vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1497     }
1498 }
1499 
1500 static void vfio_bar_setup(VFIOPCIDevice *vdev, int nr)
1501 {
1502     VFIOBAR *bar = &vdev->bars[nr];
1503 
1504     uint32_t pci_bar;
1505     uint8_t type;
1506     int ret;
1507 
1508     /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1509     if (!bar->region.size) {
1510         return;
1511     }
1512 
1513     /* Determine what type of BAR this is for registration */
1514     ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
1515                 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
1516     if (ret != sizeof(pci_bar)) {
1517         error_report("vfio: Failed to read BAR %d (%m)", nr);
1518         return;
1519     }
1520 
1521     pci_bar = le32_to_cpu(pci_bar);
1522     bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1523     bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1524     type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1525                                     ~PCI_BASE_ADDRESS_MEM_MASK);
1526 
1527     if (vfio_region_mmap(&bar->region)) {
1528         error_report("Failed to mmap %s BAR %d. Performance may be slow",
1529                      vdev->vbasedev.name, nr);
1530     }
1531 
1532     pci_register_bar(&vdev->pdev, nr, type, bar->region.mem);
1533 }
1534 
1535 static void vfio_bars_setup(VFIOPCIDevice *vdev)
1536 {
1537     int i;
1538 
1539     for (i = 0; i < PCI_ROM_SLOT; i++) {
1540         vfio_bar_setup(vdev, i);
1541     }
1542 }
1543 
1544 static void vfio_bars_exit(VFIOPCIDevice *vdev)
1545 {
1546     int i;
1547 
1548     for (i = 0; i < PCI_ROM_SLOT; i++) {
1549         vfio_bar_quirk_exit(vdev, i);
1550         vfio_region_exit(&vdev->bars[i].region);
1551     }
1552 
1553     if (vdev->vga) {
1554         pci_unregister_vga(&vdev->pdev);
1555         vfio_vga_quirk_exit(vdev);
1556     }
1557 }
1558 
1559 static void vfio_bars_finalize(VFIOPCIDevice *vdev)
1560 {
1561     int i;
1562 
1563     for (i = 0; i < PCI_ROM_SLOT; i++) {
1564         vfio_bar_quirk_finalize(vdev, i);
1565         vfio_region_finalize(&vdev->bars[i].region);
1566     }
1567 
1568     if (vdev->vga) {
1569         vfio_vga_quirk_finalize(vdev);
1570         for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
1571             object_unparent(OBJECT(&vdev->vga->region[i].mem));
1572         }
1573         g_free(vdev->vga);
1574     }
1575 }
1576 
1577 /*
1578  * General setup
1579  */
1580 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
1581 {
1582     uint8_t tmp;
1583     uint16_t next = PCI_CONFIG_SPACE_SIZE;
1584 
1585     for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
1586          tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
1587         if (tmp > pos && tmp < next) {
1588             next = tmp;
1589         }
1590     }
1591 
1592     return next - pos;
1593 }
1594 
1595 
1596 static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
1597 {
1598     uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
1599 
1600     for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
1601         tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
1602         if (tmp > pos && tmp < next) {
1603             next = tmp;
1604         }
1605     }
1606 
1607     return next - pos;
1608 }
1609 
1610 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
1611 {
1612     pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
1613 }
1614 
1615 static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
1616                                    uint16_t val, uint16_t mask)
1617 {
1618     vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
1619     vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
1620     vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
1621 }
1622 
1623 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
1624 {
1625     pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
1626 }
1627 
1628 static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
1629                                    uint32_t val, uint32_t mask)
1630 {
1631     vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
1632     vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
1633     vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
1634 }
1635 
1636 static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
1637                                Error **errp)
1638 {
1639     uint16_t flags;
1640     uint8_t type;
1641 
1642     flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
1643     type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
1644 
1645     if (type != PCI_EXP_TYPE_ENDPOINT &&
1646         type != PCI_EXP_TYPE_LEG_END &&
1647         type != PCI_EXP_TYPE_RC_END) {
1648 
1649         error_setg(errp, "assignment of PCIe type 0x%x "
1650                    "devices is not currently supported", type);
1651         return -EINVAL;
1652     }
1653 
1654     if (!pci_bus_is_express(vdev->pdev.bus)) {
1655         PCIBus *bus = vdev->pdev.bus;
1656         PCIDevice *bridge;
1657 
1658         /*
1659          * Traditionally PCI device assignment exposes the PCIe capability
1660          * as-is on non-express buses.  The reason being that some drivers
1661          * simply assume that it's there, for example tg3.  However when
1662          * we're running on a native PCIe machine type, like Q35, we need
1663          * to hide the PCIe capability.  The reason for this is twofold;
1664          * first Windows guests get a Code 10 error when the PCIe capability
1665          * is exposed in this configuration.  Therefore express devices won't
1666          * work at all unless they're attached to express buses in the VM.
1667          * Second, a native PCIe machine introduces the possibility of fine
1668          * granularity IOMMUs supporting both translation and isolation.
1669          * Guest code to discover the IOMMU visibility of a device, such as
1670          * IOMMU grouping code on Linux, is very aware of device types and
1671          * valid transitions between bus types.  An express device on a non-
1672          * express bus is not a valid combination on bare metal systems.
1673          *
1674          * Drivers that require a PCIe capability to make the device
1675          * functional are simply going to need to have their devices placed
1676          * on a PCIe bus in the VM.
1677          */
1678         while (!pci_bus_is_root(bus)) {
1679             bridge = pci_bridge_get_device(bus);
1680             bus = bridge->bus;
1681         }
1682 
1683         if (pci_bus_is_express(bus)) {
1684             return 0;
1685         }
1686 
1687     } else if (pci_bus_is_root(vdev->pdev.bus)) {
1688         /*
1689          * On a Root Complex bus Endpoints become Root Complex Integrated
1690          * Endpoints, which changes the type and clears the LNK & LNK2 fields.
1691          */
1692         if (type == PCI_EXP_TYPE_ENDPOINT) {
1693             vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1694                                    PCI_EXP_TYPE_RC_END << 4,
1695                                    PCI_EXP_FLAGS_TYPE);
1696 
1697             /* Link Capabilities, Status, and Control goes away */
1698             if (size > PCI_EXP_LNKCTL) {
1699                 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
1700                 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1701                 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
1702 
1703 #ifndef PCI_EXP_LNKCAP2
1704 #define PCI_EXP_LNKCAP2 44
1705 #endif
1706 #ifndef PCI_EXP_LNKSTA2
1707 #define PCI_EXP_LNKSTA2 50
1708 #endif
1709                 /* Link 2 Capabilities, Status, and Control goes away */
1710                 if (size > PCI_EXP_LNKCAP2) {
1711                     vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
1712                     vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
1713                     vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
1714                 }
1715             }
1716 
1717         } else if (type == PCI_EXP_TYPE_LEG_END) {
1718             /*
1719              * Legacy endpoints don't belong on the root complex.  Windows
1720              * seems to be happier with devices if we skip the capability.
1721              */
1722             return 0;
1723         }
1724 
1725     } else {
1726         /*
1727          * Convert Root Complex Integrated Endpoints to regular endpoints.
1728          * These devices don't support LNK/LNK2 capabilities, so make them up.
1729          */
1730         if (type == PCI_EXP_TYPE_RC_END) {
1731             vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1732                                    PCI_EXP_TYPE_ENDPOINT << 4,
1733                                    PCI_EXP_FLAGS_TYPE);
1734             vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
1735                                    PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25, ~0);
1736             vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1737         }
1738 
1739         /* Mark the Link Status bits as emulated to allow virtual negotiation */
1740         vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA,
1741                                pci_get_word(vdev->pdev.config + pos +
1742                                             PCI_EXP_LNKSTA),
1743                                PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS);
1744     }
1745 
1746     pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size);
1747     if (pos >= 0) {
1748         vdev->pdev.exp.exp_cap = pos;
1749     }
1750 
1751     return pos;
1752 }
1753 
1754 static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
1755 {
1756     uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
1757 
1758     if (cap & PCI_EXP_DEVCAP_FLR) {
1759         trace_vfio_check_pcie_flr(vdev->vbasedev.name);
1760         vdev->has_flr = true;
1761     }
1762 }
1763 
1764 static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
1765 {
1766     uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
1767 
1768     if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
1769         trace_vfio_check_pm_reset(vdev->vbasedev.name);
1770         vdev->has_pm_reset = true;
1771     }
1772 }
1773 
1774 static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
1775 {
1776     uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
1777 
1778     if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
1779         trace_vfio_check_af_flr(vdev->vbasedev.name);
1780         vdev->has_flr = true;
1781     }
1782 }
1783 
1784 static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp)
1785 {
1786     PCIDevice *pdev = &vdev->pdev;
1787     uint8_t cap_id, next, size;
1788     int ret;
1789 
1790     cap_id = pdev->config[pos];
1791     next = pdev->config[pos + PCI_CAP_LIST_NEXT];
1792 
1793     /*
1794      * If it becomes important to configure capabilities to their actual
1795      * size, use this as the default when it's something we don't recognize.
1796      * Since QEMU doesn't actually handle many of the config accesses,
1797      * exact size doesn't seem worthwhile.
1798      */
1799     size = vfio_std_cap_max_size(pdev, pos);
1800 
1801     /*
1802      * pci_add_capability always inserts the new capability at the head
1803      * of the chain.  Therefore to end up with a chain that matches the
1804      * physical device, we insert from the end by making this recursive.
1805      * This is also why we pre-calculate size above as cached config space
1806      * will be changed as we unwind the stack.
1807      */
1808     if (next) {
1809         ret = vfio_add_std_cap(vdev, next, errp);
1810         if (ret) {
1811             goto out;
1812         }
1813     } else {
1814         /* Begin the rebuild, use QEMU emulated list bits */
1815         pdev->config[PCI_CAPABILITY_LIST] = 0;
1816         vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
1817         vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
1818     }
1819 
1820     /* Use emulated next pointer to allow dropping caps */
1821     pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
1822 
1823     switch (cap_id) {
1824     case PCI_CAP_ID_MSI:
1825         ret = vfio_msi_setup(vdev, pos, errp);
1826         break;
1827     case PCI_CAP_ID_EXP:
1828         vfio_check_pcie_flr(vdev, pos);
1829         ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
1830         break;
1831     case PCI_CAP_ID_MSIX:
1832         ret = vfio_msix_setup(vdev, pos, errp);
1833         break;
1834     case PCI_CAP_ID_PM:
1835         vfio_check_pm_reset(vdev, pos);
1836         vdev->pm_cap = pos;
1837         ret = pci_add_capability2(pdev, cap_id, pos, size, errp);
1838         break;
1839     case PCI_CAP_ID_AF:
1840         vfio_check_af_flr(vdev, pos);
1841         ret = pci_add_capability2(pdev, cap_id, pos, size, errp);
1842         break;
1843     default:
1844         ret = pci_add_capability2(pdev, cap_id, pos, size, errp);
1845         break;
1846     }
1847 out:
1848     if (ret < 0) {
1849         error_prepend(errp,
1850                       "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
1851                       cap_id, size, pos);
1852         return ret;
1853     }
1854 
1855     return 0;
1856 }
1857 
1858 static void vfio_add_ext_cap(VFIOPCIDevice *vdev)
1859 {
1860     PCIDevice *pdev = &vdev->pdev;
1861     uint32_t header;
1862     uint16_t cap_id, next, size;
1863     uint8_t cap_ver;
1864     uint8_t *config;
1865 
1866     /* Only add extended caps if we have them and the guest can see them */
1867     if (!pci_is_express(pdev) || !pci_bus_is_express(pdev->bus) ||
1868         !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
1869         return;
1870     }
1871 
1872     /*
1873      * pcie_add_capability always inserts the new capability at the tail
1874      * of the chain.  Therefore to end up with a chain that matches the
1875      * physical device, we cache the config space to avoid overwriting
1876      * the original config space when we parse the extended capabilities.
1877      */
1878     config = g_memdup(pdev->config, vdev->config_size);
1879 
1880     /*
1881      * Extended capabilities are chained with each pointing to the next, so we
1882      * can drop anything other than the head of the chain simply by modifying
1883      * the previous next pointer.  Seed the head of the chain here such that
1884      * we can simply skip any capabilities we want to drop below, regardless
1885      * of their position in the chain.  If this stub capability still exists
1886      * after we add the capabilities we want to expose, update the capability
1887      * ID to zero.  Note that we cannot seed with the capability header being
1888      * zero as this conflicts with definition of an absent capability chain
1889      * and prevents capabilities beyond the head of the list from being added.
1890      * By replacing the dummy capability ID with zero after walking the device
1891      * chain, we also transparently mark extended capabilities as absent if
1892      * no capabilities were added.  Note that the PCIe spec defines an absence
1893      * of extended capabilities to be determined by a value of zero for the
1894      * capability ID, version, AND next pointer.  A non-zero next pointer
1895      * should be sufficient to indicate additional capabilities are present,
1896      * which will occur if we call pcie_add_capability() below.  The entire
1897      * first dword is emulated to support this.
1898      *
1899      * NB. The kernel side does similar masking, so be prepared that our
1900      * view of the device may also contain a capability ID zero in the head
1901      * of the chain.  Skip it for the same reason that we cannot seed the
1902      * chain with a zero capability.
1903      */
1904     pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
1905                  PCI_EXT_CAP(0xFFFF, 0, 0));
1906     pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
1907     pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
1908 
1909     for (next = PCI_CONFIG_SPACE_SIZE; next;
1910          next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
1911         header = pci_get_long(config + next);
1912         cap_id = PCI_EXT_CAP_ID(header);
1913         cap_ver = PCI_EXT_CAP_VER(header);
1914 
1915         /*
1916          * If it becomes important to configure extended capabilities to their
1917          * actual size, use this as the default when it's something we don't
1918          * recognize. Since QEMU doesn't actually handle many of the config
1919          * accesses, exact size doesn't seem worthwhile.
1920          */
1921         size = vfio_ext_cap_max_size(config, next);
1922 
1923         /* Use emulated next pointer to allow dropping extended caps */
1924         pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
1925                                    PCI_EXT_CAP_NEXT_MASK);
1926 
1927         switch (cap_id) {
1928         case 0: /* kernel masked capability */
1929         case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
1930         case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
1931             trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
1932             break;
1933         default:
1934             pcie_add_capability(pdev, cap_id, cap_ver, next, size);
1935         }
1936 
1937     }
1938 
1939     /* Cleanup chain head ID if necessary */
1940     if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
1941         pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
1942     }
1943 
1944     g_free(config);
1945     return;
1946 }
1947 
1948 static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
1949 {
1950     PCIDevice *pdev = &vdev->pdev;
1951     int ret;
1952 
1953     if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
1954         !pdev->config[PCI_CAPABILITY_LIST]) {
1955         return 0; /* Nothing to add */
1956     }
1957 
1958     ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp);
1959     if (ret) {
1960         return ret;
1961     }
1962 
1963     vfio_add_ext_cap(vdev);
1964     return 0;
1965 }
1966 
1967 static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
1968 {
1969     PCIDevice *pdev = &vdev->pdev;
1970     uint16_t cmd;
1971 
1972     vfio_disable_interrupts(vdev);
1973 
1974     /* Make sure the device is in D0 */
1975     if (vdev->pm_cap) {
1976         uint16_t pmcsr;
1977         uint8_t state;
1978 
1979         pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
1980         state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1981         if (state) {
1982             pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1983             vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
1984             /* vfio handles the necessary delay here */
1985             pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
1986             state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1987             if (state) {
1988                 error_report("vfio: Unable to power on device, stuck in D%d",
1989                              state);
1990             }
1991         }
1992     }
1993 
1994     /*
1995      * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
1996      * Also put INTx Disable in known state.
1997      */
1998     cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
1999     cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
2000              PCI_COMMAND_INTX_DISABLE);
2001     vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2002 }
2003 
2004 static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
2005 {
2006     Error *err = NULL;
2007     int nr;
2008 
2009     vfio_intx_enable(vdev, &err);
2010     if (err) {
2011         error_reportf_err(err, ERR_PREFIX, vdev->vbasedev.name);
2012     }
2013 
2014     for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2015         off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr);
2016         uint32_t val = 0;
2017         uint32_t len = sizeof(val);
2018 
2019         if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) {
2020             error_report("%s(%s) reset bar %d failed: %m", __func__,
2021                          vdev->vbasedev.name, nr);
2022         }
2023     }
2024 }
2025 
2026 static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
2027 {
2028     char tmp[13];
2029 
2030     sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2031             addr->bus, addr->slot, addr->function);
2032 
2033     return (strcmp(tmp, name) == 0);
2034 }
2035 
2036 static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2037 {
2038     VFIOGroup *group;
2039     struct vfio_pci_hot_reset_info *info;
2040     struct vfio_pci_dependent_device *devices;
2041     struct vfio_pci_hot_reset *reset;
2042     int32_t *fds;
2043     int ret, i, count;
2044     bool multi = false;
2045 
2046     trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
2047 
2048     if (!single) {
2049         vfio_pci_pre_reset(vdev);
2050     }
2051     vdev->vbasedev.needs_reset = false;
2052 
2053     info = g_malloc0(sizeof(*info));
2054     info->argsz = sizeof(*info);
2055 
2056     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2057     if (ret && errno != ENOSPC) {
2058         ret = -errno;
2059         if (!vdev->has_pm_reset) {
2060             error_report("vfio: Cannot reset device %s, "
2061                          "no available reset mechanism.", vdev->vbasedev.name);
2062         }
2063         goto out_single;
2064     }
2065 
2066     count = info->count;
2067     info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2068     info->argsz = sizeof(*info) + (count * sizeof(*devices));
2069     devices = &info->devices[0];
2070 
2071     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2072     if (ret) {
2073         ret = -errno;
2074         error_report("vfio: hot reset info failed: %m");
2075         goto out_single;
2076     }
2077 
2078     trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
2079 
2080     /* Verify that we have all the groups required */
2081     for (i = 0; i < info->count; i++) {
2082         PCIHostDeviceAddress host;
2083         VFIOPCIDevice *tmp;
2084         VFIODevice *vbasedev_iter;
2085 
2086         host.domain = devices[i].segment;
2087         host.bus = devices[i].bus;
2088         host.slot = PCI_SLOT(devices[i].devfn);
2089         host.function = PCI_FUNC(devices[i].devfn);
2090 
2091         trace_vfio_pci_hot_reset_dep_devices(host.domain,
2092                 host.bus, host.slot, host.function, devices[i].group_id);
2093 
2094         if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2095             continue;
2096         }
2097 
2098         QLIST_FOREACH(group, &vfio_group_list, next) {
2099             if (group->groupid == devices[i].group_id) {
2100                 break;
2101             }
2102         }
2103 
2104         if (!group) {
2105             if (!vdev->has_pm_reset) {
2106                 error_report("vfio: Cannot reset device %s, "
2107                              "depends on group %d which is not owned.",
2108                              vdev->vbasedev.name, devices[i].group_id);
2109             }
2110             ret = -EPERM;
2111             goto out;
2112         }
2113 
2114         /* Prep dependent devices for reset and clear our marker. */
2115         QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2116             if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2117                 continue;
2118             }
2119             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2120             if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2121                 if (single) {
2122                     ret = -EINVAL;
2123                     goto out_single;
2124                 }
2125                 vfio_pci_pre_reset(tmp);
2126                 tmp->vbasedev.needs_reset = false;
2127                 multi = true;
2128                 break;
2129             }
2130         }
2131     }
2132 
2133     if (!single && !multi) {
2134         ret = -EINVAL;
2135         goto out_single;
2136     }
2137 
2138     /* Determine how many group fds need to be passed */
2139     count = 0;
2140     QLIST_FOREACH(group, &vfio_group_list, next) {
2141         for (i = 0; i < info->count; i++) {
2142             if (group->groupid == devices[i].group_id) {
2143                 count++;
2144                 break;
2145             }
2146         }
2147     }
2148 
2149     reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
2150     reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
2151     fds = &reset->group_fds[0];
2152 
2153     /* Fill in group fds */
2154     QLIST_FOREACH(group, &vfio_group_list, next) {
2155         for (i = 0; i < info->count; i++) {
2156             if (group->groupid == devices[i].group_id) {
2157                 fds[reset->count++] = group->fd;
2158                 break;
2159             }
2160         }
2161     }
2162 
2163     /* Bus reset! */
2164     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
2165     g_free(reset);
2166 
2167     trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
2168                                     ret ? "%m" : "Success");
2169 
2170 out:
2171     /* Re-enable INTx on affected devices */
2172     for (i = 0; i < info->count; i++) {
2173         PCIHostDeviceAddress host;
2174         VFIOPCIDevice *tmp;
2175         VFIODevice *vbasedev_iter;
2176 
2177         host.domain = devices[i].segment;
2178         host.bus = devices[i].bus;
2179         host.slot = PCI_SLOT(devices[i].devfn);
2180         host.function = PCI_FUNC(devices[i].devfn);
2181 
2182         if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2183             continue;
2184         }
2185 
2186         QLIST_FOREACH(group, &vfio_group_list, next) {
2187             if (group->groupid == devices[i].group_id) {
2188                 break;
2189             }
2190         }
2191 
2192         if (!group) {
2193             break;
2194         }
2195 
2196         QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2197             if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2198                 continue;
2199             }
2200             tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2201             if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2202                 vfio_pci_post_reset(tmp);
2203                 break;
2204             }
2205         }
2206     }
2207 out_single:
2208     if (!single) {
2209         vfio_pci_post_reset(vdev);
2210     }
2211     g_free(info);
2212 
2213     return ret;
2214 }
2215 
2216 /*
2217  * We want to differentiate hot reset of mulitple in-use devices vs hot reset
2218  * of a single in-use device.  VFIO_DEVICE_RESET will already handle the case
2219  * of doing hot resets when there is only a single device per bus.  The in-use
2220  * here refers to how many VFIODevices are affected.  A hot reset that affects
2221  * multiple devices, but only a single in-use device, means that we can call
2222  * it from our bus ->reset() callback since the extent is effectively a single
2223  * device.  This allows us to make use of it in the hotplug path.  When there
2224  * are multiple in-use devices, we can only trigger the hot reset during a
2225  * system reset and thus from our reset handler.  We separate _one vs _multi
2226  * here so that we don't overlap and do a double reset on the system reset
2227  * path where both our reset handler and ->reset() callback are used.  Calling
2228  * _one() will only do a hot reset for the one in-use devices case, calling
2229  * _multi() will do nothing if a _one() would have been sufficient.
2230  */
2231 static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2232 {
2233     return vfio_pci_hot_reset(vdev, true);
2234 }
2235 
2236 static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2237 {
2238     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2239     return vfio_pci_hot_reset(vdev, false);
2240 }
2241 
2242 static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2243 {
2244     VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2245     if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
2246         vbasedev->needs_reset = true;
2247     }
2248 }
2249 
2250 static VFIODeviceOps vfio_pci_ops = {
2251     .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
2252     .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
2253     .vfio_eoi = vfio_intx_eoi,
2254 };
2255 
2256 int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp)
2257 {
2258     VFIODevice *vbasedev = &vdev->vbasedev;
2259     struct vfio_region_info *reg_info;
2260     int ret;
2261 
2262     ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
2263     if (ret) {
2264         error_setg_errno(errp, -ret,
2265                          "failed getting region info for VGA region index %d",
2266                          VFIO_PCI_VGA_REGION_INDEX);
2267         return ret;
2268     }
2269 
2270     if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
2271         !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
2272         reg_info->size < 0xbffff + 1) {
2273         error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2274                    (unsigned long)reg_info->flags,
2275                    (unsigned long)reg_info->size);
2276         g_free(reg_info);
2277         return -EINVAL;
2278     }
2279 
2280     vdev->vga = g_new0(VFIOVGA, 1);
2281 
2282     vdev->vga->fd_offset = reg_info->offset;
2283     vdev->vga->fd = vdev->vbasedev.fd;
2284 
2285     g_free(reg_info);
2286 
2287     vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
2288     vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
2289     QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
2290 
2291     memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2292                           OBJECT(vdev), &vfio_vga_ops,
2293                           &vdev->vga->region[QEMU_PCI_VGA_MEM],
2294                           "vfio-vga-mmio@0xa0000",
2295                           QEMU_PCI_VGA_MEM_SIZE);
2296 
2297     vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
2298     vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
2299     QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
2300 
2301     memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2302                           OBJECT(vdev), &vfio_vga_ops,
2303                           &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
2304                           "vfio-vga-io@0x3b0",
2305                           QEMU_PCI_VGA_IO_LO_SIZE);
2306 
2307     vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
2308     vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
2309     QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
2310 
2311     memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
2312                           OBJECT(vdev), &vfio_vga_ops,
2313                           &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
2314                           "vfio-vga-io@0x3c0",
2315                           QEMU_PCI_VGA_IO_HI_SIZE);
2316 
2317     pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2318                      &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2319                      &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
2320 
2321     return 0;
2322 }
2323 
2324 static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
2325 {
2326     VFIODevice *vbasedev = &vdev->vbasedev;
2327     struct vfio_region_info *reg_info;
2328     struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
2329     int i, ret = -1;
2330 
2331     /* Sanity check device */
2332     if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
2333         error_setg(errp, "this isn't a PCI device");
2334         return;
2335     }
2336 
2337     if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
2338         error_setg(errp, "unexpected number of io regions %u",
2339                    vbasedev->num_regions);
2340         return;
2341     }
2342 
2343     if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
2344         error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
2345         return;
2346     }
2347 
2348     for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2349         char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
2350 
2351         ret = vfio_region_setup(OBJECT(vdev), vbasedev,
2352                                 &vdev->bars[i].region, i, name);
2353         g_free(name);
2354 
2355         if (ret) {
2356             error_setg_errno(errp, -ret, "failed to get region %d info", i);
2357             return;
2358         }
2359 
2360         QLIST_INIT(&vdev->bars[i].quirks);
2361     }
2362 
2363     ret = vfio_get_region_info(vbasedev,
2364                                VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
2365     if (ret) {
2366         error_setg_errno(errp, -ret, "failed to get config info");
2367         return;
2368     }
2369 
2370     trace_vfio_populate_device_config(vdev->vbasedev.name,
2371                                       (unsigned long)reg_info->size,
2372                                       (unsigned long)reg_info->offset,
2373                                       (unsigned long)reg_info->flags);
2374 
2375     vdev->config_size = reg_info->size;
2376     if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
2377         vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2378     }
2379     vdev->config_offset = reg_info->offset;
2380 
2381     g_free(reg_info);
2382 
2383     if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
2384         ret = vfio_populate_vga(vdev, errp);
2385         if (ret) {
2386             error_append_hint(errp, "device does not support "
2387                               "requested feature x-vga\n");
2388             return;
2389         }
2390     }
2391 
2392     irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
2393 
2394     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
2395     if (ret) {
2396         /* This can fail for an old kernel or legacy PCI dev */
2397         trace_vfio_populate_device_get_irq_info_failure();
2398     } else if (irq_info.count == 1) {
2399         vdev->pci_aer = true;
2400     } else {
2401         error_report(WARN_PREFIX
2402                      "Could not enable error recovery for the device",
2403                      vbasedev->name);
2404     }
2405 }
2406 
2407 static void vfio_put_device(VFIOPCIDevice *vdev)
2408 {
2409     g_free(vdev->vbasedev.name);
2410     g_free(vdev->msix);
2411 
2412     vfio_put_base_device(&vdev->vbasedev);
2413 }
2414 
2415 static void vfio_err_notifier_handler(void *opaque)
2416 {
2417     VFIOPCIDevice *vdev = opaque;
2418 
2419     if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
2420         return;
2421     }
2422 
2423     /*
2424      * TBD. Retrieve the error details and decide what action
2425      * needs to be taken. One of the actions could be to pass
2426      * the error to the guest and have the guest driver recover
2427      * from the error. This requires that PCIe capabilities be
2428      * exposed to the guest. For now, we just terminate the
2429      * guest to contain the error.
2430      */
2431 
2432     error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
2433 
2434     vm_stop(RUN_STATE_INTERNAL_ERROR);
2435 }
2436 
2437 /*
2438  * Registers error notifier for devices supporting error recovery.
2439  * If we encounter a failure in this function, we report an error
2440  * and continue after disabling error recovery support for the
2441  * device.
2442  */
2443 static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
2444 {
2445     int ret;
2446     int argsz;
2447     struct vfio_irq_set *irq_set;
2448     int32_t *pfd;
2449 
2450     if (!vdev->pci_aer) {
2451         return;
2452     }
2453 
2454     if (event_notifier_init(&vdev->err_notifier, 0)) {
2455         error_report("vfio: Unable to init event notifier for error detection");
2456         vdev->pci_aer = false;
2457         return;
2458     }
2459 
2460     argsz = sizeof(*irq_set) + sizeof(*pfd);
2461 
2462     irq_set = g_malloc0(argsz);
2463     irq_set->argsz = argsz;
2464     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2465                      VFIO_IRQ_SET_ACTION_TRIGGER;
2466     irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
2467     irq_set->start = 0;
2468     irq_set->count = 1;
2469     pfd = (int32_t *)&irq_set->data;
2470 
2471     *pfd = event_notifier_get_fd(&vdev->err_notifier);
2472     qemu_set_fd_handler(*pfd, vfio_err_notifier_handler, NULL, vdev);
2473 
2474     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
2475     if (ret) {
2476         error_report("vfio: Failed to set up error notification");
2477         qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
2478         event_notifier_cleanup(&vdev->err_notifier);
2479         vdev->pci_aer = false;
2480     }
2481     g_free(irq_set);
2482 }
2483 
2484 static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
2485 {
2486     int argsz;
2487     struct vfio_irq_set *irq_set;
2488     int32_t *pfd;
2489     int ret;
2490 
2491     if (!vdev->pci_aer) {
2492         return;
2493     }
2494 
2495     argsz = sizeof(*irq_set) + sizeof(*pfd);
2496 
2497     irq_set = g_malloc0(argsz);
2498     irq_set->argsz = argsz;
2499     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2500                      VFIO_IRQ_SET_ACTION_TRIGGER;
2501     irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
2502     irq_set->start = 0;
2503     irq_set->count = 1;
2504     pfd = (int32_t *)&irq_set->data;
2505     *pfd = -1;
2506 
2507     ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
2508     if (ret) {
2509         error_report("vfio: Failed to de-assign error fd: %m");
2510     }
2511     g_free(irq_set);
2512     qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
2513                         NULL, NULL, vdev);
2514     event_notifier_cleanup(&vdev->err_notifier);
2515 }
2516 
2517 static void vfio_req_notifier_handler(void *opaque)
2518 {
2519     VFIOPCIDevice *vdev = opaque;
2520     Error *err = NULL;
2521 
2522     if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
2523         return;
2524     }
2525 
2526     qdev_unplug(&vdev->pdev.qdev, &err);
2527     if (err) {
2528         error_reportf_err(err, WARN_PREFIX, vdev->vbasedev.name);
2529     }
2530 }
2531 
2532 static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
2533 {
2534     struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
2535                                       .index = VFIO_PCI_REQ_IRQ_INDEX };
2536     int argsz;
2537     struct vfio_irq_set *irq_set;
2538     int32_t *pfd;
2539 
2540     if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
2541         return;
2542     }
2543 
2544     if (ioctl(vdev->vbasedev.fd,
2545               VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
2546         return;
2547     }
2548 
2549     if (event_notifier_init(&vdev->req_notifier, 0)) {
2550         error_report("vfio: Unable to init event notifier for device request");
2551         return;
2552     }
2553 
2554     argsz = sizeof(*irq_set) + sizeof(*pfd);
2555 
2556     irq_set = g_malloc0(argsz);
2557     irq_set->argsz = argsz;
2558     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2559                      VFIO_IRQ_SET_ACTION_TRIGGER;
2560     irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
2561     irq_set->start = 0;
2562     irq_set->count = 1;
2563     pfd = (int32_t *)&irq_set->data;
2564 
2565     *pfd = event_notifier_get_fd(&vdev->req_notifier);
2566     qemu_set_fd_handler(*pfd, vfio_req_notifier_handler, NULL, vdev);
2567 
2568     if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) {
2569         error_report("vfio: Failed to set up device request notification");
2570         qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
2571         event_notifier_cleanup(&vdev->req_notifier);
2572     } else {
2573         vdev->req_enabled = true;
2574     }
2575 
2576     g_free(irq_set);
2577 }
2578 
2579 static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
2580 {
2581     int argsz;
2582     struct vfio_irq_set *irq_set;
2583     int32_t *pfd;
2584 
2585     if (!vdev->req_enabled) {
2586         return;
2587     }
2588 
2589     argsz = sizeof(*irq_set) + sizeof(*pfd);
2590 
2591     irq_set = g_malloc0(argsz);
2592     irq_set->argsz = argsz;
2593     irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
2594                      VFIO_IRQ_SET_ACTION_TRIGGER;
2595     irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
2596     irq_set->start = 0;
2597     irq_set->count = 1;
2598     pfd = (int32_t *)&irq_set->data;
2599     *pfd = -1;
2600 
2601     if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) {
2602         error_report("vfio: Failed to de-assign device request fd: %m");
2603     }
2604     g_free(irq_set);
2605     qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
2606                         NULL, NULL, vdev);
2607     event_notifier_cleanup(&vdev->req_notifier);
2608 
2609     vdev->req_enabled = false;
2610 }
2611 
2612 static void vfio_realize(PCIDevice *pdev, Error **errp)
2613 {
2614     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
2615     VFIODevice *vbasedev_iter;
2616     VFIOGroup *group;
2617     char *tmp, group_path[PATH_MAX], *group_name;
2618     Error *err = NULL;
2619     ssize_t len;
2620     struct stat st;
2621     int groupid;
2622     int i, ret;
2623 
2624     if (!vdev->vbasedev.sysfsdev) {
2625         if (!(~vdev->host.domain || ~vdev->host.bus ||
2626               ~vdev->host.slot || ~vdev->host.function)) {
2627             error_setg(errp, "No provided host device");
2628             error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
2629                               "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2630             return;
2631         }
2632         vdev->vbasedev.sysfsdev =
2633             g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2634                             vdev->host.domain, vdev->host.bus,
2635                             vdev->host.slot, vdev->host.function);
2636     }
2637 
2638     if (stat(vdev->vbasedev.sysfsdev, &st) < 0) {
2639         error_setg_errno(errp, errno, "no such host device");
2640         error_prepend(errp, ERR_PREFIX, vdev->vbasedev.sysfsdev);
2641         return;
2642     }
2643 
2644     vdev->vbasedev.name = g_strdup(basename(vdev->vbasedev.sysfsdev));
2645     vdev->vbasedev.ops = &vfio_pci_ops;
2646     vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
2647 
2648     tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
2649     len = readlink(tmp, group_path, sizeof(group_path));
2650     g_free(tmp);
2651 
2652     if (len <= 0 || len >= sizeof(group_path)) {
2653         error_setg_errno(errp, len < 0 ? errno : ENAMETOOLONG,
2654                          "no iommu_group found");
2655         goto error;
2656     }
2657 
2658     group_path[len] = 0;
2659 
2660     group_name = basename(group_path);
2661     if (sscanf(group_name, "%d", &groupid) != 1) {
2662         error_setg_errno(errp, errno, "failed to read %s", group_path);
2663         goto error;
2664     }
2665 
2666     trace_vfio_realize(vdev->vbasedev.name, groupid);
2667 
2668     group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp);
2669     if (!group) {
2670         goto error;
2671     }
2672 
2673     QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2674         if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
2675             error_setg(errp, "device is already attached");
2676             vfio_put_group(group);
2677             goto error;
2678         }
2679     }
2680 
2681     ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev, errp);
2682     if (ret) {
2683         vfio_put_group(group);
2684         goto error;
2685     }
2686 
2687     vfio_populate_device(vdev, &err);
2688     if (err) {
2689         error_propagate(errp, err);
2690         goto error;
2691     }
2692 
2693     /* Get a copy of config space */
2694     ret = pread(vdev->vbasedev.fd, vdev->pdev.config,
2695                 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
2696                 vdev->config_offset);
2697     if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
2698         ret = ret < 0 ? -errno : -EFAULT;
2699         error_setg_errno(errp, -ret, "failed to read device config space");
2700         goto error;
2701     }
2702 
2703     /* vfio emulates a lot for us, but some bits need extra love */
2704     vdev->emulated_config_bits = g_malloc0(vdev->config_size);
2705 
2706     /* QEMU can choose to expose the ROM or not */
2707     memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
2708 
2709     /*
2710      * The PCI spec reserves vendor ID 0xffff as an invalid value.  The
2711      * device ID is managed by the vendor and need only be a 16-bit value.
2712      * Allow any 16-bit value for subsystem so they can be hidden or changed.
2713      */
2714     if (vdev->vendor_id != PCI_ANY_ID) {
2715         if (vdev->vendor_id >= 0xffff) {
2716             error_setg(errp, "invalid PCI vendor ID provided");
2717             goto error;
2718         }
2719         vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
2720         trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
2721     } else {
2722         vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
2723     }
2724 
2725     if (vdev->device_id != PCI_ANY_ID) {
2726         if (vdev->device_id > 0xffff) {
2727             error_setg(errp, "invalid PCI device ID provided");
2728             goto error;
2729         }
2730         vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
2731         trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
2732     } else {
2733         vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
2734     }
2735 
2736     if (vdev->sub_vendor_id != PCI_ANY_ID) {
2737         if (vdev->sub_vendor_id > 0xffff) {
2738             error_setg(errp, "invalid PCI subsystem vendor ID provided");
2739             goto error;
2740         }
2741         vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
2742                                vdev->sub_vendor_id, ~0);
2743         trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
2744                                               vdev->sub_vendor_id);
2745     }
2746 
2747     if (vdev->sub_device_id != PCI_ANY_ID) {
2748         if (vdev->sub_device_id > 0xffff) {
2749             error_setg(errp, "invalid PCI subsystem device ID provided");
2750             goto error;
2751         }
2752         vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
2753         trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
2754                                               vdev->sub_device_id);
2755     }
2756 
2757     /* QEMU can change multi-function devices to single function, or reverse */
2758     vdev->emulated_config_bits[PCI_HEADER_TYPE] =
2759                                               PCI_HEADER_TYPE_MULTI_FUNCTION;
2760 
2761     /* Restore or clear multifunction, this is always controlled by QEMU */
2762     if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
2763         vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
2764     } else {
2765         vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
2766     }
2767 
2768     /*
2769      * Clear host resource mapping info.  If we choose not to register a
2770      * BAR, such as might be the case with the option ROM, we can get
2771      * confusing, unwritable, residual addresses from the host here.
2772      */
2773     memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
2774     memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
2775 
2776     vfio_pci_size_rom(vdev);
2777 
2778     vfio_msix_early_setup(vdev, &err);
2779     if (err) {
2780         error_propagate(errp, err);
2781         goto error;
2782     }
2783 
2784     vfio_bars_setup(vdev);
2785 
2786     ret = vfio_add_capabilities(vdev, errp);
2787     if (ret) {
2788         goto out_teardown;
2789     }
2790 
2791     if (vdev->vga) {
2792         vfio_vga_quirk_setup(vdev);
2793     }
2794 
2795     for (i = 0; i < PCI_ROM_SLOT; i++) {
2796         vfio_bar_quirk_setup(vdev, i);
2797     }
2798 
2799     if (!vdev->igd_opregion &&
2800         vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
2801         struct vfio_region_info *opregion;
2802 
2803         if (vdev->pdev.qdev.hotplugged) {
2804             error_setg(errp,
2805                        "cannot support IGD OpRegion feature on hotplugged "
2806                        "device");
2807             goto out_teardown;
2808         }
2809 
2810         ret = vfio_get_dev_region_info(&vdev->vbasedev,
2811                         VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
2812                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
2813         if (ret) {
2814             error_setg_errno(errp, -ret,
2815                              "does not support requested IGD OpRegion feature");
2816             goto out_teardown;
2817         }
2818 
2819         ret = vfio_pci_igd_opregion_init(vdev, opregion, errp);
2820         g_free(opregion);
2821         if (ret) {
2822             goto out_teardown;
2823         }
2824     }
2825 
2826     /* QEMU emulates all of MSI & MSIX */
2827     if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
2828         memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
2829                MSIX_CAP_LENGTH);
2830     }
2831 
2832     if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
2833         memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
2834                vdev->msi_cap_size);
2835     }
2836 
2837     if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
2838         vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
2839                                                   vfio_intx_mmap_enable, vdev);
2840         pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_intx_update);
2841         ret = vfio_intx_enable(vdev, errp);
2842         if (ret) {
2843             goto out_teardown;
2844         }
2845     }
2846 
2847     vfio_register_err_notifier(vdev);
2848     vfio_register_req_notifier(vdev);
2849     vfio_setup_resetfn_quirk(vdev);
2850 
2851     return;
2852 
2853 out_teardown:
2854     pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
2855     vfio_teardown_msi(vdev);
2856     vfio_bars_exit(vdev);
2857 error:
2858     error_prepend(errp, ERR_PREFIX, vdev->vbasedev.name);
2859 }
2860 
2861 static void vfio_instance_finalize(Object *obj)
2862 {
2863     PCIDevice *pci_dev = PCI_DEVICE(obj);
2864     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pci_dev);
2865     VFIOGroup *group = vdev->vbasedev.group;
2866 
2867     vfio_bars_finalize(vdev);
2868     g_free(vdev->emulated_config_bits);
2869     g_free(vdev->rom);
2870     /*
2871      * XXX Leaking igd_opregion is not an oversight, we can't remove the
2872      * fw_cfg entry therefore leaking this allocation seems like the safest
2873      * option.
2874      *
2875      * g_free(vdev->igd_opregion);
2876      */
2877     vfio_put_device(vdev);
2878     vfio_put_group(group);
2879 }
2880 
2881 static void vfio_exitfn(PCIDevice *pdev)
2882 {
2883     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
2884 
2885     vfio_unregister_req_notifier(vdev);
2886     vfio_unregister_err_notifier(vdev);
2887     pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
2888     vfio_disable_interrupts(vdev);
2889     if (vdev->intx.mmap_timer) {
2890         timer_free(vdev->intx.mmap_timer);
2891     }
2892     vfio_teardown_msi(vdev);
2893     vfio_bars_exit(vdev);
2894 }
2895 
2896 static void vfio_pci_reset(DeviceState *dev)
2897 {
2898     PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
2899     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
2900 
2901     trace_vfio_pci_reset(vdev->vbasedev.name);
2902 
2903     vfio_pci_pre_reset(vdev);
2904 
2905     if (vdev->resetfn && !vdev->resetfn(vdev)) {
2906         goto post_reset;
2907     }
2908 
2909     if (vdev->vbasedev.reset_works &&
2910         (vdev->has_flr || !vdev->has_pm_reset) &&
2911         !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
2912         trace_vfio_pci_reset_flr(vdev->vbasedev.name);
2913         goto post_reset;
2914     }
2915 
2916     /* See if we can do our own bus reset */
2917     if (!vfio_pci_hot_reset_one(vdev)) {
2918         goto post_reset;
2919     }
2920 
2921     /* If nothing else works and the device supports PM reset, use it */
2922     if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
2923         !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
2924         trace_vfio_pci_reset_pm(vdev->vbasedev.name);
2925         goto post_reset;
2926     }
2927 
2928 post_reset:
2929     vfio_pci_post_reset(vdev);
2930 }
2931 
2932 static void vfio_instance_init(Object *obj)
2933 {
2934     PCIDevice *pci_dev = PCI_DEVICE(obj);
2935     VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, PCI_DEVICE(obj));
2936 
2937     device_add_bootindex_property(obj, &vdev->bootindex,
2938                                   "bootindex", NULL,
2939                                   &pci_dev->qdev, NULL);
2940     vdev->host.domain = ~0U;
2941     vdev->host.bus = ~0U;
2942     vdev->host.slot = ~0U;
2943     vdev->host.function = ~0U;
2944 }
2945 
2946 static Property vfio_pci_dev_properties[] = {
2947     DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
2948     DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
2949     DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
2950                        intx.mmap_timeout, 1100),
2951     DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
2952                     VFIO_FEATURE_ENABLE_VGA_BIT, false),
2953     DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
2954                     VFIO_FEATURE_ENABLE_REQ_BIT, true),
2955     DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
2956                     VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
2957     DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
2958     DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
2959     DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
2960     DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
2961     DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
2962     DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
2963     DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
2964                        sub_vendor_id, PCI_ANY_ID),
2965     DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
2966                        sub_device_id, PCI_ANY_ID),
2967     DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
2968     /*
2969      * TODO - support passed fds... is this necessary?
2970      * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
2971      * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
2972      */
2973     DEFINE_PROP_END_OF_LIST(),
2974 };
2975 
2976 static const VMStateDescription vfio_pci_vmstate = {
2977     .name = "vfio-pci",
2978     .unmigratable = 1,
2979 };
2980 
2981 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
2982 {
2983     DeviceClass *dc = DEVICE_CLASS(klass);
2984     PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
2985 
2986     dc->reset = vfio_pci_reset;
2987     dc->props = vfio_pci_dev_properties;
2988     dc->vmsd = &vfio_pci_vmstate;
2989     dc->desc = "VFIO-based PCI device assignment";
2990     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
2991     pdc->realize = vfio_realize;
2992     pdc->exit = vfio_exitfn;
2993     pdc->config_read = vfio_pci_read_config;
2994     pdc->config_write = vfio_pci_write_config;
2995     pdc->is_express = 1; /* We might be */
2996 }
2997 
2998 static const TypeInfo vfio_pci_dev_info = {
2999     .name = "vfio-pci",
3000     .parent = TYPE_PCI_DEVICE,
3001     .instance_size = sizeof(VFIOPCIDevice),
3002     .class_init = vfio_pci_dev_class_init,
3003     .instance_init = vfio_instance_init,
3004     .instance_finalize = vfio_instance_finalize,
3005 };
3006 
3007 static void register_vfio_pci_dev_type(void)
3008 {
3009     type_register_static(&vfio_pci_dev_info);
3010 }
3011 
3012 type_init(register_vfio_pci_dev_type)
3013