xref: /openbmc/qemu/hw/virtio/virtio.c (revision d5938f29fea29581725426f203a74da746ca03e7)
1 /*
2  * Virtio Support
3  *
4  * Copyright IBM, Corp. 2007
5  *
6  * Authors:
7  *  Anthony Liguori   <aliguori@us.ibm.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  */
13 
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "cpu.h"
17 #include "trace.h"
18 #include "exec/address-spaces.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "hw/virtio/virtio.h"
23 #include "migration/qemu-file-types.h"
24 #include "qemu/atomic.h"
25 #include "hw/virtio/virtio-bus.h"
26 #include "hw/qdev-properties.h"
27 #include "hw/virtio/virtio-access.h"
28 #include "sysemu/dma.h"
29 
30 /*
31  * The alignment to use between consumer and producer parts of vring.
32  * x86 pagesize again. This is the default, used by transports like PCI
33  * which don't provide a means for the guest to tell the host the alignment.
34  */
35 #define VIRTIO_PCI_VRING_ALIGN         4096
36 
37 typedef struct VRingDesc
38 {
39     uint64_t addr;
40     uint32_t len;
41     uint16_t flags;
42     uint16_t next;
43 } VRingDesc;
44 
45 typedef struct VRingAvail
46 {
47     uint16_t flags;
48     uint16_t idx;
49     uint16_t ring[0];
50 } VRingAvail;
51 
52 typedef struct VRingUsedElem
53 {
54     uint32_t id;
55     uint32_t len;
56 } VRingUsedElem;
57 
58 typedef struct VRingUsed
59 {
60     uint16_t flags;
61     uint16_t idx;
62     VRingUsedElem ring[0];
63 } VRingUsed;
64 
65 typedef struct VRingMemoryRegionCaches {
66     struct rcu_head rcu;
67     MemoryRegionCache desc;
68     MemoryRegionCache avail;
69     MemoryRegionCache used;
70 } VRingMemoryRegionCaches;
71 
72 typedef struct VRing
73 {
74     unsigned int num;
75     unsigned int num_default;
76     unsigned int align;
77     hwaddr desc;
78     hwaddr avail;
79     hwaddr used;
80     VRingMemoryRegionCaches *caches;
81 } VRing;
82 
83 struct VirtQueue
84 {
85     VRing vring;
86 
87     /* Next head to pop */
88     uint16_t last_avail_idx;
89 
90     /* Last avail_idx read from VQ. */
91     uint16_t shadow_avail_idx;
92 
93     uint16_t used_idx;
94 
95     /* Last used index value we have signalled on */
96     uint16_t signalled_used;
97 
98     /* Last used index value we have signalled on */
99     bool signalled_used_valid;
100 
101     /* Notification enabled? */
102     bool notification;
103 
104     uint16_t queue_index;
105 
106     unsigned int inuse;
107 
108     uint16_t vector;
109     VirtIOHandleOutput handle_output;
110     VirtIOHandleAIOOutput handle_aio_output;
111     VirtIODevice *vdev;
112     EventNotifier guest_notifier;
113     EventNotifier host_notifier;
114     QLIST_ENTRY(VirtQueue) node;
115 };
116 
117 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
118 {
119     if (!caches) {
120         return;
121     }
122 
123     address_space_cache_destroy(&caches->desc);
124     address_space_cache_destroy(&caches->avail);
125     address_space_cache_destroy(&caches->used);
126     g_free(caches);
127 }
128 
129 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
130 {
131     VRingMemoryRegionCaches *caches;
132 
133     caches = atomic_read(&vq->vring.caches);
134     atomic_rcu_set(&vq->vring.caches, NULL);
135     if (caches) {
136         call_rcu(caches, virtio_free_region_cache, rcu);
137     }
138 }
139 
140 static void virtio_init_region_cache(VirtIODevice *vdev, int n)
141 {
142     VirtQueue *vq = &vdev->vq[n];
143     VRingMemoryRegionCaches *old = vq->vring.caches;
144     VRingMemoryRegionCaches *new = NULL;
145     hwaddr addr, size;
146     int event_size;
147     int64_t len;
148 
149     event_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
150 
151     addr = vq->vring.desc;
152     if (!addr) {
153         goto out_no_cache;
154     }
155     new = g_new0(VRingMemoryRegionCaches, 1);
156     size = virtio_queue_get_desc_size(vdev, n);
157     len = address_space_cache_init(&new->desc, vdev->dma_as,
158                                    addr, size, false);
159     if (len < size) {
160         virtio_error(vdev, "Cannot map desc");
161         goto err_desc;
162     }
163 
164     size = virtio_queue_get_used_size(vdev, n) + event_size;
165     len = address_space_cache_init(&new->used, vdev->dma_as,
166                                    vq->vring.used, size, true);
167     if (len < size) {
168         virtio_error(vdev, "Cannot map used");
169         goto err_used;
170     }
171 
172     size = virtio_queue_get_avail_size(vdev, n) + event_size;
173     len = address_space_cache_init(&new->avail, vdev->dma_as,
174                                    vq->vring.avail, size, false);
175     if (len < size) {
176         virtio_error(vdev, "Cannot map avail");
177         goto err_avail;
178     }
179 
180     atomic_rcu_set(&vq->vring.caches, new);
181     if (old) {
182         call_rcu(old, virtio_free_region_cache, rcu);
183     }
184     return;
185 
186 err_avail:
187     address_space_cache_destroy(&new->avail);
188 err_used:
189     address_space_cache_destroy(&new->used);
190 err_desc:
191     address_space_cache_destroy(&new->desc);
192 out_no_cache:
193     g_free(new);
194     virtio_virtqueue_reset_region_cache(vq);
195 }
196 
197 /* virt queue functions */
198 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
199 {
200     VRing *vring = &vdev->vq[n].vring;
201 
202     if (!vring->num || !vring->desc || !vring->align) {
203         /* not yet setup -> nothing to do */
204         return;
205     }
206     vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
207     vring->used = vring_align(vring->avail +
208                               offsetof(VRingAvail, ring[vring->num]),
209                               vring->align);
210     virtio_init_region_cache(vdev, n);
211 }
212 
213 /* Called within rcu_read_lock().  */
214 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
215                             MemoryRegionCache *cache, int i)
216 {
217     address_space_read_cached(cache, i * sizeof(VRingDesc),
218                               desc, sizeof(VRingDesc));
219     virtio_tswap64s(vdev, &desc->addr);
220     virtio_tswap32s(vdev, &desc->len);
221     virtio_tswap16s(vdev, &desc->flags);
222     virtio_tswap16s(vdev, &desc->next);
223 }
224 
225 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
226 {
227     VRingMemoryRegionCaches *caches = atomic_rcu_read(&vq->vring.caches);
228     assert(caches != NULL);
229     return caches;
230 }
231 /* Called within rcu_read_lock().  */
232 static inline uint16_t vring_avail_flags(VirtQueue *vq)
233 {
234     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
235     hwaddr pa = offsetof(VRingAvail, flags);
236     return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
237 }
238 
239 /* Called within rcu_read_lock().  */
240 static inline uint16_t vring_avail_idx(VirtQueue *vq)
241 {
242     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
243     hwaddr pa = offsetof(VRingAvail, idx);
244     vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
245     return vq->shadow_avail_idx;
246 }
247 
248 /* Called within rcu_read_lock().  */
249 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
250 {
251     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
252     hwaddr pa = offsetof(VRingAvail, ring[i]);
253     return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
254 }
255 
256 /* Called within rcu_read_lock().  */
257 static inline uint16_t vring_get_used_event(VirtQueue *vq)
258 {
259     return vring_avail_ring(vq, vq->vring.num);
260 }
261 
262 /* Called within rcu_read_lock().  */
263 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
264                                     int i)
265 {
266     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
267     hwaddr pa = offsetof(VRingUsed, ring[i]);
268     virtio_tswap32s(vq->vdev, &uelem->id);
269     virtio_tswap32s(vq->vdev, &uelem->len);
270     address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
271     address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
272 }
273 
274 /* Called within rcu_read_lock().  */
275 static uint16_t vring_used_idx(VirtQueue *vq)
276 {
277     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
278     hwaddr pa = offsetof(VRingUsed, idx);
279     return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
280 }
281 
282 /* Called within rcu_read_lock().  */
283 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
284 {
285     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
286     hwaddr pa = offsetof(VRingUsed, idx);
287     virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
288     address_space_cache_invalidate(&caches->used, pa, sizeof(val));
289     vq->used_idx = val;
290 }
291 
292 /* Called within rcu_read_lock().  */
293 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
294 {
295     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
296     VirtIODevice *vdev = vq->vdev;
297     hwaddr pa = offsetof(VRingUsed, flags);
298     uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
299 
300     virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask);
301     address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
302 }
303 
304 /* Called within rcu_read_lock().  */
305 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
306 {
307     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
308     VirtIODevice *vdev = vq->vdev;
309     hwaddr pa = offsetof(VRingUsed, flags);
310     uint16_t flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
311 
312     virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
313     address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
314 }
315 
316 /* Called within rcu_read_lock().  */
317 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
318 {
319     VRingMemoryRegionCaches *caches;
320     hwaddr pa;
321     if (!vq->notification) {
322         return;
323     }
324 
325     caches = vring_get_region_caches(vq);
326     pa = offsetof(VRingUsed, ring[vq->vring.num]);
327     virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
328     address_space_cache_invalidate(&caches->used, pa, sizeof(val));
329 }
330 
331 void virtio_queue_set_notification(VirtQueue *vq, int enable)
332 {
333     vq->notification = enable;
334 
335     if (!vq->vring.desc) {
336         return;
337     }
338 
339     rcu_read_lock();
340     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
341         vring_set_avail_event(vq, vring_avail_idx(vq));
342     } else if (enable) {
343         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
344     } else {
345         vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
346     }
347     if (enable) {
348         /* Expose avail event/used flags before caller checks the avail idx. */
349         smp_mb();
350     }
351     rcu_read_unlock();
352 }
353 
354 int virtio_queue_ready(VirtQueue *vq)
355 {
356     return vq->vring.avail != 0;
357 }
358 
359 /* Fetch avail_idx from VQ memory only when we really need to know if
360  * guest has added some buffers.
361  * Called within rcu_read_lock().  */
362 static int virtio_queue_empty_rcu(VirtQueue *vq)
363 {
364     if (unlikely(vq->vdev->broken)) {
365         return 1;
366     }
367 
368     if (unlikely(!vq->vring.avail)) {
369         return 1;
370     }
371 
372     if (vq->shadow_avail_idx != vq->last_avail_idx) {
373         return 0;
374     }
375 
376     return vring_avail_idx(vq) == vq->last_avail_idx;
377 }
378 
379 int virtio_queue_empty(VirtQueue *vq)
380 {
381     bool empty;
382 
383     if (unlikely(vq->vdev->broken)) {
384         return 1;
385     }
386 
387     if (unlikely(!vq->vring.avail)) {
388         return 1;
389     }
390 
391     if (vq->shadow_avail_idx != vq->last_avail_idx) {
392         return 0;
393     }
394 
395     rcu_read_lock();
396     empty = vring_avail_idx(vq) == vq->last_avail_idx;
397     rcu_read_unlock();
398     return empty;
399 }
400 
401 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
402                                unsigned int len)
403 {
404     AddressSpace *dma_as = vq->vdev->dma_as;
405     unsigned int offset;
406     int i;
407 
408     offset = 0;
409     for (i = 0; i < elem->in_num; i++) {
410         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
411 
412         dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
413                          elem->in_sg[i].iov_len,
414                          DMA_DIRECTION_FROM_DEVICE, size);
415 
416         offset += size;
417     }
418 
419     for (i = 0; i < elem->out_num; i++)
420         dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
421                          elem->out_sg[i].iov_len,
422                          DMA_DIRECTION_TO_DEVICE,
423                          elem->out_sg[i].iov_len);
424 }
425 
426 /* virtqueue_detach_element:
427  * @vq: The #VirtQueue
428  * @elem: The #VirtQueueElement
429  * @len: number of bytes written
430  *
431  * Detach the element from the virtqueue.  This function is suitable for device
432  * reset or other situations where a #VirtQueueElement is simply freed and will
433  * not be pushed or discarded.
434  */
435 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,
436                               unsigned int len)
437 {
438     vq->inuse--;
439     virtqueue_unmap_sg(vq, elem, len);
440 }
441 
442 /* virtqueue_unpop:
443  * @vq: The #VirtQueue
444  * @elem: The #VirtQueueElement
445  * @len: number of bytes written
446  *
447  * Pretend the most recent element wasn't popped from the virtqueue.  The next
448  * call to virtqueue_pop() will refetch the element.
449  */
450 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,
451                      unsigned int len)
452 {
453     vq->last_avail_idx--;
454     virtqueue_detach_element(vq, elem, len);
455 }
456 
457 /* virtqueue_rewind:
458  * @vq: The #VirtQueue
459  * @num: Number of elements to push back
460  *
461  * Pretend that elements weren't popped from the virtqueue.  The next
462  * virtqueue_pop() will refetch the oldest element.
463  *
464  * Use virtqueue_unpop() instead if you have a VirtQueueElement.
465  *
466  * Returns: true on success, false if @num is greater than the number of in use
467  * elements.
468  */
469 bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
470 {
471     if (num > vq->inuse) {
472         return false;
473     }
474     vq->last_avail_idx -= num;
475     vq->inuse -= num;
476     return true;
477 }
478 
479 /* Called within rcu_read_lock().  */
480 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
481                     unsigned int len, unsigned int idx)
482 {
483     VRingUsedElem uelem;
484 
485     trace_virtqueue_fill(vq, elem, len, idx);
486 
487     virtqueue_unmap_sg(vq, elem, len);
488 
489     if (unlikely(vq->vdev->broken)) {
490         return;
491     }
492 
493     if (unlikely(!vq->vring.used)) {
494         return;
495     }
496 
497     idx = (idx + vq->used_idx) % vq->vring.num;
498 
499     uelem.id = elem->index;
500     uelem.len = len;
501     vring_used_write(vq, &uelem, idx);
502 }
503 
504 /* Called within rcu_read_lock().  */
505 void virtqueue_flush(VirtQueue *vq, unsigned int count)
506 {
507     uint16_t old, new;
508 
509     if (unlikely(vq->vdev->broken)) {
510         vq->inuse -= count;
511         return;
512     }
513 
514     if (unlikely(!vq->vring.used)) {
515         return;
516     }
517 
518     /* Make sure buffer is written before we update index. */
519     smp_wmb();
520     trace_virtqueue_flush(vq, count);
521     old = vq->used_idx;
522     new = old + count;
523     vring_used_idx_set(vq, new);
524     vq->inuse -= count;
525     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
526         vq->signalled_used_valid = false;
527 }
528 
529 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
530                     unsigned int len)
531 {
532     rcu_read_lock();
533     virtqueue_fill(vq, elem, len, 0);
534     virtqueue_flush(vq, 1);
535     rcu_read_unlock();
536 }
537 
538 /* Called within rcu_read_lock().  */
539 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
540 {
541     uint16_t num_heads = vring_avail_idx(vq) - idx;
542 
543     /* Check it isn't doing very strange things with descriptor numbers. */
544     if (num_heads > vq->vring.num) {
545         virtio_error(vq->vdev, "Guest moved used index from %u to %u",
546                      idx, vq->shadow_avail_idx);
547         return -EINVAL;
548     }
549     /* On success, callers read a descriptor at vq->last_avail_idx.
550      * Make sure descriptor read does not bypass avail index read. */
551     if (num_heads) {
552         smp_rmb();
553     }
554 
555     return num_heads;
556 }
557 
558 /* Called within rcu_read_lock().  */
559 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
560                                unsigned int *head)
561 {
562     /* Grab the next descriptor number they're advertising, and increment
563      * the index we've seen. */
564     *head = vring_avail_ring(vq, idx % vq->vring.num);
565 
566     /* If their number is silly, that's a fatal mistake. */
567     if (*head >= vq->vring.num) {
568         virtio_error(vq->vdev, "Guest says index %u is available", *head);
569         return false;
570     }
571 
572     return true;
573 }
574 
575 enum {
576     VIRTQUEUE_READ_DESC_ERROR = -1,
577     VIRTQUEUE_READ_DESC_DONE = 0,   /* end of chain */
578     VIRTQUEUE_READ_DESC_MORE = 1,   /* more buffers in chain */
579 };
580 
581 static int virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
582                                     MemoryRegionCache *desc_cache, unsigned int max,
583                                     unsigned int *next)
584 {
585     /* If this descriptor says it doesn't chain, we're done. */
586     if (!(desc->flags & VRING_DESC_F_NEXT)) {
587         return VIRTQUEUE_READ_DESC_DONE;
588     }
589 
590     /* Check they're not leading us off end of descriptors. */
591     *next = desc->next;
592     /* Make sure compiler knows to grab that: we don't want it changing! */
593     smp_wmb();
594 
595     if (*next >= max) {
596         virtio_error(vdev, "Desc next is %u", *next);
597         return VIRTQUEUE_READ_DESC_ERROR;
598     }
599 
600     vring_desc_read(vdev, desc, desc_cache, *next);
601     return VIRTQUEUE_READ_DESC_MORE;
602 }
603 
604 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
605                                unsigned int *out_bytes,
606                                unsigned max_in_bytes, unsigned max_out_bytes)
607 {
608     VirtIODevice *vdev = vq->vdev;
609     unsigned int max, idx;
610     unsigned int total_bufs, in_total, out_total;
611     VRingMemoryRegionCaches *caches;
612     MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
613     int64_t len = 0;
614     int rc;
615 
616     if (unlikely(!vq->vring.desc)) {
617         if (in_bytes) {
618             *in_bytes = 0;
619         }
620         if (out_bytes) {
621             *out_bytes = 0;
622         }
623         return;
624     }
625 
626     rcu_read_lock();
627     idx = vq->last_avail_idx;
628     total_bufs = in_total = out_total = 0;
629 
630     max = vq->vring.num;
631     caches = vring_get_region_caches(vq);
632     if (caches->desc.len < max * sizeof(VRingDesc)) {
633         virtio_error(vdev, "Cannot map descriptor ring");
634         goto err;
635     }
636 
637     while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
638         MemoryRegionCache *desc_cache = &caches->desc;
639         unsigned int num_bufs;
640         VRingDesc desc;
641         unsigned int i;
642 
643         num_bufs = total_bufs;
644 
645         if (!virtqueue_get_head(vq, idx++, &i)) {
646             goto err;
647         }
648 
649         vring_desc_read(vdev, &desc, desc_cache, i);
650 
651         if (desc.flags & VRING_DESC_F_INDIRECT) {
652             if (!desc.len || (desc.len % sizeof(VRingDesc))) {
653                 virtio_error(vdev, "Invalid size for indirect buffer table");
654                 goto err;
655             }
656 
657             /* If we've got too many, that implies a descriptor loop. */
658             if (num_bufs >= max) {
659                 virtio_error(vdev, "Looped descriptor");
660                 goto err;
661             }
662 
663             /* loop over the indirect descriptor table */
664             len = address_space_cache_init(&indirect_desc_cache,
665                                            vdev->dma_as,
666                                            desc.addr, desc.len, false);
667             desc_cache = &indirect_desc_cache;
668             if (len < desc.len) {
669                 virtio_error(vdev, "Cannot map indirect buffer");
670                 goto err;
671             }
672 
673             max = desc.len / sizeof(VRingDesc);
674             num_bufs = i = 0;
675             vring_desc_read(vdev, &desc, desc_cache, i);
676         }
677 
678         do {
679             /* If we've got too many, that implies a descriptor loop. */
680             if (++num_bufs > max) {
681                 virtio_error(vdev, "Looped descriptor");
682                 goto err;
683             }
684 
685             if (desc.flags & VRING_DESC_F_WRITE) {
686                 in_total += desc.len;
687             } else {
688                 out_total += desc.len;
689             }
690             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
691                 goto done;
692             }
693 
694             rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i);
695         } while (rc == VIRTQUEUE_READ_DESC_MORE);
696 
697         if (rc == VIRTQUEUE_READ_DESC_ERROR) {
698             goto err;
699         }
700 
701         if (desc_cache == &indirect_desc_cache) {
702             address_space_cache_destroy(&indirect_desc_cache);
703             total_bufs++;
704         } else {
705             total_bufs = num_bufs;
706         }
707     }
708 
709     if (rc < 0) {
710         goto err;
711     }
712 
713 done:
714     address_space_cache_destroy(&indirect_desc_cache);
715     if (in_bytes) {
716         *in_bytes = in_total;
717     }
718     if (out_bytes) {
719         *out_bytes = out_total;
720     }
721     rcu_read_unlock();
722     return;
723 
724 err:
725     in_total = out_total = 0;
726     goto done;
727 }
728 
729 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
730                           unsigned int out_bytes)
731 {
732     unsigned int in_total, out_total;
733 
734     virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
735     return in_bytes <= in_total && out_bytes <= out_total;
736 }
737 
738 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
739                                hwaddr *addr, struct iovec *iov,
740                                unsigned int max_num_sg, bool is_write,
741                                hwaddr pa, size_t sz)
742 {
743     bool ok = false;
744     unsigned num_sg = *p_num_sg;
745     assert(num_sg <= max_num_sg);
746 
747     if (!sz) {
748         virtio_error(vdev, "virtio: zero sized buffers are not allowed");
749         goto out;
750     }
751 
752     while (sz) {
753         hwaddr len = sz;
754 
755         if (num_sg == max_num_sg) {
756             virtio_error(vdev, "virtio: too many write descriptors in "
757                                "indirect table");
758             goto out;
759         }
760 
761         iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
762                                               is_write ?
763                                               DMA_DIRECTION_FROM_DEVICE :
764                                               DMA_DIRECTION_TO_DEVICE);
765         if (!iov[num_sg].iov_base) {
766             virtio_error(vdev, "virtio: bogus descriptor or out of resources");
767             goto out;
768         }
769 
770         iov[num_sg].iov_len = len;
771         addr[num_sg] = pa;
772 
773         sz -= len;
774         pa += len;
775         num_sg++;
776     }
777     ok = true;
778 
779 out:
780     *p_num_sg = num_sg;
781     return ok;
782 }
783 
784 /* Only used by error code paths before we have a VirtQueueElement (therefore
785  * virtqueue_unmap_sg() can't be used).  Assumes buffers weren't written to
786  * yet.
787  */
788 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num,
789                                     struct iovec *iov)
790 {
791     unsigned int i;
792 
793     for (i = 0; i < out_num + in_num; i++) {
794         int is_write = i >= out_num;
795 
796         cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0);
797         iov++;
798     }
799 }
800 
801 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
802                                 hwaddr *addr, unsigned int num_sg,
803                                 int is_write)
804 {
805     unsigned int i;
806     hwaddr len;
807 
808     for (i = 0; i < num_sg; i++) {
809         len = sg[i].iov_len;
810         sg[i].iov_base = dma_memory_map(vdev->dma_as,
811                                         addr[i], &len, is_write ?
812                                         DMA_DIRECTION_FROM_DEVICE :
813                                         DMA_DIRECTION_TO_DEVICE);
814         if (!sg[i].iov_base) {
815             error_report("virtio: error trying to map MMIO memory");
816             exit(1);
817         }
818         if (len != sg[i].iov_len) {
819             error_report("virtio: unexpected memory split");
820             exit(1);
821         }
822     }
823 }
824 
825 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem)
826 {
827     virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, 1);
828     virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num, 0);
829 }
830 
831 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
832 {
833     VirtQueueElement *elem;
834     size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
835     size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
836     size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
837     size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
838     size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
839     size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
840 
841     assert(sz >= sizeof(VirtQueueElement));
842     elem = g_malloc(out_sg_end);
843     trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
844     elem->out_num = out_num;
845     elem->in_num = in_num;
846     elem->in_addr = (void *)elem + in_addr_ofs;
847     elem->out_addr = (void *)elem + out_addr_ofs;
848     elem->in_sg = (void *)elem + in_sg_ofs;
849     elem->out_sg = (void *)elem + out_sg_ofs;
850     return elem;
851 }
852 
853 void *virtqueue_pop(VirtQueue *vq, size_t sz)
854 {
855     unsigned int i, head, max;
856     VRingMemoryRegionCaches *caches;
857     MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
858     MemoryRegionCache *desc_cache;
859     int64_t len;
860     VirtIODevice *vdev = vq->vdev;
861     VirtQueueElement *elem = NULL;
862     unsigned out_num, in_num, elem_entries;
863     hwaddr addr[VIRTQUEUE_MAX_SIZE];
864     struct iovec iov[VIRTQUEUE_MAX_SIZE];
865     VRingDesc desc;
866     int rc;
867 
868     if (unlikely(vdev->broken)) {
869         return NULL;
870     }
871     rcu_read_lock();
872     if (virtio_queue_empty_rcu(vq)) {
873         goto done;
874     }
875     /* Needed after virtio_queue_empty(), see comment in
876      * virtqueue_num_heads(). */
877     smp_rmb();
878 
879     /* When we start there are none of either input nor output. */
880     out_num = in_num = elem_entries = 0;
881 
882     max = vq->vring.num;
883 
884     if (vq->inuse >= vq->vring.num) {
885         virtio_error(vdev, "Virtqueue size exceeded");
886         goto done;
887     }
888 
889     if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
890         goto done;
891     }
892 
893     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
894         vring_set_avail_event(vq, vq->last_avail_idx);
895     }
896 
897     i = head;
898 
899     caches = vring_get_region_caches(vq);
900     if (caches->desc.len < max * sizeof(VRingDesc)) {
901         virtio_error(vdev, "Cannot map descriptor ring");
902         goto done;
903     }
904 
905     desc_cache = &caches->desc;
906     vring_desc_read(vdev, &desc, desc_cache, i);
907     if (desc.flags & VRING_DESC_F_INDIRECT) {
908         if (!desc.len || (desc.len % sizeof(VRingDesc))) {
909             virtio_error(vdev, "Invalid size for indirect buffer table");
910             goto done;
911         }
912 
913         /* loop over the indirect descriptor table */
914         len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
915                                        desc.addr, desc.len, false);
916         desc_cache = &indirect_desc_cache;
917         if (len < desc.len) {
918             virtio_error(vdev, "Cannot map indirect buffer");
919             goto done;
920         }
921 
922         max = desc.len / sizeof(VRingDesc);
923         i = 0;
924         vring_desc_read(vdev, &desc, desc_cache, i);
925     }
926 
927     /* Collect all the descriptors */
928     do {
929         bool map_ok;
930 
931         if (desc.flags & VRING_DESC_F_WRITE) {
932             map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
933                                         iov + out_num,
934                                         VIRTQUEUE_MAX_SIZE - out_num, true,
935                                         desc.addr, desc.len);
936         } else {
937             if (in_num) {
938                 virtio_error(vdev, "Incorrect order for descriptors");
939                 goto err_undo_map;
940             }
941             map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
942                                         VIRTQUEUE_MAX_SIZE, false,
943                                         desc.addr, desc.len);
944         }
945         if (!map_ok) {
946             goto err_undo_map;
947         }
948 
949         /* If we've got too many, that implies a descriptor loop. */
950         if (++elem_entries > max) {
951             virtio_error(vdev, "Looped descriptor");
952             goto err_undo_map;
953         }
954 
955         rc = virtqueue_read_next_desc(vdev, &desc, desc_cache, max, &i);
956     } while (rc == VIRTQUEUE_READ_DESC_MORE);
957 
958     if (rc == VIRTQUEUE_READ_DESC_ERROR) {
959         goto err_undo_map;
960     }
961 
962     /* Now copy what we have collected and mapped */
963     elem = virtqueue_alloc_element(sz, out_num, in_num);
964     elem->index = head;
965     for (i = 0; i < out_num; i++) {
966         elem->out_addr[i] = addr[i];
967         elem->out_sg[i] = iov[i];
968     }
969     for (i = 0; i < in_num; i++) {
970         elem->in_addr[i] = addr[out_num + i];
971         elem->in_sg[i] = iov[out_num + i];
972     }
973 
974     vq->inuse++;
975 
976     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
977 done:
978     address_space_cache_destroy(&indirect_desc_cache);
979     rcu_read_unlock();
980 
981     return elem;
982 
983 err_undo_map:
984     virtqueue_undo_map_desc(out_num, in_num, iov);
985     goto done;
986 }
987 
988 /* virtqueue_drop_all:
989  * @vq: The #VirtQueue
990  * Drops all queued buffers and indicates them to the guest
991  * as if they are done. Useful when buffers can not be
992  * processed but must be returned to the guest.
993  */
994 unsigned int virtqueue_drop_all(VirtQueue *vq)
995 {
996     unsigned int dropped = 0;
997     VirtQueueElement elem = {};
998     VirtIODevice *vdev = vq->vdev;
999     bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1000 
1001     if (unlikely(vdev->broken)) {
1002         return 0;
1003     }
1004 
1005     while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
1006         /* works similar to virtqueue_pop but does not map buffers
1007         * and does not allocate any memory */
1008         smp_rmb();
1009         if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
1010             break;
1011         }
1012         vq->inuse++;
1013         vq->last_avail_idx++;
1014         if (fEventIdx) {
1015             vring_set_avail_event(vq, vq->last_avail_idx);
1016         }
1017         /* immediately push the element, nothing to unmap
1018          * as both in_num and out_num are set to 0 */
1019         virtqueue_push(vq, &elem, 0);
1020         dropped++;
1021     }
1022 
1023     return dropped;
1024 }
1025 
1026 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1027  * it is what QEMU has always done by mistake.  We can change it sooner
1028  * or later by bumping the version number of the affected vm states.
1029  * In the meanwhile, since the in-memory layout of VirtQueueElement
1030  * has changed, we need to marshal to and from the layout that was
1031  * used before the change.
1032  */
1033 typedef struct VirtQueueElementOld {
1034     unsigned int index;
1035     unsigned int out_num;
1036     unsigned int in_num;
1037     hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
1038     hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
1039     struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
1040     struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
1041 } VirtQueueElementOld;
1042 
1043 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz)
1044 {
1045     VirtQueueElement *elem;
1046     VirtQueueElementOld data;
1047     int i;
1048 
1049     qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1050 
1051     /* TODO: teach all callers that this can fail, and return failure instead
1052      * of asserting here.
1053      * This is just one thing (there are probably more) that must be
1054      * fixed before we can allow NDEBUG compilation.
1055      */
1056     assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
1057     assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
1058 
1059     elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
1060     elem->index = data.index;
1061 
1062     for (i = 0; i < elem->in_num; i++) {
1063         elem->in_addr[i] = data.in_addr[i];
1064     }
1065 
1066     for (i = 0; i < elem->out_num; i++) {
1067         elem->out_addr[i] = data.out_addr[i];
1068     }
1069 
1070     for (i = 0; i < elem->in_num; i++) {
1071         /* Base is overwritten by virtqueue_map.  */
1072         elem->in_sg[i].iov_base = 0;
1073         elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
1074     }
1075 
1076     for (i = 0; i < elem->out_num; i++) {
1077         /* Base is overwritten by virtqueue_map.  */
1078         elem->out_sg[i].iov_base = 0;
1079         elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
1080     }
1081 
1082     virtqueue_map(vdev, elem);
1083     return elem;
1084 }
1085 
1086 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
1087 {
1088     VirtQueueElementOld data;
1089     int i;
1090 
1091     memset(&data, 0, sizeof(data));
1092     data.index = elem->index;
1093     data.in_num = elem->in_num;
1094     data.out_num = elem->out_num;
1095 
1096     for (i = 0; i < elem->in_num; i++) {
1097         data.in_addr[i] = elem->in_addr[i];
1098     }
1099 
1100     for (i = 0; i < elem->out_num; i++) {
1101         data.out_addr[i] = elem->out_addr[i];
1102     }
1103 
1104     for (i = 0; i < elem->in_num; i++) {
1105         /* Base is overwritten by virtqueue_map when loading.  Do not
1106          * save it, as it would leak the QEMU address space layout.  */
1107         data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
1108     }
1109 
1110     for (i = 0; i < elem->out_num; i++) {
1111         /* Do not save iov_base as above.  */
1112         data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
1113     }
1114     qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1115 }
1116 
1117 /* virtio device */
1118 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
1119 {
1120     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1121     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1122 
1123     if (unlikely(vdev->broken)) {
1124         return;
1125     }
1126 
1127     if (k->notify) {
1128         k->notify(qbus->parent, vector);
1129     }
1130 }
1131 
1132 void virtio_update_irq(VirtIODevice *vdev)
1133 {
1134     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1135 }
1136 
1137 static int virtio_validate_features(VirtIODevice *vdev)
1138 {
1139     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1140 
1141     if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
1142         !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1143         return -EFAULT;
1144     }
1145 
1146     if (k->validate_features) {
1147         return k->validate_features(vdev);
1148     } else {
1149         return 0;
1150     }
1151 }
1152 
1153 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
1154 {
1155     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1156     trace_virtio_set_status(vdev, val);
1157 
1158     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1159         if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
1160             val & VIRTIO_CONFIG_S_FEATURES_OK) {
1161             int ret = virtio_validate_features(vdev);
1162 
1163             if (ret) {
1164                 return ret;
1165             }
1166         }
1167     }
1168 
1169     if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) !=
1170         (val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1171         virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK);
1172     }
1173 
1174     if (k->set_status) {
1175         k->set_status(vdev, val);
1176     }
1177     vdev->status = val;
1178 
1179     return 0;
1180 }
1181 
1182 static enum virtio_device_endian virtio_default_endian(void)
1183 {
1184     if (target_words_bigendian()) {
1185         return VIRTIO_DEVICE_ENDIAN_BIG;
1186     } else {
1187         return VIRTIO_DEVICE_ENDIAN_LITTLE;
1188     }
1189 }
1190 
1191 static enum virtio_device_endian virtio_current_cpu_endian(void)
1192 {
1193     CPUClass *cc = CPU_GET_CLASS(current_cpu);
1194 
1195     if (cc->virtio_is_big_endian(current_cpu)) {
1196         return VIRTIO_DEVICE_ENDIAN_BIG;
1197     } else {
1198         return VIRTIO_DEVICE_ENDIAN_LITTLE;
1199     }
1200 }
1201 
1202 void virtio_reset(void *opaque)
1203 {
1204     VirtIODevice *vdev = opaque;
1205     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1206     int i;
1207 
1208     virtio_set_status(vdev, 0);
1209     if (current_cpu) {
1210         /* Guest initiated reset */
1211         vdev->device_endian = virtio_current_cpu_endian();
1212     } else {
1213         /* System reset */
1214         vdev->device_endian = virtio_default_endian();
1215     }
1216 
1217     if (k->reset) {
1218         k->reset(vdev);
1219     }
1220 
1221     vdev->start_on_kick = false;
1222     vdev->started = false;
1223     vdev->broken = false;
1224     vdev->guest_features = 0;
1225     vdev->queue_sel = 0;
1226     vdev->status = 0;
1227     atomic_set(&vdev->isr, 0);
1228     vdev->config_vector = VIRTIO_NO_VECTOR;
1229     virtio_notify_vector(vdev, vdev->config_vector);
1230 
1231     for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1232         vdev->vq[i].vring.desc = 0;
1233         vdev->vq[i].vring.avail = 0;
1234         vdev->vq[i].vring.used = 0;
1235         vdev->vq[i].last_avail_idx = 0;
1236         vdev->vq[i].shadow_avail_idx = 0;
1237         vdev->vq[i].used_idx = 0;
1238         virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
1239         vdev->vq[i].signalled_used = 0;
1240         vdev->vq[i].signalled_used_valid = false;
1241         vdev->vq[i].notification = true;
1242         vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
1243         vdev->vq[i].inuse = 0;
1244         virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
1245     }
1246 }
1247 
1248 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
1249 {
1250     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1251     uint8_t val;
1252 
1253     if (addr + sizeof(val) > vdev->config_len) {
1254         return (uint32_t)-1;
1255     }
1256 
1257     k->get_config(vdev, vdev->config);
1258 
1259     val = ldub_p(vdev->config + addr);
1260     return val;
1261 }
1262 
1263 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
1264 {
1265     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1266     uint16_t val;
1267 
1268     if (addr + sizeof(val) > vdev->config_len) {
1269         return (uint32_t)-1;
1270     }
1271 
1272     k->get_config(vdev, vdev->config);
1273 
1274     val = lduw_p(vdev->config + addr);
1275     return val;
1276 }
1277 
1278 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
1279 {
1280     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1281     uint32_t val;
1282 
1283     if (addr + sizeof(val) > vdev->config_len) {
1284         return (uint32_t)-1;
1285     }
1286 
1287     k->get_config(vdev, vdev->config);
1288 
1289     val = ldl_p(vdev->config + addr);
1290     return val;
1291 }
1292 
1293 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1294 {
1295     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1296     uint8_t val = data;
1297 
1298     if (addr + sizeof(val) > vdev->config_len) {
1299         return;
1300     }
1301 
1302     stb_p(vdev->config + addr, val);
1303 
1304     if (k->set_config) {
1305         k->set_config(vdev, vdev->config);
1306     }
1307 }
1308 
1309 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1310 {
1311     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1312     uint16_t val = data;
1313 
1314     if (addr + sizeof(val) > vdev->config_len) {
1315         return;
1316     }
1317 
1318     stw_p(vdev->config + addr, val);
1319 
1320     if (k->set_config) {
1321         k->set_config(vdev, vdev->config);
1322     }
1323 }
1324 
1325 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
1326 {
1327     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1328     uint32_t val = data;
1329 
1330     if (addr + sizeof(val) > vdev->config_len) {
1331         return;
1332     }
1333 
1334     stl_p(vdev->config + addr, val);
1335 
1336     if (k->set_config) {
1337         k->set_config(vdev, vdev->config);
1338     }
1339 }
1340 
1341 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
1342 {
1343     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1344     uint8_t val;
1345 
1346     if (addr + sizeof(val) > vdev->config_len) {
1347         return (uint32_t)-1;
1348     }
1349 
1350     k->get_config(vdev, vdev->config);
1351 
1352     val = ldub_p(vdev->config + addr);
1353     return val;
1354 }
1355 
1356 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
1357 {
1358     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1359     uint16_t val;
1360 
1361     if (addr + sizeof(val) > vdev->config_len) {
1362         return (uint32_t)-1;
1363     }
1364 
1365     k->get_config(vdev, vdev->config);
1366 
1367     val = lduw_le_p(vdev->config + addr);
1368     return val;
1369 }
1370 
1371 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
1372 {
1373     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1374     uint32_t val;
1375 
1376     if (addr + sizeof(val) > vdev->config_len) {
1377         return (uint32_t)-1;
1378     }
1379 
1380     k->get_config(vdev, vdev->config);
1381 
1382     val = ldl_le_p(vdev->config + addr);
1383     return val;
1384 }
1385 
1386 void virtio_config_modern_writeb(VirtIODevice *vdev,
1387                                  uint32_t addr, uint32_t data)
1388 {
1389     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1390     uint8_t val = data;
1391 
1392     if (addr + sizeof(val) > vdev->config_len) {
1393         return;
1394     }
1395 
1396     stb_p(vdev->config + addr, val);
1397 
1398     if (k->set_config) {
1399         k->set_config(vdev, vdev->config);
1400     }
1401 }
1402 
1403 void virtio_config_modern_writew(VirtIODevice *vdev,
1404                                  uint32_t addr, uint32_t data)
1405 {
1406     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1407     uint16_t val = data;
1408 
1409     if (addr + sizeof(val) > vdev->config_len) {
1410         return;
1411     }
1412 
1413     stw_le_p(vdev->config + addr, val);
1414 
1415     if (k->set_config) {
1416         k->set_config(vdev, vdev->config);
1417     }
1418 }
1419 
1420 void virtio_config_modern_writel(VirtIODevice *vdev,
1421                                  uint32_t addr, uint32_t data)
1422 {
1423     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1424     uint32_t val = data;
1425 
1426     if (addr + sizeof(val) > vdev->config_len) {
1427         return;
1428     }
1429 
1430     stl_le_p(vdev->config + addr, val);
1431 
1432     if (k->set_config) {
1433         k->set_config(vdev, vdev->config);
1434     }
1435 }
1436 
1437 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
1438 {
1439     if (!vdev->vq[n].vring.num) {
1440         return;
1441     }
1442     vdev->vq[n].vring.desc = addr;
1443     virtio_queue_update_rings(vdev, n);
1444 }
1445 
1446 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
1447 {
1448     return vdev->vq[n].vring.desc;
1449 }
1450 
1451 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
1452                             hwaddr avail, hwaddr used)
1453 {
1454     if (!vdev->vq[n].vring.num) {
1455         return;
1456     }
1457     vdev->vq[n].vring.desc = desc;
1458     vdev->vq[n].vring.avail = avail;
1459     vdev->vq[n].vring.used = used;
1460     virtio_init_region_cache(vdev, n);
1461 }
1462 
1463 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1464 {
1465     /* Don't allow guest to flip queue between existent and
1466      * nonexistent states, or to set it to an invalid size.
1467      */
1468     if (!!num != !!vdev->vq[n].vring.num ||
1469         num > VIRTQUEUE_MAX_SIZE ||
1470         num < 0) {
1471         return;
1472     }
1473     vdev->vq[n].vring.num = num;
1474 }
1475 
1476 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1477 {
1478     return QLIST_FIRST(&vdev->vector_queues[vector]);
1479 }
1480 
1481 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1482 {
1483     return QLIST_NEXT(vq, node);
1484 }
1485 
1486 int virtio_queue_get_num(VirtIODevice *vdev, int n)
1487 {
1488     return vdev->vq[n].vring.num;
1489 }
1490 
1491 int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
1492 {
1493     return vdev->vq[n].vring.num_default;
1494 }
1495 
1496 int virtio_get_num_queues(VirtIODevice *vdev)
1497 {
1498     int i;
1499 
1500     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1501         if (!virtio_queue_get_num(vdev, i)) {
1502             break;
1503         }
1504     }
1505 
1506     return i;
1507 }
1508 
1509 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1510 {
1511     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1512     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1513 
1514     /* virtio-1 compliant devices cannot change the alignment */
1515     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1516         error_report("tried to modify queue alignment for virtio-1 device");
1517         return;
1518     }
1519     /* Check that the transport told us it was going to do this
1520      * (so a buggy transport will immediately assert rather than
1521      * silently failing to migrate this state)
1522      */
1523     assert(k->has_variable_vring_alignment);
1524 
1525     if (align) {
1526         vdev->vq[n].vring.align = align;
1527         virtio_queue_update_rings(vdev, n);
1528     }
1529 }
1530 
1531 static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
1532 {
1533     bool ret = false;
1534 
1535     if (vq->vring.desc && vq->handle_aio_output) {
1536         VirtIODevice *vdev = vq->vdev;
1537 
1538         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1539         ret = vq->handle_aio_output(vdev, vq);
1540 
1541         if (unlikely(vdev->start_on_kick)) {
1542             virtio_set_started(vdev, true);
1543         }
1544     }
1545 
1546     return ret;
1547 }
1548 
1549 static void virtio_queue_notify_vq(VirtQueue *vq)
1550 {
1551     if (vq->vring.desc && vq->handle_output) {
1552         VirtIODevice *vdev = vq->vdev;
1553 
1554         if (unlikely(vdev->broken)) {
1555             return;
1556         }
1557 
1558         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1559         vq->handle_output(vdev, vq);
1560 
1561         if (unlikely(vdev->start_on_kick)) {
1562             virtio_set_started(vdev, true);
1563         }
1564     }
1565 }
1566 
1567 void virtio_queue_notify(VirtIODevice *vdev, int n)
1568 {
1569     VirtQueue *vq = &vdev->vq[n];
1570 
1571     if (unlikely(!vq->vring.desc || vdev->broken)) {
1572         return;
1573     }
1574 
1575     trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1576     if (vq->handle_aio_output) {
1577         event_notifier_set(&vq->host_notifier);
1578     } else if (vq->handle_output) {
1579         vq->handle_output(vdev, vq);
1580 
1581         if (unlikely(vdev->start_on_kick)) {
1582             virtio_set_started(vdev, true);
1583         }
1584     }
1585 }
1586 
1587 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1588 {
1589     return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1590         VIRTIO_NO_VECTOR;
1591 }
1592 
1593 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1594 {
1595     VirtQueue *vq = &vdev->vq[n];
1596 
1597     if (n < VIRTIO_QUEUE_MAX) {
1598         if (vdev->vector_queues &&
1599             vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1600             QLIST_REMOVE(vq, node);
1601         }
1602         vdev->vq[n].vector = vector;
1603         if (vdev->vector_queues &&
1604             vector != VIRTIO_NO_VECTOR) {
1605             QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1606         }
1607     }
1608 }
1609 
1610 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1611                             VirtIOHandleOutput handle_output)
1612 {
1613     int i;
1614 
1615     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1616         if (vdev->vq[i].vring.num == 0)
1617             break;
1618     }
1619 
1620     if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1621         abort();
1622 
1623     vdev->vq[i].vring.num = queue_size;
1624     vdev->vq[i].vring.num_default = queue_size;
1625     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1626     vdev->vq[i].handle_output = handle_output;
1627     vdev->vq[i].handle_aio_output = NULL;
1628 
1629     return &vdev->vq[i];
1630 }
1631 
1632 void virtio_del_queue(VirtIODevice *vdev, int n)
1633 {
1634     if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1635         abort();
1636     }
1637 
1638     vdev->vq[n].vring.num = 0;
1639     vdev->vq[n].vring.num_default = 0;
1640     vdev->vq[n].handle_output = NULL;
1641     vdev->vq[n].handle_aio_output = NULL;
1642 }
1643 
1644 static void virtio_set_isr(VirtIODevice *vdev, int value)
1645 {
1646     uint8_t old = atomic_read(&vdev->isr);
1647 
1648     /* Do not write ISR if it does not change, so that its cacheline remains
1649      * shared in the common case where the guest does not read it.
1650      */
1651     if ((old & value) != value) {
1652         atomic_or(&vdev->isr, value);
1653     }
1654 }
1655 
1656 /* Called within rcu_read_lock().  */
1657 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
1658 {
1659     uint16_t old, new;
1660     bool v;
1661     /* We need to expose used array entries before checking used event. */
1662     smp_mb();
1663     /* Always notify when queue is empty (when feature acknowledge) */
1664     if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1665         !vq->inuse && virtio_queue_empty(vq)) {
1666         return true;
1667     }
1668 
1669     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1670         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1671     }
1672 
1673     v = vq->signalled_used_valid;
1674     vq->signalled_used_valid = true;
1675     old = vq->signalled_used;
1676     new = vq->signalled_used = vq->used_idx;
1677     return !v || vring_need_event(vring_get_used_event(vq), new, old);
1678 }
1679 
1680 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
1681 {
1682     bool should_notify;
1683     rcu_read_lock();
1684     should_notify = virtio_should_notify(vdev, vq);
1685     rcu_read_unlock();
1686 
1687     if (!should_notify) {
1688         return;
1689     }
1690 
1691     trace_virtio_notify_irqfd(vdev, vq);
1692 
1693     /*
1694      * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
1695      * windows drivers included in virtio-win 1.8.0 (circa 2015) are
1696      * incorrectly polling this bit during crashdump and hibernation
1697      * in MSI mode, causing a hang if this bit is never updated.
1698      * Recent releases of Windows do not really shut down, but rather
1699      * log out and hibernate to make the next startup faster.  Hence,
1700      * this manifested as a more serious hang during shutdown with
1701      *
1702      * Next driver release from 2016 fixed this problem, so working around it
1703      * is not a must, but it's easy to do so let's do it here.
1704      *
1705      * Note: it's safe to update ISR from any thread as it was switched
1706      * to an atomic operation.
1707      */
1708     virtio_set_isr(vq->vdev, 0x1);
1709     event_notifier_set(&vq->guest_notifier);
1710 }
1711 
1712 static void virtio_irq(VirtQueue *vq)
1713 {
1714     virtio_set_isr(vq->vdev, 0x1);
1715     virtio_notify_vector(vq->vdev, vq->vector);
1716 }
1717 
1718 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1719 {
1720     bool should_notify;
1721     rcu_read_lock();
1722     should_notify = virtio_should_notify(vdev, vq);
1723     rcu_read_unlock();
1724 
1725     if (!should_notify) {
1726         return;
1727     }
1728 
1729     trace_virtio_notify(vdev, vq);
1730     virtio_irq(vq);
1731 }
1732 
1733 void virtio_notify_config(VirtIODevice *vdev)
1734 {
1735     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1736         return;
1737 
1738     virtio_set_isr(vdev, 0x3);
1739     vdev->generation++;
1740     virtio_notify_vector(vdev, vdev->config_vector);
1741 }
1742 
1743 static bool virtio_device_endian_needed(void *opaque)
1744 {
1745     VirtIODevice *vdev = opaque;
1746 
1747     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1748     if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1749         return vdev->device_endian != virtio_default_endian();
1750     }
1751     /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1752     return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1753 }
1754 
1755 static bool virtio_64bit_features_needed(void *opaque)
1756 {
1757     VirtIODevice *vdev = opaque;
1758 
1759     return (vdev->host_features >> 32) != 0;
1760 }
1761 
1762 static bool virtio_virtqueue_needed(void *opaque)
1763 {
1764     VirtIODevice *vdev = opaque;
1765 
1766     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1767 }
1768 
1769 static bool virtio_ringsize_needed(void *opaque)
1770 {
1771     VirtIODevice *vdev = opaque;
1772     int i;
1773 
1774     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1775         if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1776             return true;
1777         }
1778     }
1779     return false;
1780 }
1781 
1782 static bool virtio_extra_state_needed(void *opaque)
1783 {
1784     VirtIODevice *vdev = opaque;
1785     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1786     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1787 
1788     return k->has_extra_state &&
1789         k->has_extra_state(qbus->parent);
1790 }
1791 
1792 static bool virtio_broken_needed(void *opaque)
1793 {
1794     VirtIODevice *vdev = opaque;
1795 
1796     return vdev->broken;
1797 }
1798 
1799 static bool virtio_started_needed(void *opaque)
1800 {
1801     VirtIODevice *vdev = opaque;
1802 
1803     return vdev->started;
1804 }
1805 
1806 static const VMStateDescription vmstate_virtqueue = {
1807     .name = "virtqueue_state",
1808     .version_id = 1,
1809     .minimum_version_id = 1,
1810     .fields = (VMStateField[]) {
1811         VMSTATE_UINT64(vring.avail, struct VirtQueue),
1812         VMSTATE_UINT64(vring.used, struct VirtQueue),
1813         VMSTATE_END_OF_LIST()
1814     }
1815 };
1816 
1817 static const VMStateDescription vmstate_virtio_virtqueues = {
1818     .name = "virtio/virtqueues",
1819     .version_id = 1,
1820     .minimum_version_id = 1,
1821     .needed = &virtio_virtqueue_needed,
1822     .fields = (VMStateField[]) {
1823         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1824                       VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1825         VMSTATE_END_OF_LIST()
1826     }
1827 };
1828 
1829 static const VMStateDescription vmstate_ringsize = {
1830     .name = "ringsize_state",
1831     .version_id = 1,
1832     .minimum_version_id = 1,
1833     .fields = (VMStateField[]) {
1834         VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1835         VMSTATE_END_OF_LIST()
1836     }
1837 };
1838 
1839 static const VMStateDescription vmstate_virtio_ringsize = {
1840     .name = "virtio/ringsize",
1841     .version_id = 1,
1842     .minimum_version_id = 1,
1843     .needed = &virtio_ringsize_needed,
1844     .fields = (VMStateField[]) {
1845         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1846                       VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1847         VMSTATE_END_OF_LIST()
1848     }
1849 };
1850 
1851 static int get_extra_state(QEMUFile *f, void *pv, size_t size,
1852                            const VMStateField *field)
1853 {
1854     VirtIODevice *vdev = pv;
1855     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1856     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1857 
1858     if (!k->load_extra_state) {
1859         return -1;
1860     } else {
1861         return k->load_extra_state(qbus->parent, f);
1862     }
1863 }
1864 
1865 static int put_extra_state(QEMUFile *f, void *pv, size_t size,
1866                            const VMStateField *field, QJSON *vmdesc)
1867 {
1868     VirtIODevice *vdev = pv;
1869     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1870     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1871 
1872     k->save_extra_state(qbus->parent, f);
1873     return 0;
1874 }
1875 
1876 static const VMStateInfo vmstate_info_extra_state = {
1877     .name = "virtqueue_extra_state",
1878     .get = get_extra_state,
1879     .put = put_extra_state,
1880 };
1881 
1882 static const VMStateDescription vmstate_virtio_extra_state = {
1883     .name = "virtio/extra_state",
1884     .version_id = 1,
1885     .minimum_version_id = 1,
1886     .needed = &virtio_extra_state_needed,
1887     .fields = (VMStateField[]) {
1888         {
1889             .name         = "extra_state",
1890             .version_id   = 0,
1891             .field_exists = NULL,
1892             .size         = 0,
1893             .info         = &vmstate_info_extra_state,
1894             .flags        = VMS_SINGLE,
1895             .offset       = 0,
1896         },
1897         VMSTATE_END_OF_LIST()
1898     }
1899 };
1900 
1901 static const VMStateDescription vmstate_virtio_device_endian = {
1902     .name = "virtio/device_endian",
1903     .version_id = 1,
1904     .minimum_version_id = 1,
1905     .needed = &virtio_device_endian_needed,
1906     .fields = (VMStateField[]) {
1907         VMSTATE_UINT8(device_endian, VirtIODevice),
1908         VMSTATE_END_OF_LIST()
1909     }
1910 };
1911 
1912 static const VMStateDescription vmstate_virtio_64bit_features = {
1913     .name = "virtio/64bit_features",
1914     .version_id = 1,
1915     .minimum_version_id = 1,
1916     .needed = &virtio_64bit_features_needed,
1917     .fields = (VMStateField[]) {
1918         VMSTATE_UINT64(guest_features, VirtIODevice),
1919         VMSTATE_END_OF_LIST()
1920     }
1921 };
1922 
1923 static const VMStateDescription vmstate_virtio_broken = {
1924     .name = "virtio/broken",
1925     .version_id = 1,
1926     .minimum_version_id = 1,
1927     .needed = &virtio_broken_needed,
1928     .fields = (VMStateField[]) {
1929         VMSTATE_BOOL(broken, VirtIODevice),
1930         VMSTATE_END_OF_LIST()
1931     }
1932 };
1933 
1934 static const VMStateDescription vmstate_virtio_started = {
1935     .name = "virtio/started",
1936     .version_id = 1,
1937     .minimum_version_id = 1,
1938     .needed = &virtio_started_needed,
1939     .fields = (VMStateField[]) {
1940         VMSTATE_BOOL(started, VirtIODevice),
1941         VMSTATE_END_OF_LIST()
1942     }
1943 };
1944 
1945 static const VMStateDescription vmstate_virtio = {
1946     .name = "virtio",
1947     .version_id = 1,
1948     .minimum_version_id = 1,
1949     .minimum_version_id_old = 1,
1950     .fields = (VMStateField[]) {
1951         VMSTATE_END_OF_LIST()
1952     },
1953     .subsections = (const VMStateDescription*[]) {
1954         &vmstate_virtio_device_endian,
1955         &vmstate_virtio_64bit_features,
1956         &vmstate_virtio_virtqueues,
1957         &vmstate_virtio_ringsize,
1958         &vmstate_virtio_broken,
1959         &vmstate_virtio_extra_state,
1960         &vmstate_virtio_started,
1961         NULL
1962     }
1963 };
1964 
1965 int virtio_save(VirtIODevice *vdev, QEMUFile *f)
1966 {
1967     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1968     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1969     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1970     uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1971     int i;
1972 
1973     if (k->save_config) {
1974         k->save_config(qbus->parent, f);
1975     }
1976 
1977     qemu_put_8s(f, &vdev->status);
1978     qemu_put_8s(f, &vdev->isr);
1979     qemu_put_be16s(f, &vdev->queue_sel);
1980     qemu_put_be32s(f, &guest_features_lo);
1981     qemu_put_be32(f, vdev->config_len);
1982     qemu_put_buffer(f, vdev->config, vdev->config_len);
1983 
1984     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1985         if (vdev->vq[i].vring.num == 0)
1986             break;
1987     }
1988 
1989     qemu_put_be32(f, i);
1990 
1991     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1992         if (vdev->vq[i].vring.num == 0)
1993             break;
1994 
1995         qemu_put_be32(f, vdev->vq[i].vring.num);
1996         if (k->has_variable_vring_alignment) {
1997             qemu_put_be32(f, vdev->vq[i].vring.align);
1998         }
1999         /*
2000          * Save desc now, the rest of the ring addresses are saved in
2001          * subsections for VIRTIO-1 devices.
2002          */
2003         qemu_put_be64(f, vdev->vq[i].vring.desc);
2004         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
2005         if (k->save_queue) {
2006             k->save_queue(qbus->parent, i, f);
2007         }
2008     }
2009 
2010     if (vdc->save != NULL) {
2011         vdc->save(vdev, f);
2012     }
2013 
2014     if (vdc->vmsd) {
2015         int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL);
2016         if (ret) {
2017             return ret;
2018         }
2019     }
2020 
2021     /* Subsections */
2022     return vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
2023 }
2024 
2025 /* A wrapper for use as a VMState .put function */
2026 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
2027                               const VMStateField *field, QJSON *vmdesc)
2028 {
2029     return virtio_save(VIRTIO_DEVICE(opaque), f);
2030 }
2031 
2032 /* A wrapper for use as a VMState .get function */
2033 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size,
2034                              const VMStateField *field)
2035 {
2036     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
2037     DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
2038 
2039     return virtio_load(vdev, f, dc->vmsd->version_id);
2040 }
2041 
2042 const VMStateInfo  virtio_vmstate_info = {
2043     .name = "virtio",
2044     .get = virtio_device_get,
2045     .put = virtio_device_put,
2046 };
2047 
2048 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
2049 {
2050     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2051     bool bad = (val & ~(vdev->host_features)) != 0;
2052 
2053     val &= vdev->host_features;
2054     if (k->set_features) {
2055         k->set_features(vdev, val);
2056     }
2057     vdev->guest_features = val;
2058     return bad ? -1 : 0;
2059 }
2060 
2061 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
2062 {
2063     int ret;
2064     /*
2065      * The driver must not attempt to set features after feature negotiation
2066      * has finished.
2067      */
2068     if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
2069         return -EINVAL;
2070     }
2071     ret = virtio_set_features_nocheck(vdev, val);
2072     if (!ret) {
2073         if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2074             /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches.  */
2075             int i;
2076             for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2077                 if (vdev->vq[i].vring.num != 0) {
2078                     virtio_init_region_cache(vdev, i);
2079                 }
2080             }
2081         }
2082 
2083         if (!virtio_device_started(vdev, vdev->status) &&
2084             !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2085             vdev->start_on_kick = true;
2086         }
2087     }
2088     return ret;
2089 }
2090 
2091 size_t virtio_feature_get_config_size(VirtIOFeature *feature_sizes,
2092                                       uint64_t host_features)
2093 {
2094     size_t config_size = 0;
2095     int i;
2096 
2097     for (i = 0; feature_sizes[i].flags != 0; i++) {
2098         if (host_features & feature_sizes[i].flags) {
2099             config_size = MAX(feature_sizes[i].end, config_size);
2100         }
2101     }
2102 
2103     return config_size;
2104 }
2105 
2106 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
2107 {
2108     int i, ret;
2109     int32_t config_len;
2110     uint32_t num;
2111     uint32_t features;
2112     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2113     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2114     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
2115 
2116     /*
2117      * We poison the endianness to ensure it does not get used before
2118      * subsections have been loaded.
2119      */
2120     vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
2121 
2122     if (k->load_config) {
2123         ret = k->load_config(qbus->parent, f);
2124         if (ret)
2125             return ret;
2126     }
2127 
2128     qemu_get_8s(f, &vdev->status);
2129     qemu_get_8s(f, &vdev->isr);
2130     qemu_get_be16s(f, &vdev->queue_sel);
2131     if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
2132         return -1;
2133     }
2134     qemu_get_be32s(f, &features);
2135 
2136     /*
2137      * Temporarily set guest_features low bits - needed by
2138      * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
2139      * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
2140      *
2141      * Note: devices should always test host features in future - don't create
2142      * new dependencies like this.
2143      */
2144     vdev->guest_features = features;
2145 
2146     config_len = qemu_get_be32(f);
2147 
2148     /*
2149      * There are cases where the incoming config can be bigger or smaller
2150      * than what we have; so load what we have space for, and skip
2151      * any excess that's in the stream.
2152      */
2153     qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
2154 
2155     while (config_len > vdev->config_len) {
2156         qemu_get_byte(f);
2157         config_len--;
2158     }
2159 
2160     num = qemu_get_be32(f);
2161 
2162     if (num > VIRTIO_QUEUE_MAX) {
2163         error_report("Invalid number of virtqueues: 0x%x", num);
2164         return -1;
2165     }
2166 
2167     for (i = 0; i < num; i++) {
2168         vdev->vq[i].vring.num = qemu_get_be32(f);
2169         if (k->has_variable_vring_alignment) {
2170             vdev->vq[i].vring.align = qemu_get_be32(f);
2171         }
2172         vdev->vq[i].vring.desc = qemu_get_be64(f);
2173         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
2174         vdev->vq[i].signalled_used_valid = false;
2175         vdev->vq[i].notification = true;
2176 
2177         if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
2178             error_report("VQ %d address 0x0 "
2179                          "inconsistent with Host index 0x%x",
2180                          i, vdev->vq[i].last_avail_idx);
2181             return -1;
2182         }
2183         if (k->load_queue) {
2184             ret = k->load_queue(qbus->parent, i, f);
2185             if (ret)
2186                 return ret;
2187         }
2188     }
2189 
2190     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
2191 
2192     if (vdc->load != NULL) {
2193         ret = vdc->load(vdev, f, version_id);
2194         if (ret) {
2195             return ret;
2196         }
2197     }
2198 
2199     if (vdc->vmsd) {
2200         ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id);
2201         if (ret) {
2202             return ret;
2203         }
2204     }
2205 
2206     /* Subsections */
2207     ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
2208     if (ret) {
2209         return ret;
2210     }
2211 
2212     if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
2213         vdev->device_endian = virtio_default_endian();
2214     }
2215 
2216     if (virtio_64bit_features_needed(vdev)) {
2217         /*
2218          * Subsection load filled vdev->guest_features.  Run them
2219          * through virtio_set_features to sanity-check them against
2220          * host_features.
2221          */
2222         uint64_t features64 = vdev->guest_features;
2223         if (virtio_set_features_nocheck(vdev, features64) < 0) {
2224             error_report("Features 0x%" PRIx64 " unsupported. "
2225                          "Allowed features: 0x%" PRIx64,
2226                          features64, vdev->host_features);
2227             return -1;
2228         }
2229     } else {
2230         if (virtio_set_features_nocheck(vdev, features) < 0) {
2231             error_report("Features 0x%x unsupported. "
2232                          "Allowed features: 0x%" PRIx64,
2233                          features, vdev->host_features);
2234             return -1;
2235         }
2236     }
2237 
2238     if (!virtio_device_started(vdev, vdev->status) &&
2239         !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2240         vdev->start_on_kick = true;
2241     }
2242 
2243     rcu_read_lock();
2244     for (i = 0; i < num; i++) {
2245         if (vdev->vq[i].vring.desc) {
2246             uint16_t nheads;
2247 
2248             /*
2249              * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
2250              * only the region cache needs to be set up.  Legacy devices need
2251              * to calculate used and avail ring addresses based on the desc
2252              * address.
2253              */
2254             if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2255                 virtio_init_region_cache(vdev, i);
2256             } else {
2257                 virtio_queue_update_rings(vdev, i);
2258             }
2259 
2260             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
2261             /* Check it isn't doing strange things with descriptor numbers. */
2262             if (nheads > vdev->vq[i].vring.num) {
2263                 error_report("VQ %d size 0x%x Guest index 0x%x "
2264                              "inconsistent with Host index 0x%x: delta 0x%x",
2265                              i, vdev->vq[i].vring.num,
2266                              vring_avail_idx(&vdev->vq[i]),
2267                              vdev->vq[i].last_avail_idx, nheads);
2268                 return -1;
2269             }
2270             vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
2271             vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
2272 
2273             /*
2274              * Some devices migrate VirtQueueElements that have been popped
2275              * from the avail ring but not yet returned to the used ring.
2276              * Since max ring size < UINT16_MAX it's safe to use modulo
2277              * UINT16_MAX + 1 subtraction.
2278              */
2279             vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
2280                                 vdev->vq[i].used_idx);
2281             if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
2282                 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
2283                              "used_idx 0x%x",
2284                              i, vdev->vq[i].vring.num,
2285                              vdev->vq[i].last_avail_idx,
2286                              vdev->vq[i].used_idx);
2287                 return -1;
2288             }
2289         }
2290     }
2291     rcu_read_unlock();
2292 
2293     return 0;
2294 }
2295 
2296 void virtio_cleanup(VirtIODevice *vdev)
2297 {
2298     qemu_del_vm_change_state_handler(vdev->vmstate);
2299 }
2300 
2301 static void virtio_vmstate_change(void *opaque, int running, RunState state)
2302 {
2303     VirtIODevice *vdev = opaque;
2304     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2305     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2306     bool backend_run = running && virtio_device_started(vdev, vdev->status);
2307     vdev->vm_running = running;
2308 
2309     if (backend_run) {
2310         virtio_set_status(vdev, vdev->status);
2311     }
2312 
2313     if (k->vmstate_change) {
2314         k->vmstate_change(qbus->parent, backend_run);
2315     }
2316 
2317     if (!backend_run) {
2318         virtio_set_status(vdev, vdev->status);
2319     }
2320 }
2321 
2322 void virtio_instance_init_common(Object *proxy_obj, void *data,
2323                                  size_t vdev_size, const char *vdev_name)
2324 {
2325     DeviceState *vdev = data;
2326 
2327     object_initialize_child(proxy_obj, "virtio-backend", vdev, vdev_size,
2328                             vdev_name, &error_abort, NULL);
2329     qdev_alias_all_properties(vdev, proxy_obj);
2330 }
2331 
2332 void virtio_init(VirtIODevice *vdev, const char *name,
2333                  uint16_t device_id, size_t config_size)
2334 {
2335     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2336     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2337     int i;
2338     int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
2339 
2340     if (nvectors) {
2341         vdev->vector_queues =
2342             g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
2343     }
2344 
2345     vdev->start_on_kick = false;
2346     vdev->started = false;
2347     vdev->device_id = device_id;
2348     vdev->status = 0;
2349     atomic_set(&vdev->isr, 0);
2350     vdev->queue_sel = 0;
2351     vdev->config_vector = VIRTIO_NO_VECTOR;
2352     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
2353     vdev->vm_running = runstate_is_running();
2354     vdev->broken = false;
2355     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2356         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
2357         vdev->vq[i].vdev = vdev;
2358         vdev->vq[i].queue_index = i;
2359     }
2360 
2361     vdev->name = name;
2362     vdev->config_len = config_size;
2363     if (vdev->config_len) {
2364         vdev->config = g_malloc0(config_size);
2365     } else {
2366         vdev->config = NULL;
2367     }
2368     vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
2369             virtio_vmstate_change, vdev);
2370     vdev->device_endian = virtio_default_endian();
2371     vdev->use_guest_notifier_mask = true;
2372 }
2373 
2374 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
2375 {
2376     return vdev->vq[n].vring.desc;
2377 }
2378 
2379 bool virtio_queue_enabled(VirtIODevice *vdev, int n)
2380 {
2381     return virtio_queue_get_desc_addr(vdev, n) != 0;
2382 }
2383 
2384 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
2385 {
2386     return vdev->vq[n].vring.avail;
2387 }
2388 
2389 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
2390 {
2391     return vdev->vq[n].vring.used;
2392 }
2393 
2394 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
2395 {
2396     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
2397 }
2398 
2399 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
2400 {
2401     return offsetof(VRingAvail, ring) +
2402         sizeof(uint16_t) * vdev->vq[n].vring.num;
2403 }
2404 
2405 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
2406 {
2407     return offsetof(VRingUsed, ring) +
2408         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
2409 }
2410 
2411 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
2412 {
2413     return vdev->vq[n].last_avail_idx;
2414 }
2415 
2416 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
2417 {
2418     vdev->vq[n].last_avail_idx = idx;
2419     vdev->vq[n].shadow_avail_idx = idx;
2420 }
2421 
2422 void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n)
2423 {
2424     rcu_read_lock();
2425     if (vdev->vq[n].vring.desc) {
2426         vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]);
2427         vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx;
2428     }
2429     rcu_read_unlock();
2430 }
2431 
2432 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
2433 {
2434     rcu_read_lock();
2435     if (vdev->vq[n].vring.desc) {
2436         vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
2437     }
2438     rcu_read_unlock();
2439 }
2440 
2441 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
2442 {
2443     vdev->vq[n].signalled_used_valid = false;
2444 }
2445 
2446 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
2447 {
2448     return vdev->vq + n;
2449 }
2450 
2451 uint16_t virtio_get_queue_index(VirtQueue *vq)
2452 {
2453     return vq->queue_index;
2454 }
2455 
2456 static void virtio_queue_guest_notifier_read(EventNotifier *n)
2457 {
2458     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
2459     if (event_notifier_test_and_clear(n)) {
2460         virtio_irq(vq);
2461     }
2462 }
2463 
2464 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
2465                                                 bool with_irqfd)
2466 {
2467     if (assign && !with_irqfd) {
2468         event_notifier_set_handler(&vq->guest_notifier,
2469                                    virtio_queue_guest_notifier_read);
2470     } else {
2471         event_notifier_set_handler(&vq->guest_notifier, NULL);
2472     }
2473     if (!assign) {
2474         /* Test and clear notifier before closing it,
2475          * in case poll callback didn't have time to run. */
2476         virtio_queue_guest_notifier_read(&vq->guest_notifier);
2477     }
2478 }
2479 
2480 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
2481 {
2482     return &vq->guest_notifier;
2483 }
2484 
2485 static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
2486 {
2487     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2488     if (event_notifier_test_and_clear(n)) {
2489         virtio_queue_notify_aio_vq(vq);
2490     }
2491 }
2492 
2493 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
2494 {
2495     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2496 
2497     virtio_queue_set_notification(vq, 0);
2498 }
2499 
2500 static bool virtio_queue_host_notifier_aio_poll(void *opaque)
2501 {
2502     EventNotifier *n = opaque;
2503     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2504     bool progress;
2505 
2506     if (!vq->vring.desc || virtio_queue_empty(vq)) {
2507         return false;
2508     }
2509 
2510     progress = virtio_queue_notify_aio_vq(vq);
2511 
2512     /* In case the handler function re-enabled notifications */
2513     virtio_queue_set_notification(vq, 0);
2514     return progress;
2515 }
2516 
2517 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
2518 {
2519     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2520 
2521     /* Caller polls once more after this to catch requests that race with us */
2522     virtio_queue_set_notification(vq, 1);
2523 }
2524 
2525 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
2526                                                 VirtIOHandleAIOOutput handle_output)
2527 {
2528     if (handle_output) {
2529         vq->handle_aio_output = handle_output;
2530         aio_set_event_notifier(ctx, &vq->host_notifier, true,
2531                                virtio_queue_host_notifier_aio_read,
2532                                virtio_queue_host_notifier_aio_poll);
2533         aio_set_event_notifier_poll(ctx, &vq->host_notifier,
2534                                     virtio_queue_host_notifier_aio_poll_begin,
2535                                     virtio_queue_host_notifier_aio_poll_end);
2536     } else {
2537         aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
2538         /* Test and clear notifier before after disabling event,
2539          * in case poll callback didn't have time to run. */
2540         virtio_queue_host_notifier_aio_read(&vq->host_notifier);
2541         vq->handle_aio_output = NULL;
2542     }
2543 }
2544 
2545 void virtio_queue_host_notifier_read(EventNotifier *n)
2546 {
2547     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
2548     if (event_notifier_test_and_clear(n)) {
2549         virtio_queue_notify_vq(vq);
2550     }
2551 }
2552 
2553 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
2554 {
2555     return &vq->host_notifier;
2556 }
2557 
2558 int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n,
2559                                       MemoryRegion *mr, bool assign)
2560 {
2561     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2562     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2563 
2564     if (k->set_host_notifier_mr) {
2565         return k->set_host_notifier_mr(qbus->parent, n, mr, assign);
2566     }
2567 
2568     return -1;
2569 }
2570 
2571 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
2572 {
2573     g_free(vdev->bus_name);
2574     vdev->bus_name = g_strdup(bus_name);
2575 }
2576 
2577 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
2578 {
2579     va_list ap;
2580 
2581     va_start(ap, fmt);
2582     error_vreport(fmt, ap);
2583     va_end(ap);
2584 
2585     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2586         vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET;
2587         virtio_notify_config(vdev);
2588     }
2589 
2590     vdev->broken = true;
2591 }
2592 
2593 static void virtio_memory_listener_commit(MemoryListener *listener)
2594 {
2595     VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
2596     int i;
2597 
2598     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2599         if (vdev->vq[i].vring.num == 0) {
2600             break;
2601         }
2602         virtio_init_region_cache(vdev, i);
2603     }
2604 }
2605 
2606 static void virtio_device_realize(DeviceState *dev, Error **errp)
2607 {
2608     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
2609     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
2610     Error *err = NULL;
2611 
2612     /* Devices should either use vmsd or the load/save methods */
2613     assert(!vdc->vmsd || !vdc->load);
2614 
2615     if (vdc->realize != NULL) {
2616         vdc->realize(dev, &err);
2617         if (err != NULL) {
2618             error_propagate(errp, err);
2619             return;
2620         }
2621     }
2622 
2623     virtio_bus_device_plugged(vdev, &err);
2624     if (err != NULL) {
2625         error_propagate(errp, err);
2626         vdc->unrealize(dev, NULL);
2627         return;
2628     }
2629 
2630     vdev->listener.commit = virtio_memory_listener_commit;
2631     memory_listener_register(&vdev->listener, vdev->dma_as);
2632 }
2633 
2634 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
2635 {
2636     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
2637     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
2638     Error *err = NULL;
2639 
2640     virtio_bus_device_unplugged(vdev);
2641 
2642     if (vdc->unrealize != NULL) {
2643         vdc->unrealize(dev, &err);
2644         if (err != NULL) {
2645             error_propagate(errp, err);
2646             return;
2647         }
2648     }
2649 
2650     g_free(vdev->bus_name);
2651     vdev->bus_name = NULL;
2652 }
2653 
2654 static void virtio_device_free_virtqueues(VirtIODevice *vdev)
2655 {
2656     int i;
2657     if (!vdev->vq) {
2658         return;
2659     }
2660 
2661     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2662         if (vdev->vq[i].vring.num == 0) {
2663             break;
2664         }
2665         virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
2666     }
2667     g_free(vdev->vq);
2668 }
2669 
2670 static void virtio_device_instance_finalize(Object *obj)
2671 {
2672     VirtIODevice *vdev = VIRTIO_DEVICE(obj);
2673 
2674     memory_listener_unregister(&vdev->listener);
2675     virtio_device_free_virtqueues(vdev);
2676 
2677     g_free(vdev->config);
2678     g_free(vdev->vector_queues);
2679 }
2680 
2681 static Property virtio_properties[] = {
2682     DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
2683     DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
2684     DEFINE_PROP_END_OF_LIST(),
2685 };
2686 
2687 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
2688 {
2689     VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
2690     int i, n, r, err;
2691 
2692     memory_region_transaction_begin();
2693     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2694         VirtQueue *vq = &vdev->vq[n];
2695         if (!virtio_queue_get_num(vdev, n)) {
2696             continue;
2697         }
2698         r = virtio_bus_set_host_notifier(qbus, n, true);
2699         if (r < 0) {
2700             err = r;
2701             goto assign_error;
2702         }
2703         event_notifier_set_handler(&vq->host_notifier,
2704                                    virtio_queue_host_notifier_read);
2705     }
2706 
2707     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2708         /* Kick right away to begin processing requests already in vring */
2709         VirtQueue *vq = &vdev->vq[n];
2710         if (!vq->vring.num) {
2711             continue;
2712         }
2713         event_notifier_set(&vq->host_notifier);
2714     }
2715     memory_region_transaction_commit();
2716     return 0;
2717 
2718 assign_error:
2719     i = n; /* save n for a second iteration after transaction is committed. */
2720     while (--n >= 0) {
2721         VirtQueue *vq = &vdev->vq[n];
2722         if (!virtio_queue_get_num(vdev, n)) {
2723             continue;
2724         }
2725 
2726         event_notifier_set_handler(&vq->host_notifier, NULL);
2727         r = virtio_bus_set_host_notifier(qbus, n, false);
2728         assert(r >= 0);
2729     }
2730     memory_region_transaction_commit();
2731 
2732     while (--i >= 0) {
2733         if (!virtio_queue_get_num(vdev, i)) {
2734             continue;
2735         }
2736         virtio_bus_cleanup_host_notifier(qbus, i);
2737     }
2738     return err;
2739 }
2740 
2741 int virtio_device_start_ioeventfd(VirtIODevice *vdev)
2742 {
2743     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2744     VirtioBusState *vbus = VIRTIO_BUS(qbus);
2745 
2746     return virtio_bus_start_ioeventfd(vbus);
2747 }
2748 
2749 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
2750 {
2751     VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
2752     int n, r;
2753 
2754     memory_region_transaction_begin();
2755     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2756         VirtQueue *vq = &vdev->vq[n];
2757 
2758         if (!virtio_queue_get_num(vdev, n)) {
2759             continue;
2760         }
2761         event_notifier_set_handler(&vq->host_notifier, NULL);
2762         r = virtio_bus_set_host_notifier(qbus, n, false);
2763         assert(r >= 0);
2764     }
2765     memory_region_transaction_commit();
2766 
2767     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
2768         if (!virtio_queue_get_num(vdev, n)) {
2769             continue;
2770         }
2771         virtio_bus_cleanup_host_notifier(qbus, n);
2772     }
2773 }
2774 
2775 void virtio_device_stop_ioeventfd(VirtIODevice *vdev)
2776 {
2777     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2778     VirtioBusState *vbus = VIRTIO_BUS(qbus);
2779 
2780     virtio_bus_stop_ioeventfd(vbus);
2781 }
2782 
2783 int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
2784 {
2785     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2786     VirtioBusState *vbus = VIRTIO_BUS(qbus);
2787 
2788     return virtio_bus_grab_ioeventfd(vbus);
2789 }
2790 
2791 void virtio_device_release_ioeventfd(VirtIODevice *vdev)
2792 {
2793     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2794     VirtioBusState *vbus = VIRTIO_BUS(qbus);
2795 
2796     virtio_bus_release_ioeventfd(vbus);
2797 }
2798 
2799 static void virtio_device_class_init(ObjectClass *klass, void *data)
2800 {
2801     /* Set the default value here. */
2802     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
2803     DeviceClass *dc = DEVICE_CLASS(klass);
2804 
2805     dc->realize = virtio_device_realize;
2806     dc->unrealize = virtio_device_unrealize;
2807     dc->bus_type = TYPE_VIRTIO_BUS;
2808     dc->props = virtio_properties;
2809     vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
2810     vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
2811 
2812     vdc->legacy_features |= VIRTIO_LEGACY_FEATURES;
2813 }
2814 
2815 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
2816 {
2817     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2818     VirtioBusState *vbus = VIRTIO_BUS(qbus);
2819 
2820     return virtio_bus_ioeventfd_enabled(vbus);
2821 }
2822 
2823 static const TypeInfo virtio_device_info = {
2824     .name = TYPE_VIRTIO_DEVICE,
2825     .parent = TYPE_DEVICE,
2826     .instance_size = sizeof(VirtIODevice),
2827     .class_init = virtio_device_class_init,
2828     .instance_finalize = virtio_device_instance_finalize,
2829     .abstract = true,
2830     .class_size = sizeof(VirtioDeviceClass),
2831 };
2832 
2833 static void virtio_register_types(void)
2834 {
2835     type_register_static(&virtio_device_info);
2836 }
2837 
2838 type_init(virtio_register_types)
2839