xref: /openbmc/qemu/hw/virtio/virtio.c (revision 7acafcfa)
1 /*
2  * Virtio Support
3  *
4  * Copyright IBM, Corp. 2007
5  *
6  * Authors:
7  *  Anthony Liguori   <aliguori@us.ibm.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  */
13 
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "cpu.h"
17 #include "trace.h"
18 #include "exec/address-spaces.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "hw/virtio/virtio.h"
23 #include "migration/qemu-file-types.h"
24 #include "qemu/atomic.h"
25 #include "hw/virtio/virtio-bus.h"
26 #include "hw/qdev-properties.h"
27 #include "hw/virtio/virtio-access.h"
28 #include "sysemu/dma.h"
29 #include "sysemu/runstate.h"
30 
31 /*
32  * The alignment to use between consumer and producer parts of vring.
33  * x86 pagesize again. This is the default, used by transports like PCI
34  * which don't provide a means for the guest to tell the host the alignment.
35  */
36 #define VIRTIO_PCI_VRING_ALIGN         4096
37 
38 typedef struct VRingDesc
39 {
40     uint64_t addr;
41     uint32_t len;
42     uint16_t flags;
43     uint16_t next;
44 } VRingDesc;
45 
46 typedef struct VRingPackedDesc {
47     uint64_t addr;
48     uint32_t len;
49     uint16_t id;
50     uint16_t flags;
51 } VRingPackedDesc;
52 
53 typedef struct VRingAvail
54 {
55     uint16_t flags;
56     uint16_t idx;
57     uint16_t ring[];
58 } VRingAvail;
59 
60 typedef struct VRingUsedElem
61 {
62     uint32_t id;
63     uint32_t len;
64 } VRingUsedElem;
65 
66 typedef struct VRingUsed
67 {
68     uint16_t flags;
69     uint16_t idx;
70     VRingUsedElem ring[];
71 } VRingUsed;
72 
73 typedef struct VRingMemoryRegionCaches {
74     struct rcu_head rcu;
75     MemoryRegionCache desc;
76     MemoryRegionCache avail;
77     MemoryRegionCache used;
78 } VRingMemoryRegionCaches;
79 
80 typedef struct VRing
81 {
82     unsigned int num;
83     unsigned int num_default;
84     unsigned int align;
85     hwaddr desc;
86     hwaddr avail;
87     hwaddr used;
88     VRingMemoryRegionCaches *caches;
89 } VRing;
90 
91 typedef struct VRingPackedDescEvent {
92     uint16_t off_wrap;
93     uint16_t flags;
94 } VRingPackedDescEvent ;
95 
96 struct VirtQueue
97 {
98     VRing vring;
99     VirtQueueElement *used_elems;
100 
101     /* Next head to pop */
102     uint16_t last_avail_idx;
103     bool last_avail_wrap_counter;
104 
105     /* Last avail_idx read from VQ. */
106     uint16_t shadow_avail_idx;
107     bool shadow_avail_wrap_counter;
108 
109     uint16_t used_idx;
110     bool used_wrap_counter;
111 
112     /* Last used index value we have signalled on */
113     uint16_t signalled_used;
114 
115     /* Last used index value we have signalled on */
116     bool signalled_used_valid;
117 
118     /* Notification enabled? */
119     bool notification;
120 
121     uint16_t queue_index;
122 
123     unsigned int inuse;
124 
125     uint16_t vector;
126     VirtIOHandleOutput handle_output;
127     VirtIOHandleAIOOutput handle_aio_output;
128     VirtIODevice *vdev;
129     EventNotifier guest_notifier;
130     EventNotifier host_notifier;
131     bool host_notifier_enabled;
132     QLIST_ENTRY(VirtQueue) node;
133 };
134 
135 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
136 {
137     if (!caches) {
138         return;
139     }
140 
141     address_space_cache_destroy(&caches->desc);
142     address_space_cache_destroy(&caches->avail);
143     address_space_cache_destroy(&caches->used);
144     g_free(caches);
145 }
146 
147 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
148 {
149     VRingMemoryRegionCaches *caches;
150 
151     caches = atomic_read(&vq->vring.caches);
152     atomic_rcu_set(&vq->vring.caches, NULL);
153     if (caches) {
154         call_rcu(caches, virtio_free_region_cache, rcu);
155     }
156 }
157 
158 static void virtio_init_region_cache(VirtIODevice *vdev, int n)
159 {
160     VirtQueue *vq = &vdev->vq[n];
161     VRingMemoryRegionCaches *old = vq->vring.caches;
162     VRingMemoryRegionCaches *new = NULL;
163     hwaddr addr, size;
164     int64_t len;
165     bool packed;
166 
167 
168     addr = vq->vring.desc;
169     if (!addr) {
170         goto out_no_cache;
171     }
172     new = g_new0(VRingMemoryRegionCaches, 1);
173     size = virtio_queue_get_desc_size(vdev, n);
174     packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
175                                    true : false;
176     len = address_space_cache_init(&new->desc, vdev->dma_as,
177                                    addr, size, packed);
178     if (len < size) {
179         virtio_error(vdev, "Cannot map desc");
180         goto err_desc;
181     }
182 
183     size = virtio_queue_get_used_size(vdev, n);
184     len = address_space_cache_init(&new->used, vdev->dma_as,
185                                    vq->vring.used, size, true);
186     if (len < size) {
187         virtio_error(vdev, "Cannot map used");
188         goto err_used;
189     }
190 
191     size = virtio_queue_get_avail_size(vdev, n);
192     len = address_space_cache_init(&new->avail, vdev->dma_as,
193                                    vq->vring.avail, size, false);
194     if (len < size) {
195         virtio_error(vdev, "Cannot map avail");
196         goto err_avail;
197     }
198 
199     atomic_rcu_set(&vq->vring.caches, new);
200     if (old) {
201         call_rcu(old, virtio_free_region_cache, rcu);
202     }
203     return;
204 
205 err_avail:
206     address_space_cache_destroy(&new->avail);
207 err_used:
208     address_space_cache_destroy(&new->used);
209 err_desc:
210     address_space_cache_destroy(&new->desc);
211 out_no_cache:
212     g_free(new);
213     virtio_virtqueue_reset_region_cache(vq);
214 }
215 
216 /* virt queue functions */
217 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
218 {
219     VRing *vring = &vdev->vq[n].vring;
220 
221     if (!vring->num || !vring->desc || !vring->align) {
222         /* not yet setup -> nothing to do */
223         return;
224     }
225     vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
226     vring->used = vring_align(vring->avail +
227                               offsetof(VRingAvail, ring[vring->num]),
228                               vring->align);
229     virtio_init_region_cache(vdev, n);
230 }
231 
232 /* Called within rcu_read_lock().  */
233 static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc,
234                                   MemoryRegionCache *cache, int i)
235 {
236     address_space_read_cached(cache, i * sizeof(VRingDesc),
237                               desc, sizeof(VRingDesc));
238     virtio_tswap64s(vdev, &desc->addr);
239     virtio_tswap32s(vdev, &desc->len);
240     virtio_tswap16s(vdev, &desc->flags);
241     virtio_tswap16s(vdev, &desc->next);
242 }
243 
244 static void vring_packed_event_read(VirtIODevice *vdev,
245                                     MemoryRegionCache *cache,
246                                     VRingPackedDescEvent *e)
247 {
248     hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap);
249     hwaddr off_flags = offsetof(VRingPackedDescEvent, flags);
250 
251     address_space_read_cached(cache, off_flags, &e->flags,
252                               sizeof(e->flags));
253     /* Make sure flags is seen before off_wrap */
254     smp_rmb();
255     address_space_read_cached(cache, off_off, &e->off_wrap,
256                               sizeof(e->off_wrap));
257     virtio_tswap16s(vdev, &e->off_wrap);
258     virtio_tswap16s(vdev, &e->flags);
259 }
260 
261 static void vring_packed_off_wrap_write(VirtIODevice *vdev,
262                                         MemoryRegionCache *cache,
263                                         uint16_t off_wrap)
264 {
265     hwaddr off = offsetof(VRingPackedDescEvent, off_wrap);
266 
267     virtio_tswap16s(vdev, &off_wrap);
268     address_space_write_cached(cache, off, &off_wrap, sizeof(off_wrap));
269     address_space_cache_invalidate(cache, off, sizeof(off_wrap));
270 }
271 
272 static void vring_packed_flags_write(VirtIODevice *vdev,
273                                      MemoryRegionCache *cache, uint16_t flags)
274 {
275     hwaddr off = offsetof(VRingPackedDescEvent, flags);
276 
277     virtio_tswap16s(vdev, &flags);
278     address_space_write_cached(cache, off, &flags, sizeof(flags));
279     address_space_cache_invalidate(cache, off, sizeof(flags));
280 }
281 
282 /* Called within rcu_read_lock().  */
283 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
284 {
285     return atomic_rcu_read(&vq->vring.caches);
286 }
287 
288 /* Called within rcu_read_lock().  */
289 static inline uint16_t vring_avail_flags(VirtQueue *vq)
290 {
291     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
292     hwaddr pa = offsetof(VRingAvail, flags);
293 
294     if (!caches) {
295         return 0;
296     }
297 
298     return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
299 }
300 
301 /* Called within rcu_read_lock().  */
302 static inline uint16_t vring_avail_idx(VirtQueue *vq)
303 {
304     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
305     hwaddr pa = offsetof(VRingAvail, idx);
306 
307     if (!caches) {
308         return 0;
309     }
310 
311     vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
312     return vq->shadow_avail_idx;
313 }
314 
315 /* Called within rcu_read_lock().  */
316 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
317 {
318     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
319     hwaddr pa = offsetof(VRingAvail, ring[i]);
320 
321     if (!caches) {
322         return 0;
323     }
324 
325     return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
326 }
327 
328 /* Called within rcu_read_lock().  */
329 static inline uint16_t vring_get_used_event(VirtQueue *vq)
330 {
331     return vring_avail_ring(vq, vq->vring.num);
332 }
333 
334 /* Called within rcu_read_lock().  */
335 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
336                                     int i)
337 {
338     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
339     hwaddr pa = offsetof(VRingUsed, ring[i]);
340 
341     if (!caches) {
342         return;
343     }
344 
345     virtio_tswap32s(vq->vdev, &uelem->id);
346     virtio_tswap32s(vq->vdev, &uelem->len);
347     address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
348     address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
349 }
350 
351 /* Called within rcu_read_lock().  */
352 static uint16_t vring_used_idx(VirtQueue *vq)
353 {
354     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
355     hwaddr pa = offsetof(VRingUsed, idx);
356 
357     if (!caches) {
358         return 0;
359     }
360 
361     return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
362 }
363 
364 /* Called within rcu_read_lock().  */
365 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
366 {
367     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
368     hwaddr pa = offsetof(VRingUsed, idx);
369 
370     if (caches) {
371         virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
372         address_space_cache_invalidate(&caches->used, pa, sizeof(val));
373     }
374 
375     vq->used_idx = val;
376 }
377 
378 /* Called within rcu_read_lock().  */
379 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
380 {
381     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
382     VirtIODevice *vdev = vq->vdev;
383     hwaddr pa = offsetof(VRingUsed, flags);
384     uint16_t flags;
385 
386     if (!caches) {
387         return;
388     }
389 
390     flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
391     virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask);
392     address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
393 }
394 
395 /* Called within rcu_read_lock().  */
396 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
397 {
398     VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
399     VirtIODevice *vdev = vq->vdev;
400     hwaddr pa = offsetof(VRingUsed, flags);
401     uint16_t flags;
402 
403     if (!caches) {
404         return;
405     }
406 
407     flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
408     virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
409     address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
410 }
411 
412 /* Called within rcu_read_lock().  */
413 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
414 {
415     VRingMemoryRegionCaches *caches;
416     hwaddr pa;
417     if (!vq->notification) {
418         return;
419     }
420 
421     caches = vring_get_region_caches(vq);
422     if (!caches) {
423         return;
424     }
425 
426     pa = offsetof(VRingUsed, ring[vq->vring.num]);
427     virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
428     address_space_cache_invalidate(&caches->used, pa, sizeof(val));
429 }
430 
431 static void virtio_queue_split_set_notification(VirtQueue *vq, int enable)
432 {
433     RCU_READ_LOCK_GUARD();
434 
435     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
436         vring_set_avail_event(vq, vring_avail_idx(vq));
437     } else if (enable) {
438         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
439     } else {
440         vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
441     }
442     if (enable) {
443         /* Expose avail event/used flags before caller checks the avail idx. */
444         smp_mb();
445     }
446 }
447 
448 static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable)
449 {
450     uint16_t off_wrap;
451     VRingPackedDescEvent e;
452     VRingMemoryRegionCaches *caches;
453 
454     RCU_READ_LOCK_GUARD();
455     caches = vring_get_region_caches(vq);
456     if (!caches) {
457         return;
458     }
459 
460     vring_packed_event_read(vq->vdev, &caches->used, &e);
461 
462     if (!enable) {
463         e.flags = VRING_PACKED_EVENT_FLAG_DISABLE;
464     } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
465         off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15;
466         vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap);
467         /* Make sure off_wrap is wrote before flags */
468         smp_wmb();
469         e.flags = VRING_PACKED_EVENT_FLAG_DESC;
470     } else {
471         e.flags = VRING_PACKED_EVENT_FLAG_ENABLE;
472     }
473 
474     vring_packed_flags_write(vq->vdev, &caches->used, e.flags);
475     if (enable) {
476         /* Expose avail event/used flags before caller checks the avail idx. */
477         smp_mb();
478     }
479 }
480 
481 bool virtio_queue_get_notification(VirtQueue *vq)
482 {
483     return vq->notification;
484 }
485 
486 void virtio_queue_set_notification(VirtQueue *vq, int enable)
487 {
488     vq->notification = enable;
489 
490     if (!vq->vring.desc) {
491         return;
492     }
493 
494     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
495         virtio_queue_packed_set_notification(vq, enable);
496     } else {
497         virtio_queue_split_set_notification(vq, enable);
498     }
499 }
500 
501 int virtio_queue_ready(VirtQueue *vq)
502 {
503     return vq->vring.avail != 0;
504 }
505 
506 static void vring_packed_desc_read_flags(VirtIODevice *vdev,
507                                          uint16_t *flags,
508                                          MemoryRegionCache *cache,
509                                          int i)
510 {
511     address_space_read_cached(cache,
512                               i * sizeof(VRingPackedDesc) +
513                               offsetof(VRingPackedDesc, flags),
514                               flags, sizeof(*flags));
515     virtio_tswap16s(vdev, flags);
516 }
517 
518 static void vring_packed_desc_read(VirtIODevice *vdev,
519                                    VRingPackedDesc *desc,
520                                    MemoryRegionCache *cache,
521                                    int i, bool strict_order)
522 {
523     hwaddr off = i * sizeof(VRingPackedDesc);
524 
525     vring_packed_desc_read_flags(vdev, &desc->flags, cache, i);
526 
527     if (strict_order) {
528         /* Make sure flags is read before the rest fields. */
529         smp_rmb();
530     }
531 
532     address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr),
533                               &desc->addr, sizeof(desc->addr));
534     address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id),
535                               &desc->id, sizeof(desc->id));
536     address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len),
537                               &desc->len, sizeof(desc->len));
538     virtio_tswap64s(vdev, &desc->addr);
539     virtio_tswap16s(vdev, &desc->id);
540     virtio_tswap32s(vdev, &desc->len);
541 }
542 
543 static void vring_packed_desc_write_data(VirtIODevice *vdev,
544                                          VRingPackedDesc *desc,
545                                          MemoryRegionCache *cache,
546                                          int i)
547 {
548     hwaddr off_id = i * sizeof(VRingPackedDesc) +
549                     offsetof(VRingPackedDesc, id);
550     hwaddr off_len = i * sizeof(VRingPackedDesc) +
551                     offsetof(VRingPackedDesc, len);
552 
553     virtio_tswap32s(vdev, &desc->len);
554     virtio_tswap16s(vdev, &desc->id);
555     address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id));
556     address_space_cache_invalidate(cache, off_id, sizeof(desc->id));
557     address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len));
558     address_space_cache_invalidate(cache, off_len, sizeof(desc->len));
559 }
560 
561 static void vring_packed_desc_write_flags(VirtIODevice *vdev,
562                                           VRingPackedDesc *desc,
563                                           MemoryRegionCache *cache,
564                                           int i)
565 {
566     hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
567 
568     virtio_tswap16s(vdev, &desc->flags);
569     address_space_write_cached(cache, off, &desc->flags, sizeof(desc->flags));
570     address_space_cache_invalidate(cache, off, sizeof(desc->flags));
571 }
572 
573 static void vring_packed_desc_write(VirtIODevice *vdev,
574                                     VRingPackedDesc *desc,
575                                     MemoryRegionCache *cache,
576                                     int i, bool strict_order)
577 {
578     vring_packed_desc_write_data(vdev, desc, cache, i);
579     if (strict_order) {
580         /* Make sure data is wrote before flags. */
581         smp_wmb();
582     }
583     vring_packed_desc_write_flags(vdev, desc, cache, i);
584 }
585 
586 static inline bool is_desc_avail(uint16_t flags, bool wrap_counter)
587 {
588     bool avail, used;
589 
590     avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
591     used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
592     return (avail != used) && (avail == wrap_counter);
593 }
594 
595 /* Fetch avail_idx from VQ memory only when we really need to know if
596  * guest has added some buffers.
597  * Called within rcu_read_lock().  */
598 static int virtio_queue_empty_rcu(VirtQueue *vq)
599 {
600     if (virtio_device_disabled(vq->vdev)) {
601         return 1;
602     }
603 
604     if (unlikely(!vq->vring.avail)) {
605         return 1;
606     }
607 
608     if (vq->shadow_avail_idx != vq->last_avail_idx) {
609         return 0;
610     }
611 
612     return vring_avail_idx(vq) == vq->last_avail_idx;
613 }
614 
615 static int virtio_queue_split_empty(VirtQueue *vq)
616 {
617     bool empty;
618 
619     if (virtio_device_disabled(vq->vdev)) {
620         return 1;
621     }
622 
623     if (unlikely(!vq->vring.avail)) {
624         return 1;
625     }
626 
627     if (vq->shadow_avail_idx != vq->last_avail_idx) {
628         return 0;
629     }
630 
631     RCU_READ_LOCK_GUARD();
632     empty = vring_avail_idx(vq) == vq->last_avail_idx;
633     return empty;
634 }
635 
636 static int virtio_queue_packed_empty_rcu(VirtQueue *vq)
637 {
638     struct VRingPackedDesc desc;
639     VRingMemoryRegionCaches *cache;
640 
641     if (unlikely(!vq->vring.desc)) {
642         return 1;
643     }
644 
645     cache = vring_get_region_caches(vq);
646     if (!cache) {
647         return 1;
648     }
649 
650     vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc,
651                                  vq->last_avail_idx);
652 
653     return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter);
654 }
655 
656 static int virtio_queue_packed_empty(VirtQueue *vq)
657 {
658     RCU_READ_LOCK_GUARD();
659     return virtio_queue_packed_empty_rcu(vq);
660 }
661 
662 int virtio_queue_empty(VirtQueue *vq)
663 {
664     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
665         return virtio_queue_packed_empty(vq);
666     } else {
667         return virtio_queue_split_empty(vq);
668     }
669 }
670 
671 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
672                                unsigned int len)
673 {
674     AddressSpace *dma_as = vq->vdev->dma_as;
675     unsigned int offset;
676     int i;
677 
678     offset = 0;
679     for (i = 0; i < elem->in_num; i++) {
680         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
681 
682         dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
683                          elem->in_sg[i].iov_len,
684                          DMA_DIRECTION_FROM_DEVICE, size);
685 
686         offset += size;
687     }
688 
689     for (i = 0; i < elem->out_num; i++)
690         dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
691                          elem->out_sg[i].iov_len,
692                          DMA_DIRECTION_TO_DEVICE,
693                          elem->out_sg[i].iov_len);
694 }
695 
696 /* virtqueue_detach_element:
697  * @vq: The #VirtQueue
698  * @elem: The #VirtQueueElement
699  * @len: number of bytes written
700  *
701  * Detach the element from the virtqueue.  This function is suitable for device
702  * reset or other situations where a #VirtQueueElement is simply freed and will
703  * not be pushed or discarded.
704  */
705 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,
706                               unsigned int len)
707 {
708     vq->inuse -= elem->ndescs;
709     virtqueue_unmap_sg(vq, elem, len);
710 }
711 
712 static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num)
713 {
714     vq->last_avail_idx -= num;
715 }
716 
717 static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num)
718 {
719     if (vq->last_avail_idx < num) {
720         vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num;
721         vq->last_avail_wrap_counter ^= 1;
722     } else {
723         vq->last_avail_idx -= num;
724     }
725 }
726 
727 /* virtqueue_unpop:
728  * @vq: The #VirtQueue
729  * @elem: The #VirtQueueElement
730  * @len: number of bytes written
731  *
732  * Pretend the most recent element wasn't popped from the virtqueue.  The next
733  * call to virtqueue_pop() will refetch the element.
734  */
735 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,
736                      unsigned int len)
737 {
738 
739     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
740         virtqueue_packed_rewind(vq, 1);
741     } else {
742         virtqueue_split_rewind(vq, 1);
743     }
744 
745     virtqueue_detach_element(vq, elem, len);
746 }
747 
748 /* virtqueue_rewind:
749  * @vq: The #VirtQueue
750  * @num: Number of elements to push back
751  *
752  * Pretend that elements weren't popped from the virtqueue.  The next
753  * virtqueue_pop() will refetch the oldest element.
754  *
755  * Use virtqueue_unpop() instead if you have a VirtQueueElement.
756  *
757  * Returns: true on success, false if @num is greater than the number of in use
758  * elements.
759  */
760 bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
761 {
762     if (num > vq->inuse) {
763         return false;
764     }
765 
766     vq->inuse -= num;
767     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
768         virtqueue_packed_rewind(vq, num);
769     } else {
770         virtqueue_split_rewind(vq, num);
771     }
772     return true;
773 }
774 
775 static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem,
776                     unsigned int len, unsigned int idx)
777 {
778     VRingUsedElem uelem;
779 
780     if (unlikely(!vq->vring.used)) {
781         return;
782     }
783 
784     idx = (idx + vq->used_idx) % vq->vring.num;
785 
786     uelem.id = elem->index;
787     uelem.len = len;
788     vring_used_write(vq, &uelem, idx);
789 }
790 
791 static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem,
792                                   unsigned int len, unsigned int idx)
793 {
794     vq->used_elems[idx].index = elem->index;
795     vq->used_elems[idx].len = len;
796     vq->used_elems[idx].ndescs = elem->ndescs;
797 }
798 
799 static void virtqueue_packed_fill_desc(VirtQueue *vq,
800                                        const VirtQueueElement *elem,
801                                        unsigned int idx,
802                                        bool strict_order)
803 {
804     uint16_t head;
805     VRingMemoryRegionCaches *caches;
806     VRingPackedDesc desc = {
807         .id = elem->index,
808         .len = elem->len,
809     };
810     bool wrap_counter = vq->used_wrap_counter;
811 
812     if (unlikely(!vq->vring.desc)) {
813         return;
814     }
815 
816     head = vq->used_idx + idx;
817     if (head >= vq->vring.num) {
818         head -= vq->vring.num;
819         wrap_counter ^= 1;
820     }
821     if (wrap_counter) {
822         desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL);
823         desc.flags |= (1 << VRING_PACKED_DESC_F_USED);
824     } else {
825         desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL);
826         desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED);
827     }
828 
829     caches = vring_get_region_caches(vq);
830     if (!caches) {
831         return;
832     }
833 
834     vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order);
835 }
836 
837 /* Called within rcu_read_lock().  */
838 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
839                     unsigned int len, unsigned int idx)
840 {
841     trace_virtqueue_fill(vq, elem, len, idx);
842 
843     virtqueue_unmap_sg(vq, elem, len);
844 
845     if (virtio_device_disabled(vq->vdev)) {
846         return;
847     }
848 
849     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
850         virtqueue_packed_fill(vq, elem, len, idx);
851     } else {
852         virtqueue_split_fill(vq, elem, len, idx);
853     }
854 }
855 
856 /* Called within rcu_read_lock().  */
857 static void virtqueue_split_flush(VirtQueue *vq, unsigned int count)
858 {
859     uint16_t old, new;
860 
861     if (unlikely(!vq->vring.used)) {
862         return;
863     }
864 
865     /* Make sure buffer is written before we update index. */
866     smp_wmb();
867     trace_virtqueue_flush(vq, count);
868     old = vq->used_idx;
869     new = old + count;
870     vring_used_idx_set(vq, new);
871     vq->inuse -= count;
872     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
873         vq->signalled_used_valid = false;
874 }
875 
876 static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count)
877 {
878     unsigned int i, ndescs = 0;
879 
880     if (unlikely(!vq->vring.desc)) {
881         return;
882     }
883 
884     for (i = 1; i < count; i++) {
885         virtqueue_packed_fill_desc(vq, &vq->used_elems[i], i, false);
886         ndescs += vq->used_elems[i].ndescs;
887     }
888     virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true);
889     ndescs += vq->used_elems[0].ndescs;
890 
891     vq->inuse -= ndescs;
892     vq->used_idx += ndescs;
893     if (vq->used_idx >= vq->vring.num) {
894         vq->used_idx -= vq->vring.num;
895         vq->used_wrap_counter ^= 1;
896     }
897 }
898 
899 void virtqueue_flush(VirtQueue *vq, unsigned int count)
900 {
901     if (virtio_device_disabled(vq->vdev)) {
902         vq->inuse -= count;
903         return;
904     }
905 
906     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
907         virtqueue_packed_flush(vq, count);
908     } else {
909         virtqueue_split_flush(vq, count);
910     }
911 }
912 
913 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
914                     unsigned int len)
915 {
916     RCU_READ_LOCK_GUARD();
917     virtqueue_fill(vq, elem, len, 0);
918     virtqueue_flush(vq, 1);
919 }
920 
921 /* Called within rcu_read_lock().  */
922 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
923 {
924     uint16_t num_heads = vring_avail_idx(vq) - idx;
925 
926     /* Check it isn't doing very strange things with descriptor numbers. */
927     if (num_heads > vq->vring.num) {
928         virtio_error(vq->vdev, "Guest moved used index from %u to %u",
929                      idx, vq->shadow_avail_idx);
930         return -EINVAL;
931     }
932     /* On success, callers read a descriptor at vq->last_avail_idx.
933      * Make sure descriptor read does not bypass avail index read. */
934     if (num_heads) {
935         smp_rmb();
936     }
937 
938     return num_heads;
939 }
940 
941 /* Called within rcu_read_lock().  */
942 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
943                                unsigned int *head)
944 {
945     /* Grab the next descriptor number they're advertising, and increment
946      * the index we've seen. */
947     *head = vring_avail_ring(vq, idx % vq->vring.num);
948 
949     /* If their number is silly, that's a fatal mistake. */
950     if (*head >= vq->vring.num) {
951         virtio_error(vq->vdev, "Guest says index %u is available", *head);
952         return false;
953     }
954 
955     return true;
956 }
957 
958 enum {
959     VIRTQUEUE_READ_DESC_ERROR = -1,
960     VIRTQUEUE_READ_DESC_DONE = 0,   /* end of chain */
961     VIRTQUEUE_READ_DESC_MORE = 1,   /* more buffers in chain */
962 };
963 
964 static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
965                                           MemoryRegionCache *desc_cache,
966                                           unsigned int max, unsigned int *next)
967 {
968     /* If this descriptor says it doesn't chain, we're done. */
969     if (!(desc->flags & VRING_DESC_F_NEXT)) {
970         return VIRTQUEUE_READ_DESC_DONE;
971     }
972 
973     /* Check they're not leading us off end of descriptors. */
974     *next = desc->next;
975     /* Make sure compiler knows to grab that: we don't want it changing! */
976     smp_wmb();
977 
978     if (*next >= max) {
979         virtio_error(vdev, "Desc next is %u", *next);
980         return VIRTQUEUE_READ_DESC_ERROR;
981     }
982 
983     vring_split_desc_read(vdev, desc, desc_cache, *next);
984     return VIRTQUEUE_READ_DESC_MORE;
985 }
986 
987 static void virtqueue_split_get_avail_bytes(VirtQueue *vq,
988                             unsigned int *in_bytes, unsigned int *out_bytes,
989                             unsigned max_in_bytes, unsigned max_out_bytes)
990 {
991     VirtIODevice *vdev = vq->vdev;
992     unsigned int max, idx;
993     unsigned int total_bufs, in_total, out_total;
994     VRingMemoryRegionCaches *caches;
995     MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
996     int64_t len = 0;
997     int rc;
998 
999     RCU_READ_LOCK_GUARD();
1000 
1001     idx = vq->last_avail_idx;
1002     total_bufs = in_total = out_total = 0;
1003 
1004     max = vq->vring.num;
1005     caches = vring_get_region_caches(vq);
1006     if (!caches) {
1007         goto err;
1008     }
1009 
1010     while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
1011         MemoryRegionCache *desc_cache = &caches->desc;
1012         unsigned int num_bufs;
1013         VRingDesc desc;
1014         unsigned int i;
1015 
1016         num_bufs = total_bufs;
1017 
1018         if (!virtqueue_get_head(vq, idx++, &i)) {
1019             goto err;
1020         }
1021 
1022         vring_split_desc_read(vdev, &desc, desc_cache, i);
1023 
1024         if (desc.flags & VRING_DESC_F_INDIRECT) {
1025             if (!desc.len || (desc.len % sizeof(VRingDesc))) {
1026                 virtio_error(vdev, "Invalid size for indirect buffer table");
1027                 goto err;
1028             }
1029 
1030             /* If we've got too many, that implies a descriptor loop. */
1031             if (num_bufs >= max) {
1032                 virtio_error(vdev, "Looped descriptor");
1033                 goto err;
1034             }
1035 
1036             /* loop over the indirect descriptor table */
1037             len = address_space_cache_init(&indirect_desc_cache,
1038                                            vdev->dma_as,
1039                                            desc.addr, desc.len, false);
1040             desc_cache = &indirect_desc_cache;
1041             if (len < desc.len) {
1042                 virtio_error(vdev, "Cannot map indirect buffer");
1043                 goto err;
1044             }
1045 
1046             max = desc.len / sizeof(VRingDesc);
1047             num_bufs = i = 0;
1048             vring_split_desc_read(vdev, &desc, desc_cache, i);
1049         }
1050 
1051         do {
1052             /* If we've got too many, that implies a descriptor loop. */
1053             if (++num_bufs > max) {
1054                 virtio_error(vdev, "Looped descriptor");
1055                 goto err;
1056             }
1057 
1058             if (desc.flags & VRING_DESC_F_WRITE) {
1059                 in_total += desc.len;
1060             } else {
1061                 out_total += desc.len;
1062             }
1063             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1064                 goto done;
1065             }
1066 
1067             rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
1068         } while (rc == VIRTQUEUE_READ_DESC_MORE);
1069 
1070         if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1071             goto err;
1072         }
1073 
1074         if (desc_cache == &indirect_desc_cache) {
1075             address_space_cache_destroy(&indirect_desc_cache);
1076             total_bufs++;
1077         } else {
1078             total_bufs = num_bufs;
1079         }
1080     }
1081 
1082     if (rc < 0) {
1083         goto err;
1084     }
1085 
1086 done:
1087     address_space_cache_destroy(&indirect_desc_cache);
1088     if (in_bytes) {
1089         *in_bytes = in_total;
1090     }
1091     if (out_bytes) {
1092         *out_bytes = out_total;
1093     }
1094     return;
1095 
1096 err:
1097     in_total = out_total = 0;
1098     goto done;
1099 }
1100 
1101 static int virtqueue_packed_read_next_desc(VirtQueue *vq,
1102                                            VRingPackedDesc *desc,
1103                                            MemoryRegionCache
1104                                            *desc_cache,
1105                                            unsigned int max,
1106                                            unsigned int *next,
1107                                            bool indirect)
1108 {
1109     /* If this descriptor says it doesn't chain, we're done. */
1110     if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) {
1111         return VIRTQUEUE_READ_DESC_DONE;
1112     }
1113 
1114     ++*next;
1115     if (*next == max) {
1116         if (indirect) {
1117             return VIRTQUEUE_READ_DESC_DONE;
1118         } else {
1119             (*next) -= vq->vring.num;
1120         }
1121     }
1122 
1123     vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false);
1124     return VIRTQUEUE_READ_DESC_MORE;
1125 }
1126 
1127 static void virtqueue_packed_get_avail_bytes(VirtQueue *vq,
1128                                              unsigned int *in_bytes,
1129                                              unsigned int *out_bytes,
1130                                              unsigned max_in_bytes,
1131                                              unsigned max_out_bytes)
1132 {
1133     VirtIODevice *vdev = vq->vdev;
1134     unsigned int max, idx;
1135     unsigned int total_bufs, in_total, out_total;
1136     MemoryRegionCache *desc_cache;
1137     VRingMemoryRegionCaches *caches;
1138     MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1139     int64_t len = 0;
1140     VRingPackedDesc desc;
1141     bool wrap_counter;
1142 
1143     RCU_READ_LOCK_GUARD();
1144     idx = vq->last_avail_idx;
1145     wrap_counter = vq->last_avail_wrap_counter;
1146     total_bufs = in_total = out_total = 0;
1147 
1148     max = vq->vring.num;
1149     caches = vring_get_region_caches(vq);
1150     if (!caches) {
1151         goto err;
1152     }
1153 
1154     for (;;) {
1155         unsigned int num_bufs = total_bufs;
1156         unsigned int i = idx;
1157         int rc;
1158 
1159         desc_cache = &caches->desc;
1160         vring_packed_desc_read(vdev, &desc, desc_cache, idx, true);
1161         if (!is_desc_avail(desc.flags, wrap_counter)) {
1162             break;
1163         }
1164 
1165         if (desc.flags & VRING_DESC_F_INDIRECT) {
1166             if (desc.len % sizeof(VRingPackedDesc)) {
1167                 virtio_error(vdev, "Invalid size for indirect buffer table");
1168                 goto err;
1169             }
1170 
1171             /* If we've got too many, that implies a descriptor loop. */
1172             if (num_bufs >= max) {
1173                 virtio_error(vdev, "Looped descriptor");
1174                 goto err;
1175             }
1176 
1177             /* loop over the indirect descriptor table */
1178             len = address_space_cache_init(&indirect_desc_cache,
1179                                            vdev->dma_as,
1180                                            desc.addr, desc.len, false);
1181             desc_cache = &indirect_desc_cache;
1182             if (len < desc.len) {
1183                 virtio_error(vdev, "Cannot map indirect buffer");
1184                 goto err;
1185             }
1186 
1187             max = desc.len / sizeof(VRingPackedDesc);
1188             num_bufs = i = 0;
1189             vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1190         }
1191 
1192         do {
1193             /* If we've got too many, that implies a descriptor loop. */
1194             if (++num_bufs > max) {
1195                 virtio_error(vdev, "Looped descriptor");
1196                 goto err;
1197             }
1198 
1199             if (desc.flags & VRING_DESC_F_WRITE) {
1200                 in_total += desc.len;
1201             } else {
1202                 out_total += desc.len;
1203             }
1204             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1205                 goto done;
1206             }
1207 
1208             rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max,
1209                                                  &i, desc_cache ==
1210                                                  &indirect_desc_cache);
1211         } while (rc == VIRTQUEUE_READ_DESC_MORE);
1212 
1213         if (desc_cache == &indirect_desc_cache) {
1214             address_space_cache_destroy(&indirect_desc_cache);
1215             total_bufs++;
1216             idx++;
1217         } else {
1218             idx += num_bufs - total_bufs;
1219             total_bufs = num_bufs;
1220         }
1221 
1222         if (idx >= vq->vring.num) {
1223             idx -= vq->vring.num;
1224             wrap_counter ^= 1;
1225         }
1226     }
1227 
1228     /* Record the index and wrap counter for a kick we want */
1229     vq->shadow_avail_idx = idx;
1230     vq->shadow_avail_wrap_counter = wrap_counter;
1231 done:
1232     address_space_cache_destroy(&indirect_desc_cache);
1233     if (in_bytes) {
1234         *in_bytes = in_total;
1235     }
1236     if (out_bytes) {
1237         *out_bytes = out_total;
1238     }
1239     return;
1240 
1241 err:
1242     in_total = out_total = 0;
1243     goto done;
1244 }
1245 
1246 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
1247                                unsigned int *out_bytes,
1248                                unsigned max_in_bytes, unsigned max_out_bytes)
1249 {
1250     uint16_t desc_size;
1251     VRingMemoryRegionCaches *caches;
1252 
1253     if (unlikely(!vq->vring.desc)) {
1254         goto err;
1255     }
1256 
1257     caches = vring_get_region_caches(vq);
1258     if (!caches) {
1259         goto err;
1260     }
1261 
1262     desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
1263                                 sizeof(VRingPackedDesc) : sizeof(VRingDesc);
1264     if (caches->desc.len < vq->vring.num * desc_size) {
1265         virtio_error(vq->vdev, "Cannot map descriptor ring");
1266         goto err;
1267     }
1268 
1269     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1270         virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes,
1271                                          max_in_bytes, max_out_bytes);
1272     } else {
1273         virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes,
1274                                         max_in_bytes, max_out_bytes);
1275     }
1276 
1277     return;
1278 err:
1279     if (in_bytes) {
1280         *in_bytes = 0;
1281     }
1282     if (out_bytes) {
1283         *out_bytes = 0;
1284     }
1285 }
1286 
1287 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
1288                           unsigned int out_bytes)
1289 {
1290     unsigned int in_total, out_total;
1291 
1292     virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
1293     return in_bytes <= in_total && out_bytes <= out_total;
1294 }
1295 
1296 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
1297                                hwaddr *addr, struct iovec *iov,
1298                                unsigned int max_num_sg, bool is_write,
1299                                hwaddr pa, size_t sz)
1300 {
1301     bool ok = false;
1302     unsigned num_sg = *p_num_sg;
1303     assert(num_sg <= max_num_sg);
1304 
1305     if (!sz) {
1306         virtio_error(vdev, "virtio: zero sized buffers are not allowed");
1307         goto out;
1308     }
1309 
1310     while (sz) {
1311         hwaddr len = sz;
1312 
1313         if (num_sg == max_num_sg) {
1314             virtio_error(vdev, "virtio: too many write descriptors in "
1315                                "indirect table");
1316             goto out;
1317         }
1318 
1319         iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
1320                                               is_write ?
1321                                               DMA_DIRECTION_FROM_DEVICE :
1322                                               DMA_DIRECTION_TO_DEVICE);
1323         if (!iov[num_sg].iov_base) {
1324             virtio_error(vdev, "virtio: bogus descriptor or out of resources");
1325             goto out;
1326         }
1327 
1328         iov[num_sg].iov_len = len;
1329         addr[num_sg] = pa;
1330 
1331         sz -= len;
1332         pa += len;
1333         num_sg++;
1334     }
1335     ok = true;
1336 
1337 out:
1338     *p_num_sg = num_sg;
1339     return ok;
1340 }
1341 
1342 /* Only used by error code paths before we have a VirtQueueElement (therefore
1343  * virtqueue_unmap_sg() can't be used).  Assumes buffers weren't written to
1344  * yet.
1345  */
1346 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num,
1347                                     struct iovec *iov)
1348 {
1349     unsigned int i;
1350 
1351     for (i = 0; i < out_num + in_num; i++) {
1352         int is_write = i >= out_num;
1353 
1354         cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0);
1355         iov++;
1356     }
1357 }
1358 
1359 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
1360                                 hwaddr *addr, unsigned int num_sg,
1361                                 bool is_write)
1362 {
1363     unsigned int i;
1364     hwaddr len;
1365 
1366     for (i = 0; i < num_sg; i++) {
1367         len = sg[i].iov_len;
1368         sg[i].iov_base = dma_memory_map(vdev->dma_as,
1369                                         addr[i], &len, is_write ?
1370                                         DMA_DIRECTION_FROM_DEVICE :
1371                                         DMA_DIRECTION_TO_DEVICE);
1372         if (!sg[i].iov_base) {
1373             error_report("virtio: error trying to map MMIO memory");
1374             exit(1);
1375         }
1376         if (len != sg[i].iov_len) {
1377             error_report("virtio: unexpected memory split");
1378             exit(1);
1379         }
1380     }
1381 }
1382 
1383 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem)
1384 {
1385     virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true);
1386     virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num,
1387                                                                         false);
1388 }
1389 
1390 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
1391 {
1392     VirtQueueElement *elem;
1393     size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
1394     size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
1395     size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
1396     size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
1397     size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
1398     size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
1399 
1400     assert(sz >= sizeof(VirtQueueElement));
1401     elem = g_malloc(out_sg_end);
1402     trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
1403     elem->out_num = out_num;
1404     elem->in_num = in_num;
1405     elem->in_addr = (void *)elem + in_addr_ofs;
1406     elem->out_addr = (void *)elem + out_addr_ofs;
1407     elem->in_sg = (void *)elem + in_sg_ofs;
1408     elem->out_sg = (void *)elem + out_sg_ofs;
1409     return elem;
1410 }
1411 
1412 static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)
1413 {
1414     unsigned int i, head, max;
1415     VRingMemoryRegionCaches *caches;
1416     MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1417     MemoryRegionCache *desc_cache;
1418     int64_t len;
1419     VirtIODevice *vdev = vq->vdev;
1420     VirtQueueElement *elem = NULL;
1421     unsigned out_num, in_num, elem_entries;
1422     hwaddr addr[VIRTQUEUE_MAX_SIZE];
1423     struct iovec iov[VIRTQUEUE_MAX_SIZE];
1424     VRingDesc desc;
1425     int rc;
1426 
1427     RCU_READ_LOCK_GUARD();
1428     if (virtio_queue_empty_rcu(vq)) {
1429         goto done;
1430     }
1431     /* Needed after virtio_queue_empty(), see comment in
1432      * virtqueue_num_heads(). */
1433     smp_rmb();
1434 
1435     /* When we start there are none of either input nor output. */
1436     out_num = in_num = elem_entries = 0;
1437 
1438     max = vq->vring.num;
1439 
1440     if (vq->inuse >= vq->vring.num) {
1441         virtio_error(vdev, "Virtqueue size exceeded");
1442         goto done;
1443     }
1444 
1445     if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
1446         goto done;
1447     }
1448 
1449     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1450         vring_set_avail_event(vq, vq->last_avail_idx);
1451     }
1452 
1453     i = head;
1454 
1455     caches = vring_get_region_caches(vq);
1456     if (!caches) {
1457         virtio_error(vdev, "Region caches not initialized");
1458         goto done;
1459     }
1460 
1461     if (caches->desc.len < max * sizeof(VRingDesc)) {
1462         virtio_error(vdev, "Cannot map descriptor ring");
1463         goto done;
1464     }
1465 
1466     desc_cache = &caches->desc;
1467     vring_split_desc_read(vdev, &desc, desc_cache, i);
1468     if (desc.flags & VRING_DESC_F_INDIRECT) {
1469         if (!desc.len || (desc.len % sizeof(VRingDesc))) {
1470             virtio_error(vdev, "Invalid size for indirect buffer table");
1471             goto done;
1472         }
1473 
1474         /* loop over the indirect descriptor table */
1475         len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1476                                        desc.addr, desc.len, false);
1477         desc_cache = &indirect_desc_cache;
1478         if (len < desc.len) {
1479             virtio_error(vdev, "Cannot map indirect buffer");
1480             goto done;
1481         }
1482 
1483         max = desc.len / sizeof(VRingDesc);
1484         i = 0;
1485         vring_split_desc_read(vdev, &desc, desc_cache, i);
1486     }
1487 
1488     /* Collect all the descriptors */
1489     do {
1490         bool map_ok;
1491 
1492         if (desc.flags & VRING_DESC_F_WRITE) {
1493             map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1494                                         iov + out_num,
1495                                         VIRTQUEUE_MAX_SIZE - out_num, true,
1496                                         desc.addr, desc.len);
1497         } else {
1498             if (in_num) {
1499                 virtio_error(vdev, "Incorrect order for descriptors");
1500                 goto err_undo_map;
1501             }
1502             map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1503                                         VIRTQUEUE_MAX_SIZE, false,
1504                                         desc.addr, desc.len);
1505         }
1506         if (!map_ok) {
1507             goto err_undo_map;
1508         }
1509 
1510         /* If we've got too many, that implies a descriptor loop. */
1511         if (++elem_entries > max) {
1512             virtio_error(vdev, "Looped descriptor");
1513             goto err_undo_map;
1514         }
1515 
1516         rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
1517     } while (rc == VIRTQUEUE_READ_DESC_MORE);
1518 
1519     if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1520         goto err_undo_map;
1521     }
1522 
1523     /* Now copy what we have collected and mapped */
1524     elem = virtqueue_alloc_element(sz, out_num, in_num);
1525     elem->index = head;
1526     elem->ndescs = 1;
1527     for (i = 0; i < out_num; i++) {
1528         elem->out_addr[i] = addr[i];
1529         elem->out_sg[i] = iov[i];
1530     }
1531     for (i = 0; i < in_num; i++) {
1532         elem->in_addr[i] = addr[out_num + i];
1533         elem->in_sg[i] = iov[out_num + i];
1534     }
1535 
1536     vq->inuse++;
1537 
1538     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
1539 done:
1540     address_space_cache_destroy(&indirect_desc_cache);
1541 
1542     return elem;
1543 
1544 err_undo_map:
1545     virtqueue_undo_map_desc(out_num, in_num, iov);
1546     goto done;
1547 }
1548 
1549 static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz)
1550 {
1551     unsigned int i, max;
1552     VRingMemoryRegionCaches *caches;
1553     MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1554     MemoryRegionCache *desc_cache;
1555     int64_t len;
1556     VirtIODevice *vdev = vq->vdev;
1557     VirtQueueElement *elem = NULL;
1558     unsigned out_num, in_num, elem_entries;
1559     hwaddr addr[VIRTQUEUE_MAX_SIZE];
1560     struct iovec iov[VIRTQUEUE_MAX_SIZE];
1561     VRingPackedDesc desc;
1562     uint16_t id;
1563     int rc;
1564 
1565     RCU_READ_LOCK_GUARD();
1566     if (virtio_queue_packed_empty_rcu(vq)) {
1567         goto done;
1568     }
1569 
1570     /* When we start there are none of either input nor output. */
1571     out_num = in_num = elem_entries = 0;
1572 
1573     max = vq->vring.num;
1574 
1575     if (vq->inuse >= vq->vring.num) {
1576         virtio_error(vdev, "Virtqueue size exceeded");
1577         goto done;
1578     }
1579 
1580     i = vq->last_avail_idx;
1581 
1582     caches = vring_get_region_caches(vq);
1583     if (!caches) {
1584         virtio_error(vdev, "Region caches not initialized");
1585         goto done;
1586     }
1587 
1588     if (caches->desc.len < max * sizeof(VRingDesc)) {
1589         virtio_error(vdev, "Cannot map descriptor ring");
1590         goto done;
1591     }
1592 
1593     desc_cache = &caches->desc;
1594     vring_packed_desc_read(vdev, &desc, desc_cache, i, true);
1595     id = desc.id;
1596     if (desc.flags & VRING_DESC_F_INDIRECT) {
1597         if (desc.len % sizeof(VRingPackedDesc)) {
1598             virtio_error(vdev, "Invalid size for indirect buffer table");
1599             goto done;
1600         }
1601 
1602         /* loop over the indirect descriptor table */
1603         len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1604                                        desc.addr, desc.len, false);
1605         desc_cache = &indirect_desc_cache;
1606         if (len < desc.len) {
1607             virtio_error(vdev, "Cannot map indirect buffer");
1608             goto done;
1609         }
1610 
1611         max = desc.len / sizeof(VRingPackedDesc);
1612         i = 0;
1613         vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1614     }
1615 
1616     /* Collect all the descriptors */
1617     do {
1618         bool map_ok;
1619 
1620         if (desc.flags & VRING_DESC_F_WRITE) {
1621             map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1622                                         iov + out_num,
1623                                         VIRTQUEUE_MAX_SIZE - out_num, true,
1624                                         desc.addr, desc.len);
1625         } else {
1626             if (in_num) {
1627                 virtio_error(vdev, "Incorrect order for descriptors");
1628                 goto err_undo_map;
1629             }
1630             map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1631                                         VIRTQUEUE_MAX_SIZE, false,
1632                                         desc.addr, desc.len);
1633         }
1634         if (!map_ok) {
1635             goto err_undo_map;
1636         }
1637 
1638         /* If we've got too many, that implies a descriptor loop. */
1639         if (++elem_entries > max) {
1640             virtio_error(vdev, "Looped descriptor");
1641             goto err_undo_map;
1642         }
1643 
1644         rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i,
1645                                              desc_cache ==
1646                                              &indirect_desc_cache);
1647     } while (rc == VIRTQUEUE_READ_DESC_MORE);
1648 
1649     /* Now copy what we have collected and mapped */
1650     elem = virtqueue_alloc_element(sz, out_num, in_num);
1651     for (i = 0; i < out_num; i++) {
1652         elem->out_addr[i] = addr[i];
1653         elem->out_sg[i] = iov[i];
1654     }
1655     for (i = 0; i < in_num; i++) {
1656         elem->in_addr[i] = addr[out_num + i];
1657         elem->in_sg[i] = iov[out_num + i];
1658     }
1659 
1660     elem->index = id;
1661     elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries;
1662     vq->last_avail_idx += elem->ndescs;
1663     vq->inuse += elem->ndescs;
1664 
1665     if (vq->last_avail_idx >= vq->vring.num) {
1666         vq->last_avail_idx -= vq->vring.num;
1667         vq->last_avail_wrap_counter ^= 1;
1668     }
1669 
1670     vq->shadow_avail_idx = vq->last_avail_idx;
1671     vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter;
1672 
1673     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
1674 done:
1675     address_space_cache_destroy(&indirect_desc_cache);
1676 
1677     return elem;
1678 
1679 err_undo_map:
1680     virtqueue_undo_map_desc(out_num, in_num, iov);
1681     goto done;
1682 }
1683 
1684 void *virtqueue_pop(VirtQueue *vq, size_t sz)
1685 {
1686     if (virtio_device_disabled(vq->vdev)) {
1687         return NULL;
1688     }
1689 
1690     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1691         return virtqueue_packed_pop(vq, sz);
1692     } else {
1693         return virtqueue_split_pop(vq, sz);
1694     }
1695 }
1696 
1697 static unsigned int virtqueue_packed_drop_all(VirtQueue *vq)
1698 {
1699     VRingMemoryRegionCaches *caches;
1700     MemoryRegionCache *desc_cache;
1701     unsigned int dropped = 0;
1702     VirtQueueElement elem = {};
1703     VirtIODevice *vdev = vq->vdev;
1704     VRingPackedDesc desc;
1705 
1706     caches = vring_get_region_caches(vq);
1707     if (!caches) {
1708         return 0;
1709     }
1710 
1711     desc_cache = &caches->desc;
1712 
1713     virtio_queue_set_notification(vq, 0);
1714 
1715     while (vq->inuse < vq->vring.num) {
1716         unsigned int idx = vq->last_avail_idx;
1717         /*
1718          * works similar to virtqueue_pop but does not map buffers
1719          * and does not allocate any memory.
1720          */
1721         vring_packed_desc_read(vdev, &desc, desc_cache,
1722                                vq->last_avail_idx , true);
1723         if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) {
1724             break;
1725         }
1726         elem.index = desc.id;
1727         elem.ndescs = 1;
1728         while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache,
1729                                                vq->vring.num, &idx, false)) {
1730             ++elem.ndescs;
1731         }
1732         /*
1733          * immediately push the element, nothing to unmap
1734          * as both in_num and out_num are set to 0.
1735          */
1736         virtqueue_push(vq, &elem, 0);
1737         dropped++;
1738         vq->last_avail_idx += elem.ndescs;
1739         if (vq->last_avail_idx >= vq->vring.num) {
1740             vq->last_avail_idx -= vq->vring.num;
1741             vq->last_avail_wrap_counter ^= 1;
1742         }
1743     }
1744 
1745     return dropped;
1746 }
1747 
1748 static unsigned int virtqueue_split_drop_all(VirtQueue *vq)
1749 {
1750     unsigned int dropped = 0;
1751     VirtQueueElement elem = {};
1752     VirtIODevice *vdev = vq->vdev;
1753     bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1754 
1755     while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
1756         /* works similar to virtqueue_pop but does not map buffers
1757         * and does not allocate any memory */
1758         smp_rmb();
1759         if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
1760             break;
1761         }
1762         vq->inuse++;
1763         vq->last_avail_idx++;
1764         if (fEventIdx) {
1765             vring_set_avail_event(vq, vq->last_avail_idx);
1766         }
1767         /* immediately push the element, nothing to unmap
1768          * as both in_num and out_num are set to 0 */
1769         virtqueue_push(vq, &elem, 0);
1770         dropped++;
1771     }
1772 
1773     return dropped;
1774 }
1775 
1776 /* virtqueue_drop_all:
1777  * @vq: The #VirtQueue
1778  * Drops all queued buffers and indicates them to the guest
1779  * as if they are done. Useful when buffers can not be
1780  * processed but must be returned to the guest.
1781  */
1782 unsigned int virtqueue_drop_all(VirtQueue *vq)
1783 {
1784     struct VirtIODevice *vdev = vq->vdev;
1785 
1786     if (virtio_device_disabled(vq->vdev)) {
1787         return 0;
1788     }
1789 
1790     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1791         return virtqueue_packed_drop_all(vq);
1792     } else {
1793         return virtqueue_split_drop_all(vq);
1794     }
1795 }
1796 
1797 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1798  * it is what QEMU has always done by mistake.  We can change it sooner
1799  * or later by bumping the version number of the affected vm states.
1800  * In the meanwhile, since the in-memory layout of VirtQueueElement
1801  * has changed, we need to marshal to and from the layout that was
1802  * used before the change.
1803  */
1804 typedef struct VirtQueueElementOld {
1805     unsigned int index;
1806     unsigned int out_num;
1807     unsigned int in_num;
1808     hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
1809     hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
1810     struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
1811     struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
1812 } VirtQueueElementOld;
1813 
1814 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz)
1815 {
1816     VirtQueueElement *elem;
1817     VirtQueueElementOld data;
1818     int i;
1819 
1820     qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1821 
1822     /* TODO: teach all callers that this can fail, and return failure instead
1823      * of asserting here.
1824      * This is just one thing (there are probably more) that must be
1825      * fixed before we can allow NDEBUG compilation.
1826      */
1827     assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
1828     assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
1829 
1830     elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
1831     elem->index = data.index;
1832 
1833     for (i = 0; i < elem->in_num; i++) {
1834         elem->in_addr[i] = data.in_addr[i];
1835     }
1836 
1837     for (i = 0; i < elem->out_num; i++) {
1838         elem->out_addr[i] = data.out_addr[i];
1839     }
1840 
1841     for (i = 0; i < elem->in_num; i++) {
1842         /* Base is overwritten by virtqueue_map.  */
1843         elem->in_sg[i].iov_base = 0;
1844         elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
1845     }
1846 
1847     for (i = 0; i < elem->out_num; i++) {
1848         /* Base is overwritten by virtqueue_map.  */
1849         elem->out_sg[i].iov_base = 0;
1850         elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
1851     }
1852 
1853     if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1854         qemu_get_be32s(f, &elem->ndescs);
1855     }
1856 
1857     virtqueue_map(vdev, elem);
1858     return elem;
1859 }
1860 
1861 void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f,
1862                                 VirtQueueElement *elem)
1863 {
1864     VirtQueueElementOld data;
1865     int i;
1866 
1867     memset(&data, 0, sizeof(data));
1868     data.index = elem->index;
1869     data.in_num = elem->in_num;
1870     data.out_num = elem->out_num;
1871 
1872     for (i = 0; i < elem->in_num; i++) {
1873         data.in_addr[i] = elem->in_addr[i];
1874     }
1875 
1876     for (i = 0; i < elem->out_num; i++) {
1877         data.out_addr[i] = elem->out_addr[i];
1878     }
1879 
1880     for (i = 0; i < elem->in_num; i++) {
1881         /* Base is overwritten by virtqueue_map when loading.  Do not
1882          * save it, as it would leak the QEMU address space layout.  */
1883         data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
1884     }
1885 
1886     for (i = 0; i < elem->out_num; i++) {
1887         /* Do not save iov_base as above.  */
1888         data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
1889     }
1890 
1891     if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1892         qemu_put_be32s(f, &elem->ndescs);
1893     }
1894 
1895     qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1896 }
1897 
1898 /* virtio device */
1899 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
1900 {
1901     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1902     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1903 
1904     if (virtio_device_disabled(vdev)) {
1905         return;
1906     }
1907 
1908     if (k->notify) {
1909         k->notify(qbus->parent, vector);
1910     }
1911 }
1912 
1913 void virtio_update_irq(VirtIODevice *vdev)
1914 {
1915     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1916 }
1917 
1918 static int virtio_validate_features(VirtIODevice *vdev)
1919 {
1920     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1921 
1922     if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
1923         !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1924         return -EFAULT;
1925     }
1926 
1927     if (k->validate_features) {
1928         return k->validate_features(vdev);
1929     } else {
1930         return 0;
1931     }
1932 }
1933 
1934 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
1935 {
1936     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1937     trace_virtio_set_status(vdev, val);
1938 
1939     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1940         if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
1941             val & VIRTIO_CONFIG_S_FEATURES_OK) {
1942             int ret = virtio_validate_features(vdev);
1943 
1944             if (ret) {
1945                 return ret;
1946             }
1947         }
1948     }
1949 
1950     if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) !=
1951         (val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1952         virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK);
1953     }
1954 
1955     if (k->set_status) {
1956         k->set_status(vdev, val);
1957     }
1958     vdev->status = val;
1959 
1960     return 0;
1961 }
1962 
1963 static enum virtio_device_endian virtio_default_endian(void)
1964 {
1965     if (target_words_bigendian()) {
1966         return VIRTIO_DEVICE_ENDIAN_BIG;
1967     } else {
1968         return VIRTIO_DEVICE_ENDIAN_LITTLE;
1969     }
1970 }
1971 
1972 static enum virtio_device_endian virtio_current_cpu_endian(void)
1973 {
1974     CPUClass *cc = CPU_GET_CLASS(current_cpu);
1975 
1976     if (cc->virtio_is_big_endian(current_cpu)) {
1977         return VIRTIO_DEVICE_ENDIAN_BIG;
1978     } else {
1979         return VIRTIO_DEVICE_ENDIAN_LITTLE;
1980     }
1981 }
1982 
1983 void virtio_reset(void *opaque)
1984 {
1985     VirtIODevice *vdev = opaque;
1986     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1987     int i;
1988 
1989     virtio_set_status(vdev, 0);
1990     if (current_cpu) {
1991         /* Guest initiated reset */
1992         vdev->device_endian = virtio_current_cpu_endian();
1993     } else {
1994         /* System reset */
1995         vdev->device_endian = virtio_default_endian();
1996     }
1997 
1998     if (k->reset) {
1999         k->reset(vdev);
2000     }
2001 
2002     vdev->start_on_kick = false;
2003     vdev->started = false;
2004     vdev->broken = false;
2005     vdev->guest_features = 0;
2006     vdev->queue_sel = 0;
2007     vdev->status = 0;
2008     vdev->disabled = false;
2009     atomic_set(&vdev->isr, 0);
2010     vdev->config_vector = VIRTIO_NO_VECTOR;
2011     virtio_notify_vector(vdev, vdev->config_vector);
2012 
2013     for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2014         vdev->vq[i].vring.desc = 0;
2015         vdev->vq[i].vring.avail = 0;
2016         vdev->vq[i].vring.used = 0;
2017         vdev->vq[i].last_avail_idx = 0;
2018         vdev->vq[i].shadow_avail_idx = 0;
2019         vdev->vq[i].used_idx = 0;
2020         vdev->vq[i].last_avail_wrap_counter = true;
2021         vdev->vq[i].shadow_avail_wrap_counter = true;
2022         vdev->vq[i].used_wrap_counter = true;
2023         virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
2024         vdev->vq[i].signalled_used = 0;
2025         vdev->vq[i].signalled_used_valid = false;
2026         vdev->vq[i].notification = true;
2027         vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
2028         vdev->vq[i].inuse = 0;
2029         virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
2030     }
2031 }
2032 
2033 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
2034 {
2035     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2036     uint8_t val;
2037 
2038     if (addr + sizeof(val) > vdev->config_len) {
2039         return (uint32_t)-1;
2040     }
2041 
2042     k->get_config(vdev, vdev->config);
2043 
2044     val = ldub_p(vdev->config + addr);
2045     return val;
2046 }
2047 
2048 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
2049 {
2050     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2051     uint16_t val;
2052 
2053     if (addr + sizeof(val) > vdev->config_len) {
2054         return (uint32_t)-1;
2055     }
2056 
2057     k->get_config(vdev, vdev->config);
2058 
2059     val = lduw_p(vdev->config + addr);
2060     return val;
2061 }
2062 
2063 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
2064 {
2065     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2066     uint32_t val;
2067 
2068     if (addr + sizeof(val) > vdev->config_len) {
2069         return (uint32_t)-1;
2070     }
2071 
2072     k->get_config(vdev, vdev->config);
2073 
2074     val = ldl_p(vdev->config + addr);
2075     return val;
2076 }
2077 
2078 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
2079 {
2080     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2081     uint8_t val = data;
2082 
2083     if (addr + sizeof(val) > vdev->config_len) {
2084         return;
2085     }
2086 
2087     stb_p(vdev->config + addr, val);
2088 
2089     if (k->set_config) {
2090         k->set_config(vdev, vdev->config);
2091     }
2092 }
2093 
2094 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
2095 {
2096     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2097     uint16_t val = data;
2098 
2099     if (addr + sizeof(val) > vdev->config_len) {
2100         return;
2101     }
2102 
2103     stw_p(vdev->config + addr, val);
2104 
2105     if (k->set_config) {
2106         k->set_config(vdev, vdev->config);
2107     }
2108 }
2109 
2110 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
2111 {
2112     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2113     uint32_t val = data;
2114 
2115     if (addr + sizeof(val) > vdev->config_len) {
2116         return;
2117     }
2118 
2119     stl_p(vdev->config + addr, val);
2120 
2121     if (k->set_config) {
2122         k->set_config(vdev, vdev->config);
2123     }
2124 }
2125 
2126 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
2127 {
2128     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2129     uint8_t val;
2130 
2131     if (addr + sizeof(val) > vdev->config_len) {
2132         return (uint32_t)-1;
2133     }
2134 
2135     k->get_config(vdev, vdev->config);
2136 
2137     val = ldub_p(vdev->config + addr);
2138     return val;
2139 }
2140 
2141 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
2142 {
2143     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2144     uint16_t val;
2145 
2146     if (addr + sizeof(val) > vdev->config_len) {
2147         return (uint32_t)-1;
2148     }
2149 
2150     k->get_config(vdev, vdev->config);
2151 
2152     val = lduw_le_p(vdev->config + addr);
2153     return val;
2154 }
2155 
2156 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
2157 {
2158     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2159     uint32_t val;
2160 
2161     if (addr + sizeof(val) > vdev->config_len) {
2162         return (uint32_t)-1;
2163     }
2164 
2165     k->get_config(vdev, vdev->config);
2166 
2167     val = ldl_le_p(vdev->config + addr);
2168     return val;
2169 }
2170 
2171 void virtio_config_modern_writeb(VirtIODevice *vdev,
2172                                  uint32_t addr, uint32_t data)
2173 {
2174     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2175     uint8_t val = data;
2176 
2177     if (addr + sizeof(val) > vdev->config_len) {
2178         return;
2179     }
2180 
2181     stb_p(vdev->config + addr, val);
2182 
2183     if (k->set_config) {
2184         k->set_config(vdev, vdev->config);
2185     }
2186 }
2187 
2188 void virtio_config_modern_writew(VirtIODevice *vdev,
2189                                  uint32_t addr, uint32_t data)
2190 {
2191     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2192     uint16_t val = data;
2193 
2194     if (addr + sizeof(val) > vdev->config_len) {
2195         return;
2196     }
2197 
2198     stw_le_p(vdev->config + addr, val);
2199 
2200     if (k->set_config) {
2201         k->set_config(vdev, vdev->config);
2202     }
2203 }
2204 
2205 void virtio_config_modern_writel(VirtIODevice *vdev,
2206                                  uint32_t addr, uint32_t data)
2207 {
2208     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2209     uint32_t val = data;
2210 
2211     if (addr + sizeof(val) > vdev->config_len) {
2212         return;
2213     }
2214 
2215     stl_le_p(vdev->config + addr, val);
2216 
2217     if (k->set_config) {
2218         k->set_config(vdev, vdev->config);
2219     }
2220 }
2221 
2222 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
2223 {
2224     if (!vdev->vq[n].vring.num) {
2225         return;
2226     }
2227     vdev->vq[n].vring.desc = addr;
2228     virtio_queue_update_rings(vdev, n);
2229 }
2230 
2231 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
2232 {
2233     return vdev->vq[n].vring.desc;
2234 }
2235 
2236 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
2237                             hwaddr avail, hwaddr used)
2238 {
2239     if (!vdev->vq[n].vring.num) {
2240         return;
2241     }
2242     vdev->vq[n].vring.desc = desc;
2243     vdev->vq[n].vring.avail = avail;
2244     vdev->vq[n].vring.used = used;
2245     virtio_init_region_cache(vdev, n);
2246 }
2247 
2248 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
2249 {
2250     /* Don't allow guest to flip queue between existent and
2251      * nonexistent states, or to set it to an invalid size.
2252      */
2253     if (!!num != !!vdev->vq[n].vring.num ||
2254         num > VIRTQUEUE_MAX_SIZE ||
2255         num < 0) {
2256         return;
2257     }
2258     vdev->vq[n].vring.num = num;
2259 }
2260 
2261 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
2262 {
2263     return QLIST_FIRST(&vdev->vector_queues[vector]);
2264 }
2265 
2266 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
2267 {
2268     return QLIST_NEXT(vq, node);
2269 }
2270 
2271 int virtio_queue_get_num(VirtIODevice *vdev, int n)
2272 {
2273     return vdev->vq[n].vring.num;
2274 }
2275 
2276 int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
2277 {
2278     return vdev->vq[n].vring.num_default;
2279 }
2280 
2281 int virtio_get_num_queues(VirtIODevice *vdev)
2282 {
2283     int i;
2284 
2285     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2286         if (!virtio_queue_get_num(vdev, i)) {
2287             break;
2288         }
2289     }
2290 
2291     return i;
2292 }
2293 
2294 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
2295 {
2296     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2297     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2298 
2299     /* virtio-1 compliant devices cannot change the alignment */
2300     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2301         error_report("tried to modify queue alignment for virtio-1 device");
2302         return;
2303     }
2304     /* Check that the transport told us it was going to do this
2305      * (so a buggy transport will immediately assert rather than
2306      * silently failing to migrate this state)
2307      */
2308     assert(k->has_variable_vring_alignment);
2309 
2310     if (align) {
2311         vdev->vq[n].vring.align = align;
2312         virtio_queue_update_rings(vdev, n);
2313     }
2314 }
2315 
2316 static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
2317 {
2318     bool ret = false;
2319 
2320     if (vq->vring.desc && vq->handle_aio_output) {
2321         VirtIODevice *vdev = vq->vdev;
2322 
2323         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2324         ret = vq->handle_aio_output(vdev, vq);
2325 
2326         if (unlikely(vdev->start_on_kick)) {
2327             virtio_set_started(vdev, true);
2328         }
2329     }
2330 
2331     return ret;
2332 }
2333 
2334 static void virtio_queue_notify_vq(VirtQueue *vq)
2335 {
2336     if (vq->vring.desc && vq->handle_output) {
2337         VirtIODevice *vdev = vq->vdev;
2338 
2339         if (unlikely(vdev->broken)) {
2340             return;
2341         }
2342 
2343         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2344         vq->handle_output(vdev, vq);
2345 
2346         if (unlikely(vdev->start_on_kick)) {
2347             virtio_set_started(vdev, true);
2348         }
2349     }
2350 }
2351 
2352 void virtio_queue_notify(VirtIODevice *vdev, int n)
2353 {
2354     VirtQueue *vq = &vdev->vq[n];
2355 
2356     if (unlikely(!vq->vring.desc || vdev->broken)) {
2357         return;
2358     }
2359 
2360     trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2361     if (vq->host_notifier_enabled) {
2362         event_notifier_set(&vq->host_notifier);
2363     } else if (vq->handle_output) {
2364         vq->handle_output(vdev, vq);
2365 
2366         if (unlikely(vdev->start_on_kick)) {
2367             virtio_set_started(vdev, true);
2368         }
2369     }
2370 }
2371 
2372 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
2373 {
2374     return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
2375         VIRTIO_NO_VECTOR;
2376 }
2377 
2378 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
2379 {
2380     VirtQueue *vq = &vdev->vq[n];
2381 
2382     if (n < VIRTIO_QUEUE_MAX) {
2383         if (vdev->vector_queues &&
2384             vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
2385             QLIST_REMOVE(vq, node);
2386         }
2387         vdev->vq[n].vector = vector;
2388         if (vdev->vector_queues &&
2389             vector != VIRTIO_NO_VECTOR) {
2390             QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
2391         }
2392     }
2393 }
2394 
2395 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
2396                             VirtIOHandleOutput handle_output)
2397 {
2398     int i;
2399 
2400     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2401         if (vdev->vq[i].vring.num == 0)
2402             break;
2403     }
2404 
2405     if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
2406         abort();
2407 
2408     vdev->vq[i].vring.num = queue_size;
2409     vdev->vq[i].vring.num_default = queue_size;
2410     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
2411     vdev->vq[i].handle_output = handle_output;
2412     vdev->vq[i].handle_aio_output = NULL;
2413     vdev->vq[i].used_elems = g_malloc0(sizeof(VirtQueueElement) *
2414                                        queue_size);
2415 
2416     return &vdev->vq[i];
2417 }
2418 
2419 void virtio_delete_queue(VirtQueue *vq)
2420 {
2421     vq->vring.num = 0;
2422     vq->vring.num_default = 0;
2423     vq->handle_output = NULL;
2424     vq->handle_aio_output = NULL;
2425     g_free(vq->used_elems);
2426     vq->used_elems = NULL;
2427     virtio_virtqueue_reset_region_cache(vq);
2428 }
2429 
2430 void virtio_del_queue(VirtIODevice *vdev, int n)
2431 {
2432     if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
2433         abort();
2434     }
2435 
2436     virtio_delete_queue(&vdev->vq[n]);
2437 }
2438 
2439 static void virtio_set_isr(VirtIODevice *vdev, int value)
2440 {
2441     uint8_t old = atomic_read(&vdev->isr);
2442 
2443     /* Do not write ISR if it does not change, so that its cacheline remains
2444      * shared in the common case where the guest does not read it.
2445      */
2446     if ((old & value) != value) {
2447         atomic_or(&vdev->isr, value);
2448     }
2449 }
2450 
2451 static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2452 {
2453     uint16_t old, new;
2454     bool v;
2455     /* We need to expose used array entries before checking used event. */
2456     smp_mb();
2457     /* Always notify when queue is empty (when feature acknowledge) */
2458     if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2459         !vq->inuse && virtio_queue_empty(vq)) {
2460         return true;
2461     }
2462 
2463     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2464         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
2465     }
2466 
2467     v = vq->signalled_used_valid;
2468     vq->signalled_used_valid = true;
2469     old = vq->signalled_used;
2470     new = vq->signalled_used = vq->used_idx;
2471     return !v || vring_need_event(vring_get_used_event(vq), new, old);
2472 }
2473 
2474 static bool vring_packed_need_event(VirtQueue *vq, bool wrap,
2475                                     uint16_t off_wrap, uint16_t new,
2476                                     uint16_t old)
2477 {
2478     int off = off_wrap & ~(1 << 15);
2479 
2480     if (wrap != off_wrap >> 15) {
2481         off -= vq->vring.num;
2482     }
2483 
2484     return vring_need_event(off, new, old);
2485 }
2486 
2487 static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2488 {
2489     VRingPackedDescEvent e;
2490     uint16_t old, new;
2491     bool v;
2492     VRingMemoryRegionCaches *caches;
2493 
2494     caches = vring_get_region_caches(vq);
2495     if (!caches) {
2496         return false;
2497     }
2498 
2499     vring_packed_event_read(vdev, &caches->avail, &e);
2500 
2501     old = vq->signalled_used;
2502     new = vq->signalled_used = vq->used_idx;
2503     v = vq->signalled_used_valid;
2504     vq->signalled_used_valid = true;
2505 
2506     if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) {
2507         return false;
2508     } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) {
2509         return true;
2510     }
2511 
2512     return !v || vring_packed_need_event(vq, vq->used_wrap_counter,
2513                                          e.off_wrap, new, old);
2514 }
2515 
2516 /* Called within rcu_read_lock().  */
2517 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2518 {
2519     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2520         return virtio_packed_should_notify(vdev, vq);
2521     } else {
2522         return virtio_split_should_notify(vdev, vq);
2523     }
2524 }
2525 
2526 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
2527 {
2528     WITH_RCU_READ_LOCK_GUARD() {
2529         if (!virtio_should_notify(vdev, vq)) {
2530             return;
2531         }
2532     }
2533 
2534     trace_virtio_notify_irqfd(vdev, vq);
2535 
2536     /*
2537      * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2538      * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2539      * incorrectly polling this bit during crashdump and hibernation
2540      * in MSI mode, causing a hang if this bit is never updated.
2541      * Recent releases of Windows do not really shut down, but rather
2542      * log out and hibernate to make the next startup faster.  Hence,
2543      * this manifested as a more serious hang during shutdown with
2544      *
2545      * Next driver release from 2016 fixed this problem, so working around it
2546      * is not a must, but it's easy to do so let's do it here.
2547      *
2548      * Note: it's safe to update ISR from any thread as it was switched
2549      * to an atomic operation.
2550      */
2551     virtio_set_isr(vq->vdev, 0x1);
2552     event_notifier_set(&vq->guest_notifier);
2553 }
2554 
2555 static void virtio_irq(VirtQueue *vq)
2556 {
2557     virtio_set_isr(vq->vdev, 0x1);
2558     virtio_notify_vector(vq->vdev, vq->vector);
2559 }
2560 
2561 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
2562 {
2563     WITH_RCU_READ_LOCK_GUARD() {
2564         if (!virtio_should_notify(vdev, vq)) {
2565             return;
2566         }
2567     }
2568 
2569     trace_virtio_notify(vdev, vq);
2570     virtio_irq(vq);
2571 }
2572 
2573 void virtio_notify_config(VirtIODevice *vdev)
2574 {
2575     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
2576         return;
2577 
2578     virtio_set_isr(vdev, 0x3);
2579     vdev->generation++;
2580     virtio_notify_vector(vdev, vdev->config_vector);
2581 }
2582 
2583 static bool virtio_device_endian_needed(void *opaque)
2584 {
2585     VirtIODevice *vdev = opaque;
2586 
2587     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
2588     if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2589         return vdev->device_endian != virtio_default_endian();
2590     }
2591     /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2592     return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
2593 }
2594 
2595 static bool virtio_64bit_features_needed(void *opaque)
2596 {
2597     VirtIODevice *vdev = opaque;
2598 
2599     return (vdev->host_features >> 32) != 0;
2600 }
2601 
2602 static bool virtio_virtqueue_needed(void *opaque)
2603 {
2604     VirtIODevice *vdev = opaque;
2605 
2606     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
2607 }
2608 
2609 static bool virtio_packed_virtqueue_needed(void *opaque)
2610 {
2611     VirtIODevice *vdev = opaque;
2612 
2613     return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED);
2614 }
2615 
2616 static bool virtio_ringsize_needed(void *opaque)
2617 {
2618     VirtIODevice *vdev = opaque;
2619     int i;
2620 
2621     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2622         if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
2623             return true;
2624         }
2625     }
2626     return false;
2627 }
2628 
2629 static bool virtio_extra_state_needed(void *opaque)
2630 {
2631     VirtIODevice *vdev = opaque;
2632     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2633     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2634 
2635     return k->has_extra_state &&
2636         k->has_extra_state(qbus->parent);
2637 }
2638 
2639 static bool virtio_broken_needed(void *opaque)
2640 {
2641     VirtIODevice *vdev = opaque;
2642 
2643     return vdev->broken;
2644 }
2645 
2646 static bool virtio_started_needed(void *opaque)
2647 {
2648     VirtIODevice *vdev = opaque;
2649 
2650     return vdev->started;
2651 }
2652 
2653 static bool virtio_disabled_needed(void *opaque)
2654 {
2655     VirtIODevice *vdev = opaque;
2656 
2657     return vdev->disabled;
2658 }
2659 
2660 static const VMStateDescription vmstate_virtqueue = {
2661     .name = "virtqueue_state",
2662     .version_id = 1,
2663     .minimum_version_id = 1,
2664     .fields = (VMStateField[]) {
2665         VMSTATE_UINT64(vring.avail, struct VirtQueue),
2666         VMSTATE_UINT64(vring.used, struct VirtQueue),
2667         VMSTATE_END_OF_LIST()
2668     }
2669 };
2670 
2671 static const VMStateDescription vmstate_packed_virtqueue = {
2672     .name = "packed_virtqueue_state",
2673     .version_id = 1,
2674     .minimum_version_id = 1,
2675     .fields = (VMStateField[]) {
2676         VMSTATE_UINT16(last_avail_idx, struct VirtQueue),
2677         VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue),
2678         VMSTATE_UINT16(used_idx, struct VirtQueue),
2679         VMSTATE_BOOL(used_wrap_counter, struct VirtQueue),
2680         VMSTATE_UINT32(inuse, struct VirtQueue),
2681         VMSTATE_END_OF_LIST()
2682     }
2683 };
2684 
2685 static const VMStateDescription vmstate_virtio_virtqueues = {
2686     .name = "virtio/virtqueues",
2687     .version_id = 1,
2688     .minimum_version_id = 1,
2689     .needed = &virtio_virtqueue_needed,
2690     .fields = (VMStateField[]) {
2691         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2692                       VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
2693         VMSTATE_END_OF_LIST()
2694     }
2695 };
2696 
2697 static const VMStateDescription vmstate_virtio_packed_virtqueues = {
2698     .name = "virtio/packed_virtqueues",
2699     .version_id = 1,
2700     .minimum_version_id = 1,
2701     .needed = &virtio_packed_virtqueue_needed,
2702     .fields = (VMStateField[]) {
2703         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2704                       VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue),
2705         VMSTATE_END_OF_LIST()
2706     }
2707 };
2708 
2709 static const VMStateDescription vmstate_ringsize = {
2710     .name = "ringsize_state",
2711     .version_id = 1,
2712     .minimum_version_id = 1,
2713     .fields = (VMStateField[]) {
2714         VMSTATE_UINT32(vring.num_default, struct VirtQueue),
2715         VMSTATE_END_OF_LIST()
2716     }
2717 };
2718 
2719 static const VMStateDescription vmstate_virtio_ringsize = {
2720     .name = "virtio/ringsize",
2721     .version_id = 1,
2722     .minimum_version_id = 1,
2723     .needed = &virtio_ringsize_needed,
2724     .fields = (VMStateField[]) {
2725         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2726                       VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
2727         VMSTATE_END_OF_LIST()
2728     }
2729 };
2730 
2731 static int get_extra_state(QEMUFile *f, void *pv, size_t size,
2732                            const VMStateField *field)
2733 {
2734     VirtIODevice *vdev = pv;
2735     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2736     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2737 
2738     if (!k->load_extra_state) {
2739         return -1;
2740     } else {
2741         return k->load_extra_state(qbus->parent, f);
2742     }
2743 }
2744 
2745 static int put_extra_state(QEMUFile *f, void *pv, size_t size,
2746                            const VMStateField *field, QJSON *vmdesc)
2747 {
2748     VirtIODevice *vdev = pv;
2749     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2750     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2751 
2752     k->save_extra_state(qbus->parent, f);
2753     return 0;
2754 }
2755 
2756 static const VMStateInfo vmstate_info_extra_state = {
2757     .name = "virtqueue_extra_state",
2758     .get = get_extra_state,
2759     .put = put_extra_state,
2760 };
2761 
2762 static const VMStateDescription vmstate_virtio_extra_state = {
2763     .name = "virtio/extra_state",
2764     .version_id = 1,
2765     .minimum_version_id = 1,
2766     .needed = &virtio_extra_state_needed,
2767     .fields = (VMStateField[]) {
2768         {
2769             .name         = "extra_state",
2770             .version_id   = 0,
2771             .field_exists = NULL,
2772             .size         = 0,
2773             .info         = &vmstate_info_extra_state,
2774             .flags        = VMS_SINGLE,
2775             .offset       = 0,
2776         },
2777         VMSTATE_END_OF_LIST()
2778     }
2779 };
2780 
2781 static const VMStateDescription vmstate_virtio_device_endian = {
2782     .name = "virtio/device_endian",
2783     .version_id = 1,
2784     .minimum_version_id = 1,
2785     .needed = &virtio_device_endian_needed,
2786     .fields = (VMStateField[]) {
2787         VMSTATE_UINT8(device_endian, VirtIODevice),
2788         VMSTATE_END_OF_LIST()
2789     }
2790 };
2791 
2792 static const VMStateDescription vmstate_virtio_64bit_features = {
2793     .name = "virtio/64bit_features",
2794     .version_id = 1,
2795     .minimum_version_id = 1,
2796     .needed = &virtio_64bit_features_needed,
2797     .fields = (VMStateField[]) {
2798         VMSTATE_UINT64(guest_features, VirtIODevice),
2799         VMSTATE_END_OF_LIST()
2800     }
2801 };
2802 
2803 static const VMStateDescription vmstate_virtio_broken = {
2804     .name = "virtio/broken",
2805     .version_id = 1,
2806     .minimum_version_id = 1,
2807     .needed = &virtio_broken_needed,
2808     .fields = (VMStateField[]) {
2809         VMSTATE_BOOL(broken, VirtIODevice),
2810         VMSTATE_END_OF_LIST()
2811     }
2812 };
2813 
2814 static const VMStateDescription vmstate_virtio_started = {
2815     .name = "virtio/started",
2816     .version_id = 1,
2817     .minimum_version_id = 1,
2818     .needed = &virtio_started_needed,
2819     .fields = (VMStateField[]) {
2820         VMSTATE_BOOL(started, VirtIODevice),
2821         VMSTATE_END_OF_LIST()
2822     }
2823 };
2824 
2825 static const VMStateDescription vmstate_virtio_disabled = {
2826     .name = "virtio/disabled",
2827     .version_id = 1,
2828     .minimum_version_id = 1,
2829     .needed = &virtio_disabled_needed,
2830     .fields = (VMStateField[]) {
2831         VMSTATE_BOOL(disabled, VirtIODevice),
2832         VMSTATE_END_OF_LIST()
2833     }
2834 };
2835 
2836 static const VMStateDescription vmstate_virtio = {
2837     .name = "virtio",
2838     .version_id = 1,
2839     .minimum_version_id = 1,
2840     .minimum_version_id_old = 1,
2841     .fields = (VMStateField[]) {
2842         VMSTATE_END_OF_LIST()
2843     },
2844     .subsections = (const VMStateDescription*[]) {
2845         &vmstate_virtio_device_endian,
2846         &vmstate_virtio_64bit_features,
2847         &vmstate_virtio_virtqueues,
2848         &vmstate_virtio_ringsize,
2849         &vmstate_virtio_broken,
2850         &vmstate_virtio_extra_state,
2851         &vmstate_virtio_started,
2852         &vmstate_virtio_packed_virtqueues,
2853         &vmstate_virtio_disabled,
2854         NULL
2855     }
2856 };
2857 
2858 int virtio_save(VirtIODevice *vdev, QEMUFile *f)
2859 {
2860     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2861     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2862     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
2863     uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
2864     int i;
2865 
2866     if (k->save_config) {
2867         k->save_config(qbus->parent, f);
2868     }
2869 
2870     qemu_put_8s(f, &vdev->status);
2871     qemu_put_8s(f, &vdev->isr);
2872     qemu_put_be16s(f, &vdev->queue_sel);
2873     qemu_put_be32s(f, &guest_features_lo);
2874     qemu_put_be32(f, vdev->config_len);
2875     qemu_put_buffer(f, vdev->config, vdev->config_len);
2876 
2877     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2878         if (vdev->vq[i].vring.num == 0)
2879             break;
2880     }
2881 
2882     qemu_put_be32(f, i);
2883 
2884     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2885         if (vdev->vq[i].vring.num == 0)
2886             break;
2887 
2888         qemu_put_be32(f, vdev->vq[i].vring.num);
2889         if (k->has_variable_vring_alignment) {
2890             qemu_put_be32(f, vdev->vq[i].vring.align);
2891         }
2892         /*
2893          * Save desc now, the rest of the ring addresses are saved in
2894          * subsections for VIRTIO-1 devices.
2895          */
2896         qemu_put_be64(f, vdev->vq[i].vring.desc);
2897         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
2898         if (k->save_queue) {
2899             k->save_queue(qbus->parent, i, f);
2900         }
2901     }
2902 
2903     if (vdc->save != NULL) {
2904         vdc->save(vdev, f);
2905     }
2906 
2907     if (vdc->vmsd) {
2908         int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL);
2909         if (ret) {
2910             return ret;
2911         }
2912     }
2913 
2914     /* Subsections */
2915     return vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
2916 }
2917 
2918 /* A wrapper for use as a VMState .put function */
2919 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
2920                               const VMStateField *field, QJSON *vmdesc)
2921 {
2922     return virtio_save(VIRTIO_DEVICE(opaque), f);
2923 }
2924 
2925 /* A wrapper for use as a VMState .get function */
2926 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size,
2927                              const VMStateField *field)
2928 {
2929     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
2930     DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
2931 
2932     return virtio_load(vdev, f, dc->vmsd->version_id);
2933 }
2934 
2935 const VMStateInfo  virtio_vmstate_info = {
2936     .name = "virtio",
2937     .get = virtio_device_get,
2938     .put = virtio_device_put,
2939 };
2940 
2941 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
2942 {
2943     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2944     bool bad = (val & ~(vdev->host_features)) != 0;
2945 
2946     val &= vdev->host_features;
2947     if (k->set_features) {
2948         k->set_features(vdev, val);
2949     }
2950     vdev->guest_features = val;
2951     return bad ? -1 : 0;
2952 }
2953 
2954 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
2955 {
2956     int ret;
2957     /*
2958      * The driver must not attempt to set features after feature negotiation
2959      * has finished.
2960      */
2961     if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
2962         return -EINVAL;
2963     }
2964     ret = virtio_set_features_nocheck(vdev, val);
2965     if (!ret) {
2966         if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2967             /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches.  */
2968             int i;
2969             for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2970                 if (vdev->vq[i].vring.num != 0) {
2971                     virtio_init_region_cache(vdev, i);
2972                 }
2973             }
2974         }
2975 
2976         if (!virtio_device_started(vdev, vdev->status) &&
2977             !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2978             vdev->start_on_kick = true;
2979         }
2980     }
2981     return ret;
2982 }
2983 
2984 size_t virtio_feature_get_config_size(VirtIOFeature *feature_sizes,
2985                                       uint64_t host_features)
2986 {
2987     size_t config_size = 0;
2988     int i;
2989 
2990     for (i = 0; feature_sizes[i].flags != 0; i++) {
2991         if (host_features & feature_sizes[i].flags) {
2992             config_size = MAX(feature_sizes[i].end, config_size);
2993         }
2994     }
2995 
2996     return config_size;
2997 }
2998 
2999 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
3000 {
3001     int i, ret;
3002     int32_t config_len;
3003     uint32_t num;
3004     uint32_t features;
3005     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3006     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3007     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
3008 
3009     /*
3010      * We poison the endianness to ensure it does not get used before
3011      * subsections have been loaded.
3012      */
3013     vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
3014 
3015     if (k->load_config) {
3016         ret = k->load_config(qbus->parent, f);
3017         if (ret)
3018             return ret;
3019     }
3020 
3021     qemu_get_8s(f, &vdev->status);
3022     qemu_get_8s(f, &vdev->isr);
3023     qemu_get_be16s(f, &vdev->queue_sel);
3024     if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
3025         return -1;
3026     }
3027     qemu_get_be32s(f, &features);
3028 
3029     /*
3030      * Temporarily set guest_features low bits - needed by
3031      * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
3032      * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
3033      *
3034      * Note: devices should always test host features in future - don't create
3035      * new dependencies like this.
3036      */
3037     vdev->guest_features = features;
3038 
3039     config_len = qemu_get_be32(f);
3040 
3041     /*
3042      * There are cases where the incoming config can be bigger or smaller
3043      * than what we have; so load what we have space for, and skip
3044      * any excess that's in the stream.
3045      */
3046     qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
3047 
3048     while (config_len > vdev->config_len) {
3049         qemu_get_byte(f);
3050         config_len--;
3051     }
3052 
3053     num = qemu_get_be32(f);
3054 
3055     if (num > VIRTIO_QUEUE_MAX) {
3056         error_report("Invalid number of virtqueues: 0x%x", num);
3057         return -1;
3058     }
3059 
3060     for (i = 0; i < num; i++) {
3061         vdev->vq[i].vring.num = qemu_get_be32(f);
3062         if (k->has_variable_vring_alignment) {
3063             vdev->vq[i].vring.align = qemu_get_be32(f);
3064         }
3065         vdev->vq[i].vring.desc = qemu_get_be64(f);
3066         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
3067         vdev->vq[i].signalled_used_valid = false;
3068         vdev->vq[i].notification = true;
3069 
3070         if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
3071             error_report("VQ %d address 0x0 "
3072                          "inconsistent with Host index 0x%x",
3073                          i, vdev->vq[i].last_avail_idx);
3074             return -1;
3075         }
3076         if (k->load_queue) {
3077             ret = k->load_queue(qbus->parent, i, f);
3078             if (ret)
3079                 return ret;
3080         }
3081     }
3082 
3083     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
3084 
3085     if (vdc->load != NULL) {
3086         ret = vdc->load(vdev, f, version_id);
3087         if (ret) {
3088             return ret;
3089         }
3090     }
3091 
3092     if (vdc->vmsd) {
3093         ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id);
3094         if (ret) {
3095             return ret;
3096         }
3097     }
3098 
3099     /* Subsections */
3100     ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
3101     if (ret) {
3102         return ret;
3103     }
3104 
3105     if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
3106         vdev->device_endian = virtio_default_endian();
3107     }
3108 
3109     if (virtio_64bit_features_needed(vdev)) {
3110         /*
3111          * Subsection load filled vdev->guest_features.  Run them
3112          * through virtio_set_features to sanity-check them against
3113          * host_features.
3114          */
3115         uint64_t features64 = vdev->guest_features;
3116         if (virtio_set_features_nocheck(vdev, features64) < 0) {
3117             error_report("Features 0x%" PRIx64 " unsupported. "
3118                          "Allowed features: 0x%" PRIx64,
3119                          features64, vdev->host_features);
3120             return -1;
3121         }
3122     } else {
3123         if (virtio_set_features_nocheck(vdev, features) < 0) {
3124             error_report("Features 0x%x unsupported. "
3125                          "Allowed features: 0x%" PRIx64,
3126                          features, vdev->host_features);
3127             return -1;
3128         }
3129     }
3130 
3131     if (!virtio_device_started(vdev, vdev->status) &&
3132         !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3133         vdev->start_on_kick = true;
3134     }
3135 
3136     RCU_READ_LOCK_GUARD();
3137     for (i = 0; i < num; i++) {
3138         if (vdev->vq[i].vring.desc) {
3139             uint16_t nheads;
3140 
3141             /*
3142              * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3143              * only the region cache needs to be set up.  Legacy devices need
3144              * to calculate used and avail ring addresses based on the desc
3145              * address.
3146              */
3147             if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3148                 virtio_init_region_cache(vdev, i);
3149             } else {
3150                 virtio_queue_update_rings(vdev, i);
3151             }
3152 
3153             if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3154                 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx;
3155                 vdev->vq[i].shadow_avail_wrap_counter =
3156                                         vdev->vq[i].last_avail_wrap_counter;
3157                 continue;
3158             }
3159 
3160             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
3161             /* Check it isn't doing strange things with descriptor numbers. */
3162             if (nheads > vdev->vq[i].vring.num) {
3163                 error_report("VQ %d size 0x%x Guest index 0x%x "
3164                              "inconsistent with Host index 0x%x: delta 0x%x",
3165                              i, vdev->vq[i].vring.num,
3166                              vring_avail_idx(&vdev->vq[i]),
3167                              vdev->vq[i].last_avail_idx, nheads);
3168                 return -1;
3169             }
3170             vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
3171             vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
3172 
3173             /*
3174              * Some devices migrate VirtQueueElements that have been popped
3175              * from the avail ring but not yet returned to the used ring.
3176              * Since max ring size < UINT16_MAX it's safe to use modulo
3177              * UINT16_MAX + 1 subtraction.
3178              */
3179             vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
3180                                 vdev->vq[i].used_idx);
3181             if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
3182                 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3183                              "used_idx 0x%x",
3184                              i, vdev->vq[i].vring.num,
3185                              vdev->vq[i].last_avail_idx,
3186                              vdev->vq[i].used_idx);
3187                 return -1;
3188             }
3189         }
3190     }
3191 
3192     if (vdc->post_load) {
3193         ret = vdc->post_load(vdev);
3194         if (ret) {
3195             return ret;
3196         }
3197     }
3198 
3199     return 0;
3200 }
3201 
3202 void virtio_cleanup(VirtIODevice *vdev)
3203 {
3204     qemu_del_vm_change_state_handler(vdev->vmstate);
3205 }
3206 
3207 static void virtio_vmstate_change(void *opaque, int running, RunState state)
3208 {
3209     VirtIODevice *vdev = opaque;
3210     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3211     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3212     bool backend_run = running && virtio_device_started(vdev, vdev->status);
3213     vdev->vm_running = running;
3214 
3215     if (backend_run) {
3216         virtio_set_status(vdev, vdev->status);
3217     }
3218 
3219     if (k->vmstate_change) {
3220         k->vmstate_change(qbus->parent, backend_run);
3221     }
3222 
3223     if (!backend_run) {
3224         virtio_set_status(vdev, vdev->status);
3225     }
3226 }
3227 
3228 void virtio_instance_init_common(Object *proxy_obj, void *data,
3229                                  size_t vdev_size, const char *vdev_name)
3230 {
3231     DeviceState *vdev = data;
3232 
3233     object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev,
3234                                        vdev_size, vdev_name, &error_abort,
3235                                        NULL);
3236     qdev_alias_all_properties(vdev, proxy_obj);
3237 }
3238 
3239 void virtio_init(VirtIODevice *vdev, const char *name,
3240                  uint16_t device_id, size_t config_size)
3241 {
3242     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3243     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3244     int i;
3245     int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
3246 
3247     if (nvectors) {
3248         vdev->vector_queues =
3249             g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
3250     }
3251 
3252     vdev->start_on_kick = false;
3253     vdev->started = false;
3254     vdev->device_id = device_id;
3255     vdev->status = 0;
3256     atomic_set(&vdev->isr, 0);
3257     vdev->queue_sel = 0;
3258     vdev->config_vector = VIRTIO_NO_VECTOR;
3259     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
3260     vdev->vm_running = runstate_is_running();
3261     vdev->broken = false;
3262     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3263         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
3264         vdev->vq[i].vdev = vdev;
3265         vdev->vq[i].queue_index = i;
3266         vdev->vq[i].host_notifier_enabled = false;
3267     }
3268 
3269     vdev->name = name;
3270     vdev->config_len = config_size;
3271     if (vdev->config_len) {
3272         vdev->config = g_malloc0(config_size);
3273     } else {
3274         vdev->config = NULL;
3275     }
3276     vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
3277             virtio_vmstate_change, vdev);
3278     vdev->device_endian = virtio_default_endian();
3279     vdev->use_guest_notifier_mask = true;
3280 }
3281 
3282 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
3283 {
3284     return vdev->vq[n].vring.desc;
3285 }
3286 
3287 bool virtio_queue_enabled(VirtIODevice *vdev, int n)
3288 {
3289     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3290     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3291 
3292     if (k->queue_enabled) {
3293         return k->queue_enabled(qbus->parent, n);
3294     }
3295     return virtio_queue_get_desc_addr(vdev, n) != 0;
3296 }
3297 
3298 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
3299 {
3300     return vdev->vq[n].vring.avail;
3301 }
3302 
3303 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
3304 {
3305     return vdev->vq[n].vring.used;
3306 }
3307 
3308 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
3309 {
3310     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
3311 }
3312 
3313 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
3314 {
3315     int s;
3316 
3317     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3318         return sizeof(struct VRingPackedDescEvent);
3319     }
3320 
3321     s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3322     return offsetof(VRingAvail, ring) +
3323         sizeof(uint16_t) * vdev->vq[n].vring.num + s;
3324 }
3325 
3326 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
3327 {
3328     int s;
3329 
3330     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3331         return sizeof(struct VRingPackedDescEvent);
3332     }
3333 
3334     s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3335     return offsetof(VRingUsed, ring) +
3336         sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s;
3337 }
3338 
3339 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev,
3340                                                            int n)
3341 {
3342     unsigned int avail, used;
3343 
3344     avail = vdev->vq[n].last_avail_idx;
3345     avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15;
3346 
3347     used = vdev->vq[n].used_idx;
3348     used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15;
3349 
3350     return avail | used << 16;
3351 }
3352 
3353 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev,
3354                                                       int n)
3355 {
3356     return vdev->vq[n].last_avail_idx;
3357 }
3358 
3359 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
3360 {
3361     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3362         return virtio_queue_packed_get_last_avail_idx(vdev, n);
3363     } else {
3364         return virtio_queue_split_get_last_avail_idx(vdev, n);
3365     }
3366 }
3367 
3368 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev,
3369                                                    int n, unsigned int idx)
3370 {
3371     struct VirtQueue *vq = &vdev->vq[n];
3372 
3373     vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff;
3374     vq->last_avail_wrap_counter =
3375         vq->shadow_avail_wrap_counter = !!(idx & 0x8000);
3376     idx >>= 16;
3377     vq->used_idx = idx & 0x7ffff;
3378     vq->used_wrap_counter = !!(idx & 0x8000);
3379 }
3380 
3381 static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev,
3382                                                   int n, unsigned int idx)
3383 {
3384         vdev->vq[n].last_avail_idx = idx;
3385         vdev->vq[n].shadow_avail_idx = idx;
3386 }
3387 
3388 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n,
3389                                      unsigned int idx)
3390 {
3391     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3392         virtio_queue_packed_set_last_avail_idx(vdev, n, idx);
3393     } else {
3394         virtio_queue_split_set_last_avail_idx(vdev, n, idx);
3395     }
3396 }
3397 
3398 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev,
3399                                                        int n)
3400 {
3401     /* We don't have a reference like avail idx in shared memory */
3402     return;
3403 }
3404 
3405 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev,
3406                                                       int n)
3407 {
3408     RCU_READ_LOCK_GUARD();
3409     if (vdev->vq[n].vring.desc) {
3410         vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]);
3411         vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx;
3412     }
3413 }
3414 
3415 void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n)
3416 {
3417     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3418         virtio_queue_packed_restore_last_avail_idx(vdev, n);
3419     } else {
3420         virtio_queue_split_restore_last_avail_idx(vdev, n);
3421     }
3422 }
3423 
3424 static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n)
3425 {
3426     /* used idx was updated through set_last_avail_idx() */
3427     return;
3428 }
3429 
3430 static void virtio_split_packed_update_used_idx(VirtIODevice *vdev, int n)
3431 {
3432     RCU_READ_LOCK_GUARD();
3433     if (vdev->vq[n].vring.desc) {
3434         vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
3435     }
3436 }
3437 
3438 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
3439 {
3440     if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3441         return virtio_queue_packed_update_used_idx(vdev, n);
3442     } else {
3443         return virtio_split_packed_update_used_idx(vdev, n);
3444     }
3445 }
3446 
3447 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
3448 {
3449     vdev->vq[n].signalled_used_valid = false;
3450 }
3451 
3452 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
3453 {
3454     return vdev->vq + n;
3455 }
3456 
3457 uint16_t virtio_get_queue_index(VirtQueue *vq)
3458 {
3459     return vq->queue_index;
3460 }
3461 
3462 static void virtio_queue_guest_notifier_read(EventNotifier *n)
3463 {
3464     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
3465     if (event_notifier_test_and_clear(n)) {
3466         virtio_irq(vq);
3467     }
3468 }
3469 
3470 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
3471                                                 bool with_irqfd)
3472 {
3473     if (assign && !with_irqfd) {
3474         event_notifier_set_handler(&vq->guest_notifier,
3475                                    virtio_queue_guest_notifier_read);
3476     } else {
3477         event_notifier_set_handler(&vq->guest_notifier, NULL);
3478     }
3479     if (!assign) {
3480         /* Test and clear notifier before closing it,
3481          * in case poll callback didn't have time to run. */
3482         virtio_queue_guest_notifier_read(&vq->guest_notifier);
3483     }
3484 }
3485 
3486 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
3487 {
3488     return &vq->guest_notifier;
3489 }
3490 
3491 static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
3492 {
3493     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3494     if (event_notifier_test_and_clear(n)) {
3495         virtio_queue_notify_aio_vq(vq);
3496     }
3497 }
3498 
3499 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
3500 {
3501     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3502 
3503     virtio_queue_set_notification(vq, 0);
3504 }
3505 
3506 static bool virtio_queue_host_notifier_aio_poll(void *opaque)
3507 {
3508     EventNotifier *n = opaque;
3509     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3510 
3511     if (!vq->vring.desc || virtio_queue_empty(vq)) {
3512         return false;
3513     }
3514 
3515     return virtio_queue_notify_aio_vq(vq);
3516 }
3517 
3518 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
3519 {
3520     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3521 
3522     /* Caller polls once more after this to catch requests that race with us */
3523     virtio_queue_set_notification(vq, 1);
3524 }
3525 
3526 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
3527                                                 VirtIOHandleAIOOutput handle_output)
3528 {
3529     if (handle_output) {
3530         vq->handle_aio_output = handle_output;
3531         aio_set_event_notifier(ctx, &vq->host_notifier, true,
3532                                virtio_queue_host_notifier_aio_read,
3533                                virtio_queue_host_notifier_aio_poll);
3534         aio_set_event_notifier_poll(ctx, &vq->host_notifier,
3535                                     virtio_queue_host_notifier_aio_poll_begin,
3536                                     virtio_queue_host_notifier_aio_poll_end);
3537     } else {
3538         aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
3539         /* Test and clear notifier before after disabling event,
3540          * in case poll callback didn't have time to run. */
3541         virtio_queue_host_notifier_aio_read(&vq->host_notifier);
3542         vq->handle_aio_output = NULL;
3543     }
3544 }
3545 
3546 void virtio_queue_host_notifier_read(EventNotifier *n)
3547 {
3548     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3549     if (event_notifier_test_and_clear(n)) {
3550         virtio_queue_notify_vq(vq);
3551     }
3552 }
3553 
3554 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
3555 {
3556     return &vq->host_notifier;
3557 }
3558 
3559 void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled)
3560 {
3561     vq->host_notifier_enabled = enabled;
3562 }
3563 
3564 int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n,
3565                                       MemoryRegion *mr, bool assign)
3566 {
3567     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3568     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3569 
3570     if (k->set_host_notifier_mr) {
3571         return k->set_host_notifier_mr(qbus->parent, n, mr, assign);
3572     }
3573 
3574     return -1;
3575 }
3576 
3577 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
3578 {
3579     g_free(vdev->bus_name);
3580     vdev->bus_name = g_strdup(bus_name);
3581 }
3582 
3583 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
3584 {
3585     va_list ap;
3586 
3587     va_start(ap, fmt);
3588     error_vreport(fmt, ap);
3589     va_end(ap);
3590 
3591     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3592         vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET;
3593         virtio_notify_config(vdev);
3594     }
3595 
3596     vdev->broken = true;
3597 }
3598 
3599 static void virtio_memory_listener_commit(MemoryListener *listener)
3600 {
3601     VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
3602     int i;
3603 
3604     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3605         if (vdev->vq[i].vring.num == 0) {
3606             break;
3607         }
3608         virtio_init_region_cache(vdev, i);
3609     }
3610 }
3611 
3612 static void virtio_device_realize(DeviceState *dev, Error **errp)
3613 {
3614     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3615     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
3616     Error *err = NULL;
3617 
3618     /* Devices should either use vmsd or the load/save methods */
3619     assert(!vdc->vmsd || !vdc->load);
3620 
3621     if (vdc->realize != NULL) {
3622         vdc->realize(dev, &err);
3623         if (err != NULL) {
3624             error_propagate(errp, err);
3625             return;
3626         }
3627     }
3628 
3629     virtio_bus_device_plugged(vdev, &err);
3630     if (err != NULL) {
3631         error_propagate(errp, err);
3632         vdc->unrealize(dev);
3633         return;
3634     }
3635 
3636     vdev->listener.commit = virtio_memory_listener_commit;
3637     memory_listener_register(&vdev->listener, vdev->dma_as);
3638 }
3639 
3640 static void virtio_device_unrealize(DeviceState *dev)
3641 {
3642     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3643     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
3644 
3645     virtio_bus_device_unplugged(vdev);
3646 
3647     if (vdc->unrealize != NULL) {
3648         vdc->unrealize(dev);
3649     }
3650 
3651     g_free(vdev->bus_name);
3652     vdev->bus_name = NULL;
3653 }
3654 
3655 static void virtio_device_free_virtqueues(VirtIODevice *vdev)
3656 {
3657     int i;
3658     if (!vdev->vq) {
3659         return;
3660     }
3661 
3662     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3663         if (vdev->vq[i].vring.num == 0) {
3664             break;
3665         }
3666         virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
3667     }
3668     g_free(vdev->vq);
3669 }
3670 
3671 static void virtio_device_instance_finalize(Object *obj)
3672 {
3673     VirtIODevice *vdev = VIRTIO_DEVICE(obj);
3674 
3675     memory_listener_unregister(&vdev->listener);
3676     virtio_device_free_virtqueues(vdev);
3677 
3678     g_free(vdev->config);
3679     g_free(vdev->vector_queues);
3680 }
3681 
3682 static Property virtio_properties[] = {
3683     DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
3684     DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
3685     DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true),
3686     DEFINE_PROP_END_OF_LIST(),
3687 };
3688 
3689 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
3690 {
3691     VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
3692     int i, n, r, err;
3693 
3694     memory_region_transaction_begin();
3695     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3696         VirtQueue *vq = &vdev->vq[n];
3697         if (!virtio_queue_get_num(vdev, n)) {
3698             continue;
3699         }
3700         r = virtio_bus_set_host_notifier(qbus, n, true);
3701         if (r < 0) {
3702             err = r;
3703             goto assign_error;
3704         }
3705         event_notifier_set_handler(&vq->host_notifier,
3706                                    virtio_queue_host_notifier_read);
3707     }
3708 
3709     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3710         /* Kick right away to begin processing requests already in vring */
3711         VirtQueue *vq = &vdev->vq[n];
3712         if (!vq->vring.num) {
3713             continue;
3714         }
3715         event_notifier_set(&vq->host_notifier);
3716     }
3717     memory_region_transaction_commit();
3718     return 0;
3719 
3720 assign_error:
3721     i = n; /* save n for a second iteration after transaction is committed. */
3722     while (--n >= 0) {
3723         VirtQueue *vq = &vdev->vq[n];
3724         if (!virtio_queue_get_num(vdev, n)) {
3725             continue;
3726         }
3727 
3728         event_notifier_set_handler(&vq->host_notifier, NULL);
3729         r = virtio_bus_set_host_notifier(qbus, n, false);
3730         assert(r >= 0);
3731     }
3732     memory_region_transaction_commit();
3733 
3734     while (--i >= 0) {
3735         if (!virtio_queue_get_num(vdev, i)) {
3736             continue;
3737         }
3738         virtio_bus_cleanup_host_notifier(qbus, i);
3739     }
3740     return err;
3741 }
3742 
3743 int virtio_device_start_ioeventfd(VirtIODevice *vdev)
3744 {
3745     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3746     VirtioBusState *vbus = VIRTIO_BUS(qbus);
3747 
3748     return virtio_bus_start_ioeventfd(vbus);
3749 }
3750 
3751 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
3752 {
3753     VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
3754     int n, r;
3755 
3756     memory_region_transaction_begin();
3757     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3758         VirtQueue *vq = &vdev->vq[n];
3759 
3760         if (!virtio_queue_get_num(vdev, n)) {
3761             continue;
3762         }
3763         event_notifier_set_handler(&vq->host_notifier, NULL);
3764         r = virtio_bus_set_host_notifier(qbus, n, false);
3765         assert(r >= 0);
3766     }
3767     memory_region_transaction_commit();
3768 
3769     for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3770         if (!virtio_queue_get_num(vdev, n)) {
3771             continue;
3772         }
3773         virtio_bus_cleanup_host_notifier(qbus, n);
3774     }
3775 }
3776 
3777 int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
3778 {
3779     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3780     VirtioBusState *vbus = VIRTIO_BUS(qbus);
3781 
3782     return virtio_bus_grab_ioeventfd(vbus);
3783 }
3784 
3785 void virtio_device_release_ioeventfd(VirtIODevice *vdev)
3786 {
3787     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3788     VirtioBusState *vbus = VIRTIO_BUS(qbus);
3789 
3790     virtio_bus_release_ioeventfd(vbus);
3791 }
3792 
3793 static void virtio_device_class_init(ObjectClass *klass, void *data)
3794 {
3795     /* Set the default value here. */
3796     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
3797     DeviceClass *dc = DEVICE_CLASS(klass);
3798 
3799     dc->realize = virtio_device_realize;
3800     dc->unrealize = virtio_device_unrealize;
3801     dc->bus_type = TYPE_VIRTIO_BUS;
3802     device_class_set_props(dc, virtio_properties);
3803     vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
3804     vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
3805 
3806     vdc->legacy_features |= VIRTIO_LEGACY_FEATURES;
3807 }
3808 
3809 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
3810 {
3811     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3812     VirtioBusState *vbus = VIRTIO_BUS(qbus);
3813 
3814     return virtio_bus_ioeventfd_enabled(vbus);
3815 }
3816 
3817 static const TypeInfo virtio_device_info = {
3818     .name = TYPE_VIRTIO_DEVICE,
3819     .parent = TYPE_DEVICE,
3820     .instance_size = sizeof(VirtIODevice),
3821     .class_init = virtio_device_class_init,
3822     .instance_finalize = virtio_device_instance_finalize,
3823     .abstract = true,
3824     .class_size = sizeof(VirtioDeviceClass),
3825 };
3826 
3827 static void virtio_register_types(void)
3828 {
3829     type_register_static(&virtio_device_info);
3830 }
3831 
3832 type_init(virtio_register_types)
3833