xref: /openbmc/qemu/hw/virtio/virtio.c (revision 0e39bb022b5fa8c11964968885f3263c02ce42b0)
1 /*
2  * Virtio Support
3  *
4  * Copyright IBM, Corp. 2007
5  *
6  * Authors:
7  *  Anthony Liguori   <aliguori@us.ibm.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  */
13 
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
17 #include "cpu.h"
18 #include "trace.h"
19 #include "exec/address-spaces.h"
20 #include "qemu/error-report.h"
21 #include "hw/virtio/virtio.h"
22 #include "qemu/atomic.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "migration/migration.h"
25 #include "hw/virtio/virtio-access.h"
26 
27 /*
28  * The alignment to use between consumer and producer parts of vring.
29  * x86 pagesize again. This is the default, used by transports like PCI
30  * which don't provide a means for the guest to tell the host the alignment.
31  */
32 #define VIRTIO_PCI_VRING_ALIGN         4096
33 
34 typedef struct VRingDesc
35 {
36     uint64_t addr;
37     uint32_t len;
38     uint16_t flags;
39     uint16_t next;
40 } VRingDesc;
41 
42 typedef struct VRingAvail
43 {
44     uint16_t flags;
45     uint16_t idx;
46     uint16_t ring[0];
47 } VRingAvail;
48 
49 typedef struct VRingUsedElem
50 {
51     uint32_t id;
52     uint32_t len;
53 } VRingUsedElem;
54 
55 typedef struct VRingUsed
56 {
57     uint16_t flags;
58     uint16_t idx;
59     VRingUsedElem ring[0];
60 } VRingUsed;
61 
62 typedef struct VRing
63 {
64     unsigned int num;
65     unsigned int num_default;
66     unsigned int align;
67     hwaddr desc;
68     hwaddr avail;
69     hwaddr used;
70 } VRing;
71 
72 struct VirtQueue
73 {
74     VRing vring;
75 
76     /* Next head to pop */
77     uint16_t last_avail_idx;
78 
79     /* Last avail_idx read from VQ. */
80     uint16_t shadow_avail_idx;
81 
82     uint16_t used_idx;
83 
84     /* Last used index value we have signalled on */
85     uint16_t signalled_used;
86 
87     /* Last used index value we have signalled on */
88     bool signalled_used_valid;
89 
90     /* Notification enabled? */
91     bool notification;
92 
93     uint16_t queue_index;
94 
95     int inuse;
96 
97     uint16_t vector;
98     VirtIOHandleOutput handle_output;
99     VirtIOHandleOutput handle_aio_output;
100     bool use_aio;
101     VirtIODevice *vdev;
102     EventNotifier guest_notifier;
103     EventNotifier host_notifier;
104     QLIST_ENTRY(VirtQueue) node;
105 };
106 
107 /* virt queue functions */
108 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
109 {
110     VRing *vring = &vdev->vq[n].vring;
111 
112     if (!vring->desc) {
113         /* not yet setup -> nothing to do */
114         return;
115     }
116     vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
117     vring->used = vring_align(vring->avail +
118                               offsetof(VRingAvail, ring[vring->num]),
119                               vring->align);
120 }
121 
122 static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
123                             hwaddr desc_pa, int i)
124 {
125     address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
126                        MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
127     virtio_tswap64s(vdev, &desc->addr);
128     virtio_tswap32s(vdev, &desc->len);
129     virtio_tswap16s(vdev, &desc->flags);
130     virtio_tswap16s(vdev, &desc->next);
131 }
132 
133 static inline uint16_t vring_avail_flags(VirtQueue *vq)
134 {
135     hwaddr pa;
136     pa = vq->vring.avail + offsetof(VRingAvail, flags);
137     return virtio_lduw_phys(vq->vdev, pa);
138 }
139 
140 static inline uint16_t vring_avail_idx(VirtQueue *vq)
141 {
142     hwaddr pa;
143     pa = vq->vring.avail + offsetof(VRingAvail, idx);
144     vq->shadow_avail_idx = virtio_lduw_phys(vq->vdev, pa);
145     return vq->shadow_avail_idx;
146 }
147 
148 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
149 {
150     hwaddr pa;
151     pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
152     return virtio_lduw_phys(vq->vdev, pa);
153 }
154 
155 static inline uint16_t vring_get_used_event(VirtQueue *vq)
156 {
157     return vring_avail_ring(vq, vq->vring.num);
158 }
159 
160 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
161                                     int i)
162 {
163     hwaddr pa;
164     virtio_tswap32s(vq->vdev, &uelem->id);
165     virtio_tswap32s(vq->vdev, &uelem->len);
166     pa = vq->vring.used + offsetof(VRingUsed, ring[i]);
167     address_space_write(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
168                        (void *)uelem, sizeof(VRingUsedElem));
169 }
170 
171 static uint16_t vring_used_idx(VirtQueue *vq)
172 {
173     hwaddr pa;
174     pa = vq->vring.used + offsetof(VRingUsed, idx);
175     return virtio_lduw_phys(vq->vdev, pa);
176 }
177 
178 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
179 {
180     hwaddr pa;
181     pa = vq->vring.used + offsetof(VRingUsed, idx);
182     virtio_stw_phys(vq->vdev, pa, val);
183     vq->used_idx = val;
184 }
185 
186 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
187 {
188     VirtIODevice *vdev = vq->vdev;
189     hwaddr pa;
190     pa = vq->vring.used + offsetof(VRingUsed, flags);
191     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) | mask);
192 }
193 
194 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
195 {
196     VirtIODevice *vdev = vq->vdev;
197     hwaddr pa;
198     pa = vq->vring.used + offsetof(VRingUsed, flags);
199     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
200 }
201 
202 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
203 {
204     hwaddr pa;
205     if (!vq->notification) {
206         return;
207     }
208     pa = vq->vring.used + offsetof(VRingUsed, ring[vq->vring.num]);
209     virtio_stw_phys(vq->vdev, pa, val);
210 }
211 
212 void virtio_queue_set_notification(VirtQueue *vq, int enable)
213 {
214     vq->notification = enable;
215     if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
216         vring_set_avail_event(vq, vring_avail_idx(vq));
217     } else if (enable) {
218         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
219     } else {
220         vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
221     }
222     if (enable) {
223         /* Expose avail event/used flags before caller checks the avail idx. */
224         smp_mb();
225     }
226 }
227 
228 int virtio_queue_ready(VirtQueue *vq)
229 {
230     return vq->vring.avail != 0;
231 }
232 
233 /* Fetch avail_idx from VQ memory only when we really need to know if
234  * guest has added some buffers. */
235 int virtio_queue_empty(VirtQueue *vq)
236 {
237     if (vq->shadow_avail_idx != vq->last_avail_idx) {
238         return 0;
239     }
240 
241     return vring_avail_idx(vq) == vq->last_avail_idx;
242 }
243 
244 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
245                                unsigned int len)
246 {
247     unsigned int offset;
248     int i;
249 
250     offset = 0;
251     for (i = 0; i < elem->in_num; i++) {
252         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
253 
254         cpu_physical_memory_unmap(elem->in_sg[i].iov_base,
255                                   elem->in_sg[i].iov_len,
256                                   1, size);
257 
258         offset += size;
259     }
260 
261     for (i = 0; i < elem->out_num; i++)
262         cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
263                                   elem->out_sg[i].iov_len,
264                                   0, elem->out_sg[i].iov_len);
265 }
266 
267 void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
268                        unsigned int len)
269 {
270     vq->last_avail_idx--;
271     virtqueue_unmap_sg(vq, elem, len);
272 }
273 
274 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
275                     unsigned int len, unsigned int idx)
276 {
277     VRingUsedElem uelem;
278 
279     trace_virtqueue_fill(vq, elem, len, idx);
280 
281     virtqueue_unmap_sg(vq, elem, len);
282 
283     idx = (idx + vq->used_idx) % vq->vring.num;
284 
285     uelem.id = elem->index;
286     uelem.len = len;
287     vring_used_write(vq, &uelem, idx);
288 }
289 
290 void virtqueue_flush(VirtQueue *vq, unsigned int count)
291 {
292     uint16_t old, new;
293     /* Make sure buffer is written before we update index. */
294     smp_wmb();
295     trace_virtqueue_flush(vq, count);
296     old = vq->used_idx;
297     new = old + count;
298     vring_used_idx_set(vq, new);
299     vq->inuse -= count;
300     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
301         vq->signalled_used_valid = false;
302 }
303 
304 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
305                     unsigned int len)
306 {
307     virtqueue_fill(vq, elem, len, 0);
308     virtqueue_flush(vq, 1);
309 }
310 
311 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
312 {
313     uint16_t num_heads = vring_avail_idx(vq) - idx;
314 
315     /* Check it isn't doing very strange things with descriptor numbers. */
316     if (num_heads > vq->vring.num) {
317         error_report("Guest moved used index from %u to %u",
318                      idx, vq->shadow_avail_idx);
319         exit(1);
320     }
321     /* On success, callers read a descriptor at vq->last_avail_idx.
322      * Make sure descriptor read does not bypass avail index read. */
323     if (num_heads) {
324         smp_rmb();
325     }
326 
327     return num_heads;
328 }
329 
330 static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
331 {
332     unsigned int head;
333 
334     /* Grab the next descriptor number they're advertising, and increment
335      * the index we've seen. */
336     head = vring_avail_ring(vq, idx % vq->vring.num);
337 
338     /* If their number is silly, that's a fatal mistake. */
339     if (head >= vq->vring.num) {
340         error_report("Guest says index %u is available", head);
341         exit(1);
342     }
343 
344     return head;
345 }
346 
347 static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
348                                          hwaddr desc_pa, unsigned int max)
349 {
350     unsigned int next;
351 
352     /* If this descriptor says it doesn't chain, we're done. */
353     if (!(desc->flags & VRING_DESC_F_NEXT)) {
354         return max;
355     }
356 
357     /* Check they're not leading us off end of descriptors. */
358     next = desc->next;
359     /* Make sure compiler knows to grab that: we don't want it changing! */
360     smp_wmb();
361 
362     if (next >= max) {
363         error_report("Desc next is %u", next);
364         exit(1);
365     }
366 
367     vring_desc_read(vdev, desc, desc_pa, next);
368     return next;
369 }
370 
371 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
372                                unsigned int *out_bytes,
373                                unsigned max_in_bytes, unsigned max_out_bytes)
374 {
375     unsigned int idx;
376     unsigned int total_bufs, in_total, out_total;
377 
378     idx = vq->last_avail_idx;
379 
380     total_bufs = in_total = out_total = 0;
381     while (virtqueue_num_heads(vq, idx)) {
382         VirtIODevice *vdev = vq->vdev;
383         unsigned int max, num_bufs, indirect = 0;
384         VRingDesc desc;
385         hwaddr desc_pa;
386         int i;
387 
388         max = vq->vring.num;
389         num_bufs = total_bufs;
390         i = virtqueue_get_head(vq, idx++);
391         desc_pa = vq->vring.desc;
392         vring_desc_read(vdev, &desc, desc_pa, i);
393 
394         if (desc.flags & VRING_DESC_F_INDIRECT) {
395             if (desc.len % sizeof(VRingDesc)) {
396                 error_report("Invalid size for indirect buffer table");
397                 exit(1);
398             }
399 
400             /* If we've got too many, that implies a descriptor loop. */
401             if (num_bufs >= max) {
402                 error_report("Looped descriptor");
403                 exit(1);
404             }
405 
406             /* loop over the indirect descriptor table */
407             indirect = 1;
408             max = desc.len / sizeof(VRingDesc);
409             desc_pa = desc.addr;
410             num_bufs = i = 0;
411             vring_desc_read(vdev, &desc, desc_pa, i);
412         }
413 
414         do {
415             /* If we've got too many, that implies a descriptor loop. */
416             if (++num_bufs > max) {
417                 error_report("Looped descriptor");
418                 exit(1);
419             }
420 
421             if (desc.flags & VRING_DESC_F_WRITE) {
422                 in_total += desc.len;
423             } else {
424                 out_total += desc.len;
425             }
426             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
427                 goto done;
428             }
429         } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
430 
431         if (!indirect)
432             total_bufs = num_bufs;
433         else
434             total_bufs++;
435     }
436 done:
437     if (in_bytes) {
438         *in_bytes = in_total;
439     }
440     if (out_bytes) {
441         *out_bytes = out_total;
442     }
443 }
444 
445 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
446                           unsigned int out_bytes)
447 {
448     unsigned int in_total, out_total;
449 
450     virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
451     return in_bytes <= in_total && out_bytes <= out_total;
452 }
453 
454 static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
455                                unsigned int max_num_sg, bool is_write,
456                                hwaddr pa, size_t sz)
457 {
458     unsigned num_sg = *p_num_sg;
459     assert(num_sg <= max_num_sg);
460 
461     if (!sz) {
462         error_report("virtio: zero sized buffers are not allowed");
463         exit(1);
464     }
465 
466     while (sz) {
467         hwaddr len = sz;
468 
469         if (num_sg == max_num_sg) {
470             error_report("virtio: too many write descriptors in indirect table");
471             exit(1);
472         }
473 
474         iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
475         iov[num_sg].iov_len = len;
476         addr[num_sg] = pa;
477 
478         sz -= len;
479         pa += len;
480         num_sg++;
481     }
482     *p_num_sg = num_sg;
483 }
484 
485 static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
486                                 unsigned int *num_sg, unsigned int max_size,
487                                 int is_write)
488 {
489     unsigned int i;
490     hwaddr len;
491 
492     /* Note: this function MUST validate input, some callers
493      * are passing in num_sg values received over the network.
494      */
495     /* TODO: teach all callers that this can fail, and return failure instead
496      * of asserting here.
497      * When we do, we might be able to re-enable NDEBUG below.
498      */
499 #ifdef NDEBUG
500 #error building with NDEBUG is not supported
501 #endif
502     assert(*num_sg <= max_size);
503 
504     for (i = 0; i < *num_sg; i++) {
505         len = sg[i].iov_len;
506         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
507         if (!sg[i].iov_base) {
508             error_report("virtio: error trying to map MMIO memory");
509             exit(1);
510         }
511         if (len != sg[i].iov_len) {
512             error_report("virtio: unexpected memory split");
513             exit(1);
514         }
515     }
516 }
517 
518 void virtqueue_map(VirtQueueElement *elem)
519 {
520     virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
521                         VIRTQUEUE_MAX_SIZE, 1);
522     virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
523                         VIRTQUEUE_MAX_SIZE, 0);
524 }
525 
526 void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
527 {
528     VirtQueueElement *elem;
529     size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
530     size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
531     size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
532     size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
533     size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
534     size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
535 
536     assert(sz >= sizeof(VirtQueueElement));
537     elem = g_malloc(out_sg_end);
538     elem->out_num = out_num;
539     elem->in_num = in_num;
540     elem->in_addr = (void *)elem + in_addr_ofs;
541     elem->out_addr = (void *)elem + out_addr_ofs;
542     elem->in_sg = (void *)elem + in_sg_ofs;
543     elem->out_sg = (void *)elem + out_sg_ofs;
544     return elem;
545 }
546 
547 void *virtqueue_pop(VirtQueue *vq, size_t sz)
548 {
549     unsigned int i, head, max;
550     hwaddr desc_pa = vq->vring.desc;
551     VirtIODevice *vdev = vq->vdev;
552     VirtQueueElement *elem;
553     unsigned out_num, in_num;
554     hwaddr addr[VIRTQUEUE_MAX_SIZE];
555     struct iovec iov[VIRTQUEUE_MAX_SIZE];
556     VRingDesc desc;
557 
558     if (virtio_queue_empty(vq)) {
559         return NULL;
560     }
561     /* Needed after virtio_queue_empty(), see comment in
562      * virtqueue_num_heads(). */
563     smp_rmb();
564 
565     /* When we start there are none of either input nor output. */
566     out_num = in_num = 0;
567 
568     max = vq->vring.num;
569 
570     if (vq->inuse >= vq->vring.num) {
571         error_report("Virtqueue size exceeded");
572         exit(1);
573     }
574 
575     i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
576     if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
577         vring_set_avail_event(vq, vq->last_avail_idx);
578     }
579 
580     vring_desc_read(vdev, &desc, desc_pa, i);
581     if (desc.flags & VRING_DESC_F_INDIRECT) {
582         if (desc.len % sizeof(VRingDesc)) {
583             error_report("Invalid size for indirect buffer table");
584             exit(1);
585         }
586 
587         /* loop over the indirect descriptor table */
588         max = desc.len / sizeof(VRingDesc);
589         desc_pa = desc.addr;
590         i = 0;
591         vring_desc_read(vdev, &desc, desc_pa, i);
592     }
593 
594     /* Collect all the descriptors */
595     do {
596         if (desc.flags & VRING_DESC_F_WRITE) {
597             virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
598                                VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
599         } else {
600             if (in_num) {
601                 error_report("Incorrect order for descriptors");
602                 exit(1);
603             }
604             virtqueue_map_desc(&out_num, addr, iov,
605                                VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
606         }
607 
608         /* If we've got too many, that implies a descriptor loop. */
609         if ((in_num + out_num) > max) {
610             error_report("Looped descriptor");
611             exit(1);
612         }
613     } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
614 
615     /* Now copy what we have collected and mapped */
616     elem = virtqueue_alloc_element(sz, out_num, in_num);
617     elem->index = head;
618     for (i = 0; i < out_num; i++) {
619         elem->out_addr[i] = addr[i];
620         elem->out_sg[i] = iov[i];
621     }
622     for (i = 0; i < in_num; i++) {
623         elem->in_addr[i] = addr[out_num + i];
624         elem->in_sg[i] = iov[out_num + i];
625     }
626 
627     vq->inuse++;
628 
629     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
630     return elem;
631 }
632 
633 /* Reading and writing a structure directly to QEMUFile is *awful*, but
634  * it is what QEMU has always done by mistake.  We can change it sooner
635  * or later by bumping the version number of the affected vm states.
636  * In the meanwhile, since the in-memory layout of VirtQueueElement
637  * has changed, we need to marshal to and from the layout that was
638  * used before the change.
639  */
640 typedef struct VirtQueueElementOld {
641     unsigned int index;
642     unsigned int out_num;
643     unsigned int in_num;
644     hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
645     hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
646     struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
647     struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
648 } VirtQueueElementOld;
649 
650 void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
651 {
652     VirtQueueElement *elem;
653     VirtQueueElementOld data;
654     int i;
655 
656     qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
657 
658     elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
659     elem->index = data.index;
660 
661     for (i = 0; i < elem->in_num; i++) {
662         elem->in_addr[i] = data.in_addr[i];
663     }
664 
665     for (i = 0; i < elem->out_num; i++) {
666         elem->out_addr[i] = data.out_addr[i];
667     }
668 
669     for (i = 0; i < elem->in_num; i++) {
670         /* Base is overwritten by virtqueue_map.  */
671         elem->in_sg[i].iov_base = 0;
672         elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
673     }
674 
675     for (i = 0; i < elem->out_num; i++) {
676         /* Base is overwritten by virtqueue_map.  */
677         elem->out_sg[i].iov_base = 0;
678         elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
679     }
680 
681     virtqueue_map(elem);
682     return elem;
683 }
684 
685 void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
686 {
687     VirtQueueElementOld data;
688     int i;
689 
690     memset(&data, 0, sizeof(data));
691     data.index = elem->index;
692     data.in_num = elem->in_num;
693     data.out_num = elem->out_num;
694 
695     for (i = 0; i < elem->in_num; i++) {
696         data.in_addr[i] = elem->in_addr[i];
697     }
698 
699     for (i = 0; i < elem->out_num; i++) {
700         data.out_addr[i] = elem->out_addr[i];
701     }
702 
703     for (i = 0; i < elem->in_num; i++) {
704         /* Base is overwritten by virtqueue_map when loading.  Do not
705          * save it, as it would leak the QEMU address space layout.  */
706         data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
707     }
708 
709     for (i = 0; i < elem->out_num; i++) {
710         /* Do not save iov_base as above.  */
711         data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
712     }
713     qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
714 }
715 
716 /* virtio device */
717 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
718 {
719     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
720     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
721 
722     if (k->notify) {
723         k->notify(qbus->parent, vector);
724     }
725 }
726 
727 void virtio_update_irq(VirtIODevice *vdev)
728 {
729     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
730 }
731 
732 static int virtio_validate_features(VirtIODevice *vdev)
733 {
734     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
735 
736     if (k->validate_features) {
737         return k->validate_features(vdev);
738     } else {
739         return 0;
740     }
741 }
742 
743 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
744 {
745     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
746     trace_virtio_set_status(vdev, val);
747 
748     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
749         if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
750             val & VIRTIO_CONFIG_S_FEATURES_OK) {
751             int ret = virtio_validate_features(vdev);
752 
753             if (ret) {
754                 return ret;
755             }
756         }
757     }
758     if (k->set_status) {
759         k->set_status(vdev, val);
760     }
761     vdev->status = val;
762     return 0;
763 }
764 
765 bool target_words_bigendian(void);
766 static enum virtio_device_endian virtio_default_endian(void)
767 {
768     if (target_words_bigendian()) {
769         return VIRTIO_DEVICE_ENDIAN_BIG;
770     } else {
771         return VIRTIO_DEVICE_ENDIAN_LITTLE;
772     }
773 }
774 
775 static enum virtio_device_endian virtio_current_cpu_endian(void)
776 {
777     CPUClass *cc = CPU_GET_CLASS(current_cpu);
778 
779     if (cc->virtio_is_big_endian(current_cpu)) {
780         return VIRTIO_DEVICE_ENDIAN_BIG;
781     } else {
782         return VIRTIO_DEVICE_ENDIAN_LITTLE;
783     }
784 }
785 
786 void virtio_reset(void *opaque)
787 {
788     VirtIODevice *vdev = opaque;
789     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
790     int i;
791 
792     virtio_set_status(vdev, 0);
793     if (current_cpu) {
794         /* Guest initiated reset */
795         vdev->device_endian = virtio_current_cpu_endian();
796     } else {
797         /* System reset */
798         vdev->device_endian = virtio_default_endian();
799     }
800 
801     if (k->reset) {
802         k->reset(vdev);
803     }
804 
805     vdev->guest_features = 0;
806     vdev->queue_sel = 0;
807     vdev->status = 0;
808     vdev->isr = 0;
809     vdev->config_vector = VIRTIO_NO_VECTOR;
810     virtio_notify_vector(vdev, vdev->config_vector);
811 
812     for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
813         vdev->vq[i].vring.desc = 0;
814         vdev->vq[i].vring.avail = 0;
815         vdev->vq[i].vring.used = 0;
816         vdev->vq[i].last_avail_idx = 0;
817         vdev->vq[i].shadow_avail_idx = 0;
818         vdev->vq[i].used_idx = 0;
819         virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
820         vdev->vq[i].signalled_used = 0;
821         vdev->vq[i].signalled_used_valid = false;
822         vdev->vq[i].notification = true;
823         vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
824     }
825 }
826 
827 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
828 {
829     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
830     uint8_t val;
831 
832     if (addr + sizeof(val) > vdev->config_len) {
833         return (uint32_t)-1;
834     }
835 
836     k->get_config(vdev, vdev->config);
837 
838     val = ldub_p(vdev->config + addr);
839     return val;
840 }
841 
842 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
843 {
844     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
845     uint16_t val;
846 
847     if (addr + sizeof(val) > vdev->config_len) {
848         return (uint32_t)-1;
849     }
850 
851     k->get_config(vdev, vdev->config);
852 
853     val = lduw_p(vdev->config + addr);
854     return val;
855 }
856 
857 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
858 {
859     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
860     uint32_t val;
861 
862     if (addr + sizeof(val) > vdev->config_len) {
863         return (uint32_t)-1;
864     }
865 
866     k->get_config(vdev, vdev->config);
867 
868     val = ldl_p(vdev->config + addr);
869     return val;
870 }
871 
872 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
873 {
874     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
875     uint8_t val = data;
876 
877     if (addr + sizeof(val) > vdev->config_len) {
878         return;
879     }
880 
881     stb_p(vdev->config + addr, val);
882 
883     if (k->set_config) {
884         k->set_config(vdev, vdev->config);
885     }
886 }
887 
888 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
889 {
890     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
891     uint16_t val = data;
892 
893     if (addr + sizeof(val) > vdev->config_len) {
894         return;
895     }
896 
897     stw_p(vdev->config + addr, val);
898 
899     if (k->set_config) {
900         k->set_config(vdev, vdev->config);
901     }
902 }
903 
904 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
905 {
906     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
907     uint32_t val = data;
908 
909     if (addr + sizeof(val) > vdev->config_len) {
910         return;
911     }
912 
913     stl_p(vdev->config + addr, val);
914 
915     if (k->set_config) {
916         k->set_config(vdev, vdev->config);
917     }
918 }
919 
920 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
921 {
922     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
923     uint8_t val;
924 
925     if (addr + sizeof(val) > vdev->config_len) {
926         return (uint32_t)-1;
927     }
928 
929     k->get_config(vdev, vdev->config);
930 
931     val = ldub_p(vdev->config + addr);
932     return val;
933 }
934 
935 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
936 {
937     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
938     uint16_t val;
939 
940     if (addr + sizeof(val) > vdev->config_len) {
941         return (uint32_t)-1;
942     }
943 
944     k->get_config(vdev, vdev->config);
945 
946     val = lduw_le_p(vdev->config + addr);
947     return val;
948 }
949 
950 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
951 {
952     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
953     uint32_t val;
954 
955     if (addr + sizeof(val) > vdev->config_len) {
956         return (uint32_t)-1;
957     }
958 
959     k->get_config(vdev, vdev->config);
960 
961     val = ldl_le_p(vdev->config + addr);
962     return val;
963 }
964 
965 void virtio_config_modern_writeb(VirtIODevice *vdev,
966                                  uint32_t addr, uint32_t data)
967 {
968     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
969     uint8_t val = data;
970 
971     if (addr + sizeof(val) > vdev->config_len) {
972         return;
973     }
974 
975     stb_p(vdev->config + addr, val);
976 
977     if (k->set_config) {
978         k->set_config(vdev, vdev->config);
979     }
980 }
981 
982 void virtio_config_modern_writew(VirtIODevice *vdev,
983                                  uint32_t addr, uint32_t data)
984 {
985     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
986     uint16_t val = data;
987 
988     if (addr + sizeof(val) > vdev->config_len) {
989         return;
990     }
991 
992     stw_le_p(vdev->config + addr, val);
993 
994     if (k->set_config) {
995         k->set_config(vdev, vdev->config);
996     }
997 }
998 
999 void virtio_config_modern_writel(VirtIODevice *vdev,
1000                                  uint32_t addr, uint32_t data)
1001 {
1002     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1003     uint32_t val = data;
1004 
1005     if (addr + sizeof(val) > vdev->config_len) {
1006         return;
1007     }
1008 
1009     stl_le_p(vdev->config + addr, val);
1010 
1011     if (k->set_config) {
1012         k->set_config(vdev, vdev->config);
1013     }
1014 }
1015 
1016 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
1017 {
1018     vdev->vq[n].vring.desc = addr;
1019     virtio_queue_update_rings(vdev, n);
1020 }
1021 
1022 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
1023 {
1024     return vdev->vq[n].vring.desc;
1025 }
1026 
1027 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
1028                             hwaddr avail, hwaddr used)
1029 {
1030     vdev->vq[n].vring.desc = desc;
1031     vdev->vq[n].vring.avail = avail;
1032     vdev->vq[n].vring.used = used;
1033 }
1034 
1035 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
1036 {
1037     /* Don't allow guest to flip queue between existent and
1038      * nonexistent states, or to set it to an invalid size.
1039      */
1040     if (!!num != !!vdev->vq[n].vring.num ||
1041         num > VIRTQUEUE_MAX_SIZE ||
1042         num < 0) {
1043         return;
1044     }
1045     vdev->vq[n].vring.num = num;
1046 }
1047 
1048 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
1049 {
1050     return QLIST_FIRST(&vdev->vector_queues[vector]);
1051 }
1052 
1053 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
1054 {
1055     return QLIST_NEXT(vq, node);
1056 }
1057 
1058 int virtio_queue_get_num(VirtIODevice *vdev, int n)
1059 {
1060     return vdev->vq[n].vring.num;
1061 }
1062 
1063 int virtio_get_num_queues(VirtIODevice *vdev)
1064 {
1065     int i;
1066 
1067     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1068         if (!virtio_queue_get_num(vdev, i)) {
1069             break;
1070         }
1071     }
1072 
1073     return i;
1074 }
1075 
1076 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
1077 {
1078     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1079     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1080 
1081     /* virtio-1 compliant devices cannot change the alignment */
1082     if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1083         error_report("tried to modify queue alignment for virtio-1 device");
1084         return;
1085     }
1086     /* Check that the transport told us it was going to do this
1087      * (so a buggy transport will immediately assert rather than
1088      * silently failing to migrate this state)
1089      */
1090     assert(k->has_variable_vring_alignment);
1091 
1092     vdev->vq[n].vring.align = align;
1093     virtio_queue_update_rings(vdev, n);
1094 }
1095 
1096 static void virtio_queue_notify_aio_vq(VirtQueue *vq)
1097 {
1098     if (vq->vring.desc && vq->handle_aio_output) {
1099         VirtIODevice *vdev = vq->vdev;
1100 
1101         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1102         vq->handle_aio_output(vdev, vq);
1103     }
1104 }
1105 
1106 static void virtio_queue_notify_vq(VirtQueue *vq)
1107 {
1108     if (vq->vring.desc && vq->handle_output) {
1109         VirtIODevice *vdev = vq->vdev;
1110 
1111         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
1112         vq->handle_output(vdev, vq);
1113     }
1114 }
1115 
1116 void virtio_queue_notify(VirtIODevice *vdev, int n)
1117 {
1118     virtio_queue_notify_vq(&vdev->vq[n]);
1119 }
1120 
1121 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
1122 {
1123     return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
1124         VIRTIO_NO_VECTOR;
1125 }
1126 
1127 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
1128 {
1129     VirtQueue *vq = &vdev->vq[n];
1130 
1131     if (n < VIRTIO_QUEUE_MAX) {
1132         if (vdev->vector_queues &&
1133             vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
1134             QLIST_REMOVE(vq, node);
1135         }
1136         vdev->vq[n].vector = vector;
1137         if (vdev->vector_queues &&
1138             vector != VIRTIO_NO_VECTOR) {
1139             QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
1140         }
1141     }
1142 }
1143 
1144 static VirtQueue *virtio_add_queue_internal(VirtIODevice *vdev, int queue_size,
1145                                             VirtIOHandleOutput handle_output,
1146                                             bool use_aio)
1147 {
1148     int i;
1149 
1150     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1151         if (vdev->vq[i].vring.num == 0)
1152             break;
1153     }
1154 
1155     if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
1156         abort();
1157 
1158     vdev->vq[i].vring.num = queue_size;
1159     vdev->vq[i].vring.num_default = queue_size;
1160     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
1161     vdev->vq[i].handle_output = handle_output;
1162     vdev->vq[i].handle_aio_output = NULL;
1163     vdev->vq[i].use_aio = use_aio;
1164 
1165     return &vdev->vq[i];
1166 }
1167 
1168 /* Add a virt queue and mark AIO.
1169  * An AIO queue will use the AioContext based event interface instead of the
1170  * default IOHandler and EventNotifier interface.
1171  */
1172 VirtQueue *virtio_add_queue_aio(VirtIODevice *vdev, int queue_size,
1173                                 VirtIOHandleOutput handle_output)
1174 {
1175     return virtio_add_queue_internal(vdev, queue_size, handle_output, true);
1176 }
1177 
1178 /* Add a normal virt queue (on the contrary to the AIO version above. */
1179 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
1180                             VirtIOHandleOutput handle_output)
1181 {
1182     return virtio_add_queue_internal(vdev, queue_size, handle_output, false);
1183 }
1184 
1185 void virtio_del_queue(VirtIODevice *vdev, int n)
1186 {
1187     if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
1188         abort();
1189     }
1190 
1191     vdev->vq[n].vring.num = 0;
1192     vdev->vq[n].vring.num_default = 0;
1193 }
1194 
1195 void virtio_irq(VirtQueue *vq)
1196 {
1197     trace_virtio_irq(vq);
1198     vq->vdev->isr |= 0x01;
1199     virtio_notify_vector(vq->vdev, vq->vector);
1200 }
1201 
1202 bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
1203 {
1204     uint16_t old, new;
1205     bool v;
1206     /* We need to expose used array entries before checking used event. */
1207     smp_mb();
1208     /* Always notify when queue is empty (when feature acknowledge) */
1209     if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1210         !vq->inuse && virtio_queue_empty(vq)) {
1211         return true;
1212     }
1213 
1214     if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1215         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1216     }
1217 
1218     v = vq->signalled_used_valid;
1219     vq->signalled_used_valid = true;
1220     old = vq->signalled_used;
1221     new = vq->signalled_used = vq->used_idx;
1222     return !v || vring_need_event(vring_get_used_event(vq), new, old);
1223 }
1224 
1225 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
1226 {
1227     if (!virtio_should_notify(vdev, vq)) {
1228         return;
1229     }
1230 
1231     trace_virtio_notify(vdev, vq);
1232     vdev->isr |= 0x01;
1233     virtio_notify_vector(vdev, vq->vector);
1234 }
1235 
1236 void virtio_notify_config(VirtIODevice *vdev)
1237 {
1238     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
1239         return;
1240 
1241     vdev->isr |= 0x03;
1242     vdev->generation++;
1243     virtio_notify_vector(vdev, vdev->config_vector);
1244 }
1245 
1246 static bool virtio_device_endian_needed(void *opaque)
1247 {
1248     VirtIODevice *vdev = opaque;
1249 
1250     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
1251     if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1252         return vdev->device_endian != virtio_default_endian();
1253     }
1254     /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1255     return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
1256 }
1257 
1258 static bool virtio_64bit_features_needed(void *opaque)
1259 {
1260     VirtIODevice *vdev = opaque;
1261 
1262     return (vdev->host_features >> 32) != 0;
1263 }
1264 
1265 static bool virtio_virtqueue_needed(void *opaque)
1266 {
1267     VirtIODevice *vdev = opaque;
1268 
1269     return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
1270 }
1271 
1272 static bool virtio_ringsize_needed(void *opaque)
1273 {
1274     VirtIODevice *vdev = opaque;
1275     int i;
1276 
1277     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1278         if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
1279             return true;
1280         }
1281     }
1282     return false;
1283 }
1284 
1285 static bool virtio_extra_state_needed(void *opaque)
1286 {
1287     VirtIODevice *vdev = opaque;
1288     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1289     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1290 
1291     return k->has_extra_state &&
1292         k->has_extra_state(qbus->parent);
1293 }
1294 
1295 static const VMStateDescription vmstate_virtqueue = {
1296     .name = "virtqueue_state",
1297     .version_id = 1,
1298     .minimum_version_id = 1,
1299     .fields = (VMStateField[]) {
1300         VMSTATE_UINT64(vring.avail, struct VirtQueue),
1301         VMSTATE_UINT64(vring.used, struct VirtQueue),
1302         VMSTATE_END_OF_LIST()
1303     }
1304 };
1305 
1306 static const VMStateDescription vmstate_virtio_virtqueues = {
1307     .name = "virtio/virtqueues",
1308     .version_id = 1,
1309     .minimum_version_id = 1,
1310     .needed = &virtio_virtqueue_needed,
1311     .fields = (VMStateField[]) {
1312         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1313                       VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
1314         VMSTATE_END_OF_LIST()
1315     }
1316 };
1317 
1318 static const VMStateDescription vmstate_ringsize = {
1319     .name = "ringsize_state",
1320     .version_id = 1,
1321     .minimum_version_id = 1,
1322     .fields = (VMStateField[]) {
1323         VMSTATE_UINT32(vring.num_default, struct VirtQueue),
1324         VMSTATE_END_OF_LIST()
1325     }
1326 };
1327 
1328 static const VMStateDescription vmstate_virtio_ringsize = {
1329     .name = "virtio/ringsize",
1330     .version_id = 1,
1331     .minimum_version_id = 1,
1332     .needed = &virtio_ringsize_needed,
1333     .fields = (VMStateField[]) {
1334         VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
1335                       VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
1336         VMSTATE_END_OF_LIST()
1337     }
1338 };
1339 
1340 static int get_extra_state(QEMUFile *f, void *pv, size_t size)
1341 {
1342     VirtIODevice *vdev = pv;
1343     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1344     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1345 
1346     if (!k->load_extra_state) {
1347         return -1;
1348     } else {
1349         return k->load_extra_state(qbus->parent, f);
1350     }
1351 }
1352 
1353 static void put_extra_state(QEMUFile *f, void *pv, size_t size)
1354 {
1355     VirtIODevice *vdev = pv;
1356     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1357     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1358 
1359     k->save_extra_state(qbus->parent, f);
1360 }
1361 
1362 static const VMStateInfo vmstate_info_extra_state = {
1363     .name = "virtqueue_extra_state",
1364     .get = get_extra_state,
1365     .put = put_extra_state,
1366 };
1367 
1368 static const VMStateDescription vmstate_virtio_extra_state = {
1369     .name = "virtio/extra_state",
1370     .version_id = 1,
1371     .minimum_version_id = 1,
1372     .needed = &virtio_extra_state_needed,
1373     .fields = (VMStateField[]) {
1374         {
1375             .name         = "extra_state",
1376             .version_id   = 0,
1377             .field_exists = NULL,
1378             .size         = 0,
1379             .info         = &vmstate_info_extra_state,
1380             .flags        = VMS_SINGLE,
1381             .offset       = 0,
1382         },
1383         VMSTATE_END_OF_LIST()
1384     }
1385 };
1386 
1387 static const VMStateDescription vmstate_virtio_device_endian = {
1388     .name = "virtio/device_endian",
1389     .version_id = 1,
1390     .minimum_version_id = 1,
1391     .needed = &virtio_device_endian_needed,
1392     .fields = (VMStateField[]) {
1393         VMSTATE_UINT8(device_endian, VirtIODevice),
1394         VMSTATE_END_OF_LIST()
1395     }
1396 };
1397 
1398 static const VMStateDescription vmstate_virtio_64bit_features = {
1399     .name = "virtio/64bit_features",
1400     .version_id = 1,
1401     .minimum_version_id = 1,
1402     .needed = &virtio_64bit_features_needed,
1403     .fields = (VMStateField[]) {
1404         VMSTATE_UINT64(guest_features, VirtIODevice),
1405         VMSTATE_END_OF_LIST()
1406     }
1407 };
1408 
1409 static const VMStateDescription vmstate_virtio = {
1410     .name = "virtio",
1411     .version_id = 1,
1412     .minimum_version_id = 1,
1413     .minimum_version_id_old = 1,
1414     .fields = (VMStateField[]) {
1415         VMSTATE_END_OF_LIST()
1416     },
1417     .subsections = (const VMStateDescription*[]) {
1418         &vmstate_virtio_device_endian,
1419         &vmstate_virtio_64bit_features,
1420         &vmstate_virtio_virtqueues,
1421         &vmstate_virtio_ringsize,
1422         &vmstate_virtio_extra_state,
1423         NULL
1424     }
1425 };
1426 
1427 void virtio_save(VirtIODevice *vdev, QEMUFile *f)
1428 {
1429     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1430     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1431     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1432     uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
1433     int i;
1434 
1435     if (k->save_config) {
1436         k->save_config(qbus->parent, f);
1437     }
1438 
1439     qemu_put_8s(f, &vdev->status);
1440     qemu_put_8s(f, &vdev->isr);
1441     qemu_put_be16s(f, &vdev->queue_sel);
1442     qemu_put_be32s(f, &guest_features_lo);
1443     qemu_put_be32(f, vdev->config_len);
1444     qemu_put_buffer(f, vdev->config, vdev->config_len);
1445 
1446     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1447         if (vdev->vq[i].vring.num == 0)
1448             break;
1449     }
1450 
1451     qemu_put_be32(f, i);
1452 
1453     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1454         if (vdev->vq[i].vring.num == 0)
1455             break;
1456 
1457         qemu_put_be32(f, vdev->vq[i].vring.num);
1458         if (k->has_variable_vring_alignment) {
1459             qemu_put_be32(f, vdev->vq[i].vring.align);
1460         }
1461         /* XXX virtio-1 devices */
1462         qemu_put_be64(f, vdev->vq[i].vring.desc);
1463         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
1464         if (k->save_queue) {
1465             k->save_queue(qbus->parent, i, f);
1466         }
1467     }
1468 
1469     if (vdc->save != NULL) {
1470         vdc->save(vdev, f);
1471     }
1472 
1473     /* Subsections */
1474     vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
1475 }
1476 
1477 /* A wrapper for use as a VMState .put function */
1478 void virtio_vmstate_save(QEMUFile *f, void *opaque, size_t size)
1479 {
1480     virtio_save(VIRTIO_DEVICE(opaque), f);
1481 }
1482 
1483 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
1484 {
1485     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1486     bool bad = (val & ~(vdev->host_features)) != 0;
1487 
1488     val &= vdev->host_features;
1489     if (k->set_features) {
1490         k->set_features(vdev, val);
1491     }
1492     vdev->guest_features = val;
1493     return bad ? -1 : 0;
1494 }
1495 
1496 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
1497 {
1498    /*
1499      * The driver must not attempt to set features after feature negotiation
1500      * has finished.
1501      */
1502     if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
1503         return -EINVAL;
1504     }
1505     return virtio_set_features_nocheck(vdev, val);
1506 }
1507 
1508 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
1509 {
1510     int i, ret;
1511     int32_t config_len;
1512     uint32_t num;
1513     uint32_t features;
1514     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1515     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1516     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
1517 
1518     /*
1519      * We poison the endianness to ensure it does not get used before
1520      * subsections have been loaded.
1521      */
1522     vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
1523 
1524     if (k->load_config) {
1525         ret = k->load_config(qbus->parent, f);
1526         if (ret)
1527             return ret;
1528     }
1529 
1530     qemu_get_8s(f, &vdev->status);
1531     qemu_get_8s(f, &vdev->isr);
1532     qemu_get_be16s(f, &vdev->queue_sel);
1533     if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
1534         return -1;
1535     }
1536     qemu_get_be32s(f, &features);
1537 
1538     /*
1539      * Temporarily set guest_features low bits - needed by
1540      * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
1541      * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
1542      *
1543      * Note: devices should always test host features in future - don't create
1544      * new dependencies like this.
1545      */
1546     vdev->guest_features = features;
1547 
1548     config_len = qemu_get_be32(f);
1549 
1550     /*
1551      * There are cases where the incoming config can be bigger or smaller
1552      * than what we have; so load what we have space for, and skip
1553      * any excess that's in the stream.
1554      */
1555     qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
1556 
1557     while (config_len > vdev->config_len) {
1558         qemu_get_byte(f);
1559         config_len--;
1560     }
1561 
1562     num = qemu_get_be32(f);
1563 
1564     if (num > VIRTIO_QUEUE_MAX) {
1565         error_report("Invalid number of virtqueues: 0x%x", num);
1566         return -1;
1567     }
1568 
1569     for (i = 0; i < num; i++) {
1570         vdev->vq[i].vring.num = qemu_get_be32(f);
1571         if (k->has_variable_vring_alignment) {
1572             vdev->vq[i].vring.align = qemu_get_be32(f);
1573         }
1574         vdev->vq[i].vring.desc = qemu_get_be64(f);
1575         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
1576         vdev->vq[i].signalled_used_valid = false;
1577         vdev->vq[i].notification = true;
1578 
1579         if (vdev->vq[i].vring.desc) {
1580             /* XXX virtio-1 devices */
1581             virtio_queue_update_rings(vdev, i);
1582         } else if (vdev->vq[i].last_avail_idx) {
1583             error_report("VQ %d address 0x0 "
1584                          "inconsistent with Host index 0x%x",
1585                          i, vdev->vq[i].last_avail_idx);
1586                 return -1;
1587 	}
1588         if (k->load_queue) {
1589             ret = k->load_queue(qbus->parent, i, f);
1590             if (ret)
1591                 return ret;
1592         }
1593     }
1594 
1595     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1596 
1597     if (vdc->load != NULL) {
1598         ret = vdc->load(vdev, f, version_id);
1599         if (ret) {
1600             return ret;
1601         }
1602     }
1603 
1604     /* Subsections */
1605     ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
1606     if (ret) {
1607         return ret;
1608     }
1609 
1610     if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
1611         vdev->device_endian = virtio_default_endian();
1612     }
1613 
1614     if (virtio_64bit_features_needed(vdev)) {
1615         /*
1616          * Subsection load filled vdev->guest_features.  Run them
1617          * through virtio_set_features to sanity-check them against
1618          * host_features.
1619          */
1620         uint64_t features64 = vdev->guest_features;
1621         if (virtio_set_features_nocheck(vdev, features64) < 0) {
1622             error_report("Features 0x%" PRIx64 " unsupported. "
1623                          "Allowed features: 0x%" PRIx64,
1624                          features64, vdev->host_features);
1625             return -1;
1626         }
1627     } else {
1628         if (virtio_set_features_nocheck(vdev, features) < 0) {
1629             error_report("Features 0x%x unsupported. "
1630                          "Allowed features: 0x%" PRIx64,
1631                          features, vdev->host_features);
1632             return -1;
1633         }
1634     }
1635 
1636     for (i = 0; i < num; i++) {
1637         if (vdev->vq[i].vring.desc) {
1638             uint16_t nheads;
1639             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
1640             /* Check it isn't doing strange things with descriptor numbers. */
1641             if (nheads > vdev->vq[i].vring.num) {
1642                 error_report("VQ %d size 0x%x Guest index 0x%x "
1643                              "inconsistent with Host index 0x%x: delta 0x%x",
1644                              i, vdev->vq[i].vring.num,
1645                              vring_avail_idx(&vdev->vq[i]),
1646                              vdev->vq[i].last_avail_idx, nheads);
1647                 return -1;
1648             }
1649             vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
1650             vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
1651         }
1652     }
1653 
1654     return 0;
1655 }
1656 
1657 void virtio_cleanup(VirtIODevice *vdev)
1658 {
1659     qemu_del_vm_change_state_handler(vdev->vmstate);
1660     g_free(vdev->config);
1661     g_free(vdev->vq);
1662     g_free(vdev->vector_queues);
1663 }
1664 
1665 static void virtio_vmstate_change(void *opaque, int running, RunState state)
1666 {
1667     VirtIODevice *vdev = opaque;
1668     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1669     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1670     bool backend_run = running && (vdev->status & VIRTIO_CONFIG_S_DRIVER_OK);
1671     vdev->vm_running = running;
1672 
1673     if (backend_run) {
1674         virtio_set_status(vdev, vdev->status);
1675     }
1676 
1677     if (k->vmstate_change) {
1678         k->vmstate_change(qbus->parent, backend_run);
1679     }
1680 
1681     if (!backend_run) {
1682         virtio_set_status(vdev, vdev->status);
1683     }
1684 }
1685 
1686 void virtio_instance_init_common(Object *proxy_obj, void *data,
1687                                  size_t vdev_size, const char *vdev_name)
1688 {
1689     DeviceState *vdev = data;
1690 
1691     object_initialize(vdev, vdev_size, vdev_name);
1692     object_property_add_child(proxy_obj, "virtio-backend", OBJECT(vdev), NULL);
1693     object_unref(OBJECT(vdev));
1694     qdev_alias_all_properties(vdev, proxy_obj);
1695 }
1696 
1697 void virtio_init(VirtIODevice *vdev, const char *name,
1698                  uint16_t device_id, size_t config_size)
1699 {
1700     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1701     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1702     int i;
1703     int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
1704 
1705     if (nvectors) {
1706         vdev->vector_queues =
1707             g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
1708     }
1709 
1710     vdev->device_id = device_id;
1711     vdev->status = 0;
1712     vdev->isr = 0;
1713     vdev->queue_sel = 0;
1714     vdev->config_vector = VIRTIO_NO_VECTOR;
1715     vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
1716     vdev->vm_running = runstate_is_running();
1717     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
1718         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
1719         vdev->vq[i].vdev = vdev;
1720         vdev->vq[i].queue_index = i;
1721     }
1722 
1723     vdev->name = name;
1724     vdev->config_len = config_size;
1725     if (vdev->config_len) {
1726         vdev->config = g_malloc0(config_size);
1727     } else {
1728         vdev->config = NULL;
1729     }
1730     vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
1731                                                      vdev);
1732     vdev->device_endian = virtio_default_endian();
1733     vdev->use_guest_notifier_mask = true;
1734 }
1735 
1736 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
1737 {
1738     return vdev->vq[n].vring.desc;
1739 }
1740 
1741 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
1742 {
1743     return vdev->vq[n].vring.avail;
1744 }
1745 
1746 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
1747 {
1748     return vdev->vq[n].vring.used;
1749 }
1750 
1751 hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
1752 {
1753     return vdev->vq[n].vring.desc;
1754 }
1755 
1756 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
1757 {
1758     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
1759 }
1760 
1761 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
1762 {
1763     return offsetof(VRingAvail, ring) +
1764         sizeof(uint16_t) * vdev->vq[n].vring.num;
1765 }
1766 
1767 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
1768 {
1769     return offsetof(VRingUsed, ring) +
1770         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
1771 }
1772 
1773 hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
1774 {
1775     return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
1776 	    virtio_queue_get_used_size(vdev, n);
1777 }
1778 
1779 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
1780 {
1781     return vdev->vq[n].last_avail_idx;
1782 }
1783 
1784 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
1785 {
1786     vdev->vq[n].last_avail_idx = idx;
1787     vdev->vq[n].shadow_avail_idx = idx;
1788 }
1789 
1790 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
1791 {
1792     vdev->vq[n].signalled_used_valid = false;
1793 }
1794 
1795 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
1796 {
1797     return vdev->vq + n;
1798 }
1799 
1800 uint16_t virtio_get_queue_index(VirtQueue *vq)
1801 {
1802     return vq->queue_index;
1803 }
1804 
1805 static void virtio_queue_guest_notifier_read(EventNotifier *n)
1806 {
1807     VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
1808     if (event_notifier_test_and_clear(n)) {
1809         virtio_irq(vq);
1810     }
1811 }
1812 
1813 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
1814                                                 bool with_irqfd)
1815 {
1816     if (assign && !with_irqfd) {
1817         event_notifier_set_handler(&vq->guest_notifier, false,
1818                                    virtio_queue_guest_notifier_read);
1819     } else {
1820         event_notifier_set_handler(&vq->guest_notifier, false, NULL);
1821     }
1822     if (!assign) {
1823         /* Test and clear notifier before closing it,
1824          * in case poll callback didn't have time to run. */
1825         virtio_queue_guest_notifier_read(&vq->guest_notifier);
1826     }
1827 }
1828 
1829 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
1830 {
1831     return &vq->guest_notifier;
1832 }
1833 
1834 static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
1835 {
1836     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1837     if (event_notifier_test_and_clear(n)) {
1838         virtio_queue_notify_aio_vq(vq);
1839     }
1840 }
1841 
1842 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
1843                                                 VirtIOHandleOutput handle_output)
1844 {
1845     if (handle_output) {
1846         vq->handle_aio_output = handle_output;
1847         aio_set_event_notifier(ctx, &vq->host_notifier, true,
1848                                virtio_queue_host_notifier_aio_read);
1849     } else {
1850         aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
1851         /* Test and clear notifier before after disabling event,
1852          * in case poll callback didn't have time to run. */
1853         virtio_queue_host_notifier_aio_read(&vq->host_notifier);
1854         vq->handle_aio_output = NULL;
1855     }
1856 }
1857 
1858 static void virtio_queue_host_notifier_read(EventNotifier *n)
1859 {
1860     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
1861     if (event_notifier_test_and_clear(n)) {
1862         virtio_queue_notify_vq(vq);
1863     }
1864 }
1865 
1866 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
1867                                                bool set_handler)
1868 {
1869     AioContext *ctx = qemu_get_aio_context();
1870     if (assign && set_handler) {
1871         if (vq->use_aio) {
1872             aio_set_event_notifier(ctx, &vq->host_notifier, true,
1873                                    virtio_queue_host_notifier_read);
1874         } else {
1875             event_notifier_set_handler(&vq->host_notifier, true,
1876                                        virtio_queue_host_notifier_read);
1877         }
1878     } else {
1879         if (vq->use_aio) {
1880             aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
1881         } else {
1882             event_notifier_set_handler(&vq->host_notifier, true, NULL);
1883         }
1884     }
1885     if (!assign) {
1886         /* Test and clear notifier before after disabling event,
1887          * in case poll callback didn't have time to run. */
1888         virtio_queue_host_notifier_read(&vq->host_notifier);
1889     }
1890 }
1891 
1892 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
1893 {
1894     return &vq->host_notifier;
1895 }
1896 
1897 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
1898 {
1899     g_free(vdev->bus_name);
1900     vdev->bus_name = g_strdup(bus_name);
1901 }
1902 
1903 static void virtio_device_realize(DeviceState *dev, Error **errp)
1904 {
1905     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1906     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1907     Error *err = NULL;
1908 
1909     if (vdc->realize != NULL) {
1910         vdc->realize(dev, &err);
1911         if (err != NULL) {
1912             error_propagate(errp, err);
1913             return;
1914         }
1915     }
1916 
1917     virtio_bus_device_plugged(vdev, &err);
1918     if (err != NULL) {
1919         error_propagate(errp, err);
1920         return;
1921     }
1922 }
1923 
1924 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
1925 {
1926     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1927     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
1928     Error *err = NULL;
1929 
1930     virtio_bus_device_unplugged(vdev);
1931 
1932     if (vdc->unrealize != NULL) {
1933         vdc->unrealize(dev, &err);
1934         if (err != NULL) {
1935             error_propagate(errp, err);
1936             return;
1937         }
1938     }
1939 
1940     g_free(vdev->bus_name);
1941     vdev->bus_name = NULL;
1942 }
1943 
1944 static Property virtio_properties[] = {
1945     DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
1946     DEFINE_PROP_END_OF_LIST(),
1947 };
1948 
1949 static void virtio_device_class_init(ObjectClass *klass, void *data)
1950 {
1951     /* Set the default value here. */
1952     DeviceClass *dc = DEVICE_CLASS(klass);
1953 
1954     dc->realize = virtio_device_realize;
1955     dc->unrealize = virtio_device_unrealize;
1956     dc->bus_type = TYPE_VIRTIO_BUS;
1957     dc->props = virtio_properties;
1958 }
1959 
1960 static const TypeInfo virtio_device_info = {
1961     .name = TYPE_VIRTIO_DEVICE,
1962     .parent = TYPE_DEVICE,
1963     .instance_size = sizeof(VirtIODevice),
1964     .class_init = virtio_device_class_init,
1965     .abstract = true,
1966     .class_size = sizeof(VirtioDeviceClass),
1967 };
1968 
1969 static void virtio_register_types(void)
1970 {
1971     type_register_static(&virtio_device_info);
1972 }
1973 
1974 type_init(virtio_register_types)
1975