Lines Matching +full:- +full:- +full:disable +full:- +full:vhost +full:- +full:vsock
10 * the COPYING file in the top-level directory.
16 #include "qapi/qapi-commands-virtio.h"
18 #include "qemu/defer-call.h"
19 #include "qemu/error-report.h"
21 #include "qemu/main-loop.h"
27 #include "hw/virtio/vhost.h"
28 #include "migration/qemu-file-types.h"
30 #include "hw/virtio/virtio-bus.h"
31 #include "hw/qdev-properties.h"
32 #include "hw/virtio/virtio-access.h"
35 #include "virtio-qmp.h"
37 #include "standard-headers/linux/virtio_ids.h"
38 #include "standard-headers/linux/vhost_types.h"
39 #include "standard-headers/linux/virtio_blk.h"
40 #include "standard-headers/linux/virtio_console.h"
41 #include "standard-headers/linux/virtio_gpu.h"
42 #include "standard-headers/linux/virtio_net.h"
43 #include "standard-headers/linux/virtio_scsi.h"
44 #include "standard-headers/linux/virtio_i2c.h"
45 #include "standard-headers/linux/virtio_balloon.h"
46 #include "standard-headers/linux/virtio_iommu.h"
47 #include "standard-headers/linux/virtio_mem.h"
48 #include "standard-headers/linux/virtio_vsock.h"
159 [VIRTIO_ID_NET] = "virtio-net",
160 [VIRTIO_ID_BLOCK] = "virtio-blk",
161 [VIRTIO_ID_CONSOLE] = "virtio-serial",
162 [VIRTIO_ID_RNG] = "virtio-rng",
163 [VIRTIO_ID_BALLOON] = "virtio-balloon",
164 [VIRTIO_ID_IOMEM] = "virtio-iomem",
165 [VIRTIO_ID_RPMSG] = "virtio-rpmsg",
166 [VIRTIO_ID_SCSI] = "virtio-scsi",
167 [VIRTIO_ID_9P] = "virtio-9p",
168 [VIRTIO_ID_MAC80211_WLAN] = "virtio-mac-wlan",
169 [VIRTIO_ID_RPROC_SERIAL] = "virtio-rproc-serial",
170 [VIRTIO_ID_CAIF] = "virtio-caif",
171 [VIRTIO_ID_MEMORY_BALLOON] = "virtio-mem-balloon",
172 [VIRTIO_ID_GPU] = "virtio-gpu",
173 [VIRTIO_ID_CLOCK] = "virtio-clk",
174 [VIRTIO_ID_INPUT] = "virtio-input",
175 [VIRTIO_ID_VSOCK] = "vhost-vsock",
176 [VIRTIO_ID_CRYPTO] = "virtio-crypto",
177 [VIRTIO_ID_SIGNAL_DIST] = "virtio-signal",
178 [VIRTIO_ID_PSTORE] = "virtio-pstore",
179 [VIRTIO_ID_IOMMU] = "virtio-iommu",
180 [VIRTIO_ID_MEM] = "virtio-mem",
181 [VIRTIO_ID_SOUND] = "virtio-sound",
182 [VIRTIO_ID_FS] = "virtio-user-fs",
183 [VIRTIO_ID_PMEM] = "virtio-pmem",
184 [VIRTIO_ID_RPMB] = "virtio-rpmb",
185 [VIRTIO_ID_MAC80211_HWSIM] = "virtio-mac-hwsim",
186 [VIRTIO_ID_VIDEO_ENCODER] = "virtio-vid-encoder",
187 [VIRTIO_ID_VIDEO_DECODER] = "virtio-vid-decoder",
188 [VIRTIO_ID_SCMI] = "virtio-scmi",
189 [VIRTIO_ID_NITRO_SEC_MOD] = "virtio-nitro-sec-mod",
190 [VIRTIO_ID_I2C_ADAPTER] = "vhost-user-i2c",
191 [VIRTIO_ID_WATCHDOG] = "virtio-watchdog",
192 [VIRTIO_ID_CAN] = "virtio-can",
193 [VIRTIO_ID_DMABUF] = "virtio-dmabuf",
194 [VIRTIO_ID_PARAM_SERV] = "virtio-param-serv",
195 [VIRTIO_ID_AUDIO_POLICY] = "virtio-audio-pol",
196 [VIRTIO_ID_BT] = "virtio-bluetooth",
197 [VIRTIO_ID_GPIO] = "virtio-gpio"
212 address_space_cache_destroy(&caches->desc); in virtio_free_region_cache()
213 address_space_cache_destroy(&caches->avail); in virtio_free_region_cache()
214 address_space_cache_destroy(&caches->used); in virtio_free_region_cache()
222 caches = qatomic_read(&vq->vring.caches); in virtio_virtqueue_reset_region_cache()
223 qatomic_rcu_set(&vq->vring.caches, NULL); in virtio_virtqueue_reset_region_cache()
231 VirtQueue *vq = &vdev->vq[n]; in virtio_init_region_cache()
232 VRingMemoryRegionCaches *old = vq->vring.caches; in virtio_init_region_cache()
239 addr = vq->vring.desc; in virtio_init_region_cache()
245 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? in virtio_init_region_cache()
247 len = address_space_cache_init(&new->desc, vdev->dma_as, in virtio_init_region_cache()
255 len = address_space_cache_init(&new->used, vdev->dma_as, in virtio_init_region_cache()
256 vq->vring.used, size, true); in virtio_init_region_cache()
263 len = address_space_cache_init(&new->avail, vdev->dma_as, in virtio_init_region_cache()
264 vq->vring.avail, size, false); in virtio_init_region_cache()
270 qatomic_rcu_set(&vq->vring.caches, new); in virtio_init_region_cache()
277 address_space_cache_destroy(&new->avail); in virtio_init_region_cache()
279 address_space_cache_destroy(&new->used); in virtio_init_region_cache()
281 address_space_cache_destroy(&new->desc); in virtio_init_region_cache()
290 VRing *vring = &vdev->vq[n].vring; in virtio_queue_update_rings()
292 if (!vring->num || !vring->desc || !vring->align) { in virtio_queue_update_rings()
293 /* not yet setup -> nothing to do */ in virtio_queue_update_rings()
296 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); in virtio_queue_update_rings()
297 vring->used = vring_align(vring->avail + in virtio_queue_update_rings()
298 offsetof(VRingAvail, ring[vring->num]), in virtio_queue_update_rings()
299 vring->align); in virtio_queue_update_rings()
309 virtio_tswap64s(vdev, &desc->addr); in vring_split_desc_read()
310 virtio_tswap32s(vdev, &desc->len); in vring_split_desc_read()
311 virtio_tswap16s(vdev, &desc->flags); in vring_split_desc_read()
312 virtio_tswap16s(vdev, &desc->next); in vring_split_desc_read()
322 e->flags = virtio_lduw_phys_cached(vdev, cache, off_flags); in vring_packed_event_read()
325 e->off_wrap = virtio_lduw_phys_cached(vdev, cache, off_off); in vring_packed_event_read()
350 return qatomic_rcu_read(&vq->vring.caches); in vring_get_region_caches()
363 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); in vring_avail_flags()
376 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); in vring_avail_idx()
377 return vq->shadow_avail_idx; in vring_avail_idx()
390 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa); in vring_avail_ring()
396 return vring_avail_ring(vq, vq->vring.num); in vring_get_used_event()
410 virtio_tswap32s(vq->vdev, &uelem->id); in vring_used_write()
411 virtio_tswap32s(vq->vdev, &uelem->len); in vring_used_write()
412 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem)); in vring_used_write()
413 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem)); in vring_used_write()
426 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_flags()
439 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_idx()
449 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); in vring_used_idx_set()
450 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); in vring_used_idx_set()
453 vq->used_idx = val; in vring_used_idx_set()
460 VirtIODevice *vdev = vq->vdev; in vring_used_flags_set_bit()
468 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_flags_set_bit()
469 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask); in vring_used_flags_set_bit()
470 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); in vring_used_flags_set_bit()
477 VirtIODevice *vdev = vq->vdev; in vring_used_flags_unset_bit()
485 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa); in vring_used_flags_unset_bit()
486 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask); in vring_used_flags_unset_bit()
487 address_space_cache_invalidate(&caches->used, pa, sizeof(flags)); in vring_used_flags_unset_bit()
495 if (!vq->notification) { in vring_set_avail_event()
504 pa = offsetof(VRingUsed, ring[vq->vring.num]); in vring_set_avail_event()
505 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val); in vring_set_avail_event()
506 address_space_cache_invalidate(&caches->used, pa, sizeof(val)); in vring_set_avail_event()
513 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in virtio_queue_split_set_notification()
538 vring_packed_event_read(vq->vdev, &caches->used, &e); in virtio_queue_packed_set_notification()
542 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) { in virtio_queue_packed_set_notification()
543 off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15; in virtio_queue_packed_set_notification()
544 vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap); in virtio_queue_packed_set_notification()
552 vring_packed_flags_write(vq->vdev, &caches->used, e.flags); in virtio_queue_packed_set_notification()
561 return vq->notification; in virtio_queue_get_notification()
566 vq->notification = enable; in virtio_queue_set_notification()
568 if (!vq->vring.desc) { in virtio_queue_set_notification()
572 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_set_notification()
581 return vq->vring.avail != 0; in virtio_queue_ready()
601 vring_packed_desc_read_flags(vdev, &desc->flags, cache, i); in vring_packed_desc_read()
609 &desc->addr, sizeof(desc->addr)); in vring_packed_desc_read()
611 &desc->id, sizeof(desc->id)); in vring_packed_desc_read()
613 &desc->len, sizeof(desc->len)); in vring_packed_desc_read()
614 virtio_tswap64s(vdev, &desc->addr); in vring_packed_desc_read()
615 virtio_tswap16s(vdev, &desc->id); in vring_packed_desc_read()
616 virtio_tswap32s(vdev, &desc->len); in vring_packed_desc_read()
629 virtio_tswap32s(vdev, &desc->len); in vring_packed_desc_write_data()
630 virtio_tswap16s(vdev, &desc->id); in vring_packed_desc_write_data()
631 address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id)); in vring_packed_desc_write_data()
632 address_space_cache_invalidate(cache, off_id, sizeof(desc->id)); in vring_packed_desc_write_data()
633 address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len)); in vring_packed_desc_write_data()
634 address_space_cache_invalidate(cache, off_len, sizeof(desc->len)); in vring_packed_desc_write_data()
644 virtio_stw_phys_cached(vdev, cache, off, desc->flags); in vring_packed_desc_write_flags()
645 address_space_cache_invalidate(cache, off, sizeof(desc->flags)); in vring_packed_desc_write_flags()
675 if (virtio_device_disabled(vq->vdev)) { in virtio_queue_empty_rcu()
679 if (unlikely(!vq->vring.avail)) { in virtio_queue_empty_rcu()
683 if (vq->shadow_avail_idx != vq->last_avail_idx) { in virtio_queue_empty_rcu()
687 return vring_avail_idx(vq) == vq->last_avail_idx; in virtio_queue_empty_rcu()
694 if (virtio_device_disabled(vq->vdev)) { in virtio_queue_split_empty()
698 if (unlikely(!vq->vring.avail)) { in virtio_queue_split_empty()
702 if (vq->shadow_avail_idx != vq->last_avail_idx) { in virtio_queue_split_empty()
707 empty = vring_avail_idx(vq) == vq->last_avail_idx; in virtio_queue_split_empty()
717 if (unlikely(!vq->vring.desc)) { in virtio_queue_packed_empty_rcu()
726 vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc, in virtio_queue_packed_empty_rcu()
727 vq->last_avail_idx); in virtio_queue_packed_empty_rcu()
729 return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter); in virtio_queue_packed_empty_rcu()
740 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_empty()
749 if (unlikely(!vq->vring.avail)) { in virtio_queue_split_poll()
761 if (unlikely(!vq->vring.desc)) { in virtio_queue_packed_poll()
770 vring_packed_desc_read(vq->vdev, &desc, &caches->desc, in virtio_queue_packed_poll()
773 return is_desc_avail(desc.flags, vq->shadow_avail_wrap_counter); in virtio_queue_packed_poll()
778 if (virtio_device_disabled(vq->vdev)) { in virtio_queue_poll()
782 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_poll()
804 AddressSpace *dma_as = vq->vdev->dma_as; in virtqueue_unmap_sg()
809 for (i = 0; i < elem->in_num; i++) { in virtqueue_unmap_sg()
810 size_t size = MIN(len - offset, elem->in_sg[i].iov_len); in virtqueue_unmap_sg()
812 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base, in virtqueue_unmap_sg()
813 elem->in_sg[i].iov_len, in virtqueue_unmap_sg()
819 for (i = 0; i < elem->out_num; i++) in virtqueue_unmap_sg()
820 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base, in virtqueue_unmap_sg()
821 elem->out_sg[i].iov_len, in virtqueue_unmap_sg()
823 elem->out_sg[i].iov_len); in virtqueue_unmap_sg()
838 vq->inuse -= elem->ndescs; in virtqueue_detach_element()
844 vq->last_avail_idx -= num; in virtqueue_split_rewind()
849 if (vq->last_avail_idx < num) { in virtqueue_packed_rewind()
850 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num; in virtqueue_packed_rewind()
851 vq->last_avail_wrap_counter ^= 1; in virtqueue_packed_rewind()
853 vq->last_avail_idx -= num; in virtqueue_packed_rewind()
869 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_unpop()
892 if (num > vq->inuse) { in virtqueue_rewind()
896 vq->inuse -= num; in virtqueue_rewind()
897 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_rewind()
910 if (unlikely(!vq->vring.used)) { in virtqueue_split_fill()
914 idx = (idx + vq->used_idx) % vq->vring.num; in virtqueue_split_fill()
916 uelem.id = elem->index; in virtqueue_split_fill()
924 vq->used_elems[idx].index = elem->index; in virtqueue_packed_fill()
925 vq->used_elems[idx].len = len; in virtqueue_packed_fill()
926 vq->used_elems[idx].ndescs = elem->ndescs; in virtqueue_packed_fill()
934 i = vq->used_idx % vq->vring.num; in virtqueue_ordered_fill()
940 max_steps = (vq->last_avail_idx - vq->used_idx) % vq->vring.num; in virtqueue_ordered_fill()
942 /* Search for element in vq->used_elems */ in virtqueue_ordered_fill()
945 if (vq->used_elems[i].index == elem->index) { in virtqueue_ordered_fill()
946 vq->used_elems[i].len = len; in virtqueue_ordered_fill()
947 vq->used_elems[i].in_order_filled = true; in virtqueue_ordered_fill()
951 i += vq->used_elems[i].ndescs; in virtqueue_ordered_fill()
952 steps += vq->used_elems[i].ndescs; in virtqueue_ordered_fill()
954 if (i >= vq->vring.num) { in virtqueue_ordered_fill()
955 i -= vq->vring.num; in virtqueue_ordered_fill()
965 __func__, vq->vdev->name, elem->index); in virtqueue_ordered_fill()
977 .id = elem->index, in virtqueue_packed_fill_desc()
978 .len = elem->len, in virtqueue_packed_fill_desc()
980 bool wrap_counter = vq->used_wrap_counter; in virtqueue_packed_fill_desc()
982 if (unlikely(!vq->vring.desc)) { in virtqueue_packed_fill_desc()
986 head = vq->used_idx + idx; in virtqueue_packed_fill_desc()
987 if (head >= vq->vring.num) { in virtqueue_packed_fill_desc()
988 head -= vq->vring.num; in virtqueue_packed_fill_desc()
1004 vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order); in virtqueue_packed_fill_desc()
1015 if (virtio_device_disabled(vq->vdev)) { in virtqueue_fill()
1019 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) { in virtqueue_fill()
1021 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_fill()
1033 if (unlikely(!vq->vring.used)) { in virtqueue_split_flush()
1040 old = vq->used_idx; in virtqueue_split_flush()
1043 vq->inuse -= count; in virtqueue_split_flush()
1044 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) in virtqueue_split_flush()
1045 vq->signalled_used_valid = false; in virtqueue_split_flush()
1052 if (unlikely(!vq->vring.desc)) { in virtqueue_packed_flush()
1062 * the value of 'vq->used_idx' plus the 'ndescs'. in virtqueue_packed_flush()
1064 ndescs += vq->used_elems[0].ndescs; in virtqueue_packed_flush()
1066 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false); in virtqueue_packed_flush()
1067 ndescs += vq->used_elems[i].ndescs; in virtqueue_packed_flush()
1069 virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true); in virtqueue_packed_flush()
1071 vq->inuse -= ndescs; in virtqueue_packed_flush()
1072 vq->used_idx += ndescs; in virtqueue_packed_flush()
1073 if (vq->used_idx >= vq->vring.num) { in virtqueue_packed_flush()
1074 vq->used_idx -= vq->vring.num; in virtqueue_packed_flush()
1075 vq->used_wrap_counter ^= 1; in virtqueue_packed_flush()
1076 vq->signalled_used_valid = false; in virtqueue_packed_flush()
1082 unsigned int i = vq->used_idx % vq->vring.num; in virtqueue_ordered_flush()
1084 uint16_t old = vq->used_idx; in virtqueue_ordered_flush()
1089 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED); in virtqueue_ordered_flush()
1092 if (unlikely(!vq->vring.desc)) { in virtqueue_ordered_flush()
1095 } else if (unlikely(!vq->vring.used)) { in virtqueue_ordered_flush()
1099 /* First expected in-order element isn't ready, nothing to do */ in virtqueue_ordered_flush()
1100 if (!vq->used_elems[i].in_order_filled) { in virtqueue_ordered_flush()
1104 /* Search for filled elements in-order */ in virtqueue_ordered_flush()
1105 while (vq->used_elems[i].in_order_filled) { in virtqueue_ordered_flush()
1110 if (packed && i != vq->used_idx) { in virtqueue_ordered_flush()
1111 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], ndescs, false); in virtqueue_ordered_flush()
1113 uelem.id = vq->used_elems[i].index; in virtqueue_ordered_flush()
1114 uelem.len = vq->used_elems[i].len; in virtqueue_ordered_flush()
1118 vq->used_elems[i].in_order_filled = false; in virtqueue_ordered_flush()
1119 ndescs += vq->used_elems[i].ndescs; in virtqueue_ordered_flush()
1120 i += vq->used_elems[i].ndescs; in virtqueue_ordered_flush()
1121 if (i >= vq->vring.num) { in virtqueue_ordered_flush()
1122 i -= vq->vring.num; in virtqueue_ordered_flush()
1127 virtqueue_packed_fill_desc(vq, &vq->used_elems[vq->used_idx], 0, true); in virtqueue_ordered_flush()
1128 vq->used_idx += ndescs; in virtqueue_ordered_flush()
1129 if (vq->used_idx >= vq->vring.num) { in virtqueue_ordered_flush()
1130 vq->used_idx -= vq->vring.num; in virtqueue_ordered_flush()
1131 vq->used_wrap_counter ^= 1; in virtqueue_ordered_flush()
1132 vq->signalled_used_valid = false; in virtqueue_ordered_flush()
1139 if (unlikely((int16_t)(new - vq->signalled_used) < in virtqueue_ordered_flush()
1140 (uint16_t)(new - old))) { in virtqueue_ordered_flush()
1141 vq->signalled_used_valid = false; in virtqueue_ordered_flush()
1144 vq->inuse -= ndescs; in virtqueue_ordered_flush()
1149 if (virtio_device_disabled(vq->vdev)) { in virtqueue_flush()
1150 vq->inuse -= count; in virtqueue_flush()
1154 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_IN_ORDER)) { in virtqueue_flush()
1156 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_flush()
1177 avail_idx = (vq->shadow_avail_idx != idx) ? vq->shadow_avail_idx in virtqueue_num_heads()
1179 num_heads = avail_idx - idx; in virtqueue_num_heads()
1182 if (num_heads > vq->vring.num) { in virtqueue_num_heads()
1183 virtio_error(vq->vdev, "Guest moved used index from %u to %u", in virtqueue_num_heads()
1184 idx, vq->shadow_avail_idx); in virtqueue_num_heads()
1185 return -EINVAL; in virtqueue_num_heads()
1188 * On success, callers read a descriptor at vq->last_avail_idx. in virtqueue_num_heads()
1209 *head = vring_avail_ring(vq, idx % vq->vring.num); in virtqueue_get_head()
1212 if (*head >= vq->vring.num) { in virtqueue_get_head()
1213 virtio_error(vq->vdev, "Guest says index %u is available", *head); in virtqueue_get_head()
1221 VIRTQUEUE_READ_DESC_ERROR = -1,
1226 /* Reads the 'desc->next' descriptor into '*desc'. */
1232 if (!(desc->flags & VRING_DESC_F_NEXT)) { in virtqueue_split_read_next_desc()
1237 if (desc->next >= max) { in virtqueue_split_read_next_desc()
1238 virtio_error(vdev, "Desc next is %u", desc->next); in virtqueue_split_read_next_desc()
1242 vring_split_desc_read(vdev, desc, desc_cache, desc->next); in virtqueue_split_read_next_desc()
1252 VirtIODevice *vdev = vq->vdev; in virtqueue_split_get_avail_bytes()
1261 idx = vq->last_avail_idx; in virtqueue_split_get_avail_bytes()
1265 MemoryRegionCache *desc_cache = &caches->desc; in virtqueue_split_get_avail_bytes()
1269 unsigned int max = vq->vring.num; in virtqueue_split_get_avail_bytes()
1293 vdev->dma_as, in virtqueue_split_get_avail_bytes()
1365 if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) { in virtqueue_packed_read_next_desc()
1374 (*next) -= vq->vring.num; in virtqueue_packed_read_next_desc()
1378 vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false); in virtqueue_packed_read_next_desc()
1390 VirtIODevice *vdev = vq->vdev; in virtqueue_packed_get_avail_bytes()
1401 idx = vq->last_avail_idx; in virtqueue_packed_get_avail_bytes()
1402 wrap_counter = vq->last_avail_wrap_counter; in virtqueue_packed_get_avail_bytes()
1409 unsigned int max = vq->vring.num; in virtqueue_packed_get_avail_bytes()
1411 desc_cache = &caches->desc; in virtqueue_packed_get_avail_bytes()
1432 vdev->dma_as, in virtqueue_packed_get_avail_bytes()
1471 idx += num_bufs - total_bufs; in virtqueue_packed_get_avail_bytes()
1475 if (idx >= vq->vring.num) { in virtqueue_packed_get_avail_bytes()
1476 idx -= vq->vring.num; in virtqueue_packed_get_avail_bytes()
1482 vq->shadow_avail_idx = idx; in virtqueue_packed_get_avail_bytes()
1483 vq->shadow_avail_wrap_counter = wrap_counter; in virtqueue_packed_get_avail_bytes()
1508 if (unlikely(!vq->vring.desc)) { in virtqueue_get_avail_bytes()
1517 desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ? in virtqueue_get_avail_bytes()
1519 if (caches->desc.len < vq->vring.num * desc_size) { in virtqueue_get_avail_bytes()
1520 virtio_error(vq->vdev, "Cannot map descriptor ring"); in virtqueue_get_avail_bytes()
1524 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_get_avail_bytes()
1534 return (int)vq->shadow_avail_idx; in virtqueue_get_avail_bytes()
1543 return -1; in virtqueue_get_avail_bytes()
1578 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len, in virtqueue_map_desc()
1591 sz -= len; in virtqueue_map_desc()
1614 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); in virtqueue_undo_map_desc()
1628 sg[i].iov_base = dma_memory_map(vdev->dma_as, in virtqueue_map_iovec()
1646 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true); in virtqueue_map()
1647 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num, in virtqueue_map()
1654 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0])); in virtqueue_alloc_element()
1655 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]); in virtqueue_alloc_element()
1656 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]); in virtqueue_alloc_element()
1657 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0])); in virtqueue_alloc_element()
1658 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]); in virtqueue_alloc_element()
1659 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]); in virtqueue_alloc_element()
1664 elem->out_num = out_num; in virtqueue_alloc_element()
1665 elem->in_num = in_num; in virtqueue_alloc_element()
1666 elem->in_addr = (void *)elem + in_addr_ofs; in virtqueue_alloc_element()
1667 elem->out_addr = (void *)elem + out_addr_ofs; in virtqueue_alloc_element()
1668 elem->in_sg = (void *)elem + in_sg_ofs; in virtqueue_alloc_element()
1669 elem->out_sg = (void *)elem + out_sg_ofs; in virtqueue_alloc_element()
1680 VirtIODevice *vdev = vq->vdev; in virtqueue_split_pop()
1701 max = vq->vring.num; in virtqueue_split_pop()
1703 if (vq->inuse >= vq->vring.num) { in virtqueue_split_pop()
1708 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) { in virtqueue_split_pop()
1713 vring_set_avail_event(vq, vq->last_avail_idx); in virtqueue_split_pop()
1724 if (caches->desc.len < max * sizeof(VRingDesc)) { in virtqueue_split_pop()
1729 desc_cache = &caches->desc; in virtqueue_split_pop()
1738 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, in virtqueue_split_pop()
1758 VIRTQUEUE_MAX_SIZE - out_num, true, in virtqueue_split_pop()
1788 elem->index = head; in virtqueue_split_pop()
1789 elem->ndescs = 1; in virtqueue_split_pop()
1791 elem->out_addr[i] = addr[i]; in virtqueue_split_pop()
1792 elem->out_sg[i] = iov[i]; in virtqueue_split_pop()
1795 elem->in_addr[i] = addr[out_num + i]; in virtqueue_split_pop()
1796 elem->in_sg[i] = iov[out_num + i]; in virtqueue_split_pop()
1800 idx = (vq->last_avail_idx - 1) % vq->vring.num; in virtqueue_split_pop()
1801 vq->used_elems[idx].index = elem->index; in virtqueue_split_pop()
1802 vq->used_elems[idx].len = elem->len; in virtqueue_split_pop()
1803 vq->used_elems[idx].ndescs = elem->ndescs; in virtqueue_split_pop()
1806 vq->inuse++; in virtqueue_split_pop()
1808 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); in virtqueue_split_pop()
1826 VirtIODevice *vdev = vq->vdev; in virtqueue_packed_pop()
1845 max = vq->vring.num; in virtqueue_packed_pop()
1847 if (vq->inuse >= vq->vring.num) { in virtqueue_packed_pop()
1852 i = vq->last_avail_idx; in virtqueue_packed_pop()
1860 if (caches->desc.len < max * sizeof(VRingDesc)) { in virtqueue_packed_pop()
1865 desc_cache = &caches->desc; in virtqueue_packed_pop()
1875 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, in virtqueue_packed_pop()
1895 VIRTQUEUE_MAX_SIZE - out_num, true, in virtqueue_packed_pop()
1929 elem->out_addr[i] = addr[i]; in virtqueue_packed_pop()
1930 elem->out_sg[i] = iov[i]; in virtqueue_packed_pop()
1933 elem->in_addr[i] = addr[out_num + i]; in virtqueue_packed_pop()
1934 elem->in_sg[i] = iov[out_num + i]; in virtqueue_packed_pop()
1937 elem->index = id; in virtqueue_packed_pop()
1938 elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries; in virtqueue_packed_pop()
1941 vq->used_elems[vq->last_avail_idx].index = elem->index; in virtqueue_packed_pop()
1942 vq->used_elems[vq->last_avail_idx].len = elem->len; in virtqueue_packed_pop()
1943 vq->used_elems[vq->last_avail_idx].ndescs = elem->ndescs; in virtqueue_packed_pop()
1946 vq->last_avail_idx += elem->ndescs; in virtqueue_packed_pop()
1947 vq->inuse += elem->ndescs; in virtqueue_packed_pop()
1949 if (vq->last_avail_idx >= vq->vring.num) { in virtqueue_packed_pop()
1950 vq->last_avail_idx -= vq->vring.num; in virtqueue_packed_pop()
1951 vq->last_avail_wrap_counter ^= 1; in virtqueue_packed_pop()
1954 vq->shadow_avail_idx = vq->last_avail_idx; in virtqueue_packed_pop()
1955 vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter; in virtqueue_packed_pop()
1957 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); in virtqueue_packed_pop()
1970 if (virtio_device_disabled(vq->vdev)) { in virtqueue_pop()
1974 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtqueue_pop()
1987 VirtIODevice *vdev = vq->vdev; in virtqueue_packed_drop_all()
1997 desc_cache = &caches->desc; in virtqueue_packed_drop_all()
2001 while (vq->inuse < vq->vring.num) { in virtqueue_packed_drop_all()
2002 unsigned int idx = vq->last_avail_idx; in virtqueue_packed_drop_all()
2008 vq->last_avail_idx , true); in virtqueue_packed_drop_all()
2009 if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) { in virtqueue_packed_drop_all()
2015 vq->vring.num, &idx, false)) { in virtqueue_packed_drop_all()
2024 vq->last_avail_idx += elem.ndescs; in virtqueue_packed_drop_all()
2025 if (vq->last_avail_idx >= vq->vring.num) { in virtqueue_packed_drop_all()
2026 vq->last_avail_idx -= vq->vring.num; in virtqueue_packed_drop_all()
2027 vq->last_avail_wrap_counter ^= 1; in virtqueue_packed_drop_all()
2038 VirtIODevice *vdev = vq->vdev; in virtqueue_split_drop_all()
2041 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { in virtqueue_split_drop_all()
2045 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) { in virtqueue_split_drop_all()
2048 vq->inuse++; in virtqueue_split_drop_all()
2049 vq->last_avail_idx++; in virtqueue_split_drop_all()
2051 vring_set_avail_event(vq, vq->last_avail_idx); in virtqueue_split_drop_all()
2070 struct VirtIODevice *vdev = vq->vdev; in virtqueue_drop_all()
2072 if (virtio_device_disabled(vq->vdev)) { in virtqueue_drop_all()
2086 * In the meanwhile, since the in-memory layout of VirtQueueElement
2117 elem->index = data.index; in qemu_get_virtqueue_element()
2119 for (i = 0; i < elem->in_num; i++) { in qemu_get_virtqueue_element()
2120 elem->in_addr[i] = data.in_addr[i]; in qemu_get_virtqueue_element()
2123 for (i = 0; i < elem->out_num; i++) { in qemu_get_virtqueue_element()
2124 elem->out_addr[i] = data.out_addr[i]; in qemu_get_virtqueue_element()
2127 for (i = 0; i < elem->in_num; i++) { in qemu_get_virtqueue_element()
2129 elem->in_sg[i].iov_base = 0; in qemu_get_virtqueue_element()
2130 elem->in_sg[i].iov_len = data.in_sg[i].iov_len; in qemu_get_virtqueue_element()
2133 for (i = 0; i < elem->out_num; i++) { in qemu_get_virtqueue_element()
2135 elem->out_sg[i].iov_base = 0; in qemu_get_virtqueue_element()
2136 elem->out_sg[i].iov_len = data.out_sg[i].iov_len; in qemu_get_virtqueue_element()
2140 qemu_get_be32s(f, &elem->ndescs); in qemu_get_virtqueue_element()
2154 data.index = elem->index; in qemu_put_virtqueue_element()
2155 data.in_num = elem->in_num; in qemu_put_virtqueue_element()
2156 data.out_num = elem->out_num; in qemu_put_virtqueue_element()
2158 for (i = 0; i < elem->in_num; i++) { in qemu_put_virtqueue_element()
2159 data.in_addr[i] = elem->in_addr[i]; in qemu_put_virtqueue_element()
2162 for (i = 0; i < elem->out_num; i++) { in qemu_put_virtqueue_element()
2163 data.out_addr[i] = elem->out_addr[i]; in qemu_put_virtqueue_element()
2166 for (i = 0; i < elem->in_num; i++) { in qemu_put_virtqueue_element()
2169 data.in_sg[i].iov_len = elem->in_sg[i].iov_len; in qemu_put_virtqueue_element()
2172 for (i = 0; i < elem->out_num; i++) { in qemu_put_virtqueue_element()
2174 data.out_sg[i].iov_len = elem->out_sg[i].iov_len; in qemu_put_virtqueue_element()
2178 qemu_put_be32s(f, &elem->ndescs); in qemu_put_virtqueue_element()
2194 if (k->notify) { in virtio_notify_vector()
2195 k->notify(qbus->parent, vector); in virtio_notify_vector()
2210 return -EFAULT; in virtio_validate_features()
2213 if (k->validate_features) { in virtio_validate_features()
2214 return k->validate_features(vdev); in virtio_validate_features()
2226 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) && in virtio_set_status()
2236 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) != in virtio_set_status()
2241 if (k->set_status) { in virtio_set_status()
2242 k->set_status(vdev, val); in virtio_set_status()
2244 vdev->status = val; in virtio_set_status()
2269 vdev->vq[i].vring.desc = 0; in __virtio_queue_reset()
2270 vdev->vq[i].vring.avail = 0; in __virtio_queue_reset()
2271 vdev->vq[i].vring.used = 0; in __virtio_queue_reset()
2272 vdev->vq[i].last_avail_idx = 0; in __virtio_queue_reset()
2273 vdev->vq[i].shadow_avail_idx = 0; in __virtio_queue_reset()
2274 vdev->vq[i].used_idx = 0; in __virtio_queue_reset()
2275 vdev->vq[i].last_avail_wrap_counter = true; in __virtio_queue_reset()
2276 vdev->vq[i].shadow_avail_wrap_counter = true; in __virtio_queue_reset()
2277 vdev->vq[i].used_wrap_counter = true; in __virtio_queue_reset()
2279 vdev->vq[i].signalled_used = 0; in __virtio_queue_reset()
2280 vdev->vq[i].signalled_used_valid = false; in __virtio_queue_reset()
2281 vdev->vq[i].notification = true; in __virtio_queue_reset()
2282 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; in __virtio_queue_reset()
2283 vdev->vq[i].inuse = 0; in __virtio_queue_reset()
2284 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); in __virtio_queue_reset()
2291 if (k->queue_reset) { in virtio_queue_reset()
2292 k->queue_reset(vdev, queue_index); in virtio_queue_reset()
2305 * be re-enabled for new machine types only, and also after in virtio_queue_enable()
2314 if (k->queue_enable) { in virtio_queue_enable()
2315 k->queue_enable(vdev, queue_index); in virtio_queue_enable()
2328 vdev->device_endian = virtio_current_cpu_endian(); in virtio_reset()
2331 vdev->device_endian = virtio_default_endian(); in virtio_reset()
2334 if (k->get_vhost) { in virtio_reset()
2335 struct vhost_dev *hdev = k->get_vhost(vdev); in virtio_reset()
2336 /* Only reset when vhost back-end is connected */ in virtio_reset()
2337 if (hdev && hdev->vhost_ops) { in virtio_reset()
2342 if (k->reset) { in virtio_reset()
2343 k->reset(vdev); in virtio_reset()
2346 vdev->start_on_kick = false; in virtio_reset()
2347 vdev->started = false; in virtio_reset()
2348 vdev->broken = false; in virtio_reset()
2349 vdev->guest_features = 0; in virtio_reset()
2350 vdev->queue_sel = 0; in virtio_reset()
2351 vdev->status = 0; in virtio_reset()
2352 vdev->disabled = false; in virtio_reset()
2353 qatomic_set(&vdev->isr, 0); in virtio_reset()
2354 vdev->config_vector = VIRTIO_NO_VECTOR; in virtio_reset()
2355 virtio_notify_vector(vdev, vdev->config_vector); in virtio_reset()
2364 if (!vdev->vq[n].vring.num) { in virtio_queue_set_addr()
2367 vdev->vq[n].vring.desc = addr; in virtio_queue_set_addr()
2373 return vdev->vq[n].vring.desc; in virtio_queue_get_addr()
2379 if (!vdev->vq[n].vring.num) { in virtio_queue_set_rings()
2382 vdev->vq[n].vring.desc = desc; in virtio_queue_set_rings()
2383 vdev->vq[n].vring.avail = avail; in virtio_queue_set_rings()
2384 vdev->vq[n].vring.used = used; in virtio_queue_set_rings()
2393 if (!!num != !!vdev->vq[n].vring.num || in virtio_queue_set_num()
2398 vdev->vq[n].vring.num = num; in virtio_queue_set_num()
2403 return QLIST_FIRST(&vdev->vector_queues[vector]); in virtio_vector_first_queue()
2413 return vdev->vq[n].vring.num; in virtio_queue_get_num()
2418 return vdev->vq[n].vring.num_default; in virtio_queue_get_max_num()
2439 /* virtio-1 compliant devices cannot change the alignment */ in virtio_queue_set_align()
2441 error_report("tried to modify queue alignment for virtio-1 device"); in virtio_queue_set_align()
2448 assert(k->has_variable_vring_alignment); in virtio_queue_set_align()
2451 vdev->vq[n].vring.align = align; in virtio_queue_set_align()
2458 if (!vq->vring.desc) { in virtio_queue_set_shadow_avail_idx()
2463 * 16-bit data for packed VQs include 1-bit wrap counter and in virtio_queue_set_shadow_avail_idx()
2464 * 15-bit shadow_avail_idx. in virtio_queue_set_shadow_avail_idx()
2466 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) { in virtio_queue_set_shadow_avail_idx()
2467 vq->shadow_avail_wrap_counter = (shadow_avail_idx >> 15) & 0x1; in virtio_queue_set_shadow_avail_idx()
2468 vq->shadow_avail_idx = shadow_avail_idx & 0x7FFF; in virtio_queue_set_shadow_avail_idx()
2470 vq->shadow_avail_idx = shadow_avail_idx; in virtio_queue_set_shadow_avail_idx()
2476 if (vq->vring.desc && vq->handle_output) { in virtio_queue_notify_vq()
2477 VirtIODevice *vdev = vq->vdev; in virtio_queue_notify_vq()
2479 if (unlikely(vdev->broken)) { in virtio_queue_notify_vq()
2483 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); in virtio_queue_notify_vq()
2484 vq->handle_output(vdev, vq); in virtio_queue_notify_vq()
2486 if (unlikely(vdev->start_on_kick)) { in virtio_queue_notify_vq()
2494 VirtQueue *vq = &vdev->vq[n]; in virtio_queue_notify()
2496 if (unlikely(!vq->vring.desc || vdev->broken)) { in virtio_queue_notify()
2500 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq); in virtio_queue_notify()
2501 if (vq->host_notifier_enabled) { in virtio_queue_notify()
2502 event_notifier_set(&vq->host_notifier); in virtio_queue_notify()
2503 } else if (vq->handle_output) { in virtio_queue_notify()
2504 vq->handle_output(vdev, vq); in virtio_queue_notify()
2506 if (unlikely(vdev->start_on_kick)) { in virtio_queue_notify()
2514 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector : in virtio_queue_vector()
2520 VirtQueue *vq = &vdev->vq[n]; in virtio_queue_set_vector()
2523 if (vdev->vector_queues && in virtio_queue_set_vector()
2524 vdev->vq[n].vector != VIRTIO_NO_VECTOR) { in virtio_queue_set_vector()
2527 vdev->vq[n].vector = vector; in virtio_queue_set_vector()
2528 if (vdev->vector_queues && in virtio_queue_set_vector()
2530 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node); in virtio_queue_set_vector()
2541 if (vdev->vq[i].vring.num == 0) in virtio_add_queue()
2548 vdev->vq[i].vring.num = queue_size; in virtio_add_queue()
2549 vdev->vq[i].vring.num_default = queue_size; in virtio_add_queue()
2550 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; in virtio_add_queue()
2551 vdev->vq[i].handle_output = handle_output; in virtio_add_queue()
2552 vdev->vq[i].used_elems = g_new0(VirtQueueElement, queue_size); in virtio_add_queue()
2554 return &vdev->vq[i]; in virtio_add_queue()
2559 vq->vring.num = 0; in virtio_delete_queue()
2560 vq->vring.num_default = 0; in virtio_delete_queue()
2561 vq->handle_output = NULL; in virtio_delete_queue()
2562 g_free(vq->used_elems); in virtio_delete_queue()
2563 vq->used_elems = NULL; in virtio_delete_queue()
2573 virtio_delete_queue(&vdev->vq[n]); in virtio_del_queue()
2578 uint8_t old = qatomic_read(&vdev->isr); in virtio_set_isr()
2584 qatomic_or(&vdev->isr, value); in virtio_set_isr()
2597 !vq->inuse && virtio_queue_empty(vq)) { in virtio_split_should_notify()
2605 v = vq->signalled_used_valid; in virtio_split_should_notify()
2606 vq->signalled_used_valid = true; in virtio_split_should_notify()
2607 old = vq->signalled_used; in virtio_split_should_notify()
2608 new = vq->signalled_used = vq->used_idx; in virtio_split_should_notify()
2619 off -= vq->vring.num; in vring_packed_need_event()
2638 vring_packed_event_read(vdev, &caches->avail, &e); in virtio_packed_should_notify()
2640 old = vq->signalled_used; in virtio_packed_should_notify()
2641 new = vq->signalled_used = vq->used_idx; in virtio_packed_should_notify()
2642 v = vq->signalled_used_valid; in virtio_packed_should_notify()
2643 vq->signalled_used_valid = true; in virtio_packed_should_notify()
2651 return !v || vring_packed_need_event(vq, vq->used_wrap_counter, in virtio_packed_should_notify()
2671 trace_virtio_notify_irqfd_deferred_fn(vq->vdev, vq); in virtio_notify_irqfd_deferred_fn()
2687 * windows drivers included in virtio-win 1.8.0 (circa 2015) are in virtio_notify_irqfd()
2700 virtio_set_isr(vq->vdev, 0x1); in virtio_notify_irqfd()
2701 defer_call(virtio_notify_irqfd_deferred_fn, &vq->guest_notifier); in virtio_notify_irqfd()
2706 virtio_set_isr(vq->vdev, 0x1); in virtio_irq()
2707 virtio_notify_vector(vq->vdev, vq->vector); in virtio_irq()
2724 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) in virtio_notify_config()
2728 vdev->generation++; in virtio_notify_config()
2729 virtio_notify_vector(vdev, vdev->config_vector); in virtio_notify_config()
2736 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN); in virtio_device_endian_needed()
2738 return vdev->device_endian != virtio_default_endian(); in virtio_device_endian_needed()
2741 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE; in virtio_device_endian_needed()
2748 return (vdev->host_features >> 32) != 0; in virtio_64bit_features_needed()
2771 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { in virtio_ringsize_needed()
2784 return k->has_extra_state && in virtio_extra_state_needed()
2785 k->has_extra_state(qbus->parent); in virtio_extra_state_needed()
2792 return vdev->broken; in virtio_broken_needed()
2799 return vdev->started; in virtio_started_needed()
2806 return vdev->disabled; in virtio_disabled_needed()
2887 if (!k->load_extra_state) { in get_extra_state()
2888 return -1; in get_extra_state()
2890 return k->load_extra_state(qbus->parent, f); in get_extra_state()
2901 k->save_extra_state(qbus->parent, f); in put_extra_state()
3011 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff); in virtio_save()
3014 if (k->save_config) { in virtio_save()
3015 k->save_config(qbus->parent, f); in virtio_save()
3018 qemu_put_8s(f, &vdev->status); in virtio_save()
3019 qemu_put_8s(f, &vdev->isr); in virtio_save()
3020 qemu_put_be16s(f, &vdev->queue_sel); in virtio_save()
3022 qemu_put_be32(f, vdev->config_len); in virtio_save()
3023 qemu_put_buffer(f, vdev->config, vdev->config_len); in virtio_save()
3026 if (vdev->vq[i].vring.num == 0) in virtio_save()
3033 if (vdev->vq[i].vring.num == 0) in virtio_save()
3036 qemu_put_be32(f, vdev->vq[i].vring.num); in virtio_save()
3037 if (k->has_variable_vring_alignment) { in virtio_save()
3038 qemu_put_be32(f, vdev->vq[i].vring.align); in virtio_save()
3042 * subsections for VIRTIO-1 devices. in virtio_save()
3044 qemu_put_be64(f, vdev->vq[i].vring.desc); in virtio_save()
3045 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); in virtio_save()
3046 if (k->save_queue) { in virtio_save()
3047 k->save_queue(qbus->parent, i, f); in virtio_save()
3051 if (vdc->save != NULL) { in virtio_save()
3052 vdc->save(vdev, f); in virtio_save()
3055 if (vdc->vmsd) { in virtio_save()
3056 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL); in virtio_save()
3081 return virtio_load(vdev, f, dc->vmsd->version_id); in virtio_device_get()
3093 bool bad = (val & ~(vdev->host_features)) != 0; in virtio_set_features_nocheck()
3095 val &= vdev->host_features; in virtio_set_features_nocheck()
3096 if (k->set_features) { in virtio_set_features_nocheck()
3097 k->set_features(vdev, val); in virtio_set_features_nocheck()
3099 vdev->guest_features = val; in virtio_set_features_nocheck()
3100 return bad ? -1 : 0; in virtio_set_features_nocheck()
3114 data->ret = virtio_set_features_nocheck(data->vdev, data->val); in virtio_set_features_nocheck_bh()
3115 aio_co_wake(data->co); in virtio_set_features_nocheck_bh()
3143 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) { in virtio_set_features()
3144 return -EINVAL; in virtio_set_features()
3150 __func__, vdev->name); in virtio_set_features()
3158 if (vdev->vq[i].vring.num != 0) { in virtio_set_features()
3164 if (!virtio_device_started(vdev, vdev->status) && in virtio_set_features()
3166 vdev->start_on_kick = true; in virtio_set_features()
3177 DeviceState *proxy = DEVICE(BUS(bus)->parent); in virtio_device_check_notification_compatibility()
3180 k->ioeventfd_enabled(proxy)) { in virtio_device_check_notification_compatibility()
3189 size_t config_size = params->min_size; in virtio_get_config_size()
3190 const VirtIOFeature *feature_sizes = params->feature_sizes; in virtio_get_config_size()
3199 assert(config_size <= params->max_size); in virtio_get_config_size()
3218 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN; in virtio_load()
3220 if (k->load_config) { in virtio_load()
3221 ret = k->load_config(qbus->parent, f); in virtio_load()
3226 qemu_get_8s(f, &vdev->status); in virtio_load()
3227 qemu_get_8s(f, &vdev->isr); in virtio_load()
3228 qemu_get_be16s(f, &vdev->queue_sel); in virtio_load()
3229 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) { in virtio_load()
3230 return -1; in virtio_load()
3235 * Temporarily set guest_features low bits - needed by in virtio_load()
3239 * Note: devices should always test host features in future - don't create in virtio_load()
3242 vdev->guest_features = features; in virtio_load()
3251 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len)); in virtio_load()
3253 while (config_len > vdev->config_len) { in virtio_load()
3255 config_len--; in virtio_load()
3258 if (vdc->pre_load_queues) { in virtio_load()
3259 ret = vdc->pre_load_queues(vdev); in virtio_load()
3269 return -1; in virtio_load()
3273 vdev->vq[i].vring.num = qemu_get_be32(f); in virtio_load()
3274 if (k->has_variable_vring_alignment) { in virtio_load()
3275 vdev->vq[i].vring.align = qemu_get_be32(f); in virtio_load()
3277 vdev->vq[i].vring.desc = qemu_get_be64(f); in virtio_load()
3278 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); in virtio_load()
3279 vdev->vq[i].signalled_used_valid = false; in virtio_load()
3280 vdev->vq[i].notification = true; in virtio_load()
3282 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { in virtio_load()
3285 i, vdev->vq[i].last_avail_idx); in virtio_load()
3286 return -1; in virtio_load()
3288 if (k->load_queue) { in virtio_load()
3289 ret = k->load_queue(qbus->parent, i, f); in virtio_load()
3297 if (vdc->load != NULL) { in virtio_load()
3298 ret = vdc->load(vdev, f, version_id); in virtio_load()
3304 if (vdc->vmsd) { in virtio_load()
3305 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id); in virtio_load()
3317 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) { in virtio_load()
3318 vdev->device_endian = virtio_default_endian(); in virtio_load()
3323 * Subsection load filled vdev->guest_features. Run them in virtio_load()
3324 * through virtio_set_features to sanity-check them against in virtio_load()
3327 uint64_t features64 = vdev->guest_features; in virtio_load()
3331 features64, vdev->host_features); in virtio_load()
3332 return -1; in virtio_load()
3338 features, vdev->host_features); in virtio_load()
3339 return -1; in virtio_load()
3343 if (!virtio_device_started(vdev, vdev->status) && in virtio_load()
3345 vdev->start_on_kick = true; in virtio_load()
3350 if (vdev->vq[i].vring.desc) { in virtio_load()
3354 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so in virtio_load()
3366 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx; in virtio_load()
3367 vdev->vq[i].shadow_avail_wrap_counter = in virtio_load()
3368 vdev->vq[i].last_avail_wrap_counter; in virtio_load()
3372 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; in virtio_load()
3374 if (nheads > vdev->vq[i].vring.num) { in virtio_load()
3377 i, vdev->vq[i].vring.num, in virtio_load()
3378 vring_avail_idx(&vdev->vq[i]), in virtio_load()
3379 vdev->vq[i].last_avail_idx, nheads); in virtio_load()
3380 vdev->vq[i].used_idx = 0; in virtio_load()
3381 vdev->vq[i].shadow_avail_idx = 0; in virtio_load()
3382 vdev->vq[i].inuse = 0; in virtio_load()
3385 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]); in virtio_load()
3386 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]); in virtio_load()
3394 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx - in virtio_load()
3395 vdev->vq[i].used_idx); in virtio_load()
3396 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { in virtio_load()
3397 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - " in virtio_load()
3399 i, vdev->vq[i].vring.num, in virtio_load()
3400 vdev->vq[i].last_avail_idx, in virtio_load()
3401 vdev->vq[i].used_idx); in virtio_load()
3402 return -1; in virtio_load()
3407 if (vdc->post_load) { in virtio_load()
3408 ret = vdc->post_load(vdev); in virtio_load()
3419 qemu_del_vm_change_state_handler(vdev->vmstate); in virtio_cleanup()
3427 bool backend_run = running && virtio_device_started(vdev, vdev->status); in virtio_vmstate_change()
3428 vdev->vm_running = running; in virtio_vmstate_change()
3431 virtio_set_status(vdev, vdev->status); in virtio_vmstate_change()
3434 if (k->vmstate_change) { in virtio_vmstate_change()
3435 k->vmstate_change(qbus->parent, backend_run); in virtio_vmstate_change()
3439 virtio_set_status(vdev, vdev->status); in virtio_vmstate_change()
3448 object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev, in virtio_instance_init_common()
3459 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0; in virtio_init()
3462 vdev->vector_queues = in virtio_init()
3463 g_malloc0(sizeof(*vdev->vector_queues) * nvectors); in virtio_init()
3466 vdev->start_on_kick = false; in virtio_init()
3467 vdev->started = false; in virtio_init()
3468 vdev->vhost_started = false; in virtio_init()
3469 vdev->device_id = device_id; in virtio_init()
3470 vdev->status = 0; in virtio_init()
3471 qatomic_set(&vdev->isr, 0); in virtio_init()
3472 vdev->queue_sel = 0; in virtio_init()
3473 vdev->config_vector = VIRTIO_NO_VECTOR; in virtio_init()
3474 vdev->vq = g_new0(VirtQueue, VIRTIO_QUEUE_MAX); in virtio_init()
3475 vdev->vm_running = runstate_is_running(); in virtio_init()
3476 vdev->broken = false; in virtio_init()
3478 vdev->vq[i].vector = VIRTIO_NO_VECTOR; in virtio_init()
3479 vdev->vq[i].vdev = vdev; in virtio_init()
3480 vdev->vq[i].queue_index = i; in virtio_init()
3481 vdev->vq[i].host_notifier_enabled = false; in virtio_init()
3484 vdev->name = virtio_id_to_name(device_id); in virtio_init()
3485 vdev->config_len = config_size; in virtio_init()
3486 if (vdev->config_len) { in virtio_init()
3487 vdev->config = g_malloc0(config_size); in virtio_init()
3489 vdev->config = NULL; in virtio_init()
3491 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev), in virtio_init()
3493 vdev->device_endian = virtio_default_endian(); in virtio_init()
3494 vdev->use_guest_notifier_mask = true; in virtio_init()
3504 switch (vdev->device_id) { in virtio_legacy_allowed()
3523 return vdev->disable_legacy_check; in virtio_legacy_check_disabled()
3528 return vdev->vq[n].vring.desc; in virtio_queue_get_desc_addr()
3541 if (k->queue_enabled) { in virtio_queue_enabled()
3542 return k->queue_enabled(qbus->parent, n); in virtio_queue_enabled()
3549 return vdev->vq[n].vring.avail; in virtio_queue_get_avail_addr()
3554 return vdev->vq[n].vring.used; in virtio_queue_get_used_addr()
3559 return sizeof(VRingDesc) * vdev->vq[n].vring.num; in virtio_queue_get_desc_size()
3572 sizeof(uint16_t) * vdev->vq[n].vring.num + s; in virtio_queue_get_avail_size()
3585 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s; in virtio_queue_get_used_size()
3593 avail = vdev->vq[n].last_avail_idx; in virtio_queue_packed_get_last_avail_idx()
3594 avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15; in virtio_queue_packed_get_last_avail_idx()
3596 used = vdev->vq[n].used_idx; in virtio_queue_packed_get_last_avail_idx()
3597 used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15; in virtio_queue_packed_get_last_avail_idx()
3605 return vdev->vq[n].last_avail_idx; in virtio_queue_split_get_last_avail_idx()
3620 struct VirtQueue *vq = &vdev->vq[n]; in virtio_queue_packed_set_last_avail_idx()
3622 vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff; in virtio_queue_packed_set_last_avail_idx()
3623 vq->last_avail_wrap_counter = in virtio_queue_packed_set_last_avail_idx()
3624 vq->shadow_avail_wrap_counter = !!(idx & 0x8000); in virtio_queue_packed_set_last_avail_idx()
3626 vq->used_idx = idx & 0x7fff; in virtio_queue_packed_set_last_avail_idx()
3627 vq->used_wrap_counter = !!(idx & 0x8000); in virtio_queue_packed_set_last_avail_idx()
3633 vdev->vq[n].last_avail_idx = idx; in virtio_queue_split_set_last_avail_idx()
3634 vdev->vq[n].shadow_avail_idx = idx; in virtio_queue_split_set_last_avail_idx()
3658 if (vdev->vq[n].vring.desc) { in virtio_queue_split_restore_last_avail_idx()
3659 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]); in virtio_queue_split_restore_last_avail_idx()
3660 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx; in virtio_queue_split_restore_last_avail_idx()
3682 if (vdev->vq[n].vring.desc) { in virtio_queue_split_update_used_idx()
3683 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]); in virtio_queue_split_update_used_idx()
3698 vdev->vq[n].signalled_used_valid = false; in virtio_queue_invalidate_signalled_used()
3703 return vdev->vq + n; in virtio_get_queue()
3708 return vq->queue_index; in virtio_get_queue_index()
3730 event_notifier_set_handler(&vq->guest_notifier, in virtio_queue_set_guest_notifier_fd_handler()
3733 event_notifier_set_handler(&vq->guest_notifier, NULL); in virtio_queue_set_guest_notifier_fd_handler()
3738 virtio_queue_guest_notifier_read(&vq->guest_notifier); in virtio_queue_set_guest_notifier_fd_handler()
3746 n = &vdev->config_notifier; in virtio_config_set_guest_notifier_fd_handler()
3761 return &vq->guest_notifier; in virtio_queue_get_guest_notifier()
3776 return vq->vring.desc && !virtio_queue_empty(vq); in virtio_queue_host_notifier_aio_poll()
3798 * Re-enable them. (And if detach has not been used before, notifications in virtio_queue_aio_attach_host_notifier()
3807 aio_set_event_notifier(ctx, &vq->host_notifier, in virtio_queue_aio_attach_host_notifier()
3811 aio_set_event_notifier_poll(ctx, &vq->host_notifier, in virtio_queue_aio_attach_host_notifier()
3820 event_notifier_set(&vq->host_notifier); in virtio_queue_aio_attach_host_notifier()
3826 * function does not pop all elements. When the virtqueue is left non-empty
3836 aio_set_event_notifier(ctx, &vq->host_notifier, in virtio_queue_aio_attach_host_notifier_no_poll()
3846 event_notifier_set(&vq->host_notifier); in virtio_queue_aio_attach_host_notifier_no_poll()
3851 aio_set_event_notifier(ctx, &vq->host_notifier, NULL, NULL, NULL); in virtio_queue_aio_detach_host_notifier()
3860 * we potentially re-attach it. The attach_host_notifier functions will in virtio_queue_aio_detach_host_notifier()
3875 return &vq->host_notifier; in virtio_queue_get_host_notifier()
3880 return &vdev->config_notifier; in virtio_config_get_guest_notifier()
3885 vq->host_notifier_enabled = enabled; in virtio_queue_set_host_notifier_enabled()
3894 if (k->set_host_notifier_mr) { in virtio_queue_set_host_notifier_mr()
3895 return k->set_host_notifier_mr(qbus->parent, n, mr, assign); in virtio_queue_set_host_notifier_mr()
3898 return -1; in virtio_queue_set_host_notifier_mr()
3903 g_free(vdev->bus_name); in virtio_device_set_child_bus_name()
3904 vdev->bus_name = g_strdup(bus_name); in virtio_device_set_child_bus_name()
3916 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET; in virtio_error()
3920 vdev->broken = true; in virtio_error()
3929 if (vdev->vq[i].vring.num == 0) { in virtio_memory_listener_commit()
3943 assert(!vdc->vmsd || !vdc->load); in virtio_device_realize()
3945 if (vdc->realize != NULL) { in virtio_device_realize()
3946 vdc->realize(dev, &err); in virtio_device_realize()
3957 vdc->unrealize(dev); in virtio_device_realize()
3964 vdc->unrealize(dev); in virtio_device_realize()
3968 vdev->listener.commit = virtio_memory_listener_commit; in virtio_device_realize()
3969 vdev->listener.name = "virtio"; in virtio_device_realize()
3970 memory_listener_register(&vdev->listener, vdev->dma_as); in virtio_device_realize()
3978 memory_listener_unregister(&vdev->listener); in virtio_device_unrealize()
3981 if (vdc->unrealize != NULL) { in virtio_device_unrealize()
3982 vdc->unrealize(dev); in virtio_device_unrealize()
3985 g_free(vdev->bus_name); in virtio_device_unrealize()
3986 vdev->bus_name = NULL; in virtio_device_unrealize()
3992 if (!vdev->vq) { in virtio_device_free_virtqueues()
3997 if (vdev->vq[i].vring.num == 0) { in virtio_device_free_virtqueues()
4000 virtio_virtqueue_reset_region_cache(&vdev->vq[i]); in virtio_device_free_virtqueues()
4002 g_free(vdev->vq); in virtio_device_free_virtqueues()
4011 g_free(vdev->config); in virtio_device_instance_finalize()
4012 g_free(vdev->vector_queues); in virtio_device_instance_finalize()
4017 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
4018 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true),
4019 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice,
4035 VirtQueue *vq = &vdev->vq[n]; in virtio_device_start_ioeventfd_impl()
4044 event_notifier_set_handler(&vq->host_notifier, in virtio_device_start_ioeventfd_impl()
4050 VirtQueue *vq = &vdev->vq[n]; in virtio_device_start_ioeventfd_impl()
4051 if (!vq->vring.num) { in virtio_device_start_ioeventfd_impl()
4054 event_notifier_set(&vq->host_notifier); in virtio_device_start_ioeventfd_impl()
4061 while (--n >= 0) { in virtio_device_start_ioeventfd_impl()
4062 VirtQueue *vq = &vdev->vq[n]; in virtio_device_start_ioeventfd_impl()
4067 event_notifier_set_handler(&vq->host_notifier, NULL); in virtio_device_start_ioeventfd_impl()
4077 while (--i >= 0) { in virtio_device_start_ioeventfd_impl()
4105 VirtQueue *vq = &vdev->vq[n]; in virtio_device_stop_ioeventfd_impl()
4110 event_notifier_set_handler(&vq->host_notifier, NULL); in virtio_device_stop_ioeventfd_impl()
4150 dc->realize = virtio_device_realize; in virtio_device_class_init()
4151 dc->unrealize = virtio_device_unrealize; in virtio_device_class_init()
4152 dc->bus_type = TYPE_VIRTIO_BUS; in virtio_device_class_init()
4154 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl; in virtio_device_class_init()
4155 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl; in virtio_device_class_init()
4157 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES; in virtio_device_class_init()
4187 status->name = g_strdup(vdev->name); in qmp_x_query_virtio_queue_status()
4188 status->queue_index = vdev->vq[queue].queue_index; in qmp_x_query_virtio_queue_status()
4189 status->inuse = vdev->vq[queue].inuse; in qmp_x_query_virtio_queue_status()
4190 status->vring_num = vdev->vq[queue].vring.num; in qmp_x_query_virtio_queue_status()
4191 status->vring_num_default = vdev->vq[queue].vring.num_default; in qmp_x_query_virtio_queue_status()
4192 status->vring_align = vdev->vq[queue].vring.align; in qmp_x_query_virtio_queue_status()
4193 status->vring_desc = vdev->vq[queue].vring.desc; in qmp_x_query_virtio_queue_status()
4194 status->vring_avail = vdev->vq[queue].vring.avail; in qmp_x_query_virtio_queue_status()
4195 status->vring_used = vdev->vq[queue].vring.used; in qmp_x_query_virtio_queue_status()
4196 status->used_idx = vdev->vq[queue].used_idx; in qmp_x_query_virtio_queue_status()
4197 status->signalled_used = vdev->vq[queue].signalled_used; in qmp_x_query_virtio_queue_status()
4198 status->signalled_used_valid = vdev->vq[queue].signalled_used_valid; in qmp_x_query_virtio_queue_status()
4200 if (vdev->vhost_started) { in qmp_x_query_virtio_queue_status()
4202 struct vhost_dev *hdev = vdc->get_vhost(vdev); in qmp_x_query_virtio_queue_status()
4204 /* check if vq index exists for vhost as well */ in qmp_x_query_virtio_queue_status()
4205 if (queue >= hdev->vq_index && queue < hdev->vq_index + hdev->nvqs) { in qmp_x_query_virtio_queue_status()
4206 status->has_last_avail_idx = true; in qmp_x_query_virtio_queue_status()
4209 hdev->vhost_ops->vhost_get_vq_index(hdev, queue); in qmp_x_query_virtio_queue_status()
4214 status->last_avail_idx = in qmp_x_query_virtio_queue_status()
4215 hdev->vhost_ops->vhost_get_vring_base(hdev, &state); in qmp_x_query_virtio_queue_status()
4218 status->has_shadow_avail_idx = true; in qmp_x_query_virtio_queue_status()
4219 status->has_last_avail_idx = true; in qmp_x_query_virtio_queue_status()
4220 status->last_avail_idx = vdev->vq[queue].last_avail_idx; in qmp_x_query_virtio_queue_status()
4221 status->shadow_avail_idx = vdev->vq[queue].shadow_avail_idx; in qmp_x_query_virtio_queue_status()
4250 node->value = g_strdup(map[i].value); in qmp_decode_vring_desc_flags()
4251 node->next = list; in qmp_decode_vring_desc_flags()
4278 vq = &vdev->vq[queue]; in qmp_x_query_virtio_queue_element()
4297 max = vq->vring.num; in qmp_x_query_virtio_queue_element()
4300 head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num); in qmp_x_query_virtio_queue_element()
4302 head = vring_avail_ring(vq, index % vq->vring.num); in qmp_x_query_virtio_queue_element()
4311 if (caches->desc.len < max * sizeof(VRingDesc)) { in qmp_x_query_virtio_queue_element()
4316 desc_cache = &caches->desc; in qmp_x_query_virtio_queue_element()
4320 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as, in qmp_x_query_virtio_queue_element()
4334 element->avail = g_new0(VirtioRingAvail, 1); in qmp_x_query_virtio_queue_element()
4335 element->used = g_new0(VirtioRingUsed, 1); in qmp_x_query_virtio_queue_element()
4336 element->name = g_strdup(vdev->name); in qmp_x_query_virtio_queue_element()
4337 element->index = head; in qmp_x_query_virtio_queue_element()
4338 element->avail->flags = vring_avail_flags(vq); in qmp_x_query_virtio_queue_element()
4339 element->avail->idx = vring_avail_idx(vq); in qmp_x_query_virtio_queue_element()
4340 element->avail->ring = head; in qmp_x_query_virtio_queue_element()
4341 element->used->flags = vring_used_flags(vq); in qmp_x_query_virtio_queue_element()
4342 element->used->idx = vring_used_idx(vq); in qmp_x_query_virtio_queue_element()
4351 node->value = g_new0(VirtioRingDesc, 1); in qmp_x_query_virtio_queue_element()
4352 node->value->addr = desc.addr; in qmp_x_query_virtio_queue_element()
4353 node->value->len = desc.len; in qmp_x_query_virtio_queue_element()
4354 node->value->flags = qmp_decode_vring_desc_flags(desc.flags); in qmp_x_query_virtio_queue_element()
4355 node->next = list; in qmp_x_query_virtio_queue_element()
4361 element->descs = list; in qmp_x_query_virtio_queue_element()
4390 DeviceState *transport = qdev_get_parent_bus(dev)->parent; in type_init()
4393 &transport->mem_reentrancy_guard); in type_init()