Lines Matching full:vring
56 * The alignment to use between consumer and producer parts of vring.
104 typedef struct VRing struct
113 } VRing; argument
122 VRing vring; member
222 caches = qatomic_read(&vq->vring.caches); in virtio_virtqueue_reset_region_cache()
223 qatomic_rcu_set(&vq->vring.caches, NULL); in virtio_virtqueue_reset_region_cache()
232 VRingMemoryRegionCaches *old = vq->vring.caches; in virtio_init_region_cache()
239 addr = vq->vring.desc; in virtio_init_region_cache()
256 vq->vring.used, size, true); in virtio_init_region_cache()
264 vq->vring.avail, size, false); in virtio_init_region_cache()
270 qatomic_rcu_set(&vq->vring.caches, new); in virtio_init_region_cache()
290 VRing *vring = &vdev->vq[n].vring; in virtio_queue_update_rings() local
292 if (!vring->num || !vring->desc || !vring->align) { in virtio_queue_update_rings()
296 vring->avail = vring->desc + vring->num * sizeof(VRingDesc); in virtio_queue_update_rings()
297 vring->used = vring_align(vring->avail + in virtio_queue_update_rings()
298 offsetof(VRingAvail, ring[vring->num]), in virtio_queue_update_rings()
299 vring->align); in virtio_queue_update_rings()
350 return qatomic_rcu_read(&vq->vring.caches); in vring_get_region_caches()
396 return vring_avail_ring(vq, vq->vring.num); in vring_get_used_event()
504 pa = offsetof(VRingUsed, ring[vq->vring.num]); in vring_set_avail_event()
568 if (!vq->vring.desc) { in virtio_queue_set_notification()
581 return vq->vring.avail != 0; in virtio_queue_ready()
679 if (unlikely(!vq->vring.avail)) { in virtio_queue_empty_rcu()
698 if (unlikely(!vq->vring.avail)) { in virtio_queue_split_empty()
717 if (unlikely(!vq->vring.desc)) { in virtio_queue_packed_empty_rcu()
749 if (unlikely(!vq->vring.avail)) { in virtio_queue_split_poll()
761 if (unlikely(!vq->vring.desc)) { in virtio_queue_packed_poll()
850 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num; in virtqueue_packed_rewind()
910 if (unlikely(!vq->vring.used)) { in virtqueue_split_fill()
914 idx = (idx + vq->used_idx) % vq->vring.num; in virtqueue_split_fill()
934 i = vq->used_idx % vq->vring.num; in virtqueue_ordered_fill()
940 max_steps = (vq->last_avail_idx - vq->used_idx) % vq->vring.num; in virtqueue_ordered_fill()
954 if (i >= vq->vring.num) { in virtqueue_ordered_fill()
955 i -= vq->vring.num; in virtqueue_ordered_fill()
982 if (unlikely(!vq->vring.desc)) { in virtqueue_packed_fill_desc()
987 if (head >= vq->vring.num) { in virtqueue_packed_fill_desc()
988 head -= vq->vring.num; in virtqueue_packed_fill_desc()
1033 if (unlikely(!vq->vring.used)) { in virtqueue_split_flush()
1052 if (unlikely(!vq->vring.desc)) { in virtqueue_packed_flush()
1073 if (vq->used_idx >= vq->vring.num) { in virtqueue_packed_flush()
1074 vq->used_idx -= vq->vring.num; in virtqueue_packed_flush()
1082 unsigned int i = vq->used_idx % vq->vring.num; in virtqueue_ordered_flush()
1092 if (unlikely(!vq->vring.desc)) { in virtqueue_ordered_flush()
1095 } else if (unlikely(!vq->vring.used)) { in virtqueue_ordered_flush()
1121 if (i >= vq->vring.num) { in virtqueue_ordered_flush()
1122 i -= vq->vring.num; in virtqueue_ordered_flush()
1129 if (vq->used_idx >= vq->vring.num) { in virtqueue_ordered_flush()
1130 vq->used_idx -= vq->vring.num; in virtqueue_ordered_flush()
1182 if (num_heads > vq->vring.num) { in virtqueue_num_heads()
1209 *head = vring_avail_ring(vq, idx % vq->vring.num); in virtqueue_get_head()
1212 if (*head >= vq->vring.num) { in virtqueue_get_head()
1269 unsigned int max = vq->vring.num; in virtqueue_split_get_avail_bytes()
1374 (*next) -= vq->vring.num; in virtqueue_packed_read_next_desc()
1409 unsigned int max = vq->vring.num; in virtqueue_packed_get_avail_bytes()
1475 if (idx >= vq->vring.num) { in virtqueue_packed_get_avail_bytes()
1476 idx -= vq->vring.num; in virtqueue_packed_get_avail_bytes()
1508 if (unlikely(!vq->vring.desc)) { in virtqueue_get_avail_bytes()
1519 if (caches->desc.len < vq->vring.num * desc_size) { in virtqueue_get_avail_bytes()
1701 max = vq->vring.num; in virtqueue_split_pop()
1703 if (vq->inuse >= vq->vring.num) { in virtqueue_split_pop()
1800 idx = (vq->last_avail_idx - 1) % vq->vring.num; in virtqueue_split_pop()
1845 max = vq->vring.num; in virtqueue_packed_pop()
1847 if (vq->inuse >= vq->vring.num) { in virtqueue_packed_pop()
1949 if (vq->last_avail_idx >= vq->vring.num) { in virtqueue_packed_pop()
1950 vq->last_avail_idx -= vq->vring.num; in virtqueue_packed_pop()
2001 while (vq->inuse < vq->vring.num) { in virtqueue_packed_drop_all()
2015 vq->vring.num, &idx, false)) { in virtqueue_packed_drop_all()
2025 if (vq->last_avail_idx >= vq->vring.num) { in virtqueue_packed_drop_all()
2026 vq->last_avail_idx -= vq->vring.num; in virtqueue_packed_drop_all()
2041 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) { in virtqueue_split_drop_all()
2269 vdev->vq[i].vring.desc = 0; in __virtio_queue_reset()
2270 vdev->vq[i].vring.avail = 0; in __virtio_queue_reset()
2271 vdev->vq[i].vring.used = 0; in __virtio_queue_reset()
2282 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default; in __virtio_queue_reset()
2364 if (!vdev->vq[n].vring.num) { in virtio_queue_set_addr()
2367 vdev->vq[n].vring.desc = addr; in virtio_queue_set_addr()
2373 return vdev->vq[n].vring.desc; in virtio_queue_get_addr()
2379 if (!vdev->vq[n].vring.num) { in virtio_queue_set_rings()
2382 vdev->vq[n].vring.desc = desc; in virtio_queue_set_rings()
2383 vdev->vq[n].vring.avail = avail; in virtio_queue_set_rings()
2384 vdev->vq[n].vring.used = used; in virtio_queue_set_rings()
2393 if (!!num != !!vdev->vq[n].vring.num || in virtio_queue_set_num()
2398 vdev->vq[n].vring.num = num; in virtio_queue_set_num()
2413 return vdev->vq[n].vring.num; in virtio_queue_get_num()
2418 return vdev->vq[n].vring.num_default; in virtio_queue_get_max_num()
2451 vdev->vq[n].vring.align = align; in virtio_queue_set_align()
2458 if (!vq->vring.desc) { in virtio_queue_set_shadow_avail_idx()
2476 if (vq->vring.desc && vq->handle_output) { in virtio_queue_notify_vq()
2496 if (unlikely(!vq->vring.desc || vdev->broken)) { in virtio_queue_notify()
2541 if (vdev->vq[i].vring.num == 0) in virtio_add_queue()
2548 vdev->vq[i].vring.num = queue_size; in virtio_add_queue()
2549 vdev->vq[i].vring.num_default = queue_size; in virtio_add_queue()
2550 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN; in virtio_add_queue()
2559 vq->vring.num = 0; in virtio_delete_queue()
2560 vq->vring.num_default = 0; in virtio_delete_queue()
2619 off -= vq->vring.num; in vring_packed_need_event()
2771 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) { in virtio_ringsize_needed()
2814 VMSTATE_UINT64(vring.avail, struct VirtQueue),
2815 VMSTATE_UINT64(vring.used, struct VirtQueue),
2863 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
3026 if (vdev->vq[i].vring.num == 0) in virtio_save()
3033 if (vdev->vq[i].vring.num == 0) in virtio_save()
3036 qemu_put_be32(f, vdev->vq[i].vring.num); in virtio_save()
3038 qemu_put_be32(f, vdev->vq[i].vring.align); in virtio_save()
3044 qemu_put_be64(f, vdev->vq[i].vring.desc); in virtio_save()
3158 if (vdev->vq[i].vring.num != 0) { in virtio_set_features()
3273 vdev->vq[i].vring.num = qemu_get_be32(f); in virtio_load()
3275 vdev->vq[i].vring.align = qemu_get_be32(f); in virtio_load()
3277 vdev->vq[i].vring.desc = qemu_get_be64(f); in virtio_load()
3282 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) { in virtio_load()
3350 if (vdev->vq[i].vring.desc) { in virtio_load()
3374 if (nheads > vdev->vq[i].vring.num) { in virtio_load()
3377 i, vdev->vq[i].vring.num, in virtio_load()
3396 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) { in virtio_load()
3399 i, vdev->vq[i].vring.num, in virtio_load()
3528 return vdev->vq[n].vring.desc; in virtio_queue_get_desc_addr()
3549 return vdev->vq[n].vring.avail; in virtio_queue_get_avail_addr()
3554 return vdev->vq[n].vring.used; in virtio_queue_get_used_addr()
3559 return sizeof(VRingDesc) * vdev->vq[n].vring.num; in virtio_queue_get_desc_size()
3572 sizeof(uint16_t) * vdev->vq[n].vring.num + s; in virtio_queue_get_avail_size()
3585 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s; in virtio_queue_get_used_size()
3658 if (vdev->vq[n].vring.desc) { in virtio_queue_split_restore_last_avail_idx()
3682 if (vdev->vq[n].vring.desc) { in virtio_queue_split_update_used_idx()
3776 return vq->vring.desc && !virtio_queue_empty(vq); in virtio_queue_host_notifier_aio_poll()
3929 if (vdev->vq[i].vring.num == 0) { in virtio_memory_listener_commit()
3997 if (vdev->vq[i].vring.num == 0) { in virtio_device_free_virtqueues()
4049 /* Kick right away to begin processing requests already in vring */ in virtio_device_start_ioeventfd_impl()
4051 if (!vq->vring.num) { in virtio_device_start_ioeventfd_impl()
4190 status->vring_num = vdev->vq[queue].vring.num; in qmp_x_query_virtio_queue_status()
4191 status->vring_num_default = vdev->vq[queue].vring.num_default; in qmp_x_query_virtio_queue_status()
4192 status->vring_align = vdev->vq[queue].vring.align; in qmp_x_query_virtio_queue_status()
4193 status->vring_desc = vdev->vq[queue].vring.desc; in qmp_x_query_virtio_queue_status()
4194 status->vring_avail = vdev->vq[queue].vring.avail; in qmp_x_query_virtio_queue_status()
4195 status->vring_used = vdev->vq[queue].vring.used; in qmp_x_query_virtio_queue_status()
4297 max = vq->vring.num; in qmp_x_query_virtio_queue_element()
4300 head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num); in qmp_x_query_virtio_queue_element()
4302 head = vring_avail_ring(vq, index % vq->vring.num); in qmp_x_query_virtio_queue_element()