122     VRing vring;  member
222     caches = qatomic_read(&vq->vring.caches);  in virtio_virtqueue_reset_region_cache()
223     qatomic_rcu_set(&vq->vring.caches, NULL);  in virtio_virtqueue_reset_region_cache()
232     VRingMemoryRegionCaches *old = vq->vring.caches;  in virtio_init_region_cache()
239     addr = vq->vring.desc;  in virtio_init_region_cache()
256                                    vq->vring.used, size, true);  in virtio_init_region_cache()
264                                    vq->vring.avail, size, false);  in virtio_init_region_cache()
270     qatomic_rcu_set(&vq->vring.caches, new);  in virtio_init_region_cache()
290     VRing *vring = &vdev->vq[n].vring;  in virtio_queue_update_rings()  local
292     if (!vring->num || !vring->desc || !vring->align) {  in virtio_queue_update_rings()
296     vring->avail = vring->desc + vring->num * sizeof(VRingDesc);  in virtio_queue_update_rings()
297     vring->used = vring_align(vring->avail +  in virtio_queue_update_rings()
298                               offsetof(VRingAvail, ring[vring->num]),  in virtio_queue_update_rings()
299                               vring->align);  in virtio_queue_update_rings()
350     return qatomic_rcu_read(&vq->vring.caches);  in vring_get_region_caches()
396     return vring_avail_ring(vq, vq->vring.num);  in vring_get_used_event()
504     pa = offsetof(VRingUsed, ring[vq->vring.num]);  in vring_set_avail_event()
568     if (!vq->vring.desc) {  in virtio_queue_set_notification()
581     return vq->vring.avail != 0;  in virtio_queue_ready()
679     if (unlikely(!vq->vring.avail)) {  in virtio_queue_empty_rcu()
698     if (unlikely(!vq->vring.avail)) {  in virtio_queue_split_empty()
717     if (unlikely(!vq->vring.desc)) {  in virtio_queue_packed_empty_rcu()
749     if (unlikely(!vq->vring.avail)) {  in virtio_queue_split_poll()
761     if (unlikely(!vq->vring.desc)) {  in virtio_queue_packed_poll()
850         vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num;  in virtqueue_packed_rewind()
910     if (unlikely(!vq->vring.used)) {  in virtqueue_split_fill()
914     idx = (idx + vq->used_idx) % vq->vring.num;  in virtqueue_split_fill()
934     i = vq->used_idx % vq->vring.num;  in virtqueue_ordered_fill()
940     max_steps = (vq->last_avail_idx - vq->used_idx) % vq->vring.num;  in virtqueue_ordered_fill()
954         if (i >= vq->vring.num) {  in virtqueue_ordered_fill()
955             i -= vq->vring.num;  in virtqueue_ordered_fill()
982     if (unlikely(!vq->vring.desc)) {  in virtqueue_packed_fill_desc()
987     if (head >= vq->vring.num) {  in virtqueue_packed_fill_desc()
988         head -= vq->vring.num;  in virtqueue_packed_fill_desc()
1033     if (unlikely(!vq->vring.used)) {  in virtqueue_split_flush()
1052     if (unlikely(!vq->vring.desc)) {  in virtqueue_packed_flush()
1073     if (vq->used_idx >= vq->vring.num) {  in virtqueue_packed_flush()
1074         vq->used_idx -= vq->vring.num;  in virtqueue_packed_flush()
1082     unsigned int i = vq->used_idx % vq->vring.num;  in virtqueue_ordered_flush()
1092         if (unlikely(!vq->vring.desc)) {  in virtqueue_ordered_flush()
1095     } else if (unlikely(!vq->vring.used)) {  in virtqueue_ordered_flush()
1121         if (i >= vq->vring.num) {  in virtqueue_ordered_flush()
1122             i -= vq->vring.num;  in virtqueue_ordered_flush()
1129         if (vq->used_idx >= vq->vring.num) {  in virtqueue_ordered_flush()
1130             vq->used_idx -= vq->vring.num;  in virtqueue_ordered_flush()
1182     if (num_heads > vq->vring.num) {  in virtqueue_num_heads()
1209     *head = vring_avail_ring(vq, idx % vq->vring.num);  in virtqueue_get_head()
1212     if (*head >= vq->vring.num) {  in virtqueue_get_head()
1269         unsigned int max = vq->vring.num;  in virtqueue_split_get_avail_bytes()
1374             (*next) -= vq->vring.num;  in virtqueue_packed_read_next_desc()
1409         unsigned int max = vq->vring.num;  in virtqueue_packed_get_avail_bytes()
1475         if (idx >= vq->vring.num) {  in virtqueue_packed_get_avail_bytes()
1476             idx -= vq->vring.num;  in virtqueue_packed_get_avail_bytes()
1508     if (unlikely(!vq->vring.desc)) {  in virtqueue_get_avail_bytes()
1519     if (caches->desc.len < vq->vring.num * desc_size) {  in virtqueue_get_avail_bytes()
1701     max = vq->vring.num;  in virtqueue_split_pop()
1703     if (vq->inuse >= vq->vring.num) {  in virtqueue_split_pop()
1800         idx = (vq->last_avail_idx - 1) % vq->vring.num;  in virtqueue_split_pop()
1845     max = vq->vring.num;  in virtqueue_packed_pop()
1847     if (vq->inuse >= vq->vring.num) {  in virtqueue_packed_pop()
1949     if (vq->last_avail_idx >= vq->vring.num) {  in virtqueue_packed_pop()
1950         vq->last_avail_idx -= vq->vring.num;  in virtqueue_packed_pop()
2001     while (vq->inuse < vq->vring.num) {  in virtqueue_packed_drop_all()
2015                                                vq->vring.num, &idx, false)) {  in virtqueue_packed_drop_all()
2025         if (vq->last_avail_idx >= vq->vring.num) {  in virtqueue_packed_drop_all()
2026             vq->last_avail_idx -= vq->vring.num;  in virtqueue_packed_drop_all()
2041     while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {  in virtqueue_split_drop_all()
2269     vdev->vq[i].vring.desc = 0;  in __virtio_queue_reset()
2270     vdev->vq[i].vring.avail = 0;  in __virtio_queue_reset()
2271     vdev->vq[i].vring.used = 0;  in __virtio_queue_reset()
2282     vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;  in __virtio_queue_reset()
2360     if (!vdev->vq[n].vring.num) {  in virtio_queue_set_addr()
2363     vdev->vq[n].vring.desc = addr;  in virtio_queue_set_addr()
2369     return vdev->vq[n].vring.desc;  in virtio_queue_get_addr()
2375     if (!vdev->vq[n].vring.num) {  in virtio_queue_set_rings()
2378     vdev->vq[n].vring.desc = desc;  in virtio_queue_set_rings()
2379     vdev->vq[n].vring.avail = avail;  in virtio_queue_set_rings()
2380     vdev->vq[n].vring.used = used;  in virtio_queue_set_rings()
2389     if (!!num != !!vdev->vq[n].vring.num ||  in virtio_queue_set_num()
2394     vdev->vq[n].vring.num = num;  in virtio_queue_set_num()
2409     return vdev->vq[n].vring.num;  in virtio_queue_get_num()
2414     return vdev->vq[n].vring.num_default;  in virtio_queue_get_max_num()
2447         vdev->vq[n].vring.align = align;  in virtio_queue_set_align()
2454     if (!vq->vring.desc) {  in virtio_queue_set_shadow_avail_idx()
2472     if (vq->vring.desc && vq->handle_output) {  in virtio_queue_notify_vq()
2492     if (unlikely(!vq->vring.desc || vdev->broken)) {  in virtio_queue_notify()
2537         if (vdev->vq[i].vring.num == 0)  in virtio_add_queue()
2544     vdev->vq[i].vring.num = queue_size;  in virtio_add_queue()
2545     vdev->vq[i].vring.num_default = queue_size;  in virtio_add_queue()
2546     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;  in virtio_add_queue()
2555     vq->vring.num = 0;  in virtio_delete_queue()
2556     vq->vring.num_default = 0;  in virtio_delete_queue()
2615         off -= vq->vring.num;  in vring_packed_need_event()
2767         if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {  in virtio_ringsize_needed()
2810         VMSTATE_UINT64(vring.avail, struct VirtQueue),
2811         VMSTATE_UINT64(vring.used, struct VirtQueue),
2859         VMSTATE_UINT32(vring.num_default, struct VirtQueue),
3022         if (vdev->vq[i].vring.num == 0)  in virtio_save()
3029         if (vdev->vq[i].vring.num == 0)  in virtio_save()
3032         qemu_put_be32(f, vdev->vq[i].vring.num);  in virtio_save()
3034             qemu_put_be32(f, vdev->vq[i].vring.align);  in virtio_save()
3040         qemu_put_be64(f, vdev->vq[i].vring.desc);  in virtio_save()
3154             if (vdev->vq[i].vring.num != 0) {  in virtio_set_features()
3262         vdev->vq[i].vring.num = qemu_get_be32(f);  in virtio_load()
3264             vdev->vq[i].vring.align = qemu_get_be32(f);  in virtio_load()
3266         vdev->vq[i].vring.desc = qemu_get_be64(f);  in virtio_load()
3271         if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {  in virtio_load()
3339         if (vdev->vq[i].vring.desc) {  in virtio_load()
3363             if (nheads > vdev->vq[i].vring.num) {  in virtio_load()
3366                              i, vdev->vq[i].vring.num,  in virtio_load()
3385             if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {  in virtio_load()
3388                              i, vdev->vq[i].vring.num,  in virtio_load()
3517     return vdev->vq[n].vring.desc;  in virtio_queue_get_desc_addr()
3538     return vdev->vq[n].vring.avail;  in virtio_queue_get_avail_addr()
3543     return vdev->vq[n].vring.used;  in virtio_queue_get_used_addr()
3548     return sizeof(VRingDesc) * vdev->vq[n].vring.num;  in virtio_queue_get_desc_size()
3561         sizeof(uint16_t) * vdev->vq[n].vring.num + s;  in virtio_queue_get_avail_size()
3574         sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s;  in virtio_queue_get_used_size()
3647     if (vdev->vq[n].vring.desc) {  in virtio_queue_split_restore_last_avail_idx()
3671     if (vdev->vq[n].vring.desc) {  in virtio_split_packed_update_used_idx()
3765     return vq->vring.desc && !virtio_queue_empty(vq);  in virtio_queue_host_notifier_aio_poll()
3918         if (vdev->vq[i].vring.num == 0) {  in virtio_memory_listener_commit()
3986         if (vdev->vq[i].vring.num == 0) {  in virtio_device_free_virtqueues()
4040         if (!vq->vring.num) {  in virtio_device_start_ioeventfd_impl()
4179     status->vring_num = vdev->vq[queue].vring.num;  in qmp_x_query_virtio_queue_status()
4180     status->vring_num_default = vdev->vq[queue].vring.num_default;  in qmp_x_query_virtio_queue_status()
4181     status->vring_align = vdev->vq[queue].vring.align;  in qmp_x_query_virtio_queue_status()
4182     status->vring_desc = vdev->vq[queue].vring.desc;  in qmp_x_query_virtio_queue_status()
4183     status->vring_avail = vdev->vq[queue].vring.avail;  in qmp_x_query_virtio_queue_status()
4184     status->vring_used = vdev->vq[queue].vring.used;  in qmp_x_query_virtio_queue_status()
4286         max = vq->vring.num;  in qmp_x_query_virtio_queue_element()
4289             head = vring_avail_ring(vq, vq->last_avail_idx % vq->vring.num);  in qmp_x_query_virtio_queue_element()
4291             head = vring_avail_ring(vq, index % vq->vring.num);  in qmp_x_query_virtio_queue_element()