185 VduseDev *vduse_queue_get_dev(VduseVirtq *vq)  in vduse_queue_get_dev()  argument
187     return vq->dev;  in vduse_queue_get_dev()
190 int vduse_queue_get_fd(VduseVirtq *vq)  in vduse_queue_get_fd()  argument
192     return vq->fd;  in vduse_queue_get_fd()
228 static int vduse_queue_check_inflights(VduseVirtq *vq)  in vduse_queue_check_inflights()  argument
231     VduseDev *dev = vq->dev;  in vduse_queue_check_inflights()
233     vq->used_idx = le16toh(vq->vring.used->idx);  in vduse_queue_check_inflights()
234     vq->resubmit_num = 0;  in vduse_queue_check_inflights()
235     vq->resubmit_list = NULL;  in vduse_queue_check_inflights()
236     vq->counter = 0;  in vduse_queue_check_inflights()
238     if (unlikely(vq->log->inflight.used_idx != vq->used_idx)) {  in vduse_queue_check_inflights()
239         if (vq->log->inflight.last_batch_head > VIRTQUEUE_MAX_SIZE) {  in vduse_queue_check_inflights()
243         vq->log->inflight.desc[vq->log->inflight.last_batch_head].inflight = 0;  in vduse_queue_check_inflights()
247         vq->log->inflight.used_idx = vq->used_idx;  in vduse_queue_check_inflights()
250     for (i = 0; i < vq->log->inflight.desc_num; i++) {  in vduse_queue_check_inflights()
251         if (vq->log->inflight.desc[i].inflight == 1) {  in vduse_queue_check_inflights()
252             vq->inuse++;  in vduse_queue_check_inflights()
256     vq->shadow_avail_idx = vq->last_avail_idx = vq->inuse + vq->used_idx;  in vduse_queue_check_inflights()
258     if (vq->inuse) {  in vduse_queue_check_inflights()
259         vq->resubmit_list = calloc(vq->inuse, sizeof(VduseVirtqInflightDesc));  in vduse_queue_check_inflights()
260         if (!vq->resubmit_list) {  in vduse_queue_check_inflights()
264         for (i = 0; i < vq->log->inflight.desc_num; i++) {  in vduse_queue_check_inflights()
265             if (vq->log->inflight.desc[i].inflight) {  in vduse_queue_check_inflights()
266                 vq->resubmit_list[vq->resubmit_num].index = i;  in vduse_queue_check_inflights()
267                 vq->resubmit_list[vq->resubmit_num].counter =  in vduse_queue_check_inflights()
268                                         vq->log->inflight.desc[i].counter;  in vduse_queue_check_inflights()
269                 vq->resubmit_num++;  in vduse_queue_check_inflights()
273         if (vq->resubmit_num > 1) {  in vduse_queue_check_inflights()
274             qsort(vq->resubmit_list, vq->resubmit_num,  in vduse_queue_check_inflights()
277         vq->counter = vq->resubmit_list[0].counter + 1;  in vduse_queue_check_inflights()
280     vduse_inject_irq(dev, vq->index);  in vduse_queue_check_inflights()
285 static int vduse_queue_inflight_get(VduseVirtq *vq, int desc_idx)  in vduse_queue_inflight_get()  argument
287     vq->log->inflight.desc[desc_idx].counter = vq->counter++;  in vduse_queue_inflight_get()
291     vq->log->inflight.desc[desc_idx].inflight = 1;  in vduse_queue_inflight_get()
296 static int vduse_queue_inflight_pre_put(VduseVirtq *vq, int desc_idx)  in vduse_queue_inflight_pre_put()  argument
298     vq->log->inflight.last_batch_head = desc_idx;  in vduse_queue_inflight_pre_put()
303 static int vduse_queue_inflight_post_put(VduseVirtq *vq, int desc_idx)  in vduse_queue_inflight_post_put()  argument
305     vq->log->inflight.desc[desc_idx].inflight = 0;  in vduse_queue_inflight_post_put()
309     vq->log->inflight.used_idx = vq->used_idx;  in vduse_queue_inflight_post_put()
424 static inline uint16_t vring_avail_flags(VduseVirtq *vq)  in vring_avail_flags()  argument
426     return le16toh(vq->vring.avail->flags);  in vring_avail_flags()
429 static inline uint16_t vring_avail_idx(VduseVirtq *vq)  in vring_avail_idx()  argument
431     vq->shadow_avail_idx = le16toh(vq->vring.avail->idx);  in vring_avail_idx()
433     return vq->shadow_avail_idx;  in vring_avail_idx()
436 static inline uint16_t vring_avail_ring(VduseVirtq *vq, int i)  in vring_avail_ring()  argument
438     return le16toh(vq->vring.avail->ring[i]);  in vring_avail_ring()
441 static inline uint16_t vring_get_used_event(VduseVirtq *vq)  in vring_get_used_event()  argument
443     return vring_avail_ring(vq, vq->vring.num);  in vring_get_used_event()
446 static bool vduse_queue_get_head(VduseVirtq *vq, unsigned int idx,  in vduse_queue_get_head()  argument
453     *head = vring_avail_ring(vq, idx % vq->vring.num);  in vduse_queue_get_head()
456     if (*head >= vq->vring.num) {  in vduse_queue_get_head()
526 static bool vduse_queue_empty(VduseVirtq *vq)  in vduse_queue_empty()  argument
528     if (unlikely(!vq->vring.avail)) {  in vduse_queue_empty()
532     if (vq->shadow_avail_idx != vq->last_avail_idx) {  in vduse_queue_empty()
536     return vring_avail_idx(vq) == vq->last_avail_idx;  in vduse_queue_empty()
539 static bool vduse_queue_should_notify(VduseVirtq *vq)  in vduse_queue_should_notify()  argument
541     VduseDev *dev = vq->dev;  in vduse_queue_should_notify()
550         !vq->inuse && vduse_queue_empty(vq)) {  in vduse_queue_should_notify()
555         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);  in vduse_queue_should_notify()
558     v = vq->signalled_used_valid;  in vduse_queue_should_notify()
559     vq->signalled_used_valid = true;  in vduse_queue_should_notify()
560     old = vq->signalled_used;  in vduse_queue_should_notify()
561     new = vq->signalled_used = vq->used_idx;  in vduse_queue_should_notify()
562     return !v || vring_need_event(vring_get_used_event(vq), new, old);  in vduse_queue_should_notify()
565 void vduse_queue_notify(VduseVirtq *vq)  in vduse_queue_notify()  argument
567     VduseDev *dev = vq->dev;  in vduse_queue_notify()
569     if (unlikely(!vq->vring.avail)) {  in vduse_queue_notify()
573     if (!vduse_queue_should_notify(vq)) {  in vduse_queue_notify()
577     if (vduse_inject_irq(dev, vq->index) < 0) {  in vduse_queue_notify()
579                 vq->index, strerror(errno));  in vduse_queue_notify()
583 static inline void vring_set_avail_event(VduseVirtq *vq, uint16_t val)  in vring_set_avail_event()  argument
586     memcpy(&vq->vring.used->ring[vq->vring.num], &val_le, sizeof(uint16_t));  in vring_set_avail_event()
589 static bool vduse_queue_map_single_desc(VduseVirtq *vq, unsigned int *p_num_sg,  in vduse_queue_map_single_desc()  argument
594     VduseDev *dev = vq->dev;  in vduse_queue_map_single_desc()
646 static void *vduse_queue_map_desc(VduseVirtq *vq, unsigned int idx, size_t sz)  in vduse_queue_map_desc()  argument
648     struct vring_desc *desc = vq->vring.desc;  in vduse_queue_map_desc()
649     VduseDev *dev = vq->dev;  in vduse_queue_map_desc()
652     unsigned int max = vq->vring.num;  in vduse_queue_map_desc()
691             if (!vduse_queue_map_single_desc(vq, &in_num, iov + out_num,  in vduse_queue_map_desc()
702             if (!vduse_queue_map_single_desc(vq, &out_num, iov,  in vduse_queue_map_desc()
740 void *vduse_queue_pop(VduseVirtq *vq, size_t sz)  in vduse_queue_pop()  argument
744     VduseDev *dev = vq->dev;  in vduse_queue_pop()
747     if (unlikely(!vq->vring.avail)) {  in vduse_queue_pop()
751     if (unlikely(vq->resubmit_list && vq->resubmit_num > 0)) {  in vduse_queue_pop()
752         i = (--vq->resubmit_num);  in vduse_queue_pop()
753         elem = vduse_queue_map_desc(vq, vq->resubmit_list[i].index, sz);  in vduse_queue_pop()
755         if (!vq->resubmit_num) {  in vduse_queue_pop()
756             free(vq->resubmit_list);  in vduse_queue_pop()
757             vq->resubmit_list = NULL;  in vduse_queue_pop()
763     if (vduse_queue_empty(vq)) {  in vduse_queue_pop()
769     if (vq->inuse >= vq->vring.num) {  in vduse_queue_pop()
770         fprintf(stderr, "Virtqueue size exceeded: %d\n", vq->inuse);  in vduse_queue_pop()
774     if (!vduse_queue_get_head(vq, vq->last_avail_idx++, &head)) {  in vduse_queue_pop()
779         vring_set_avail_event(vq, vq->last_avail_idx);  in vduse_queue_pop()
782     elem = vduse_queue_map_desc(vq, head, sz);  in vduse_queue_pop()
788     vq->inuse++;  in vduse_queue_pop()
790     vduse_queue_inflight_get(vq, head);  in vduse_queue_pop()
795 static inline void vring_used_write(VduseVirtq *vq,  in vring_used_write()  argument
798     struct vring_used *used = vq->vring.used;  in vring_used_write()
803 static void vduse_queue_fill(VduseVirtq *vq, const VduseVirtqElement *elem,  in vduse_queue_fill()  argument
808     if (unlikely(!vq->vring.used)) {  in vduse_queue_fill()
812     idx = (idx + vq->used_idx) % vq->vring.num;  in vduse_queue_fill()
816     vring_used_write(vq, &uelem, idx);  in vduse_queue_fill()
819 static inline void vring_used_idx_set(VduseVirtq *vq, uint16_t val)  in vring_used_idx_set()  argument
821     vq->vring.used->idx = htole16(val);  in vring_used_idx_set()
822     vq->used_idx = val;  in vring_used_idx_set()
825 static void vduse_queue_flush(VduseVirtq *vq, unsigned int count)  in vduse_queue_flush()  argument
829     if (unlikely(!vq->vring.used)) {  in vduse_queue_flush()
836     old = vq->used_idx;  in vduse_queue_flush()
838     vring_used_idx_set(vq, new);  in vduse_queue_flush()
839     vq->inuse -= count;  in vduse_queue_flush()
840     if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) {  in vduse_queue_flush()
841         vq->signalled_used_valid = false;  in vduse_queue_flush()
845 void vduse_queue_push(VduseVirtq *vq, const VduseVirtqElement *elem,  in vduse_queue_push()  argument
848     vduse_queue_fill(vq, elem, len, 0);  in vduse_queue_push()
849     vduse_queue_inflight_pre_put(vq, elem->index);  in vduse_queue_push()
850     vduse_queue_flush(vq, 1);  in vduse_queue_push()
851     vduse_queue_inflight_post_put(vq, elem->index);  in vduse_queue_push()
854 static int vduse_queue_update_vring(VduseVirtq *vq, uint64_t desc_addr,  in vduse_queue_update_vring()  argument
857     struct VduseDev *dev = vq->dev;  in vduse_queue_update_vring()
861     vq->vring.desc = iova_to_va(dev, &len, desc_addr);  in vduse_queue_update_vring()
867     vq->vring.avail = iova_to_va(dev, &len, avail_addr);  in vduse_queue_update_vring()
873     vq->vring.used = iova_to_va(dev, &len, used_addr);  in vduse_queue_update_vring()
878     if (!vq->vring.desc || !vq->vring.avail || !vq->vring.used) {  in vduse_queue_update_vring()
879         fprintf(stderr, "Failed to get vq[%d] iova mapping\n", vq->index);  in vduse_queue_update_vring()
886 static void vduse_queue_enable(VduseVirtq *vq)  in vduse_queue_enable()  argument
888     struct VduseDev *dev = vq->dev;  in vduse_queue_enable()
893     vq_info.index = vq->index;  in vduse_queue_enable()
896                 vq->index, strerror(errno));  in vduse_queue_enable()
904     vq->vring.num = vq_info.num;  in vduse_queue_enable()
905     vq->vring.desc_addr = vq_info.desc_addr;  in vduse_queue_enable()
906     vq->vring.avail_addr = vq_info.driver_addr;  in vduse_queue_enable()
907     vq->vring.used_addr = vq_info.device_addr;  in vduse_queue_enable()
909     if (vduse_queue_update_vring(vq, vq_info.desc_addr,  in vduse_queue_enable()
911         fprintf(stderr, "Failed to update vring for vq[%d]\n", vq->index);  in vduse_queue_enable()
917         fprintf(stderr, "Failed to init eventfd for vq[%d]\n", vq->index);  in vduse_queue_enable()
921     vq_eventfd.index = vq->index;  in vduse_queue_enable()
924         fprintf(stderr, "Failed to setup kick fd for vq[%d]\n", vq->index);  in vduse_queue_enable()
929     vq->fd = fd;  in vduse_queue_enable()
930     vq->signalled_used_valid = false;  in vduse_queue_enable()
931     vq->ready = true;  in vduse_queue_enable()
933     if (vduse_queue_check_inflights(vq)) {  in vduse_queue_enable()
934         fprintf(stderr, "Failed to check inflights for vq[%d]\n", vq->index);  in vduse_queue_enable()
939     dev->ops->enable_queue(dev, vq);  in vduse_queue_enable()
942 static void vduse_queue_disable(VduseVirtq *vq)  in vduse_queue_disable()  argument
944     struct VduseDev *dev = vq->dev;  in vduse_queue_disable()
947     if (!vq->ready) {  in vduse_queue_disable()
951     dev->ops->disable_queue(dev, vq);  in vduse_queue_disable()
953     eventfd.index = vq->index;  in vduse_queue_disable()
956     close(vq->fd);  in vduse_queue_disable()
958     assert(vq->inuse == 0);  in vduse_queue_disable()
960     vq->vring.num = 0;  in vduse_queue_disable()
961     vq->vring.desc_addr = 0;  in vduse_queue_disable()
962     vq->vring.avail_addr = 0;  in vduse_queue_disable()
963     vq->vring.used_addr = 0;  in vduse_queue_disable()
964     vq->vring.desc = 0;  in vduse_queue_disable()
965     vq->vring.avail = 0;  in vduse_queue_disable()
966     vq->vring.used = 0;  in vduse_queue_disable()
967     vq->ready = false;  in vduse_queue_disable()
968     vq->fd = -1;  in vduse_queue_disable()
1005     VduseVirtq *vq;  in vduse_dev_handler()  local
1018         vq = &dev->vqs[req.vq_state.index];  in vduse_dev_handler()
1019         resp.vq_state.split.avail_index = vq->last_avail_idx;  in vduse_dev_handler()
1034             vq = &dev->vqs[i];  in vduse_dev_handler()
1035             if (vq->ready) {  in vduse_dev_handler()
1036                 if (vduse_queue_update_vring(vq, vq->vring.desc_addr,  in vduse_dev_handler()
1037                                              vq->vring.avail_addr,  in vduse_dev_handler()
1038                                              vq->vring.used_addr)) {  in vduse_dev_handler()
1040                             vq->index);  in vduse_dev_handler()
1091     VduseVirtq *vq = &dev->vqs[index];  in vduse_dev_setup_queue()  local
1098     vq_config.index = vq->index;  in vduse_dev_setup_queue()
1105     vduse_queue_enable(vq);  in vduse_dev_setup_queue()