Lines Matching refs:vq
16 int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[], in virtqueue_add() argument
26 head = vq->free_head; in virtqueue_add()
28 desc = vq->vring.desc; in virtqueue_add()
32 if (vq->num_free < descs_used) { in virtqueue_add()
34 descs_used, vq->num_free); in virtqueue_add()
41 virtio_notify(vq->vdev, vq); in virtqueue_add()
48 desc[i].flags = cpu_to_virtio16(vq->vdev, VRING_DESC_F_NEXT); in virtqueue_add()
49 desc[i].addr = cpu_to_virtio64(vq->vdev, (u64)(size_t)sg->addr); in virtqueue_add()
50 desc[i].len = cpu_to_virtio32(vq->vdev, sg->length); in virtqueue_add()
53 i = virtio16_to_cpu(vq->vdev, desc[i].next); in virtqueue_add()
58 desc[i].flags = cpu_to_virtio16(vq->vdev, VRING_DESC_F_NEXT | in virtqueue_add()
60 desc[i].addr = cpu_to_virtio64(vq->vdev, in virtqueue_add()
62 desc[i].len = cpu_to_virtio32(vq->vdev, sg->length); in virtqueue_add()
65 i = virtio16_to_cpu(vq->vdev, desc[i].next); in virtqueue_add()
68 desc[prev].flags &= cpu_to_virtio16(vq->vdev, ~VRING_DESC_F_NEXT); in virtqueue_add()
71 vq->num_free -= descs_used; in virtqueue_add()
74 vq->free_head = i; in virtqueue_add()
80 avail = vq->avail_idx_shadow & (vq->vring.num - 1); in virtqueue_add()
81 vq->vring.avail->ring[avail] = cpu_to_virtio16(vq->vdev, head); in virtqueue_add()
88 vq->avail_idx_shadow++; in virtqueue_add()
89 vq->vring.avail->idx = cpu_to_virtio16(vq->vdev, vq->avail_idx_shadow); in virtqueue_add()
90 vq->num_added++; in virtqueue_add()
96 if (unlikely(vq->num_added == (1 << 16) - 1)) in virtqueue_add()
97 virtqueue_kick(vq); in virtqueue_add()
102 static bool virtqueue_kick_prepare(struct virtqueue *vq) in virtqueue_kick_prepare() argument
113 old = vq->avail_idx_shadow - vq->num_added; in virtqueue_kick_prepare()
114 new = vq->avail_idx_shadow; in virtqueue_kick_prepare()
115 vq->num_added = 0; in virtqueue_kick_prepare()
117 if (vq->event) { in virtqueue_kick_prepare()
118 needs_kick = vring_need_event(virtio16_to_cpu(vq->vdev, in virtqueue_kick_prepare()
119 vring_avail_event(&vq->vring)), new, old); in virtqueue_kick_prepare()
121 needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(vq->vdev, in virtqueue_kick_prepare()
128 void virtqueue_kick(struct virtqueue *vq) in virtqueue_kick() argument
130 if (virtqueue_kick_prepare(vq)) in virtqueue_kick()
131 virtio_notify(vq->vdev, vq); in virtqueue_kick()
134 static void detach_buf(struct virtqueue *vq, unsigned int head) in detach_buf() argument
137 __virtio16 nextflag = cpu_to_virtio16(vq->vdev, VRING_DESC_F_NEXT); in detach_buf()
142 while (vq->vring.desc[i].flags & nextflag) { in detach_buf()
143 i = virtio16_to_cpu(vq->vdev, vq->vring.desc[i].next); in detach_buf()
144 vq->num_free++; in detach_buf()
147 vq->vring.desc[i].next = cpu_to_virtio16(vq->vdev, vq->free_head); in detach_buf()
148 vq->free_head = head; in detach_buf()
151 vq->num_free++; in detach_buf()
154 static inline bool more_used(const struct virtqueue *vq) in more_used() argument
156 return vq->last_used_idx != virtio16_to_cpu(vq->vdev, in more_used()
157 vq->vring.used->idx); in more_used()
160 void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len) in virtqueue_get_buf() argument
165 if (!more_used(vq)) { in virtqueue_get_buf()
167 vq->vdev->name, vq->index); in virtqueue_get_buf()
174 last_used = (vq->last_used_idx & (vq->vring.num - 1)); in virtqueue_get_buf()
175 i = virtio32_to_cpu(vq->vdev, vq->vring.used->ring[last_used].id); in virtqueue_get_buf()
177 *len = virtio32_to_cpu(vq->vdev, in virtqueue_get_buf()
178 vq->vring.used->ring[last_used].len); in virtqueue_get_buf()
180 vq->vdev->name, vq->index, i, *len); in virtqueue_get_buf()
183 if (unlikely(i >= vq->vring.num)) { in virtqueue_get_buf()
185 vq->vdev->name, vq->index, i); in virtqueue_get_buf()
189 detach_buf(vq, i); in virtqueue_get_buf()
190 vq->last_used_idx++; in virtqueue_get_buf()
196 if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) in virtqueue_get_buf()
197 virtio_store_mb(&vring_used_event(&vq->vring), in virtqueue_get_buf()
198 cpu_to_virtio16(vq->vdev, vq->last_used_idx)); in virtqueue_get_buf()
200 return (void *)(uintptr_t)virtio64_to_cpu(vq->vdev, in virtqueue_get_buf()
201 vq->vring.desc[i].addr); in virtqueue_get_buf()
209 struct virtqueue *vq; in __vring_new_virtqueue() local
213 vq = malloc(sizeof(*vq)); in __vring_new_virtqueue()
214 if (!vq) in __vring_new_virtqueue()
217 vq->vdev = vdev; in __vring_new_virtqueue()
218 vq->index = index; in __vring_new_virtqueue()
219 vq->num_free = vring.num; in __vring_new_virtqueue()
220 vq->vring = vring; in __vring_new_virtqueue()
221 vq->last_used_idx = 0; in __vring_new_virtqueue()
222 vq->avail_flags_shadow = 0; in __vring_new_virtqueue()
223 vq->avail_idx_shadow = 0; in __vring_new_virtqueue()
224 vq->num_added = 0; in __vring_new_virtqueue()
225 list_add_tail(&vq->list, &uc_priv->vqs); in __vring_new_virtqueue()
227 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); in __vring_new_virtqueue()
230 vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in __vring_new_virtqueue()
231 if (!vq->event) in __vring_new_virtqueue()
232 vq->vring.avail->flags = cpu_to_virtio16(vdev, in __vring_new_virtqueue()
233 vq->avail_flags_shadow); in __vring_new_virtqueue()
236 vq->free_head = 0; in __vring_new_virtqueue()
238 vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1); in __vring_new_virtqueue()
240 return vq; in __vring_new_virtqueue()
247 struct virtqueue *vq; in vring_create_virtqueue() local
277 vq = __vring_new_virtqueue(index, vring, udev); in vring_create_virtqueue()
278 if (!vq) { in vring_create_virtqueue()
283 queue, vq, num); in vring_create_virtqueue()
285 return vq; in vring_create_virtqueue()
288 void vring_del_virtqueue(struct virtqueue *vq) in vring_del_virtqueue() argument
290 free(vq->vring.desc); in vring_del_virtqueue()
291 list_del(&vq->list); in vring_del_virtqueue()
292 free(vq); in vring_del_virtqueue()
295 unsigned int virtqueue_get_vring_size(struct virtqueue *vq) in virtqueue_get_vring_size() argument
297 return vq->vring.num; in virtqueue_get_vring_size()
300 ulong virtqueue_get_desc_addr(struct virtqueue *vq) in virtqueue_get_desc_addr() argument
302 return (ulong)vq->vring.desc; in virtqueue_get_desc_addr()
305 ulong virtqueue_get_avail_addr(struct virtqueue *vq) in virtqueue_get_avail_addr() argument
307 return (ulong)vq->vring.desc + in virtqueue_get_avail_addr()
308 ((char *)vq->vring.avail - (char *)vq->vring.desc); in virtqueue_get_avail_addr()
311 ulong virtqueue_get_used_addr(struct virtqueue *vq) in virtqueue_get_used_addr() argument
313 return (ulong)vq->vring.desc + in virtqueue_get_used_addr()
314 ((char *)vq->vring.used - (char *)vq->vring.desc); in virtqueue_get_used_addr()
317 bool virtqueue_poll(struct virtqueue *vq, u16 last_used_idx) in virtqueue_poll() argument
321 return last_used_idx != virtio16_to_cpu(vq->vdev, vq->vring.used->idx); in virtqueue_poll()
324 void virtqueue_dump(struct virtqueue *vq) in virtqueue_dump() argument
328 printf("virtqueue %p for dev %s:\n", vq, vq->vdev->name); in virtqueue_dump()
330 vq->index, vq->vring.desc, vq->vring.num); in virtqueue_dump()
332 vq->free_head, vq->num_added, vq->num_free); in virtqueue_dump()
334 vq->last_used_idx, vq->avail_flags_shadow, vq->avail_idx_shadow); in virtqueue_dump()
337 for (i = 0; i < vq->vring.num; i++) { in virtqueue_dump()
339 i, vq->vring.desc[i].addr, vq->vring.desc[i].len, in virtqueue_dump()
340 vq->vring.desc[i].flags, vq->vring.desc[i].next); in virtqueue_dump()
345 vq->vring.avail->flags, vq->vring.avail->idx); in virtqueue_dump()
346 for (i = 0; i < vq->vring.num; i++) { in virtqueue_dump()
348 i, vq->vring.avail->ring[i]); in virtqueue_dump()
353 vq->vring.used->flags, vq->vring.used->idx); in virtqueue_dump()
354 for (i = 0; i < vq->vring.num; i++) { in virtqueue_dump()
356 vq->vring.used->ring[i].id, vq->vring.used->ring[i].len); in virtqueue_dump()