virtio_ring.c (818b930bc15077fc00ff16bb22c5df1857f05afa) virtio_ring.c (06ca287dbac9cc19d04ac2901b8c4882c03795ff)
1/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.

--- 79 unchanged lines hidden (view full) ---

88 bool broken;
89
90 /* Host supports indirect buffers */
91 bool indirect;
92
93 /* Host publishes avail event idx */
94 bool event;
95
1/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.

--- 79 unchanged lines hidden (view full) ---

88 bool broken;
89
90 /* Host supports indirect buffers */
91 bool indirect;
92
93 /* Host publishes avail event idx */
94 bool event;
95
96 /* Number of free buffers */
97 unsigned int num_free;
98 /* Head of free buffer list. */
99 unsigned int free_head;
100 /* Number we've added since last sync. */
101 unsigned int num_added;
102
103 /* Last used index we've seen. */
104 u16 last_used_idx;
105
106 /* How to notify other side. FIXME: commonalize hcalls! */
107 void (*notify)(struct virtqueue *vq);
108
96 /* Head of free buffer list. */
97 unsigned int free_head;
98 /* Number we've added since last sync. */
99 unsigned int num_added;
100
101 /* Last used index we've seen. */
102 u16 last_used_idx;
103
104 /* How to notify other side. FIXME: commonalize hcalls! */
105 void (*notify)(struct virtqueue *vq);
106
109 /* Index of the queue */
110 int queue_index;
111
112#ifdef DEBUG
113 /* They're supposed to lock for us. */
114 unsigned int in_use;
115
116 /* Figure out if their kicks are too delayed. */
117 bool last_add_time_valid;
118 ktime_t last_add_time;
119#endif

--- 10 unchanged lines hidden (view full) ---

130 unsigned int out,
131 unsigned int in,
132 gfp_t gfp)
133{
134 struct vring_desc *desc;
135 unsigned head;
136 int i;
137
107#ifdef DEBUG
108 /* They're supposed to lock for us. */
109 unsigned int in_use;
110
111 /* Figure out if their kicks are too delayed. */
112 bool last_add_time_valid;
113 ktime_t last_add_time;
114#endif

--- 10 unchanged lines hidden (view full) ---

125 unsigned int out,
126 unsigned int in,
127 gfp_t gfp)
128{
129 struct vring_desc *desc;
130 unsigned head;
131 int i;
132
133 /*
134 * We require lowmem mappings for the descriptors because
135 * otherwise virt_to_phys will give us bogus addresses in the
136 * virtqueue.
137 */
138 gfp &= ~(__GFP_HIGHMEM | __GFP_HIGH);
139
138 desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
139 if (!desc)
140 return -ENOMEM;
141
142 /* Transfer entries from the sg list into the indirect page */
143 for (i = 0; i < out; i++) {
144 desc[i].flags = VRING_DESC_F_NEXT;
145 desc[i].addr = sg_phys(sg);

--- 9 unchanged lines hidden (view full) ---

155 sg++;
156 }
157
158 /* Last one doesn't continue. */
159 desc[i-1].flags &= ~VRING_DESC_F_NEXT;
160 desc[i-1].next = 0;
161
162 /* We're about to use a buffer */
140 desc = kmalloc((out + in) * sizeof(struct vring_desc), gfp);
141 if (!desc)
142 return -ENOMEM;
143
144 /* Transfer entries from the sg list into the indirect page */
145 for (i = 0; i < out; i++) {
146 desc[i].flags = VRING_DESC_F_NEXT;
147 desc[i].addr = sg_phys(sg);

--- 9 unchanged lines hidden (view full) ---

157 sg++;
158 }
159
160 /* Last one doesn't continue. */
161 desc[i-1].flags &= ~VRING_DESC_F_NEXT;
162 desc[i-1].next = 0;
163
164 /* We're about to use a buffer */
163 vq->num_free--;
165 vq->vq.num_free--;
164
165 /* Use a single buffer which doesn't continue */
166 head = vq->free_head;
167 vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT;
168 vq->vring.desc[head].addr = virt_to_phys(desc);
169 vq->vring.desc[head].len = i * sizeof(struct vring_desc);
170
171 /* Update free pointer */
172 vq->free_head = vq->vring.desc[head].next;
173
174 return head;
175}
176
166
167 /* Use a single buffer which doesn't continue */
168 head = vq->free_head;
169 vq->vring.desc[head].flags = VRING_DESC_F_INDIRECT;
170 vq->vring.desc[head].addr = virt_to_phys(desc);
171 vq->vring.desc[head].len = i * sizeof(struct vring_desc);
172
173 /* Update free pointer */
174 vq->free_head = vq->vring.desc[head].next;
175
176 return head;
177}
178
177int virtqueue_get_queue_index(struct virtqueue *_vq)
178{
179 struct vring_virtqueue *vq = to_vvq(_vq);
180 return vq->queue_index;
181}
182EXPORT_SYMBOL_GPL(virtqueue_get_queue_index);
183
184/**
185 * virtqueue_add_buf - expose buffer to other end
186 * @vq: the struct virtqueue we're talking about.
187 * @sg: the description of the buffer(s).
188 * @out_num: the number of sg readable by other side
189 * @in_num: the number of sg which are writable (after readable ones)
190 * @data: the token identifying the buffer.
191 * @gfp: how to do memory allocations (if necessary).

--- 31 unchanged lines hidden (view full) ---

223 > 100);
224 vq->last_add_time = now;
225 vq->last_add_time_valid = true;
226 }
227#endif
228
229 /* If the host supports indirect descriptor tables, and we have multiple
230 * buffers, then go indirect. FIXME: tune this threshold */
179/**
180 * virtqueue_add_buf - expose buffer to other end
181 * @vq: the struct virtqueue we're talking about.
182 * @sg: the description of the buffer(s).
183 * @out_num: the number of sg readable by other side
184 * @in_num: the number of sg which are writable (after readable ones)
185 * @data: the token identifying the buffer.
186 * @gfp: how to do memory allocations (if necessary).

--- 31 unchanged lines hidden (view full) ---

218 > 100);
219 vq->last_add_time = now;
220 vq->last_add_time_valid = true;
221 }
222#endif
223
224 /* If the host supports indirect descriptor tables, and we have multiple
225 * buffers, then go indirect. FIXME: tune this threshold */
231 if (vq->indirect && (out + in) > 1 && vq->num_free) {
226 if (vq->indirect && (out + in) > 1 && vq->vq.num_free) {
232 head = vring_add_indirect(vq, sg, out, in, gfp);
233 if (likely(head >= 0))
234 goto add_head;
235 }
236
237 BUG_ON(out + in > vq->vring.num);
238 BUG_ON(out + in == 0);
239
227 head = vring_add_indirect(vq, sg, out, in, gfp);
228 if (likely(head >= 0))
229 goto add_head;
230 }
231
232 BUG_ON(out + in > vq->vring.num);
233 BUG_ON(out + in == 0);
234
240 if (vq->num_free < out + in) {
235 if (vq->vq.num_free < out + in) {
241 pr_debug("Can't add buf len %i - avail = %i\n",
236 pr_debug("Can't add buf len %i - avail = %i\n",
242 out + in, vq->num_free);
237 out + in, vq->vq.num_free);
243 /* FIXME: for historical reasons, we force a notify here if
244 * there are outgoing parts to the buffer. Presumably the
245 * host should service the ring ASAP. */
246 if (out)
247 vq->notify(&vq->vq);
248 END_USE(vq);
249 return -ENOSPC;
250 }
251
252 /* We're about to use some buffers from the free list. */
238 /* FIXME: for historical reasons, we force a notify here if
239 * there are outgoing parts to the buffer. Presumably the
240 * host should service the ring ASAP. */
241 if (out)
242 vq->notify(&vq->vq);
243 END_USE(vq);
244 return -ENOSPC;
245 }
246
247 /* We're about to use some buffers from the free list. */
253 vq->num_free -= out + in;
248 vq->vq.num_free -= out + in;
254
255 head = vq->free_head;
256 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) {
257 vq->vring.desc[i].flags = VRING_DESC_F_NEXT;
258 vq->vring.desc[i].addr = sg_phys(sg);
259 vq->vring.desc[i].len = sg->length;
260 prev = i;
261 sg++;

--- 29 unchanged lines hidden (view full) ---

291 /* This is very unlikely, but theoretically possible. Kick
292 * just in case. */
293 if (unlikely(vq->num_added == (1 << 16) - 1))
294 virtqueue_kick(_vq);
295
296 pr_debug("Added buffer head %i to %p\n", head, vq);
297 END_USE(vq);
298
249
250 head = vq->free_head;
251 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) {
252 vq->vring.desc[i].flags = VRING_DESC_F_NEXT;
253 vq->vring.desc[i].addr = sg_phys(sg);
254 vq->vring.desc[i].len = sg->length;
255 prev = i;
256 sg++;

--- 29 unchanged lines hidden (view full) ---

286 /* This is very unlikely, but theoretically possible. Kick
287 * just in case. */
288 if (unlikely(vq->num_added == (1 << 16) - 1))
289 virtqueue_kick(_vq);
290
291 pr_debug("Added buffer head %i to %p\n", head, vq);
292 END_USE(vq);
293
299 return vq->num_free;
294 return vq->vq.num_free;
300}
301EXPORT_SYMBOL_GPL(virtqueue_add_buf);
302
303/**
304 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
305 * @vq: the struct virtqueue
306 *
307 * Instead of virtqueue_kick(), you can do:

--- 80 unchanged lines hidden (view full) ---

388 i = head;
389
390 /* Free the indirect table */
391 if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
392 kfree(phys_to_virt(vq->vring.desc[i].addr));
393
394 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
395 i = vq->vring.desc[i].next;
295}
296EXPORT_SYMBOL_GPL(virtqueue_add_buf);
297
298/**
299 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
300 * @vq: the struct virtqueue
301 *
302 * Instead of virtqueue_kick(), you can do:

--- 80 unchanged lines hidden (view full) ---

383 i = head;
384
385 /* Free the indirect table */
386 if (vq->vring.desc[i].flags & VRING_DESC_F_INDIRECT)
387 kfree(phys_to_virt(vq->vring.desc[i].addr));
388
389 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) {
390 i = vq->vring.desc[i].next;
396 vq->num_free++;
391 vq->vq.num_free++;
397 }
398
399 vq->vring.desc[i].next = vq->free_head;
400 vq->free_head = head;
401 /* Plus final descriptor */
392 }
393
394 vq->vring.desc[i].next = vq->free_head;
395 vq->free_head = head;
396 /* Plus final descriptor */
402 vq->num_free++;
397 vq->vq.num_free++;
403}
404
405static inline bool more_used(const struct vring_virtqueue *vq)
406{
407 return vq->last_used_idx != vq->vring.used->idx;
408}
409
410/**

--- 183 unchanged lines hidden (view full) ---

594 /* detach_buf clears data, so grab it now. */
595 buf = vq->data[i];
596 detach_buf(vq, i);
597 vq->vring.avail->idx--;
598 END_USE(vq);
599 return buf;
600 }
601 /* That should have freed everything. */
398}
399
400static inline bool more_used(const struct vring_virtqueue *vq)
401{
402 return vq->last_used_idx != vq->vring.used->idx;
403}
404
405/**

--- 183 unchanged lines hidden (view full) ---

589 /* detach_buf clears data, so grab it now. */
590 buf = vq->data[i];
591 detach_buf(vq, i);
592 vq->vring.avail->idx--;
593 END_USE(vq);
594 return buf;
595 }
596 /* That should have freed everything. */
602 BUG_ON(vq->num_free != vq->vring.num);
597 BUG_ON(vq->vq.num_free != vq->vring.num);
603
604 END_USE(vq);
605 return NULL;
606}
607EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
608
609irqreturn_t vring_interrupt(int irq, void *_vq)
610{

--- 37 unchanged lines hidden (view full) ---

648 vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);
649 if (!vq)
650 return NULL;
651
652 vring_init(&vq->vring, num, pages, vring_align);
653 vq->vq.callback = callback;
654 vq->vq.vdev = vdev;
655 vq->vq.name = name;
598
599 END_USE(vq);
600 return NULL;
601}
602EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
603
604irqreturn_t vring_interrupt(int irq, void *_vq)
605{

--- 37 unchanged lines hidden (view full) ---

643 vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL);
644 if (!vq)
645 return NULL;
646
647 vring_init(&vq->vring, num, pages, vring_align);
648 vq->vq.callback = callback;
649 vq->vq.vdev = vdev;
650 vq->vq.name = name;
651 vq->vq.num_free = num;
652 vq->vq.index = index;
656 vq->notify = notify;
657 vq->weak_barriers = weak_barriers;
658 vq->broken = false;
659 vq->last_used_idx = 0;
660 vq->num_added = 0;
653 vq->notify = notify;
654 vq->weak_barriers = weak_barriers;
655 vq->broken = false;
656 vq->last_used_idx = 0;
657 vq->num_added = 0;
661 vq->queue_index = index;
662 list_add_tail(&vq->vq.list, &vdev->vqs);
663#ifdef DEBUG
664 vq->in_use = false;
665 vq->last_add_time_valid = false;
666#endif
667
668 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
669 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
670
671 /* No callback? Tell other side not to bother us. */
672 if (!callback)
673 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
674
675 /* Put everything in free lists. */
658 list_add_tail(&vq->vq.list, &vdev->vqs);
659#ifdef DEBUG
660 vq->in_use = false;
661 vq->last_add_time_valid = false;
662#endif
663
664 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC);
665 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
666
667 /* No callback? Tell other side not to bother us. */
668 if (!callback)
669 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
670
671 /* Put everything in free lists. */
676 vq->num_free = num;
677 vq->free_head = 0;
678 for (i = 0; i < num-1; i++) {
679 vq->vring.desc[i].next = i+1;
680 vq->data[i] = NULL;
681 }
682 vq->data[i] = NULL;
683
684 return &vq->vq;

--- 46 unchanged lines hidden --- 		672 vq->free_head = 0;
673 for (i = 0; i < num-1; i++) {
674 vq->vring.desc[i].next = i+1;
675 vq->data[i] = NULL;
676 }
677 vq->data[i] = NULL;
678
679 return &vq->vq;

--- 46 unchanged lines hidden ---