1 /* Virtio ring implementation. 2 * 3 * Copyright 2007 Rusty Russell IBM Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 18 */ 19 #include <linux/virtio.h> 20 #include <linux/virtio_ring.h> 21 #include <linux/virtio_config.h> 22 #include <linux/device.h> 23 24 #ifdef DEBUG 25 /* For development, we want to crash whenever the ring is screwed. */ 26 #define BAD_RING(_vq, fmt...) \ 27 do { dev_err(&(_vq)->vq.vdev->dev, fmt); BUG(); } while(0) 28 /* Caller is supposed to guarantee no reentry. */ 29 #define START_USE(_vq) \ 30 do { \ 31 if ((_vq)->in_use) \ 32 panic("in_use = %i\n", (_vq)->in_use); \ 33 (_vq)->in_use = __LINE__; \ 34 mb(); \ 35 } while(0) 36 #define END_USE(_vq) \ 37 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; mb(); } while(0) 38 #else 39 #define BAD_RING(_vq, fmt...) \ 40 do { dev_err(&_vq->vq.vdev->dev, fmt); (_vq)->broken = true; } while(0) 41 #define START_USE(vq) 42 #define END_USE(vq) 43 #endif 44 45 struct vring_virtqueue 46 { 47 struct virtqueue vq; 48 49 /* Actual memory layout for this queue */ 50 struct vring vring; 51 52 /* Other side has made a mess, don't try any more. */ 53 bool broken; 54 55 /* Number of free buffers */ 56 unsigned int num_free; 57 /* Head of free buffer list. */ 58 unsigned int free_head; 59 /* Number we've added since last sync. */ 60 unsigned int num_added; 61 62 /* Last used index we've seen. */ 63 u16 last_used_idx; 64 65 /* How to notify other side. FIXME: commonalize hcalls! */ 66 void (*notify)(struct virtqueue *vq); 67 68 #ifdef DEBUG 69 /* They're supposed to lock for us. */ 70 unsigned int in_use; 71 #endif 72 73 /* Tokens for callbacks. */ 74 void *data[]; 75 }; 76 77 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) 78 79 static int vring_add_buf(struct virtqueue *_vq, 80 struct scatterlist sg[], 81 unsigned int out, 82 unsigned int in, 83 void *data) 84 { 85 struct vring_virtqueue *vq = to_vvq(_vq); 86 unsigned int i, avail, head, uninitialized_var(prev); 87 88 BUG_ON(data == NULL); 89 BUG_ON(out + in > vq->vring.num); 90 BUG_ON(out + in == 0); 91 92 START_USE(vq); 93 94 if (vq->num_free < out + in) { 95 pr_debug("Can't add buf len %i - avail = %i\n", 96 out + in, vq->num_free); 97 /* FIXME: for historical reasons, we force a notify here if 98 * there are outgoing parts to the buffer. Presumably the 99 * host should service the ring ASAP. */ 100 if (out) 101 vq->notify(&vq->vq); 102 END_USE(vq); 103 return -ENOSPC; 104 } 105 106 /* We're about to use some buffers from the free list. */ 107 vq->num_free -= out + in; 108 109 head = vq->free_head; 110 for (i = vq->free_head; out; i = vq->vring.desc[i].next, out--) { 111 vq->vring.desc[i].flags = VRING_DESC_F_NEXT; 112 vq->vring.desc[i].addr = sg_phys(sg); 113 vq->vring.desc[i].len = sg->length; 114 prev = i; 115 sg++; 116 } 117 for (; in; i = vq->vring.desc[i].next, in--) { 118 vq->vring.desc[i].flags = VRING_DESC_F_NEXT|VRING_DESC_F_WRITE; 119 vq->vring.desc[i].addr = sg_phys(sg); 120 vq->vring.desc[i].len = sg->length; 121 prev = i; 122 sg++; 123 } 124 /* Last one doesn't continue. */ 125 vq->vring.desc[prev].flags &= ~VRING_DESC_F_NEXT; 126 127 /* Update free pointer */ 128 vq->free_head = i; 129 130 /* Set token. */ 131 vq->data[head] = data; 132 133 /* Put entry in available array (but don't update avail->idx until they 134 * do sync). FIXME: avoid modulus here? */ 135 avail = (vq->vring.avail->idx + vq->num_added++) % vq->vring.num; 136 vq->vring.avail->ring[avail] = head; 137 138 pr_debug("Added buffer head %i to %p\n", head, vq); 139 END_USE(vq); 140 return 0; 141 } 142 143 static void vring_kick(struct virtqueue *_vq) 144 { 145 struct vring_virtqueue *vq = to_vvq(_vq); 146 START_USE(vq); 147 /* Descriptors and available array need to be set before we expose the 148 * new available array entries. */ 149 wmb(); 150 151 vq->vring.avail->idx += vq->num_added; 152 vq->num_added = 0; 153 154 /* Need to update avail index before checking if we should notify */ 155 mb(); 156 157 if (!(vq->vring.used->flags & VRING_USED_F_NO_NOTIFY)) 158 /* Prod other side to tell it about changes. */ 159 vq->notify(&vq->vq); 160 161 END_USE(vq); 162 } 163 164 static void detach_buf(struct vring_virtqueue *vq, unsigned int head) 165 { 166 unsigned int i; 167 168 /* Clear data ptr. */ 169 vq->data[head] = NULL; 170 171 /* Put back on free list: find end */ 172 i = head; 173 while (vq->vring.desc[i].flags & VRING_DESC_F_NEXT) { 174 i = vq->vring.desc[i].next; 175 vq->num_free++; 176 } 177 178 vq->vring.desc[i].next = vq->free_head; 179 vq->free_head = head; 180 /* Plus final descriptor */ 181 vq->num_free++; 182 } 183 184 static inline bool more_used(const struct vring_virtqueue *vq) 185 { 186 return vq->last_used_idx != vq->vring.used->idx; 187 } 188 189 static void *vring_get_buf(struct virtqueue *_vq, unsigned int *len) 190 { 191 struct vring_virtqueue *vq = to_vvq(_vq); 192 void *ret; 193 unsigned int i; 194 195 START_USE(vq); 196 197 if (unlikely(vq->broken)) { 198 END_USE(vq); 199 return NULL; 200 } 201 202 if (!more_used(vq)) { 203 pr_debug("No more buffers in queue\n"); 204 END_USE(vq); 205 return NULL; 206 } 207 208 i = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].id; 209 *len = vq->vring.used->ring[vq->last_used_idx%vq->vring.num].len; 210 211 if (unlikely(i >= vq->vring.num)) { 212 BAD_RING(vq, "id %u out of range\n", i); 213 return NULL; 214 } 215 if (unlikely(!vq->data[i])) { 216 BAD_RING(vq, "id %u is not a head!\n", i); 217 return NULL; 218 } 219 220 /* detach_buf clears data, so grab it now. */ 221 ret = vq->data[i]; 222 detach_buf(vq, i); 223 vq->last_used_idx++; 224 END_USE(vq); 225 return ret; 226 } 227 228 static void vring_disable_cb(struct virtqueue *_vq) 229 { 230 struct vring_virtqueue *vq = to_vvq(_vq); 231 232 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 233 } 234 235 static bool vring_enable_cb(struct virtqueue *_vq) 236 { 237 struct vring_virtqueue *vq = to_vvq(_vq); 238 239 START_USE(vq); 240 241 /* We optimistically turn back on interrupts, then check if there was 242 * more to do. */ 243 vq->vring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 244 mb(); 245 if (unlikely(more_used(vq))) { 246 END_USE(vq); 247 return false; 248 } 249 250 END_USE(vq); 251 return true; 252 } 253 254 irqreturn_t vring_interrupt(int irq, void *_vq) 255 { 256 struct vring_virtqueue *vq = to_vvq(_vq); 257 258 if (!more_used(vq)) { 259 pr_debug("virtqueue interrupt with no work for %p\n", vq); 260 return IRQ_NONE; 261 } 262 263 if (unlikely(vq->broken)) 264 return IRQ_HANDLED; 265 266 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); 267 if (vq->vq.callback) 268 vq->vq.callback(&vq->vq); 269 270 return IRQ_HANDLED; 271 } 272 EXPORT_SYMBOL_GPL(vring_interrupt); 273 274 static struct virtqueue_ops vring_vq_ops = { 275 .add_buf = vring_add_buf, 276 .get_buf = vring_get_buf, 277 .kick = vring_kick, 278 .disable_cb = vring_disable_cb, 279 .enable_cb = vring_enable_cb, 280 }; 281 282 struct virtqueue *vring_new_virtqueue(unsigned int num, 283 unsigned int vring_align, 284 struct virtio_device *vdev, 285 void *pages, 286 void (*notify)(struct virtqueue *), 287 void (*callback)(struct virtqueue *)) 288 { 289 struct vring_virtqueue *vq; 290 unsigned int i; 291 292 /* We assume num is a power of 2. */ 293 if (num & (num - 1)) { 294 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); 295 return NULL; 296 } 297 298 vq = kmalloc(sizeof(*vq) + sizeof(void *)*num, GFP_KERNEL); 299 if (!vq) 300 return NULL; 301 302 vring_init(&vq->vring, num, pages, vring_align); 303 vq->vq.callback = callback; 304 vq->vq.vdev = vdev; 305 vq->vq.vq_ops = &vring_vq_ops; 306 vq->notify = notify; 307 vq->broken = false; 308 vq->last_used_idx = 0; 309 vq->num_added = 0; 310 #ifdef DEBUG 311 vq->in_use = false; 312 #endif 313 314 /* No callback? Tell other side not to bother us. */ 315 if (!callback) 316 vq->vring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 317 318 /* Put everything in free lists. */ 319 vq->num_free = num; 320 vq->free_head = 0; 321 for (i = 0; i < num-1; i++) 322 vq->vring.desc[i].next = i+1; 323 324 return &vq->vq; 325 } 326 EXPORT_SYMBOL_GPL(vring_new_virtqueue); 327 328 void vring_del_virtqueue(struct virtqueue *vq) 329 { 330 kfree(to_vvq(vq)); 331 } 332 EXPORT_SYMBOL_GPL(vring_del_virtqueue); 333 334 /* Manipulates transport-specific feature bits. */ 335 void vring_transport_features(struct virtio_device *vdev) 336 { 337 unsigned int i; 338 339 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) { 340 switch (i) { 341 default: 342 /* We don't understand this bit. */ 343 clear_bit(i, vdev->features); 344 } 345 } 346 } 347 EXPORT_SYMBOL_GPL(vring_transport_features); 348 349 MODULE_LICENSE("GPL"); 350