virtio_ring.c (2f18c2d153fb6fc889665688f266c780353002ec) virtio_ring.c (cbeedb72b97ad826e31e68e0717b763e2db0806d)
1/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.

--- 64 unchanged lines hidden (view full) ---

73 } while (0)
74#define START_USE(vq)
75#define END_USE(vq)
76#define LAST_ADD_TIME_UPDATE(vq)
77#define LAST_ADD_TIME_CHECK(vq)
78#define LAST_ADD_TIME_INVALID(vq)
79#endif
80
1/* Virtio ring implementation.
2 *
3 * Copyright 2007 Rusty Russell IBM Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.

--- 64 unchanged lines hidden (view full) ---

73 } while (0)
74#define START_USE(vq)
75#define END_USE(vq)
76#define LAST_ADD_TIME_UPDATE(vq)
77#define LAST_ADD_TIME_CHECK(vq)
78#define LAST_ADD_TIME_INVALID(vq)
79#endif
80
81struct vring_desc_state {
81struct vring_desc_state_split {
82 void *data; /* Data for callback. */
83 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
84};
85
86struct vring_virtqueue {
87 struct virtqueue vq;
88
89 /* Can we use weak barriers? */

--- 20 unchanged lines hidden (view full) ---

110 /* Actual memory layout for this queue */
111 struct vring vring;
112
113 /* Last written value to avail->flags */
114 u16 avail_flags_shadow;
115
116 /* Last written value to avail->idx in guest byte order */
117 u16 avail_idx_shadow;
82 void *data; /* Data for callback. */
83 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */
84};
85
86struct vring_virtqueue {
87 struct virtqueue vq;
88
89 /* Can we use weak barriers? */

--- 20 unchanged lines hidden (view full) ---

110 /* Actual memory layout for this queue */
111 struct vring vring;
112
113 /* Last written value to avail->flags */
114 u16 avail_flags_shadow;
115
116 /* Last written value to avail->idx in guest byte order */
117 u16 avail_idx_shadow;
118
119 /* Per-descriptor state. */
120 struct vring_desc_state_split *desc_state;
118 } split;
119
120 /* How to notify other side. FIXME: commonalize hcalls! */
121 bool (*notify)(struct virtqueue *vq);
122
123 /* DMA, allocation, and size information */
124 bool we_own_ring;
125 size_t queue_size_in_bytes;
126 dma_addr_t queue_dma_addr;
127
128#ifdef DEBUG
129 /* They're supposed to lock for us. */
130 unsigned int in_use;
131
132 /* Figure out if their kicks are too delayed. */
133 bool last_add_time_valid;
134 ktime_t last_add_time;
135#endif
121 } split;
122
123 /* How to notify other side. FIXME: commonalize hcalls! */
124 bool (*notify)(struct virtqueue *vq);
125
126 /* DMA, allocation, and size information */
127 bool we_own_ring;
128 size_t queue_size_in_bytes;
129 dma_addr_t queue_dma_addr;
130
131#ifdef DEBUG
132 /* They're supposed to lock for us. */
133 unsigned int in_use;
134
135 /* Figure out if their kicks are too delayed. */
136 bool last_add_time_valid;
137 ktime_t last_add_time;
138#endif
136
137 /* Per-descriptor state. */
138 struct vring_desc_state desc_state[];
139};
140
141
142/*
143 * Helpers.
144 */
145
146#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)

--- 275 unchanged lines hidden (view full) ---

422 /* Update free pointer */
423 if (indirect)
424 vq->free_head = virtio16_to_cpu(_vq->vdev,
425 vq->split.vring.desc[head].next);
426 else
427 vq->free_head = i;
428
429 /* Store token and indirect buffer state. */
139};
140
141
142/*
143 * Helpers.
144 */
145
146#define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)

--- 275 unchanged lines hidden (view full) ---

422 /* Update free pointer */
423 if (indirect)
424 vq->free_head = virtio16_to_cpu(_vq->vdev,
425 vq->split.vring.desc[head].next);
426 else
427 vq->free_head = i;
428
429 /* Store token and indirect buffer state. */
430 vq->desc_state[head].data = data;
430 vq->split.desc_state[head].data = data;
431 if (indirect)
431 if (indirect)
432 vq->desc_state[head].indir_desc = desc;
432 vq->split.desc_state[head].indir_desc = desc;
433 else
433 else
434 vq->desc_state[head].indir_desc = ctx;
434 vq->split.desc_state[head].indir_desc = ctx;
435
436 /* Put entry in available array (but don't update avail->idx until they
437 * do sync). */
438 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
439 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
440
441 /* Descriptors and available array need to be set before we expose the
442 * new available array entries. */

--- 64 unchanged lines hidden (view full) ---

507
508static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
509 void **ctx)
510{
511 unsigned int i, j;
512 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
513
514 /* Clear data ptr. */
435
436 /* Put entry in available array (but don't update avail->idx until they
437 * do sync). */
438 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
439 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
440
441 /* Descriptors and available array need to be set before we expose the
442 * new available array entries. */

--- 64 unchanged lines hidden (view full) ---

507
508static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
509 void **ctx)
510{
511 unsigned int i, j;
512 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
513
514 /* Clear data ptr. */
515 vq->desc_state[head].data = NULL;
515 vq->split.desc_state[head].data = NULL;
516
517 /* Put back on free list: unmap first-level descriptors and find end */
518 i = head;
519
520 while (vq->split.vring.desc[i].flags & nextflag) {
521 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
522 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
523 vq->vq.num_free++;
524 }
525
526 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
527 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
528 vq->free_head);
529 vq->free_head = head;
530
531 /* Plus final descriptor */
532 vq->vq.num_free++;
533
534 if (vq->indirect) {
516
517 /* Put back on free list: unmap first-level descriptors and find end */
518 i = head;
519
520 while (vq->split.vring.desc[i].flags & nextflag) {
521 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
522 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
523 vq->vq.num_free++;
524 }
525
526 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
527 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
528 vq->free_head);
529 vq->free_head = head;
530
531 /* Plus final descriptor */
532 vq->vq.num_free++;
533
534 if (vq->indirect) {
535 struct vring_desc *indir_desc = vq->desc_state[head].indir_desc;
535 struct vring_desc *indir_desc =
536 vq->split.desc_state[head].indir_desc;
536 u32 len;
537
538 /* Free the indirect table, if any, now that it's unmapped. */
539 if (!indir_desc)
540 return;
541
542 len = virtio32_to_cpu(vq->vq.vdev,
543 vq->split.vring.desc[head].len);
544
545 BUG_ON(!(vq->split.vring.desc[head].flags &
546 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
547 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
548
549 for (j = 0; j < len / sizeof(struct vring_desc); j++)
550 vring_unmap_one_split(vq, &indir_desc[j]);
551
552 kfree(indir_desc);
537 u32 len;
538
539 /* Free the indirect table, if any, now that it's unmapped. */
540 if (!indir_desc)
541 return;
542
543 len = virtio32_to_cpu(vq->vq.vdev,
544 vq->split.vring.desc[head].len);
545
546 BUG_ON(!(vq->split.vring.desc[head].flags &
547 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
548 BUG_ON(len == 0 || len % sizeof(struct vring_desc));
549
550 for (j = 0; j < len / sizeof(struct vring_desc); j++)
551 vring_unmap_one_split(vq, &indir_desc[j]);
552
553 kfree(indir_desc);
553 vq->desc_state[head].indir_desc = NULL;
554 vq->split.desc_state[head].indir_desc = NULL;
554 } else if (ctx) {
555 } else if (ctx) {
555 *ctx = vq->desc_state[head].indir_desc;
556 *ctx = vq->split.desc_state[head].indir_desc;
556 }
557}
558
559static inline bool more_used_split(const struct vring_virtqueue *vq)
560{
561 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
562 vq->split.vring.used->idx);
563}

--- 28 unchanged lines hidden (view full) ---

592 vq->split.vring.used->ring[last_used].id);
593 *len = virtio32_to_cpu(_vq->vdev,
594 vq->split.vring.used->ring[last_used].len);
595
596 if (unlikely(i >= vq->split.vring.num)) {
597 BAD_RING(vq, "id %u out of range\n", i);
598 return NULL;
599 }
557 }
558}
559
560static inline bool more_used_split(const struct vring_virtqueue *vq)
561{
562 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
563 vq->split.vring.used->idx);
564}

--- 28 unchanged lines hidden (view full) ---

593 vq->split.vring.used->ring[last_used].id);
594 *len = virtio32_to_cpu(_vq->vdev,
595 vq->split.vring.used->ring[last_used].len);
596
597 if (unlikely(i >= vq->split.vring.num)) {
598 BAD_RING(vq, "id %u out of range\n", i);
599 return NULL;
600 }
600 if (unlikely(!vq->desc_state[i].data)) {
601 if (unlikely(!vq->split.desc_state[i].data)) {
601 BAD_RING(vq, "id %u is not a head!\n", i);
602 return NULL;
603 }
604
605 /* detach_buf_split clears data, so grab it now. */
602 BAD_RING(vq, "id %u is not a head!\n", i);
603 return NULL;
604 }
605
606 /* detach_buf_split clears data, so grab it now. */
606 ret = vq->desc_state[i].data;
607 ret = vq->split.desc_state[i].data;
607 detach_buf_split(vq, i, ctx);
608 vq->last_used_idx++;
609 /* If we expect an interrupt for the next entry, tell host
610 * by writing event index and flush out the write before
611 * the read in the next get_buf call. */
612 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
613 virtio_store_mb(vq->weak_barriers,
614 &vring_used_event(&vq->split.vring),

--- 91 unchanged lines hidden (view full) ---

706{
707 struct vring_virtqueue *vq = to_vvq(_vq);
708 unsigned int i;
709 void *buf;
710
711 START_USE(vq);
712
713 for (i = 0; i < vq->split.vring.num; i++) {
608 detach_buf_split(vq, i, ctx);
609 vq->last_used_idx++;
610 /* If we expect an interrupt for the next entry, tell host
611 * by writing event index and flush out the write before
612 * the read in the next get_buf call. */
613 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
614 virtio_store_mb(vq->weak_barriers,
615 &vring_used_event(&vq->split.vring),

--- 91 unchanged lines hidden (view full) ---

707{
708 struct vring_virtqueue *vq = to_vvq(_vq);
709 unsigned int i;
710 void *buf;
711
712 START_USE(vq);
713
714 for (i = 0; i < vq->split.vring.num; i++) {
714 if (!vq->desc_state[i].data)
715 if (!vq->split.desc_state[i].data)
715 continue;
716 /* detach_buf_split clears data, so grab it now. */
716 continue;
717 /* detach_buf_split clears data, so grab it now. */
717 buf = vq->desc_state[i].data;
718 buf = vq->split.desc_state[i].data;
718 detach_buf_split(vq, i, NULL);
719 vq->split.avail_idx_shadow--;
720 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
721 vq->split.avail_idx_shadow);
722 END_USE(vq);
723 return buf;
724 }
725 /* That should have freed everything. */

--- 349 unchanged lines hidden (view full) ---

1075 bool context,
1076 bool (*notify)(struct virtqueue *),
1077 void (*callback)(struct virtqueue *),
1078 const char *name)
1079{
1080 unsigned int i;
1081 struct vring_virtqueue *vq;
1082
719 detach_buf_split(vq, i, NULL);
720 vq->split.avail_idx_shadow--;
721 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
722 vq->split.avail_idx_shadow);
723 END_USE(vq);
724 return buf;
725 }
726 /* That should have freed everything. */

--- 349 unchanged lines hidden (view full) ---

1076 bool context,
1077 bool (*notify)(struct virtqueue *),
1078 void (*callback)(struct virtqueue *),
1079 const char *name)
1080{
1081 unsigned int i;
1082 struct vring_virtqueue *vq;
1083
1083 vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state),
1084 GFP_KERNEL);
1084 vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1085 if (!vq)
1086 return NULL;
1087
1088 vq->vq.callback = callback;
1089 vq->vq.vdev = vdev;
1090 vq->vq.name = name;
1091 vq->vq.num_free = vring.num;
1092 vq->vq.index = index;

--- 22 unchanged lines hidden (view full) ---

1115 /* No callback? Tell other side not to bother us. */
1116 if (!callback) {
1117 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
1118 if (!vq->event)
1119 vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
1120 vq->split.avail_flags_shadow);
1121 }
1122
1085 if (!vq)
1086 return NULL;
1087
1088 vq->vq.callback = callback;
1089 vq->vq.vdev = vdev;
1090 vq->vq.name = name;
1091 vq->vq.num_free = vring.num;
1092 vq->vq.index = index;

--- 22 unchanged lines hidden (view full) ---

1115 /* No callback? Tell other side not to bother us. */
1116 if (!callback) {
1117 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
1118 if (!vq->event)
1119 vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
1120 vq->split.avail_flags_shadow);
1121 }
1122
1123 vq->split.desc_state = kmalloc_array(vring.num,
1124 sizeof(struct vring_desc_state_split), GFP_KERNEL);
1125 if (!vq->split.desc_state) {
1126 kfree(vq);
1127 return NULL;
1128 }
1129
1123 /* Put everything in free lists. */
1124 vq->free_head = 0;
1125 for (i = 0; i < vring.num-1; i++)
1126 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
1130 /* Put everything in free lists. */
1131 vq->free_head = 0;
1132 for (i = 0; i < vring.num-1; i++)
1133 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
1127 memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state));
1134 memset(vq->split.desc_state, 0, vring.num *
1135 sizeof(struct vring_desc_state_split));
1128
1129 return &vq->vq;
1130}
1131EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
1132
1133static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
1134 dma_addr_t *dma_handle, gfp_t flag)
1135{

--- 119 unchanged lines hidden (view full) ---

1255
1256void vring_del_virtqueue(struct virtqueue *_vq)
1257{
1258 struct vring_virtqueue *vq = to_vvq(_vq);
1259
1260 if (vq->we_own_ring) {
1261 vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes,
1262 vq->split.vring.desc, vq->queue_dma_addr);
1136
1137 return &vq->vq;
1138}
1139EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
1140
1141static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
1142 dma_addr_t *dma_handle, gfp_t flag)
1143{

--- 119 unchanged lines hidden (view full) ---

1263
1264void vring_del_virtqueue(struct virtqueue *_vq)
1265{
1266 struct vring_virtqueue *vq = to_vvq(_vq);
1267
1268 if (vq->we_own_ring) {
1269 vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes,
1270 vq->split.vring.desc, vq->queue_dma_addr);
1271 kfree(vq->split.desc_state);
1263 }
1264 list_del(&_vq->list);
1265 kfree(vq);
1266}
1267EXPORT_SYMBOL_GPL(vring_del_virtqueue);
1268
1269/* Manipulates transport-specific feature bits. */
1270void vring_transport_features(struct virtio_device *vdev)

--- 99 unchanged lines hidden --- 		1272 }
1273 list_del(&_vq->list);
1274 kfree(vq);
1275}
1276EXPORT_SYMBOL_GPL(vring_del_virtqueue);
1277
1278/* Manipulates transport-specific feature bits. */
1279void vring_transport_features(struct virtio_device *vdev)

--- 99 unchanged lines hidden ---