1 /* 2 * Dedicated thread for virtio-blk I/O processing 3 * 4 * Copyright 2012 IBM, Corp. 5 * Copyright 2012 Red Hat, Inc. and/or its affiliates 6 * 7 * Authors: 8 * Stefan Hajnoczi <stefanha@redhat.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2 or later. 11 * See the COPYING file in the top-level directory. 12 * 13 */ 14 15 #include "qemu/osdep.h" 16 #include "qapi/error.h" 17 #include "trace.h" 18 #include "qemu/iov.h" 19 #include "qemu/thread.h" 20 #include "qemu/error-report.h" 21 #include "hw/virtio/virtio-access.h" 22 #include "hw/virtio/virtio-blk.h" 23 #include "virtio-blk.h" 24 #include "block/aio.h" 25 #include "hw/virtio/virtio-bus.h" 26 #include "qom/object_interfaces.h" 27 28 struct VirtIOBlockDataPlane { 29 bool starting; 30 bool stopping; 31 32 VirtIOBlkConf *conf; 33 VirtIODevice *vdev; 34 QEMUBH *bh; /* bh for guest notification */ 35 unsigned long *batch_notify_vqs; 36 bool batch_notifications; 37 38 /* Note that these EventNotifiers are assigned by value. This is 39 * fine as long as you do not call event_notifier_cleanup on them 40 * (because you don't own the file descriptor or handle; you just 41 * use it). 42 */ 43 IOThread *iothread; 44 AioContext *ctx; 45 }; 46 47 /* Raise an interrupt to signal guest, if necessary */ 48 void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) 49 { 50 if (s->batch_notifications) { 51 set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs); 52 qemu_bh_schedule(s->bh); 53 } else { 54 virtio_notify_irqfd(s->vdev, vq); 55 } 56 } 57 58 static void notify_guest_bh(void *opaque) 59 { 60 VirtIOBlockDataPlane *s = opaque; 61 unsigned nvqs = s->conf->num_queues; 62 unsigned long bitmap[BITS_TO_LONGS(nvqs)]; 63 unsigned j; 64 65 memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap)); 66 memset(s->batch_notify_vqs, 0, sizeof(bitmap)); 67 68 for (j = 0; j < nvqs; j += BITS_PER_LONG) { 69 unsigned long bits = bitmap[j]; 70 71 while (bits != 0) { 72 unsigned i = j + ctzl(bits); 73 VirtQueue *vq = virtio_get_queue(s->vdev, i); 74 75 virtio_notify_irqfd(s->vdev, vq); 76 77 bits &= bits - 1; /* clear right-most bit */ 78 } 79 } 80 } 81 82 /* Context: QEMU global mutex held */ 83 bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, 84 VirtIOBlockDataPlane **dataplane, 85 Error **errp) 86 { 87 VirtIOBlockDataPlane *s; 88 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); 89 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 90 91 *dataplane = NULL; 92 93 if (conf->iothread) { 94 if (!k->set_guest_notifiers || !k->ioeventfd_assign) { 95 error_setg(errp, 96 "device is incompatible with iothread " 97 "(transport does not support notifiers)"); 98 return false; 99 } 100 if (!virtio_device_ioeventfd_enabled(vdev)) { 101 error_setg(errp, "ioeventfd is required for iothread"); 102 return false; 103 } 104 105 /* If dataplane is (re-)enabled while the guest is running there could 106 * be block jobs that can conflict. 107 */ 108 if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { 109 error_prepend(errp, "cannot start virtio-blk dataplane: "); 110 return false; 111 } 112 } 113 /* Don't try if transport does not support notifiers. */ 114 if (!virtio_device_ioeventfd_enabled(vdev)) { 115 return false; 116 } 117 118 s = g_new0(VirtIOBlockDataPlane, 1); 119 s->vdev = vdev; 120 s->conf = conf; 121 122 if (conf->iothread) { 123 s->iothread = conf->iothread; 124 object_ref(OBJECT(s->iothread)); 125 s->ctx = iothread_get_aio_context(s->iothread); 126 } else { 127 s->ctx = qemu_get_aio_context(); 128 } 129 s->bh = aio_bh_new(s->ctx, notify_guest_bh, s); 130 s->batch_notify_vqs = bitmap_new(conf->num_queues); 131 132 *dataplane = s; 133 134 return true; 135 } 136 137 /* Context: QEMU global mutex held */ 138 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) 139 { 140 VirtIOBlock *vblk; 141 142 if (!s) { 143 return; 144 } 145 146 vblk = VIRTIO_BLK(s->vdev); 147 assert(!vblk->dataplane_started); 148 g_free(s->batch_notify_vqs); 149 qemu_bh_delete(s->bh); 150 if (s->iothread) { 151 object_unref(OBJECT(s->iothread)); 152 } 153 g_free(s); 154 } 155 156 static bool virtio_blk_data_plane_handle_output(VirtIODevice *vdev, 157 VirtQueue *vq) 158 { 159 VirtIOBlock *s = (VirtIOBlock *)vdev; 160 161 assert(s->dataplane); 162 assert(s->dataplane_started); 163 164 return virtio_blk_handle_vq(s, vq); 165 } 166 167 /* Context: QEMU global mutex held */ 168 int virtio_blk_data_plane_start(VirtIODevice *vdev) 169 { 170 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 171 VirtIOBlockDataPlane *s = vblk->dataplane; 172 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); 173 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 174 unsigned i; 175 unsigned nvqs = s->conf->num_queues; 176 int r; 177 178 if (vblk->dataplane_started || s->starting) { 179 return 0; 180 } 181 182 s->starting = true; 183 184 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 185 s->batch_notifications = true; 186 } else { 187 s->batch_notifications = false; 188 } 189 190 /* Set up guest notifier (irq) */ 191 r = k->set_guest_notifiers(qbus->parent, nvqs, true); 192 if (r != 0) { 193 error_report("virtio-blk failed to set guest notifier (%d), " 194 "ensure -accel kvm is set.", r); 195 goto fail_guest_notifiers; 196 } 197 198 /* Set up virtqueue notify */ 199 for (i = 0; i < nvqs; i++) { 200 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); 201 if (r != 0) { 202 fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); 203 while (i--) { 204 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 205 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 206 } 207 goto fail_guest_notifiers; 208 } 209 } 210 211 s->starting = false; 212 vblk->dataplane_started = true; 213 trace_virtio_blk_data_plane_start(s); 214 215 blk_set_aio_context(s->conf->conf.blk, s->ctx); 216 217 /* Kick right away to begin processing requests already in vring */ 218 for (i = 0; i < nvqs; i++) { 219 VirtQueue *vq = virtio_get_queue(s->vdev, i); 220 221 event_notifier_set(virtio_queue_get_host_notifier(vq)); 222 } 223 224 /* Get this show started by hooking up our callbacks */ 225 aio_context_acquire(s->ctx); 226 for (i = 0; i < nvqs; i++) { 227 VirtQueue *vq = virtio_get_queue(s->vdev, i); 228 229 virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, 230 virtio_blk_data_plane_handle_output); 231 } 232 aio_context_release(s->ctx); 233 return 0; 234 235 fail_guest_notifiers: 236 vblk->dataplane_disabled = true; 237 s->starting = false; 238 vblk->dataplane_started = true; 239 return -ENOSYS; 240 } 241 242 /* Stop notifications for new requests from guest. 243 * 244 * Context: BH in IOThread 245 */ 246 static void virtio_blk_data_plane_stop_bh(void *opaque) 247 { 248 VirtIOBlockDataPlane *s = opaque; 249 unsigned i; 250 251 for (i = 0; i < s->conf->num_queues; i++) { 252 VirtQueue *vq = virtio_get_queue(s->vdev, i); 253 254 virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL); 255 } 256 } 257 258 /* Context: QEMU global mutex held */ 259 void virtio_blk_data_plane_stop(VirtIODevice *vdev) 260 { 261 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 262 VirtIOBlockDataPlane *s = vblk->dataplane; 263 BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); 264 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 265 unsigned i; 266 unsigned nvqs = s->conf->num_queues; 267 268 if (!vblk->dataplane_started || s->stopping) { 269 return; 270 } 271 272 /* Better luck next time. */ 273 if (vblk->dataplane_disabled) { 274 vblk->dataplane_disabled = false; 275 vblk->dataplane_started = false; 276 return; 277 } 278 s->stopping = true; 279 trace_virtio_blk_data_plane_stop(s); 280 281 aio_context_acquire(s->ctx); 282 aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); 283 284 /* Drain and switch bs back to the QEMU main loop */ 285 blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context()); 286 287 aio_context_release(s->ctx); 288 289 for (i = 0; i < nvqs; i++) { 290 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 291 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 292 } 293 294 /* Clean up guest notifier (irq) */ 295 k->set_guest_notifiers(qbus->parent, nvqs, false); 296 297 vblk->dataplane_started = false; 298 s->stopping = false; 299 } 300