1 /* 2 * Dedicated thread for virtio-blk I/O processing 3 * 4 * Copyright 2012 IBM, Corp. 5 * Copyright 2012 Red Hat, Inc. and/or its affiliates 6 * 7 * Authors: 8 * Stefan Hajnoczi <stefanha@redhat.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2 or later. 11 * See the COPYING file in the top-level directory. 12 * 13 */ 14 15 #include "qemu/osdep.h" 16 #include "qapi/error.h" 17 #include "trace.h" 18 #include "qemu/iov.h" 19 #include "qemu/main-loop.h" 20 #include "qemu/thread.h" 21 #include "qemu/error-report.h" 22 #include "hw/virtio/virtio-blk.h" 23 #include "virtio-blk.h" 24 #include "block/aio.h" 25 #include "hw/virtio/virtio-bus.h" 26 #include "qom/object_interfaces.h" 27 28 struct VirtIOBlockDataPlane { 29 bool starting; 30 bool stopping; 31 32 VirtIOBlkConf *conf; 33 VirtIODevice *vdev; 34 35 /* Note that these EventNotifiers are assigned by value. This is 36 * fine as long as you do not call event_notifier_cleanup on them 37 * (because you don't own the file descriptor or handle; you just 38 * use it). 39 */ 40 IOThread *iothread; 41 AioContext *ctx; 42 }; 43 44 /* Raise an interrupt to signal guest, if necessary */ 45 void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) 46 { 47 virtio_notify_irqfd(s->vdev, vq); 48 } 49 50 /* Context: QEMU global mutex held */ 51 bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, 52 VirtIOBlockDataPlane **dataplane, 53 Error **errp) 54 { 55 VirtIOBlockDataPlane *s; 56 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); 57 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 58 59 *dataplane = NULL; 60 61 if (conf->iothread) { 62 if (!k->set_guest_notifiers || !k->ioeventfd_assign) { 63 error_setg(errp, 64 "device is incompatible with iothread " 65 "(transport does not support notifiers)"); 66 return false; 67 } 68 if (!virtio_device_ioeventfd_enabled(vdev)) { 69 error_setg(errp, "ioeventfd is required for iothread"); 70 return false; 71 } 72 73 /* If dataplane is (re-)enabled while the guest is running there could 74 * be block jobs that can conflict. 75 */ 76 if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { 77 error_prepend(errp, "cannot start virtio-blk dataplane: "); 78 return false; 79 } 80 } 81 /* Don't try if transport does not support notifiers. */ 82 if (!virtio_device_ioeventfd_enabled(vdev)) { 83 return false; 84 } 85 86 s = g_new0(VirtIOBlockDataPlane, 1); 87 s->vdev = vdev; 88 s->conf = conf; 89 90 if (conf->iothread) { 91 s->iothread = conf->iothread; 92 object_ref(OBJECT(s->iothread)); 93 s->ctx = iothread_get_aio_context(s->iothread); 94 } else { 95 s->ctx = qemu_get_aio_context(); 96 } 97 98 *dataplane = s; 99 100 return true; 101 } 102 103 /* Context: QEMU global mutex held */ 104 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) 105 { 106 VirtIOBlock *vblk; 107 108 if (!s) { 109 return; 110 } 111 112 vblk = VIRTIO_BLK(s->vdev); 113 assert(!vblk->dataplane_started); 114 if (s->iothread) { 115 object_unref(OBJECT(s->iothread)); 116 } 117 g_free(s); 118 } 119 120 /* Context: QEMU global mutex held */ 121 int virtio_blk_data_plane_start(VirtIODevice *vdev) 122 { 123 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 124 VirtIOBlockDataPlane *s = vblk->dataplane; 125 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); 126 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 127 AioContext *old_context; 128 unsigned i; 129 unsigned nvqs = s->conf->num_queues; 130 Error *local_err = NULL; 131 int r; 132 133 if (vblk->dataplane_started || s->starting) { 134 return 0; 135 } 136 137 s->starting = true; 138 139 /* Set up guest notifier (irq) */ 140 r = k->set_guest_notifiers(qbus->parent, nvqs, true); 141 if (r != 0) { 142 error_report("virtio-blk failed to set guest notifier (%d), " 143 "ensure -accel kvm is set.", r); 144 goto fail_guest_notifiers; 145 } 146 147 /* 148 * Batch all the host notifiers in a single transaction to avoid 149 * quadratic time complexity in address_space_update_ioeventfds(). 150 */ 151 memory_region_transaction_begin(); 152 153 /* Set up virtqueue notify */ 154 for (i = 0; i < nvqs; i++) { 155 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); 156 if (r != 0) { 157 int j = i; 158 159 fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); 160 while (i--) { 161 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 162 } 163 164 /* 165 * The transaction expects the ioeventfds to be open when it 166 * commits. Do it now, before the cleanup loop. 167 */ 168 memory_region_transaction_commit(); 169 170 while (j--) { 171 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j); 172 } 173 goto fail_host_notifiers; 174 } 175 } 176 177 memory_region_transaction_commit(); 178 179 trace_virtio_blk_data_plane_start(s); 180 181 old_context = blk_get_aio_context(s->conf->conf.blk); 182 aio_context_acquire(old_context); 183 r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); 184 aio_context_release(old_context); 185 if (r < 0) { 186 error_report_err(local_err); 187 goto fail_aio_context; 188 } 189 190 /* Kick right away to begin processing requests already in vring */ 191 for (i = 0; i < nvqs; i++) { 192 VirtQueue *vq = virtio_get_queue(s->vdev, i); 193 194 event_notifier_set(virtio_queue_get_host_notifier(vq)); 195 } 196 197 /* 198 * These fields must be visible to the IOThread when it processes the 199 * virtqueue, otherwise it will think dataplane has not started yet. 200 * 201 * Make sure ->dataplane_started is false when blk_set_aio_context() is 202 * called above so that draining does not cause the host notifier to be 203 * detached/attached prematurely. 204 */ 205 s->starting = false; 206 vblk->dataplane_started = true; 207 smp_wmb(); /* paired with aio_notify_accept() on the read side */ 208 209 /* Get this show started by hooking up our callbacks */ 210 if (!blk_in_drain(s->conf->conf.blk)) { 211 aio_context_acquire(s->ctx); 212 for (i = 0; i < nvqs; i++) { 213 VirtQueue *vq = virtio_get_queue(s->vdev, i); 214 215 virtio_queue_aio_attach_host_notifier(vq, s->ctx); 216 } 217 aio_context_release(s->ctx); 218 } 219 return 0; 220 221 fail_aio_context: 222 memory_region_transaction_begin(); 223 224 for (i = 0; i < nvqs; i++) { 225 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 226 } 227 228 memory_region_transaction_commit(); 229 230 for (i = 0; i < nvqs; i++) { 231 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 232 } 233 fail_host_notifiers: 234 k->set_guest_notifiers(qbus->parent, nvqs, false); 235 fail_guest_notifiers: 236 vblk->dataplane_disabled = true; 237 s->starting = false; 238 return -ENOSYS; 239 } 240 241 /* Stop notifications for new requests from guest. 242 * 243 * Context: BH in IOThread 244 */ 245 static void virtio_blk_data_plane_stop_bh(void *opaque) 246 { 247 VirtIOBlockDataPlane *s = opaque; 248 unsigned i; 249 250 for (i = 0; i < s->conf->num_queues; i++) { 251 VirtQueue *vq = virtio_get_queue(s->vdev, i); 252 EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); 253 254 virtio_queue_aio_detach_host_notifier(vq, s->ctx); 255 256 /* 257 * Test and clear notifier after disabling event, in case poll callback 258 * didn't have time to run. 259 */ 260 virtio_queue_host_notifier_read(host_notifier); 261 } 262 } 263 264 /* Context: QEMU global mutex held */ 265 void virtio_blk_data_plane_stop(VirtIODevice *vdev) 266 { 267 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 268 VirtIOBlockDataPlane *s = vblk->dataplane; 269 BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); 270 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 271 unsigned i; 272 unsigned nvqs = s->conf->num_queues; 273 274 if (!vblk->dataplane_started || s->stopping) { 275 return; 276 } 277 278 /* Better luck next time. */ 279 if (vblk->dataplane_disabled) { 280 vblk->dataplane_disabled = false; 281 vblk->dataplane_started = false; 282 return; 283 } 284 s->stopping = true; 285 trace_virtio_blk_data_plane_stop(s); 286 287 if (!blk_in_drain(s->conf->conf.blk)) { 288 aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); 289 } 290 291 /* 292 * Batch all the host notifiers in a single transaction to avoid 293 * quadratic time complexity in address_space_update_ioeventfds(). 294 */ 295 memory_region_transaction_begin(); 296 297 for (i = 0; i < nvqs; i++) { 298 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 299 } 300 301 /* 302 * The transaction expects the ioeventfds to be open when it 303 * commits. Do it now, before the cleanup loop. 304 */ 305 memory_region_transaction_commit(); 306 307 for (i = 0; i < nvqs; i++) { 308 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 309 } 310 311 /* 312 * Set ->dataplane_started to false before draining so that host notifiers 313 * are not detached/attached anymore. 314 */ 315 vblk->dataplane_started = false; 316 317 aio_context_acquire(s->ctx); 318 319 /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */ 320 blk_drain(s->conf->conf.blk); 321 322 /* 323 * Try to switch bs back to the QEMU main loop. If other users keep the 324 * BlockBackend in the iothread, that's ok 325 */ 326 blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); 327 328 aio_context_release(s->ctx); 329 330 /* Clean up guest notifier (irq) */ 331 k->set_guest_notifiers(qbus->parent, nvqs, false); 332 333 s->stopping = false; 334 } 335