1 /* 2 * Dedicated thread for virtio-blk I/O processing 3 * 4 * Copyright 2012 IBM, Corp. 5 * Copyright 2012 Red Hat, Inc. and/or its affiliates 6 * 7 * Authors: 8 * Stefan Hajnoczi <stefanha@redhat.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2 or later. 11 * See the COPYING file in the top-level directory. 12 * 13 */ 14 15 #include "qemu/osdep.h" 16 #include "qapi/error.h" 17 #include "trace.h" 18 #include "qemu/iov.h" 19 #include "qemu/main-loop.h" 20 #include "qemu/thread.h" 21 #include "qemu/error-report.h" 22 #include "hw/virtio/virtio-blk.h" 23 #include "virtio-blk.h" 24 #include "block/aio.h" 25 #include "hw/virtio/virtio-bus.h" 26 #include "qom/object_interfaces.h" 27 28 struct VirtIOBlockDataPlane { 29 bool starting; 30 bool stopping; 31 32 VirtIOBlkConf *conf; 33 VirtIODevice *vdev; 34 QEMUBH *bh; /* bh for guest notification */ 35 unsigned long *batch_notify_vqs; 36 bool batch_notifications; 37 38 /* Note that these EventNotifiers are assigned by value. This is 39 * fine as long as you do not call event_notifier_cleanup on them 40 * (because you don't own the file descriptor or handle; you just 41 * use it). 42 */ 43 IOThread *iothread; 44 AioContext *ctx; 45 }; 46 47 /* Raise an interrupt to signal guest, if necessary */ 48 void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) 49 { 50 if (s->batch_notifications) { 51 set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs); 52 qemu_bh_schedule(s->bh); 53 } else { 54 virtio_notify_irqfd(s->vdev, vq); 55 } 56 } 57 58 static void notify_guest_bh(void *opaque) 59 { 60 VirtIOBlockDataPlane *s = opaque; 61 unsigned nvqs = s->conf->num_queues; 62 unsigned long bitmap[BITS_TO_LONGS(nvqs)]; 63 unsigned j; 64 65 memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap)); 66 memset(s->batch_notify_vqs, 0, sizeof(bitmap)); 67 68 for (j = 0; j < nvqs; j += BITS_PER_LONG) { 69 unsigned long bits = bitmap[j / BITS_PER_LONG]; 70 71 while (bits != 0) { 72 unsigned i = j + ctzl(bits); 73 VirtQueue *vq = virtio_get_queue(s->vdev, i); 74 75 virtio_notify_irqfd(s->vdev, vq); 76 77 bits &= bits - 1; /* clear right-most bit */ 78 } 79 } 80 } 81 82 /* Context: QEMU global mutex held */ 83 bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, 84 VirtIOBlockDataPlane **dataplane, 85 Error **errp) 86 { 87 VirtIOBlockDataPlane *s; 88 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); 89 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 90 91 *dataplane = NULL; 92 93 if (conf->iothread) { 94 if (!k->set_guest_notifiers || !k->ioeventfd_assign) { 95 error_setg(errp, 96 "device is incompatible with iothread " 97 "(transport does not support notifiers)"); 98 return false; 99 } 100 if (!virtio_device_ioeventfd_enabled(vdev)) { 101 error_setg(errp, "ioeventfd is required for iothread"); 102 return false; 103 } 104 105 /* If dataplane is (re-)enabled while the guest is running there could 106 * be block jobs that can conflict. 107 */ 108 if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { 109 error_prepend(errp, "cannot start virtio-blk dataplane: "); 110 return false; 111 } 112 } 113 /* Don't try if transport does not support notifiers. */ 114 if (!virtio_device_ioeventfd_enabled(vdev)) { 115 return false; 116 } 117 118 s = g_new0(VirtIOBlockDataPlane, 1); 119 s->vdev = vdev; 120 s->conf = conf; 121 122 if (conf->iothread) { 123 s->iothread = conf->iothread; 124 object_ref(OBJECT(s->iothread)); 125 s->ctx = iothread_get_aio_context(s->iothread); 126 } else { 127 s->ctx = qemu_get_aio_context(); 128 } 129 s->bh = aio_bh_new_guarded(s->ctx, notify_guest_bh, s, 130 &DEVICE(vdev)->mem_reentrancy_guard); 131 s->batch_notify_vqs = bitmap_new(conf->num_queues); 132 133 *dataplane = s; 134 135 return true; 136 } 137 138 /* Context: QEMU global mutex held */ 139 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) 140 { 141 VirtIOBlock *vblk; 142 143 if (!s) { 144 return; 145 } 146 147 vblk = VIRTIO_BLK(s->vdev); 148 assert(!vblk->dataplane_started); 149 g_free(s->batch_notify_vqs); 150 qemu_bh_delete(s->bh); 151 if (s->iothread) { 152 object_unref(OBJECT(s->iothread)); 153 } 154 g_free(s); 155 } 156 157 /* Context: QEMU global mutex held */ 158 int virtio_blk_data_plane_start(VirtIODevice *vdev) 159 { 160 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 161 VirtIOBlockDataPlane *s = vblk->dataplane; 162 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); 163 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 164 AioContext *old_context; 165 unsigned i; 166 unsigned nvqs = s->conf->num_queues; 167 Error *local_err = NULL; 168 int r; 169 170 if (vblk->dataplane_started || s->starting) { 171 return 0; 172 } 173 174 s->starting = true; 175 176 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 177 s->batch_notifications = true; 178 } else { 179 s->batch_notifications = false; 180 } 181 182 /* Set up guest notifier (irq) */ 183 r = k->set_guest_notifiers(qbus->parent, nvqs, true); 184 if (r != 0) { 185 error_report("virtio-blk failed to set guest notifier (%d), " 186 "ensure -accel kvm is set.", r); 187 goto fail_guest_notifiers; 188 } 189 190 /* 191 * Batch all the host notifiers in a single transaction to avoid 192 * quadratic time complexity in address_space_update_ioeventfds(). 193 */ 194 memory_region_transaction_begin(); 195 196 /* Set up virtqueue notify */ 197 for (i = 0; i < nvqs; i++) { 198 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); 199 if (r != 0) { 200 int j = i; 201 202 fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); 203 while (i--) { 204 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 205 } 206 207 /* 208 * The transaction expects the ioeventfds to be open when it 209 * commits. Do it now, before the cleanup loop. 210 */ 211 memory_region_transaction_commit(); 212 213 while (j--) { 214 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j); 215 } 216 goto fail_host_notifiers; 217 } 218 } 219 220 memory_region_transaction_commit(); 221 222 trace_virtio_blk_data_plane_start(s); 223 224 old_context = blk_get_aio_context(s->conf->conf.blk); 225 aio_context_acquire(old_context); 226 r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); 227 aio_context_release(old_context); 228 if (r < 0) { 229 error_report_err(local_err); 230 goto fail_aio_context; 231 } 232 233 /* Kick right away to begin processing requests already in vring */ 234 for (i = 0; i < nvqs; i++) { 235 VirtQueue *vq = virtio_get_queue(s->vdev, i); 236 237 event_notifier_set(virtio_queue_get_host_notifier(vq)); 238 } 239 240 /* 241 * These fields must be visible to the IOThread when it processes the 242 * virtqueue, otherwise it will think dataplane has not started yet. 243 * 244 * Make sure ->dataplane_started is false when blk_set_aio_context() is 245 * called above so that draining does not cause the host notifier to be 246 * detached/attached prematurely. 247 */ 248 s->starting = false; 249 vblk->dataplane_started = true; 250 smp_wmb(); /* paired with aio_notify_accept() on the read side */ 251 252 /* Get this show started by hooking up our callbacks */ 253 if (!blk_in_drain(s->conf->conf.blk)) { 254 aio_context_acquire(s->ctx); 255 for (i = 0; i < nvqs; i++) { 256 VirtQueue *vq = virtio_get_queue(s->vdev, i); 257 258 virtio_queue_aio_attach_host_notifier(vq, s->ctx); 259 } 260 aio_context_release(s->ctx); 261 } 262 return 0; 263 264 fail_aio_context: 265 memory_region_transaction_begin(); 266 267 for (i = 0; i < nvqs; i++) { 268 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 269 } 270 271 memory_region_transaction_commit(); 272 273 for (i = 0; i < nvqs; i++) { 274 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 275 } 276 fail_host_notifiers: 277 k->set_guest_notifiers(qbus->parent, nvqs, false); 278 fail_guest_notifiers: 279 vblk->dataplane_disabled = true; 280 s->starting = false; 281 return -ENOSYS; 282 } 283 284 /* Stop notifications for new requests from guest. 285 * 286 * Context: BH in IOThread 287 */ 288 static void virtio_blk_data_plane_stop_bh(void *opaque) 289 { 290 VirtIOBlockDataPlane *s = opaque; 291 unsigned i; 292 293 for (i = 0; i < s->conf->num_queues; i++) { 294 VirtQueue *vq = virtio_get_queue(s->vdev, i); 295 EventNotifier *host_notifier = virtio_queue_get_host_notifier(vq); 296 297 virtio_queue_aio_detach_host_notifier(vq, s->ctx); 298 299 /* 300 * Test and clear notifier after disabling event, in case poll callback 301 * didn't have time to run. 302 */ 303 virtio_queue_host_notifier_read(host_notifier); 304 } 305 } 306 307 /* Context: QEMU global mutex held */ 308 void virtio_blk_data_plane_stop(VirtIODevice *vdev) 309 { 310 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 311 VirtIOBlockDataPlane *s = vblk->dataplane; 312 BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); 313 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 314 unsigned i; 315 unsigned nvqs = s->conf->num_queues; 316 317 if (!vblk->dataplane_started || s->stopping) { 318 return; 319 } 320 321 /* Better luck next time. */ 322 if (vblk->dataplane_disabled) { 323 vblk->dataplane_disabled = false; 324 vblk->dataplane_started = false; 325 return; 326 } 327 s->stopping = true; 328 trace_virtio_blk_data_plane_stop(s); 329 330 if (!blk_in_drain(s->conf->conf.blk)) { 331 aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); 332 } 333 334 /* 335 * Batch all the host notifiers in a single transaction to avoid 336 * quadratic time complexity in address_space_update_ioeventfds(). 337 */ 338 memory_region_transaction_begin(); 339 340 for (i = 0; i < nvqs; i++) { 341 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 342 } 343 344 /* 345 * The transaction expects the ioeventfds to be open when it 346 * commits. Do it now, before the cleanup loop. 347 */ 348 memory_region_transaction_commit(); 349 350 for (i = 0; i < nvqs; i++) { 351 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 352 } 353 354 /* 355 * Set ->dataplane_started to false before draining so that host notifiers 356 * are not detached/attached anymore. 357 */ 358 vblk->dataplane_started = false; 359 360 aio_context_acquire(s->ctx); 361 362 /* Wait for virtio_blk_dma_restart_bh() and in flight I/O to complete */ 363 blk_drain(s->conf->conf.blk); 364 365 /* 366 * Try to switch bs back to the QEMU main loop. If other users keep the 367 * BlockBackend in the iothread, that's ok 368 */ 369 blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); 370 371 aio_context_release(s->ctx); 372 373 qemu_bh_cancel(s->bh); 374 notify_guest_bh(s); /* final chance to notify guest */ 375 376 /* Clean up guest notifier (irq) */ 377 k->set_guest_notifiers(qbus->parent, nvqs, false); 378 379 s->stopping = false; 380 } 381