1 /* 2 * Dedicated thread for virtio-blk I/O processing 3 * 4 * Copyright 2012 IBM, Corp. 5 * Copyright 2012 Red Hat, Inc. and/or its affiliates 6 * 7 * Authors: 8 * Stefan Hajnoczi <stefanha@redhat.com> 9 * 10 * This work is licensed under the terms of the GNU GPL, version 2 or later. 11 * See the COPYING file in the top-level directory. 12 * 13 */ 14 15 #include "qemu/osdep.h" 16 #include "qapi/error.h" 17 #include "trace.h" 18 #include "qemu/iov.h" 19 #include "qemu/main-loop.h" 20 #include "qemu/thread.h" 21 #include "qemu/error-report.h" 22 #include "hw/virtio/virtio-access.h" 23 #include "hw/virtio/virtio-blk.h" 24 #include "virtio-blk.h" 25 #include "block/aio.h" 26 #include "hw/virtio/virtio-bus.h" 27 #include "qom/object_interfaces.h" 28 29 struct VirtIOBlockDataPlane { 30 bool starting; 31 bool stopping; 32 33 VirtIOBlkConf *conf; 34 VirtIODevice *vdev; 35 QEMUBH *bh; /* bh for guest notification */ 36 unsigned long *batch_notify_vqs; 37 bool batch_notifications; 38 39 /* Note that these EventNotifiers are assigned by value. This is 40 * fine as long as you do not call event_notifier_cleanup on them 41 * (because you don't own the file descriptor or handle; you just 42 * use it). 43 */ 44 IOThread *iothread; 45 AioContext *ctx; 46 }; 47 48 /* Raise an interrupt to signal guest, if necessary */ 49 void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq) 50 { 51 if (s->batch_notifications) { 52 set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs); 53 qemu_bh_schedule(s->bh); 54 } else { 55 virtio_notify_irqfd(s->vdev, vq); 56 } 57 } 58 59 static void notify_guest_bh(void *opaque) 60 { 61 VirtIOBlockDataPlane *s = opaque; 62 unsigned nvqs = s->conf->num_queues; 63 unsigned long bitmap[BITS_TO_LONGS(nvqs)]; 64 unsigned j; 65 66 memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap)); 67 memset(s->batch_notify_vqs, 0, sizeof(bitmap)); 68 69 for (j = 0; j < nvqs; j += BITS_PER_LONG) { 70 unsigned long bits = bitmap[j / BITS_PER_LONG]; 71 72 while (bits != 0) { 73 unsigned i = j + ctzl(bits); 74 VirtQueue *vq = virtio_get_queue(s->vdev, i); 75 76 virtio_notify_irqfd(s->vdev, vq); 77 78 bits &= bits - 1; /* clear right-most bit */ 79 } 80 } 81 } 82 83 /* Context: QEMU global mutex held */ 84 bool virtio_blk_data_plane_create(VirtIODevice *vdev, VirtIOBlkConf *conf, 85 VirtIOBlockDataPlane **dataplane, 86 Error **errp) 87 { 88 VirtIOBlockDataPlane *s; 89 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); 90 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 91 92 *dataplane = NULL; 93 94 if (conf->iothread) { 95 if (!k->set_guest_notifiers || !k->ioeventfd_assign) { 96 error_setg(errp, 97 "device is incompatible with iothread " 98 "(transport does not support notifiers)"); 99 return false; 100 } 101 if (!virtio_device_ioeventfd_enabled(vdev)) { 102 error_setg(errp, "ioeventfd is required for iothread"); 103 return false; 104 } 105 106 /* If dataplane is (re-)enabled while the guest is running there could 107 * be block jobs that can conflict. 108 */ 109 if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) { 110 error_prepend(errp, "cannot start virtio-blk dataplane: "); 111 return false; 112 } 113 } 114 /* Don't try if transport does not support notifiers. */ 115 if (!virtio_device_ioeventfd_enabled(vdev)) { 116 return false; 117 } 118 119 s = g_new0(VirtIOBlockDataPlane, 1); 120 s->vdev = vdev; 121 s->conf = conf; 122 123 if (conf->iothread) { 124 s->iothread = conf->iothread; 125 object_ref(OBJECT(s->iothread)); 126 s->ctx = iothread_get_aio_context(s->iothread); 127 } else { 128 s->ctx = qemu_get_aio_context(); 129 } 130 s->bh = aio_bh_new(s->ctx, notify_guest_bh, s); 131 s->batch_notify_vqs = bitmap_new(conf->num_queues); 132 133 *dataplane = s; 134 135 return true; 136 } 137 138 /* Context: QEMU global mutex held */ 139 void virtio_blk_data_plane_destroy(VirtIOBlockDataPlane *s) 140 { 141 VirtIOBlock *vblk; 142 143 if (!s) { 144 return; 145 } 146 147 vblk = VIRTIO_BLK(s->vdev); 148 assert(!vblk->dataplane_started); 149 g_free(s->batch_notify_vqs); 150 qemu_bh_delete(s->bh); 151 if (s->iothread) { 152 object_unref(OBJECT(s->iothread)); 153 } 154 g_free(s); 155 } 156 157 /* Context: QEMU global mutex held */ 158 int virtio_blk_data_plane_start(VirtIODevice *vdev) 159 { 160 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 161 VirtIOBlockDataPlane *s = vblk->dataplane; 162 BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vblk))); 163 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 164 AioContext *old_context; 165 unsigned i; 166 unsigned nvqs = s->conf->num_queues; 167 Error *local_err = NULL; 168 int r; 169 170 if (vblk->dataplane_started || s->starting) { 171 return 0; 172 } 173 174 s->starting = true; 175 176 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) { 177 s->batch_notifications = true; 178 } else { 179 s->batch_notifications = false; 180 } 181 182 /* Set up guest notifier (irq) */ 183 r = k->set_guest_notifiers(qbus->parent, nvqs, true); 184 if (r != 0) { 185 error_report("virtio-blk failed to set guest notifier (%d), " 186 "ensure -accel kvm is set.", r); 187 goto fail_guest_notifiers; 188 } 189 190 /* 191 * Batch all the host notifiers in a single transaction to avoid 192 * quadratic time complexity in address_space_update_ioeventfds(). 193 */ 194 memory_region_transaction_begin(); 195 196 /* Set up virtqueue notify */ 197 for (i = 0; i < nvqs; i++) { 198 r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true); 199 if (r != 0) { 200 int j = i; 201 202 fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r); 203 while (i--) { 204 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 205 } 206 207 /* 208 * The transaction expects the ioeventfds to be open when it 209 * commits. Do it now, before the cleanup loop. 210 */ 211 memory_region_transaction_commit(); 212 213 while (j--) { 214 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), j); 215 } 216 goto fail_host_notifiers; 217 } 218 } 219 220 memory_region_transaction_commit(); 221 222 s->starting = false; 223 vblk->dataplane_started = true; 224 trace_virtio_blk_data_plane_start(s); 225 226 old_context = blk_get_aio_context(s->conf->conf.blk); 227 aio_context_acquire(old_context); 228 r = blk_set_aio_context(s->conf->conf.blk, s->ctx, &local_err); 229 aio_context_release(old_context); 230 if (r < 0) { 231 error_report_err(local_err); 232 goto fail_aio_context; 233 } 234 235 /* Process queued requests before the ones in vring */ 236 virtio_blk_process_queued_requests(vblk, false); 237 238 /* Kick right away to begin processing requests already in vring */ 239 for (i = 0; i < nvqs; i++) { 240 VirtQueue *vq = virtio_get_queue(s->vdev, i); 241 242 event_notifier_set(virtio_queue_get_host_notifier(vq)); 243 } 244 245 /* Get this show started by hooking up our callbacks */ 246 aio_context_acquire(s->ctx); 247 for (i = 0; i < nvqs; i++) { 248 VirtQueue *vq = virtio_get_queue(s->vdev, i); 249 250 virtio_queue_aio_attach_host_notifier(vq, s->ctx); 251 } 252 aio_context_release(s->ctx); 253 return 0; 254 255 fail_aio_context: 256 memory_region_transaction_begin(); 257 258 for (i = 0; i < nvqs; i++) { 259 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 260 } 261 262 memory_region_transaction_commit(); 263 264 for (i = 0; i < nvqs; i++) { 265 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 266 } 267 fail_host_notifiers: 268 k->set_guest_notifiers(qbus->parent, nvqs, false); 269 fail_guest_notifiers: 270 /* 271 * If we failed to set up the guest notifiers queued requests will be 272 * processed on the main context. 273 */ 274 virtio_blk_process_queued_requests(vblk, false); 275 vblk->dataplane_disabled = true; 276 s->starting = false; 277 vblk->dataplane_started = true; 278 return -ENOSYS; 279 } 280 281 /* Stop notifications for new requests from guest. 282 * 283 * Context: BH in IOThread 284 */ 285 static void virtio_blk_data_plane_stop_bh(void *opaque) 286 { 287 VirtIOBlockDataPlane *s = opaque; 288 unsigned i; 289 290 for (i = 0; i < s->conf->num_queues; i++) { 291 VirtQueue *vq = virtio_get_queue(s->vdev, i); 292 293 virtio_queue_aio_detach_host_notifier(vq, s->ctx); 294 } 295 } 296 297 /* Context: QEMU global mutex held */ 298 void virtio_blk_data_plane_stop(VirtIODevice *vdev) 299 { 300 VirtIOBlock *vblk = VIRTIO_BLK(vdev); 301 VirtIOBlockDataPlane *s = vblk->dataplane; 302 BusState *qbus = qdev_get_parent_bus(DEVICE(vblk)); 303 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus); 304 unsigned i; 305 unsigned nvqs = s->conf->num_queues; 306 307 if (!vblk->dataplane_started || s->stopping) { 308 return; 309 } 310 311 /* Better luck next time. */ 312 if (vblk->dataplane_disabled) { 313 vblk->dataplane_disabled = false; 314 vblk->dataplane_started = false; 315 return; 316 } 317 s->stopping = true; 318 trace_virtio_blk_data_plane_stop(s); 319 320 aio_context_acquire(s->ctx); 321 aio_wait_bh_oneshot(s->ctx, virtio_blk_data_plane_stop_bh, s); 322 323 /* Drain and try to switch bs back to the QEMU main loop. If other users 324 * keep the BlockBackend in the iothread, that's ok */ 325 blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context(), NULL); 326 327 aio_context_release(s->ctx); 328 329 /* 330 * Batch all the host notifiers in a single transaction to avoid 331 * quadratic time complexity in address_space_update_ioeventfds(). 332 */ 333 memory_region_transaction_begin(); 334 335 for (i = 0; i < nvqs; i++) { 336 virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false); 337 } 338 339 /* 340 * The transaction expects the ioeventfds to be open when it 341 * commits. Do it now, before the cleanup loop. 342 */ 343 memory_region_transaction_commit(); 344 345 for (i = 0; i < nvqs; i++) { 346 virtio_bus_cleanup_host_notifier(VIRTIO_BUS(qbus), i); 347 } 348 349 qemu_bh_cancel(s->bh); 350 notify_guest_bh(s); /* final chance to notify guest */ 351 352 /* Clean up guest notifier (irq) */ 353 k->set_guest_notifiers(qbus->parent, nvqs, false); 354 355 vblk->dataplane_started = false; 356 s->stopping = false; 357 } 358