xref: /openbmc/qemu/block/io.c (revision 9c5ce8db2e5c2769ed2fd3d91928dd1853b5ce7c)
1 /*
2  * Block layer I/O functions
3  *
4  * Copyright (c) 2003 Fabrice Bellard
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/blockjob.h"
29 #include "block/block_int.h"
30 #include "qemu/cutils.h"
31 #include "qapi/error.h"
32 #include "qemu/error-report.h"
33 
34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
35 
36 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
37                                           int64_t offset,
38                                           QEMUIOVector *qiov,
39                                           BdrvRequestFlags flags,
40                                           BlockCompletionFunc *cb,
41                                           void *opaque,
42                                           bool is_write);
43 static void coroutine_fn bdrv_co_do_rw(void *opaque);
44 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
45     int64_t offset, int count, BdrvRequestFlags flags);
46 
47 static void bdrv_parent_drained_begin(BlockDriverState *bs)
48 {
49     BdrvChild *c;
50 
51     QLIST_FOREACH(c, &bs->parents, next_parent) {
52         if (c->role->drained_begin) {
53             c->role->drained_begin(c);
54         }
55     }
56 }
57 
58 static void bdrv_parent_drained_end(BlockDriverState *bs)
59 {
60     BdrvChild *c;
61 
62     QLIST_FOREACH(c, &bs->parents, next_parent) {
63         if (c->role->drained_end) {
64             c->role->drained_end(c);
65         }
66     }
67 }
68 
69 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
70 {
71     dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
72     dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
73     dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
74                                  src->opt_mem_alignment);
75     dst->min_mem_alignment = MAX(dst->min_mem_alignment,
76                                  src->min_mem_alignment);
77     dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
78 }
79 
80 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
81 {
82     BlockDriver *drv = bs->drv;
83     Error *local_err = NULL;
84 
85     memset(&bs->bl, 0, sizeof(bs->bl));
86 
87     if (!drv) {
88         return;
89     }
90 
91     /* Default alignment based on whether driver has byte interface */
92     bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512;
93 
94     /* Take some limits from the children as a default */
95     if (bs->file) {
96         bdrv_refresh_limits(bs->file->bs, &local_err);
97         if (local_err) {
98             error_propagate(errp, local_err);
99             return;
100         }
101         bdrv_merge_limits(&bs->bl, &bs->file->bs->bl);
102     } else {
103         bs->bl.min_mem_alignment = 512;
104         bs->bl.opt_mem_alignment = getpagesize();
105 
106         /* Safe default since most protocols use readv()/writev()/etc */
107         bs->bl.max_iov = IOV_MAX;
108     }
109 
110     if (bs->backing) {
111         bdrv_refresh_limits(bs->backing->bs, &local_err);
112         if (local_err) {
113             error_propagate(errp, local_err);
114             return;
115         }
116         bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl);
117     }
118 
119     /* Then let the driver override it */
120     if (drv->bdrv_refresh_limits) {
121         drv->bdrv_refresh_limits(bs, errp);
122     }
123 }
124 
125 /**
126  * The copy-on-read flag is actually a reference count so multiple users may
127  * use the feature without worrying about clobbering its previous state.
128  * Copy-on-read stays enabled until all users have called to disable it.
129  */
130 void bdrv_enable_copy_on_read(BlockDriverState *bs)
131 {
132     bs->copy_on_read++;
133 }
134 
135 void bdrv_disable_copy_on_read(BlockDriverState *bs)
136 {
137     assert(bs->copy_on_read > 0);
138     bs->copy_on_read--;
139 }
140 
141 /* Check if any requests are in-flight (including throttled requests) */
142 bool bdrv_requests_pending(BlockDriverState *bs)
143 {
144     BdrvChild *child;
145 
146     if (!QLIST_EMPTY(&bs->tracked_requests)) {
147         return true;
148     }
149 
150     QLIST_FOREACH(child, &bs->children, next) {
151         if (bdrv_requests_pending(child->bs)) {
152             return true;
153         }
154     }
155 
156     return false;
157 }
158 
159 static void bdrv_drain_recurse(BlockDriverState *bs)
160 {
161     BdrvChild *child;
162 
163     if (bs->drv && bs->drv->bdrv_drain) {
164         bs->drv->bdrv_drain(bs);
165     }
166     QLIST_FOREACH(child, &bs->children, next) {
167         bdrv_drain_recurse(child->bs);
168     }
169 }
170 
171 typedef struct {
172     Coroutine *co;
173     BlockDriverState *bs;
174     QEMUBH *bh;
175     bool done;
176 } BdrvCoDrainData;
177 
178 static void bdrv_drain_poll(BlockDriverState *bs)
179 {
180     bool busy = true;
181 
182     while (busy) {
183         /* Keep iterating */
184         busy = bdrv_requests_pending(bs);
185         busy |= aio_poll(bdrv_get_aio_context(bs), busy);
186     }
187 }
188 
189 static void bdrv_co_drain_bh_cb(void *opaque)
190 {
191     BdrvCoDrainData *data = opaque;
192     Coroutine *co = data->co;
193 
194     qemu_bh_delete(data->bh);
195     bdrv_drain_poll(data->bs);
196     data->done = true;
197     qemu_coroutine_enter(co);
198 }
199 
200 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
201 {
202     BdrvCoDrainData data;
203 
204     /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
205      * other coroutines run if they were queued from
206      * qemu_co_queue_run_restart(). */
207 
208     assert(qemu_in_coroutine());
209     data = (BdrvCoDrainData) {
210         .co = qemu_coroutine_self(),
211         .bs = bs,
212         .done = false,
213         .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
214     };
215     qemu_bh_schedule(data.bh);
216 
217     qemu_coroutine_yield();
218     /* If we are resumed from some other event (such as an aio completion or a
219      * timer callback), it is a bug in the caller that should be fixed. */
220     assert(data.done);
221 }
222 
223 void bdrv_drained_begin(BlockDriverState *bs)
224 {
225     if (!bs->quiesce_counter++) {
226         aio_disable_external(bdrv_get_aio_context(bs));
227         bdrv_parent_drained_begin(bs);
228     }
229 
230     bdrv_io_unplugged_begin(bs);
231     bdrv_drain_recurse(bs);
232     if (qemu_in_coroutine()) {
233         bdrv_co_yield_to_drain(bs);
234     } else {
235         bdrv_drain_poll(bs);
236     }
237     bdrv_io_unplugged_end(bs);
238 }
239 
240 void bdrv_drained_end(BlockDriverState *bs)
241 {
242     assert(bs->quiesce_counter > 0);
243     if (--bs->quiesce_counter > 0) {
244         return;
245     }
246 
247     bdrv_parent_drained_end(bs);
248     aio_enable_external(bdrv_get_aio_context(bs));
249 }
250 
251 /*
252  * Wait for pending requests to complete on a single BlockDriverState subtree,
253  * and suspend block driver's internal I/O until next request arrives.
254  *
255  * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
256  * AioContext.
257  *
258  * Only this BlockDriverState's AioContext is run, so in-flight requests must
259  * not depend on events in other AioContexts.  In that case, use
260  * bdrv_drain_all() instead.
261  */
262 void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
263 {
264     assert(qemu_in_coroutine());
265     bdrv_drained_begin(bs);
266     bdrv_drained_end(bs);
267 }
268 
269 void bdrv_drain(BlockDriverState *bs)
270 {
271     bdrv_drained_begin(bs);
272     bdrv_drained_end(bs);
273 }
274 
275 /*
276  * Wait for pending requests to complete across all BlockDriverStates
277  *
278  * This function does not flush data to disk, use bdrv_flush_all() for that
279  * after calling this function.
280  */
281 void bdrv_drain_all(void)
282 {
283     /* Always run first iteration so any pending completion BHs run */
284     bool busy = true;
285     BlockDriverState *bs;
286     BdrvNextIterator it;
287     BlockJob *job = NULL;
288     GSList *aio_ctxs = NULL, *ctx;
289 
290     while ((job = block_job_next(job))) {
291         AioContext *aio_context = blk_get_aio_context(job->blk);
292 
293         aio_context_acquire(aio_context);
294         block_job_pause(job);
295         aio_context_release(aio_context);
296     }
297 
298     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
299         AioContext *aio_context = bdrv_get_aio_context(bs);
300 
301         aio_context_acquire(aio_context);
302         bdrv_parent_drained_begin(bs);
303         bdrv_io_unplugged_begin(bs);
304         bdrv_drain_recurse(bs);
305         aio_context_release(aio_context);
306 
307         if (!g_slist_find(aio_ctxs, aio_context)) {
308             aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
309         }
310     }
311 
312     /* Note that completion of an asynchronous I/O operation can trigger any
313      * number of other I/O operations on other devices---for example a
314      * coroutine can submit an I/O request to another device in response to
315      * request completion.  Therefore we must keep looping until there was no
316      * more activity rather than simply draining each device independently.
317      */
318     while (busy) {
319         busy = false;
320 
321         for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
322             AioContext *aio_context = ctx->data;
323 
324             aio_context_acquire(aio_context);
325             for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
326                 if (aio_context == bdrv_get_aio_context(bs)) {
327                     if (bdrv_requests_pending(bs)) {
328                         busy = true;
329                         aio_poll(aio_context, busy);
330                     }
331                 }
332             }
333             busy |= aio_poll(aio_context, false);
334             aio_context_release(aio_context);
335         }
336     }
337 
338     for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
339         AioContext *aio_context = bdrv_get_aio_context(bs);
340 
341         aio_context_acquire(aio_context);
342         bdrv_io_unplugged_end(bs);
343         bdrv_parent_drained_end(bs);
344         aio_context_release(aio_context);
345     }
346     g_slist_free(aio_ctxs);
347 
348     job = NULL;
349     while ((job = block_job_next(job))) {
350         AioContext *aio_context = blk_get_aio_context(job->blk);
351 
352         aio_context_acquire(aio_context);
353         block_job_resume(job);
354         aio_context_release(aio_context);
355     }
356 }
357 
358 /**
359  * Remove an active request from the tracked requests list
360  *
361  * This function should be called when a tracked request is completing.
362  */
363 static void tracked_request_end(BdrvTrackedRequest *req)
364 {
365     if (req->serialising) {
366         req->bs->serialising_in_flight--;
367     }
368 
369     QLIST_REMOVE(req, list);
370     qemu_co_queue_restart_all(&req->wait_queue);
371 }
372 
373 /**
374  * Add an active request to the tracked requests list
375  */
376 static void tracked_request_begin(BdrvTrackedRequest *req,
377                                   BlockDriverState *bs,
378                                   int64_t offset,
379                                   unsigned int bytes,
380                                   enum BdrvTrackedRequestType type)
381 {
382     *req = (BdrvTrackedRequest){
383         .bs = bs,
384         .offset         = offset,
385         .bytes          = bytes,
386         .type           = type,
387         .co             = qemu_coroutine_self(),
388         .serialising    = false,
389         .overlap_offset = offset,
390         .overlap_bytes  = bytes,
391     };
392 
393     qemu_co_queue_init(&req->wait_queue);
394 
395     QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
396 }
397 
398 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
399 {
400     int64_t overlap_offset = req->offset & ~(align - 1);
401     unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
402                                - overlap_offset;
403 
404     if (!req->serialising) {
405         req->bs->serialising_in_flight++;
406         req->serialising = true;
407     }
408 
409     req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
410     req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
411 }
412 
413 /**
414  * Round a region to cluster boundaries (sector-based)
415  */
416 void bdrv_round_sectors_to_clusters(BlockDriverState *bs,
417                                     int64_t sector_num, int nb_sectors,
418                                     int64_t *cluster_sector_num,
419                                     int *cluster_nb_sectors)
420 {
421     BlockDriverInfo bdi;
422 
423     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
424         *cluster_sector_num = sector_num;
425         *cluster_nb_sectors = nb_sectors;
426     } else {
427         int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
428         *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
429         *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
430                                             nb_sectors, c);
431     }
432 }
433 
434 /**
435  * Round a region to cluster boundaries
436  */
437 void bdrv_round_to_clusters(BlockDriverState *bs,
438                             int64_t offset, unsigned int bytes,
439                             int64_t *cluster_offset,
440                             unsigned int *cluster_bytes)
441 {
442     BlockDriverInfo bdi;
443 
444     if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
445         *cluster_offset = offset;
446         *cluster_bytes = bytes;
447     } else {
448         int64_t c = bdi.cluster_size;
449         *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
450         *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
451     }
452 }
453 
454 static int bdrv_get_cluster_size(BlockDriverState *bs)
455 {
456     BlockDriverInfo bdi;
457     int ret;
458 
459     ret = bdrv_get_info(bs, &bdi);
460     if (ret < 0 || bdi.cluster_size == 0) {
461         return bs->bl.request_alignment;
462     } else {
463         return bdi.cluster_size;
464     }
465 }
466 
467 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
468                                      int64_t offset, unsigned int bytes)
469 {
470     /*        aaaa   bbbb */
471     if (offset >= req->overlap_offset + req->overlap_bytes) {
472         return false;
473     }
474     /* bbbb   aaaa        */
475     if (req->overlap_offset >= offset + bytes) {
476         return false;
477     }
478     return true;
479 }
480 
481 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
482 {
483     BlockDriverState *bs = self->bs;
484     BdrvTrackedRequest *req;
485     bool retry;
486     bool waited = false;
487 
488     if (!bs->serialising_in_flight) {
489         return false;
490     }
491 
492     do {
493         retry = false;
494         QLIST_FOREACH(req, &bs->tracked_requests, list) {
495             if (req == self || (!req->serialising && !self->serialising)) {
496                 continue;
497             }
498             if (tracked_request_overlaps(req, self->overlap_offset,
499                                          self->overlap_bytes))
500             {
501                 /* Hitting this means there was a reentrant request, for
502                  * example, a block driver issuing nested requests.  This must
503                  * never happen since it means deadlock.
504                  */
505                 assert(qemu_coroutine_self() != req->co);
506 
507                 /* If the request is already (indirectly) waiting for us, or
508                  * will wait for us as soon as it wakes up, then just go on
509                  * (instead of producing a deadlock in the former case). */
510                 if (!req->waiting_for) {
511                     self->waiting_for = req;
512                     qemu_co_queue_wait(&req->wait_queue);
513                     self->waiting_for = NULL;
514                     retry = true;
515                     waited = true;
516                     break;
517                 }
518             }
519         }
520     } while (retry);
521 
522     return waited;
523 }
524 
525 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
526                                    size_t size)
527 {
528     if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
529         return -EIO;
530     }
531 
532     if (!bdrv_is_inserted(bs)) {
533         return -ENOMEDIUM;
534     }
535 
536     if (offset < 0) {
537         return -EIO;
538     }
539 
540     return 0;
541 }
542 
543 typedef struct RwCo {
544     BdrvChild *child;
545     int64_t offset;
546     QEMUIOVector *qiov;
547     bool is_write;
548     int ret;
549     BdrvRequestFlags flags;
550 } RwCo;
551 
552 static void coroutine_fn bdrv_rw_co_entry(void *opaque)
553 {
554     RwCo *rwco = opaque;
555 
556     if (!rwco->is_write) {
557         rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset,
558                                    rwco->qiov->size, rwco->qiov,
559                                    rwco->flags);
560     } else {
561         rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset,
562                                     rwco->qiov->size, rwco->qiov,
563                                     rwco->flags);
564     }
565 }
566 
567 /*
568  * Process a vectored synchronous request using coroutines
569  */
570 static int bdrv_prwv_co(BdrvChild *child, int64_t offset,
571                         QEMUIOVector *qiov, bool is_write,
572                         BdrvRequestFlags flags)
573 {
574     Coroutine *co;
575     RwCo rwco = {
576         .child = child,
577         .offset = offset,
578         .qiov = qiov,
579         .is_write = is_write,
580         .ret = NOT_DONE,
581         .flags = flags,
582     };
583 
584     if (qemu_in_coroutine()) {
585         /* Fast-path if already in coroutine context */
586         bdrv_rw_co_entry(&rwco);
587     } else {
588         AioContext *aio_context = bdrv_get_aio_context(child->bs);
589 
590         co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco);
591         qemu_coroutine_enter(co);
592         while (rwco.ret == NOT_DONE) {
593             aio_poll(aio_context, true);
594         }
595     }
596     return rwco.ret;
597 }
598 
599 /*
600  * Process a synchronous request using coroutines
601  */
602 static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf,
603                       int nb_sectors, bool is_write, BdrvRequestFlags flags)
604 {
605     QEMUIOVector qiov;
606     struct iovec iov = {
607         .iov_base = (void *)buf,
608         .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
609     };
610 
611     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
612         return -EINVAL;
613     }
614 
615     qemu_iovec_init_external(&qiov, &iov, 1);
616     return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS,
617                         &qiov, is_write, flags);
618 }
619 
620 /* return < 0 if error. See bdrv_write() for the return codes */
621 int bdrv_read(BdrvChild *child, int64_t sector_num,
622               uint8_t *buf, int nb_sectors)
623 {
624     return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0);
625 }
626 
627 /* Return < 0 if error. Important errors are:
628   -EIO         generic I/O error (may happen for all errors)
629   -ENOMEDIUM   No media inserted.
630   -EINVAL      Invalid sector number or nb_sectors
631   -EACCES      Trying to write a read-only device
632 */
633 int bdrv_write(BdrvChild *child, int64_t sector_num,
634                const uint8_t *buf, int nb_sectors)
635 {
636     return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
637 }
638 
639 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
640                        int count, BdrvRequestFlags flags)
641 {
642     QEMUIOVector qiov;
643     struct iovec iov = {
644         .iov_base = NULL,
645         .iov_len = count,
646     };
647 
648     qemu_iovec_init_external(&qiov, &iov, 1);
649     return bdrv_prwv_co(child, offset, &qiov, true,
650                         BDRV_REQ_ZERO_WRITE | flags);
651 }
652 
653 /*
654  * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
655  * The operation is sped up by checking the block status and only writing
656  * zeroes to the device if they currently do not return zeroes. Optional
657  * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
658  * BDRV_REQ_FUA).
659  *
660  * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
661  */
662 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
663 {
664     int64_t target_sectors, ret, nb_sectors, sector_num = 0;
665     BlockDriverState *bs = child->bs;
666     BlockDriverState *file;
667     int n;
668 
669     target_sectors = bdrv_nb_sectors(bs);
670     if (target_sectors < 0) {
671         return target_sectors;
672     }
673 
674     for (;;) {
675         nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
676         if (nb_sectors <= 0) {
677             return 0;
678         }
679         ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
680         if (ret < 0) {
681             error_report("error getting block status at sector %" PRId64 ": %s",
682                          sector_num, strerror(-ret));
683             return ret;
684         }
685         if (ret & BDRV_BLOCK_ZERO) {
686             sector_num += n;
687             continue;
688         }
689         ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS,
690                                  n << BDRV_SECTOR_BITS, flags);
691         if (ret < 0) {
692             error_report("error writing zeroes at sector %" PRId64 ": %s",
693                          sector_num, strerror(-ret));
694             return ret;
695         }
696         sector_num += n;
697     }
698 }
699 
700 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
701 {
702     int ret;
703 
704     ret = bdrv_prwv_co(child, offset, qiov, false, 0);
705     if (ret < 0) {
706         return ret;
707     }
708 
709     return qiov->size;
710 }
711 
712 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
713 {
714     QEMUIOVector qiov;
715     struct iovec iov = {
716         .iov_base = (void *)buf,
717         .iov_len = bytes,
718     };
719 
720     if (bytes < 0) {
721         return -EINVAL;
722     }
723 
724     qemu_iovec_init_external(&qiov, &iov, 1);
725     return bdrv_preadv(child, offset, &qiov);
726 }
727 
728 int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
729 {
730     int ret;
731 
732     ret = bdrv_prwv_co(child, offset, qiov, true, 0);
733     if (ret < 0) {
734         return ret;
735     }
736 
737     return qiov->size;
738 }
739 
740 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
741 {
742     QEMUIOVector qiov;
743     struct iovec iov = {
744         .iov_base   = (void *) buf,
745         .iov_len    = bytes,
746     };
747 
748     if (bytes < 0) {
749         return -EINVAL;
750     }
751 
752     qemu_iovec_init_external(&qiov, &iov, 1);
753     return bdrv_pwritev(child, offset, &qiov);
754 }
755 
756 /*
757  * Writes to the file and ensures that no writes are reordered across this
758  * request (acts as a barrier)
759  *
760  * Returns 0 on success, -errno in error cases.
761  */
762 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
763                      const void *buf, int count)
764 {
765     int ret;
766 
767     ret = bdrv_pwrite(child, offset, buf, count);
768     if (ret < 0) {
769         return ret;
770     }
771 
772     ret = bdrv_flush(child->bs);
773     if (ret < 0) {
774         return ret;
775     }
776 
777     return 0;
778 }
779 
780 typedef struct CoroutineIOCompletion {
781     Coroutine *coroutine;
782     int ret;
783 } CoroutineIOCompletion;
784 
785 static void bdrv_co_io_em_complete(void *opaque, int ret)
786 {
787     CoroutineIOCompletion *co = opaque;
788 
789     co->ret = ret;
790     qemu_coroutine_enter(co->coroutine);
791 }
792 
793 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
794                                            uint64_t offset, uint64_t bytes,
795                                            QEMUIOVector *qiov, int flags)
796 {
797     BlockDriver *drv = bs->drv;
798     int64_t sector_num;
799     unsigned int nb_sectors;
800 
801     assert(!(flags & ~BDRV_REQ_MASK));
802 
803     if (drv->bdrv_co_preadv) {
804         return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
805     }
806 
807     sector_num = offset >> BDRV_SECTOR_BITS;
808     nb_sectors = bytes >> BDRV_SECTOR_BITS;
809 
810     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
811     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
812     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
813 
814     if (drv->bdrv_co_readv) {
815         return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
816     } else {
817         BlockAIOCB *acb;
818         CoroutineIOCompletion co = {
819             .coroutine = qemu_coroutine_self(),
820         };
821 
822         acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
823                                       bdrv_co_io_em_complete, &co);
824         if (acb == NULL) {
825             return -EIO;
826         } else {
827             qemu_coroutine_yield();
828             return co.ret;
829         }
830     }
831 }
832 
833 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
834                                             uint64_t offset, uint64_t bytes,
835                                             QEMUIOVector *qiov, int flags)
836 {
837     BlockDriver *drv = bs->drv;
838     int64_t sector_num;
839     unsigned int nb_sectors;
840     int ret;
841 
842     assert(!(flags & ~BDRV_REQ_MASK));
843 
844     if (drv->bdrv_co_pwritev) {
845         ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
846                                    flags & bs->supported_write_flags);
847         flags &= ~bs->supported_write_flags;
848         goto emulate_flags;
849     }
850 
851     sector_num = offset >> BDRV_SECTOR_BITS;
852     nb_sectors = bytes >> BDRV_SECTOR_BITS;
853 
854     assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
855     assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
856     assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
857 
858     if (drv->bdrv_co_writev_flags) {
859         ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
860                                         flags & bs->supported_write_flags);
861         flags &= ~bs->supported_write_flags;
862     } else if (drv->bdrv_co_writev) {
863         assert(!bs->supported_write_flags);
864         ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
865     } else {
866         BlockAIOCB *acb;
867         CoroutineIOCompletion co = {
868             .coroutine = qemu_coroutine_self(),
869         };
870 
871         acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
872                                        bdrv_co_io_em_complete, &co);
873         if (acb == NULL) {
874             ret = -EIO;
875         } else {
876             qemu_coroutine_yield();
877             ret = co.ret;
878         }
879     }
880 
881 emulate_flags:
882     if (ret == 0 && (flags & BDRV_REQ_FUA)) {
883         ret = bdrv_co_flush(bs);
884     }
885 
886     return ret;
887 }
888 
889 static int coroutine_fn
890 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset,
891                                uint64_t bytes, QEMUIOVector *qiov)
892 {
893     BlockDriver *drv = bs->drv;
894 
895     if (!drv->bdrv_co_pwritev_compressed) {
896         return -ENOTSUP;
897     }
898 
899     return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
900 }
901 
902 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
903         int64_t offset, unsigned int bytes, QEMUIOVector *qiov)
904 {
905     /* Perform I/O through a temporary buffer so that users who scribble over
906      * their read buffer while the operation is in progress do not end up
907      * modifying the image file.  This is critical for zero-copy guest I/O
908      * where anything might happen inside guest memory.
909      */
910     void *bounce_buffer;
911 
912     BlockDriver *drv = bs->drv;
913     struct iovec iov;
914     QEMUIOVector bounce_qiov;
915     int64_t cluster_offset;
916     unsigned int cluster_bytes;
917     size_t skip_bytes;
918     int ret;
919 
920     /* Cover entire cluster so no additional backing file I/O is required when
921      * allocating cluster in the image file.
922      */
923     bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
924 
925     trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
926                                    cluster_offset, cluster_bytes);
927 
928     iov.iov_len = cluster_bytes;
929     iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
930     if (bounce_buffer == NULL) {
931         ret = -ENOMEM;
932         goto err;
933     }
934 
935     qemu_iovec_init_external(&bounce_qiov, &iov, 1);
936 
937     ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes,
938                              &bounce_qiov, 0);
939     if (ret < 0) {
940         goto err;
941     }
942 
943     if (drv->bdrv_co_pwrite_zeroes &&
944         buffer_is_zero(bounce_buffer, iov.iov_len)) {
945         /* FIXME: Should we (perhaps conditionally) be setting
946          * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
947          * that still correctly reads as zero? */
948         ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0);
949     } else {
950         /* This does not change the data on the disk, it is not necessary
951          * to flush even in cache=writethrough mode.
952          */
953         ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes,
954                                   &bounce_qiov, 0);
955     }
956 
957     if (ret < 0) {
958         /* It might be okay to ignore write errors for guest requests.  If this
959          * is a deliberate copy-on-read then we don't want to ignore the error.
960          * Simply report it in all cases.
961          */
962         goto err;
963     }
964 
965     skip_bytes = offset - cluster_offset;
966     qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes);
967 
968 err:
969     qemu_vfree(bounce_buffer);
970     return ret;
971 }
972 
973 /*
974  * Forwards an already correctly aligned request to the BlockDriver. This
975  * handles copy on read, zeroing after EOF, and fragmentation of large
976  * reads; any other features must be implemented by the caller.
977  */
978 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
979     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
980     int64_t align, QEMUIOVector *qiov, int flags)
981 {
982     int64_t total_bytes, max_bytes;
983     int ret = 0;
984     uint64_t bytes_remaining = bytes;
985     int max_transfer;
986 
987     assert(is_power_of_2(align));
988     assert((offset & (align - 1)) == 0);
989     assert((bytes & (align - 1)) == 0);
990     assert(!qiov || bytes == qiov->size);
991     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
992     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
993                                    align);
994 
995     /* TODO: We would need a per-BDS .supported_read_flags and
996      * potential fallback support, if we ever implement any read flags
997      * to pass through to drivers.  For now, there aren't any
998      * passthrough flags.  */
999     assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ)));
1000 
1001     /* Handle Copy on Read and associated serialisation */
1002     if (flags & BDRV_REQ_COPY_ON_READ) {
1003         /* If we touch the same cluster it counts as an overlap.  This
1004          * guarantees that allocating writes will be serialized and not race
1005          * with each other for the same cluster.  For example, in copy-on-read
1006          * it ensures that the CoR read and write operations are atomic and
1007          * guest writes cannot interleave between them. */
1008         mark_request_serialising(req, bdrv_get_cluster_size(bs));
1009     }
1010 
1011     if (!(flags & BDRV_REQ_NO_SERIALISING)) {
1012         wait_serialising_requests(req);
1013     }
1014 
1015     if (flags & BDRV_REQ_COPY_ON_READ) {
1016         int64_t start_sector = offset >> BDRV_SECTOR_BITS;
1017         int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1018         unsigned int nb_sectors = end_sector - start_sector;
1019         int pnum;
1020 
1021         ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum);
1022         if (ret < 0) {
1023             goto out;
1024         }
1025 
1026         if (!ret || pnum != nb_sectors) {
1027             ret = bdrv_co_do_copy_on_readv(bs, offset, bytes, qiov);
1028             goto out;
1029         }
1030     }
1031 
1032     /* Forward the request to the BlockDriver, possibly fragmenting it */
1033     total_bytes = bdrv_getlength(bs);
1034     if (total_bytes < 0) {
1035         ret = total_bytes;
1036         goto out;
1037     }
1038 
1039     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1040     if (bytes <= max_bytes && bytes <= max_transfer) {
1041         ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
1042         goto out;
1043     }
1044 
1045     while (bytes_remaining) {
1046         int num;
1047 
1048         if (max_bytes) {
1049             QEMUIOVector local_qiov;
1050 
1051             num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1052             assert(num);
1053             qemu_iovec_init(&local_qiov, qiov->niov);
1054             qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
1055 
1056             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1057                                      num, &local_qiov, 0);
1058             max_bytes -= num;
1059             qemu_iovec_destroy(&local_qiov);
1060         } else {
1061             num = bytes_remaining;
1062             ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0,
1063                                     bytes_remaining);
1064         }
1065         if (ret < 0) {
1066             goto out;
1067         }
1068         bytes_remaining -= num;
1069     }
1070 
1071 out:
1072     return ret < 0 ? ret : 0;
1073 }
1074 
1075 /*
1076  * Handle a read request in coroutine context
1077  */
1078 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1079     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1080     BdrvRequestFlags flags)
1081 {
1082     BlockDriverState *bs = child->bs;
1083     BlockDriver *drv = bs->drv;
1084     BdrvTrackedRequest req;
1085 
1086     uint64_t align = bs->bl.request_alignment;
1087     uint8_t *head_buf = NULL;
1088     uint8_t *tail_buf = NULL;
1089     QEMUIOVector local_qiov;
1090     bool use_local_qiov = false;
1091     int ret;
1092 
1093     if (!drv) {
1094         return -ENOMEDIUM;
1095     }
1096 
1097     ret = bdrv_check_byte_request(bs, offset, bytes);
1098     if (ret < 0) {
1099         return ret;
1100     }
1101 
1102     /* Don't do copy-on-read if we read data before write operation */
1103     if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
1104         flags |= BDRV_REQ_COPY_ON_READ;
1105     }
1106 
1107     /* Align read if necessary by padding qiov */
1108     if (offset & (align - 1)) {
1109         head_buf = qemu_blockalign(bs, align);
1110         qemu_iovec_init(&local_qiov, qiov->niov + 2);
1111         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1112         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1113         use_local_qiov = true;
1114 
1115         bytes += offset & (align - 1);
1116         offset = offset & ~(align - 1);
1117     }
1118 
1119     if ((offset + bytes) & (align - 1)) {
1120         if (!use_local_qiov) {
1121             qemu_iovec_init(&local_qiov, qiov->niov + 1);
1122             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1123             use_local_qiov = true;
1124         }
1125         tail_buf = qemu_blockalign(bs, align);
1126         qemu_iovec_add(&local_qiov, tail_buf,
1127                        align - ((offset + bytes) & (align - 1)));
1128 
1129         bytes = ROUND_UP(bytes, align);
1130     }
1131 
1132     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1133     ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
1134                               use_local_qiov ? &local_qiov : qiov,
1135                               flags);
1136     tracked_request_end(&req);
1137 
1138     if (use_local_qiov) {
1139         qemu_iovec_destroy(&local_qiov);
1140         qemu_vfree(head_buf);
1141         qemu_vfree(tail_buf);
1142     }
1143 
1144     return ret;
1145 }
1146 
1147 static int coroutine_fn bdrv_co_do_readv(BdrvChild *child,
1148     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1149     BdrvRequestFlags flags)
1150 {
1151     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1152         return -EINVAL;
1153     }
1154 
1155     return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS,
1156                           nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1157 }
1158 
1159 int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
1160                                int nb_sectors, QEMUIOVector *qiov)
1161 {
1162     trace_bdrv_co_readv(child->bs, sector_num, nb_sectors);
1163 
1164     return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0);
1165 }
1166 
1167 /* Maximum buffer for write zeroes fallback, in bytes */
1168 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
1169 
1170 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1171     int64_t offset, int count, BdrvRequestFlags flags)
1172 {
1173     BlockDriver *drv = bs->drv;
1174     QEMUIOVector qiov;
1175     struct iovec iov = {0};
1176     int ret = 0;
1177     bool need_flush = false;
1178     int head = 0;
1179     int tail = 0;
1180 
1181     int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
1182     int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1183                         bs->bl.request_alignment);
1184 
1185     assert(alignment % bs->bl.request_alignment == 0);
1186     head = offset % alignment;
1187     tail = (offset + count) % alignment;
1188     max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1189     assert(max_write_zeroes >= bs->bl.request_alignment);
1190 
1191     while (count > 0 && !ret) {
1192         int num = count;
1193 
1194         /* Align request.  Block drivers can expect the "bulk" of the request
1195          * to be aligned, and that unaligned requests do not cross cluster
1196          * boundaries.
1197          */
1198         if (head) {
1199             /* Make a small request up to the first aligned sector.  */
1200             num = MIN(count, alignment - head);
1201             head = 0;
1202         } else if (tail && num > alignment) {
1203             /* Shorten the request to the last aligned sector.  */
1204             num -= tail;
1205         }
1206 
1207         /* limit request size */
1208         if (num > max_write_zeroes) {
1209             num = max_write_zeroes;
1210         }
1211 
1212         ret = -ENOTSUP;
1213         /* First try the efficient write zeroes operation */
1214         if (drv->bdrv_co_pwrite_zeroes) {
1215             ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1216                                              flags & bs->supported_zero_flags);
1217             if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1218                 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1219                 need_flush = true;
1220             }
1221         } else {
1222             assert(!bs->supported_zero_flags);
1223         }
1224 
1225         if (ret == -ENOTSUP) {
1226             /* Fall back to bounce buffer if write zeroes is unsupported */
1227             int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1228                                             MAX_WRITE_ZEROES_BOUNCE_BUFFER);
1229             BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1230 
1231             if ((flags & BDRV_REQ_FUA) &&
1232                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1233                 /* No need for bdrv_driver_pwrite() to do a fallback
1234                  * flush on each chunk; use just one at the end */
1235                 write_flags &= ~BDRV_REQ_FUA;
1236                 need_flush = true;
1237             }
1238             num = MIN(num, max_transfer);
1239             iov.iov_len = num;
1240             if (iov.iov_base == NULL) {
1241                 iov.iov_base = qemu_try_blockalign(bs, num);
1242                 if (iov.iov_base == NULL) {
1243                     ret = -ENOMEM;
1244                     goto fail;
1245                 }
1246                 memset(iov.iov_base, 0, num);
1247             }
1248             qemu_iovec_init_external(&qiov, &iov, 1);
1249 
1250             ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags);
1251 
1252             /* Keep bounce buffer around if it is big enough for all
1253              * all future requests.
1254              */
1255             if (num < max_transfer) {
1256                 qemu_vfree(iov.iov_base);
1257                 iov.iov_base = NULL;
1258             }
1259         }
1260 
1261         offset += num;
1262         count -= num;
1263     }
1264 
1265 fail:
1266     if (ret == 0 && need_flush) {
1267         ret = bdrv_co_flush(bs);
1268     }
1269     qemu_vfree(iov.iov_base);
1270     return ret;
1271 }
1272 
1273 /*
1274  * Forwards an already correctly aligned write request to the BlockDriver,
1275  * after possibly fragmenting it.
1276  */
1277 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
1278     BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
1279     int64_t align, QEMUIOVector *qiov, int flags)
1280 {
1281     BlockDriver *drv = bs->drv;
1282     bool waited;
1283     int ret;
1284 
1285     int64_t start_sector = offset >> BDRV_SECTOR_BITS;
1286     int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1287     uint64_t bytes_remaining = bytes;
1288     int max_transfer;
1289 
1290     assert(is_power_of_2(align));
1291     assert((offset & (align - 1)) == 0);
1292     assert((bytes & (align - 1)) == 0);
1293     assert(!qiov || bytes == qiov->size);
1294     assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1295     assert(!(flags & ~BDRV_REQ_MASK));
1296     max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1297                                    align);
1298 
1299     waited = wait_serialising_requests(req);
1300     assert(!waited || !req->serialising);
1301     assert(req->overlap_offset <= offset);
1302     assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1303 
1304     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
1305 
1306     if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1307         !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
1308         qemu_iovec_is_zero(qiov)) {
1309         flags |= BDRV_REQ_ZERO_WRITE;
1310         if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1311             flags |= BDRV_REQ_MAY_UNMAP;
1312         }
1313     }
1314 
1315     if (ret < 0) {
1316         /* Do nothing, write notifier decided to fail this request */
1317     } else if (flags & BDRV_REQ_ZERO_WRITE) {
1318         bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1319         ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
1320     } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
1321         ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov);
1322     } else if (bytes <= max_transfer) {
1323         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1324         ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
1325     } else {
1326         bdrv_debug_event(bs, BLKDBG_PWRITEV);
1327         while (bytes_remaining) {
1328             int num = MIN(bytes_remaining, max_transfer);
1329             QEMUIOVector local_qiov;
1330             int local_flags = flags;
1331 
1332             assert(num);
1333             if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
1334                 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1335                 /* If FUA is going to be emulated by flush, we only
1336                  * need to flush on the last iteration */
1337                 local_flags &= ~BDRV_REQ_FUA;
1338             }
1339             qemu_iovec_init(&local_qiov, qiov->niov);
1340             qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
1341 
1342             ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
1343                                       num, &local_qiov, local_flags);
1344             qemu_iovec_destroy(&local_qiov);
1345             if (ret < 0) {
1346                 break;
1347             }
1348             bytes_remaining -= num;
1349         }
1350     }
1351     bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1352 
1353     ++bs->write_gen;
1354     bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
1355 
1356     if (bs->wr_highest_offset < offset + bytes) {
1357         bs->wr_highest_offset = offset + bytes;
1358     }
1359 
1360     if (ret >= 0) {
1361         bs->total_sectors = MAX(bs->total_sectors, end_sector);
1362         ret = 0;
1363     }
1364 
1365     return ret;
1366 }
1367 
1368 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
1369                                                 int64_t offset,
1370                                                 unsigned int bytes,
1371                                                 BdrvRequestFlags flags,
1372                                                 BdrvTrackedRequest *req)
1373 {
1374     uint8_t *buf = NULL;
1375     QEMUIOVector local_qiov;
1376     struct iovec iov;
1377     uint64_t align = bs->bl.request_alignment;
1378     unsigned int head_padding_bytes, tail_padding_bytes;
1379     int ret = 0;
1380 
1381     head_padding_bytes = offset & (align - 1);
1382     tail_padding_bytes = align - ((offset + bytes) & (align - 1));
1383 
1384 
1385     assert(flags & BDRV_REQ_ZERO_WRITE);
1386     if (head_padding_bytes || tail_padding_bytes) {
1387         buf = qemu_blockalign(bs, align);
1388         iov = (struct iovec) {
1389             .iov_base   = buf,
1390             .iov_len    = align,
1391         };
1392         qemu_iovec_init_external(&local_qiov, &iov, 1);
1393     }
1394     if (head_padding_bytes) {
1395         uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
1396 
1397         /* RMW the unaligned part before head. */
1398         mark_request_serialising(req, align);
1399         wait_serialising_requests(req);
1400         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1401         ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
1402                                   align, &local_qiov, 0);
1403         if (ret < 0) {
1404             goto fail;
1405         }
1406         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1407 
1408         memset(buf + head_padding_bytes, 0, zero_bytes);
1409         ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
1410                                    align, &local_qiov,
1411                                    flags & ~BDRV_REQ_ZERO_WRITE);
1412         if (ret < 0) {
1413             goto fail;
1414         }
1415         offset += zero_bytes;
1416         bytes -= zero_bytes;
1417     }
1418 
1419     assert(!bytes || (offset & (align - 1)) == 0);
1420     if (bytes >= align) {
1421         /* Write the aligned part in the middle. */
1422         uint64_t aligned_bytes = bytes & ~(align - 1);
1423         ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, align,
1424                                    NULL, flags);
1425         if (ret < 0) {
1426             goto fail;
1427         }
1428         bytes -= aligned_bytes;
1429         offset += aligned_bytes;
1430     }
1431 
1432     assert(!bytes || (offset & (align - 1)) == 0);
1433     if (bytes) {
1434         assert(align == tail_padding_bytes + bytes);
1435         /* RMW the unaligned part after tail. */
1436         mark_request_serialising(req, align);
1437         wait_serialising_requests(req);
1438         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1439         ret = bdrv_aligned_preadv(bs, req, offset, align,
1440                                   align, &local_qiov, 0);
1441         if (ret < 0) {
1442             goto fail;
1443         }
1444         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1445 
1446         memset(buf, 0, bytes);
1447         ret = bdrv_aligned_pwritev(bs, req, offset, align, align,
1448                                    &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
1449     }
1450 fail:
1451     qemu_vfree(buf);
1452     return ret;
1453 
1454 }
1455 
1456 /*
1457  * Handle a write request in coroutine context
1458  */
1459 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
1460     int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
1461     BdrvRequestFlags flags)
1462 {
1463     BlockDriverState *bs = child->bs;
1464     BdrvTrackedRequest req;
1465     uint64_t align = bs->bl.request_alignment;
1466     uint8_t *head_buf = NULL;
1467     uint8_t *tail_buf = NULL;
1468     QEMUIOVector local_qiov;
1469     bool use_local_qiov = false;
1470     int ret;
1471 
1472     if (!bs->drv) {
1473         return -ENOMEDIUM;
1474     }
1475     if (bs->read_only) {
1476         return -EPERM;
1477     }
1478     assert(!(bs->open_flags & BDRV_O_INACTIVE));
1479 
1480     ret = bdrv_check_byte_request(bs, offset, bytes);
1481     if (ret < 0) {
1482         return ret;
1483     }
1484 
1485     /*
1486      * Align write if necessary by performing a read-modify-write cycle.
1487      * Pad qiov with the read parts and be sure to have a tracked request not
1488      * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
1489      */
1490     tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
1491 
1492     if (!qiov) {
1493         ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
1494         goto out;
1495     }
1496 
1497     if (offset & (align - 1)) {
1498         QEMUIOVector head_qiov;
1499         struct iovec head_iov;
1500 
1501         mark_request_serialising(&req, align);
1502         wait_serialising_requests(&req);
1503 
1504         head_buf = qemu_blockalign(bs, align);
1505         head_iov = (struct iovec) {
1506             .iov_base   = head_buf,
1507             .iov_len    = align,
1508         };
1509         qemu_iovec_init_external(&head_qiov, &head_iov, 1);
1510 
1511         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1512         ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
1513                                   align, &head_qiov, 0);
1514         if (ret < 0) {
1515             goto fail;
1516         }
1517         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1518 
1519         qemu_iovec_init(&local_qiov, qiov->niov + 2);
1520         qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
1521         qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1522         use_local_qiov = true;
1523 
1524         bytes += offset & (align - 1);
1525         offset = offset & ~(align - 1);
1526 
1527         /* We have read the tail already if the request is smaller
1528          * than one aligned block.
1529          */
1530         if (bytes < align) {
1531             qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes);
1532             bytes = align;
1533         }
1534     }
1535 
1536     if ((offset + bytes) & (align - 1)) {
1537         QEMUIOVector tail_qiov;
1538         struct iovec tail_iov;
1539         size_t tail_bytes;
1540         bool waited;
1541 
1542         mark_request_serialising(&req, align);
1543         waited = wait_serialising_requests(&req);
1544         assert(!waited || !use_local_qiov);
1545 
1546         tail_buf = qemu_blockalign(bs, align);
1547         tail_iov = (struct iovec) {
1548             .iov_base   = tail_buf,
1549             .iov_len    = align,
1550         };
1551         qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
1552 
1553         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1554         ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
1555                                   align, &tail_qiov, 0);
1556         if (ret < 0) {
1557             goto fail;
1558         }
1559         bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1560 
1561         if (!use_local_qiov) {
1562             qemu_iovec_init(&local_qiov, qiov->niov + 1);
1563             qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
1564             use_local_qiov = true;
1565         }
1566 
1567         tail_bytes = (offset + bytes) & (align - 1);
1568         qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
1569 
1570         bytes = ROUND_UP(bytes, align);
1571     }
1572 
1573     ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, align,
1574                                use_local_qiov ? &local_qiov : qiov,
1575                                flags);
1576 
1577 fail:
1578 
1579     if (use_local_qiov) {
1580         qemu_iovec_destroy(&local_qiov);
1581     }
1582     qemu_vfree(head_buf);
1583     qemu_vfree(tail_buf);
1584 out:
1585     tracked_request_end(&req);
1586     return ret;
1587 }
1588 
1589 static int coroutine_fn bdrv_co_do_writev(BdrvChild *child,
1590     int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
1591     BdrvRequestFlags flags)
1592 {
1593     if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
1594         return -EINVAL;
1595     }
1596 
1597     return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS,
1598                            nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
1599 }
1600 
1601 int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
1602     int nb_sectors, QEMUIOVector *qiov)
1603 {
1604     trace_bdrv_co_writev(child->bs, sector_num, nb_sectors);
1605 
1606     return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0);
1607 }
1608 
1609 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
1610                                        int count, BdrvRequestFlags flags)
1611 {
1612     trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags);
1613 
1614     if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
1615         flags &= ~BDRV_REQ_MAY_UNMAP;
1616     }
1617 
1618     return bdrv_co_pwritev(child, offset, count, NULL,
1619                            BDRV_REQ_ZERO_WRITE | flags);
1620 }
1621 
1622 typedef struct BdrvCoGetBlockStatusData {
1623     BlockDriverState *bs;
1624     BlockDriverState *base;
1625     BlockDriverState **file;
1626     int64_t sector_num;
1627     int nb_sectors;
1628     int *pnum;
1629     int64_t ret;
1630     bool done;
1631 } BdrvCoGetBlockStatusData;
1632 
1633 /*
1634  * Returns the allocation status of the specified sectors.
1635  * Drivers not implementing the functionality are assumed to not support
1636  * backing files, hence all their sectors are reported as allocated.
1637  *
1638  * If 'sector_num' is beyond the end of the disk image the return value is 0
1639  * and 'pnum' is set to 0.
1640  *
1641  * 'pnum' is set to the number of sectors (including and immediately following
1642  * the specified sector) that are known to be in the same
1643  * allocated/unallocated state.
1644  *
1645  * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
1646  * beyond the end of the disk image it will be clamped.
1647  *
1648  * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
1649  * points to the BDS which the sector range is allocated in.
1650  */
1651 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
1652                                                      int64_t sector_num,
1653                                                      int nb_sectors, int *pnum,
1654                                                      BlockDriverState **file)
1655 {
1656     int64_t total_sectors;
1657     int64_t n;
1658     int64_t ret, ret2;
1659 
1660     total_sectors = bdrv_nb_sectors(bs);
1661     if (total_sectors < 0) {
1662         return total_sectors;
1663     }
1664 
1665     if (sector_num >= total_sectors) {
1666         *pnum = 0;
1667         return 0;
1668     }
1669 
1670     n = total_sectors - sector_num;
1671     if (n < nb_sectors) {
1672         nb_sectors = n;
1673     }
1674 
1675     if (!bs->drv->bdrv_co_get_block_status) {
1676         *pnum = nb_sectors;
1677         ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
1678         if (bs->drv->protocol_name) {
1679             ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
1680         }
1681         return ret;
1682     }
1683 
1684     *file = NULL;
1685     ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
1686                                             file);
1687     if (ret < 0) {
1688         *pnum = 0;
1689         return ret;
1690     }
1691 
1692     if (ret & BDRV_BLOCK_RAW) {
1693         assert(ret & BDRV_BLOCK_OFFSET_VALID);
1694         return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
1695                                      *pnum, pnum, file);
1696     }
1697 
1698     if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
1699         ret |= BDRV_BLOCK_ALLOCATED;
1700     } else {
1701         if (bdrv_unallocated_blocks_are_zero(bs)) {
1702             ret |= BDRV_BLOCK_ZERO;
1703         } else if (bs->backing) {
1704             BlockDriverState *bs2 = bs->backing->bs;
1705             int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
1706             if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
1707                 ret |= BDRV_BLOCK_ZERO;
1708             }
1709         }
1710     }
1711 
1712     if (*file && *file != bs &&
1713         (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
1714         (ret & BDRV_BLOCK_OFFSET_VALID)) {
1715         BlockDriverState *file2;
1716         int file_pnum;
1717 
1718         ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
1719                                         *pnum, &file_pnum, &file2);
1720         if (ret2 >= 0) {
1721             /* Ignore errors.  This is just providing extra information, it
1722              * is useful but not necessary.
1723              */
1724             if (!file_pnum) {
1725                 /* !file_pnum indicates an offset at or beyond the EOF; it is
1726                  * perfectly valid for the format block driver to point to such
1727                  * offsets, so catch it and mark everything as zero */
1728                 ret |= BDRV_BLOCK_ZERO;
1729             } else {
1730                 /* Limit request to the range reported by the protocol driver */
1731                 *pnum = file_pnum;
1732                 ret |= (ret2 & BDRV_BLOCK_ZERO);
1733             }
1734         }
1735     }
1736 
1737     return ret;
1738 }
1739 
1740 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
1741         BlockDriverState *base,
1742         int64_t sector_num,
1743         int nb_sectors,
1744         int *pnum,
1745         BlockDriverState **file)
1746 {
1747     BlockDriverState *p;
1748     int64_t ret = 0;
1749 
1750     assert(bs != base);
1751     for (p = bs; p != base; p = backing_bs(p)) {
1752         ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
1753         if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
1754             break;
1755         }
1756         /* [sector_num, pnum] unallocated on this layer, which could be only
1757          * the first part of [sector_num, nb_sectors].  */
1758         nb_sectors = MIN(nb_sectors, *pnum);
1759     }
1760     return ret;
1761 }
1762 
1763 /* Coroutine wrapper for bdrv_get_block_status_above() */
1764 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
1765 {
1766     BdrvCoGetBlockStatusData *data = opaque;
1767 
1768     data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
1769                                                data->sector_num,
1770                                                data->nb_sectors,
1771                                                data->pnum,
1772                                                data->file);
1773     data->done = true;
1774 }
1775 
1776 /*
1777  * Synchronous wrapper around bdrv_co_get_block_status_above().
1778  *
1779  * See bdrv_co_get_block_status_above() for details.
1780  */
1781 int64_t bdrv_get_block_status_above(BlockDriverState *bs,
1782                                     BlockDriverState *base,
1783                                     int64_t sector_num,
1784                                     int nb_sectors, int *pnum,
1785                                     BlockDriverState **file)
1786 {
1787     Coroutine *co;
1788     BdrvCoGetBlockStatusData data = {
1789         .bs = bs,
1790         .base = base,
1791         .file = file,
1792         .sector_num = sector_num,
1793         .nb_sectors = nb_sectors,
1794         .pnum = pnum,
1795         .done = false,
1796     };
1797 
1798     if (qemu_in_coroutine()) {
1799         /* Fast-path if already in coroutine context */
1800         bdrv_get_block_status_above_co_entry(&data);
1801     } else {
1802         AioContext *aio_context = bdrv_get_aio_context(bs);
1803 
1804         co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry,
1805                                    &data);
1806         qemu_coroutine_enter(co);
1807         while (!data.done) {
1808             aio_poll(aio_context, true);
1809         }
1810     }
1811     return data.ret;
1812 }
1813 
1814 int64_t bdrv_get_block_status(BlockDriverState *bs,
1815                               int64_t sector_num,
1816                               int nb_sectors, int *pnum,
1817                               BlockDriverState **file)
1818 {
1819     return bdrv_get_block_status_above(bs, backing_bs(bs),
1820                                        sector_num, nb_sectors, pnum, file);
1821 }
1822 
1823 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
1824                                    int nb_sectors, int *pnum)
1825 {
1826     BlockDriverState *file;
1827     int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
1828                                         &file);
1829     if (ret < 0) {
1830         return ret;
1831     }
1832     return !!(ret & BDRV_BLOCK_ALLOCATED);
1833 }
1834 
1835 /*
1836  * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
1837  *
1838  * Return true if the given sector is allocated in any image between
1839  * BASE and TOP (inclusive).  BASE can be NULL to check if the given
1840  * sector is allocated in any image of the chain.  Return false otherwise.
1841  *
1842  * 'pnum' is set to the number of sectors (including and immediately following
1843  *  the specified sector) that are known to be in the same
1844  *  allocated/unallocated state.
1845  *
1846  */
1847 int bdrv_is_allocated_above(BlockDriverState *top,
1848                             BlockDriverState *base,
1849                             int64_t sector_num,
1850                             int nb_sectors, int *pnum)
1851 {
1852     BlockDriverState *intermediate;
1853     int ret, n = nb_sectors;
1854 
1855     intermediate = top;
1856     while (intermediate && intermediate != base) {
1857         int pnum_inter;
1858         ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
1859                                 &pnum_inter);
1860         if (ret < 0) {
1861             return ret;
1862         } else if (ret) {
1863             *pnum = pnum_inter;
1864             return 1;
1865         }
1866 
1867         /*
1868          * [sector_num, nb_sectors] is unallocated on top but intermediate
1869          * might have
1870          *
1871          * [sector_num+x, nr_sectors] allocated.
1872          */
1873         if (n > pnum_inter &&
1874             (intermediate == top ||
1875              sector_num + pnum_inter < intermediate->total_sectors)) {
1876             n = pnum_inter;
1877         }
1878 
1879         intermediate = backing_bs(intermediate);
1880     }
1881 
1882     *pnum = n;
1883     return 0;
1884 }
1885 
1886 typedef struct BdrvVmstateCo {
1887     BlockDriverState    *bs;
1888     QEMUIOVector        *qiov;
1889     int64_t             pos;
1890     bool                is_read;
1891     int                 ret;
1892 } BdrvVmstateCo;
1893 
1894 static int coroutine_fn
1895 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
1896                    bool is_read)
1897 {
1898     BlockDriver *drv = bs->drv;
1899 
1900     if (!drv) {
1901         return -ENOMEDIUM;
1902     } else if (drv->bdrv_load_vmstate) {
1903         return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos)
1904                        : drv->bdrv_save_vmstate(bs, qiov, pos);
1905     } else if (bs->file) {
1906         return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read);
1907     }
1908 
1909     return -ENOTSUP;
1910 }
1911 
1912 static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque)
1913 {
1914     BdrvVmstateCo *co = opaque;
1915     co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read);
1916 }
1917 
1918 static inline int
1919 bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
1920                 bool is_read)
1921 {
1922     if (qemu_in_coroutine()) {
1923         return bdrv_co_rw_vmstate(bs, qiov, pos, is_read);
1924     } else {
1925         BdrvVmstateCo data = {
1926             .bs         = bs,
1927             .qiov       = qiov,
1928             .pos        = pos,
1929             .is_read    = is_read,
1930             .ret        = -EINPROGRESS,
1931         };
1932         Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data);
1933 
1934         qemu_coroutine_enter(co);
1935         while (data.ret == -EINPROGRESS) {
1936             aio_poll(bdrv_get_aio_context(bs), true);
1937         }
1938         return data.ret;
1939     }
1940 }
1941 
1942 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
1943                       int64_t pos, int size)
1944 {
1945     QEMUIOVector qiov;
1946     struct iovec iov = {
1947         .iov_base   = (void *) buf,
1948         .iov_len    = size,
1949     };
1950     int ret;
1951 
1952     qemu_iovec_init_external(&qiov, &iov, 1);
1953 
1954     ret = bdrv_writev_vmstate(bs, &qiov, pos);
1955     if (ret < 0) {
1956         return ret;
1957     }
1958 
1959     return size;
1960 }
1961 
1962 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1963 {
1964     return bdrv_rw_vmstate(bs, qiov, pos, false);
1965 }
1966 
1967 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
1968                       int64_t pos, int size)
1969 {
1970     QEMUIOVector qiov;
1971     struct iovec iov = {
1972         .iov_base   = buf,
1973         .iov_len    = size,
1974     };
1975     int ret;
1976 
1977     qemu_iovec_init_external(&qiov, &iov, 1);
1978     ret = bdrv_readv_vmstate(bs, &qiov, pos);
1979     if (ret < 0) {
1980         return ret;
1981     }
1982 
1983     return size;
1984 }
1985 
1986 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
1987 {
1988     return bdrv_rw_vmstate(bs, qiov, pos, true);
1989 }
1990 
1991 /**************************************************************/
1992 /* async I/Os */
1993 
1994 BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num,
1995                            QEMUIOVector *qiov, int nb_sectors,
1996                            BlockCompletionFunc *cb, void *opaque)
1997 {
1998     trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque);
1999 
2000     assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
2001     return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
2002                                   0, cb, opaque, false);
2003 }
2004 
2005 BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num,
2006                             QEMUIOVector *qiov, int nb_sectors,
2007                             BlockCompletionFunc *cb, void *opaque)
2008 {
2009     trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque);
2010 
2011     assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
2012     return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
2013                                   0, cb, opaque, true);
2014 }
2015 
2016 void bdrv_aio_cancel(BlockAIOCB *acb)
2017 {
2018     qemu_aio_ref(acb);
2019     bdrv_aio_cancel_async(acb);
2020     while (acb->refcnt > 1) {
2021         if (acb->aiocb_info->get_aio_context) {
2022             aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2023         } else if (acb->bs) {
2024             aio_poll(bdrv_get_aio_context(acb->bs), true);
2025         } else {
2026             abort();
2027         }
2028     }
2029     qemu_aio_unref(acb);
2030 }
2031 
2032 /* Async version of aio cancel. The caller is not blocked if the acb implements
2033  * cancel_async, otherwise we do nothing and let the request normally complete.
2034  * In either case the completion callback must be called. */
2035 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2036 {
2037     if (acb->aiocb_info->cancel_async) {
2038         acb->aiocb_info->cancel_async(acb);
2039     }
2040 }
2041 
2042 /**************************************************************/
2043 /* async block device emulation */
2044 
2045 typedef struct BlockRequest {
2046     union {
2047         /* Used during read, write, trim */
2048         struct {
2049             int64_t offset;
2050             int bytes;
2051             int flags;
2052             QEMUIOVector *qiov;
2053         };
2054         /* Used during ioctl */
2055         struct {
2056             int req;
2057             void *buf;
2058         };
2059     };
2060     BlockCompletionFunc *cb;
2061     void *opaque;
2062 
2063     int error;
2064 } BlockRequest;
2065 
2066 typedef struct BlockAIOCBCoroutine {
2067     BlockAIOCB common;
2068     BdrvChild *child;
2069     BlockRequest req;
2070     bool is_write;
2071     bool need_bh;
2072     bool *done;
2073     QEMUBH* bh;
2074 } BlockAIOCBCoroutine;
2075 
2076 static const AIOCBInfo bdrv_em_co_aiocb_info = {
2077     .aiocb_size         = sizeof(BlockAIOCBCoroutine),
2078 };
2079 
2080 static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
2081 {
2082     if (!acb->need_bh) {
2083         acb->common.cb(acb->common.opaque, acb->req.error);
2084         qemu_aio_unref(acb);
2085     }
2086 }
2087 
2088 static void bdrv_co_em_bh(void *opaque)
2089 {
2090     BlockAIOCBCoroutine *acb = opaque;
2091 
2092     assert(!acb->need_bh);
2093     qemu_bh_delete(acb->bh);
2094     bdrv_co_complete(acb);
2095 }
2096 
2097 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
2098 {
2099     acb->need_bh = false;
2100     if (acb->req.error != -EINPROGRESS) {
2101         BlockDriverState *bs = acb->common.bs;
2102 
2103         acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
2104         qemu_bh_schedule(acb->bh);
2105     }
2106 }
2107 
2108 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
2109 static void coroutine_fn bdrv_co_do_rw(void *opaque)
2110 {
2111     BlockAIOCBCoroutine *acb = opaque;
2112 
2113     if (!acb->is_write) {
2114         acb->req.error = bdrv_co_preadv(acb->child, acb->req.offset,
2115             acb->req.qiov->size, acb->req.qiov, acb->req.flags);
2116     } else {
2117         acb->req.error = bdrv_co_pwritev(acb->child, acb->req.offset,
2118             acb->req.qiov->size, acb->req.qiov, acb->req.flags);
2119     }
2120 
2121     bdrv_co_complete(acb);
2122 }
2123 
2124 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
2125                                           int64_t offset,
2126                                           QEMUIOVector *qiov,
2127                                           BdrvRequestFlags flags,
2128                                           BlockCompletionFunc *cb,
2129                                           void *opaque,
2130                                           bool is_write)
2131 {
2132     Coroutine *co;
2133     BlockAIOCBCoroutine *acb;
2134 
2135     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque);
2136     acb->child = child;
2137     acb->need_bh = true;
2138     acb->req.error = -EINPROGRESS;
2139     acb->req.offset = offset;
2140     acb->req.qiov = qiov;
2141     acb->req.flags = flags;
2142     acb->is_write = is_write;
2143 
2144     co = qemu_coroutine_create(bdrv_co_do_rw, acb);
2145     qemu_coroutine_enter(co);
2146 
2147     bdrv_co_maybe_schedule_bh(acb);
2148     return &acb->common;
2149 }
2150 
2151 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
2152 {
2153     BlockAIOCBCoroutine *acb = opaque;
2154     BlockDriverState *bs = acb->common.bs;
2155 
2156     acb->req.error = bdrv_co_flush(bs);
2157     bdrv_co_complete(acb);
2158 }
2159 
2160 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
2161         BlockCompletionFunc *cb, void *opaque)
2162 {
2163     trace_bdrv_aio_flush(bs, opaque);
2164 
2165     Coroutine *co;
2166     BlockAIOCBCoroutine *acb;
2167 
2168     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2169     acb->need_bh = true;
2170     acb->req.error = -EINPROGRESS;
2171 
2172     co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb);
2173     qemu_coroutine_enter(co);
2174 
2175     bdrv_co_maybe_schedule_bh(acb);
2176     return &acb->common;
2177 }
2178 
2179 static void coroutine_fn bdrv_aio_pdiscard_co_entry(void *opaque)
2180 {
2181     BlockAIOCBCoroutine *acb = opaque;
2182     BlockDriverState *bs = acb->common.bs;
2183 
2184     acb->req.error = bdrv_co_pdiscard(bs, acb->req.offset, acb->req.bytes);
2185     bdrv_co_complete(acb);
2186 }
2187 
2188 BlockAIOCB *bdrv_aio_pdiscard(BlockDriverState *bs, int64_t offset, int count,
2189                               BlockCompletionFunc *cb, void *opaque)
2190 {
2191     Coroutine *co;
2192     BlockAIOCBCoroutine *acb;
2193 
2194     trace_bdrv_aio_pdiscard(bs, offset, count, opaque);
2195 
2196     acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
2197     acb->need_bh = true;
2198     acb->req.error = -EINPROGRESS;
2199     acb->req.offset = offset;
2200     acb->req.bytes = count;
2201     co = qemu_coroutine_create(bdrv_aio_pdiscard_co_entry, acb);
2202     qemu_coroutine_enter(co);
2203 
2204     bdrv_co_maybe_schedule_bh(acb);
2205     return &acb->common;
2206 }
2207 
2208 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
2209                    BlockCompletionFunc *cb, void *opaque)
2210 {
2211     BlockAIOCB *acb;
2212 
2213     acb = g_malloc(aiocb_info->aiocb_size);
2214     acb->aiocb_info = aiocb_info;
2215     acb->bs = bs;
2216     acb->cb = cb;
2217     acb->opaque = opaque;
2218     acb->refcnt = 1;
2219     return acb;
2220 }
2221 
2222 void qemu_aio_ref(void *p)
2223 {
2224     BlockAIOCB *acb = p;
2225     acb->refcnt++;
2226 }
2227 
2228 void qemu_aio_unref(void *p)
2229 {
2230     BlockAIOCB *acb = p;
2231     assert(acb->refcnt > 0);
2232     if (--acb->refcnt == 0) {
2233         g_free(acb);
2234     }
2235 }
2236 
2237 /**************************************************************/
2238 /* Coroutine block device emulation */
2239 
2240 typedef struct FlushCo {
2241     BlockDriverState *bs;
2242     int ret;
2243 } FlushCo;
2244 
2245 
2246 static void coroutine_fn bdrv_flush_co_entry(void *opaque)
2247 {
2248     FlushCo *rwco = opaque;
2249 
2250     rwco->ret = bdrv_co_flush(rwco->bs);
2251 }
2252 
2253 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2254 {
2255     int ret;
2256     BdrvTrackedRequest req;
2257 
2258     if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2259         bdrv_is_sg(bs)) {
2260         return 0;
2261     }
2262 
2263     tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
2264 
2265     int current_gen = bs->write_gen;
2266 
2267     /* Wait until any previous flushes are completed */
2268     while (bs->active_flush_req != NULL) {
2269         qemu_co_queue_wait(&bs->flush_queue);
2270     }
2271 
2272     bs->active_flush_req = &req;
2273 
2274     /* Write back all layers by calling one driver function */
2275     if (bs->drv->bdrv_co_flush) {
2276         ret = bs->drv->bdrv_co_flush(bs);
2277         goto out;
2278     }
2279 
2280     /* Write back cached data to the OS even with cache=unsafe */
2281     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
2282     if (bs->drv->bdrv_co_flush_to_os) {
2283         ret = bs->drv->bdrv_co_flush_to_os(bs);
2284         if (ret < 0) {
2285             goto out;
2286         }
2287     }
2288 
2289     /* But don't actually force it to the disk with cache=unsafe */
2290     if (bs->open_flags & BDRV_O_NO_FLUSH) {
2291         goto flush_parent;
2292     }
2293 
2294     /* Check if we really need to flush anything */
2295     if (bs->flushed_gen == current_gen) {
2296         goto flush_parent;
2297     }
2298 
2299     BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
2300     if (bs->drv->bdrv_co_flush_to_disk) {
2301         ret = bs->drv->bdrv_co_flush_to_disk(bs);
2302     } else if (bs->drv->bdrv_aio_flush) {
2303         BlockAIOCB *acb;
2304         CoroutineIOCompletion co = {
2305             .coroutine = qemu_coroutine_self(),
2306         };
2307 
2308         acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2309         if (acb == NULL) {
2310             ret = -EIO;
2311         } else {
2312             qemu_coroutine_yield();
2313             ret = co.ret;
2314         }
2315     } else {
2316         /*
2317          * Some block drivers always operate in either writethrough or unsafe
2318          * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2319          * know how the server works (because the behaviour is hardcoded or
2320          * depends on server-side configuration), so we can't ensure that
2321          * everything is safe on disk. Returning an error doesn't work because
2322          * that would break guests even if the server operates in writethrough
2323          * mode.
2324          *
2325          * Let's hope the user knows what he's doing.
2326          */
2327         ret = 0;
2328     }
2329 
2330     if (ret < 0) {
2331         goto out;
2332     }
2333 
2334     /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
2335      * in the case of cache=unsafe, so there are no useless flushes.
2336      */
2337 flush_parent:
2338     ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
2339 out:
2340     /* Notify any pending flushes that we have completed */
2341     bs->flushed_gen = current_gen;
2342     bs->active_flush_req = NULL;
2343     /* Return value is ignored - it's ok if wait queue is empty */
2344     qemu_co_queue_next(&bs->flush_queue);
2345 
2346     tracked_request_end(&req);
2347     return ret;
2348 }
2349 
2350 int bdrv_flush(BlockDriverState *bs)
2351 {
2352     Coroutine *co;
2353     FlushCo flush_co = {
2354         .bs = bs,
2355         .ret = NOT_DONE,
2356     };
2357 
2358     if (qemu_in_coroutine()) {
2359         /* Fast-path if already in coroutine context */
2360         bdrv_flush_co_entry(&flush_co);
2361     } else {
2362         AioContext *aio_context = bdrv_get_aio_context(bs);
2363 
2364         co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co);
2365         qemu_coroutine_enter(co);
2366         while (flush_co.ret == NOT_DONE) {
2367             aio_poll(aio_context, true);
2368         }
2369     }
2370 
2371     return flush_co.ret;
2372 }
2373 
2374 typedef struct DiscardCo {
2375     BlockDriverState *bs;
2376     int64_t offset;
2377     int count;
2378     int ret;
2379 } DiscardCo;
2380 static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque)
2381 {
2382     DiscardCo *rwco = opaque;
2383 
2384     rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->count);
2385 }
2386 
2387 int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset,
2388                                   int count)
2389 {
2390     BdrvTrackedRequest req;
2391     int max_pdiscard, ret;
2392     int head, align;
2393 
2394     if (!bs->drv) {
2395         return -ENOMEDIUM;
2396     }
2397 
2398     ret = bdrv_check_byte_request(bs, offset, count);
2399     if (ret < 0) {
2400         return ret;
2401     } else if (bs->read_only) {
2402         return -EPERM;
2403     }
2404     assert(!(bs->open_flags & BDRV_O_INACTIVE));
2405 
2406     /* Do nothing if disabled.  */
2407     if (!(bs->open_flags & BDRV_O_UNMAP)) {
2408         return 0;
2409     }
2410 
2411     if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2412         return 0;
2413     }
2414 
2415     /* Discard is advisory, so ignore any unaligned head or tail */
2416     align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2417     assert(align % bs->bl.request_alignment == 0);
2418     head = offset % align;
2419     if (head) {
2420         head = MIN(count, align - head);
2421         count -= head;
2422         offset += head;
2423     }
2424     count = QEMU_ALIGN_DOWN(count, align);
2425     if (!count) {
2426         return 0;
2427     }
2428 
2429     tracked_request_begin(&req, bs, offset, count, BDRV_TRACKED_DISCARD);
2430 
2431     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req);
2432     if (ret < 0) {
2433         goto out;
2434     }
2435 
2436     max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
2437                                    align);
2438     assert(max_pdiscard);
2439 
2440     while (count > 0) {
2441         int ret;
2442         int num = MIN(count, max_pdiscard);
2443 
2444         if (bs->drv->bdrv_co_pdiscard) {
2445             ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
2446         } else {
2447             BlockAIOCB *acb;
2448             CoroutineIOCompletion co = {
2449                 .coroutine = qemu_coroutine_self(),
2450             };
2451 
2452             acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
2453                                              bdrv_co_io_em_complete, &co);
2454             if (acb == NULL) {
2455                 ret = -EIO;
2456                 goto out;
2457             } else {
2458                 qemu_coroutine_yield();
2459                 ret = co.ret;
2460             }
2461         }
2462         if (ret && ret != -ENOTSUP) {
2463             goto out;
2464         }
2465 
2466         offset += num;
2467         count -= num;
2468     }
2469     ret = 0;
2470 out:
2471     ++bs->write_gen;
2472     bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
2473                    req.bytes >> BDRV_SECTOR_BITS);
2474     tracked_request_end(&req);
2475     return ret;
2476 }
2477 
2478 int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count)
2479 {
2480     Coroutine *co;
2481     DiscardCo rwco = {
2482         .bs = bs,
2483         .offset = offset,
2484         .count = count,
2485         .ret = NOT_DONE,
2486     };
2487 
2488     if (qemu_in_coroutine()) {
2489         /* Fast-path if already in coroutine context */
2490         bdrv_pdiscard_co_entry(&rwco);
2491     } else {
2492         AioContext *aio_context = bdrv_get_aio_context(bs);
2493 
2494         co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco);
2495         qemu_coroutine_enter(co);
2496         while (rwco.ret == NOT_DONE) {
2497             aio_poll(aio_context, true);
2498         }
2499     }
2500 
2501     return rwco.ret;
2502 }
2503 
2504 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
2505 {
2506     BlockDriver *drv = bs->drv;
2507     BdrvTrackedRequest tracked_req;
2508     CoroutineIOCompletion co = {
2509         .coroutine = qemu_coroutine_self(),
2510     };
2511     BlockAIOCB *acb;
2512 
2513     tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
2514     if (!drv || !drv->bdrv_aio_ioctl) {
2515         co.ret = -ENOTSUP;
2516         goto out;
2517     }
2518 
2519     acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
2520     if (!acb) {
2521         co.ret = -ENOTSUP;
2522         goto out;
2523     }
2524     qemu_coroutine_yield();
2525 out:
2526     tracked_request_end(&tracked_req);
2527     return co.ret;
2528 }
2529 
2530 typedef struct {
2531     BlockDriverState *bs;
2532     int req;
2533     void *buf;
2534     int ret;
2535 } BdrvIoctlCoData;
2536 
2537 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
2538 {
2539     BdrvIoctlCoData *data = opaque;
2540     data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
2541 }
2542 
2543 /* needed for generic scsi interface */
2544 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
2545 {
2546     BdrvIoctlCoData data = {
2547         .bs = bs,
2548         .req = req,
2549         .buf = buf,
2550         .ret = -EINPROGRESS,
2551     };
2552 
2553     if (qemu_in_coroutine()) {
2554         /* Fast-path if already in coroutine context */
2555         bdrv_co_ioctl_entry(&data);
2556     } else {
2557         Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry, &data);
2558 
2559         qemu_coroutine_enter(co);
2560         while (data.ret == -EINPROGRESS) {
2561             aio_poll(bdrv_get_aio_context(bs), true);
2562         }
2563     }
2564     return data.ret;
2565 }
2566 
2567 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
2568 {
2569     BlockAIOCBCoroutine *acb = opaque;
2570     acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
2571                                       acb->req.req, acb->req.buf);
2572     bdrv_co_complete(acb);
2573 }
2574 
2575 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
2576         unsigned long int req, void *buf,
2577         BlockCompletionFunc *cb, void *opaque)
2578 {
2579     BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
2580                                             bs, cb, opaque);
2581     Coroutine *co;
2582 
2583     acb->need_bh = true;
2584     acb->req.error = -EINPROGRESS;
2585     acb->req.req = req;
2586     acb->req.buf = buf;
2587     co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry, acb);
2588     qemu_coroutine_enter(co);
2589 
2590     bdrv_co_maybe_schedule_bh(acb);
2591     return &acb->common;
2592 }
2593 
2594 void *qemu_blockalign(BlockDriverState *bs, size_t size)
2595 {
2596     return qemu_memalign(bdrv_opt_mem_align(bs), size);
2597 }
2598 
2599 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
2600 {
2601     return memset(qemu_blockalign(bs, size), 0, size);
2602 }
2603 
2604 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
2605 {
2606     size_t align = bdrv_opt_mem_align(bs);
2607 
2608     /* Ensure that NULL is never returned on success */
2609     assert(align > 0);
2610     if (size == 0) {
2611         size = align;
2612     }
2613 
2614     return qemu_try_memalign(align, size);
2615 }
2616 
2617 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
2618 {
2619     void *mem = qemu_try_blockalign(bs, size);
2620 
2621     if (mem) {
2622         memset(mem, 0, size);
2623     }
2624 
2625     return mem;
2626 }
2627 
2628 /*
2629  * Check if all memory in this vector is sector aligned.
2630  */
2631 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
2632 {
2633     int i;
2634     size_t alignment = bdrv_min_mem_align(bs);
2635 
2636     for (i = 0; i < qiov->niov; i++) {
2637         if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
2638             return false;
2639         }
2640         if (qiov->iov[i].iov_len % alignment) {
2641             return false;
2642         }
2643     }
2644 
2645     return true;
2646 }
2647 
2648 void bdrv_add_before_write_notifier(BlockDriverState *bs,
2649                                     NotifierWithReturn *notifier)
2650 {
2651     notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
2652 }
2653 
2654 void bdrv_io_plug(BlockDriverState *bs)
2655 {
2656     BdrvChild *child;
2657 
2658     QLIST_FOREACH(child, &bs->children, next) {
2659         bdrv_io_plug(child->bs);
2660     }
2661 
2662     if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
2663         BlockDriver *drv = bs->drv;
2664         if (drv && drv->bdrv_io_plug) {
2665             drv->bdrv_io_plug(bs);
2666         }
2667     }
2668 }
2669 
2670 void bdrv_io_unplug(BlockDriverState *bs)
2671 {
2672     BdrvChild *child;
2673 
2674     assert(bs->io_plugged);
2675     if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
2676         BlockDriver *drv = bs->drv;
2677         if (drv && drv->bdrv_io_unplug) {
2678             drv->bdrv_io_unplug(bs);
2679         }
2680     }
2681 
2682     QLIST_FOREACH(child, &bs->children, next) {
2683         bdrv_io_unplug(child->bs);
2684     }
2685 }
2686 
2687 void bdrv_io_unplugged_begin(BlockDriverState *bs)
2688 {
2689     BdrvChild *child;
2690 
2691     if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
2692         BlockDriver *drv = bs->drv;
2693         if (drv && drv->bdrv_io_unplug) {
2694             drv->bdrv_io_unplug(bs);
2695         }
2696     }
2697 
2698     QLIST_FOREACH(child, &bs->children, next) {
2699         bdrv_io_unplugged_begin(child->bs);
2700     }
2701 }
2702 
2703 void bdrv_io_unplugged_end(BlockDriverState *bs)
2704 {
2705     BdrvChild *child;
2706 
2707     assert(bs->io_plug_disabled);
2708     QLIST_FOREACH(child, &bs->children, next) {
2709         bdrv_io_unplugged_end(child->bs);
2710     }
2711 
2712     if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
2713         BlockDriver *drv = bs->drv;
2714         if (drv && drv->bdrv_io_plug) {
2715             drv->bdrv_io_plug(bs);
2716         }
2717     }
2718 }
2719