1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/blockjob.h" 29 #include "block/block_int.h" 30 #include "qemu/cutils.h" 31 #include "qapi/error.h" 32 #include "qemu/error-report.h" 33 34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 35 36 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child, 37 int64_t offset, 38 QEMUIOVector *qiov, 39 BdrvRequestFlags flags, 40 BlockCompletionFunc *cb, 41 void *opaque, 42 bool is_write); 43 static void coroutine_fn bdrv_co_do_rw(void *opaque); 44 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 45 int64_t offset, int count, BdrvRequestFlags flags); 46 47 static void bdrv_parent_drained_begin(BlockDriverState *bs) 48 { 49 BdrvChild *c; 50 51 QLIST_FOREACH(c, &bs->parents, next_parent) { 52 if (c->role->drained_begin) { 53 c->role->drained_begin(c); 54 } 55 } 56 } 57 58 static void bdrv_parent_drained_end(BlockDriverState *bs) 59 { 60 BdrvChild *c; 61 62 QLIST_FOREACH(c, &bs->parents, next_parent) { 63 if (c->role->drained_end) { 64 c->role->drained_end(c); 65 } 66 } 67 } 68 69 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 70 { 71 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 72 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 73 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 74 src->opt_mem_alignment); 75 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 76 src->min_mem_alignment); 77 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 78 } 79 80 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 81 { 82 BlockDriver *drv = bs->drv; 83 Error *local_err = NULL; 84 85 memset(&bs->bl, 0, sizeof(bs->bl)); 86 87 if (!drv) { 88 return; 89 } 90 91 /* Default alignment based on whether driver has byte interface */ 92 bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512; 93 94 /* Take some limits from the children as a default */ 95 if (bs->file) { 96 bdrv_refresh_limits(bs->file->bs, &local_err); 97 if (local_err) { 98 error_propagate(errp, local_err); 99 return; 100 } 101 bdrv_merge_limits(&bs->bl, &bs->file->bs->bl); 102 } else { 103 bs->bl.min_mem_alignment = 512; 104 bs->bl.opt_mem_alignment = getpagesize(); 105 106 /* Safe default since most protocols use readv()/writev()/etc */ 107 bs->bl.max_iov = IOV_MAX; 108 } 109 110 if (bs->backing) { 111 bdrv_refresh_limits(bs->backing->bs, &local_err); 112 if (local_err) { 113 error_propagate(errp, local_err); 114 return; 115 } 116 bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl); 117 } 118 119 /* Then let the driver override it */ 120 if (drv->bdrv_refresh_limits) { 121 drv->bdrv_refresh_limits(bs, errp); 122 } 123 } 124 125 /** 126 * The copy-on-read flag is actually a reference count so multiple users may 127 * use the feature without worrying about clobbering its previous state. 128 * Copy-on-read stays enabled until all users have called to disable it. 129 */ 130 void bdrv_enable_copy_on_read(BlockDriverState *bs) 131 { 132 bs->copy_on_read++; 133 } 134 135 void bdrv_disable_copy_on_read(BlockDriverState *bs) 136 { 137 assert(bs->copy_on_read > 0); 138 bs->copy_on_read--; 139 } 140 141 /* Check if any requests are in-flight (including throttled requests) */ 142 bool bdrv_requests_pending(BlockDriverState *bs) 143 { 144 BdrvChild *child; 145 146 if (!QLIST_EMPTY(&bs->tracked_requests)) { 147 return true; 148 } 149 150 QLIST_FOREACH(child, &bs->children, next) { 151 if (bdrv_requests_pending(child->bs)) { 152 return true; 153 } 154 } 155 156 return false; 157 } 158 159 static void bdrv_drain_recurse(BlockDriverState *bs) 160 { 161 BdrvChild *child; 162 163 if (bs->drv && bs->drv->bdrv_drain) { 164 bs->drv->bdrv_drain(bs); 165 } 166 QLIST_FOREACH(child, &bs->children, next) { 167 bdrv_drain_recurse(child->bs); 168 } 169 } 170 171 typedef struct { 172 Coroutine *co; 173 BlockDriverState *bs; 174 QEMUBH *bh; 175 bool done; 176 } BdrvCoDrainData; 177 178 static void bdrv_drain_poll(BlockDriverState *bs) 179 { 180 bool busy = true; 181 182 while (busy) { 183 /* Keep iterating */ 184 busy = bdrv_requests_pending(bs); 185 busy |= aio_poll(bdrv_get_aio_context(bs), busy); 186 } 187 } 188 189 static void bdrv_co_drain_bh_cb(void *opaque) 190 { 191 BdrvCoDrainData *data = opaque; 192 Coroutine *co = data->co; 193 194 qemu_bh_delete(data->bh); 195 bdrv_drain_poll(data->bs); 196 data->done = true; 197 qemu_coroutine_enter(co); 198 } 199 200 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs) 201 { 202 BdrvCoDrainData data; 203 204 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 205 * other coroutines run if they were queued from 206 * qemu_co_queue_run_restart(). */ 207 208 assert(qemu_in_coroutine()); 209 data = (BdrvCoDrainData) { 210 .co = qemu_coroutine_self(), 211 .bs = bs, 212 .done = false, 213 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data), 214 }; 215 qemu_bh_schedule(data.bh); 216 217 qemu_coroutine_yield(); 218 /* If we are resumed from some other event (such as an aio completion or a 219 * timer callback), it is a bug in the caller that should be fixed. */ 220 assert(data.done); 221 } 222 223 void bdrv_drained_begin(BlockDriverState *bs) 224 { 225 if (!bs->quiesce_counter++) { 226 aio_disable_external(bdrv_get_aio_context(bs)); 227 bdrv_parent_drained_begin(bs); 228 } 229 230 bdrv_io_unplugged_begin(bs); 231 bdrv_drain_recurse(bs); 232 if (qemu_in_coroutine()) { 233 bdrv_co_yield_to_drain(bs); 234 } else { 235 bdrv_drain_poll(bs); 236 } 237 bdrv_io_unplugged_end(bs); 238 } 239 240 void bdrv_drained_end(BlockDriverState *bs) 241 { 242 assert(bs->quiesce_counter > 0); 243 if (--bs->quiesce_counter > 0) { 244 return; 245 } 246 247 bdrv_parent_drained_end(bs); 248 aio_enable_external(bdrv_get_aio_context(bs)); 249 } 250 251 /* 252 * Wait for pending requests to complete on a single BlockDriverState subtree, 253 * and suspend block driver's internal I/O until next request arrives. 254 * 255 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 256 * AioContext. 257 * 258 * Only this BlockDriverState's AioContext is run, so in-flight requests must 259 * not depend on events in other AioContexts. In that case, use 260 * bdrv_drain_all() instead. 261 */ 262 void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 263 { 264 assert(qemu_in_coroutine()); 265 bdrv_drained_begin(bs); 266 bdrv_drained_end(bs); 267 } 268 269 void bdrv_drain(BlockDriverState *bs) 270 { 271 bdrv_drained_begin(bs); 272 bdrv_drained_end(bs); 273 } 274 275 /* 276 * Wait for pending requests to complete across all BlockDriverStates 277 * 278 * This function does not flush data to disk, use bdrv_flush_all() for that 279 * after calling this function. 280 */ 281 void bdrv_drain_all(void) 282 { 283 /* Always run first iteration so any pending completion BHs run */ 284 bool busy = true; 285 BlockDriverState *bs; 286 BdrvNextIterator it; 287 BlockJob *job = NULL; 288 GSList *aio_ctxs = NULL, *ctx; 289 290 while ((job = block_job_next(job))) { 291 AioContext *aio_context = blk_get_aio_context(job->blk); 292 293 aio_context_acquire(aio_context); 294 block_job_pause(job); 295 aio_context_release(aio_context); 296 } 297 298 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 299 AioContext *aio_context = bdrv_get_aio_context(bs); 300 301 aio_context_acquire(aio_context); 302 bdrv_parent_drained_begin(bs); 303 bdrv_io_unplugged_begin(bs); 304 bdrv_drain_recurse(bs); 305 aio_context_release(aio_context); 306 307 if (!g_slist_find(aio_ctxs, aio_context)) { 308 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 309 } 310 } 311 312 /* Note that completion of an asynchronous I/O operation can trigger any 313 * number of other I/O operations on other devices---for example a 314 * coroutine can submit an I/O request to another device in response to 315 * request completion. Therefore we must keep looping until there was no 316 * more activity rather than simply draining each device independently. 317 */ 318 while (busy) { 319 busy = false; 320 321 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 322 AioContext *aio_context = ctx->data; 323 324 aio_context_acquire(aio_context); 325 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 326 if (aio_context == bdrv_get_aio_context(bs)) { 327 if (bdrv_requests_pending(bs)) { 328 busy = true; 329 aio_poll(aio_context, busy); 330 } 331 } 332 } 333 busy |= aio_poll(aio_context, false); 334 aio_context_release(aio_context); 335 } 336 } 337 338 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 339 AioContext *aio_context = bdrv_get_aio_context(bs); 340 341 aio_context_acquire(aio_context); 342 bdrv_io_unplugged_end(bs); 343 bdrv_parent_drained_end(bs); 344 aio_context_release(aio_context); 345 } 346 g_slist_free(aio_ctxs); 347 348 job = NULL; 349 while ((job = block_job_next(job))) { 350 AioContext *aio_context = blk_get_aio_context(job->blk); 351 352 aio_context_acquire(aio_context); 353 block_job_resume(job); 354 aio_context_release(aio_context); 355 } 356 } 357 358 /** 359 * Remove an active request from the tracked requests list 360 * 361 * This function should be called when a tracked request is completing. 362 */ 363 static void tracked_request_end(BdrvTrackedRequest *req) 364 { 365 if (req->serialising) { 366 req->bs->serialising_in_flight--; 367 } 368 369 QLIST_REMOVE(req, list); 370 qemu_co_queue_restart_all(&req->wait_queue); 371 } 372 373 /** 374 * Add an active request to the tracked requests list 375 */ 376 static void tracked_request_begin(BdrvTrackedRequest *req, 377 BlockDriverState *bs, 378 int64_t offset, 379 unsigned int bytes, 380 enum BdrvTrackedRequestType type) 381 { 382 *req = (BdrvTrackedRequest){ 383 .bs = bs, 384 .offset = offset, 385 .bytes = bytes, 386 .type = type, 387 .co = qemu_coroutine_self(), 388 .serialising = false, 389 .overlap_offset = offset, 390 .overlap_bytes = bytes, 391 }; 392 393 qemu_co_queue_init(&req->wait_queue); 394 395 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 396 } 397 398 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 399 { 400 int64_t overlap_offset = req->offset & ~(align - 1); 401 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 402 - overlap_offset; 403 404 if (!req->serialising) { 405 req->bs->serialising_in_flight++; 406 req->serialising = true; 407 } 408 409 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 410 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 411 } 412 413 /** 414 * Round a region to cluster boundaries (sector-based) 415 */ 416 void bdrv_round_sectors_to_clusters(BlockDriverState *bs, 417 int64_t sector_num, int nb_sectors, 418 int64_t *cluster_sector_num, 419 int *cluster_nb_sectors) 420 { 421 BlockDriverInfo bdi; 422 423 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 424 *cluster_sector_num = sector_num; 425 *cluster_nb_sectors = nb_sectors; 426 } else { 427 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 428 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 429 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 430 nb_sectors, c); 431 } 432 } 433 434 /** 435 * Round a region to cluster boundaries 436 */ 437 void bdrv_round_to_clusters(BlockDriverState *bs, 438 int64_t offset, unsigned int bytes, 439 int64_t *cluster_offset, 440 unsigned int *cluster_bytes) 441 { 442 BlockDriverInfo bdi; 443 444 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 445 *cluster_offset = offset; 446 *cluster_bytes = bytes; 447 } else { 448 int64_t c = bdi.cluster_size; 449 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 450 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 451 } 452 } 453 454 static int bdrv_get_cluster_size(BlockDriverState *bs) 455 { 456 BlockDriverInfo bdi; 457 int ret; 458 459 ret = bdrv_get_info(bs, &bdi); 460 if (ret < 0 || bdi.cluster_size == 0) { 461 return bs->bl.request_alignment; 462 } else { 463 return bdi.cluster_size; 464 } 465 } 466 467 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 468 int64_t offset, unsigned int bytes) 469 { 470 /* aaaa bbbb */ 471 if (offset >= req->overlap_offset + req->overlap_bytes) { 472 return false; 473 } 474 /* bbbb aaaa */ 475 if (req->overlap_offset >= offset + bytes) { 476 return false; 477 } 478 return true; 479 } 480 481 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 482 { 483 BlockDriverState *bs = self->bs; 484 BdrvTrackedRequest *req; 485 bool retry; 486 bool waited = false; 487 488 if (!bs->serialising_in_flight) { 489 return false; 490 } 491 492 do { 493 retry = false; 494 QLIST_FOREACH(req, &bs->tracked_requests, list) { 495 if (req == self || (!req->serialising && !self->serialising)) { 496 continue; 497 } 498 if (tracked_request_overlaps(req, self->overlap_offset, 499 self->overlap_bytes)) 500 { 501 /* Hitting this means there was a reentrant request, for 502 * example, a block driver issuing nested requests. This must 503 * never happen since it means deadlock. 504 */ 505 assert(qemu_coroutine_self() != req->co); 506 507 /* If the request is already (indirectly) waiting for us, or 508 * will wait for us as soon as it wakes up, then just go on 509 * (instead of producing a deadlock in the former case). */ 510 if (!req->waiting_for) { 511 self->waiting_for = req; 512 qemu_co_queue_wait(&req->wait_queue); 513 self->waiting_for = NULL; 514 retry = true; 515 waited = true; 516 break; 517 } 518 } 519 } 520 } while (retry); 521 522 return waited; 523 } 524 525 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 526 size_t size) 527 { 528 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 529 return -EIO; 530 } 531 532 if (!bdrv_is_inserted(bs)) { 533 return -ENOMEDIUM; 534 } 535 536 if (offset < 0) { 537 return -EIO; 538 } 539 540 return 0; 541 } 542 543 typedef struct RwCo { 544 BdrvChild *child; 545 int64_t offset; 546 QEMUIOVector *qiov; 547 bool is_write; 548 int ret; 549 BdrvRequestFlags flags; 550 } RwCo; 551 552 static void coroutine_fn bdrv_rw_co_entry(void *opaque) 553 { 554 RwCo *rwco = opaque; 555 556 if (!rwco->is_write) { 557 rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset, 558 rwco->qiov->size, rwco->qiov, 559 rwco->flags); 560 } else { 561 rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset, 562 rwco->qiov->size, rwco->qiov, 563 rwco->flags); 564 } 565 } 566 567 /* 568 * Process a vectored synchronous request using coroutines 569 */ 570 static int bdrv_prwv_co(BdrvChild *child, int64_t offset, 571 QEMUIOVector *qiov, bool is_write, 572 BdrvRequestFlags flags) 573 { 574 Coroutine *co; 575 RwCo rwco = { 576 .child = child, 577 .offset = offset, 578 .qiov = qiov, 579 .is_write = is_write, 580 .ret = NOT_DONE, 581 .flags = flags, 582 }; 583 584 if (qemu_in_coroutine()) { 585 /* Fast-path if already in coroutine context */ 586 bdrv_rw_co_entry(&rwco); 587 } else { 588 AioContext *aio_context = bdrv_get_aio_context(child->bs); 589 590 co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco); 591 qemu_coroutine_enter(co); 592 while (rwco.ret == NOT_DONE) { 593 aio_poll(aio_context, true); 594 } 595 } 596 return rwco.ret; 597 } 598 599 /* 600 * Process a synchronous request using coroutines 601 */ 602 static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf, 603 int nb_sectors, bool is_write, BdrvRequestFlags flags) 604 { 605 QEMUIOVector qiov; 606 struct iovec iov = { 607 .iov_base = (void *)buf, 608 .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 609 }; 610 611 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 612 return -EINVAL; 613 } 614 615 qemu_iovec_init_external(&qiov, &iov, 1); 616 return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS, 617 &qiov, is_write, flags); 618 } 619 620 /* return < 0 if error. See bdrv_write() for the return codes */ 621 int bdrv_read(BdrvChild *child, int64_t sector_num, 622 uint8_t *buf, int nb_sectors) 623 { 624 return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0); 625 } 626 627 /* Return < 0 if error. Important errors are: 628 -EIO generic I/O error (may happen for all errors) 629 -ENOMEDIUM No media inserted. 630 -EINVAL Invalid sector number or nb_sectors 631 -EACCES Trying to write a read-only device 632 */ 633 int bdrv_write(BdrvChild *child, int64_t sector_num, 634 const uint8_t *buf, int nb_sectors) 635 { 636 return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 637 } 638 639 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 640 int count, BdrvRequestFlags flags) 641 { 642 QEMUIOVector qiov; 643 struct iovec iov = { 644 .iov_base = NULL, 645 .iov_len = count, 646 }; 647 648 qemu_iovec_init_external(&qiov, &iov, 1); 649 return bdrv_prwv_co(child, offset, &qiov, true, 650 BDRV_REQ_ZERO_WRITE | flags); 651 } 652 653 /* 654 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 655 * The operation is sped up by checking the block status and only writing 656 * zeroes to the device if they currently do not return zeroes. Optional 657 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 658 * BDRV_REQ_FUA). 659 * 660 * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 661 */ 662 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 663 { 664 int64_t target_sectors, ret, nb_sectors, sector_num = 0; 665 BlockDriverState *bs = child->bs; 666 BlockDriverState *file; 667 int n; 668 669 target_sectors = bdrv_nb_sectors(bs); 670 if (target_sectors < 0) { 671 return target_sectors; 672 } 673 674 for (;;) { 675 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 676 if (nb_sectors <= 0) { 677 return 0; 678 } 679 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file); 680 if (ret < 0) { 681 error_report("error getting block status at sector %" PRId64 ": %s", 682 sector_num, strerror(-ret)); 683 return ret; 684 } 685 if (ret & BDRV_BLOCK_ZERO) { 686 sector_num += n; 687 continue; 688 } 689 ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS, 690 n << BDRV_SECTOR_BITS, flags); 691 if (ret < 0) { 692 error_report("error writing zeroes at sector %" PRId64 ": %s", 693 sector_num, strerror(-ret)); 694 return ret; 695 } 696 sector_num += n; 697 } 698 } 699 700 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 701 { 702 int ret; 703 704 ret = bdrv_prwv_co(child, offset, qiov, false, 0); 705 if (ret < 0) { 706 return ret; 707 } 708 709 return qiov->size; 710 } 711 712 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 713 { 714 QEMUIOVector qiov; 715 struct iovec iov = { 716 .iov_base = (void *)buf, 717 .iov_len = bytes, 718 }; 719 720 if (bytes < 0) { 721 return -EINVAL; 722 } 723 724 qemu_iovec_init_external(&qiov, &iov, 1); 725 return bdrv_preadv(child, offset, &qiov); 726 } 727 728 int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 729 { 730 int ret; 731 732 ret = bdrv_prwv_co(child, offset, qiov, true, 0); 733 if (ret < 0) { 734 return ret; 735 } 736 737 return qiov->size; 738 } 739 740 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 741 { 742 QEMUIOVector qiov; 743 struct iovec iov = { 744 .iov_base = (void *) buf, 745 .iov_len = bytes, 746 }; 747 748 if (bytes < 0) { 749 return -EINVAL; 750 } 751 752 qemu_iovec_init_external(&qiov, &iov, 1); 753 return bdrv_pwritev(child, offset, &qiov); 754 } 755 756 /* 757 * Writes to the file and ensures that no writes are reordered across this 758 * request (acts as a barrier) 759 * 760 * Returns 0 on success, -errno in error cases. 761 */ 762 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 763 const void *buf, int count) 764 { 765 int ret; 766 767 ret = bdrv_pwrite(child, offset, buf, count); 768 if (ret < 0) { 769 return ret; 770 } 771 772 ret = bdrv_flush(child->bs); 773 if (ret < 0) { 774 return ret; 775 } 776 777 return 0; 778 } 779 780 typedef struct CoroutineIOCompletion { 781 Coroutine *coroutine; 782 int ret; 783 } CoroutineIOCompletion; 784 785 static void bdrv_co_io_em_complete(void *opaque, int ret) 786 { 787 CoroutineIOCompletion *co = opaque; 788 789 co->ret = ret; 790 qemu_coroutine_enter(co->coroutine); 791 } 792 793 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 794 uint64_t offset, uint64_t bytes, 795 QEMUIOVector *qiov, int flags) 796 { 797 BlockDriver *drv = bs->drv; 798 int64_t sector_num; 799 unsigned int nb_sectors; 800 801 assert(!(flags & ~BDRV_REQ_MASK)); 802 803 if (drv->bdrv_co_preadv) { 804 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 805 } 806 807 sector_num = offset >> BDRV_SECTOR_BITS; 808 nb_sectors = bytes >> BDRV_SECTOR_BITS; 809 810 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 811 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 812 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 813 814 if (drv->bdrv_co_readv) { 815 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 816 } else { 817 BlockAIOCB *acb; 818 CoroutineIOCompletion co = { 819 .coroutine = qemu_coroutine_self(), 820 }; 821 822 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 823 bdrv_co_io_em_complete, &co); 824 if (acb == NULL) { 825 return -EIO; 826 } else { 827 qemu_coroutine_yield(); 828 return co.ret; 829 } 830 } 831 } 832 833 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 834 uint64_t offset, uint64_t bytes, 835 QEMUIOVector *qiov, int flags) 836 { 837 BlockDriver *drv = bs->drv; 838 int64_t sector_num; 839 unsigned int nb_sectors; 840 int ret; 841 842 assert(!(flags & ~BDRV_REQ_MASK)); 843 844 if (drv->bdrv_co_pwritev) { 845 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 846 flags & bs->supported_write_flags); 847 flags &= ~bs->supported_write_flags; 848 goto emulate_flags; 849 } 850 851 sector_num = offset >> BDRV_SECTOR_BITS; 852 nb_sectors = bytes >> BDRV_SECTOR_BITS; 853 854 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 855 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 856 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 857 858 if (drv->bdrv_co_writev_flags) { 859 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 860 flags & bs->supported_write_flags); 861 flags &= ~bs->supported_write_flags; 862 } else if (drv->bdrv_co_writev) { 863 assert(!bs->supported_write_flags); 864 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 865 } else { 866 BlockAIOCB *acb; 867 CoroutineIOCompletion co = { 868 .coroutine = qemu_coroutine_self(), 869 }; 870 871 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 872 bdrv_co_io_em_complete, &co); 873 if (acb == NULL) { 874 ret = -EIO; 875 } else { 876 qemu_coroutine_yield(); 877 ret = co.ret; 878 } 879 } 880 881 emulate_flags: 882 if (ret == 0 && (flags & BDRV_REQ_FUA)) { 883 ret = bdrv_co_flush(bs); 884 } 885 886 return ret; 887 } 888 889 static int coroutine_fn 890 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 891 uint64_t bytes, QEMUIOVector *qiov) 892 { 893 BlockDriver *drv = bs->drv; 894 895 if (!drv->bdrv_co_pwritev_compressed) { 896 return -ENOTSUP; 897 } 898 899 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 900 } 901 902 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs, 903 int64_t offset, unsigned int bytes, QEMUIOVector *qiov) 904 { 905 /* Perform I/O through a temporary buffer so that users who scribble over 906 * their read buffer while the operation is in progress do not end up 907 * modifying the image file. This is critical for zero-copy guest I/O 908 * where anything might happen inside guest memory. 909 */ 910 void *bounce_buffer; 911 912 BlockDriver *drv = bs->drv; 913 struct iovec iov; 914 QEMUIOVector bounce_qiov; 915 int64_t cluster_offset; 916 unsigned int cluster_bytes; 917 size_t skip_bytes; 918 int ret; 919 920 /* Cover entire cluster so no additional backing file I/O is required when 921 * allocating cluster in the image file. 922 */ 923 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 924 925 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 926 cluster_offset, cluster_bytes); 927 928 iov.iov_len = cluster_bytes; 929 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 930 if (bounce_buffer == NULL) { 931 ret = -ENOMEM; 932 goto err; 933 } 934 935 qemu_iovec_init_external(&bounce_qiov, &iov, 1); 936 937 ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes, 938 &bounce_qiov, 0); 939 if (ret < 0) { 940 goto err; 941 } 942 943 if (drv->bdrv_co_pwrite_zeroes && 944 buffer_is_zero(bounce_buffer, iov.iov_len)) { 945 /* FIXME: Should we (perhaps conditionally) be setting 946 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 947 * that still correctly reads as zero? */ 948 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0); 949 } else { 950 /* This does not change the data on the disk, it is not necessary 951 * to flush even in cache=writethrough mode. 952 */ 953 ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes, 954 &bounce_qiov, 0); 955 } 956 957 if (ret < 0) { 958 /* It might be okay to ignore write errors for guest requests. If this 959 * is a deliberate copy-on-read then we don't want to ignore the error. 960 * Simply report it in all cases. 961 */ 962 goto err; 963 } 964 965 skip_bytes = offset - cluster_offset; 966 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes); 967 968 err: 969 qemu_vfree(bounce_buffer); 970 return ret; 971 } 972 973 /* 974 * Forwards an already correctly aligned request to the BlockDriver. This 975 * handles copy on read, zeroing after EOF, and fragmentation of large 976 * reads; any other features must be implemented by the caller. 977 */ 978 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs, 979 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 980 int64_t align, QEMUIOVector *qiov, int flags) 981 { 982 int64_t total_bytes, max_bytes; 983 int ret = 0; 984 uint64_t bytes_remaining = bytes; 985 int max_transfer; 986 987 assert(is_power_of_2(align)); 988 assert((offset & (align - 1)) == 0); 989 assert((bytes & (align - 1)) == 0); 990 assert(!qiov || bytes == qiov->size); 991 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 992 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 993 align); 994 995 /* TODO: We would need a per-BDS .supported_read_flags and 996 * potential fallback support, if we ever implement any read flags 997 * to pass through to drivers. For now, there aren't any 998 * passthrough flags. */ 999 assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ))); 1000 1001 /* Handle Copy on Read and associated serialisation */ 1002 if (flags & BDRV_REQ_COPY_ON_READ) { 1003 /* If we touch the same cluster it counts as an overlap. This 1004 * guarantees that allocating writes will be serialized and not race 1005 * with each other for the same cluster. For example, in copy-on-read 1006 * it ensures that the CoR read and write operations are atomic and 1007 * guest writes cannot interleave between them. */ 1008 mark_request_serialising(req, bdrv_get_cluster_size(bs)); 1009 } 1010 1011 if (!(flags & BDRV_REQ_NO_SERIALISING)) { 1012 wait_serialising_requests(req); 1013 } 1014 1015 if (flags & BDRV_REQ_COPY_ON_READ) { 1016 int64_t start_sector = offset >> BDRV_SECTOR_BITS; 1017 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1018 unsigned int nb_sectors = end_sector - start_sector; 1019 int pnum; 1020 1021 ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum); 1022 if (ret < 0) { 1023 goto out; 1024 } 1025 1026 if (!ret || pnum != nb_sectors) { 1027 ret = bdrv_co_do_copy_on_readv(bs, offset, bytes, qiov); 1028 goto out; 1029 } 1030 } 1031 1032 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1033 total_bytes = bdrv_getlength(bs); 1034 if (total_bytes < 0) { 1035 ret = total_bytes; 1036 goto out; 1037 } 1038 1039 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1040 if (bytes <= max_bytes && bytes <= max_transfer) { 1041 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 1042 goto out; 1043 } 1044 1045 while (bytes_remaining) { 1046 int num; 1047 1048 if (max_bytes) { 1049 QEMUIOVector local_qiov; 1050 1051 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1052 assert(num); 1053 qemu_iovec_init(&local_qiov, qiov->niov); 1054 qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 1055 1056 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1057 num, &local_qiov, 0); 1058 max_bytes -= num; 1059 qemu_iovec_destroy(&local_qiov); 1060 } else { 1061 num = bytes_remaining; 1062 ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0, 1063 bytes_remaining); 1064 } 1065 if (ret < 0) { 1066 goto out; 1067 } 1068 bytes_remaining -= num; 1069 } 1070 1071 out: 1072 return ret < 0 ? ret : 0; 1073 } 1074 1075 /* 1076 * Handle a read request in coroutine context 1077 */ 1078 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1079 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1080 BdrvRequestFlags flags) 1081 { 1082 BlockDriverState *bs = child->bs; 1083 BlockDriver *drv = bs->drv; 1084 BdrvTrackedRequest req; 1085 1086 uint64_t align = bs->bl.request_alignment; 1087 uint8_t *head_buf = NULL; 1088 uint8_t *tail_buf = NULL; 1089 QEMUIOVector local_qiov; 1090 bool use_local_qiov = false; 1091 int ret; 1092 1093 if (!drv) { 1094 return -ENOMEDIUM; 1095 } 1096 1097 ret = bdrv_check_byte_request(bs, offset, bytes); 1098 if (ret < 0) { 1099 return ret; 1100 } 1101 1102 /* Don't do copy-on-read if we read data before write operation */ 1103 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 1104 flags |= BDRV_REQ_COPY_ON_READ; 1105 } 1106 1107 /* Align read if necessary by padding qiov */ 1108 if (offset & (align - 1)) { 1109 head_buf = qemu_blockalign(bs, align); 1110 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1111 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1112 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1113 use_local_qiov = true; 1114 1115 bytes += offset & (align - 1); 1116 offset = offset & ~(align - 1); 1117 } 1118 1119 if ((offset + bytes) & (align - 1)) { 1120 if (!use_local_qiov) { 1121 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1122 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1123 use_local_qiov = true; 1124 } 1125 tail_buf = qemu_blockalign(bs, align); 1126 qemu_iovec_add(&local_qiov, tail_buf, 1127 align - ((offset + bytes) & (align - 1))); 1128 1129 bytes = ROUND_UP(bytes, align); 1130 } 1131 1132 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1133 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align, 1134 use_local_qiov ? &local_qiov : qiov, 1135 flags); 1136 tracked_request_end(&req); 1137 1138 if (use_local_qiov) { 1139 qemu_iovec_destroy(&local_qiov); 1140 qemu_vfree(head_buf); 1141 qemu_vfree(tail_buf); 1142 } 1143 1144 return ret; 1145 } 1146 1147 static int coroutine_fn bdrv_co_do_readv(BdrvChild *child, 1148 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1149 BdrvRequestFlags flags) 1150 { 1151 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1152 return -EINVAL; 1153 } 1154 1155 return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS, 1156 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1157 } 1158 1159 int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num, 1160 int nb_sectors, QEMUIOVector *qiov) 1161 { 1162 trace_bdrv_co_readv(child->bs, sector_num, nb_sectors); 1163 1164 return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0); 1165 } 1166 1167 /* Maximum buffer for write zeroes fallback, in bytes */ 1168 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 1169 1170 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1171 int64_t offset, int count, BdrvRequestFlags flags) 1172 { 1173 BlockDriver *drv = bs->drv; 1174 QEMUIOVector qiov; 1175 struct iovec iov = {0}; 1176 int ret = 0; 1177 bool need_flush = false; 1178 int head = 0; 1179 int tail = 0; 1180 1181 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1182 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1183 bs->bl.request_alignment); 1184 1185 assert(alignment % bs->bl.request_alignment == 0); 1186 head = offset % alignment; 1187 tail = (offset + count) % alignment; 1188 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1189 assert(max_write_zeroes >= bs->bl.request_alignment); 1190 1191 while (count > 0 && !ret) { 1192 int num = count; 1193 1194 /* Align request. Block drivers can expect the "bulk" of the request 1195 * to be aligned, and that unaligned requests do not cross cluster 1196 * boundaries. 1197 */ 1198 if (head) { 1199 /* Make a small request up to the first aligned sector. */ 1200 num = MIN(count, alignment - head); 1201 head = 0; 1202 } else if (tail && num > alignment) { 1203 /* Shorten the request to the last aligned sector. */ 1204 num -= tail; 1205 } 1206 1207 /* limit request size */ 1208 if (num > max_write_zeroes) { 1209 num = max_write_zeroes; 1210 } 1211 1212 ret = -ENOTSUP; 1213 /* First try the efficient write zeroes operation */ 1214 if (drv->bdrv_co_pwrite_zeroes) { 1215 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1216 flags & bs->supported_zero_flags); 1217 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1218 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1219 need_flush = true; 1220 } 1221 } else { 1222 assert(!bs->supported_zero_flags); 1223 } 1224 1225 if (ret == -ENOTSUP) { 1226 /* Fall back to bounce buffer if write zeroes is unsupported */ 1227 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1228 MAX_WRITE_ZEROES_BOUNCE_BUFFER); 1229 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1230 1231 if ((flags & BDRV_REQ_FUA) && 1232 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1233 /* No need for bdrv_driver_pwrite() to do a fallback 1234 * flush on each chunk; use just one at the end */ 1235 write_flags &= ~BDRV_REQ_FUA; 1236 need_flush = true; 1237 } 1238 num = MIN(num, max_transfer); 1239 iov.iov_len = num; 1240 if (iov.iov_base == NULL) { 1241 iov.iov_base = qemu_try_blockalign(bs, num); 1242 if (iov.iov_base == NULL) { 1243 ret = -ENOMEM; 1244 goto fail; 1245 } 1246 memset(iov.iov_base, 0, num); 1247 } 1248 qemu_iovec_init_external(&qiov, &iov, 1); 1249 1250 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags); 1251 1252 /* Keep bounce buffer around if it is big enough for all 1253 * all future requests. 1254 */ 1255 if (num < max_transfer) { 1256 qemu_vfree(iov.iov_base); 1257 iov.iov_base = NULL; 1258 } 1259 } 1260 1261 offset += num; 1262 count -= num; 1263 } 1264 1265 fail: 1266 if (ret == 0 && need_flush) { 1267 ret = bdrv_co_flush(bs); 1268 } 1269 qemu_vfree(iov.iov_base); 1270 return ret; 1271 } 1272 1273 /* 1274 * Forwards an already correctly aligned write request to the BlockDriver, 1275 * after possibly fragmenting it. 1276 */ 1277 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs, 1278 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1279 int64_t align, QEMUIOVector *qiov, int flags) 1280 { 1281 BlockDriver *drv = bs->drv; 1282 bool waited; 1283 int ret; 1284 1285 int64_t start_sector = offset >> BDRV_SECTOR_BITS; 1286 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1287 uint64_t bytes_remaining = bytes; 1288 int max_transfer; 1289 1290 assert(is_power_of_2(align)); 1291 assert((offset & (align - 1)) == 0); 1292 assert((bytes & (align - 1)) == 0); 1293 assert(!qiov || bytes == qiov->size); 1294 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1295 assert(!(flags & ~BDRV_REQ_MASK)); 1296 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1297 align); 1298 1299 waited = wait_serialising_requests(req); 1300 assert(!waited || !req->serialising); 1301 assert(req->overlap_offset <= offset); 1302 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1303 1304 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 1305 1306 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1307 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 1308 qemu_iovec_is_zero(qiov)) { 1309 flags |= BDRV_REQ_ZERO_WRITE; 1310 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1311 flags |= BDRV_REQ_MAY_UNMAP; 1312 } 1313 } 1314 1315 if (ret < 0) { 1316 /* Do nothing, write notifier decided to fail this request */ 1317 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1318 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1319 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 1320 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 1321 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, qiov); 1322 } else if (bytes <= max_transfer) { 1323 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1324 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 1325 } else { 1326 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1327 while (bytes_remaining) { 1328 int num = MIN(bytes_remaining, max_transfer); 1329 QEMUIOVector local_qiov; 1330 int local_flags = flags; 1331 1332 assert(num); 1333 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 1334 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1335 /* If FUA is going to be emulated by flush, we only 1336 * need to flush on the last iteration */ 1337 local_flags &= ~BDRV_REQ_FUA; 1338 } 1339 qemu_iovec_init(&local_qiov, qiov->niov); 1340 qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num); 1341 1342 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 1343 num, &local_qiov, local_flags); 1344 qemu_iovec_destroy(&local_qiov); 1345 if (ret < 0) { 1346 break; 1347 } 1348 bytes_remaining -= num; 1349 } 1350 } 1351 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 1352 1353 ++bs->write_gen; 1354 bdrv_set_dirty(bs, start_sector, end_sector - start_sector); 1355 1356 if (bs->wr_highest_offset < offset + bytes) { 1357 bs->wr_highest_offset = offset + bytes; 1358 } 1359 1360 if (ret >= 0) { 1361 bs->total_sectors = MAX(bs->total_sectors, end_sector); 1362 ret = 0; 1363 } 1364 1365 return ret; 1366 } 1367 1368 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs, 1369 int64_t offset, 1370 unsigned int bytes, 1371 BdrvRequestFlags flags, 1372 BdrvTrackedRequest *req) 1373 { 1374 uint8_t *buf = NULL; 1375 QEMUIOVector local_qiov; 1376 struct iovec iov; 1377 uint64_t align = bs->bl.request_alignment; 1378 unsigned int head_padding_bytes, tail_padding_bytes; 1379 int ret = 0; 1380 1381 head_padding_bytes = offset & (align - 1); 1382 tail_padding_bytes = align - ((offset + bytes) & (align - 1)); 1383 1384 1385 assert(flags & BDRV_REQ_ZERO_WRITE); 1386 if (head_padding_bytes || tail_padding_bytes) { 1387 buf = qemu_blockalign(bs, align); 1388 iov = (struct iovec) { 1389 .iov_base = buf, 1390 .iov_len = align, 1391 }; 1392 qemu_iovec_init_external(&local_qiov, &iov, 1); 1393 } 1394 if (head_padding_bytes) { 1395 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 1396 1397 /* RMW the unaligned part before head. */ 1398 mark_request_serialising(req, align); 1399 wait_serialising_requests(req); 1400 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1401 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align, 1402 align, &local_qiov, 0); 1403 if (ret < 0) { 1404 goto fail; 1405 } 1406 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1407 1408 memset(buf + head_padding_bytes, 0, zero_bytes); 1409 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align, 1410 align, &local_qiov, 1411 flags & ~BDRV_REQ_ZERO_WRITE); 1412 if (ret < 0) { 1413 goto fail; 1414 } 1415 offset += zero_bytes; 1416 bytes -= zero_bytes; 1417 } 1418 1419 assert(!bytes || (offset & (align - 1)) == 0); 1420 if (bytes >= align) { 1421 /* Write the aligned part in the middle. */ 1422 uint64_t aligned_bytes = bytes & ~(align - 1); 1423 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, align, 1424 NULL, flags); 1425 if (ret < 0) { 1426 goto fail; 1427 } 1428 bytes -= aligned_bytes; 1429 offset += aligned_bytes; 1430 } 1431 1432 assert(!bytes || (offset & (align - 1)) == 0); 1433 if (bytes) { 1434 assert(align == tail_padding_bytes + bytes); 1435 /* RMW the unaligned part after tail. */ 1436 mark_request_serialising(req, align); 1437 wait_serialising_requests(req); 1438 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1439 ret = bdrv_aligned_preadv(bs, req, offset, align, 1440 align, &local_qiov, 0); 1441 if (ret < 0) { 1442 goto fail; 1443 } 1444 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1445 1446 memset(buf, 0, bytes); 1447 ret = bdrv_aligned_pwritev(bs, req, offset, align, align, 1448 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 1449 } 1450 fail: 1451 qemu_vfree(buf); 1452 return ret; 1453 1454 } 1455 1456 /* 1457 * Handle a write request in coroutine context 1458 */ 1459 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 1460 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1461 BdrvRequestFlags flags) 1462 { 1463 BlockDriverState *bs = child->bs; 1464 BdrvTrackedRequest req; 1465 uint64_t align = bs->bl.request_alignment; 1466 uint8_t *head_buf = NULL; 1467 uint8_t *tail_buf = NULL; 1468 QEMUIOVector local_qiov; 1469 bool use_local_qiov = false; 1470 int ret; 1471 1472 if (!bs->drv) { 1473 return -ENOMEDIUM; 1474 } 1475 if (bs->read_only) { 1476 return -EPERM; 1477 } 1478 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1479 1480 ret = bdrv_check_byte_request(bs, offset, bytes); 1481 if (ret < 0) { 1482 return ret; 1483 } 1484 1485 /* 1486 * Align write if necessary by performing a read-modify-write cycle. 1487 * Pad qiov with the read parts and be sure to have a tracked request not 1488 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 1489 */ 1490 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 1491 1492 if (!qiov) { 1493 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req); 1494 goto out; 1495 } 1496 1497 if (offset & (align - 1)) { 1498 QEMUIOVector head_qiov; 1499 struct iovec head_iov; 1500 1501 mark_request_serialising(&req, align); 1502 wait_serialising_requests(&req); 1503 1504 head_buf = qemu_blockalign(bs, align); 1505 head_iov = (struct iovec) { 1506 .iov_base = head_buf, 1507 .iov_len = align, 1508 }; 1509 qemu_iovec_init_external(&head_qiov, &head_iov, 1); 1510 1511 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1512 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align, 1513 align, &head_qiov, 0); 1514 if (ret < 0) { 1515 goto fail; 1516 } 1517 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1518 1519 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1520 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1521 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1522 use_local_qiov = true; 1523 1524 bytes += offset & (align - 1); 1525 offset = offset & ~(align - 1); 1526 1527 /* We have read the tail already if the request is smaller 1528 * than one aligned block. 1529 */ 1530 if (bytes < align) { 1531 qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes); 1532 bytes = align; 1533 } 1534 } 1535 1536 if ((offset + bytes) & (align - 1)) { 1537 QEMUIOVector tail_qiov; 1538 struct iovec tail_iov; 1539 size_t tail_bytes; 1540 bool waited; 1541 1542 mark_request_serialising(&req, align); 1543 waited = wait_serialising_requests(&req); 1544 assert(!waited || !use_local_qiov); 1545 1546 tail_buf = qemu_blockalign(bs, align); 1547 tail_iov = (struct iovec) { 1548 .iov_base = tail_buf, 1549 .iov_len = align, 1550 }; 1551 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 1552 1553 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1554 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align, 1555 align, &tail_qiov, 0); 1556 if (ret < 0) { 1557 goto fail; 1558 } 1559 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1560 1561 if (!use_local_qiov) { 1562 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1563 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1564 use_local_qiov = true; 1565 } 1566 1567 tail_bytes = (offset + bytes) & (align - 1); 1568 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 1569 1570 bytes = ROUND_UP(bytes, align); 1571 } 1572 1573 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, align, 1574 use_local_qiov ? &local_qiov : qiov, 1575 flags); 1576 1577 fail: 1578 1579 if (use_local_qiov) { 1580 qemu_iovec_destroy(&local_qiov); 1581 } 1582 qemu_vfree(head_buf); 1583 qemu_vfree(tail_buf); 1584 out: 1585 tracked_request_end(&req); 1586 return ret; 1587 } 1588 1589 static int coroutine_fn bdrv_co_do_writev(BdrvChild *child, 1590 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1591 BdrvRequestFlags flags) 1592 { 1593 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1594 return -EINVAL; 1595 } 1596 1597 return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS, 1598 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1599 } 1600 1601 int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num, 1602 int nb_sectors, QEMUIOVector *qiov) 1603 { 1604 trace_bdrv_co_writev(child->bs, sector_num, nb_sectors); 1605 1606 return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0); 1607 } 1608 1609 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 1610 int count, BdrvRequestFlags flags) 1611 { 1612 trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags); 1613 1614 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 1615 flags &= ~BDRV_REQ_MAY_UNMAP; 1616 } 1617 1618 return bdrv_co_pwritev(child, offset, count, NULL, 1619 BDRV_REQ_ZERO_WRITE | flags); 1620 } 1621 1622 typedef struct BdrvCoGetBlockStatusData { 1623 BlockDriverState *bs; 1624 BlockDriverState *base; 1625 BlockDriverState **file; 1626 int64_t sector_num; 1627 int nb_sectors; 1628 int *pnum; 1629 int64_t ret; 1630 bool done; 1631 } BdrvCoGetBlockStatusData; 1632 1633 /* 1634 * Returns the allocation status of the specified sectors. 1635 * Drivers not implementing the functionality are assumed to not support 1636 * backing files, hence all their sectors are reported as allocated. 1637 * 1638 * If 'sector_num' is beyond the end of the disk image the return value is 0 1639 * and 'pnum' is set to 0. 1640 * 1641 * 'pnum' is set to the number of sectors (including and immediately following 1642 * the specified sector) that are known to be in the same 1643 * allocated/unallocated state. 1644 * 1645 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 1646 * beyond the end of the disk image it will be clamped. 1647 * 1648 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file' 1649 * points to the BDS which the sector range is allocated in. 1650 */ 1651 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 1652 int64_t sector_num, 1653 int nb_sectors, int *pnum, 1654 BlockDriverState **file) 1655 { 1656 int64_t total_sectors; 1657 int64_t n; 1658 int64_t ret, ret2; 1659 1660 total_sectors = bdrv_nb_sectors(bs); 1661 if (total_sectors < 0) { 1662 return total_sectors; 1663 } 1664 1665 if (sector_num >= total_sectors) { 1666 *pnum = 0; 1667 return 0; 1668 } 1669 1670 n = total_sectors - sector_num; 1671 if (n < nb_sectors) { 1672 nb_sectors = n; 1673 } 1674 1675 if (!bs->drv->bdrv_co_get_block_status) { 1676 *pnum = nb_sectors; 1677 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 1678 if (bs->drv->protocol_name) { 1679 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 1680 } 1681 return ret; 1682 } 1683 1684 *file = NULL; 1685 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 1686 file); 1687 if (ret < 0) { 1688 *pnum = 0; 1689 return ret; 1690 } 1691 1692 if (ret & BDRV_BLOCK_RAW) { 1693 assert(ret & BDRV_BLOCK_OFFSET_VALID); 1694 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 1695 *pnum, pnum, file); 1696 } 1697 1698 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 1699 ret |= BDRV_BLOCK_ALLOCATED; 1700 } else { 1701 if (bdrv_unallocated_blocks_are_zero(bs)) { 1702 ret |= BDRV_BLOCK_ZERO; 1703 } else if (bs->backing) { 1704 BlockDriverState *bs2 = bs->backing->bs; 1705 int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 1706 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 1707 ret |= BDRV_BLOCK_ZERO; 1708 } 1709 } 1710 } 1711 1712 if (*file && *file != bs && 1713 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 1714 (ret & BDRV_BLOCK_OFFSET_VALID)) { 1715 BlockDriverState *file2; 1716 int file_pnum; 1717 1718 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 1719 *pnum, &file_pnum, &file2); 1720 if (ret2 >= 0) { 1721 /* Ignore errors. This is just providing extra information, it 1722 * is useful but not necessary. 1723 */ 1724 if (!file_pnum) { 1725 /* !file_pnum indicates an offset at or beyond the EOF; it is 1726 * perfectly valid for the format block driver to point to such 1727 * offsets, so catch it and mark everything as zero */ 1728 ret |= BDRV_BLOCK_ZERO; 1729 } else { 1730 /* Limit request to the range reported by the protocol driver */ 1731 *pnum = file_pnum; 1732 ret |= (ret2 & BDRV_BLOCK_ZERO); 1733 } 1734 } 1735 } 1736 1737 return ret; 1738 } 1739 1740 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1741 BlockDriverState *base, 1742 int64_t sector_num, 1743 int nb_sectors, 1744 int *pnum, 1745 BlockDriverState **file) 1746 { 1747 BlockDriverState *p; 1748 int64_t ret = 0; 1749 1750 assert(bs != base); 1751 for (p = bs; p != base; p = backing_bs(p)) { 1752 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file); 1753 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1754 break; 1755 } 1756 /* [sector_num, pnum] unallocated on this layer, which could be only 1757 * the first part of [sector_num, nb_sectors]. */ 1758 nb_sectors = MIN(nb_sectors, *pnum); 1759 } 1760 return ret; 1761 } 1762 1763 /* Coroutine wrapper for bdrv_get_block_status_above() */ 1764 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 1765 { 1766 BdrvCoGetBlockStatusData *data = opaque; 1767 1768 data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1769 data->sector_num, 1770 data->nb_sectors, 1771 data->pnum, 1772 data->file); 1773 data->done = true; 1774 } 1775 1776 /* 1777 * Synchronous wrapper around bdrv_co_get_block_status_above(). 1778 * 1779 * See bdrv_co_get_block_status_above() for details. 1780 */ 1781 int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1782 BlockDriverState *base, 1783 int64_t sector_num, 1784 int nb_sectors, int *pnum, 1785 BlockDriverState **file) 1786 { 1787 Coroutine *co; 1788 BdrvCoGetBlockStatusData data = { 1789 .bs = bs, 1790 .base = base, 1791 .file = file, 1792 .sector_num = sector_num, 1793 .nb_sectors = nb_sectors, 1794 .pnum = pnum, 1795 .done = false, 1796 }; 1797 1798 if (qemu_in_coroutine()) { 1799 /* Fast-path if already in coroutine context */ 1800 bdrv_get_block_status_above_co_entry(&data); 1801 } else { 1802 AioContext *aio_context = bdrv_get_aio_context(bs); 1803 1804 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry, 1805 &data); 1806 qemu_coroutine_enter(co); 1807 while (!data.done) { 1808 aio_poll(aio_context, true); 1809 } 1810 } 1811 return data.ret; 1812 } 1813 1814 int64_t bdrv_get_block_status(BlockDriverState *bs, 1815 int64_t sector_num, 1816 int nb_sectors, int *pnum, 1817 BlockDriverState **file) 1818 { 1819 return bdrv_get_block_status_above(bs, backing_bs(bs), 1820 sector_num, nb_sectors, pnum, file); 1821 } 1822 1823 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 1824 int nb_sectors, int *pnum) 1825 { 1826 BlockDriverState *file; 1827 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum, 1828 &file); 1829 if (ret < 0) { 1830 return ret; 1831 } 1832 return !!(ret & BDRV_BLOCK_ALLOCATED); 1833 } 1834 1835 /* 1836 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 1837 * 1838 * Return true if the given sector is allocated in any image between 1839 * BASE and TOP (inclusive). BASE can be NULL to check if the given 1840 * sector is allocated in any image of the chain. Return false otherwise. 1841 * 1842 * 'pnum' is set to the number of sectors (including and immediately following 1843 * the specified sector) that are known to be in the same 1844 * allocated/unallocated state. 1845 * 1846 */ 1847 int bdrv_is_allocated_above(BlockDriverState *top, 1848 BlockDriverState *base, 1849 int64_t sector_num, 1850 int nb_sectors, int *pnum) 1851 { 1852 BlockDriverState *intermediate; 1853 int ret, n = nb_sectors; 1854 1855 intermediate = top; 1856 while (intermediate && intermediate != base) { 1857 int pnum_inter; 1858 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 1859 &pnum_inter); 1860 if (ret < 0) { 1861 return ret; 1862 } else if (ret) { 1863 *pnum = pnum_inter; 1864 return 1; 1865 } 1866 1867 /* 1868 * [sector_num, nb_sectors] is unallocated on top but intermediate 1869 * might have 1870 * 1871 * [sector_num+x, nr_sectors] allocated. 1872 */ 1873 if (n > pnum_inter && 1874 (intermediate == top || 1875 sector_num + pnum_inter < intermediate->total_sectors)) { 1876 n = pnum_inter; 1877 } 1878 1879 intermediate = backing_bs(intermediate); 1880 } 1881 1882 *pnum = n; 1883 return 0; 1884 } 1885 1886 typedef struct BdrvVmstateCo { 1887 BlockDriverState *bs; 1888 QEMUIOVector *qiov; 1889 int64_t pos; 1890 bool is_read; 1891 int ret; 1892 } BdrvVmstateCo; 1893 1894 static int coroutine_fn 1895 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 1896 bool is_read) 1897 { 1898 BlockDriver *drv = bs->drv; 1899 1900 if (!drv) { 1901 return -ENOMEDIUM; 1902 } else if (drv->bdrv_load_vmstate) { 1903 return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos) 1904 : drv->bdrv_save_vmstate(bs, qiov, pos); 1905 } else if (bs->file) { 1906 return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read); 1907 } 1908 1909 return -ENOTSUP; 1910 } 1911 1912 static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque) 1913 { 1914 BdrvVmstateCo *co = opaque; 1915 co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read); 1916 } 1917 1918 static inline int 1919 bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 1920 bool is_read) 1921 { 1922 if (qemu_in_coroutine()) { 1923 return bdrv_co_rw_vmstate(bs, qiov, pos, is_read); 1924 } else { 1925 BdrvVmstateCo data = { 1926 .bs = bs, 1927 .qiov = qiov, 1928 .pos = pos, 1929 .is_read = is_read, 1930 .ret = -EINPROGRESS, 1931 }; 1932 Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data); 1933 1934 qemu_coroutine_enter(co); 1935 while (data.ret == -EINPROGRESS) { 1936 aio_poll(bdrv_get_aio_context(bs), true); 1937 } 1938 return data.ret; 1939 } 1940 } 1941 1942 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 1943 int64_t pos, int size) 1944 { 1945 QEMUIOVector qiov; 1946 struct iovec iov = { 1947 .iov_base = (void *) buf, 1948 .iov_len = size, 1949 }; 1950 int ret; 1951 1952 qemu_iovec_init_external(&qiov, &iov, 1); 1953 1954 ret = bdrv_writev_vmstate(bs, &qiov, pos); 1955 if (ret < 0) { 1956 return ret; 1957 } 1958 1959 return size; 1960 } 1961 1962 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 1963 { 1964 return bdrv_rw_vmstate(bs, qiov, pos, false); 1965 } 1966 1967 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 1968 int64_t pos, int size) 1969 { 1970 QEMUIOVector qiov; 1971 struct iovec iov = { 1972 .iov_base = buf, 1973 .iov_len = size, 1974 }; 1975 int ret; 1976 1977 qemu_iovec_init_external(&qiov, &iov, 1); 1978 ret = bdrv_readv_vmstate(bs, &qiov, pos); 1979 if (ret < 0) { 1980 return ret; 1981 } 1982 1983 return size; 1984 } 1985 1986 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 1987 { 1988 return bdrv_rw_vmstate(bs, qiov, pos, true); 1989 } 1990 1991 /**************************************************************/ 1992 /* async I/Os */ 1993 1994 BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num, 1995 QEMUIOVector *qiov, int nb_sectors, 1996 BlockCompletionFunc *cb, void *opaque) 1997 { 1998 trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque); 1999 2000 assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size); 2001 return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov, 2002 0, cb, opaque, false); 2003 } 2004 2005 BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num, 2006 QEMUIOVector *qiov, int nb_sectors, 2007 BlockCompletionFunc *cb, void *opaque) 2008 { 2009 trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque); 2010 2011 assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size); 2012 return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov, 2013 0, cb, opaque, true); 2014 } 2015 2016 void bdrv_aio_cancel(BlockAIOCB *acb) 2017 { 2018 qemu_aio_ref(acb); 2019 bdrv_aio_cancel_async(acb); 2020 while (acb->refcnt > 1) { 2021 if (acb->aiocb_info->get_aio_context) { 2022 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2023 } else if (acb->bs) { 2024 aio_poll(bdrv_get_aio_context(acb->bs), true); 2025 } else { 2026 abort(); 2027 } 2028 } 2029 qemu_aio_unref(acb); 2030 } 2031 2032 /* Async version of aio cancel. The caller is not blocked if the acb implements 2033 * cancel_async, otherwise we do nothing and let the request normally complete. 2034 * In either case the completion callback must be called. */ 2035 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2036 { 2037 if (acb->aiocb_info->cancel_async) { 2038 acb->aiocb_info->cancel_async(acb); 2039 } 2040 } 2041 2042 /**************************************************************/ 2043 /* async block device emulation */ 2044 2045 typedef struct BlockRequest { 2046 union { 2047 /* Used during read, write, trim */ 2048 struct { 2049 int64_t offset; 2050 int bytes; 2051 int flags; 2052 QEMUIOVector *qiov; 2053 }; 2054 /* Used during ioctl */ 2055 struct { 2056 int req; 2057 void *buf; 2058 }; 2059 }; 2060 BlockCompletionFunc *cb; 2061 void *opaque; 2062 2063 int error; 2064 } BlockRequest; 2065 2066 typedef struct BlockAIOCBCoroutine { 2067 BlockAIOCB common; 2068 BdrvChild *child; 2069 BlockRequest req; 2070 bool is_write; 2071 bool need_bh; 2072 bool *done; 2073 QEMUBH* bh; 2074 } BlockAIOCBCoroutine; 2075 2076 static const AIOCBInfo bdrv_em_co_aiocb_info = { 2077 .aiocb_size = sizeof(BlockAIOCBCoroutine), 2078 }; 2079 2080 static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 2081 { 2082 if (!acb->need_bh) { 2083 acb->common.cb(acb->common.opaque, acb->req.error); 2084 qemu_aio_unref(acb); 2085 } 2086 } 2087 2088 static void bdrv_co_em_bh(void *opaque) 2089 { 2090 BlockAIOCBCoroutine *acb = opaque; 2091 2092 assert(!acb->need_bh); 2093 qemu_bh_delete(acb->bh); 2094 bdrv_co_complete(acb); 2095 } 2096 2097 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 2098 { 2099 acb->need_bh = false; 2100 if (acb->req.error != -EINPROGRESS) { 2101 BlockDriverState *bs = acb->common.bs; 2102 2103 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 2104 qemu_bh_schedule(acb->bh); 2105 } 2106 } 2107 2108 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 2109 static void coroutine_fn bdrv_co_do_rw(void *opaque) 2110 { 2111 BlockAIOCBCoroutine *acb = opaque; 2112 2113 if (!acb->is_write) { 2114 acb->req.error = bdrv_co_preadv(acb->child, acb->req.offset, 2115 acb->req.qiov->size, acb->req.qiov, acb->req.flags); 2116 } else { 2117 acb->req.error = bdrv_co_pwritev(acb->child, acb->req.offset, 2118 acb->req.qiov->size, acb->req.qiov, acb->req.flags); 2119 } 2120 2121 bdrv_co_complete(acb); 2122 } 2123 2124 static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child, 2125 int64_t offset, 2126 QEMUIOVector *qiov, 2127 BdrvRequestFlags flags, 2128 BlockCompletionFunc *cb, 2129 void *opaque, 2130 bool is_write) 2131 { 2132 Coroutine *co; 2133 BlockAIOCBCoroutine *acb; 2134 2135 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque); 2136 acb->child = child; 2137 acb->need_bh = true; 2138 acb->req.error = -EINPROGRESS; 2139 acb->req.offset = offset; 2140 acb->req.qiov = qiov; 2141 acb->req.flags = flags; 2142 acb->is_write = is_write; 2143 2144 co = qemu_coroutine_create(bdrv_co_do_rw, acb); 2145 qemu_coroutine_enter(co); 2146 2147 bdrv_co_maybe_schedule_bh(acb); 2148 return &acb->common; 2149 } 2150 2151 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 2152 { 2153 BlockAIOCBCoroutine *acb = opaque; 2154 BlockDriverState *bs = acb->common.bs; 2155 2156 acb->req.error = bdrv_co_flush(bs); 2157 bdrv_co_complete(acb); 2158 } 2159 2160 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 2161 BlockCompletionFunc *cb, void *opaque) 2162 { 2163 trace_bdrv_aio_flush(bs, opaque); 2164 2165 Coroutine *co; 2166 BlockAIOCBCoroutine *acb; 2167 2168 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2169 acb->need_bh = true; 2170 acb->req.error = -EINPROGRESS; 2171 2172 co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb); 2173 qemu_coroutine_enter(co); 2174 2175 bdrv_co_maybe_schedule_bh(acb); 2176 return &acb->common; 2177 } 2178 2179 static void coroutine_fn bdrv_aio_pdiscard_co_entry(void *opaque) 2180 { 2181 BlockAIOCBCoroutine *acb = opaque; 2182 BlockDriverState *bs = acb->common.bs; 2183 2184 acb->req.error = bdrv_co_pdiscard(bs, acb->req.offset, acb->req.bytes); 2185 bdrv_co_complete(acb); 2186 } 2187 2188 BlockAIOCB *bdrv_aio_pdiscard(BlockDriverState *bs, int64_t offset, int count, 2189 BlockCompletionFunc *cb, void *opaque) 2190 { 2191 Coroutine *co; 2192 BlockAIOCBCoroutine *acb; 2193 2194 trace_bdrv_aio_pdiscard(bs, offset, count, opaque); 2195 2196 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2197 acb->need_bh = true; 2198 acb->req.error = -EINPROGRESS; 2199 acb->req.offset = offset; 2200 acb->req.bytes = count; 2201 co = qemu_coroutine_create(bdrv_aio_pdiscard_co_entry, acb); 2202 qemu_coroutine_enter(co); 2203 2204 bdrv_co_maybe_schedule_bh(acb); 2205 return &acb->common; 2206 } 2207 2208 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs, 2209 BlockCompletionFunc *cb, void *opaque) 2210 { 2211 BlockAIOCB *acb; 2212 2213 acb = g_malloc(aiocb_info->aiocb_size); 2214 acb->aiocb_info = aiocb_info; 2215 acb->bs = bs; 2216 acb->cb = cb; 2217 acb->opaque = opaque; 2218 acb->refcnt = 1; 2219 return acb; 2220 } 2221 2222 void qemu_aio_ref(void *p) 2223 { 2224 BlockAIOCB *acb = p; 2225 acb->refcnt++; 2226 } 2227 2228 void qemu_aio_unref(void *p) 2229 { 2230 BlockAIOCB *acb = p; 2231 assert(acb->refcnt > 0); 2232 if (--acb->refcnt == 0) { 2233 g_free(acb); 2234 } 2235 } 2236 2237 /**************************************************************/ 2238 /* Coroutine block device emulation */ 2239 2240 typedef struct FlushCo { 2241 BlockDriverState *bs; 2242 int ret; 2243 } FlushCo; 2244 2245 2246 static void coroutine_fn bdrv_flush_co_entry(void *opaque) 2247 { 2248 FlushCo *rwco = opaque; 2249 2250 rwco->ret = bdrv_co_flush(rwco->bs); 2251 } 2252 2253 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2254 { 2255 int ret; 2256 BdrvTrackedRequest req; 2257 2258 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2259 bdrv_is_sg(bs)) { 2260 return 0; 2261 } 2262 2263 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH); 2264 2265 int current_gen = bs->write_gen; 2266 2267 /* Wait until any previous flushes are completed */ 2268 while (bs->active_flush_req != NULL) { 2269 qemu_co_queue_wait(&bs->flush_queue); 2270 } 2271 2272 bs->active_flush_req = &req; 2273 2274 /* Write back all layers by calling one driver function */ 2275 if (bs->drv->bdrv_co_flush) { 2276 ret = bs->drv->bdrv_co_flush(bs); 2277 goto out; 2278 } 2279 2280 /* Write back cached data to the OS even with cache=unsafe */ 2281 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 2282 if (bs->drv->bdrv_co_flush_to_os) { 2283 ret = bs->drv->bdrv_co_flush_to_os(bs); 2284 if (ret < 0) { 2285 goto out; 2286 } 2287 } 2288 2289 /* But don't actually force it to the disk with cache=unsafe */ 2290 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2291 goto flush_parent; 2292 } 2293 2294 /* Check if we really need to flush anything */ 2295 if (bs->flushed_gen == current_gen) { 2296 goto flush_parent; 2297 } 2298 2299 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 2300 if (bs->drv->bdrv_co_flush_to_disk) { 2301 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2302 } else if (bs->drv->bdrv_aio_flush) { 2303 BlockAIOCB *acb; 2304 CoroutineIOCompletion co = { 2305 .coroutine = qemu_coroutine_self(), 2306 }; 2307 2308 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2309 if (acb == NULL) { 2310 ret = -EIO; 2311 } else { 2312 qemu_coroutine_yield(); 2313 ret = co.ret; 2314 } 2315 } else { 2316 /* 2317 * Some block drivers always operate in either writethrough or unsafe 2318 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2319 * know how the server works (because the behaviour is hardcoded or 2320 * depends on server-side configuration), so we can't ensure that 2321 * everything is safe on disk. Returning an error doesn't work because 2322 * that would break guests even if the server operates in writethrough 2323 * mode. 2324 * 2325 * Let's hope the user knows what he's doing. 2326 */ 2327 ret = 0; 2328 } 2329 2330 if (ret < 0) { 2331 goto out; 2332 } 2333 2334 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2335 * in the case of cache=unsafe, so there are no useless flushes. 2336 */ 2337 flush_parent: 2338 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2339 out: 2340 /* Notify any pending flushes that we have completed */ 2341 bs->flushed_gen = current_gen; 2342 bs->active_flush_req = NULL; 2343 /* Return value is ignored - it's ok if wait queue is empty */ 2344 qemu_co_queue_next(&bs->flush_queue); 2345 2346 tracked_request_end(&req); 2347 return ret; 2348 } 2349 2350 int bdrv_flush(BlockDriverState *bs) 2351 { 2352 Coroutine *co; 2353 FlushCo flush_co = { 2354 .bs = bs, 2355 .ret = NOT_DONE, 2356 }; 2357 2358 if (qemu_in_coroutine()) { 2359 /* Fast-path if already in coroutine context */ 2360 bdrv_flush_co_entry(&flush_co); 2361 } else { 2362 AioContext *aio_context = bdrv_get_aio_context(bs); 2363 2364 co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co); 2365 qemu_coroutine_enter(co); 2366 while (flush_co.ret == NOT_DONE) { 2367 aio_poll(aio_context, true); 2368 } 2369 } 2370 2371 return flush_co.ret; 2372 } 2373 2374 typedef struct DiscardCo { 2375 BlockDriverState *bs; 2376 int64_t offset; 2377 int count; 2378 int ret; 2379 } DiscardCo; 2380 static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque) 2381 { 2382 DiscardCo *rwco = opaque; 2383 2384 rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->count); 2385 } 2386 2387 int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset, 2388 int count) 2389 { 2390 BdrvTrackedRequest req; 2391 int max_pdiscard, ret; 2392 int head, align; 2393 2394 if (!bs->drv) { 2395 return -ENOMEDIUM; 2396 } 2397 2398 ret = bdrv_check_byte_request(bs, offset, count); 2399 if (ret < 0) { 2400 return ret; 2401 } else if (bs->read_only) { 2402 return -EPERM; 2403 } 2404 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 2405 2406 /* Do nothing if disabled. */ 2407 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2408 return 0; 2409 } 2410 2411 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 2412 return 0; 2413 } 2414 2415 /* Discard is advisory, so ignore any unaligned head or tail */ 2416 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2417 assert(align % bs->bl.request_alignment == 0); 2418 head = offset % align; 2419 if (head) { 2420 head = MIN(count, align - head); 2421 count -= head; 2422 offset += head; 2423 } 2424 count = QEMU_ALIGN_DOWN(count, align); 2425 if (!count) { 2426 return 0; 2427 } 2428 2429 tracked_request_begin(&req, bs, offset, count, BDRV_TRACKED_DISCARD); 2430 2431 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req); 2432 if (ret < 0) { 2433 goto out; 2434 } 2435 2436 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 2437 align); 2438 assert(max_pdiscard); 2439 2440 while (count > 0) { 2441 int ret; 2442 int num = MIN(count, max_pdiscard); 2443 2444 if (bs->drv->bdrv_co_pdiscard) { 2445 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 2446 } else { 2447 BlockAIOCB *acb; 2448 CoroutineIOCompletion co = { 2449 .coroutine = qemu_coroutine_self(), 2450 }; 2451 2452 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 2453 bdrv_co_io_em_complete, &co); 2454 if (acb == NULL) { 2455 ret = -EIO; 2456 goto out; 2457 } else { 2458 qemu_coroutine_yield(); 2459 ret = co.ret; 2460 } 2461 } 2462 if (ret && ret != -ENOTSUP) { 2463 goto out; 2464 } 2465 2466 offset += num; 2467 count -= num; 2468 } 2469 ret = 0; 2470 out: 2471 ++bs->write_gen; 2472 bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS, 2473 req.bytes >> BDRV_SECTOR_BITS); 2474 tracked_request_end(&req); 2475 return ret; 2476 } 2477 2478 int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count) 2479 { 2480 Coroutine *co; 2481 DiscardCo rwco = { 2482 .bs = bs, 2483 .offset = offset, 2484 .count = count, 2485 .ret = NOT_DONE, 2486 }; 2487 2488 if (qemu_in_coroutine()) { 2489 /* Fast-path if already in coroutine context */ 2490 bdrv_pdiscard_co_entry(&rwco); 2491 } else { 2492 AioContext *aio_context = bdrv_get_aio_context(bs); 2493 2494 co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco); 2495 qemu_coroutine_enter(co); 2496 while (rwco.ret == NOT_DONE) { 2497 aio_poll(aio_context, true); 2498 } 2499 } 2500 2501 return rwco.ret; 2502 } 2503 2504 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf) 2505 { 2506 BlockDriver *drv = bs->drv; 2507 BdrvTrackedRequest tracked_req; 2508 CoroutineIOCompletion co = { 2509 .coroutine = qemu_coroutine_self(), 2510 }; 2511 BlockAIOCB *acb; 2512 2513 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL); 2514 if (!drv || !drv->bdrv_aio_ioctl) { 2515 co.ret = -ENOTSUP; 2516 goto out; 2517 } 2518 2519 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 2520 if (!acb) { 2521 co.ret = -ENOTSUP; 2522 goto out; 2523 } 2524 qemu_coroutine_yield(); 2525 out: 2526 tracked_request_end(&tracked_req); 2527 return co.ret; 2528 } 2529 2530 typedef struct { 2531 BlockDriverState *bs; 2532 int req; 2533 void *buf; 2534 int ret; 2535 } BdrvIoctlCoData; 2536 2537 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque) 2538 { 2539 BdrvIoctlCoData *data = opaque; 2540 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf); 2541 } 2542 2543 /* needed for generic scsi interface */ 2544 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) 2545 { 2546 BdrvIoctlCoData data = { 2547 .bs = bs, 2548 .req = req, 2549 .buf = buf, 2550 .ret = -EINPROGRESS, 2551 }; 2552 2553 if (qemu_in_coroutine()) { 2554 /* Fast-path if already in coroutine context */ 2555 bdrv_co_ioctl_entry(&data); 2556 } else { 2557 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry, &data); 2558 2559 qemu_coroutine_enter(co); 2560 while (data.ret == -EINPROGRESS) { 2561 aio_poll(bdrv_get_aio_context(bs), true); 2562 } 2563 } 2564 return data.ret; 2565 } 2566 2567 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque) 2568 { 2569 BlockAIOCBCoroutine *acb = opaque; 2570 acb->req.error = bdrv_co_do_ioctl(acb->common.bs, 2571 acb->req.req, acb->req.buf); 2572 bdrv_co_complete(acb); 2573 } 2574 2575 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs, 2576 unsigned long int req, void *buf, 2577 BlockCompletionFunc *cb, void *opaque) 2578 { 2579 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info, 2580 bs, cb, opaque); 2581 Coroutine *co; 2582 2583 acb->need_bh = true; 2584 acb->req.error = -EINPROGRESS; 2585 acb->req.req = req; 2586 acb->req.buf = buf; 2587 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry, acb); 2588 qemu_coroutine_enter(co); 2589 2590 bdrv_co_maybe_schedule_bh(acb); 2591 return &acb->common; 2592 } 2593 2594 void *qemu_blockalign(BlockDriverState *bs, size_t size) 2595 { 2596 return qemu_memalign(bdrv_opt_mem_align(bs), size); 2597 } 2598 2599 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 2600 { 2601 return memset(qemu_blockalign(bs, size), 0, size); 2602 } 2603 2604 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 2605 { 2606 size_t align = bdrv_opt_mem_align(bs); 2607 2608 /* Ensure that NULL is never returned on success */ 2609 assert(align > 0); 2610 if (size == 0) { 2611 size = align; 2612 } 2613 2614 return qemu_try_memalign(align, size); 2615 } 2616 2617 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 2618 { 2619 void *mem = qemu_try_blockalign(bs, size); 2620 2621 if (mem) { 2622 memset(mem, 0, size); 2623 } 2624 2625 return mem; 2626 } 2627 2628 /* 2629 * Check if all memory in this vector is sector aligned. 2630 */ 2631 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 2632 { 2633 int i; 2634 size_t alignment = bdrv_min_mem_align(bs); 2635 2636 for (i = 0; i < qiov->niov; i++) { 2637 if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 2638 return false; 2639 } 2640 if (qiov->iov[i].iov_len % alignment) { 2641 return false; 2642 } 2643 } 2644 2645 return true; 2646 } 2647 2648 void bdrv_add_before_write_notifier(BlockDriverState *bs, 2649 NotifierWithReturn *notifier) 2650 { 2651 notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 2652 } 2653 2654 void bdrv_io_plug(BlockDriverState *bs) 2655 { 2656 BdrvChild *child; 2657 2658 QLIST_FOREACH(child, &bs->children, next) { 2659 bdrv_io_plug(child->bs); 2660 } 2661 2662 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) { 2663 BlockDriver *drv = bs->drv; 2664 if (drv && drv->bdrv_io_plug) { 2665 drv->bdrv_io_plug(bs); 2666 } 2667 } 2668 } 2669 2670 void bdrv_io_unplug(BlockDriverState *bs) 2671 { 2672 BdrvChild *child; 2673 2674 assert(bs->io_plugged); 2675 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) { 2676 BlockDriver *drv = bs->drv; 2677 if (drv && drv->bdrv_io_unplug) { 2678 drv->bdrv_io_unplug(bs); 2679 } 2680 } 2681 2682 QLIST_FOREACH(child, &bs->children, next) { 2683 bdrv_io_unplug(child->bs); 2684 } 2685 } 2686 2687 void bdrv_io_unplugged_begin(BlockDriverState *bs) 2688 { 2689 BdrvChild *child; 2690 2691 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) { 2692 BlockDriver *drv = bs->drv; 2693 if (drv && drv->bdrv_io_unplug) { 2694 drv->bdrv_io_unplug(bs); 2695 } 2696 } 2697 2698 QLIST_FOREACH(child, &bs->children, next) { 2699 bdrv_io_unplugged_begin(child->bs); 2700 } 2701 } 2702 2703 void bdrv_io_unplugged_end(BlockDriverState *bs) 2704 { 2705 BdrvChild *child; 2706 2707 assert(bs->io_plug_disabled); 2708 QLIST_FOREACH(child, &bs->children, next) { 2709 bdrv_io_unplugged_end(child->bs); 2710 } 2711 2712 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) { 2713 BlockDriver *drv = bs->drv; 2714 if (drv && drv->bdrv_io_plug) { 2715 drv->bdrv_io_plug(bs); 2716 } 2717 } 2718 } 2719