1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/blockjob.h" 29 #include "block/block_int.h" 30 #include "qemu/cutils.h" 31 #include "qapi/error.h" 32 #include "qemu/error-report.h" 33 34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 35 36 static BlockAIOCB *bdrv_co_aio_rw_vector(BdrvChild *child, 37 int64_t sector_num, 38 QEMUIOVector *qiov, 39 int nb_sectors, 40 BdrvRequestFlags flags, 41 BlockCompletionFunc *cb, 42 void *opaque, 43 bool is_write); 44 static void coroutine_fn bdrv_co_do_rw(void *opaque); 45 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 46 int64_t offset, int count, BdrvRequestFlags flags); 47 48 static void bdrv_parent_drained_begin(BlockDriverState *bs) 49 { 50 BdrvChild *c; 51 52 QLIST_FOREACH(c, &bs->parents, next_parent) { 53 if (c->role->drained_begin) { 54 c->role->drained_begin(c); 55 } 56 } 57 } 58 59 static void bdrv_parent_drained_end(BlockDriverState *bs) 60 { 61 BdrvChild *c; 62 63 QLIST_FOREACH(c, &bs->parents, next_parent) { 64 if (c->role->drained_end) { 65 c->role->drained_end(c); 66 } 67 } 68 } 69 70 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 71 { 72 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 73 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 74 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 75 src->opt_mem_alignment); 76 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 77 src->min_mem_alignment); 78 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 79 } 80 81 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 82 { 83 BlockDriver *drv = bs->drv; 84 Error *local_err = NULL; 85 86 memset(&bs->bl, 0, sizeof(bs->bl)); 87 88 if (!drv) { 89 return; 90 } 91 92 /* Default alignment based on whether driver has byte interface */ 93 bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512; 94 95 /* Take some limits from the children as a default */ 96 if (bs->file) { 97 bdrv_refresh_limits(bs->file->bs, &local_err); 98 if (local_err) { 99 error_propagate(errp, local_err); 100 return; 101 } 102 bdrv_merge_limits(&bs->bl, &bs->file->bs->bl); 103 } else { 104 bs->bl.min_mem_alignment = 512; 105 bs->bl.opt_mem_alignment = getpagesize(); 106 107 /* Safe default since most protocols use readv()/writev()/etc */ 108 bs->bl.max_iov = IOV_MAX; 109 } 110 111 if (bs->backing) { 112 bdrv_refresh_limits(bs->backing->bs, &local_err); 113 if (local_err) { 114 error_propagate(errp, local_err); 115 return; 116 } 117 bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl); 118 } 119 120 /* Then let the driver override it */ 121 if (drv->bdrv_refresh_limits) { 122 drv->bdrv_refresh_limits(bs, errp); 123 } 124 } 125 126 /** 127 * The copy-on-read flag is actually a reference count so multiple users may 128 * use the feature without worrying about clobbering its previous state. 129 * Copy-on-read stays enabled until all users have called to disable it. 130 */ 131 void bdrv_enable_copy_on_read(BlockDriverState *bs) 132 { 133 bs->copy_on_read++; 134 } 135 136 void bdrv_disable_copy_on_read(BlockDriverState *bs) 137 { 138 assert(bs->copy_on_read > 0); 139 bs->copy_on_read--; 140 } 141 142 /* Check if any requests are in-flight (including throttled requests) */ 143 bool bdrv_requests_pending(BlockDriverState *bs) 144 { 145 BdrvChild *child; 146 147 if (!QLIST_EMPTY(&bs->tracked_requests)) { 148 return true; 149 } 150 151 QLIST_FOREACH(child, &bs->children, next) { 152 if (bdrv_requests_pending(child->bs)) { 153 return true; 154 } 155 } 156 157 return false; 158 } 159 160 static void bdrv_drain_recurse(BlockDriverState *bs) 161 { 162 BdrvChild *child; 163 164 if (bs->drv && bs->drv->bdrv_drain) { 165 bs->drv->bdrv_drain(bs); 166 } 167 QLIST_FOREACH(child, &bs->children, next) { 168 bdrv_drain_recurse(child->bs); 169 } 170 } 171 172 typedef struct { 173 Coroutine *co; 174 BlockDriverState *bs; 175 QEMUBH *bh; 176 bool done; 177 } BdrvCoDrainData; 178 179 static void bdrv_drain_poll(BlockDriverState *bs) 180 { 181 bool busy = true; 182 183 while (busy) { 184 /* Keep iterating */ 185 busy = bdrv_requests_pending(bs); 186 busy |= aio_poll(bdrv_get_aio_context(bs), busy); 187 } 188 } 189 190 static void bdrv_co_drain_bh_cb(void *opaque) 191 { 192 BdrvCoDrainData *data = opaque; 193 Coroutine *co = data->co; 194 195 qemu_bh_delete(data->bh); 196 bdrv_drain_poll(data->bs); 197 data->done = true; 198 qemu_coroutine_enter(co); 199 } 200 201 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs) 202 { 203 BdrvCoDrainData data; 204 205 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 206 * other coroutines run if they were queued from 207 * qemu_co_queue_run_restart(). */ 208 209 assert(qemu_in_coroutine()); 210 data = (BdrvCoDrainData) { 211 .co = qemu_coroutine_self(), 212 .bs = bs, 213 .done = false, 214 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data), 215 }; 216 qemu_bh_schedule(data.bh); 217 218 qemu_coroutine_yield(); 219 /* If we are resumed from some other event (such as an aio completion or a 220 * timer callback), it is a bug in the caller that should be fixed. */ 221 assert(data.done); 222 } 223 224 void bdrv_drained_begin(BlockDriverState *bs) 225 { 226 if (!bs->quiesce_counter++) { 227 aio_disable_external(bdrv_get_aio_context(bs)); 228 bdrv_parent_drained_begin(bs); 229 } 230 231 bdrv_io_unplugged_begin(bs); 232 bdrv_drain_recurse(bs); 233 if (qemu_in_coroutine()) { 234 bdrv_co_yield_to_drain(bs); 235 } else { 236 bdrv_drain_poll(bs); 237 } 238 bdrv_io_unplugged_end(bs); 239 } 240 241 void bdrv_drained_end(BlockDriverState *bs) 242 { 243 assert(bs->quiesce_counter > 0); 244 if (--bs->quiesce_counter > 0) { 245 return; 246 } 247 248 bdrv_parent_drained_end(bs); 249 aio_enable_external(bdrv_get_aio_context(bs)); 250 } 251 252 /* 253 * Wait for pending requests to complete on a single BlockDriverState subtree, 254 * and suspend block driver's internal I/O until next request arrives. 255 * 256 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 257 * AioContext. 258 * 259 * Only this BlockDriverState's AioContext is run, so in-flight requests must 260 * not depend on events in other AioContexts. In that case, use 261 * bdrv_drain_all() instead. 262 */ 263 void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 264 { 265 assert(qemu_in_coroutine()); 266 bdrv_drained_begin(bs); 267 bdrv_drained_end(bs); 268 } 269 270 void bdrv_drain(BlockDriverState *bs) 271 { 272 bdrv_drained_begin(bs); 273 bdrv_drained_end(bs); 274 } 275 276 /* 277 * Wait for pending requests to complete across all BlockDriverStates 278 * 279 * This function does not flush data to disk, use bdrv_flush_all() for that 280 * after calling this function. 281 */ 282 void bdrv_drain_all(void) 283 { 284 /* Always run first iteration so any pending completion BHs run */ 285 bool busy = true; 286 BlockDriverState *bs; 287 BdrvNextIterator it; 288 BlockJob *job = NULL; 289 GSList *aio_ctxs = NULL, *ctx; 290 291 while ((job = block_job_next(job))) { 292 AioContext *aio_context = blk_get_aio_context(job->blk); 293 294 aio_context_acquire(aio_context); 295 block_job_pause(job); 296 aio_context_release(aio_context); 297 } 298 299 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 300 AioContext *aio_context = bdrv_get_aio_context(bs); 301 302 aio_context_acquire(aio_context); 303 bdrv_parent_drained_begin(bs); 304 bdrv_io_unplugged_begin(bs); 305 bdrv_drain_recurse(bs); 306 aio_context_release(aio_context); 307 308 if (!g_slist_find(aio_ctxs, aio_context)) { 309 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 310 } 311 } 312 313 /* Note that completion of an asynchronous I/O operation can trigger any 314 * number of other I/O operations on other devices---for example a 315 * coroutine can submit an I/O request to another device in response to 316 * request completion. Therefore we must keep looping until there was no 317 * more activity rather than simply draining each device independently. 318 */ 319 while (busy) { 320 busy = false; 321 322 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 323 AioContext *aio_context = ctx->data; 324 325 aio_context_acquire(aio_context); 326 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 327 if (aio_context == bdrv_get_aio_context(bs)) { 328 if (bdrv_requests_pending(bs)) { 329 busy = true; 330 aio_poll(aio_context, busy); 331 } 332 } 333 } 334 busy |= aio_poll(aio_context, false); 335 aio_context_release(aio_context); 336 } 337 } 338 339 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 340 AioContext *aio_context = bdrv_get_aio_context(bs); 341 342 aio_context_acquire(aio_context); 343 bdrv_io_unplugged_end(bs); 344 bdrv_parent_drained_end(bs); 345 aio_context_release(aio_context); 346 } 347 g_slist_free(aio_ctxs); 348 349 job = NULL; 350 while ((job = block_job_next(job))) { 351 AioContext *aio_context = blk_get_aio_context(job->blk); 352 353 aio_context_acquire(aio_context); 354 block_job_resume(job); 355 aio_context_release(aio_context); 356 } 357 } 358 359 /** 360 * Remove an active request from the tracked requests list 361 * 362 * This function should be called when a tracked request is completing. 363 */ 364 static void tracked_request_end(BdrvTrackedRequest *req) 365 { 366 if (req->serialising) { 367 req->bs->serialising_in_flight--; 368 } 369 370 QLIST_REMOVE(req, list); 371 qemu_co_queue_restart_all(&req->wait_queue); 372 } 373 374 /** 375 * Add an active request to the tracked requests list 376 */ 377 static void tracked_request_begin(BdrvTrackedRequest *req, 378 BlockDriverState *bs, 379 int64_t offset, 380 unsigned int bytes, 381 enum BdrvTrackedRequestType type) 382 { 383 *req = (BdrvTrackedRequest){ 384 .bs = bs, 385 .offset = offset, 386 .bytes = bytes, 387 .type = type, 388 .co = qemu_coroutine_self(), 389 .serialising = false, 390 .overlap_offset = offset, 391 .overlap_bytes = bytes, 392 }; 393 394 qemu_co_queue_init(&req->wait_queue); 395 396 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 397 } 398 399 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 400 { 401 int64_t overlap_offset = req->offset & ~(align - 1); 402 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 403 - overlap_offset; 404 405 if (!req->serialising) { 406 req->bs->serialising_in_flight++; 407 req->serialising = true; 408 } 409 410 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 411 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 412 } 413 414 /** 415 * Round a region to cluster boundaries (sector-based) 416 */ 417 void bdrv_round_sectors_to_clusters(BlockDriverState *bs, 418 int64_t sector_num, int nb_sectors, 419 int64_t *cluster_sector_num, 420 int *cluster_nb_sectors) 421 { 422 BlockDriverInfo bdi; 423 424 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 425 *cluster_sector_num = sector_num; 426 *cluster_nb_sectors = nb_sectors; 427 } else { 428 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 429 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 430 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 431 nb_sectors, c); 432 } 433 } 434 435 /** 436 * Round a region to cluster boundaries 437 */ 438 void bdrv_round_to_clusters(BlockDriverState *bs, 439 int64_t offset, unsigned int bytes, 440 int64_t *cluster_offset, 441 unsigned int *cluster_bytes) 442 { 443 BlockDriverInfo bdi; 444 445 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 446 *cluster_offset = offset; 447 *cluster_bytes = bytes; 448 } else { 449 int64_t c = bdi.cluster_size; 450 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 451 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 452 } 453 } 454 455 static int bdrv_get_cluster_size(BlockDriverState *bs) 456 { 457 BlockDriverInfo bdi; 458 int ret; 459 460 ret = bdrv_get_info(bs, &bdi); 461 if (ret < 0 || bdi.cluster_size == 0) { 462 return bs->bl.request_alignment; 463 } else { 464 return bdi.cluster_size; 465 } 466 } 467 468 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 469 int64_t offset, unsigned int bytes) 470 { 471 /* aaaa bbbb */ 472 if (offset >= req->overlap_offset + req->overlap_bytes) { 473 return false; 474 } 475 /* bbbb aaaa */ 476 if (req->overlap_offset >= offset + bytes) { 477 return false; 478 } 479 return true; 480 } 481 482 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 483 { 484 BlockDriverState *bs = self->bs; 485 BdrvTrackedRequest *req; 486 bool retry; 487 bool waited = false; 488 489 if (!bs->serialising_in_flight) { 490 return false; 491 } 492 493 do { 494 retry = false; 495 QLIST_FOREACH(req, &bs->tracked_requests, list) { 496 if (req == self || (!req->serialising && !self->serialising)) { 497 continue; 498 } 499 if (tracked_request_overlaps(req, self->overlap_offset, 500 self->overlap_bytes)) 501 { 502 /* Hitting this means there was a reentrant request, for 503 * example, a block driver issuing nested requests. This must 504 * never happen since it means deadlock. 505 */ 506 assert(qemu_coroutine_self() != req->co); 507 508 /* If the request is already (indirectly) waiting for us, or 509 * will wait for us as soon as it wakes up, then just go on 510 * (instead of producing a deadlock in the former case). */ 511 if (!req->waiting_for) { 512 self->waiting_for = req; 513 qemu_co_queue_wait(&req->wait_queue); 514 self->waiting_for = NULL; 515 retry = true; 516 waited = true; 517 break; 518 } 519 } 520 } 521 } while (retry); 522 523 return waited; 524 } 525 526 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 527 size_t size) 528 { 529 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 530 return -EIO; 531 } 532 533 if (!bdrv_is_inserted(bs)) { 534 return -ENOMEDIUM; 535 } 536 537 if (offset < 0) { 538 return -EIO; 539 } 540 541 return 0; 542 } 543 544 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num, 545 int nb_sectors) 546 { 547 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 548 return -EIO; 549 } 550 551 return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE, 552 nb_sectors * BDRV_SECTOR_SIZE); 553 } 554 555 typedef struct RwCo { 556 BdrvChild *child; 557 int64_t offset; 558 QEMUIOVector *qiov; 559 bool is_write; 560 int ret; 561 BdrvRequestFlags flags; 562 } RwCo; 563 564 static void coroutine_fn bdrv_rw_co_entry(void *opaque) 565 { 566 RwCo *rwco = opaque; 567 568 if (!rwco->is_write) { 569 rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset, 570 rwco->qiov->size, rwco->qiov, 571 rwco->flags); 572 } else { 573 rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset, 574 rwco->qiov->size, rwco->qiov, 575 rwco->flags); 576 } 577 } 578 579 /* 580 * Process a vectored synchronous request using coroutines 581 */ 582 static int bdrv_prwv_co(BdrvChild *child, int64_t offset, 583 QEMUIOVector *qiov, bool is_write, 584 BdrvRequestFlags flags) 585 { 586 Coroutine *co; 587 RwCo rwco = { 588 .child = child, 589 .offset = offset, 590 .qiov = qiov, 591 .is_write = is_write, 592 .ret = NOT_DONE, 593 .flags = flags, 594 }; 595 596 if (qemu_in_coroutine()) { 597 /* Fast-path if already in coroutine context */ 598 bdrv_rw_co_entry(&rwco); 599 } else { 600 AioContext *aio_context = bdrv_get_aio_context(child->bs); 601 602 co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco); 603 qemu_coroutine_enter(co); 604 while (rwco.ret == NOT_DONE) { 605 aio_poll(aio_context, true); 606 } 607 } 608 return rwco.ret; 609 } 610 611 /* 612 * Process a synchronous request using coroutines 613 */ 614 static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf, 615 int nb_sectors, bool is_write, BdrvRequestFlags flags) 616 { 617 QEMUIOVector qiov; 618 struct iovec iov = { 619 .iov_base = (void *)buf, 620 .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 621 }; 622 623 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 624 return -EINVAL; 625 } 626 627 qemu_iovec_init_external(&qiov, &iov, 1); 628 return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS, 629 &qiov, is_write, flags); 630 } 631 632 /* return < 0 if error. See bdrv_write() for the return codes */ 633 int bdrv_read(BdrvChild *child, int64_t sector_num, 634 uint8_t *buf, int nb_sectors) 635 { 636 return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0); 637 } 638 639 /* Return < 0 if error. Important errors are: 640 -EIO generic I/O error (may happen for all errors) 641 -ENOMEDIUM No media inserted. 642 -EINVAL Invalid sector number or nb_sectors 643 -EACCES Trying to write a read-only device 644 */ 645 int bdrv_write(BdrvChild *child, int64_t sector_num, 646 const uint8_t *buf, int nb_sectors) 647 { 648 return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 649 } 650 651 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 652 int count, BdrvRequestFlags flags) 653 { 654 QEMUIOVector qiov; 655 struct iovec iov = { 656 .iov_base = NULL, 657 .iov_len = count, 658 }; 659 660 qemu_iovec_init_external(&qiov, &iov, 1); 661 return bdrv_prwv_co(child, offset, &qiov, true, 662 BDRV_REQ_ZERO_WRITE | flags); 663 } 664 665 /* 666 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 667 * The operation is sped up by checking the block status and only writing 668 * zeroes to the device if they currently do not return zeroes. Optional 669 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 670 * BDRV_REQ_FUA). 671 * 672 * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 673 */ 674 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 675 { 676 int64_t target_sectors, ret, nb_sectors, sector_num = 0; 677 BlockDriverState *bs = child->bs; 678 BlockDriverState *file; 679 int n; 680 681 target_sectors = bdrv_nb_sectors(bs); 682 if (target_sectors < 0) { 683 return target_sectors; 684 } 685 686 for (;;) { 687 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 688 if (nb_sectors <= 0) { 689 return 0; 690 } 691 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file); 692 if (ret < 0) { 693 error_report("error getting block status at sector %" PRId64 ": %s", 694 sector_num, strerror(-ret)); 695 return ret; 696 } 697 if (ret & BDRV_BLOCK_ZERO) { 698 sector_num += n; 699 continue; 700 } 701 ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS, 702 n << BDRV_SECTOR_BITS, flags); 703 if (ret < 0) { 704 error_report("error writing zeroes at sector %" PRId64 ": %s", 705 sector_num, strerror(-ret)); 706 return ret; 707 } 708 sector_num += n; 709 } 710 } 711 712 int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 713 { 714 int ret; 715 716 ret = bdrv_prwv_co(child, offset, qiov, false, 0); 717 if (ret < 0) { 718 return ret; 719 } 720 721 return qiov->size; 722 } 723 724 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 725 { 726 QEMUIOVector qiov; 727 struct iovec iov = { 728 .iov_base = (void *)buf, 729 .iov_len = bytes, 730 }; 731 732 if (bytes < 0) { 733 return -EINVAL; 734 } 735 736 qemu_iovec_init_external(&qiov, &iov, 1); 737 return bdrv_preadv(child, offset, &qiov); 738 } 739 740 int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov) 741 { 742 int ret; 743 744 ret = bdrv_prwv_co(child, offset, qiov, true, 0); 745 if (ret < 0) { 746 return ret; 747 } 748 749 return qiov->size; 750 } 751 752 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 753 { 754 QEMUIOVector qiov; 755 struct iovec iov = { 756 .iov_base = (void *) buf, 757 .iov_len = bytes, 758 }; 759 760 if (bytes < 0) { 761 return -EINVAL; 762 } 763 764 qemu_iovec_init_external(&qiov, &iov, 1); 765 return bdrv_pwritev(child, offset, &qiov); 766 } 767 768 /* 769 * Writes to the file and ensures that no writes are reordered across this 770 * request (acts as a barrier) 771 * 772 * Returns 0 on success, -errno in error cases. 773 */ 774 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 775 const void *buf, int count) 776 { 777 int ret; 778 779 ret = bdrv_pwrite(child, offset, buf, count); 780 if (ret < 0) { 781 return ret; 782 } 783 784 ret = bdrv_flush(child->bs); 785 if (ret < 0) { 786 return ret; 787 } 788 789 return 0; 790 } 791 792 typedef struct CoroutineIOCompletion { 793 Coroutine *coroutine; 794 int ret; 795 } CoroutineIOCompletion; 796 797 static void bdrv_co_io_em_complete(void *opaque, int ret) 798 { 799 CoroutineIOCompletion *co = opaque; 800 801 co->ret = ret; 802 qemu_coroutine_enter(co->coroutine); 803 } 804 805 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 806 uint64_t offset, uint64_t bytes, 807 QEMUIOVector *qiov, int flags) 808 { 809 BlockDriver *drv = bs->drv; 810 int64_t sector_num; 811 unsigned int nb_sectors; 812 813 assert(!(flags & ~BDRV_REQ_MASK)); 814 815 if (drv->bdrv_co_preadv) { 816 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 817 } 818 819 sector_num = offset >> BDRV_SECTOR_BITS; 820 nb_sectors = bytes >> BDRV_SECTOR_BITS; 821 822 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 823 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 824 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 825 826 if (drv->bdrv_co_readv) { 827 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 828 } else { 829 BlockAIOCB *acb; 830 CoroutineIOCompletion co = { 831 .coroutine = qemu_coroutine_self(), 832 }; 833 834 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 835 bdrv_co_io_em_complete, &co); 836 if (acb == NULL) { 837 return -EIO; 838 } else { 839 qemu_coroutine_yield(); 840 return co.ret; 841 } 842 } 843 } 844 845 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 846 uint64_t offset, uint64_t bytes, 847 QEMUIOVector *qiov, int flags) 848 { 849 BlockDriver *drv = bs->drv; 850 int64_t sector_num; 851 unsigned int nb_sectors; 852 int ret; 853 854 assert(!(flags & ~BDRV_REQ_MASK)); 855 856 if (drv->bdrv_co_pwritev) { 857 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 858 flags & bs->supported_write_flags); 859 flags &= ~bs->supported_write_flags; 860 goto emulate_flags; 861 } 862 863 sector_num = offset >> BDRV_SECTOR_BITS; 864 nb_sectors = bytes >> BDRV_SECTOR_BITS; 865 866 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 867 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 868 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 869 870 if (drv->bdrv_co_writev_flags) { 871 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 872 flags & bs->supported_write_flags); 873 flags &= ~bs->supported_write_flags; 874 } else if (drv->bdrv_co_writev) { 875 assert(!bs->supported_write_flags); 876 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 877 } else { 878 BlockAIOCB *acb; 879 CoroutineIOCompletion co = { 880 .coroutine = qemu_coroutine_self(), 881 }; 882 883 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 884 bdrv_co_io_em_complete, &co); 885 if (acb == NULL) { 886 ret = -EIO; 887 } else { 888 qemu_coroutine_yield(); 889 ret = co.ret; 890 } 891 } 892 893 emulate_flags: 894 if (ret == 0 && (flags & BDRV_REQ_FUA)) { 895 ret = bdrv_co_flush(bs); 896 } 897 898 return ret; 899 } 900 901 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs, 902 int64_t offset, unsigned int bytes, QEMUIOVector *qiov) 903 { 904 /* Perform I/O through a temporary buffer so that users who scribble over 905 * their read buffer while the operation is in progress do not end up 906 * modifying the image file. This is critical for zero-copy guest I/O 907 * where anything might happen inside guest memory. 908 */ 909 void *bounce_buffer; 910 911 BlockDriver *drv = bs->drv; 912 struct iovec iov; 913 QEMUIOVector bounce_qiov; 914 int64_t cluster_offset; 915 unsigned int cluster_bytes; 916 size_t skip_bytes; 917 int ret; 918 919 /* Cover entire cluster so no additional backing file I/O is required when 920 * allocating cluster in the image file. 921 */ 922 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 923 924 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 925 cluster_offset, cluster_bytes); 926 927 iov.iov_len = cluster_bytes; 928 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 929 if (bounce_buffer == NULL) { 930 ret = -ENOMEM; 931 goto err; 932 } 933 934 qemu_iovec_init_external(&bounce_qiov, &iov, 1); 935 936 ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes, 937 &bounce_qiov, 0); 938 if (ret < 0) { 939 goto err; 940 } 941 942 if (drv->bdrv_co_pwrite_zeroes && 943 buffer_is_zero(bounce_buffer, iov.iov_len)) { 944 /* FIXME: Should we (perhaps conditionally) be setting 945 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 946 * that still correctly reads as zero? */ 947 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0); 948 } else { 949 /* This does not change the data on the disk, it is not necessary 950 * to flush even in cache=writethrough mode. 951 */ 952 ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes, 953 &bounce_qiov, 0); 954 } 955 956 if (ret < 0) { 957 /* It might be okay to ignore write errors for guest requests. If this 958 * is a deliberate copy-on-read then we don't want to ignore the error. 959 * Simply report it in all cases. 960 */ 961 goto err; 962 } 963 964 skip_bytes = offset - cluster_offset; 965 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes); 966 967 err: 968 qemu_vfree(bounce_buffer); 969 return ret; 970 } 971 972 /* 973 * Forwards an already correctly aligned request to the BlockDriver. This 974 * handles copy on read and zeroing after EOF; any other features must be 975 * implemented by the caller. 976 */ 977 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs, 978 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 979 int64_t align, QEMUIOVector *qiov, int flags) 980 { 981 int64_t total_bytes, max_bytes; 982 int ret; 983 984 assert(is_power_of_2(align)); 985 assert((offset & (align - 1)) == 0); 986 assert((bytes & (align - 1)) == 0); 987 assert(!qiov || bytes == qiov->size); 988 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 989 990 /* TODO: We would need a per-BDS .supported_read_flags and 991 * potential fallback support, if we ever implement any read flags 992 * to pass through to drivers. For now, there aren't any 993 * passthrough flags. */ 994 assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ))); 995 996 /* Handle Copy on Read and associated serialisation */ 997 if (flags & BDRV_REQ_COPY_ON_READ) { 998 /* If we touch the same cluster it counts as an overlap. This 999 * guarantees that allocating writes will be serialized and not race 1000 * with each other for the same cluster. For example, in copy-on-read 1001 * it ensures that the CoR read and write operations are atomic and 1002 * guest writes cannot interleave between them. */ 1003 mark_request_serialising(req, bdrv_get_cluster_size(bs)); 1004 } 1005 1006 if (!(flags & BDRV_REQ_NO_SERIALISING)) { 1007 wait_serialising_requests(req); 1008 } 1009 1010 if (flags & BDRV_REQ_COPY_ON_READ) { 1011 int64_t start_sector = offset >> BDRV_SECTOR_BITS; 1012 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1013 unsigned int nb_sectors = end_sector - start_sector; 1014 int pnum; 1015 1016 ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum); 1017 if (ret < 0) { 1018 goto out; 1019 } 1020 1021 if (!ret || pnum != nb_sectors) { 1022 ret = bdrv_co_do_copy_on_readv(bs, offset, bytes, qiov); 1023 goto out; 1024 } 1025 } 1026 1027 /* Forward the request to the BlockDriver */ 1028 total_bytes = bdrv_getlength(bs); 1029 if (total_bytes < 0) { 1030 ret = total_bytes; 1031 goto out; 1032 } 1033 1034 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1035 if (bytes <= max_bytes) { 1036 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 1037 } else if (max_bytes > 0) { 1038 QEMUIOVector local_qiov; 1039 1040 qemu_iovec_init(&local_qiov, qiov->niov); 1041 qemu_iovec_concat(&local_qiov, qiov, 0, max_bytes); 1042 1043 ret = bdrv_driver_preadv(bs, offset, max_bytes, &local_qiov, 0); 1044 1045 qemu_iovec_destroy(&local_qiov); 1046 } else { 1047 ret = 0; 1048 } 1049 1050 /* Reading beyond end of file is supposed to produce zeroes */ 1051 if (ret == 0 && total_bytes < offset + bytes) { 1052 uint64_t zero_offset = MAX(0, total_bytes - offset); 1053 uint64_t zero_bytes = offset + bytes - zero_offset; 1054 qemu_iovec_memset(qiov, zero_offset, 0, zero_bytes); 1055 } 1056 1057 out: 1058 return ret; 1059 } 1060 1061 /* 1062 * Handle a read request in coroutine context 1063 */ 1064 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1065 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1066 BdrvRequestFlags flags) 1067 { 1068 BlockDriverState *bs = child->bs; 1069 BlockDriver *drv = bs->drv; 1070 BdrvTrackedRequest req; 1071 1072 uint64_t align = bs->bl.request_alignment; 1073 uint8_t *head_buf = NULL; 1074 uint8_t *tail_buf = NULL; 1075 QEMUIOVector local_qiov; 1076 bool use_local_qiov = false; 1077 int ret; 1078 1079 if (!drv) { 1080 return -ENOMEDIUM; 1081 } 1082 1083 ret = bdrv_check_byte_request(bs, offset, bytes); 1084 if (ret < 0) { 1085 return ret; 1086 } 1087 1088 /* Don't do copy-on-read if we read data before write operation */ 1089 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 1090 flags |= BDRV_REQ_COPY_ON_READ; 1091 } 1092 1093 /* Align read if necessary by padding qiov */ 1094 if (offset & (align - 1)) { 1095 head_buf = qemu_blockalign(bs, align); 1096 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1097 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1098 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1099 use_local_qiov = true; 1100 1101 bytes += offset & (align - 1); 1102 offset = offset & ~(align - 1); 1103 } 1104 1105 if ((offset + bytes) & (align - 1)) { 1106 if (!use_local_qiov) { 1107 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1108 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1109 use_local_qiov = true; 1110 } 1111 tail_buf = qemu_blockalign(bs, align); 1112 qemu_iovec_add(&local_qiov, tail_buf, 1113 align - ((offset + bytes) & (align - 1))); 1114 1115 bytes = ROUND_UP(bytes, align); 1116 } 1117 1118 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1119 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align, 1120 use_local_qiov ? &local_qiov : qiov, 1121 flags); 1122 tracked_request_end(&req); 1123 1124 if (use_local_qiov) { 1125 qemu_iovec_destroy(&local_qiov); 1126 qemu_vfree(head_buf); 1127 qemu_vfree(tail_buf); 1128 } 1129 1130 return ret; 1131 } 1132 1133 static int coroutine_fn bdrv_co_do_readv(BdrvChild *child, 1134 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1135 BdrvRequestFlags flags) 1136 { 1137 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1138 return -EINVAL; 1139 } 1140 1141 return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS, 1142 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1143 } 1144 1145 int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num, 1146 int nb_sectors, QEMUIOVector *qiov) 1147 { 1148 trace_bdrv_co_readv(child->bs, sector_num, nb_sectors); 1149 1150 return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0); 1151 } 1152 1153 /* Maximum buffer for write zeroes fallback, in bytes */ 1154 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 1155 1156 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1157 int64_t offset, int count, BdrvRequestFlags flags) 1158 { 1159 BlockDriver *drv = bs->drv; 1160 QEMUIOVector qiov; 1161 struct iovec iov = {0}; 1162 int ret = 0; 1163 bool need_flush = false; 1164 int head = 0; 1165 int tail = 0; 1166 1167 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1168 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1169 bs->bl.request_alignment); 1170 1171 assert(is_power_of_2(alignment)); 1172 head = offset & (alignment - 1); 1173 tail = (offset + count) & (alignment - 1); 1174 max_write_zeroes &= ~(alignment - 1); 1175 1176 while (count > 0 && !ret) { 1177 int num = count; 1178 1179 /* Align request. Block drivers can expect the "bulk" of the request 1180 * to be aligned, and that unaligned requests do not cross cluster 1181 * boundaries. 1182 */ 1183 if (head) { 1184 /* Make a small request up to the first aligned sector. */ 1185 num = MIN(count, alignment - head); 1186 head = 0; 1187 } else if (tail && num > alignment) { 1188 /* Shorten the request to the last aligned sector. */ 1189 num -= tail; 1190 } 1191 1192 /* limit request size */ 1193 if (num > max_write_zeroes) { 1194 num = max_write_zeroes; 1195 } 1196 1197 ret = -ENOTSUP; 1198 /* First try the efficient write zeroes operation */ 1199 if (drv->bdrv_co_pwrite_zeroes) { 1200 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1201 flags & bs->supported_zero_flags); 1202 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1203 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1204 need_flush = true; 1205 } 1206 } else { 1207 assert(!bs->supported_zero_flags); 1208 } 1209 1210 if (ret == -ENOTSUP) { 1211 /* Fall back to bounce buffer if write zeroes is unsupported */ 1212 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1213 MAX_WRITE_ZEROES_BOUNCE_BUFFER); 1214 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1215 1216 if ((flags & BDRV_REQ_FUA) && 1217 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1218 /* No need for bdrv_driver_pwrite() to do a fallback 1219 * flush on each chunk; use just one at the end */ 1220 write_flags &= ~BDRV_REQ_FUA; 1221 need_flush = true; 1222 } 1223 num = MIN(num, max_transfer); 1224 iov.iov_len = num; 1225 if (iov.iov_base == NULL) { 1226 iov.iov_base = qemu_try_blockalign(bs, num); 1227 if (iov.iov_base == NULL) { 1228 ret = -ENOMEM; 1229 goto fail; 1230 } 1231 memset(iov.iov_base, 0, num); 1232 } 1233 qemu_iovec_init_external(&qiov, &iov, 1); 1234 1235 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags); 1236 1237 /* Keep bounce buffer around if it is big enough for all 1238 * all future requests. 1239 */ 1240 if (num < max_transfer) { 1241 qemu_vfree(iov.iov_base); 1242 iov.iov_base = NULL; 1243 } 1244 } 1245 1246 offset += num; 1247 count -= num; 1248 } 1249 1250 fail: 1251 if (ret == 0 && need_flush) { 1252 ret = bdrv_co_flush(bs); 1253 } 1254 qemu_vfree(iov.iov_base); 1255 return ret; 1256 } 1257 1258 /* 1259 * Forwards an already correctly aligned write request to the BlockDriver. 1260 */ 1261 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs, 1262 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1263 int64_t align, QEMUIOVector *qiov, int flags) 1264 { 1265 BlockDriver *drv = bs->drv; 1266 bool waited; 1267 int ret; 1268 1269 int64_t start_sector = offset >> BDRV_SECTOR_BITS; 1270 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1271 1272 assert(is_power_of_2(align)); 1273 assert((offset & (align - 1)) == 0); 1274 assert((bytes & (align - 1)) == 0); 1275 assert(!qiov || bytes == qiov->size); 1276 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1277 assert(!(flags & ~BDRV_REQ_MASK)); 1278 1279 waited = wait_serialising_requests(req); 1280 assert(!waited || !req->serialising); 1281 assert(req->overlap_offset <= offset); 1282 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1283 1284 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 1285 1286 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1287 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 1288 qemu_iovec_is_zero(qiov)) { 1289 flags |= BDRV_REQ_ZERO_WRITE; 1290 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1291 flags |= BDRV_REQ_MAY_UNMAP; 1292 } 1293 } 1294 1295 if (ret < 0) { 1296 /* Do nothing, write notifier decided to fail this request */ 1297 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1298 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1299 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 1300 } else { 1301 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1302 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 1303 } 1304 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 1305 1306 ++bs->write_gen; 1307 bdrv_set_dirty(bs, start_sector, end_sector - start_sector); 1308 1309 if (bs->wr_highest_offset < offset + bytes) { 1310 bs->wr_highest_offset = offset + bytes; 1311 } 1312 1313 if (ret >= 0) { 1314 bs->total_sectors = MAX(bs->total_sectors, end_sector); 1315 } 1316 1317 return ret; 1318 } 1319 1320 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs, 1321 int64_t offset, 1322 unsigned int bytes, 1323 BdrvRequestFlags flags, 1324 BdrvTrackedRequest *req) 1325 { 1326 uint8_t *buf = NULL; 1327 QEMUIOVector local_qiov; 1328 struct iovec iov; 1329 uint64_t align = bs->bl.request_alignment; 1330 unsigned int head_padding_bytes, tail_padding_bytes; 1331 int ret = 0; 1332 1333 head_padding_bytes = offset & (align - 1); 1334 tail_padding_bytes = align - ((offset + bytes) & (align - 1)); 1335 1336 1337 assert(flags & BDRV_REQ_ZERO_WRITE); 1338 if (head_padding_bytes || tail_padding_bytes) { 1339 buf = qemu_blockalign(bs, align); 1340 iov = (struct iovec) { 1341 .iov_base = buf, 1342 .iov_len = align, 1343 }; 1344 qemu_iovec_init_external(&local_qiov, &iov, 1); 1345 } 1346 if (head_padding_bytes) { 1347 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 1348 1349 /* RMW the unaligned part before head. */ 1350 mark_request_serialising(req, align); 1351 wait_serialising_requests(req); 1352 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1353 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align, 1354 align, &local_qiov, 0); 1355 if (ret < 0) { 1356 goto fail; 1357 } 1358 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1359 1360 memset(buf + head_padding_bytes, 0, zero_bytes); 1361 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align, 1362 align, &local_qiov, 1363 flags & ~BDRV_REQ_ZERO_WRITE); 1364 if (ret < 0) { 1365 goto fail; 1366 } 1367 offset += zero_bytes; 1368 bytes -= zero_bytes; 1369 } 1370 1371 assert(!bytes || (offset & (align - 1)) == 0); 1372 if (bytes >= align) { 1373 /* Write the aligned part in the middle. */ 1374 uint64_t aligned_bytes = bytes & ~(align - 1); 1375 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, align, 1376 NULL, flags); 1377 if (ret < 0) { 1378 goto fail; 1379 } 1380 bytes -= aligned_bytes; 1381 offset += aligned_bytes; 1382 } 1383 1384 assert(!bytes || (offset & (align - 1)) == 0); 1385 if (bytes) { 1386 assert(align == tail_padding_bytes + bytes); 1387 /* RMW the unaligned part after tail. */ 1388 mark_request_serialising(req, align); 1389 wait_serialising_requests(req); 1390 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1391 ret = bdrv_aligned_preadv(bs, req, offset, align, 1392 align, &local_qiov, 0); 1393 if (ret < 0) { 1394 goto fail; 1395 } 1396 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1397 1398 memset(buf, 0, bytes); 1399 ret = bdrv_aligned_pwritev(bs, req, offset, align, align, 1400 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 1401 } 1402 fail: 1403 qemu_vfree(buf); 1404 return ret; 1405 1406 } 1407 1408 /* 1409 * Handle a write request in coroutine context 1410 */ 1411 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 1412 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1413 BdrvRequestFlags flags) 1414 { 1415 BlockDriverState *bs = child->bs; 1416 BdrvTrackedRequest req; 1417 uint64_t align = bs->bl.request_alignment; 1418 uint8_t *head_buf = NULL; 1419 uint8_t *tail_buf = NULL; 1420 QEMUIOVector local_qiov; 1421 bool use_local_qiov = false; 1422 int ret; 1423 1424 if (!bs->drv) { 1425 return -ENOMEDIUM; 1426 } 1427 if (bs->read_only) { 1428 return -EPERM; 1429 } 1430 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1431 1432 ret = bdrv_check_byte_request(bs, offset, bytes); 1433 if (ret < 0) { 1434 return ret; 1435 } 1436 1437 /* 1438 * Align write if necessary by performing a read-modify-write cycle. 1439 * Pad qiov with the read parts and be sure to have a tracked request not 1440 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 1441 */ 1442 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 1443 1444 if (!qiov) { 1445 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req); 1446 goto out; 1447 } 1448 1449 if (offset & (align - 1)) { 1450 QEMUIOVector head_qiov; 1451 struct iovec head_iov; 1452 1453 mark_request_serialising(&req, align); 1454 wait_serialising_requests(&req); 1455 1456 head_buf = qemu_blockalign(bs, align); 1457 head_iov = (struct iovec) { 1458 .iov_base = head_buf, 1459 .iov_len = align, 1460 }; 1461 qemu_iovec_init_external(&head_qiov, &head_iov, 1); 1462 1463 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1464 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align, 1465 align, &head_qiov, 0); 1466 if (ret < 0) { 1467 goto fail; 1468 } 1469 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1470 1471 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1472 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1473 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1474 use_local_qiov = true; 1475 1476 bytes += offset & (align - 1); 1477 offset = offset & ~(align - 1); 1478 1479 /* We have read the tail already if the request is smaller 1480 * than one aligned block. 1481 */ 1482 if (bytes < align) { 1483 qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes); 1484 bytes = align; 1485 } 1486 } 1487 1488 if ((offset + bytes) & (align - 1)) { 1489 QEMUIOVector tail_qiov; 1490 struct iovec tail_iov; 1491 size_t tail_bytes; 1492 bool waited; 1493 1494 mark_request_serialising(&req, align); 1495 waited = wait_serialising_requests(&req); 1496 assert(!waited || !use_local_qiov); 1497 1498 tail_buf = qemu_blockalign(bs, align); 1499 tail_iov = (struct iovec) { 1500 .iov_base = tail_buf, 1501 .iov_len = align, 1502 }; 1503 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 1504 1505 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1506 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align, 1507 align, &tail_qiov, 0); 1508 if (ret < 0) { 1509 goto fail; 1510 } 1511 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1512 1513 if (!use_local_qiov) { 1514 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1515 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1516 use_local_qiov = true; 1517 } 1518 1519 tail_bytes = (offset + bytes) & (align - 1); 1520 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 1521 1522 bytes = ROUND_UP(bytes, align); 1523 } 1524 1525 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, align, 1526 use_local_qiov ? &local_qiov : qiov, 1527 flags); 1528 1529 fail: 1530 1531 if (use_local_qiov) { 1532 qemu_iovec_destroy(&local_qiov); 1533 } 1534 qemu_vfree(head_buf); 1535 qemu_vfree(tail_buf); 1536 out: 1537 tracked_request_end(&req); 1538 return ret; 1539 } 1540 1541 static int coroutine_fn bdrv_co_do_writev(BdrvChild *child, 1542 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1543 BdrvRequestFlags flags) 1544 { 1545 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1546 return -EINVAL; 1547 } 1548 1549 return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS, 1550 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1551 } 1552 1553 int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num, 1554 int nb_sectors, QEMUIOVector *qiov) 1555 { 1556 trace_bdrv_co_writev(child->bs, sector_num, nb_sectors); 1557 1558 return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0); 1559 } 1560 1561 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 1562 int count, BdrvRequestFlags flags) 1563 { 1564 trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags); 1565 1566 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 1567 flags &= ~BDRV_REQ_MAY_UNMAP; 1568 } 1569 1570 return bdrv_co_pwritev(child, offset, count, NULL, 1571 BDRV_REQ_ZERO_WRITE | flags); 1572 } 1573 1574 typedef struct BdrvCoGetBlockStatusData { 1575 BlockDriverState *bs; 1576 BlockDriverState *base; 1577 BlockDriverState **file; 1578 int64_t sector_num; 1579 int nb_sectors; 1580 int *pnum; 1581 int64_t ret; 1582 bool done; 1583 } BdrvCoGetBlockStatusData; 1584 1585 /* 1586 * Returns the allocation status of the specified sectors. 1587 * Drivers not implementing the functionality are assumed to not support 1588 * backing files, hence all their sectors are reported as allocated. 1589 * 1590 * If 'sector_num' is beyond the end of the disk image the return value is 0 1591 * and 'pnum' is set to 0. 1592 * 1593 * 'pnum' is set to the number of sectors (including and immediately following 1594 * the specified sector) that are known to be in the same 1595 * allocated/unallocated state. 1596 * 1597 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 1598 * beyond the end of the disk image it will be clamped. 1599 * 1600 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file' 1601 * points to the BDS which the sector range is allocated in. 1602 */ 1603 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 1604 int64_t sector_num, 1605 int nb_sectors, int *pnum, 1606 BlockDriverState **file) 1607 { 1608 int64_t total_sectors; 1609 int64_t n; 1610 int64_t ret, ret2; 1611 1612 total_sectors = bdrv_nb_sectors(bs); 1613 if (total_sectors < 0) { 1614 return total_sectors; 1615 } 1616 1617 if (sector_num >= total_sectors) { 1618 *pnum = 0; 1619 return 0; 1620 } 1621 1622 n = total_sectors - sector_num; 1623 if (n < nb_sectors) { 1624 nb_sectors = n; 1625 } 1626 1627 if (!bs->drv->bdrv_co_get_block_status) { 1628 *pnum = nb_sectors; 1629 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 1630 if (bs->drv->protocol_name) { 1631 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 1632 } 1633 return ret; 1634 } 1635 1636 *file = NULL; 1637 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 1638 file); 1639 if (ret < 0) { 1640 *pnum = 0; 1641 return ret; 1642 } 1643 1644 if (ret & BDRV_BLOCK_RAW) { 1645 assert(ret & BDRV_BLOCK_OFFSET_VALID); 1646 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 1647 *pnum, pnum, file); 1648 } 1649 1650 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 1651 ret |= BDRV_BLOCK_ALLOCATED; 1652 } else { 1653 if (bdrv_unallocated_blocks_are_zero(bs)) { 1654 ret |= BDRV_BLOCK_ZERO; 1655 } else if (bs->backing) { 1656 BlockDriverState *bs2 = bs->backing->bs; 1657 int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 1658 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 1659 ret |= BDRV_BLOCK_ZERO; 1660 } 1661 } 1662 } 1663 1664 if (*file && *file != bs && 1665 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 1666 (ret & BDRV_BLOCK_OFFSET_VALID)) { 1667 BlockDriverState *file2; 1668 int file_pnum; 1669 1670 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 1671 *pnum, &file_pnum, &file2); 1672 if (ret2 >= 0) { 1673 /* Ignore errors. This is just providing extra information, it 1674 * is useful but not necessary. 1675 */ 1676 if (!file_pnum) { 1677 /* !file_pnum indicates an offset at or beyond the EOF; it is 1678 * perfectly valid for the format block driver to point to such 1679 * offsets, so catch it and mark everything as zero */ 1680 ret |= BDRV_BLOCK_ZERO; 1681 } else { 1682 /* Limit request to the range reported by the protocol driver */ 1683 *pnum = file_pnum; 1684 ret |= (ret2 & BDRV_BLOCK_ZERO); 1685 } 1686 } 1687 } 1688 1689 return ret; 1690 } 1691 1692 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1693 BlockDriverState *base, 1694 int64_t sector_num, 1695 int nb_sectors, 1696 int *pnum, 1697 BlockDriverState **file) 1698 { 1699 BlockDriverState *p; 1700 int64_t ret = 0; 1701 1702 assert(bs != base); 1703 for (p = bs; p != base; p = backing_bs(p)) { 1704 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file); 1705 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1706 break; 1707 } 1708 /* [sector_num, pnum] unallocated on this layer, which could be only 1709 * the first part of [sector_num, nb_sectors]. */ 1710 nb_sectors = MIN(nb_sectors, *pnum); 1711 } 1712 return ret; 1713 } 1714 1715 /* Coroutine wrapper for bdrv_get_block_status_above() */ 1716 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 1717 { 1718 BdrvCoGetBlockStatusData *data = opaque; 1719 1720 data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1721 data->sector_num, 1722 data->nb_sectors, 1723 data->pnum, 1724 data->file); 1725 data->done = true; 1726 } 1727 1728 /* 1729 * Synchronous wrapper around bdrv_co_get_block_status_above(). 1730 * 1731 * See bdrv_co_get_block_status_above() for details. 1732 */ 1733 int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1734 BlockDriverState *base, 1735 int64_t sector_num, 1736 int nb_sectors, int *pnum, 1737 BlockDriverState **file) 1738 { 1739 Coroutine *co; 1740 BdrvCoGetBlockStatusData data = { 1741 .bs = bs, 1742 .base = base, 1743 .file = file, 1744 .sector_num = sector_num, 1745 .nb_sectors = nb_sectors, 1746 .pnum = pnum, 1747 .done = false, 1748 }; 1749 1750 if (qemu_in_coroutine()) { 1751 /* Fast-path if already in coroutine context */ 1752 bdrv_get_block_status_above_co_entry(&data); 1753 } else { 1754 AioContext *aio_context = bdrv_get_aio_context(bs); 1755 1756 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry, 1757 &data); 1758 qemu_coroutine_enter(co); 1759 while (!data.done) { 1760 aio_poll(aio_context, true); 1761 } 1762 } 1763 return data.ret; 1764 } 1765 1766 int64_t bdrv_get_block_status(BlockDriverState *bs, 1767 int64_t sector_num, 1768 int nb_sectors, int *pnum, 1769 BlockDriverState **file) 1770 { 1771 return bdrv_get_block_status_above(bs, backing_bs(bs), 1772 sector_num, nb_sectors, pnum, file); 1773 } 1774 1775 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 1776 int nb_sectors, int *pnum) 1777 { 1778 BlockDriverState *file; 1779 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum, 1780 &file); 1781 if (ret < 0) { 1782 return ret; 1783 } 1784 return !!(ret & BDRV_BLOCK_ALLOCATED); 1785 } 1786 1787 /* 1788 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 1789 * 1790 * Return true if the given sector is allocated in any image between 1791 * BASE and TOP (inclusive). BASE can be NULL to check if the given 1792 * sector is allocated in any image of the chain. Return false otherwise. 1793 * 1794 * 'pnum' is set to the number of sectors (including and immediately following 1795 * the specified sector) that are known to be in the same 1796 * allocated/unallocated state. 1797 * 1798 */ 1799 int bdrv_is_allocated_above(BlockDriverState *top, 1800 BlockDriverState *base, 1801 int64_t sector_num, 1802 int nb_sectors, int *pnum) 1803 { 1804 BlockDriverState *intermediate; 1805 int ret, n = nb_sectors; 1806 1807 intermediate = top; 1808 while (intermediate && intermediate != base) { 1809 int pnum_inter; 1810 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 1811 &pnum_inter); 1812 if (ret < 0) { 1813 return ret; 1814 } else if (ret) { 1815 *pnum = pnum_inter; 1816 return 1; 1817 } 1818 1819 /* 1820 * [sector_num, nb_sectors] is unallocated on top but intermediate 1821 * might have 1822 * 1823 * [sector_num+x, nr_sectors] allocated. 1824 */ 1825 if (n > pnum_inter && 1826 (intermediate == top || 1827 sector_num + pnum_inter < intermediate->total_sectors)) { 1828 n = pnum_inter; 1829 } 1830 1831 intermediate = backing_bs(intermediate); 1832 } 1833 1834 *pnum = n; 1835 return 0; 1836 } 1837 1838 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num, 1839 const uint8_t *buf, int nb_sectors) 1840 { 1841 BlockDriver *drv = bs->drv; 1842 int ret; 1843 1844 if (!drv) { 1845 return -ENOMEDIUM; 1846 } 1847 if (!drv->bdrv_write_compressed) { 1848 return -ENOTSUP; 1849 } 1850 ret = bdrv_check_request(bs, sector_num, nb_sectors); 1851 if (ret < 0) { 1852 return ret; 1853 } 1854 1855 assert(QLIST_EMPTY(&bs->dirty_bitmaps)); 1856 1857 return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors); 1858 } 1859 1860 typedef struct BdrvVmstateCo { 1861 BlockDriverState *bs; 1862 QEMUIOVector *qiov; 1863 int64_t pos; 1864 bool is_read; 1865 int ret; 1866 } BdrvVmstateCo; 1867 1868 static int coroutine_fn 1869 bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 1870 bool is_read) 1871 { 1872 BlockDriver *drv = bs->drv; 1873 1874 if (!drv) { 1875 return -ENOMEDIUM; 1876 } else if (drv->bdrv_load_vmstate) { 1877 return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos) 1878 : drv->bdrv_save_vmstate(bs, qiov, pos); 1879 } else if (bs->file) { 1880 return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read); 1881 } 1882 1883 return -ENOTSUP; 1884 } 1885 1886 static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque) 1887 { 1888 BdrvVmstateCo *co = opaque; 1889 co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read); 1890 } 1891 1892 static inline int 1893 bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos, 1894 bool is_read) 1895 { 1896 if (qemu_in_coroutine()) { 1897 return bdrv_co_rw_vmstate(bs, qiov, pos, is_read); 1898 } else { 1899 BdrvVmstateCo data = { 1900 .bs = bs, 1901 .qiov = qiov, 1902 .pos = pos, 1903 .is_read = is_read, 1904 .ret = -EINPROGRESS, 1905 }; 1906 Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data); 1907 1908 qemu_coroutine_enter(co); 1909 while (data.ret == -EINPROGRESS) { 1910 aio_poll(bdrv_get_aio_context(bs), true); 1911 } 1912 return data.ret; 1913 } 1914 } 1915 1916 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 1917 int64_t pos, int size) 1918 { 1919 QEMUIOVector qiov; 1920 struct iovec iov = { 1921 .iov_base = (void *) buf, 1922 .iov_len = size, 1923 }; 1924 int ret; 1925 1926 qemu_iovec_init_external(&qiov, &iov, 1); 1927 1928 ret = bdrv_writev_vmstate(bs, &qiov, pos); 1929 if (ret < 0) { 1930 return ret; 1931 } 1932 1933 return size; 1934 } 1935 1936 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 1937 { 1938 return bdrv_rw_vmstate(bs, qiov, pos, false); 1939 } 1940 1941 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 1942 int64_t pos, int size) 1943 { 1944 QEMUIOVector qiov; 1945 struct iovec iov = { 1946 .iov_base = buf, 1947 .iov_len = size, 1948 }; 1949 int ret; 1950 1951 qemu_iovec_init_external(&qiov, &iov, 1); 1952 ret = bdrv_readv_vmstate(bs, &qiov, pos); 1953 if (ret < 0) { 1954 return ret; 1955 } 1956 1957 return size; 1958 } 1959 1960 int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 1961 { 1962 return bdrv_rw_vmstate(bs, qiov, pos, true); 1963 } 1964 1965 /**************************************************************/ 1966 /* async I/Os */ 1967 1968 BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num, 1969 QEMUIOVector *qiov, int nb_sectors, 1970 BlockCompletionFunc *cb, void *opaque) 1971 { 1972 trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque); 1973 1974 return bdrv_co_aio_rw_vector(child, sector_num, qiov, nb_sectors, 0, 1975 cb, opaque, false); 1976 } 1977 1978 BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num, 1979 QEMUIOVector *qiov, int nb_sectors, 1980 BlockCompletionFunc *cb, void *opaque) 1981 { 1982 trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque); 1983 1984 return bdrv_co_aio_rw_vector(child, sector_num, qiov, nb_sectors, 0, 1985 cb, opaque, true); 1986 } 1987 1988 void bdrv_aio_cancel(BlockAIOCB *acb) 1989 { 1990 qemu_aio_ref(acb); 1991 bdrv_aio_cancel_async(acb); 1992 while (acb->refcnt > 1) { 1993 if (acb->aiocb_info->get_aio_context) { 1994 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 1995 } else if (acb->bs) { 1996 aio_poll(bdrv_get_aio_context(acb->bs), true); 1997 } else { 1998 abort(); 1999 } 2000 } 2001 qemu_aio_unref(acb); 2002 } 2003 2004 /* Async version of aio cancel. The caller is not blocked if the acb implements 2005 * cancel_async, otherwise we do nothing and let the request normally complete. 2006 * In either case the completion callback must be called. */ 2007 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2008 { 2009 if (acb->aiocb_info->cancel_async) { 2010 acb->aiocb_info->cancel_async(acb); 2011 } 2012 } 2013 2014 /**************************************************************/ 2015 /* async block device emulation */ 2016 2017 typedef struct BlockRequest { 2018 union { 2019 /* Used during read, write, trim */ 2020 struct { 2021 int64_t sector; 2022 int nb_sectors; 2023 int flags; 2024 QEMUIOVector *qiov; 2025 }; 2026 /* Used during ioctl */ 2027 struct { 2028 int req; 2029 void *buf; 2030 }; 2031 }; 2032 BlockCompletionFunc *cb; 2033 void *opaque; 2034 2035 int error; 2036 } BlockRequest; 2037 2038 typedef struct BlockAIOCBCoroutine { 2039 BlockAIOCB common; 2040 BdrvChild *child; 2041 BlockRequest req; 2042 bool is_write; 2043 bool need_bh; 2044 bool *done; 2045 QEMUBH* bh; 2046 } BlockAIOCBCoroutine; 2047 2048 static const AIOCBInfo bdrv_em_co_aiocb_info = { 2049 .aiocb_size = sizeof(BlockAIOCBCoroutine), 2050 }; 2051 2052 static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 2053 { 2054 if (!acb->need_bh) { 2055 acb->common.cb(acb->common.opaque, acb->req.error); 2056 qemu_aio_unref(acb); 2057 } 2058 } 2059 2060 static void bdrv_co_em_bh(void *opaque) 2061 { 2062 BlockAIOCBCoroutine *acb = opaque; 2063 2064 assert(!acb->need_bh); 2065 qemu_bh_delete(acb->bh); 2066 bdrv_co_complete(acb); 2067 } 2068 2069 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 2070 { 2071 acb->need_bh = false; 2072 if (acb->req.error != -EINPROGRESS) { 2073 BlockDriverState *bs = acb->common.bs; 2074 2075 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 2076 qemu_bh_schedule(acb->bh); 2077 } 2078 } 2079 2080 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 2081 static void coroutine_fn bdrv_co_do_rw(void *opaque) 2082 { 2083 BlockAIOCBCoroutine *acb = opaque; 2084 2085 if (!acb->is_write) { 2086 acb->req.error = bdrv_co_do_readv(acb->child, acb->req.sector, 2087 acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 2088 } else { 2089 acb->req.error = bdrv_co_do_writev(acb->child, acb->req.sector, 2090 acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 2091 } 2092 2093 bdrv_co_complete(acb); 2094 } 2095 2096 static BlockAIOCB *bdrv_co_aio_rw_vector(BdrvChild *child, 2097 int64_t sector_num, 2098 QEMUIOVector *qiov, 2099 int nb_sectors, 2100 BdrvRequestFlags flags, 2101 BlockCompletionFunc *cb, 2102 void *opaque, 2103 bool is_write) 2104 { 2105 Coroutine *co; 2106 BlockAIOCBCoroutine *acb; 2107 2108 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque); 2109 acb->child = child; 2110 acb->need_bh = true; 2111 acb->req.error = -EINPROGRESS; 2112 acb->req.sector = sector_num; 2113 acb->req.nb_sectors = nb_sectors; 2114 acb->req.qiov = qiov; 2115 acb->req.flags = flags; 2116 acb->is_write = is_write; 2117 2118 co = qemu_coroutine_create(bdrv_co_do_rw, acb); 2119 qemu_coroutine_enter(co); 2120 2121 bdrv_co_maybe_schedule_bh(acb); 2122 return &acb->common; 2123 } 2124 2125 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 2126 { 2127 BlockAIOCBCoroutine *acb = opaque; 2128 BlockDriverState *bs = acb->common.bs; 2129 2130 acb->req.error = bdrv_co_flush(bs); 2131 bdrv_co_complete(acb); 2132 } 2133 2134 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 2135 BlockCompletionFunc *cb, void *opaque) 2136 { 2137 trace_bdrv_aio_flush(bs, opaque); 2138 2139 Coroutine *co; 2140 BlockAIOCBCoroutine *acb; 2141 2142 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2143 acb->need_bh = true; 2144 acb->req.error = -EINPROGRESS; 2145 2146 co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb); 2147 qemu_coroutine_enter(co); 2148 2149 bdrv_co_maybe_schedule_bh(acb); 2150 return &acb->common; 2151 } 2152 2153 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque) 2154 { 2155 BlockAIOCBCoroutine *acb = opaque; 2156 BlockDriverState *bs = acb->common.bs; 2157 2158 acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors); 2159 bdrv_co_complete(acb); 2160 } 2161 2162 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs, 2163 int64_t sector_num, int nb_sectors, 2164 BlockCompletionFunc *cb, void *opaque) 2165 { 2166 Coroutine *co; 2167 BlockAIOCBCoroutine *acb; 2168 2169 trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque); 2170 2171 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2172 acb->need_bh = true; 2173 acb->req.error = -EINPROGRESS; 2174 acb->req.sector = sector_num; 2175 acb->req.nb_sectors = nb_sectors; 2176 co = qemu_coroutine_create(bdrv_aio_discard_co_entry, acb); 2177 qemu_coroutine_enter(co); 2178 2179 bdrv_co_maybe_schedule_bh(acb); 2180 return &acb->common; 2181 } 2182 2183 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs, 2184 BlockCompletionFunc *cb, void *opaque) 2185 { 2186 BlockAIOCB *acb; 2187 2188 acb = g_malloc(aiocb_info->aiocb_size); 2189 acb->aiocb_info = aiocb_info; 2190 acb->bs = bs; 2191 acb->cb = cb; 2192 acb->opaque = opaque; 2193 acb->refcnt = 1; 2194 return acb; 2195 } 2196 2197 void qemu_aio_ref(void *p) 2198 { 2199 BlockAIOCB *acb = p; 2200 acb->refcnt++; 2201 } 2202 2203 void qemu_aio_unref(void *p) 2204 { 2205 BlockAIOCB *acb = p; 2206 assert(acb->refcnt > 0); 2207 if (--acb->refcnt == 0) { 2208 g_free(acb); 2209 } 2210 } 2211 2212 /**************************************************************/ 2213 /* Coroutine block device emulation */ 2214 2215 typedef struct FlushCo { 2216 BlockDriverState *bs; 2217 int ret; 2218 } FlushCo; 2219 2220 2221 static void coroutine_fn bdrv_flush_co_entry(void *opaque) 2222 { 2223 FlushCo *rwco = opaque; 2224 2225 rwco->ret = bdrv_co_flush(rwco->bs); 2226 } 2227 2228 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2229 { 2230 int ret; 2231 BdrvTrackedRequest req; 2232 2233 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2234 bdrv_is_sg(bs)) { 2235 return 0; 2236 } 2237 2238 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH); 2239 2240 int current_gen = bs->write_gen; 2241 2242 /* Wait until any previous flushes are completed */ 2243 while (bs->flush_started_gen != bs->flushed_gen) { 2244 qemu_co_queue_wait(&bs->flush_queue); 2245 } 2246 2247 bs->flush_started_gen = current_gen; 2248 2249 /* Write back all layers by calling one driver function */ 2250 if (bs->drv->bdrv_co_flush) { 2251 ret = bs->drv->bdrv_co_flush(bs); 2252 goto out; 2253 } 2254 2255 /* Write back cached data to the OS even with cache=unsafe */ 2256 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 2257 if (bs->drv->bdrv_co_flush_to_os) { 2258 ret = bs->drv->bdrv_co_flush_to_os(bs); 2259 if (ret < 0) { 2260 goto out; 2261 } 2262 } 2263 2264 /* But don't actually force it to the disk with cache=unsafe */ 2265 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2266 goto flush_parent; 2267 } 2268 2269 /* Check if we really need to flush anything */ 2270 if (bs->flushed_gen == current_gen) { 2271 goto flush_parent; 2272 } 2273 2274 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 2275 if (bs->drv->bdrv_co_flush_to_disk) { 2276 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2277 } else if (bs->drv->bdrv_aio_flush) { 2278 BlockAIOCB *acb; 2279 CoroutineIOCompletion co = { 2280 .coroutine = qemu_coroutine_self(), 2281 }; 2282 2283 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2284 if (acb == NULL) { 2285 ret = -EIO; 2286 } else { 2287 qemu_coroutine_yield(); 2288 ret = co.ret; 2289 } 2290 } else { 2291 /* 2292 * Some block drivers always operate in either writethrough or unsafe 2293 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2294 * know how the server works (because the behaviour is hardcoded or 2295 * depends on server-side configuration), so we can't ensure that 2296 * everything is safe on disk. Returning an error doesn't work because 2297 * that would break guests even if the server operates in writethrough 2298 * mode. 2299 * 2300 * Let's hope the user knows what he's doing. 2301 */ 2302 ret = 0; 2303 } 2304 2305 if (ret < 0) { 2306 goto out; 2307 } 2308 2309 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2310 * in the case of cache=unsafe, so there are no useless flushes. 2311 */ 2312 flush_parent: 2313 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2314 out: 2315 /* Notify any pending flushes that we have completed */ 2316 bs->flushed_gen = current_gen; 2317 qemu_co_queue_restart_all(&bs->flush_queue); 2318 2319 tracked_request_end(&req); 2320 return ret; 2321 } 2322 2323 int bdrv_flush(BlockDriverState *bs) 2324 { 2325 Coroutine *co; 2326 FlushCo flush_co = { 2327 .bs = bs, 2328 .ret = NOT_DONE, 2329 }; 2330 2331 if (qemu_in_coroutine()) { 2332 /* Fast-path if already in coroutine context */ 2333 bdrv_flush_co_entry(&flush_co); 2334 } else { 2335 AioContext *aio_context = bdrv_get_aio_context(bs); 2336 2337 co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co); 2338 qemu_coroutine_enter(co); 2339 while (flush_co.ret == NOT_DONE) { 2340 aio_poll(aio_context, true); 2341 } 2342 } 2343 2344 return flush_co.ret; 2345 } 2346 2347 typedef struct DiscardCo { 2348 BlockDriverState *bs; 2349 int64_t sector_num; 2350 int nb_sectors; 2351 int ret; 2352 } DiscardCo; 2353 static void coroutine_fn bdrv_discard_co_entry(void *opaque) 2354 { 2355 DiscardCo *rwco = opaque; 2356 2357 rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors); 2358 } 2359 2360 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, 2361 int nb_sectors) 2362 { 2363 BdrvTrackedRequest req; 2364 int max_discard, ret; 2365 2366 if (!bs->drv) { 2367 return -ENOMEDIUM; 2368 } 2369 2370 ret = bdrv_check_request(bs, sector_num, nb_sectors); 2371 if (ret < 0) { 2372 return ret; 2373 } else if (bs->read_only) { 2374 return -EPERM; 2375 } 2376 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 2377 2378 /* Do nothing if disabled. */ 2379 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2380 return 0; 2381 } 2382 2383 if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) { 2384 return 0; 2385 } 2386 2387 tracked_request_begin(&req, bs, sector_num << BDRV_SECTOR_BITS, 2388 nb_sectors << BDRV_SECTOR_BITS, BDRV_TRACKED_DISCARD); 2389 2390 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req); 2391 if (ret < 0) { 2392 goto out; 2393 } 2394 2395 max_discard = MIN_NON_ZERO(bs->bl.max_pdiscard >> BDRV_SECTOR_BITS, 2396 BDRV_REQUEST_MAX_SECTORS); 2397 while (nb_sectors > 0) { 2398 int ret; 2399 int num = nb_sectors; 2400 int discard_alignment = bs->bl.pdiscard_alignment >> BDRV_SECTOR_BITS; 2401 2402 /* align request */ 2403 if (discard_alignment && 2404 num >= discard_alignment && 2405 sector_num % discard_alignment) { 2406 if (num > discard_alignment) { 2407 num = discard_alignment; 2408 } 2409 num -= sector_num % discard_alignment; 2410 } 2411 2412 /* limit request size */ 2413 if (num > max_discard) { 2414 num = max_discard; 2415 } 2416 2417 if (bs->drv->bdrv_co_discard) { 2418 ret = bs->drv->bdrv_co_discard(bs, sector_num, num); 2419 } else { 2420 BlockAIOCB *acb; 2421 CoroutineIOCompletion co = { 2422 .coroutine = qemu_coroutine_self(), 2423 }; 2424 2425 acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors, 2426 bdrv_co_io_em_complete, &co); 2427 if (acb == NULL) { 2428 ret = -EIO; 2429 goto out; 2430 } else { 2431 qemu_coroutine_yield(); 2432 ret = co.ret; 2433 } 2434 } 2435 if (ret && ret != -ENOTSUP) { 2436 goto out; 2437 } 2438 2439 sector_num += num; 2440 nb_sectors -= num; 2441 } 2442 ret = 0; 2443 out: 2444 ++bs->write_gen; 2445 bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS, 2446 req.bytes >> BDRV_SECTOR_BITS); 2447 tracked_request_end(&req); 2448 return ret; 2449 } 2450 2451 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors) 2452 { 2453 Coroutine *co; 2454 DiscardCo rwco = { 2455 .bs = bs, 2456 .sector_num = sector_num, 2457 .nb_sectors = nb_sectors, 2458 .ret = NOT_DONE, 2459 }; 2460 2461 if (qemu_in_coroutine()) { 2462 /* Fast-path if already in coroutine context */ 2463 bdrv_discard_co_entry(&rwco); 2464 } else { 2465 AioContext *aio_context = bdrv_get_aio_context(bs); 2466 2467 co = qemu_coroutine_create(bdrv_discard_co_entry, &rwco); 2468 qemu_coroutine_enter(co); 2469 while (rwco.ret == NOT_DONE) { 2470 aio_poll(aio_context, true); 2471 } 2472 } 2473 2474 return rwco.ret; 2475 } 2476 2477 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf) 2478 { 2479 BlockDriver *drv = bs->drv; 2480 BdrvTrackedRequest tracked_req; 2481 CoroutineIOCompletion co = { 2482 .coroutine = qemu_coroutine_self(), 2483 }; 2484 BlockAIOCB *acb; 2485 2486 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL); 2487 if (!drv || !drv->bdrv_aio_ioctl) { 2488 co.ret = -ENOTSUP; 2489 goto out; 2490 } 2491 2492 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 2493 if (!acb) { 2494 co.ret = -ENOTSUP; 2495 goto out; 2496 } 2497 qemu_coroutine_yield(); 2498 out: 2499 tracked_request_end(&tracked_req); 2500 return co.ret; 2501 } 2502 2503 typedef struct { 2504 BlockDriverState *bs; 2505 int req; 2506 void *buf; 2507 int ret; 2508 } BdrvIoctlCoData; 2509 2510 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque) 2511 { 2512 BdrvIoctlCoData *data = opaque; 2513 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf); 2514 } 2515 2516 /* needed for generic scsi interface */ 2517 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) 2518 { 2519 BdrvIoctlCoData data = { 2520 .bs = bs, 2521 .req = req, 2522 .buf = buf, 2523 .ret = -EINPROGRESS, 2524 }; 2525 2526 if (qemu_in_coroutine()) { 2527 /* Fast-path if already in coroutine context */ 2528 bdrv_co_ioctl_entry(&data); 2529 } else { 2530 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry, &data); 2531 2532 qemu_coroutine_enter(co); 2533 while (data.ret == -EINPROGRESS) { 2534 aio_poll(bdrv_get_aio_context(bs), true); 2535 } 2536 } 2537 return data.ret; 2538 } 2539 2540 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque) 2541 { 2542 BlockAIOCBCoroutine *acb = opaque; 2543 acb->req.error = bdrv_co_do_ioctl(acb->common.bs, 2544 acb->req.req, acb->req.buf); 2545 bdrv_co_complete(acb); 2546 } 2547 2548 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs, 2549 unsigned long int req, void *buf, 2550 BlockCompletionFunc *cb, void *opaque) 2551 { 2552 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info, 2553 bs, cb, opaque); 2554 Coroutine *co; 2555 2556 acb->need_bh = true; 2557 acb->req.error = -EINPROGRESS; 2558 acb->req.req = req; 2559 acb->req.buf = buf; 2560 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry, acb); 2561 qemu_coroutine_enter(co); 2562 2563 bdrv_co_maybe_schedule_bh(acb); 2564 return &acb->common; 2565 } 2566 2567 void *qemu_blockalign(BlockDriverState *bs, size_t size) 2568 { 2569 return qemu_memalign(bdrv_opt_mem_align(bs), size); 2570 } 2571 2572 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 2573 { 2574 return memset(qemu_blockalign(bs, size), 0, size); 2575 } 2576 2577 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 2578 { 2579 size_t align = bdrv_opt_mem_align(bs); 2580 2581 /* Ensure that NULL is never returned on success */ 2582 assert(align > 0); 2583 if (size == 0) { 2584 size = align; 2585 } 2586 2587 return qemu_try_memalign(align, size); 2588 } 2589 2590 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 2591 { 2592 void *mem = qemu_try_blockalign(bs, size); 2593 2594 if (mem) { 2595 memset(mem, 0, size); 2596 } 2597 2598 return mem; 2599 } 2600 2601 /* 2602 * Check if all memory in this vector is sector aligned. 2603 */ 2604 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 2605 { 2606 int i; 2607 size_t alignment = bdrv_min_mem_align(bs); 2608 2609 for (i = 0; i < qiov->niov; i++) { 2610 if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 2611 return false; 2612 } 2613 if (qiov->iov[i].iov_len % alignment) { 2614 return false; 2615 } 2616 } 2617 2618 return true; 2619 } 2620 2621 void bdrv_add_before_write_notifier(BlockDriverState *bs, 2622 NotifierWithReturn *notifier) 2623 { 2624 notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 2625 } 2626 2627 void bdrv_io_plug(BlockDriverState *bs) 2628 { 2629 BdrvChild *child; 2630 2631 QLIST_FOREACH(child, &bs->children, next) { 2632 bdrv_io_plug(child->bs); 2633 } 2634 2635 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) { 2636 BlockDriver *drv = bs->drv; 2637 if (drv && drv->bdrv_io_plug) { 2638 drv->bdrv_io_plug(bs); 2639 } 2640 } 2641 } 2642 2643 void bdrv_io_unplug(BlockDriverState *bs) 2644 { 2645 BdrvChild *child; 2646 2647 assert(bs->io_plugged); 2648 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) { 2649 BlockDriver *drv = bs->drv; 2650 if (drv && drv->bdrv_io_unplug) { 2651 drv->bdrv_io_unplug(bs); 2652 } 2653 } 2654 2655 QLIST_FOREACH(child, &bs->children, next) { 2656 bdrv_io_unplug(child->bs); 2657 } 2658 } 2659 2660 void bdrv_io_unplugged_begin(BlockDriverState *bs) 2661 { 2662 BdrvChild *child; 2663 2664 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) { 2665 BlockDriver *drv = bs->drv; 2666 if (drv && drv->bdrv_io_unplug) { 2667 drv->bdrv_io_unplug(bs); 2668 } 2669 } 2670 2671 QLIST_FOREACH(child, &bs->children, next) { 2672 bdrv_io_unplugged_begin(child->bs); 2673 } 2674 } 2675 2676 void bdrv_io_unplugged_end(BlockDriverState *bs) 2677 { 2678 BdrvChild *child; 2679 2680 assert(bs->io_plug_disabled); 2681 QLIST_FOREACH(child, &bs->children, next) { 2682 bdrv_io_unplugged_end(child->bs); 2683 } 2684 2685 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) { 2686 BlockDriver *drv = bs->drv; 2687 if (drv && drv->bdrv_io_plug) { 2688 drv->bdrv_io_plug(bs); 2689 } 2690 } 2691 } 2692