1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/blockjob.h" 29 #include "block/block_int.h" 30 #include "qemu/cutils.h" 31 #include "qapi/error.h" 32 #include "qemu/error-report.h" 33 34 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 35 36 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 37 int64_t sector_num, 38 QEMUIOVector *qiov, 39 int nb_sectors, 40 BdrvRequestFlags flags, 41 BlockCompletionFunc *cb, 42 void *opaque, 43 bool is_write); 44 static void coroutine_fn bdrv_co_do_rw(void *opaque); 45 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 46 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags); 47 48 static void bdrv_parent_drained_begin(BlockDriverState *bs) 49 { 50 BdrvChild *c; 51 52 QLIST_FOREACH(c, &bs->parents, next_parent) { 53 if (c->role->drained_begin) { 54 c->role->drained_begin(c); 55 } 56 } 57 } 58 59 static void bdrv_parent_drained_end(BlockDriverState *bs) 60 { 61 BdrvChild *c; 62 63 QLIST_FOREACH(c, &bs->parents, next_parent) { 64 if (c->role->drained_end) { 65 c->role->drained_end(c); 66 } 67 } 68 } 69 70 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 71 { 72 BlockDriver *drv = bs->drv; 73 Error *local_err = NULL; 74 75 memset(&bs->bl, 0, sizeof(bs->bl)); 76 77 if (!drv) { 78 return; 79 } 80 81 /* Take some limits from the children as a default */ 82 if (bs->file) { 83 bdrv_refresh_limits(bs->file->bs, &local_err); 84 if (local_err) { 85 error_propagate(errp, local_err); 86 return; 87 } 88 bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length; 89 bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length; 90 bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment; 91 bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment; 92 bs->bl.max_iov = bs->file->bs->bl.max_iov; 93 } else { 94 bs->bl.min_mem_alignment = 512; 95 bs->bl.opt_mem_alignment = getpagesize(); 96 97 /* Safe default since most protocols use readv()/writev()/etc */ 98 bs->bl.max_iov = IOV_MAX; 99 } 100 101 if (bs->backing) { 102 bdrv_refresh_limits(bs->backing->bs, &local_err); 103 if (local_err) { 104 error_propagate(errp, local_err); 105 return; 106 } 107 bs->bl.opt_transfer_length = 108 MAX(bs->bl.opt_transfer_length, 109 bs->backing->bs->bl.opt_transfer_length); 110 bs->bl.max_transfer_length = 111 MIN_NON_ZERO(bs->bl.max_transfer_length, 112 bs->backing->bs->bl.max_transfer_length); 113 bs->bl.opt_mem_alignment = 114 MAX(bs->bl.opt_mem_alignment, 115 bs->backing->bs->bl.opt_mem_alignment); 116 bs->bl.min_mem_alignment = 117 MAX(bs->bl.min_mem_alignment, 118 bs->backing->bs->bl.min_mem_alignment); 119 bs->bl.max_iov = 120 MIN(bs->bl.max_iov, 121 bs->backing->bs->bl.max_iov); 122 } 123 124 /* Then let the driver override it */ 125 if (drv->bdrv_refresh_limits) { 126 drv->bdrv_refresh_limits(bs, errp); 127 } 128 } 129 130 /** 131 * The copy-on-read flag is actually a reference count so multiple users may 132 * use the feature without worrying about clobbering its previous state. 133 * Copy-on-read stays enabled until all users have called to disable it. 134 */ 135 void bdrv_enable_copy_on_read(BlockDriverState *bs) 136 { 137 bs->copy_on_read++; 138 } 139 140 void bdrv_disable_copy_on_read(BlockDriverState *bs) 141 { 142 assert(bs->copy_on_read > 0); 143 bs->copy_on_read--; 144 } 145 146 /* Check if any requests are in-flight (including throttled requests) */ 147 bool bdrv_requests_pending(BlockDriverState *bs) 148 { 149 BdrvChild *child; 150 151 if (!QLIST_EMPTY(&bs->tracked_requests)) { 152 return true; 153 } 154 155 QLIST_FOREACH(child, &bs->children, next) { 156 if (bdrv_requests_pending(child->bs)) { 157 return true; 158 } 159 } 160 161 return false; 162 } 163 164 static void bdrv_drain_recurse(BlockDriverState *bs) 165 { 166 BdrvChild *child; 167 168 if (bs->drv && bs->drv->bdrv_drain) { 169 bs->drv->bdrv_drain(bs); 170 } 171 QLIST_FOREACH(child, &bs->children, next) { 172 bdrv_drain_recurse(child->bs); 173 } 174 } 175 176 typedef struct { 177 Coroutine *co; 178 BlockDriverState *bs; 179 QEMUBH *bh; 180 bool done; 181 } BdrvCoDrainData; 182 183 static void bdrv_drain_poll(BlockDriverState *bs) 184 { 185 bool busy = true; 186 187 while (busy) { 188 /* Keep iterating */ 189 busy = bdrv_requests_pending(bs); 190 busy |= aio_poll(bdrv_get_aio_context(bs), busy); 191 } 192 } 193 194 static void bdrv_co_drain_bh_cb(void *opaque) 195 { 196 BdrvCoDrainData *data = opaque; 197 Coroutine *co = data->co; 198 199 qemu_bh_delete(data->bh); 200 bdrv_drain_poll(data->bs); 201 data->done = true; 202 qemu_coroutine_enter(co, NULL); 203 } 204 205 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs) 206 { 207 BdrvCoDrainData data; 208 209 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 210 * other coroutines run if they were queued from 211 * qemu_co_queue_run_restart(). */ 212 213 assert(qemu_in_coroutine()); 214 data = (BdrvCoDrainData) { 215 .co = qemu_coroutine_self(), 216 .bs = bs, 217 .done = false, 218 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data), 219 }; 220 qemu_bh_schedule(data.bh); 221 222 qemu_coroutine_yield(); 223 /* If we are resumed from some other event (such as an aio completion or a 224 * timer callback), it is a bug in the caller that should be fixed. */ 225 assert(data.done); 226 } 227 228 /* 229 * Wait for pending requests to complete on a single BlockDriverState subtree, 230 * and suspend block driver's internal I/O until next request arrives. 231 * 232 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 233 * AioContext. 234 * 235 * Only this BlockDriverState's AioContext is run, so in-flight requests must 236 * not depend on events in other AioContexts. In that case, use 237 * bdrv_drain_all() instead. 238 */ 239 void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 240 { 241 bdrv_parent_drained_begin(bs); 242 bdrv_io_unplugged_begin(bs); 243 bdrv_drain_recurse(bs); 244 bdrv_co_yield_to_drain(bs); 245 bdrv_io_unplugged_end(bs); 246 bdrv_parent_drained_end(bs); 247 } 248 249 void bdrv_drain(BlockDriverState *bs) 250 { 251 bdrv_parent_drained_begin(bs); 252 bdrv_io_unplugged_begin(bs); 253 bdrv_drain_recurse(bs); 254 if (qemu_in_coroutine()) { 255 bdrv_co_yield_to_drain(bs); 256 } else { 257 bdrv_drain_poll(bs); 258 } 259 bdrv_io_unplugged_end(bs); 260 bdrv_parent_drained_end(bs); 261 } 262 263 /* 264 * Wait for pending requests to complete across all BlockDriverStates 265 * 266 * This function does not flush data to disk, use bdrv_flush_all() for that 267 * after calling this function. 268 */ 269 void bdrv_drain_all(void) 270 { 271 /* Always run first iteration so any pending completion BHs run */ 272 bool busy = true; 273 BlockDriverState *bs; 274 BdrvNextIterator *it = NULL; 275 GSList *aio_ctxs = NULL, *ctx; 276 277 while ((it = bdrv_next(it, &bs))) { 278 AioContext *aio_context = bdrv_get_aio_context(bs); 279 280 aio_context_acquire(aio_context); 281 if (bs->job) { 282 block_job_pause(bs->job); 283 } 284 bdrv_parent_drained_begin(bs); 285 bdrv_io_unplugged_begin(bs); 286 bdrv_drain_recurse(bs); 287 aio_context_release(aio_context); 288 289 if (!g_slist_find(aio_ctxs, aio_context)) { 290 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 291 } 292 } 293 294 /* Note that completion of an asynchronous I/O operation can trigger any 295 * number of other I/O operations on other devices---for example a 296 * coroutine can submit an I/O request to another device in response to 297 * request completion. Therefore we must keep looping until there was no 298 * more activity rather than simply draining each device independently. 299 */ 300 while (busy) { 301 busy = false; 302 303 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 304 AioContext *aio_context = ctx->data; 305 it = NULL; 306 307 aio_context_acquire(aio_context); 308 while ((it = bdrv_next(it, &bs))) { 309 if (aio_context == bdrv_get_aio_context(bs)) { 310 if (bdrv_requests_pending(bs)) { 311 busy = true; 312 aio_poll(aio_context, busy); 313 } 314 } 315 } 316 busy |= aio_poll(aio_context, false); 317 aio_context_release(aio_context); 318 } 319 } 320 321 it = NULL; 322 while ((it = bdrv_next(it, &bs))) { 323 AioContext *aio_context = bdrv_get_aio_context(bs); 324 325 aio_context_acquire(aio_context); 326 bdrv_io_unplugged_end(bs); 327 bdrv_parent_drained_end(bs); 328 if (bs->job) { 329 block_job_resume(bs->job); 330 } 331 aio_context_release(aio_context); 332 } 333 g_slist_free(aio_ctxs); 334 } 335 336 /** 337 * Remove an active request from the tracked requests list 338 * 339 * This function should be called when a tracked request is completing. 340 */ 341 static void tracked_request_end(BdrvTrackedRequest *req) 342 { 343 if (req->serialising) { 344 req->bs->serialising_in_flight--; 345 } 346 347 QLIST_REMOVE(req, list); 348 qemu_co_queue_restart_all(&req->wait_queue); 349 } 350 351 /** 352 * Add an active request to the tracked requests list 353 */ 354 static void tracked_request_begin(BdrvTrackedRequest *req, 355 BlockDriverState *bs, 356 int64_t offset, 357 unsigned int bytes, 358 enum BdrvTrackedRequestType type) 359 { 360 *req = (BdrvTrackedRequest){ 361 .bs = bs, 362 .offset = offset, 363 .bytes = bytes, 364 .type = type, 365 .co = qemu_coroutine_self(), 366 .serialising = false, 367 .overlap_offset = offset, 368 .overlap_bytes = bytes, 369 }; 370 371 qemu_co_queue_init(&req->wait_queue); 372 373 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 374 } 375 376 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 377 { 378 int64_t overlap_offset = req->offset & ~(align - 1); 379 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 380 - overlap_offset; 381 382 if (!req->serialising) { 383 req->bs->serialising_in_flight++; 384 req->serialising = true; 385 } 386 387 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 388 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 389 } 390 391 /** 392 * Round a region to cluster boundaries 393 */ 394 void bdrv_round_to_clusters(BlockDriverState *bs, 395 int64_t sector_num, int nb_sectors, 396 int64_t *cluster_sector_num, 397 int *cluster_nb_sectors) 398 { 399 BlockDriverInfo bdi; 400 401 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 402 *cluster_sector_num = sector_num; 403 *cluster_nb_sectors = nb_sectors; 404 } else { 405 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 406 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 407 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 408 nb_sectors, c); 409 } 410 } 411 412 static int bdrv_get_cluster_size(BlockDriverState *bs) 413 { 414 BlockDriverInfo bdi; 415 int ret; 416 417 ret = bdrv_get_info(bs, &bdi); 418 if (ret < 0 || bdi.cluster_size == 0) { 419 return bs->request_alignment; 420 } else { 421 return bdi.cluster_size; 422 } 423 } 424 425 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 426 int64_t offset, unsigned int bytes) 427 { 428 /* aaaa bbbb */ 429 if (offset >= req->overlap_offset + req->overlap_bytes) { 430 return false; 431 } 432 /* bbbb aaaa */ 433 if (req->overlap_offset >= offset + bytes) { 434 return false; 435 } 436 return true; 437 } 438 439 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 440 { 441 BlockDriverState *bs = self->bs; 442 BdrvTrackedRequest *req; 443 bool retry; 444 bool waited = false; 445 446 if (!bs->serialising_in_flight) { 447 return false; 448 } 449 450 do { 451 retry = false; 452 QLIST_FOREACH(req, &bs->tracked_requests, list) { 453 if (req == self || (!req->serialising && !self->serialising)) { 454 continue; 455 } 456 if (tracked_request_overlaps(req, self->overlap_offset, 457 self->overlap_bytes)) 458 { 459 /* Hitting this means there was a reentrant request, for 460 * example, a block driver issuing nested requests. This must 461 * never happen since it means deadlock. 462 */ 463 assert(qemu_coroutine_self() != req->co); 464 465 /* If the request is already (indirectly) waiting for us, or 466 * will wait for us as soon as it wakes up, then just go on 467 * (instead of producing a deadlock in the former case). */ 468 if (!req->waiting_for) { 469 self->waiting_for = req; 470 qemu_co_queue_wait(&req->wait_queue); 471 self->waiting_for = NULL; 472 retry = true; 473 waited = true; 474 break; 475 } 476 } 477 } 478 } while (retry); 479 480 return waited; 481 } 482 483 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 484 size_t size) 485 { 486 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 487 return -EIO; 488 } 489 490 if (!bdrv_is_inserted(bs)) { 491 return -ENOMEDIUM; 492 } 493 494 if (offset < 0) { 495 return -EIO; 496 } 497 498 return 0; 499 } 500 501 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num, 502 int nb_sectors) 503 { 504 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 505 return -EIO; 506 } 507 508 return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE, 509 nb_sectors * BDRV_SECTOR_SIZE); 510 } 511 512 typedef struct RwCo { 513 BlockDriverState *bs; 514 int64_t offset; 515 QEMUIOVector *qiov; 516 bool is_write; 517 int ret; 518 BdrvRequestFlags flags; 519 } RwCo; 520 521 static void coroutine_fn bdrv_rw_co_entry(void *opaque) 522 { 523 RwCo *rwco = opaque; 524 525 if (!rwco->is_write) { 526 rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset, 527 rwco->qiov->size, rwco->qiov, 528 rwco->flags); 529 } else { 530 rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset, 531 rwco->qiov->size, rwco->qiov, 532 rwco->flags); 533 } 534 } 535 536 /* 537 * Process a vectored synchronous request using coroutines 538 */ 539 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset, 540 QEMUIOVector *qiov, bool is_write, 541 BdrvRequestFlags flags) 542 { 543 Coroutine *co; 544 RwCo rwco = { 545 .bs = bs, 546 .offset = offset, 547 .qiov = qiov, 548 .is_write = is_write, 549 .ret = NOT_DONE, 550 .flags = flags, 551 }; 552 553 if (qemu_in_coroutine()) { 554 /* Fast-path if already in coroutine context */ 555 bdrv_rw_co_entry(&rwco); 556 } else { 557 AioContext *aio_context = bdrv_get_aio_context(bs); 558 559 co = qemu_coroutine_create(bdrv_rw_co_entry); 560 qemu_coroutine_enter(co, &rwco); 561 while (rwco.ret == NOT_DONE) { 562 aio_poll(aio_context, true); 563 } 564 } 565 return rwco.ret; 566 } 567 568 /* 569 * Process a synchronous request using coroutines 570 */ 571 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf, 572 int nb_sectors, bool is_write, BdrvRequestFlags flags) 573 { 574 QEMUIOVector qiov; 575 struct iovec iov = { 576 .iov_base = (void *)buf, 577 .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 578 }; 579 580 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 581 return -EINVAL; 582 } 583 584 qemu_iovec_init_external(&qiov, &iov, 1); 585 return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS, 586 &qiov, is_write, flags); 587 } 588 589 /* return < 0 if error. See bdrv_write() for the return codes */ 590 int bdrv_read(BlockDriverState *bs, int64_t sector_num, 591 uint8_t *buf, int nb_sectors) 592 { 593 return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0); 594 } 595 596 /* Return < 0 if error. Important errors are: 597 -EIO generic I/O error (may happen for all errors) 598 -ENOMEDIUM No media inserted. 599 -EINVAL Invalid sector number or nb_sectors 600 -EACCES Trying to write a read-only device 601 */ 602 int bdrv_write(BlockDriverState *bs, int64_t sector_num, 603 const uint8_t *buf, int nb_sectors) 604 { 605 return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 606 } 607 608 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num, 609 int nb_sectors, BdrvRequestFlags flags) 610 { 611 return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true, 612 BDRV_REQ_ZERO_WRITE | flags); 613 } 614 615 /* 616 * Completely zero out a block device with the help of bdrv_write_zeroes. 617 * The operation is sped up by checking the block status and only writing 618 * zeroes to the device if they currently do not return zeroes. Optional 619 * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 620 * BDRV_REQ_FUA). 621 * 622 * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 623 */ 624 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags) 625 { 626 int64_t target_sectors, ret, nb_sectors, sector_num = 0; 627 BlockDriverState *file; 628 int n; 629 630 target_sectors = bdrv_nb_sectors(bs); 631 if (target_sectors < 0) { 632 return target_sectors; 633 } 634 635 for (;;) { 636 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 637 if (nb_sectors <= 0) { 638 return 0; 639 } 640 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file); 641 if (ret < 0) { 642 error_report("error getting block status at sector %" PRId64 ": %s", 643 sector_num, strerror(-ret)); 644 return ret; 645 } 646 if (ret & BDRV_BLOCK_ZERO) { 647 sector_num += n; 648 continue; 649 } 650 ret = bdrv_write_zeroes(bs, sector_num, n, flags); 651 if (ret < 0) { 652 error_report("error writing zeroes at sector %" PRId64 ": %s", 653 sector_num, strerror(-ret)); 654 return ret; 655 } 656 sector_num += n; 657 } 658 } 659 660 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes) 661 { 662 QEMUIOVector qiov; 663 struct iovec iov = { 664 .iov_base = (void *)buf, 665 .iov_len = bytes, 666 }; 667 int ret; 668 669 if (bytes < 0) { 670 return -EINVAL; 671 } 672 673 qemu_iovec_init_external(&qiov, &iov, 1); 674 ret = bdrv_prwv_co(bs, offset, &qiov, false, 0); 675 if (ret < 0) { 676 return ret; 677 } 678 679 return bytes; 680 } 681 682 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov) 683 { 684 int ret; 685 686 ret = bdrv_prwv_co(bs, offset, qiov, true, 0); 687 if (ret < 0) { 688 return ret; 689 } 690 691 return qiov->size; 692 } 693 694 int bdrv_pwrite(BlockDriverState *bs, int64_t offset, 695 const void *buf, int bytes) 696 { 697 QEMUIOVector qiov; 698 struct iovec iov = { 699 .iov_base = (void *) buf, 700 .iov_len = bytes, 701 }; 702 703 if (bytes < 0) { 704 return -EINVAL; 705 } 706 707 qemu_iovec_init_external(&qiov, &iov, 1); 708 return bdrv_pwritev(bs, offset, &qiov); 709 } 710 711 /* 712 * Writes to the file and ensures that no writes are reordered across this 713 * request (acts as a barrier) 714 * 715 * Returns 0 on success, -errno in error cases. 716 */ 717 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset, 718 const void *buf, int count) 719 { 720 int ret; 721 722 ret = bdrv_pwrite(bs, offset, buf, count); 723 if (ret < 0) { 724 return ret; 725 } 726 727 ret = bdrv_flush(bs); 728 if (ret < 0) { 729 return ret; 730 } 731 732 return 0; 733 } 734 735 typedef struct CoroutineIOCompletion { 736 Coroutine *coroutine; 737 int ret; 738 } CoroutineIOCompletion; 739 740 static void bdrv_co_io_em_complete(void *opaque, int ret) 741 { 742 CoroutineIOCompletion *co = opaque; 743 744 co->ret = ret; 745 qemu_coroutine_enter(co->coroutine, NULL); 746 } 747 748 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 749 uint64_t offset, uint64_t bytes, 750 QEMUIOVector *qiov, int flags) 751 { 752 BlockDriver *drv = bs->drv; 753 int64_t sector_num; 754 unsigned int nb_sectors; 755 756 if (drv->bdrv_co_preadv) { 757 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 758 } 759 760 sector_num = offset >> BDRV_SECTOR_BITS; 761 nb_sectors = bytes >> BDRV_SECTOR_BITS; 762 763 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 764 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 765 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 766 767 if (drv->bdrv_co_readv) { 768 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 769 } else { 770 BlockAIOCB *acb; 771 CoroutineIOCompletion co = { 772 .coroutine = qemu_coroutine_self(), 773 }; 774 775 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 776 bdrv_co_io_em_complete, &co); 777 if (acb == NULL) { 778 return -EIO; 779 } else { 780 qemu_coroutine_yield(); 781 return co.ret; 782 } 783 } 784 } 785 786 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 787 uint64_t offset, uint64_t bytes, 788 QEMUIOVector *qiov, int flags) 789 { 790 BlockDriver *drv = bs->drv; 791 int64_t sector_num; 792 unsigned int nb_sectors; 793 int ret; 794 795 if (drv->bdrv_co_pwritev) { 796 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags); 797 goto emulate_flags; 798 } 799 800 sector_num = offset >> BDRV_SECTOR_BITS; 801 nb_sectors = bytes >> BDRV_SECTOR_BITS; 802 803 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 804 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 805 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 806 807 if (drv->bdrv_co_writev_flags) { 808 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 809 flags & bs->supported_write_flags); 810 flags &= ~bs->supported_write_flags; 811 } else if (drv->bdrv_co_writev) { 812 assert(!bs->supported_write_flags); 813 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 814 } else { 815 BlockAIOCB *acb; 816 CoroutineIOCompletion co = { 817 .coroutine = qemu_coroutine_self(), 818 }; 819 820 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 821 bdrv_co_io_em_complete, &co); 822 if (acb == NULL) { 823 ret = -EIO; 824 } else { 825 qemu_coroutine_yield(); 826 ret = co.ret; 827 } 828 } 829 830 emulate_flags: 831 if (ret == 0 && (flags & BDRV_REQ_FUA)) { 832 ret = bdrv_co_flush(bs); 833 } 834 835 return ret; 836 } 837 838 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs, 839 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 840 { 841 /* Perform I/O through a temporary buffer so that users who scribble over 842 * their read buffer while the operation is in progress do not end up 843 * modifying the image file. This is critical for zero-copy guest I/O 844 * where anything might happen inside guest memory. 845 */ 846 void *bounce_buffer; 847 848 BlockDriver *drv = bs->drv; 849 struct iovec iov; 850 QEMUIOVector bounce_qiov; 851 int64_t cluster_sector_num; 852 int cluster_nb_sectors; 853 size_t skip_bytes; 854 int ret; 855 856 /* Cover entire cluster so no additional backing file I/O is required when 857 * allocating cluster in the image file. 858 */ 859 bdrv_round_to_clusters(bs, sector_num, nb_sectors, 860 &cluster_sector_num, &cluster_nb_sectors); 861 862 trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, 863 cluster_sector_num, cluster_nb_sectors); 864 865 iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE; 866 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 867 if (bounce_buffer == NULL) { 868 ret = -ENOMEM; 869 goto err; 870 } 871 872 qemu_iovec_init_external(&bounce_qiov, &iov, 1); 873 874 ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE, 875 cluster_nb_sectors * BDRV_SECTOR_SIZE, 876 &bounce_qiov, 0); 877 if (ret < 0) { 878 goto err; 879 } 880 881 if (drv->bdrv_co_write_zeroes && 882 buffer_is_zero(bounce_buffer, iov.iov_len)) { 883 ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num, 884 cluster_nb_sectors, 0); 885 } else { 886 /* This does not change the data on the disk, it is not necessary 887 * to flush even in cache=writethrough mode. 888 */ 889 ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE, 890 cluster_nb_sectors * BDRV_SECTOR_SIZE, 891 &bounce_qiov, 0); 892 } 893 894 if (ret < 0) { 895 /* It might be okay to ignore write errors for guest requests. If this 896 * is a deliberate copy-on-read then we don't want to ignore the error. 897 * Simply report it in all cases. 898 */ 899 goto err; 900 } 901 902 skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE; 903 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, 904 nb_sectors * BDRV_SECTOR_SIZE); 905 906 err: 907 qemu_vfree(bounce_buffer); 908 return ret; 909 } 910 911 /* 912 * Forwards an already correctly aligned request to the BlockDriver. This 913 * handles copy on read and zeroing after EOF; any other features must be 914 * implemented by the caller. 915 */ 916 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs, 917 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 918 int64_t align, QEMUIOVector *qiov, int flags) 919 { 920 int ret; 921 922 int64_t sector_num = offset >> BDRV_SECTOR_BITS; 923 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 924 925 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 926 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 927 assert(!qiov || bytes == qiov->size); 928 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 929 930 /* Handle Copy on Read and associated serialisation */ 931 if (flags & BDRV_REQ_COPY_ON_READ) { 932 /* If we touch the same cluster it counts as an overlap. This 933 * guarantees that allocating writes will be serialized and not race 934 * with each other for the same cluster. For example, in copy-on-read 935 * it ensures that the CoR read and write operations are atomic and 936 * guest writes cannot interleave between them. */ 937 mark_request_serialising(req, bdrv_get_cluster_size(bs)); 938 } 939 940 if (!(flags & BDRV_REQ_NO_SERIALISING)) { 941 wait_serialising_requests(req); 942 } 943 944 if (flags & BDRV_REQ_COPY_ON_READ) { 945 int pnum; 946 947 ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum); 948 if (ret < 0) { 949 goto out; 950 } 951 952 if (!ret || pnum != nb_sectors) { 953 ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov); 954 goto out; 955 } 956 } 957 958 /* Forward the request to the BlockDriver */ 959 if (!bs->zero_beyond_eof) { 960 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 961 } else { 962 /* Read zeros after EOF */ 963 int64_t total_sectors, max_nb_sectors; 964 965 total_sectors = bdrv_nb_sectors(bs); 966 if (total_sectors < 0) { 967 ret = total_sectors; 968 goto out; 969 } 970 971 max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num), 972 align >> BDRV_SECTOR_BITS); 973 if (nb_sectors < max_nb_sectors) { 974 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 975 } else if (max_nb_sectors > 0) { 976 QEMUIOVector local_qiov; 977 978 qemu_iovec_init(&local_qiov, qiov->niov); 979 qemu_iovec_concat(&local_qiov, qiov, 0, 980 max_nb_sectors * BDRV_SECTOR_SIZE); 981 982 ret = bdrv_driver_preadv(bs, offset, 983 max_nb_sectors * BDRV_SECTOR_SIZE, 984 &local_qiov, 0); 985 986 qemu_iovec_destroy(&local_qiov); 987 } else { 988 ret = 0; 989 } 990 991 /* Reading beyond end of file is supposed to produce zeroes */ 992 if (ret == 0 && total_sectors < sector_num + nb_sectors) { 993 uint64_t offset = MAX(0, total_sectors - sector_num); 994 uint64_t bytes = (sector_num + nb_sectors - offset) * 995 BDRV_SECTOR_SIZE; 996 qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes); 997 } 998 } 999 1000 out: 1001 return ret; 1002 } 1003 1004 /* 1005 * Handle a read request in coroutine context 1006 */ 1007 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs, 1008 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1009 BdrvRequestFlags flags) 1010 { 1011 BlockDriver *drv = bs->drv; 1012 BdrvTrackedRequest req; 1013 1014 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 1015 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 1016 uint8_t *head_buf = NULL; 1017 uint8_t *tail_buf = NULL; 1018 QEMUIOVector local_qiov; 1019 bool use_local_qiov = false; 1020 int ret; 1021 1022 if (!drv) { 1023 return -ENOMEDIUM; 1024 } 1025 1026 ret = bdrv_check_byte_request(bs, offset, bytes); 1027 if (ret < 0) { 1028 return ret; 1029 } 1030 1031 /* Don't do copy-on-read if we read data before write operation */ 1032 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 1033 flags |= BDRV_REQ_COPY_ON_READ; 1034 } 1035 1036 /* Align read if necessary by padding qiov */ 1037 if (offset & (align - 1)) { 1038 head_buf = qemu_blockalign(bs, align); 1039 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1040 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1041 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1042 use_local_qiov = true; 1043 1044 bytes += offset & (align - 1); 1045 offset = offset & ~(align - 1); 1046 } 1047 1048 if ((offset + bytes) & (align - 1)) { 1049 if (!use_local_qiov) { 1050 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1051 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1052 use_local_qiov = true; 1053 } 1054 tail_buf = qemu_blockalign(bs, align); 1055 qemu_iovec_add(&local_qiov, tail_buf, 1056 align - ((offset + bytes) & (align - 1))); 1057 1058 bytes = ROUND_UP(bytes, align); 1059 } 1060 1061 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1062 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align, 1063 use_local_qiov ? &local_qiov : qiov, 1064 flags); 1065 tracked_request_end(&req); 1066 1067 if (use_local_qiov) { 1068 qemu_iovec_destroy(&local_qiov); 1069 qemu_vfree(head_buf); 1070 qemu_vfree(tail_buf); 1071 } 1072 1073 return ret; 1074 } 1075 1076 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs, 1077 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1078 BdrvRequestFlags flags) 1079 { 1080 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1081 return -EINVAL; 1082 } 1083 1084 return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS, 1085 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1086 } 1087 1088 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num, 1089 int nb_sectors, QEMUIOVector *qiov) 1090 { 1091 trace_bdrv_co_readv(bs, sector_num, nb_sectors); 1092 1093 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0); 1094 } 1095 1096 int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs, 1097 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 1098 { 1099 trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors); 1100 1101 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 1102 BDRV_REQ_NO_SERIALISING); 1103 } 1104 1105 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs, 1106 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 1107 { 1108 trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors); 1109 1110 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 1111 BDRV_REQ_COPY_ON_READ); 1112 } 1113 1114 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768 1115 1116 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 1117 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags) 1118 { 1119 BlockDriver *drv = bs->drv; 1120 QEMUIOVector qiov; 1121 struct iovec iov = {0}; 1122 int ret = 0; 1123 bool need_flush = false; 1124 1125 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes, 1126 BDRV_REQUEST_MAX_SECTORS); 1127 1128 while (nb_sectors > 0 && !ret) { 1129 int num = nb_sectors; 1130 1131 /* Align request. Block drivers can expect the "bulk" of the request 1132 * to be aligned. 1133 */ 1134 if (bs->bl.write_zeroes_alignment 1135 && num > bs->bl.write_zeroes_alignment) { 1136 if (sector_num % bs->bl.write_zeroes_alignment != 0) { 1137 /* Make a small request up to the first aligned sector. */ 1138 num = bs->bl.write_zeroes_alignment; 1139 num -= sector_num % bs->bl.write_zeroes_alignment; 1140 } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) { 1141 /* Shorten the request to the last aligned sector. num cannot 1142 * underflow because num > bs->bl.write_zeroes_alignment. 1143 */ 1144 num -= (sector_num + num) % bs->bl.write_zeroes_alignment; 1145 } 1146 } 1147 1148 /* limit request size */ 1149 if (num > max_write_zeroes) { 1150 num = max_write_zeroes; 1151 } 1152 1153 ret = -ENOTSUP; 1154 /* First try the efficient write zeroes operation */ 1155 if (drv->bdrv_co_write_zeroes) { 1156 ret = drv->bdrv_co_write_zeroes(bs, sector_num, num, 1157 flags & bs->supported_zero_flags); 1158 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1159 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1160 need_flush = true; 1161 } 1162 } else { 1163 assert(!bs->supported_zero_flags); 1164 } 1165 1166 if (ret == -ENOTSUP) { 1167 /* Fall back to bounce buffer if write zeroes is unsupported */ 1168 int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length, 1169 MAX_WRITE_ZEROES_BOUNCE_BUFFER); 1170 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1171 1172 if ((flags & BDRV_REQ_FUA) && 1173 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1174 /* No need for bdrv_driver_pwrite() to do a fallback 1175 * flush on each chunk; use just one at the end */ 1176 write_flags &= ~BDRV_REQ_FUA; 1177 need_flush = true; 1178 } 1179 num = MIN(num, max_xfer_len); 1180 iov.iov_len = num * BDRV_SECTOR_SIZE; 1181 if (iov.iov_base == NULL) { 1182 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE); 1183 if (iov.iov_base == NULL) { 1184 ret = -ENOMEM; 1185 goto fail; 1186 } 1187 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE); 1188 } 1189 qemu_iovec_init_external(&qiov, &iov, 1); 1190 1191 ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE, 1192 num * BDRV_SECTOR_SIZE, &qiov, 1193 write_flags); 1194 1195 /* Keep bounce buffer around if it is big enough for all 1196 * all future requests. 1197 */ 1198 if (num < max_xfer_len) { 1199 qemu_vfree(iov.iov_base); 1200 iov.iov_base = NULL; 1201 } 1202 } 1203 1204 sector_num += num; 1205 nb_sectors -= num; 1206 } 1207 1208 fail: 1209 if (ret == 0 && need_flush) { 1210 ret = bdrv_co_flush(bs); 1211 } 1212 qemu_vfree(iov.iov_base); 1213 return ret; 1214 } 1215 1216 /* 1217 * Forwards an already correctly aligned write request to the BlockDriver. 1218 */ 1219 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs, 1220 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1221 QEMUIOVector *qiov, int flags) 1222 { 1223 BlockDriver *drv = bs->drv; 1224 bool waited; 1225 int ret; 1226 1227 int64_t sector_num = offset >> BDRV_SECTOR_BITS; 1228 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 1229 1230 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 1231 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 1232 assert(!qiov || bytes == qiov->size); 1233 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1234 1235 waited = wait_serialising_requests(req); 1236 assert(!waited || !req->serialising); 1237 assert(req->overlap_offset <= offset); 1238 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1239 1240 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 1241 1242 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1243 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes && 1244 qemu_iovec_is_zero(qiov)) { 1245 flags |= BDRV_REQ_ZERO_WRITE; 1246 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1247 flags |= BDRV_REQ_MAY_UNMAP; 1248 } 1249 } 1250 1251 if (ret < 0) { 1252 /* Do nothing, write notifier decided to fail this request */ 1253 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1254 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1255 ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags); 1256 } else { 1257 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1258 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 1259 } 1260 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 1261 1262 bdrv_set_dirty(bs, sector_num, nb_sectors); 1263 1264 if (bs->wr_highest_offset < offset + bytes) { 1265 bs->wr_highest_offset = offset + bytes; 1266 } 1267 1268 if (ret >= 0) { 1269 bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors); 1270 } 1271 1272 return ret; 1273 } 1274 1275 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs, 1276 int64_t offset, 1277 unsigned int bytes, 1278 BdrvRequestFlags flags, 1279 BdrvTrackedRequest *req) 1280 { 1281 uint8_t *buf = NULL; 1282 QEMUIOVector local_qiov; 1283 struct iovec iov; 1284 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 1285 unsigned int head_padding_bytes, tail_padding_bytes; 1286 int ret = 0; 1287 1288 head_padding_bytes = offset & (align - 1); 1289 tail_padding_bytes = align - ((offset + bytes) & (align - 1)); 1290 1291 1292 assert(flags & BDRV_REQ_ZERO_WRITE); 1293 if (head_padding_bytes || tail_padding_bytes) { 1294 buf = qemu_blockalign(bs, align); 1295 iov = (struct iovec) { 1296 .iov_base = buf, 1297 .iov_len = align, 1298 }; 1299 qemu_iovec_init_external(&local_qiov, &iov, 1); 1300 } 1301 if (head_padding_bytes) { 1302 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 1303 1304 /* RMW the unaligned part before head. */ 1305 mark_request_serialising(req, align); 1306 wait_serialising_requests(req); 1307 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1308 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align, 1309 align, &local_qiov, 0); 1310 if (ret < 0) { 1311 goto fail; 1312 } 1313 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1314 1315 memset(buf + head_padding_bytes, 0, zero_bytes); 1316 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align, 1317 &local_qiov, 1318 flags & ~BDRV_REQ_ZERO_WRITE); 1319 if (ret < 0) { 1320 goto fail; 1321 } 1322 offset += zero_bytes; 1323 bytes -= zero_bytes; 1324 } 1325 1326 assert(!bytes || (offset & (align - 1)) == 0); 1327 if (bytes >= align) { 1328 /* Write the aligned part in the middle. */ 1329 uint64_t aligned_bytes = bytes & ~(align - 1); 1330 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, 1331 NULL, flags); 1332 if (ret < 0) { 1333 goto fail; 1334 } 1335 bytes -= aligned_bytes; 1336 offset += aligned_bytes; 1337 } 1338 1339 assert(!bytes || (offset & (align - 1)) == 0); 1340 if (bytes) { 1341 assert(align == tail_padding_bytes + bytes); 1342 /* RMW the unaligned part after tail. */ 1343 mark_request_serialising(req, align); 1344 wait_serialising_requests(req); 1345 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1346 ret = bdrv_aligned_preadv(bs, req, offset, align, 1347 align, &local_qiov, 0); 1348 if (ret < 0) { 1349 goto fail; 1350 } 1351 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1352 1353 memset(buf, 0, bytes); 1354 ret = bdrv_aligned_pwritev(bs, req, offset, align, 1355 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 1356 } 1357 fail: 1358 qemu_vfree(buf); 1359 return ret; 1360 1361 } 1362 1363 /* 1364 * Handle a write request in coroutine context 1365 */ 1366 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs, 1367 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1368 BdrvRequestFlags flags) 1369 { 1370 BdrvTrackedRequest req; 1371 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 1372 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 1373 uint8_t *head_buf = NULL; 1374 uint8_t *tail_buf = NULL; 1375 QEMUIOVector local_qiov; 1376 bool use_local_qiov = false; 1377 int ret; 1378 1379 if (!bs->drv) { 1380 return -ENOMEDIUM; 1381 } 1382 if (bs->read_only) { 1383 return -EPERM; 1384 } 1385 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1386 1387 ret = bdrv_check_byte_request(bs, offset, bytes); 1388 if (ret < 0) { 1389 return ret; 1390 } 1391 1392 /* 1393 * Align write if necessary by performing a read-modify-write cycle. 1394 * Pad qiov with the read parts and be sure to have a tracked request not 1395 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 1396 */ 1397 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 1398 1399 if (!qiov) { 1400 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req); 1401 goto out; 1402 } 1403 1404 if (offset & (align - 1)) { 1405 QEMUIOVector head_qiov; 1406 struct iovec head_iov; 1407 1408 mark_request_serialising(&req, align); 1409 wait_serialising_requests(&req); 1410 1411 head_buf = qemu_blockalign(bs, align); 1412 head_iov = (struct iovec) { 1413 .iov_base = head_buf, 1414 .iov_len = align, 1415 }; 1416 qemu_iovec_init_external(&head_qiov, &head_iov, 1); 1417 1418 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1419 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align, 1420 align, &head_qiov, 0); 1421 if (ret < 0) { 1422 goto fail; 1423 } 1424 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1425 1426 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1427 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1428 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1429 use_local_qiov = true; 1430 1431 bytes += offset & (align - 1); 1432 offset = offset & ~(align - 1); 1433 } 1434 1435 if ((offset + bytes) & (align - 1)) { 1436 QEMUIOVector tail_qiov; 1437 struct iovec tail_iov; 1438 size_t tail_bytes; 1439 bool waited; 1440 1441 mark_request_serialising(&req, align); 1442 waited = wait_serialising_requests(&req); 1443 assert(!waited || !use_local_qiov); 1444 1445 tail_buf = qemu_blockalign(bs, align); 1446 tail_iov = (struct iovec) { 1447 .iov_base = tail_buf, 1448 .iov_len = align, 1449 }; 1450 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 1451 1452 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1453 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align, 1454 align, &tail_qiov, 0); 1455 if (ret < 0) { 1456 goto fail; 1457 } 1458 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1459 1460 if (!use_local_qiov) { 1461 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1462 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1463 use_local_qiov = true; 1464 } 1465 1466 tail_bytes = (offset + bytes) & (align - 1); 1467 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 1468 1469 bytes = ROUND_UP(bytes, align); 1470 } 1471 1472 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, 1473 use_local_qiov ? &local_qiov : qiov, 1474 flags); 1475 1476 fail: 1477 1478 if (use_local_qiov) { 1479 qemu_iovec_destroy(&local_qiov); 1480 } 1481 qemu_vfree(head_buf); 1482 qemu_vfree(tail_buf); 1483 out: 1484 tracked_request_end(&req); 1485 return ret; 1486 } 1487 1488 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs, 1489 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1490 BdrvRequestFlags flags) 1491 { 1492 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1493 return -EINVAL; 1494 } 1495 1496 return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS, 1497 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1498 } 1499 1500 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num, 1501 int nb_sectors, QEMUIOVector *qiov) 1502 { 1503 trace_bdrv_co_writev(bs, sector_num, nb_sectors); 1504 1505 return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0); 1506 } 1507 1508 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, 1509 int64_t sector_num, int nb_sectors, 1510 BdrvRequestFlags flags) 1511 { 1512 trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags); 1513 1514 if (!(bs->open_flags & BDRV_O_UNMAP)) { 1515 flags &= ~BDRV_REQ_MAY_UNMAP; 1516 } 1517 1518 return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL, 1519 BDRV_REQ_ZERO_WRITE | flags); 1520 } 1521 1522 typedef struct BdrvCoGetBlockStatusData { 1523 BlockDriverState *bs; 1524 BlockDriverState *base; 1525 BlockDriverState **file; 1526 int64_t sector_num; 1527 int nb_sectors; 1528 int *pnum; 1529 int64_t ret; 1530 bool done; 1531 } BdrvCoGetBlockStatusData; 1532 1533 /* 1534 * Returns the allocation status of the specified sectors. 1535 * Drivers not implementing the functionality are assumed to not support 1536 * backing files, hence all their sectors are reported as allocated. 1537 * 1538 * If 'sector_num' is beyond the end of the disk image the return value is 0 1539 * and 'pnum' is set to 0. 1540 * 1541 * 'pnum' is set to the number of sectors (including and immediately following 1542 * the specified sector) that are known to be in the same 1543 * allocated/unallocated state. 1544 * 1545 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 1546 * beyond the end of the disk image it will be clamped. 1547 * 1548 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file' 1549 * points to the BDS which the sector range is allocated in. 1550 */ 1551 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 1552 int64_t sector_num, 1553 int nb_sectors, int *pnum, 1554 BlockDriverState **file) 1555 { 1556 int64_t total_sectors; 1557 int64_t n; 1558 int64_t ret, ret2; 1559 1560 total_sectors = bdrv_nb_sectors(bs); 1561 if (total_sectors < 0) { 1562 return total_sectors; 1563 } 1564 1565 if (sector_num >= total_sectors) { 1566 *pnum = 0; 1567 return 0; 1568 } 1569 1570 n = total_sectors - sector_num; 1571 if (n < nb_sectors) { 1572 nb_sectors = n; 1573 } 1574 1575 if (!bs->drv->bdrv_co_get_block_status) { 1576 *pnum = nb_sectors; 1577 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 1578 if (bs->drv->protocol_name) { 1579 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 1580 } 1581 return ret; 1582 } 1583 1584 *file = NULL; 1585 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 1586 file); 1587 if (ret < 0) { 1588 *pnum = 0; 1589 return ret; 1590 } 1591 1592 if (ret & BDRV_BLOCK_RAW) { 1593 assert(ret & BDRV_BLOCK_OFFSET_VALID); 1594 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 1595 *pnum, pnum, file); 1596 } 1597 1598 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 1599 ret |= BDRV_BLOCK_ALLOCATED; 1600 } else { 1601 if (bdrv_unallocated_blocks_are_zero(bs)) { 1602 ret |= BDRV_BLOCK_ZERO; 1603 } else if (bs->backing) { 1604 BlockDriverState *bs2 = bs->backing->bs; 1605 int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 1606 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 1607 ret |= BDRV_BLOCK_ZERO; 1608 } 1609 } 1610 } 1611 1612 if (*file && *file != bs && 1613 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 1614 (ret & BDRV_BLOCK_OFFSET_VALID)) { 1615 BlockDriverState *file2; 1616 int file_pnum; 1617 1618 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 1619 *pnum, &file_pnum, &file2); 1620 if (ret2 >= 0) { 1621 /* Ignore errors. This is just providing extra information, it 1622 * is useful but not necessary. 1623 */ 1624 if (!file_pnum) { 1625 /* !file_pnum indicates an offset at or beyond the EOF; it is 1626 * perfectly valid for the format block driver to point to such 1627 * offsets, so catch it and mark everything as zero */ 1628 ret |= BDRV_BLOCK_ZERO; 1629 } else { 1630 /* Limit request to the range reported by the protocol driver */ 1631 *pnum = file_pnum; 1632 ret |= (ret2 & BDRV_BLOCK_ZERO); 1633 } 1634 } 1635 } 1636 1637 return ret; 1638 } 1639 1640 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1641 BlockDriverState *base, 1642 int64_t sector_num, 1643 int nb_sectors, 1644 int *pnum, 1645 BlockDriverState **file) 1646 { 1647 BlockDriverState *p; 1648 int64_t ret = 0; 1649 1650 assert(bs != base); 1651 for (p = bs; p != base; p = backing_bs(p)) { 1652 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file); 1653 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1654 break; 1655 } 1656 /* [sector_num, pnum] unallocated on this layer, which could be only 1657 * the first part of [sector_num, nb_sectors]. */ 1658 nb_sectors = MIN(nb_sectors, *pnum); 1659 } 1660 return ret; 1661 } 1662 1663 /* Coroutine wrapper for bdrv_get_block_status_above() */ 1664 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 1665 { 1666 BdrvCoGetBlockStatusData *data = opaque; 1667 1668 data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1669 data->sector_num, 1670 data->nb_sectors, 1671 data->pnum, 1672 data->file); 1673 data->done = true; 1674 } 1675 1676 /* 1677 * Synchronous wrapper around bdrv_co_get_block_status_above(). 1678 * 1679 * See bdrv_co_get_block_status_above() for details. 1680 */ 1681 int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1682 BlockDriverState *base, 1683 int64_t sector_num, 1684 int nb_sectors, int *pnum, 1685 BlockDriverState **file) 1686 { 1687 Coroutine *co; 1688 BdrvCoGetBlockStatusData data = { 1689 .bs = bs, 1690 .base = base, 1691 .file = file, 1692 .sector_num = sector_num, 1693 .nb_sectors = nb_sectors, 1694 .pnum = pnum, 1695 .done = false, 1696 }; 1697 1698 if (qemu_in_coroutine()) { 1699 /* Fast-path if already in coroutine context */ 1700 bdrv_get_block_status_above_co_entry(&data); 1701 } else { 1702 AioContext *aio_context = bdrv_get_aio_context(bs); 1703 1704 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry); 1705 qemu_coroutine_enter(co, &data); 1706 while (!data.done) { 1707 aio_poll(aio_context, true); 1708 } 1709 } 1710 return data.ret; 1711 } 1712 1713 int64_t bdrv_get_block_status(BlockDriverState *bs, 1714 int64_t sector_num, 1715 int nb_sectors, int *pnum, 1716 BlockDriverState **file) 1717 { 1718 return bdrv_get_block_status_above(bs, backing_bs(bs), 1719 sector_num, nb_sectors, pnum, file); 1720 } 1721 1722 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 1723 int nb_sectors, int *pnum) 1724 { 1725 BlockDriverState *file; 1726 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum, 1727 &file); 1728 if (ret < 0) { 1729 return ret; 1730 } 1731 return !!(ret & BDRV_BLOCK_ALLOCATED); 1732 } 1733 1734 /* 1735 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 1736 * 1737 * Return true if the given sector is allocated in any image between 1738 * BASE and TOP (inclusive). BASE can be NULL to check if the given 1739 * sector is allocated in any image of the chain. Return false otherwise. 1740 * 1741 * 'pnum' is set to the number of sectors (including and immediately following 1742 * the specified sector) that are known to be in the same 1743 * allocated/unallocated state. 1744 * 1745 */ 1746 int bdrv_is_allocated_above(BlockDriverState *top, 1747 BlockDriverState *base, 1748 int64_t sector_num, 1749 int nb_sectors, int *pnum) 1750 { 1751 BlockDriverState *intermediate; 1752 int ret, n = nb_sectors; 1753 1754 intermediate = top; 1755 while (intermediate && intermediate != base) { 1756 int pnum_inter; 1757 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 1758 &pnum_inter); 1759 if (ret < 0) { 1760 return ret; 1761 } else if (ret) { 1762 *pnum = pnum_inter; 1763 return 1; 1764 } 1765 1766 /* 1767 * [sector_num, nb_sectors] is unallocated on top but intermediate 1768 * might have 1769 * 1770 * [sector_num+x, nr_sectors] allocated. 1771 */ 1772 if (n > pnum_inter && 1773 (intermediate == top || 1774 sector_num + pnum_inter < intermediate->total_sectors)) { 1775 n = pnum_inter; 1776 } 1777 1778 intermediate = backing_bs(intermediate); 1779 } 1780 1781 *pnum = n; 1782 return 0; 1783 } 1784 1785 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num, 1786 const uint8_t *buf, int nb_sectors) 1787 { 1788 BlockDriver *drv = bs->drv; 1789 int ret; 1790 1791 if (!drv) { 1792 return -ENOMEDIUM; 1793 } 1794 if (!drv->bdrv_write_compressed) { 1795 return -ENOTSUP; 1796 } 1797 ret = bdrv_check_request(bs, sector_num, nb_sectors); 1798 if (ret < 0) { 1799 return ret; 1800 } 1801 1802 assert(QLIST_EMPTY(&bs->dirty_bitmaps)); 1803 1804 return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors); 1805 } 1806 1807 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 1808 int64_t pos, int size) 1809 { 1810 QEMUIOVector qiov; 1811 struct iovec iov = { 1812 .iov_base = (void *) buf, 1813 .iov_len = size, 1814 }; 1815 1816 qemu_iovec_init_external(&qiov, &iov, 1); 1817 return bdrv_writev_vmstate(bs, &qiov, pos); 1818 } 1819 1820 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 1821 { 1822 BlockDriver *drv = bs->drv; 1823 1824 if (!drv) { 1825 return -ENOMEDIUM; 1826 } else if (drv->bdrv_save_vmstate) { 1827 return drv->bdrv_save_vmstate(bs, qiov, pos); 1828 } else if (bs->file) { 1829 return bdrv_writev_vmstate(bs->file->bs, qiov, pos); 1830 } 1831 1832 return -ENOTSUP; 1833 } 1834 1835 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 1836 int64_t pos, int size) 1837 { 1838 BlockDriver *drv = bs->drv; 1839 if (!drv) 1840 return -ENOMEDIUM; 1841 if (drv->bdrv_load_vmstate) 1842 return drv->bdrv_load_vmstate(bs, buf, pos, size); 1843 if (bs->file) 1844 return bdrv_load_vmstate(bs->file->bs, buf, pos, size); 1845 return -ENOTSUP; 1846 } 1847 1848 /**************************************************************/ 1849 /* async I/Os */ 1850 1851 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num, 1852 QEMUIOVector *qiov, int nb_sectors, 1853 BlockCompletionFunc *cb, void *opaque) 1854 { 1855 trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque); 1856 1857 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 1858 cb, opaque, false); 1859 } 1860 1861 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num, 1862 QEMUIOVector *qiov, int nb_sectors, 1863 BlockCompletionFunc *cb, void *opaque) 1864 { 1865 trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque); 1866 1867 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 1868 cb, opaque, true); 1869 } 1870 1871 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs, 1872 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags, 1873 BlockCompletionFunc *cb, void *opaque) 1874 { 1875 trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque); 1876 1877 return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors, 1878 BDRV_REQ_ZERO_WRITE | flags, 1879 cb, opaque, true); 1880 } 1881 1882 void bdrv_aio_cancel(BlockAIOCB *acb) 1883 { 1884 qemu_aio_ref(acb); 1885 bdrv_aio_cancel_async(acb); 1886 while (acb->refcnt > 1) { 1887 if (acb->aiocb_info->get_aio_context) { 1888 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 1889 } else if (acb->bs) { 1890 aio_poll(bdrv_get_aio_context(acb->bs), true); 1891 } else { 1892 abort(); 1893 } 1894 } 1895 qemu_aio_unref(acb); 1896 } 1897 1898 /* Async version of aio cancel. The caller is not blocked if the acb implements 1899 * cancel_async, otherwise we do nothing and let the request normally complete. 1900 * In either case the completion callback must be called. */ 1901 void bdrv_aio_cancel_async(BlockAIOCB *acb) 1902 { 1903 if (acb->aiocb_info->cancel_async) { 1904 acb->aiocb_info->cancel_async(acb); 1905 } 1906 } 1907 1908 /**************************************************************/ 1909 /* async block device emulation */ 1910 1911 typedef struct BlockAIOCBCoroutine { 1912 BlockAIOCB common; 1913 BlockRequest req; 1914 bool is_write; 1915 bool need_bh; 1916 bool *done; 1917 QEMUBH* bh; 1918 } BlockAIOCBCoroutine; 1919 1920 static const AIOCBInfo bdrv_em_co_aiocb_info = { 1921 .aiocb_size = sizeof(BlockAIOCBCoroutine), 1922 }; 1923 1924 static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 1925 { 1926 if (!acb->need_bh) { 1927 acb->common.cb(acb->common.opaque, acb->req.error); 1928 qemu_aio_unref(acb); 1929 } 1930 } 1931 1932 static void bdrv_co_em_bh(void *opaque) 1933 { 1934 BlockAIOCBCoroutine *acb = opaque; 1935 1936 assert(!acb->need_bh); 1937 qemu_bh_delete(acb->bh); 1938 bdrv_co_complete(acb); 1939 } 1940 1941 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 1942 { 1943 acb->need_bh = false; 1944 if (acb->req.error != -EINPROGRESS) { 1945 BlockDriverState *bs = acb->common.bs; 1946 1947 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 1948 qemu_bh_schedule(acb->bh); 1949 } 1950 } 1951 1952 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 1953 static void coroutine_fn bdrv_co_do_rw(void *opaque) 1954 { 1955 BlockAIOCBCoroutine *acb = opaque; 1956 BlockDriverState *bs = acb->common.bs; 1957 1958 if (!acb->is_write) { 1959 acb->req.error = bdrv_co_do_readv(bs, acb->req.sector, 1960 acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 1961 } else { 1962 acb->req.error = bdrv_co_do_writev(bs, acb->req.sector, 1963 acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 1964 } 1965 1966 bdrv_co_complete(acb); 1967 } 1968 1969 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 1970 int64_t sector_num, 1971 QEMUIOVector *qiov, 1972 int nb_sectors, 1973 BdrvRequestFlags flags, 1974 BlockCompletionFunc *cb, 1975 void *opaque, 1976 bool is_write) 1977 { 1978 Coroutine *co; 1979 BlockAIOCBCoroutine *acb; 1980 1981 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 1982 acb->need_bh = true; 1983 acb->req.error = -EINPROGRESS; 1984 acb->req.sector = sector_num; 1985 acb->req.nb_sectors = nb_sectors; 1986 acb->req.qiov = qiov; 1987 acb->req.flags = flags; 1988 acb->is_write = is_write; 1989 1990 co = qemu_coroutine_create(bdrv_co_do_rw); 1991 qemu_coroutine_enter(co, acb); 1992 1993 bdrv_co_maybe_schedule_bh(acb); 1994 return &acb->common; 1995 } 1996 1997 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 1998 { 1999 BlockAIOCBCoroutine *acb = opaque; 2000 BlockDriverState *bs = acb->common.bs; 2001 2002 acb->req.error = bdrv_co_flush(bs); 2003 bdrv_co_complete(acb); 2004 } 2005 2006 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 2007 BlockCompletionFunc *cb, void *opaque) 2008 { 2009 trace_bdrv_aio_flush(bs, opaque); 2010 2011 Coroutine *co; 2012 BlockAIOCBCoroutine *acb; 2013 2014 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2015 acb->need_bh = true; 2016 acb->req.error = -EINPROGRESS; 2017 2018 co = qemu_coroutine_create(bdrv_aio_flush_co_entry); 2019 qemu_coroutine_enter(co, acb); 2020 2021 bdrv_co_maybe_schedule_bh(acb); 2022 return &acb->common; 2023 } 2024 2025 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque) 2026 { 2027 BlockAIOCBCoroutine *acb = opaque; 2028 BlockDriverState *bs = acb->common.bs; 2029 2030 acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors); 2031 bdrv_co_complete(acb); 2032 } 2033 2034 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs, 2035 int64_t sector_num, int nb_sectors, 2036 BlockCompletionFunc *cb, void *opaque) 2037 { 2038 Coroutine *co; 2039 BlockAIOCBCoroutine *acb; 2040 2041 trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque); 2042 2043 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2044 acb->need_bh = true; 2045 acb->req.error = -EINPROGRESS; 2046 acb->req.sector = sector_num; 2047 acb->req.nb_sectors = nb_sectors; 2048 co = qemu_coroutine_create(bdrv_aio_discard_co_entry); 2049 qemu_coroutine_enter(co, acb); 2050 2051 bdrv_co_maybe_schedule_bh(acb); 2052 return &acb->common; 2053 } 2054 2055 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs, 2056 BlockCompletionFunc *cb, void *opaque) 2057 { 2058 BlockAIOCB *acb; 2059 2060 acb = g_malloc(aiocb_info->aiocb_size); 2061 acb->aiocb_info = aiocb_info; 2062 acb->bs = bs; 2063 acb->cb = cb; 2064 acb->opaque = opaque; 2065 acb->refcnt = 1; 2066 return acb; 2067 } 2068 2069 void qemu_aio_ref(void *p) 2070 { 2071 BlockAIOCB *acb = p; 2072 acb->refcnt++; 2073 } 2074 2075 void qemu_aio_unref(void *p) 2076 { 2077 BlockAIOCB *acb = p; 2078 assert(acb->refcnt > 0); 2079 if (--acb->refcnt == 0) { 2080 g_free(acb); 2081 } 2082 } 2083 2084 /**************************************************************/ 2085 /* Coroutine block device emulation */ 2086 2087 static void coroutine_fn bdrv_flush_co_entry(void *opaque) 2088 { 2089 RwCo *rwco = opaque; 2090 2091 rwco->ret = bdrv_co_flush(rwco->bs); 2092 } 2093 2094 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2095 { 2096 int ret; 2097 BdrvTrackedRequest req; 2098 2099 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2100 bdrv_is_sg(bs)) { 2101 return 0; 2102 } 2103 2104 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH); 2105 2106 /* Write back all layers by calling one driver function */ 2107 if (bs->drv->bdrv_co_flush) { 2108 ret = bs->drv->bdrv_co_flush(bs); 2109 goto out; 2110 } 2111 2112 /* Write back cached data to the OS even with cache=unsafe */ 2113 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 2114 if (bs->drv->bdrv_co_flush_to_os) { 2115 ret = bs->drv->bdrv_co_flush_to_os(bs); 2116 if (ret < 0) { 2117 goto out; 2118 } 2119 } 2120 2121 /* But don't actually force it to the disk with cache=unsafe */ 2122 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2123 goto flush_parent; 2124 } 2125 2126 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 2127 if (bs->drv->bdrv_co_flush_to_disk) { 2128 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2129 } else if (bs->drv->bdrv_aio_flush) { 2130 BlockAIOCB *acb; 2131 CoroutineIOCompletion co = { 2132 .coroutine = qemu_coroutine_self(), 2133 }; 2134 2135 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2136 if (acb == NULL) { 2137 ret = -EIO; 2138 } else { 2139 qemu_coroutine_yield(); 2140 ret = co.ret; 2141 } 2142 } else { 2143 /* 2144 * Some block drivers always operate in either writethrough or unsafe 2145 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2146 * know how the server works (because the behaviour is hardcoded or 2147 * depends on server-side configuration), so we can't ensure that 2148 * everything is safe on disk. Returning an error doesn't work because 2149 * that would break guests even if the server operates in writethrough 2150 * mode. 2151 * 2152 * Let's hope the user knows what he's doing. 2153 */ 2154 ret = 0; 2155 } 2156 if (ret < 0) { 2157 goto out; 2158 } 2159 2160 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2161 * in the case of cache=unsafe, so there are no useless flushes. 2162 */ 2163 flush_parent: 2164 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2165 out: 2166 tracked_request_end(&req); 2167 return ret; 2168 } 2169 2170 int bdrv_flush(BlockDriverState *bs) 2171 { 2172 Coroutine *co; 2173 RwCo rwco = { 2174 .bs = bs, 2175 .ret = NOT_DONE, 2176 }; 2177 2178 if (qemu_in_coroutine()) { 2179 /* Fast-path if already in coroutine context */ 2180 bdrv_flush_co_entry(&rwco); 2181 } else { 2182 AioContext *aio_context = bdrv_get_aio_context(bs); 2183 2184 co = qemu_coroutine_create(bdrv_flush_co_entry); 2185 qemu_coroutine_enter(co, &rwco); 2186 while (rwco.ret == NOT_DONE) { 2187 aio_poll(aio_context, true); 2188 } 2189 } 2190 2191 return rwco.ret; 2192 } 2193 2194 typedef struct DiscardCo { 2195 BlockDriverState *bs; 2196 int64_t sector_num; 2197 int nb_sectors; 2198 int ret; 2199 } DiscardCo; 2200 static void coroutine_fn bdrv_discard_co_entry(void *opaque) 2201 { 2202 DiscardCo *rwco = opaque; 2203 2204 rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors); 2205 } 2206 2207 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, 2208 int nb_sectors) 2209 { 2210 BdrvTrackedRequest req; 2211 int max_discard, ret; 2212 2213 if (!bs->drv) { 2214 return -ENOMEDIUM; 2215 } 2216 2217 ret = bdrv_check_request(bs, sector_num, nb_sectors); 2218 if (ret < 0) { 2219 return ret; 2220 } else if (bs->read_only) { 2221 return -EPERM; 2222 } 2223 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 2224 2225 /* Do nothing if disabled. */ 2226 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2227 return 0; 2228 } 2229 2230 if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) { 2231 return 0; 2232 } 2233 2234 tracked_request_begin(&req, bs, sector_num, nb_sectors, 2235 BDRV_TRACKED_DISCARD); 2236 bdrv_set_dirty(bs, sector_num, nb_sectors); 2237 2238 max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS); 2239 while (nb_sectors > 0) { 2240 int ret; 2241 int num = nb_sectors; 2242 2243 /* align request */ 2244 if (bs->bl.discard_alignment && 2245 num >= bs->bl.discard_alignment && 2246 sector_num % bs->bl.discard_alignment) { 2247 if (num > bs->bl.discard_alignment) { 2248 num = bs->bl.discard_alignment; 2249 } 2250 num -= sector_num % bs->bl.discard_alignment; 2251 } 2252 2253 /* limit request size */ 2254 if (num > max_discard) { 2255 num = max_discard; 2256 } 2257 2258 if (bs->drv->bdrv_co_discard) { 2259 ret = bs->drv->bdrv_co_discard(bs, sector_num, num); 2260 } else { 2261 BlockAIOCB *acb; 2262 CoroutineIOCompletion co = { 2263 .coroutine = qemu_coroutine_self(), 2264 }; 2265 2266 acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors, 2267 bdrv_co_io_em_complete, &co); 2268 if (acb == NULL) { 2269 ret = -EIO; 2270 goto out; 2271 } else { 2272 qemu_coroutine_yield(); 2273 ret = co.ret; 2274 } 2275 } 2276 if (ret && ret != -ENOTSUP) { 2277 goto out; 2278 } 2279 2280 sector_num += num; 2281 nb_sectors -= num; 2282 } 2283 ret = 0; 2284 out: 2285 tracked_request_end(&req); 2286 return ret; 2287 } 2288 2289 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors) 2290 { 2291 Coroutine *co; 2292 DiscardCo rwco = { 2293 .bs = bs, 2294 .sector_num = sector_num, 2295 .nb_sectors = nb_sectors, 2296 .ret = NOT_DONE, 2297 }; 2298 2299 if (qemu_in_coroutine()) { 2300 /* Fast-path if already in coroutine context */ 2301 bdrv_discard_co_entry(&rwco); 2302 } else { 2303 AioContext *aio_context = bdrv_get_aio_context(bs); 2304 2305 co = qemu_coroutine_create(bdrv_discard_co_entry); 2306 qemu_coroutine_enter(co, &rwco); 2307 while (rwco.ret == NOT_DONE) { 2308 aio_poll(aio_context, true); 2309 } 2310 } 2311 2312 return rwco.ret; 2313 } 2314 2315 typedef struct { 2316 CoroutineIOCompletion *co; 2317 QEMUBH *bh; 2318 } BdrvIoctlCompletionData; 2319 2320 static void bdrv_ioctl_bh_cb(void *opaque) 2321 { 2322 BdrvIoctlCompletionData *data = opaque; 2323 2324 bdrv_co_io_em_complete(data->co, -ENOTSUP); 2325 qemu_bh_delete(data->bh); 2326 } 2327 2328 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf) 2329 { 2330 BlockDriver *drv = bs->drv; 2331 BdrvTrackedRequest tracked_req; 2332 CoroutineIOCompletion co = { 2333 .coroutine = qemu_coroutine_self(), 2334 }; 2335 BlockAIOCB *acb; 2336 2337 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL); 2338 if (!drv || !drv->bdrv_aio_ioctl) { 2339 co.ret = -ENOTSUP; 2340 goto out; 2341 } 2342 2343 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 2344 if (!acb) { 2345 BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1); 2346 data->bh = aio_bh_new(bdrv_get_aio_context(bs), 2347 bdrv_ioctl_bh_cb, data); 2348 data->co = &co; 2349 qemu_bh_schedule(data->bh); 2350 } 2351 qemu_coroutine_yield(); 2352 out: 2353 tracked_request_end(&tracked_req); 2354 return co.ret; 2355 } 2356 2357 typedef struct { 2358 BlockDriverState *bs; 2359 int req; 2360 void *buf; 2361 int ret; 2362 } BdrvIoctlCoData; 2363 2364 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque) 2365 { 2366 BdrvIoctlCoData *data = opaque; 2367 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf); 2368 } 2369 2370 /* needed for generic scsi interface */ 2371 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) 2372 { 2373 BdrvIoctlCoData data = { 2374 .bs = bs, 2375 .req = req, 2376 .buf = buf, 2377 .ret = -EINPROGRESS, 2378 }; 2379 2380 if (qemu_in_coroutine()) { 2381 /* Fast-path if already in coroutine context */ 2382 bdrv_co_ioctl_entry(&data); 2383 } else { 2384 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry); 2385 2386 qemu_coroutine_enter(co, &data); 2387 while (data.ret == -EINPROGRESS) { 2388 aio_poll(bdrv_get_aio_context(bs), true); 2389 } 2390 } 2391 return data.ret; 2392 } 2393 2394 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque) 2395 { 2396 BlockAIOCBCoroutine *acb = opaque; 2397 acb->req.error = bdrv_co_do_ioctl(acb->common.bs, 2398 acb->req.req, acb->req.buf); 2399 bdrv_co_complete(acb); 2400 } 2401 2402 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs, 2403 unsigned long int req, void *buf, 2404 BlockCompletionFunc *cb, void *opaque) 2405 { 2406 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info, 2407 bs, cb, opaque); 2408 Coroutine *co; 2409 2410 acb->need_bh = true; 2411 acb->req.error = -EINPROGRESS; 2412 acb->req.req = req; 2413 acb->req.buf = buf; 2414 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry); 2415 qemu_coroutine_enter(co, acb); 2416 2417 bdrv_co_maybe_schedule_bh(acb); 2418 return &acb->common; 2419 } 2420 2421 void *qemu_blockalign(BlockDriverState *bs, size_t size) 2422 { 2423 return qemu_memalign(bdrv_opt_mem_align(bs), size); 2424 } 2425 2426 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 2427 { 2428 return memset(qemu_blockalign(bs, size), 0, size); 2429 } 2430 2431 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 2432 { 2433 size_t align = bdrv_opt_mem_align(bs); 2434 2435 /* Ensure that NULL is never returned on success */ 2436 assert(align > 0); 2437 if (size == 0) { 2438 size = align; 2439 } 2440 2441 return qemu_try_memalign(align, size); 2442 } 2443 2444 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 2445 { 2446 void *mem = qemu_try_blockalign(bs, size); 2447 2448 if (mem) { 2449 memset(mem, 0, size); 2450 } 2451 2452 return mem; 2453 } 2454 2455 /* 2456 * Check if all memory in this vector is sector aligned. 2457 */ 2458 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 2459 { 2460 int i; 2461 size_t alignment = bdrv_min_mem_align(bs); 2462 2463 for (i = 0; i < qiov->niov; i++) { 2464 if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 2465 return false; 2466 } 2467 if (qiov->iov[i].iov_len % alignment) { 2468 return false; 2469 } 2470 } 2471 2472 return true; 2473 } 2474 2475 void bdrv_add_before_write_notifier(BlockDriverState *bs, 2476 NotifierWithReturn *notifier) 2477 { 2478 notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 2479 } 2480 2481 void bdrv_io_plug(BlockDriverState *bs) 2482 { 2483 BdrvChild *child; 2484 2485 QLIST_FOREACH(child, &bs->children, next) { 2486 bdrv_io_plug(child->bs); 2487 } 2488 2489 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) { 2490 BlockDriver *drv = bs->drv; 2491 if (drv && drv->bdrv_io_plug) { 2492 drv->bdrv_io_plug(bs); 2493 } 2494 } 2495 } 2496 2497 void bdrv_io_unplug(BlockDriverState *bs) 2498 { 2499 BdrvChild *child; 2500 2501 assert(bs->io_plugged); 2502 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) { 2503 BlockDriver *drv = bs->drv; 2504 if (drv && drv->bdrv_io_unplug) { 2505 drv->bdrv_io_unplug(bs); 2506 } 2507 } 2508 2509 QLIST_FOREACH(child, &bs->children, next) { 2510 bdrv_io_unplug(child->bs); 2511 } 2512 } 2513 2514 void bdrv_io_unplugged_begin(BlockDriverState *bs) 2515 { 2516 BdrvChild *child; 2517 2518 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) { 2519 BlockDriver *drv = bs->drv; 2520 if (drv && drv->bdrv_io_unplug) { 2521 drv->bdrv_io_unplug(bs); 2522 } 2523 } 2524 2525 QLIST_FOREACH(child, &bs->children, next) { 2526 bdrv_io_unplugged_begin(child->bs); 2527 } 2528 } 2529 2530 void bdrv_io_unplugged_end(BlockDriverState *bs) 2531 { 2532 BdrvChild *child; 2533 2534 assert(bs->io_plug_disabled); 2535 QLIST_FOREACH(child, &bs->children, next) { 2536 bdrv_io_unplugged_end(child->bs); 2537 } 2538 2539 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) { 2540 BlockDriver *drv = bs->drv; 2541 if (drv && drv->bdrv_io_plug) { 2542 drv->bdrv_io_plug(bs); 2543 } 2544 } 2545 } 2546 2547 void bdrv_drained_begin(BlockDriverState *bs) 2548 { 2549 if (!bs->quiesce_counter++) { 2550 aio_disable_external(bdrv_get_aio_context(bs)); 2551 } 2552 bdrv_parent_drained_begin(bs); 2553 bdrv_drain(bs); 2554 } 2555 2556 void bdrv_drained_end(BlockDriverState *bs) 2557 { 2558 bdrv_parent_drained_end(bs); 2559 2560 assert(bs->quiesce_counter > 0); 2561 if (--bs->quiesce_counter > 0) { 2562 return; 2563 } 2564 aio_enable_external(bdrv_get_aio_context(bs)); 2565 } 2566