1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/blockjob.h" 29 #include "block/block_int.h" 30 #include "block/throttle-groups.h" 31 #include "qemu/cutils.h" 32 #include "qapi/error.h" 33 #include "qemu/error-report.h" 34 35 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */ 36 37 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 38 int64_t sector_num, 39 QEMUIOVector *qiov, 40 int nb_sectors, 41 BdrvRequestFlags flags, 42 BlockCompletionFunc *cb, 43 void *opaque, 44 bool is_write); 45 static void coroutine_fn bdrv_co_do_rw(void *opaque); 46 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 47 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags); 48 49 /* throttling disk I/O limits */ 50 void bdrv_set_io_limits(BlockDriverState *bs, 51 ThrottleConfig *cfg) 52 { 53 throttle_group_config(bs, cfg); 54 } 55 56 void bdrv_no_throttling_begin(BlockDriverState *bs) 57 { 58 if (bs->io_limits_disabled++ == 0) { 59 throttle_group_restart_bs(bs); 60 } 61 } 62 63 void bdrv_no_throttling_end(BlockDriverState *bs) 64 { 65 assert(bs->io_limits_disabled); 66 --bs->io_limits_disabled; 67 } 68 69 void bdrv_io_limits_disable(BlockDriverState *bs) 70 { 71 assert(bs->throttle_state); 72 bdrv_no_throttling_begin(bs); 73 throttle_group_unregister_bs(bs); 74 bdrv_no_throttling_end(bs); 75 } 76 77 /* should be called before bdrv_set_io_limits if a limit is set */ 78 void bdrv_io_limits_enable(BlockDriverState *bs, const char *group) 79 { 80 assert(!bs->throttle_state); 81 throttle_group_register_bs(bs, group); 82 } 83 84 void bdrv_io_limits_update_group(BlockDriverState *bs, const char *group) 85 { 86 /* this bs is not part of any group */ 87 if (!bs->throttle_state) { 88 return; 89 } 90 91 /* this bs is a part of the same group than the one we want */ 92 if (!g_strcmp0(throttle_group_get_name(bs), group)) { 93 return; 94 } 95 96 /* need to change the group this bs belong to */ 97 bdrv_io_limits_disable(bs); 98 bdrv_io_limits_enable(bs, group); 99 } 100 101 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 102 { 103 BlockDriver *drv = bs->drv; 104 Error *local_err = NULL; 105 106 memset(&bs->bl, 0, sizeof(bs->bl)); 107 108 if (!drv) { 109 return; 110 } 111 112 /* Take some limits from the children as a default */ 113 if (bs->file) { 114 bdrv_refresh_limits(bs->file->bs, &local_err); 115 if (local_err) { 116 error_propagate(errp, local_err); 117 return; 118 } 119 bs->bl.opt_transfer_length = bs->file->bs->bl.opt_transfer_length; 120 bs->bl.max_transfer_length = bs->file->bs->bl.max_transfer_length; 121 bs->bl.min_mem_alignment = bs->file->bs->bl.min_mem_alignment; 122 bs->bl.opt_mem_alignment = bs->file->bs->bl.opt_mem_alignment; 123 bs->bl.max_iov = bs->file->bs->bl.max_iov; 124 } else { 125 bs->bl.min_mem_alignment = 512; 126 bs->bl.opt_mem_alignment = getpagesize(); 127 128 /* Safe default since most protocols use readv()/writev()/etc */ 129 bs->bl.max_iov = IOV_MAX; 130 } 131 132 if (bs->backing) { 133 bdrv_refresh_limits(bs->backing->bs, &local_err); 134 if (local_err) { 135 error_propagate(errp, local_err); 136 return; 137 } 138 bs->bl.opt_transfer_length = 139 MAX(bs->bl.opt_transfer_length, 140 bs->backing->bs->bl.opt_transfer_length); 141 bs->bl.max_transfer_length = 142 MIN_NON_ZERO(bs->bl.max_transfer_length, 143 bs->backing->bs->bl.max_transfer_length); 144 bs->bl.opt_mem_alignment = 145 MAX(bs->bl.opt_mem_alignment, 146 bs->backing->bs->bl.opt_mem_alignment); 147 bs->bl.min_mem_alignment = 148 MAX(bs->bl.min_mem_alignment, 149 bs->backing->bs->bl.min_mem_alignment); 150 bs->bl.max_iov = 151 MIN(bs->bl.max_iov, 152 bs->backing->bs->bl.max_iov); 153 } 154 155 /* Then let the driver override it */ 156 if (drv->bdrv_refresh_limits) { 157 drv->bdrv_refresh_limits(bs, errp); 158 } 159 } 160 161 /** 162 * The copy-on-read flag is actually a reference count so multiple users may 163 * use the feature without worrying about clobbering its previous state. 164 * Copy-on-read stays enabled until all users have called to disable it. 165 */ 166 void bdrv_enable_copy_on_read(BlockDriverState *bs) 167 { 168 bs->copy_on_read++; 169 } 170 171 void bdrv_disable_copy_on_read(BlockDriverState *bs) 172 { 173 assert(bs->copy_on_read > 0); 174 bs->copy_on_read--; 175 } 176 177 /* Check if any requests are in-flight (including throttled requests) */ 178 bool bdrv_requests_pending(BlockDriverState *bs) 179 { 180 BdrvChild *child; 181 182 if (!QLIST_EMPTY(&bs->tracked_requests)) { 183 return true; 184 } 185 if (!qemu_co_queue_empty(&bs->throttled_reqs[0])) { 186 return true; 187 } 188 if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) { 189 return true; 190 } 191 192 QLIST_FOREACH(child, &bs->children, next) { 193 if (bdrv_requests_pending(child->bs)) { 194 return true; 195 } 196 } 197 198 return false; 199 } 200 201 static void bdrv_drain_recurse(BlockDriverState *bs) 202 { 203 BdrvChild *child; 204 205 if (bs->drv && bs->drv->bdrv_drain) { 206 bs->drv->bdrv_drain(bs); 207 } 208 QLIST_FOREACH(child, &bs->children, next) { 209 bdrv_drain_recurse(child->bs); 210 } 211 } 212 213 typedef struct { 214 Coroutine *co; 215 BlockDriverState *bs; 216 QEMUBH *bh; 217 bool done; 218 } BdrvCoDrainData; 219 220 static void bdrv_drain_poll(BlockDriverState *bs) 221 { 222 bool busy = true; 223 224 while (busy) { 225 /* Keep iterating */ 226 busy = bdrv_requests_pending(bs); 227 busy |= aio_poll(bdrv_get_aio_context(bs), busy); 228 } 229 } 230 231 static void bdrv_co_drain_bh_cb(void *opaque) 232 { 233 BdrvCoDrainData *data = opaque; 234 Coroutine *co = data->co; 235 236 qemu_bh_delete(data->bh); 237 bdrv_drain_poll(data->bs); 238 data->done = true; 239 qemu_coroutine_enter(co, NULL); 240 } 241 242 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs) 243 { 244 BdrvCoDrainData data; 245 246 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 247 * other coroutines run if they were queued from 248 * qemu_co_queue_run_restart(). */ 249 250 assert(qemu_in_coroutine()); 251 data = (BdrvCoDrainData) { 252 .co = qemu_coroutine_self(), 253 .bs = bs, 254 .done = false, 255 .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data), 256 }; 257 qemu_bh_schedule(data.bh); 258 259 qemu_coroutine_yield(); 260 /* If we are resumed from some other event (such as an aio completion or a 261 * timer callback), it is a bug in the caller that should be fixed. */ 262 assert(data.done); 263 } 264 265 /* 266 * Wait for pending requests to complete on a single BlockDriverState subtree, 267 * and suspend block driver's internal I/O until next request arrives. 268 * 269 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 270 * AioContext. 271 * 272 * Only this BlockDriverState's AioContext is run, so in-flight requests must 273 * not depend on events in other AioContexts. In that case, use 274 * bdrv_drain_all() instead. 275 */ 276 void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 277 { 278 bdrv_no_throttling_begin(bs); 279 bdrv_io_unplugged_begin(bs); 280 bdrv_drain_recurse(bs); 281 bdrv_co_yield_to_drain(bs); 282 bdrv_io_unplugged_end(bs); 283 bdrv_no_throttling_end(bs); 284 } 285 286 void bdrv_drain(BlockDriverState *bs) 287 { 288 bdrv_no_throttling_begin(bs); 289 bdrv_io_unplugged_begin(bs); 290 bdrv_drain_recurse(bs); 291 if (qemu_in_coroutine()) { 292 bdrv_co_yield_to_drain(bs); 293 } else { 294 bdrv_drain_poll(bs); 295 } 296 bdrv_io_unplugged_end(bs); 297 bdrv_no_throttling_end(bs); 298 } 299 300 /* 301 * Wait for pending requests to complete across all BlockDriverStates 302 * 303 * This function does not flush data to disk, use bdrv_flush_all() for that 304 * after calling this function. 305 */ 306 void bdrv_drain_all(void) 307 { 308 /* Always run first iteration so any pending completion BHs run */ 309 bool busy = true; 310 BlockDriverState *bs = NULL; 311 GSList *aio_ctxs = NULL, *ctx; 312 313 while ((bs = bdrv_next(bs))) { 314 AioContext *aio_context = bdrv_get_aio_context(bs); 315 316 aio_context_acquire(aio_context); 317 if (bs->job) { 318 block_job_pause(bs->job); 319 } 320 bdrv_no_throttling_begin(bs); 321 bdrv_io_unplugged_begin(bs); 322 bdrv_drain_recurse(bs); 323 aio_context_release(aio_context); 324 325 if (!g_slist_find(aio_ctxs, aio_context)) { 326 aio_ctxs = g_slist_prepend(aio_ctxs, aio_context); 327 } 328 } 329 330 /* Note that completion of an asynchronous I/O operation can trigger any 331 * number of other I/O operations on other devices---for example a 332 * coroutine can submit an I/O request to another device in response to 333 * request completion. Therefore we must keep looping until there was no 334 * more activity rather than simply draining each device independently. 335 */ 336 while (busy) { 337 busy = false; 338 339 for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) { 340 AioContext *aio_context = ctx->data; 341 bs = NULL; 342 343 aio_context_acquire(aio_context); 344 while ((bs = bdrv_next(bs))) { 345 if (aio_context == bdrv_get_aio_context(bs)) { 346 if (bdrv_requests_pending(bs)) { 347 busy = true; 348 aio_poll(aio_context, busy); 349 } 350 } 351 } 352 busy |= aio_poll(aio_context, false); 353 aio_context_release(aio_context); 354 } 355 } 356 357 bs = NULL; 358 while ((bs = bdrv_next(bs))) { 359 AioContext *aio_context = bdrv_get_aio_context(bs); 360 361 aio_context_acquire(aio_context); 362 bdrv_io_unplugged_end(bs); 363 bdrv_no_throttling_end(bs); 364 if (bs->job) { 365 block_job_resume(bs->job); 366 } 367 aio_context_release(aio_context); 368 } 369 g_slist_free(aio_ctxs); 370 } 371 372 /** 373 * Remove an active request from the tracked requests list 374 * 375 * This function should be called when a tracked request is completing. 376 */ 377 static void tracked_request_end(BdrvTrackedRequest *req) 378 { 379 if (req->serialising) { 380 req->bs->serialising_in_flight--; 381 } 382 383 QLIST_REMOVE(req, list); 384 qemu_co_queue_restart_all(&req->wait_queue); 385 } 386 387 /** 388 * Add an active request to the tracked requests list 389 */ 390 static void tracked_request_begin(BdrvTrackedRequest *req, 391 BlockDriverState *bs, 392 int64_t offset, 393 unsigned int bytes, 394 enum BdrvTrackedRequestType type) 395 { 396 *req = (BdrvTrackedRequest){ 397 .bs = bs, 398 .offset = offset, 399 .bytes = bytes, 400 .type = type, 401 .co = qemu_coroutine_self(), 402 .serialising = false, 403 .overlap_offset = offset, 404 .overlap_bytes = bytes, 405 }; 406 407 qemu_co_queue_init(&req->wait_queue); 408 409 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 410 } 411 412 static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 413 { 414 int64_t overlap_offset = req->offset & ~(align - 1); 415 unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 416 - overlap_offset; 417 418 if (!req->serialising) { 419 req->bs->serialising_in_flight++; 420 req->serialising = true; 421 } 422 423 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 424 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 425 } 426 427 /** 428 * Round a region to cluster boundaries 429 */ 430 void bdrv_round_to_clusters(BlockDriverState *bs, 431 int64_t sector_num, int nb_sectors, 432 int64_t *cluster_sector_num, 433 int *cluster_nb_sectors) 434 { 435 BlockDriverInfo bdi; 436 437 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 438 *cluster_sector_num = sector_num; 439 *cluster_nb_sectors = nb_sectors; 440 } else { 441 int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE; 442 *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c); 443 *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num + 444 nb_sectors, c); 445 } 446 } 447 448 static int bdrv_get_cluster_size(BlockDriverState *bs) 449 { 450 BlockDriverInfo bdi; 451 int ret; 452 453 ret = bdrv_get_info(bs, &bdi); 454 if (ret < 0 || bdi.cluster_size == 0) { 455 return bs->request_alignment; 456 } else { 457 return bdi.cluster_size; 458 } 459 } 460 461 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 462 int64_t offset, unsigned int bytes) 463 { 464 /* aaaa bbbb */ 465 if (offset >= req->overlap_offset + req->overlap_bytes) { 466 return false; 467 } 468 /* bbbb aaaa */ 469 if (req->overlap_offset >= offset + bytes) { 470 return false; 471 } 472 return true; 473 } 474 475 static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self) 476 { 477 BlockDriverState *bs = self->bs; 478 BdrvTrackedRequest *req; 479 bool retry; 480 bool waited = false; 481 482 if (!bs->serialising_in_flight) { 483 return false; 484 } 485 486 do { 487 retry = false; 488 QLIST_FOREACH(req, &bs->tracked_requests, list) { 489 if (req == self || (!req->serialising && !self->serialising)) { 490 continue; 491 } 492 if (tracked_request_overlaps(req, self->overlap_offset, 493 self->overlap_bytes)) 494 { 495 /* Hitting this means there was a reentrant request, for 496 * example, a block driver issuing nested requests. This must 497 * never happen since it means deadlock. 498 */ 499 assert(qemu_coroutine_self() != req->co); 500 501 /* If the request is already (indirectly) waiting for us, or 502 * will wait for us as soon as it wakes up, then just go on 503 * (instead of producing a deadlock in the former case). */ 504 if (!req->waiting_for) { 505 self->waiting_for = req; 506 qemu_co_queue_wait(&req->wait_queue); 507 self->waiting_for = NULL; 508 retry = true; 509 waited = true; 510 break; 511 } 512 } 513 } 514 } while (retry); 515 516 return waited; 517 } 518 519 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 520 size_t size) 521 { 522 if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) { 523 return -EIO; 524 } 525 526 if (!bdrv_is_inserted(bs)) { 527 return -ENOMEDIUM; 528 } 529 530 if (offset < 0) { 531 return -EIO; 532 } 533 534 return 0; 535 } 536 537 static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num, 538 int nb_sectors) 539 { 540 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 541 return -EIO; 542 } 543 544 return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE, 545 nb_sectors * BDRV_SECTOR_SIZE); 546 } 547 548 typedef struct RwCo { 549 BlockDriverState *bs; 550 int64_t offset; 551 QEMUIOVector *qiov; 552 bool is_write; 553 int ret; 554 BdrvRequestFlags flags; 555 } RwCo; 556 557 static void coroutine_fn bdrv_rw_co_entry(void *opaque) 558 { 559 RwCo *rwco = opaque; 560 561 if (!rwco->is_write) { 562 rwco->ret = bdrv_co_preadv(rwco->bs, rwco->offset, 563 rwco->qiov->size, rwco->qiov, 564 rwco->flags); 565 } else { 566 rwco->ret = bdrv_co_pwritev(rwco->bs, rwco->offset, 567 rwco->qiov->size, rwco->qiov, 568 rwco->flags); 569 } 570 } 571 572 /* 573 * Process a vectored synchronous request using coroutines 574 */ 575 static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset, 576 QEMUIOVector *qiov, bool is_write, 577 BdrvRequestFlags flags) 578 { 579 Coroutine *co; 580 RwCo rwco = { 581 .bs = bs, 582 .offset = offset, 583 .qiov = qiov, 584 .is_write = is_write, 585 .ret = NOT_DONE, 586 .flags = flags, 587 }; 588 589 if (qemu_in_coroutine()) { 590 /* Fast-path if already in coroutine context */ 591 bdrv_rw_co_entry(&rwco); 592 } else { 593 AioContext *aio_context = bdrv_get_aio_context(bs); 594 595 co = qemu_coroutine_create(bdrv_rw_co_entry); 596 qemu_coroutine_enter(co, &rwco); 597 while (rwco.ret == NOT_DONE) { 598 aio_poll(aio_context, true); 599 } 600 } 601 return rwco.ret; 602 } 603 604 /* 605 * Process a synchronous request using coroutines 606 */ 607 static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf, 608 int nb_sectors, bool is_write, BdrvRequestFlags flags) 609 { 610 QEMUIOVector qiov; 611 struct iovec iov = { 612 .iov_base = (void *)buf, 613 .iov_len = nb_sectors * BDRV_SECTOR_SIZE, 614 }; 615 616 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 617 return -EINVAL; 618 } 619 620 qemu_iovec_init_external(&qiov, &iov, 1); 621 return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS, 622 &qiov, is_write, flags); 623 } 624 625 /* return < 0 if error. See bdrv_write() for the return codes */ 626 int bdrv_read(BlockDriverState *bs, int64_t sector_num, 627 uint8_t *buf, int nb_sectors) 628 { 629 return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0); 630 } 631 632 /* Return < 0 if error. Important errors are: 633 -EIO generic I/O error (may happen for all errors) 634 -ENOMEDIUM No media inserted. 635 -EINVAL Invalid sector number or nb_sectors 636 -EACCES Trying to write a read-only device 637 */ 638 int bdrv_write(BlockDriverState *bs, int64_t sector_num, 639 const uint8_t *buf, int nb_sectors) 640 { 641 return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0); 642 } 643 644 int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num, 645 int nb_sectors, BdrvRequestFlags flags) 646 { 647 return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true, 648 BDRV_REQ_ZERO_WRITE | flags); 649 } 650 651 /* 652 * Completely zero out a block device with the help of bdrv_write_zeroes. 653 * The operation is sped up by checking the block status and only writing 654 * zeroes to the device if they currently do not return zeroes. Optional 655 * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 656 * BDRV_REQ_FUA). 657 * 658 * Returns < 0 on error, 0 on success. For error codes see bdrv_write(). 659 */ 660 int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags) 661 { 662 int64_t target_sectors, ret, nb_sectors, sector_num = 0; 663 BlockDriverState *file; 664 int n; 665 666 target_sectors = bdrv_nb_sectors(bs); 667 if (target_sectors < 0) { 668 return target_sectors; 669 } 670 671 for (;;) { 672 nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS); 673 if (nb_sectors <= 0) { 674 return 0; 675 } 676 ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file); 677 if (ret < 0) { 678 error_report("error getting block status at sector %" PRId64 ": %s", 679 sector_num, strerror(-ret)); 680 return ret; 681 } 682 if (ret & BDRV_BLOCK_ZERO) { 683 sector_num += n; 684 continue; 685 } 686 ret = bdrv_write_zeroes(bs, sector_num, n, flags); 687 if (ret < 0) { 688 error_report("error writing zeroes at sector %" PRId64 ": %s", 689 sector_num, strerror(-ret)); 690 return ret; 691 } 692 sector_num += n; 693 } 694 } 695 696 int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes) 697 { 698 QEMUIOVector qiov; 699 struct iovec iov = { 700 .iov_base = (void *)buf, 701 .iov_len = bytes, 702 }; 703 int ret; 704 705 if (bytes < 0) { 706 return -EINVAL; 707 } 708 709 qemu_iovec_init_external(&qiov, &iov, 1); 710 ret = bdrv_prwv_co(bs, offset, &qiov, false, 0); 711 if (ret < 0) { 712 return ret; 713 } 714 715 return bytes; 716 } 717 718 int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov) 719 { 720 int ret; 721 722 ret = bdrv_prwv_co(bs, offset, qiov, true, 0); 723 if (ret < 0) { 724 return ret; 725 } 726 727 return qiov->size; 728 } 729 730 int bdrv_pwrite(BlockDriverState *bs, int64_t offset, 731 const void *buf, int bytes) 732 { 733 QEMUIOVector qiov; 734 struct iovec iov = { 735 .iov_base = (void *) buf, 736 .iov_len = bytes, 737 }; 738 739 if (bytes < 0) { 740 return -EINVAL; 741 } 742 743 qemu_iovec_init_external(&qiov, &iov, 1); 744 return bdrv_pwritev(bs, offset, &qiov); 745 } 746 747 /* 748 * Writes to the file and ensures that no writes are reordered across this 749 * request (acts as a barrier) 750 * 751 * Returns 0 on success, -errno in error cases. 752 */ 753 int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset, 754 const void *buf, int count) 755 { 756 int ret; 757 758 ret = bdrv_pwrite(bs, offset, buf, count); 759 if (ret < 0) { 760 return ret; 761 } 762 763 ret = bdrv_flush(bs); 764 if (ret < 0) { 765 return ret; 766 } 767 768 return 0; 769 } 770 771 typedef struct CoroutineIOCompletion { 772 Coroutine *coroutine; 773 int ret; 774 } CoroutineIOCompletion; 775 776 static void bdrv_co_io_em_complete(void *opaque, int ret) 777 { 778 CoroutineIOCompletion *co = opaque; 779 780 co->ret = ret; 781 qemu_coroutine_enter(co->coroutine, NULL); 782 } 783 784 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 785 uint64_t offset, uint64_t bytes, 786 QEMUIOVector *qiov, int flags) 787 { 788 BlockDriver *drv = bs->drv; 789 int64_t sector_num; 790 unsigned int nb_sectors; 791 792 if (drv->bdrv_co_preadv) { 793 return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 794 } 795 796 sector_num = offset >> BDRV_SECTOR_BITS; 797 nb_sectors = bytes >> BDRV_SECTOR_BITS; 798 799 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 800 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 801 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 802 803 if (drv->bdrv_co_readv) { 804 return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 805 } else { 806 BlockAIOCB *acb; 807 CoroutineIOCompletion co = { 808 .coroutine = qemu_coroutine_self(), 809 }; 810 811 acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors, 812 bdrv_co_io_em_complete, &co); 813 if (acb == NULL) { 814 return -EIO; 815 } else { 816 qemu_coroutine_yield(); 817 return co.ret; 818 } 819 } 820 } 821 822 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 823 uint64_t offset, uint64_t bytes, 824 QEMUIOVector *qiov, int flags) 825 { 826 BlockDriver *drv = bs->drv; 827 int64_t sector_num; 828 unsigned int nb_sectors; 829 int ret; 830 831 if (drv->bdrv_co_pwritev) { 832 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags); 833 goto emulate_flags; 834 } 835 836 sector_num = offset >> BDRV_SECTOR_BITS; 837 nb_sectors = bytes >> BDRV_SECTOR_BITS; 838 839 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 840 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 841 assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS); 842 843 if (drv->bdrv_co_writev_flags) { 844 ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov, 845 flags & bs->supported_write_flags); 846 flags &= ~bs->supported_write_flags; 847 } else if (drv->bdrv_co_writev) { 848 assert(!bs->supported_write_flags); 849 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov); 850 } else { 851 BlockAIOCB *acb; 852 CoroutineIOCompletion co = { 853 .coroutine = qemu_coroutine_self(), 854 }; 855 856 acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors, 857 bdrv_co_io_em_complete, &co); 858 if (acb == NULL) { 859 ret = -EIO; 860 } else { 861 qemu_coroutine_yield(); 862 ret = co.ret; 863 } 864 } 865 866 emulate_flags: 867 if (ret == 0 && (flags & BDRV_REQ_FUA)) { 868 ret = bdrv_co_flush(bs); 869 } 870 871 return ret; 872 } 873 874 static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs, 875 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 876 { 877 /* Perform I/O through a temporary buffer so that users who scribble over 878 * their read buffer while the operation is in progress do not end up 879 * modifying the image file. This is critical for zero-copy guest I/O 880 * where anything might happen inside guest memory. 881 */ 882 void *bounce_buffer; 883 884 BlockDriver *drv = bs->drv; 885 struct iovec iov; 886 QEMUIOVector bounce_qiov; 887 int64_t cluster_sector_num; 888 int cluster_nb_sectors; 889 size_t skip_bytes; 890 int ret; 891 892 /* Cover entire cluster so no additional backing file I/O is required when 893 * allocating cluster in the image file. 894 */ 895 bdrv_round_to_clusters(bs, sector_num, nb_sectors, 896 &cluster_sector_num, &cluster_nb_sectors); 897 898 trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, 899 cluster_sector_num, cluster_nb_sectors); 900 901 iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE; 902 iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len); 903 if (bounce_buffer == NULL) { 904 ret = -ENOMEM; 905 goto err; 906 } 907 908 qemu_iovec_init_external(&bounce_qiov, &iov, 1); 909 910 ret = bdrv_driver_preadv(bs, cluster_sector_num * BDRV_SECTOR_SIZE, 911 cluster_nb_sectors * BDRV_SECTOR_SIZE, 912 &bounce_qiov, 0); 913 if (ret < 0) { 914 goto err; 915 } 916 917 if (drv->bdrv_co_write_zeroes && 918 buffer_is_zero(bounce_buffer, iov.iov_len)) { 919 ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num, 920 cluster_nb_sectors, 0); 921 } else { 922 /* This does not change the data on the disk, it is not necessary 923 * to flush even in cache=writethrough mode. 924 */ 925 ret = bdrv_driver_pwritev(bs, cluster_sector_num * BDRV_SECTOR_SIZE, 926 cluster_nb_sectors * BDRV_SECTOR_SIZE, 927 &bounce_qiov, 0); 928 } 929 930 if (ret < 0) { 931 /* It might be okay to ignore write errors for guest requests. If this 932 * is a deliberate copy-on-read then we don't want to ignore the error. 933 * Simply report it in all cases. 934 */ 935 goto err; 936 } 937 938 skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE; 939 qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, 940 nb_sectors * BDRV_SECTOR_SIZE); 941 942 err: 943 qemu_vfree(bounce_buffer); 944 return ret; 945 } 946 947 /* 948 * Forwards an already correctly aligned request to the BlockDriver. This 949 * handles copy on read and zeroing after EOF; any other features must be 950 * implemented by the caller. 951 */ 952 static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs, 953 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 954 int64_t align, QEMUIOVector *qiov, int flags) 955 { 956 int ret; 957 958 int64_t sector_num = offset >> BDRV_SECTOR_BITS; 959 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 960 961 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 962 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 963 assert(!qiov || bytes == qiov->size); 964 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 965 966 /* Handle Copy on Read and associated serialisation */ 967 if (flags & BDRV_REQ_COPY_ON_READ) { 968 /* If we touch the same cluster it counts as an overlap. This 969 * guarantees that allocating writes will be serialized and not race 970 * with each other for the same cluster. For example, in copy-on-read 971 * it ensures that the CoR read and write operations are atomic and 972 * guest writes cannot interleave between them. */ 973 mark_request_serialising(req, bdrv_get_cluster_size(bs)); 974 } 975 976 if (!(flags & BDRV_REQ_NO_SERIALISING)) { 977 wait_serialising_requests(req); 978 } 979 980 if (flags & BDRV_REQ_COPY_ON_READ) { 981 int pnum; 982 983 ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum); 984 if (ret < 0) { 985 goto out; 986 } 987 988 if (!ret || pnum != nb_sectors) { 989 ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov); 990 goto out; 991 } 992 } 993 994 /* Forward the request to the BlockDriver */ 995 if (!bs->zero_beyond_eof) { 996 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 997 } else { 998 /* Read zeros after EOF */ 999 int64_t total_sectors, max_nb_sectors; 1000 1001 total_sectors = bdrv_nb_sectors(bs); 1002 if (total_sectors < 0) { 1003 ret = total_sectors; 1004 goto out; 1005 } 1006 1007 max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num), 1008 align >> BDRV_SECTOR_BITS); 1009 if (nb_sectors < max_nb_sectors) { 1010 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0); 1011 } else if (max_nb_sectors > 0) { 1012 QEMUIOVector local_qiov; 1013 1014 qemu_iovec_init(&local_qiov, qiov->niov); 1015 qemu_iovec_concat(&local_qiov, qiov, 0, 1016 max_nb_sectors * BDRV_SECTOR_SIZE); 1017 1018 ret = bdrv_driver_preadv(bs, offset, 1019 max_nb_sectors * BDRV_SECTOR_SIZE, 1020 &local_qiov, 0); 1021 1022 qemu_iovec_destroy(&local_qiov); 1023 } else { 1024 ret = 0; 1025 } 1026 1027 /* Reading beyond end of file is supposed to produce zeroes */ 1028 if (ret == 0 && total_sectors < sector_num + nb_sectors) { 1029 uint64_t offset = MAX(0, total_sectors - sector_num); 1030 uint64_t bytes = (sector_num + nb_sectors - offset) * 1031 BDRV_SECTOR_SIZE; 1032 qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes); 1033 } 1034 } 1035 1036 out: 1037 return ret; 1038 } 1039 1040 /* 1041 * Handle a read request in coroutine context 1042 */ 1043 int coroutine_fn bdrv_co_preadv(BlockDriverState *bs, 1044 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1045 BdrvRequestFlags flags) 1046 { 1047 BlockDriver *drv = bs->drv; 1048 BdrvTrackedRequest req; 1049 1050 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 1051 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 1052 uint8_t *head_buf = NULL; 1053 uint8_t *tail_buf = NULL; 1054 QEMUIOVector local_qiov; 1055 bool use_local_qiov = false; 1056 int ret; 1057 1058 if (!drv) { 1059 return -ENOMEDIUM; 1060 } 1061 1062 ret = bdrv_check_byte_request(bs, offset, bytes); 1063 if (ret < 0) { 1064 return ret; 1065 } 1066 1067 /* Don't do copy-on-read if we read data before write operation */ 1068 if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) { 1069 flags |= BDRV_REQ_COPY_ON_READ; 1070 } 1071 1072 /* throttling disk I/O */ 1073 if (bs->throttle_state) { 1074 throttle_group_co_io_limits_intercept(bs, bytes, false); 1075 } 1076 1077 /* Align read if necessary by padding qiov */ 1078 if (offset & (align - 1)) { 1079 head_buf = qemu_blockalign(bs, align); 1080 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1081 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1082 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1083 use_local_qiov = true; 1084 1085 bytes += offset & (align - 1); 1086 offset = offset & ~(align - 1); 1087 } 1088 1089 if ((offset + bytes) & (align - 1)) { 1090 if (!use_local_qiov) { 1091 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1092 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1093 use_local_qiov = true; 1094 } 1095 tail_buf = qemu_blockalign(bs, align); 1096 qemu_iovec_add(&local_qiov, tail_buf, 1097 align - ((offset + bytes) & (align - 1))); 1098 1099 bytes = ROUND_UP(bytes, align); 1100 } 1101 1102 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1103 ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align, 1104 use_local_qiov ? &local_qiov : qiov, 1105 flags); 1106 tracked_request_end(&req); 1107 1108 if (use_local_qiov) { 1109 qemu_iovec_destroy(&local_qiov); 1110 qemu_vfree(head_buf); 1111 qemu_vfree(tail_buf); 1112 } 1113 1114 return ret; 1115 } 1116 1117 static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs, 1118 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1119 BdrvRequestFlags flags) 1120 { 1121 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1122 return -EINVAL; 1123 } 1124 1125 return bdrv_co_preadv(bs, sector_num << BDRV_SECTOR_BITS, 1126 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1127 } 1128 1129 int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num, 1130 int nb_sectors, QEMUIOVector *qiov) 1131 { 1132 trace_bdrv_co_readv(bs, sector_num, nb_sectors); 1133 1134 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0); 1135 } 1136 1137 int coroutine_fn bdrv_co_readv_no_serialising(BlockDriverState *bs, 1138 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 1139 { 1140 trace_bdrv_co_readv_no_serialising(bs, sector_num, nb_sectors); 1141 1142 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 1143 BDRV_REQ_NO_SERIALISING); 1144 } 1145 1146 int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs, 1147 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov) 1148 { 1149 trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors); 1150 1151 return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 1152 BDRV_REQ_COPY_ON_READ); 1153 } 1154 1155 #define MAX_WRITE_ZEROES_BOUNCE_BUFFER 32768 1156 1157 static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs, 1158 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags) 1159 { 1160 BlockDriver *drv = bs->drv; 1161 QEMUIOVector qiov; 1162 struct iovec iov = {0}; 1163 int ret = 0; 1164 bool need_flush = false; 1165 1166 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_write_zeroes, 1167 BDRV_REQUEST_MAX_SECTORS); 1168 1169 while (nb_sectors > 0 && !ret) { 1170 int num = nb_sectors; 1171 1172 /* Align request. Block drivers can expect the "bulk" of the request 1173 * to be aligned. 1174 */ 1175 if (bs->bl.write_zeroes_alignment 1176 && num > bs->bl.write_zeroes_alignment) { 1177 if (sector_num % bs->bl.write_zeroes_alignment != 0) { 1178 /* Make a small request up to the first aligned sector. */ 1179 num = bs->bl.write_zeroes_alignment; 1180 num -= sector_num % bs->bl.write_zeroes_alignment; 1181 } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) { 1182 /* Shorten the request to the last aligned sector. num cannot 1183 * underflow because num > bs->bl.write_zeroes_alignment. 1184 */ 1185 num -= (sector_num + num) % bs->bl.write_zeroes_alignment; 1186 } 1187 } 1188 1189 /* limit request size */ 1190 if (num > max_write_zeroes) { 1191 num = max_write_zeroes; 1192 } 1193 1194 ret = -ENOTSUP; 1195 /* First try the efficient write zeroes operation */ 1196 if (drv->bdrv_co_write_zeroes) { 1197 ret = drv->bdrv_co_write_zeroes(bs, sector_num, num, 1198 flags & bs->supported_zero_flags); 1199 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1200 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1201 need_flush = true; 1202 } 1203 } else { 1204 assert(!bs->supported_zero_flags); 1205 } 1206 1207 if (ret == -ENOTSUP) { 1208 /* Fall back to bounce buffer if write zeroes is unsupported */ 1209 int max_xfer_len = MIN_NON_ZERO(bs->bl.max_transfer_length, 1210 MAX_WRITE_ZEROES_BOUNCE_BUFFER); 1211 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1212 1213 if ((flags & BDRV_REQ_FUA) && 1214 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1215 /* No need for bdrv_driver_pwrite() to do a fallback 1216 * flush on each chunk; use just one at the end */ 1217 write_flags &= ~BDRV_REQ_FUA; 1218 need_flush = true; 1219 } 1220 num = MIN(num, max_xfer_len); 1221 iov.iov_len = num * BDRV_SECTOR_SIZE; 1222 if (iov.iov_base == NULL) { 1223 iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE); 1224 if (iov.iov_base == NULL) { 1225 ret = -ENOMEM; 1226 goto fail; 1227 } 1228 memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE); 1229 } 1230 qemu_iovec_init_external(&qiov, &iov, 1); 1231 1232 ret = bdrv_driver_pwritev(bs, sector_num * BDRV_SECTOR_SIZE, 1233 num * BDRV_SECTOR_SIZE, &qiov, 1234 write_flags); 1235 1236 /* Keep bounce buffer around if it is big enough for all 1237 * all future requests. 1238 */ 1239 if (num < max_xfer_len) { 1240 qemu_vfree(iov.iov_base); 1241 iov.iov_base = NULL; 1242 } 1243 } 1244 1245 sector_num += num; 1246 nb_sectors -= num; 1247 } 1248 1249 fail: 1250 if (ret == 0 && need_flush) { 1251 ret = bdrv_co_flush(bs); 1252 } 1253 qemu_vfree(iov.iov_base); 1254 return ret; 1255 } 1256 1257 /* 1258 * Forwards an already correctly aligned write request to the BlockDriver. 1259 */ 1260 static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs, 1261 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1262 QEMUIOVector *qiov, int flags) 1263 { 1264 BlockDriver *drv = bs->drv; 1265 bool waited; 1266 int ret; 1267 1268 int64_t sector_num = offset >> BDRV_SECTOR_BITS; 1269 unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS; 1270 1271 assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0); 1272 assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0); 1273 assert(!qiov || bytes == qiov->size); 1274 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1275 1276 waited = wait_serialising_requests(req); 1277 assert(!waited || !req->serialising); 1278 assert(req->overlap_offset <= offset); 1279 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1280 1281 ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req); 1282 1283 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1284 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes && 1285 qemu_iovec_is_zero(qiov)) { 1286 flags |= BDRV_REQ_ZERO_WRITE; 1287 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1288 flags |= BDRV_REQ_MAY_UNMAP; 1289 } 1290 } 1291 1292 if (ret < 0) { 1293 /* Do nothing, write notifier decided to fail this request */ 1294 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1295 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1296 ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags); 1297 } else { 1298 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1299 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags); 1300 } 1301 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 1302 1303 bdrv_set_dirty(bs, sector_num, nb_sectors); 1304 1305 if (bs->wr_highest_offset < offset + bytes) { 1306 bs->wr_highest_offset = offset + bytes; 1307 } 1308 1309 if (ret >= 0) { 1310 bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors); 1311 } 1312 1313 return ret; 1314 } 1315 1316 static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs, 1317 int64_t offset, 1318 unsigned int bytes, 1319 BdrvRequestFlags flags, 1320 BdrvTrackedRequest *req) 1321 { 1322 uint8_t *buf = NULL; 1323 QEMUIOVector local_qiov; 1324 struct iovec iov; 1325 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 1326 unsigned int head_padding_bytes, tail_padding_bytes; 1327 int ret = 0; 1328 1329 head_padding_bytes = offset & (align - 1); 1330 tail_padding_bytes = align - ((offset + bytes) & (align - 1)); 1331 1332 1333 assert(flags & BDRV_REQ_ZERO_WRITE); 1334 if (head_padding_bytes || tail_padding_bytes) { 1335 buf = qemu_blockalign(bs, align); 1336 iov = (struct iovec) { 1337 .iov_base = buf, 1338 .iov_len = align, 1339 }; 1340 qemu_iovec_init_external(&local_qiov, &iov, 1); 1341 } 1342 if (head_padding_bytes) { 1343 uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes); 1344 1345 /* RMW the unaligned part before head. */ 1346 mark_request_serialising(req, align); 1347 wait_serialising_requests(req); 1348 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1349 ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align, 1350 align, &local_qiov, 0); 1351 if (ret < 0) { 1352 goto fail; 1353 } 1354 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1355 1356 memset(buf + head_padding_bytes, 0, zero_bytes); 1357 ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align, 1358 &local_qiov, 1359 flags & ~BDRV_REQ_ZERO_WRITE); 1360 if (ret < 0) { 1361 goto fail; 1362 } 1363 offset += zero_bytes; 1364 bytes -= zero_bytes; 1365 } 1366 1367 assert(!bytes || (offset & (align - 1)) == 0); 1368 if (bytes >= align) { 1369 /* Write the aligned part in the middle. */ 1370 uint64_t aligned_bytes = bytes & ~(align - 1); 1371 ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, 1372 NULL, flags); 1373 if (ret < 0) { 1374 goto fail; 1375 } 1376 bytes -= aligned_bytes; 1377 offset += aligned_bytes; 1378 } 1379 1380 assert(!bytes || (offset & (align - 1)) == 0); 1381 if (bytes) { 1382 assert(align == tail_padding_bytes + bytes); 1383 /* RMW the unaligned part after tail. */ 1384 mark_request_serialising(req, align); 1385 wait_serialising_requests(req); 1386 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1387 ret = bdrv_aligned_preadv(bs, req, offset, align, 1388 align, &local_qiov, 0); 1389 if (ret < 0) { 1390 goto fail; 1391 } 1392 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1393 1394 memset(buf, 0, bytes); 1395 ret = bdrv_aligned_pwritev(bs, req, offset, align, 1396 &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE); 1397 } 1398 fail: 1399 qemu_vfree(buf); 1400 return ret; 1401 1402 } 1403 1404 /* 1405 * Handle a write request in coroutine context 1406 */ 1407 int coroutine_fn bdrv_co_pwritev(BlockDriverState *bs, 1408 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1409 BdrvRequestFlags flags) 1410 { 1411 BdrvTrackedRequest req; 1412 /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */ 1413 uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment); 1414 uint8_t *head_buf = NULL; 1415 uint8_t *tail_buf = NULL; 1416 QEMUIOVector local_qiov; 1417 bool use_local_qiov = false; 1418 int ret; 1419 1420 if (!bs->drv) { 1421 return -ENOMEDIUM; 1422 } 1423 if (bs->read_only) { 1424 return -EPERM; 1425 } 1426 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1427 1428 ret = bdrv_check_byte_request(bs, offset, bytes); 1429 if (ret < 0) { 1430 return ret; 1431 } 1432 1433 /* throttling disk I/O */ 1434 if (bs->throttle_state) { 1435 throttle_group_co_io_limits_intercept(bs, bytes, true); 1436 } 1437 1438 /* 1439 * Align write if necessary by performing a read-modify-write cycle. 1440 * Pad qiov with the read parts and be sure to have a tracked request not 1441 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 1442 */ 1443 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 1444 1445 if (!qiov) { 1446 ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req); 1447 goto out; 1448 } 1449 1450 if (offset & (align - 1)) { 1451 QEMUIOVector head_qiov; 1452 struct iovec head_iov; 1453 1454 mark_request_serialising(&req, align); 1455 wait_serialising_requests(&req); 1456 1457 head_buf = qemu_blockalign(bs, align); 1458 head_iov = (struct iovec) { 1459 .iov_base = head_buf, 1460 .iov_len = align, 1461 }; 1462 qemu_iovec_init_external(&head_qiov, &head_iov, 1); 1463 1464 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1465 ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align, 1466 align, &head_qiov, 0); 1467 if (ret < 0) { 1468 goto fail; 1469 } 1470 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1471 1472 qemu_iovec_init(&local_qiov, qiov->niov + 2); 1473 qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1)); 1474 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1475 use_local_qiov = true; 1476 1477 bytes += offset & (align - 1); 1478 offset = offset & ~(align - 1); 1479 } 1480 1481 if ((offset + bytes) & (align - 1)) { 1482 QEMUIOVector tail_qiov; 1483 struct iovec tail_iov; 1484 size_t tail_bytes; 1485 bool waited; 1486 1487 mark_request_serialising(&req, align); 1488 waited = wait_serialising_requests(&req); 1489 assert(!waited || !use_local_qiov); 1490 1491 tail_buf = qemu_blockalign(bs, align); 1492 tail_iov = (struct iovec) { 1493 .iov_base = tail_buf, 1494 .iov_len = align, 1495 }; 1496 qemu_iovec_init_external(&tail_qiov, &tail_iov, 1); 1497 1498 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1499 ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align, 1500 align, &tail_qiov, 0); 1501 if (ret < 0) { 1502 goto fail; 1503 } 1504 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1505 1506 if (!use_local_qiov) { 1507 qemu_iovec_init(&local_qiov, qiov->niov + 1); 1508 qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size); 1509 use_local_qiov = true; 1510 } 1511 1512 tail_bytes = (offset + bytes) & (align - 1); 1513 qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes); 1514 1515 bytes = ROUND_UP(bytes, align); 1516 } 1517 1518 ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, 1519 use_local_qiov ? &local_qiov : qiov, 1520 flags); 1521 1522 fail: 1523 1524 if (use_local_qiov) { 1525 qemu_iovec_destroy(&local_qiov); 1526 } 1527 qemu_vfree(head_buf); 1528 qemu_vfree(tail_buf); 1529 out: 1530 tracked_request_end(&req); 1531 return ret; 1532 } 1533 1534 static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs, 1535 int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, 1536 BdrvRequestFlags flags) 1537 { 1538 if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) { 1539 return -EINVAL; 1540 } 1541 1542 return bdrv_co_pwritev(bs, sector_num << BDRV_SECTOR_BITS, 1543 nb_sectors << BDRV_SECTOR_BITS, qiov, flags); 1544 } 1545 1546 int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num, 1547 int nb_sectors, QEMUIOVector *qiov) 1548 { 1549 trace_bdrv_co_writev(bs, sector_num, nb_sectors); 1550 1551 return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0); 1552 } 1553 1554 int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, 1555 int64_t sector_num, int nb_sectors, 1556 BdrvRequestFlags flags) 1557 { 1558 trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags); 1559 1560 if (!(bs->open_flags & BDRV_O_UNMAP)) { 1561 flags &= ~BDRV_REQ_MAY_UNMAP; 1562 } 1563 1564 return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL, 1565 BDRV_REQ_ZERO_WRITE | flags); 1566 } 1567 1568 typedef struct BdrvCoGetBlockStatusData { 1569 BlockDriverState *bs; 1570 BlockDriverState *base; 1571 BlockDriverState **file; 1572 int64_t sector_num; 1573 int nb_sectors; 1574 int *pnum; 1575 int64_t ret; 1576 bool done; 1577 } BdrvCoGetBlockStatusData; 1578 1579 /* 1580 * Returns the allocation status of the specified sectors. 1581 * Drivers not implementing the functionality are assumed to not support 1582 * backing files, hence all their sectors are reported as allocated. 1583 * 1584 * If 'sector_num' is beyond the end of the disk image the return value is 0 1585 * and 'pnum' is set to 0. 1586 * 1587 * 'pnum' is set to the number of sectors (including and immediately following 1588 * the specified sector) that are known to be in the same 1589 * allocated/unallocated state. 1590 * 1591 * 'nb_sectors' is the max value 'pnum' should be set to. If nb_sectors goes 1592 * beyond the end of the disk image it will be clamped. 1593 * 1594 * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file' 1595 * points to the BDS which the sector range is allocated in. 1596 */ 1597 static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs, 1598 int64_t sector_num, 1599 int nb_sectors, int *pnum, 1600 BlockDriverState **file) 1601 { 1602 int64_t total_sectors; 1603 int64_t n; 1604 int64_t ret, ret2; 1605 1606 total_sectors = bdrv_nb_sectors(bs); 1607 if (total_sectors < 0) { 1608 return total_sectors; 1609 } 1610 1611 if (sector_num >= total_sectors) { 1612 *pnum = 0; 1613 return 0; 1614 } 1615 1616 n = total_sectors - sector_num; 1617 if (n < nb_sectors) { 1618 nb_sectors = n; 1619 } 1620 1621 if (!bs->drv->bdrv_co_get_block_status) { 1622 *pnum = nb_sectors; 1623 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 1624 if (bs->drv->protocol_name) { 1625 ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE); 1626 } 1627 return ret; 1628 } 1629 1630 *file = NULL; 1631 ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum, 1632 file); 1633 if (ret < 0) { 1634 *pnum = 0; 1635 return ret; 1636 } 1637 1638 if (ret & BDRV_BLOCK_RAW) { 1639 assert(ret & BDRV_BLOCK_OFFSET_VALID); 1640 return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS, 1641 *pnum, pnum, file); 1642 } 1643 1644 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 1645 ret |= BDRV_BLOCK_ALLOCATED; 1646 } else { 1647 if (bdrv_unallocated_blocks_are_zero(bs)) { 1648 ret |= BDRV_BLOCK_ZERO; 1649 } else if (bs->backing) { 1650 BlockDriverState *bs2 = bs->backing->bs; 1651 int64_t nb_sectors2 = bdrv_nb_sectors(bs2); 1652 if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) { 1653 ret |= BDRV_BLOCK_ZERO; 1654 } 1655 } 1656 } 1657 1658 if (*file && *file != bs && 1659 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 1660 (ret & BDRV_BLOCK_OFFSET_VALID)) { 1661 BlockDriverState *file2; 1662 int file_pnum; 1663 1664 ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS, 1665 *pnum, &file_pnum, &file2); 1666 if (ret2 >= 0) { 1667 /* Ignore errors. This is just providing extra information, it 1668 * is useful but not necessary. 1669 */ 1670 if (!file_pnum) { 1671 /* !file_pnum indicates an offset at or beyond the EOF; it is 1672 * perfectly valid for the format block driver to point to such 1673 * offsets, so catch it and mark everything as zero */ 1674 ret |= BDRV_BLOCK_ZERO; 1675 } else { 1676 /* Limit request to the range reported by the protocol driver */ 1677 *pnum = file_pnum; 1678 ret |= (ret2 & BDRV_BLOCK_ZERO); 1679 } 1680 } 1681 } 1682 1683 return ret; 1684 } 1685 1686 static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs, 1687 BlockDriverState *base, 1688 int64_t sector_num, 1689 int nb_sectors, 1690 int *pnum, 1691 BlockDriverState **file) 1692 { 1693 BlockDriverState *p; 1694 int64_t ret = 0; 1695 1696 assert(bs != base); 1697 for (p = bs; p != base; p = backing_bs(p)) { 1698 ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file); 1699 if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) { 1700 break; 1701 } 1702 /* [sector_num, pnum] unallocated on this layer, which could be only 1703 * the first part of [sector_num, nb_sectors]. */ 1704 nb_sectors = MIN(nb_sectors, *pnum); 1705 } 1706 return ret; 1707 } 1708 1709 /* Coroutine wrapper for bdrv_get_block_status_above() */ 1710 static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque) 1711 { 1712 BdrvCoGetBlockStatusData *data = opaque; 1713 1714 data->ret = bdrv_co_get_block_status_above(data->bs, data->base, 1715 data->sector_num, 1716 data->nb_sectors, 1717 data->pnum, 1718 data->file); 1719 data->done = true; 1720 } 1721 1722 /* 1723 * Synchronous wrapper around bdrv_co_get_block_status_above(). 1724 * 1725 * See bdrv_co_get_block_status_above() for details. 1726 */ 1727 int64_t bdrv_get_block_status_above(BlockDriverState *bs, 1728 BlockDriverState *base, 1729 int64_t sector_num, 1730 int nb_sectors, int *pnum, 1731 BlockDriverState **file) 1732 { 1733 Coroutine *co; 1734 BdrvCoGetBlockStatusData data = { 1735 .bs = bs, 1736 .base = base, 1737 .file = file, 1738 .sector_num = sector_num, 1739 .nb_sectors = nb_sectors, 1740 .pnum = pnum, 1741 .done = false, 1742 }; 1743 1744 if (qemu_in_coroutine()) { 1745 /* Fast-path if already in coroutine context */ 1746 bdrv_get_block_status_above_co_entry(&data); 1747 } else { 1748 AioContext *aio_context = bdrv_get_aio_context(bs); 1749 1750 co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry); 1751 qemu_coroutine_enter(co, &data); 1752 while (!data.done) { 1753 aio_poll(aio_context, true); 1754 } 1755 } 1756 return data.ret; 1757 } 1758 1759 int64_t bdrv_get_block_status(BlockDriverState *bs, 1760 int64_t sector_num, 1761 int nb_sectors, int *pnum, 1762 BlockDriverState **file) 1763 { 1764 return bdrv_get_block_status_above(bs, backing_bs(bs), 1765 sector_num, nb_sectors, pnum, file); 1766 } 1767 1768 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, 1769 int nb_sectors, int *pnum) 1770 { 1771 BlockDriverState *file; 1772 int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum, 1773 &file); 1774 if (ret < 0) { 1775 return ret; 1776 } 1777 return !!(ret & BDRV_BLOCK_ALLOCATED); 1778 } 1779 1780 /* 1781 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 1782 * 1783 * Return true if the given sector is allocated in any image between 1784 * BASE and TOP (inclusive). BASE can be NULL to check if the given 1785 * sector is allocated in any image of the chain. Return false otherwise. 1786 * 1787 * 'pnum' is set to the number of sectors (including and immediately following 1788 * the specified sector) that are known to be in the same 1789 * allocated/unallocated state. 1790 * 1791 */ 1792 int bdrv_is_allocated_above(BlockDriverState *top, 1793 BlockDriverState *base, 1794 int64_t sector_num, 1795 int nb_sectors, int *pnum) 1796 { 1797 BlockDriverState *intermediate; 1798 int ret, n = nb_sectors; 1799 1800 intermediate = top; 1801 while (intermediate && intermediate != base) { 1802 int pnum_inter; 1803 ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors, 1804 &pnum_inter); 1805 if (ret < 0) { 1806 return ret; 1807 } else if (ret) { 1808 *pnum = pnum_inter; 1809 return 1; 1810 } 1811 1812 /* 1813 * [sector_num, nb_sectors] is unallocated on top but intermediate 1814 * might have 1815 * 1816 * [sector_num+x, nr_sectors] allocated. 1817 */ 1818 if (n > pnum_inter && 1819 (intermediate == top || 1820 sector_num + pnum_inter < intermediate->total_sectors)) { 1821 n = pnum_inter; 1822 } 1823 1824 intermediate = backing_bs(intermediate); 1825 } 1826 1827 *pnum = n; 1828 return 0; 1829 } 1830 1831 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num, 1832 const uint8_t *buf, int nb_sectors) 1833 { 1834 BlockDriver *drv = bs->drv; 1835 int ret; 1836 1837 if (!drv) { 1838 return -ENOMEDIUM; 1839 } 1840 if (!drv->bdrv_write_compressed) { 1841 return -ENOTSUP; 1842 } 1843 ret = bdrv_check_request(bs, sector_num, nb_sectors); 1844 if (ret < 0) { 1845 return ret; 1846 } 1847 1848 assert(QLIST_EMPTY(&bs->dirty_bitmaps)); 1849 1850 return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors); 1851 } 1852 1853 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 1854 int64_t pos, int size) 1855 { 1856 QEMUIOVector qiov; 1857 struct iovec iov = { 1858 .iov_base = (void *) buf, 1859 .iov_len = size, 1860 }; 1861 1862 qemu_iovec_init_external(&qiov, &iov, 1); 1863 return bdrv_writev_vmstate(bs, &qiov, pos); 1864 } 1865 1866 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 1867 { 1868 BlockDriver *drv = bs->drv; 1869 1870 if (!drv) { 1871 return -ENOMEDIUM; 1872 } else if (drv->bdrv_save_vmstate) { 1873 return drv->bdrv_save_vmstate(bs, qiov, pos); 1874 } else if (bs->file) { 1875 return bdrv_writev_vmstate(bs->file->bs, qiov, pos); 1876 } 1877 1878 return -ENOTSUP; 1879 } 1880 1881 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 1882 int64_t pos, int size) 1883 { 1884 BlockDriver *drv = bs->drv; 1885 if (!drv) 1886 return -ENOMEDIUM; 1887 if (drv->bdrv_load_vmstate) 1888 return drv->bdrv_load_vmstate(bs, buf, pos, size); 1889 if (bs->file) 1890 return bdrv_load_vmstate(bs->file->bs, buf, pos, size); 1891 return -ENOTSUP; 1892 } 1893 1894 /**************************************************************/ 1895 /* async I/Os */ 1896 1897 BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num, 1898 QEMUIOVector *qiov, int nb_sectors, 1899 BlockCompletionFunc *cb, void *opaque) 1900 { 1901 trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque); 1902 1903 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 1904 cb, opaque, false); 1905 } 1906 1907 BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num, 1908 QEMUIOVector *qiov, int nb_sectors, 1909 BlockCompletionFunc *cb, void *opaque) 1910 { 1911 trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque); 1912 1913 return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0, 1914 cb, opaque, true); 1915 } 1916 1917 BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs, 1918 int64_t sector_num, int nb_sectors, BdrvRequestFlags flags, 1919 BlockCompletionFunc *cb, void *opaque) 1920 { 1921 trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque); 1922 1923 return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors, 1924 BDRV_REQ_ZERO_WRITE | flags, 1925 cb, opaque, true); 1926 } 1927 1928 1929 typedef struct MultiwriteCB { 1930 int error; 1931 int num_requests; 1932 int num_callbacks; 1933 struct { 1934 BlockCompletionFunc *cb; 1935 void *opaque; 1936 QEMUIOVector *free_qiov; 1937 } callbacks[]; 1938 } MultiwriteCB; 1939 1940 static void multiwrite_user_cb(MultiwriteCB *mcb) 1941 { 1942 int i; 1943 1944 for (i = 0; i < mcb->num_callbacks; i++) { 1945 mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error); 1946 if (mcb->callbacks[i].free_qiov) { 1947 qemu_iovec_destroy(mcb->callbacks[i].free_qiov); 1948 } 1949 g_free(mcb->callbacks[i].free_qiov); 1950 } 1951 } 1952 1953 static void multiwrite_cb(void *opaque, int ret) 1954 { 1955 MultiwriteCB *mcb = opaque; 1956 1957 trace_multiwrite_cb(mcb, ret); 1958 1959 if (ret < 0 && !mcb->error) { 1960 mcb->error = ret; 1961 } 1962 1963 mcb->num_requests--; 1964 if (mcb->num_requests == 0) { 1965 multiwrite_user_cb(mcb); 1966 g_free(mcb); 1967 } 1968 } 1969 1970 static int multiwrite_req_compare(const void *a, const void *b) 1971 { 1972 const BlockRequest *req1 = a, *req2 = b; 1973 1974 /* 1975 * Note that we can't simply subtract req2->sector from req1->sector 1976 * here as that could overflow the return value. 1977 */ 1978 if (req1->sector > req2->sector) { 1979 return 1; 1980 } else if (req1->sector < req2->sector) { 1981 return -1; 1982 } else { 1983 return 0; 1984 } 1985 } 1986 1987 /* 1988 * Takes a bunch of requests and tries to merge them. Returns the number of 1989 * requests that remain after merging. 1990 */ 1991 static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs, 1992 int num_reqs, MultiwriteCB *mcb) 1993 { 1994 int i, outidx; 1995 1996 // Sort requests by start sector 1997 qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare); 1998 1999 // Check if adjacent requests touch the same clusters. If so, combine them, 2000 // filling up gaps with zero sectors. 2001 outidx = 0; 2002 for (i = 1; i < num_reqs; i++) { 2003 int merge = 0; 2004 int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors; 2005 2006 // Handle exactly sequential writes and overlapping writes. 2007 if (reqs[i].sector <= oldreq_last) { 2008 merge = 1; 2009 } 2010 2011 if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 > 2012 bs->bl.max_iov) { 2013 merge = 0; 2014 } 2015 2016 if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors + 2017 reqs[i].nb_sectors > bs->bl.max_transfer_length) { 2018 merge = 0; 2019 } 2020 2021 if (merge) { 2022 size_t size; 2023 QEMUIOVector *qiov = g_malloc0(sizeof(*qiov)); 2024 qemu_iovec_init(qiov, 2025 reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1); 2026 2027 // Add the first request to the merged one. If the requests are 2028 // overlapping, drop the last sectors of the first request. 2029 size = (reqs[i].sector - reqs[outidx].sector) << 9; 2030 qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size); 2031 2032 // We should need to add any zeros between the two requests 2033 assert (reqs[i].sector <= oldreq_last); 2034 2035 // Add the second request 2036 qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size); 2037 2038 // Add tail of first request, if necessary 2039 if (qiov->size < reqs[outidx].qiov->size) { 2040 qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size, 2041 reqs[outidx].qiov->size - qiov->size); 2042 } 2043 2044 reqs[outidx].nb_sectors = qiov->size >> 9; 2045 reqs[outidx].qiov = qiov; 2046 2047 mcb->callbacks[i].free_qiov = reqs[outidx].qiov; 2048 } else { 2049 outidx++; 2050 reqs[outidx].sector = reqs[i].sector; 2051 reqs[outidx].nb_sectors = reqs[i].nb_sectors; 2052 reqs[outidx].qiov = reqs[i].qiov; 2053 } 2054 } 2055 2056 if (bs->blk) { 2057 block_acct_merge_done(blk_get_stats(bs->blk), BLOCK_ACCT_WRITE, 2058 num_reqs - outidx - 1); 2059 } 2060 2061 return outidx + 1; 2062 } 2063 2064 /* 2065 * Submit multiple AIO write requests at once. 2066 * 2067 * On success, the function returns 0 and all requests in the reqs array have 2068 * been submitted. In error case this function returns -1, and any of the 2069 * requests may or may not be submitted yet. In particular, this means that the 2070 * callback will be called for some of the requests, for others it won't. The 2071 * caller must check the error field of the BlockRequest to wait for the right 2072 * callbacks (if error != 0, no callback will be called). 2073 * 2074 * The implementation may modify the contents of the reqs array, e.g. to merge 2075 * requests. However, the fields opaque and error are left unmodified as they 2076 * are used to signal failure for a single request to the caller. 2077 */ 2078 int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs) 2079 { 2080 MultiwriteCB *mcb; 2081 int i; 2082 2083 /* don't submit writes if we don't have a medium */ 2084 if (bs->drv == NULL) { 2085 for (i = 0; i < num_reqs; i++) { 2086 reqs[i].error = -ENOMEDIUM; 2087 } 2088 return -1; 2089 } 2090 2091 if (num_reqs == 0) { 2092 return 0; 2093 } 2094 2095 // Create MultiwriteCB structure 2096 mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks)); 2097 mcb->num_requests = 0; 2098 mcb->num_callbacks = num_reqs; 2099 2100 for (i = 0; i < num_reqs; i++) { 2101 mcb->callbacks[i].cb = reqs[i].cb; 2102 mcb->callbacks[i].opaque = reqs[i].opaque; 2103 } 2104 2105 // Check for mergable requests 2106 num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb); 2107 2108 trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs); 2109 2110 /* Run the aio requests. */ 2111 mcb->num_requests = num_reqs; 2112 for (i = 0; i < num_reqs; i++) { 2113 bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov, 2114 reqs[i].nb_sectors, reqs[i].flags, 2115 multiwrite_cb, mcb, 2116 true); 2117 } 2118 2119 return 0; 2120 } 2121 2122 void bdrv_aio_cancel(BlockAIOCB *acb) 2123 { 2124 qemu_aio_ref(acb); 2125 bdrv_aio_cancel_async(acb); 2126 while (acb->refcnt > 1) { 2127 if (acb->aiocb_info->get_aio_context) { 2128 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2129 } else if (acb->bs) { 2130 aio_poll(bdrv_get_aio_context(acb->bs), true); 2131 } else { 2132 abort(); 2133 } 2134 } 2135 qemu_aio_unref(acb); 2136 } 2137 2138 /* Async version of aio cancel. The caller is not blocked if the acb implements 2139 * cancel_async, otherwise we do nothing and let the request normally complete. 2140 * In either case the completion callback must be called. */ 2141 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2142 { 2143 if (acb->aiocb_info->cancel_async) { 2144 acb->aiocb_info->cancel_async(acb); 2145 } 2146 } 2147 2148 /**************************************************************/ 2149 /* async block device emulation */ 2150 2151 typedef struct BlockAIOCBCoroutine { 2152 BlockAIOCB common; 2153 BlockRequest req; 2154 bool is_write; 2155 bool need_bh; 2156 bool *done; 2157 QEMUBH* bh; 2158 } BlockAIOCBCoroutine; 2159 2160 static const AIOCBInfo bdrv_em_co_aiocb_info = { 2161 .aiocb_size = sizeof(BlockAIOCBCoroutine), 2162 }; 2163 2164 static void bdrv_co_complete(BlockAIOCBCoroutine *acb) 2165 { 2166 if (!acb->need_bh) { 2167 acb->common.cb(acb->common.opaque, acb->req.error); 2168 qemu_aio_unref(acb); 2169 } 2170 } 2171 2172 static void bdrv_co_em_bh(void *opaque) 2173 { 2174 BlockAIOCBCoroutine *acb = opaque; 2175 2176 assert(!acb->need_bh); 2177 qemu_bh_delete(acb->bh); 2178 bdrv_co_complete(acb); 2179 } 2180 2181 static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb) 2182 { 2183 acb->need_bh = false; 2184 if (acb->req.error != -EINPROGRESS) { 2185 BlockDriverState *bs = acb->common.bs; 2186 2187 acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb); 2188 qemu_bh_schedule(acb->bh); 2189 } 2190 } 2191 2192 /* Invoke bdrv_co_do_readv/bdrv_co_do_writev */ 2193 static void coroutine_fn bdrv_co_do_rw(void *opaque) 2194 { 2195 BlockAIOCBCoroutine *acb = opaque; 2196 BlockDriverState *bs = acb->common.bs; 2197 2198 if (!acb->is_write) { 2199 acb->req.error = bdrv_co_do_readv(bs, acb->req.sector, 2200 acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 2201 } else { 2202 acb->req.error = bdrv_co_do_writev(bs, acb->req.sector, 2203 acb->req.nb_sectors, acb->req.qiov, acb->req.flags); 2204 } 2205 2206 bdrv_co_complete(acb); 2207 } 2208 2209 static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs, 2210 int64_t sector_num, 2211 QEMUIOVector *qiov, 2212 int nb_sectors, 2213 BdrvRequestFlags flags, 2214 BlockCompletionFunc *cb, 2215 void *opaque, 2216 bool is_write) 2217 { 2218 Coroutine *co; 2219 BlockAIOCBCoroutine *acb; 2220 2221 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2222 acb->need_bh = true; 2223 acb->req.error = -EINPROGRESS; 2224 acb->req.sector = sector_num; 2225 acb->req.nb_sectors = nb_sectors; 2226 acb->req.qiov = qiov; 2227 acb->req.flags = flags; 2228 acb->is_write = is_write; 2229 2230 co = qemu_coroutine_create(bdrv_co_do_rw); 2231 qemu_coroutine_enter(co, acb); 2232 2233 bdrv_co_maybe_schedule_bh(acb); 2234 return &acb->common; 2235 } 2236 2237 static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque) 2238 { 2239 BlockAIOCBCoroutine *acb = opaque; 2240 BlockDriverState *bs = acb->common.bs; 2241 2242 acb->req.error = bdrv_co_flush(bs); 2243 bdrv_co_complete(acb); 2244 } 2245 2246 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs, 2247 BlockCompletionFunc *cb, void *opaque) 2248 { 2249 trace_bdrv_aio_flush(bs, opaque); 2250 2251 Coroutine *co; 2252 BlockAIOCBCoroutine *acb; 2253 2254 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2255 acb->need_bh = true; 2256 acb->req.error = -EINPROGRESS; 2257 2258 co = qemu_coroutine_create(bdrv_aio_flush_co_entry); 2259 qemu_coroutine_enter(co, acb); 2260 2261 bdrv_co_maybe_schedule_bh(acb); 2262 return &acb->common; 2263 } 2264 2265 static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque) 2266 { 2267 BlockAIOCBCoroutine *acb = opaque; 2268 BlockDriverState *bs = acb->common.bs; 2269 2270 acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors); 2271 bdrv_co_complete(acb); 2272 } 2273 2274 BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs, 2275 int64_t sector_num, int nb_sectors, 2276 BlockCompletionFunc *cb, void *opaque) 2277 { 2278 Coroutine *co; 2279 BlockAIOCBCoroutine *acb; 2280 2281 trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque); 2282 2283 acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque); 2284 acb->need_bh = true; 2285 acb->req.error = -EINPROGRESS; 2286 acb->req.sector = sector_num; 2287 acb->req.nb_sectors = nb_sectors; 2288 co = qemu_coroutine_create(bdrv_aio_discard_co_entry); 2289 qemu_coroutine_enter(co, acb); 2290 2291 bdrv_co_maybe_schedule_bh(acb); 2292 return &acb->common; 2293 } 2294 2295 void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs, 2296 BlockCompletionFunc *cb, void *opaque) 2297 { 2298 BlockAIOCB *acb; 2299 2300 acb = g_malloc(aiocb_info->aiocb_size); 2301 acb->aiocb_info = aiocb_info; 2302 acb->bs = bs; 2303 acb->cb = cb; 2304 acb->opaque = opaque; 2305 acb->refcnt = 1; 2306 return acb; 2307 } 2308 2309 void qemu_aio_ref(void *p) 2310 { 2311 BlockAIOCB *acb = p; 2312 acb->refcnt++; 2313 } 2314 2315 void qemu_aio_unref(void *p) 2316 { 2317 BlockAIOCB *acb = p; 2318 assert(acb->refcnt > 0); 2319 if (--acb->refcnt == 0) { 2320 g_free(acb); 2321 } 2322 } 2323 2324 /**************************************************************/ 2325 /* Coroutine block device emulation */ 2326 2327 static void coroutine_fn bdrv_flush_co_entry(void *opaque) 2328 { 2329 RwCo *rwco = opaque; 2330 2331 rwco->ret = bdrv_co_flush(rwco->bs); 2332 } 2333 2334 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2335 { 2336 int ret; 2337 BdrvTrackedRequest req; 2338 2339 if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2340 bdrv_is_sg(bs)) { 2341 return 0; 2342 } 2343 2344 tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH); 2345 2346 /* Write back all layers by calling one driver function */ 2347 if (bs->drv->bdrv_co_flush) { 2348 ret = bs->drv->bdrv_co_flush(bs); 2349 goto out; 2350 } 2351 2352 /* Write back cached data to the OS even with cache=unsafe */ 2353 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS); 2354 if (bs->drv->bdrv_co_flush_to_os) { 2355 ret = bs->drv->bdrv_co_flush_to_os(bs); 2356 if (ret < 0) { 2357 goto out; 2358 } 2359 } 2360 2361 /* But don't actually force it to the disk with cache=unsafe */ 2362 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2363 goto flush_parent; 2364 } 2365 2366 BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK); 2367 if (bs->drv->bdrv_co_flush_to_disk) { 2368 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2369 } else if (bs->drv->bdrv_aio_flush) { 2370 BlockAIOCB *acb; 2371 CoroutineIOCompletion co = { 2372 .coroutine = qemu_coroutine_self(), 2373 }; 2374 2375 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2376 if (acb == NULL) { 2377 ret = -EIO; 2378 } else { 2379 qemu_coroutine_yield(); 2380 ret = co.ret; 2381 } 2382 } else { 2383 /* 2384 * Some block drivers always operate in either writethrough or unsafe 2385 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2386 * know how the server works (because the behaviour is hardcoded or 2387 * depends on server-side configuration), so we can't ensure that 2388 * everything is safe on disk. Returning an error doesn't work because 2389 * that would break guests even if the server operates in writethrough 2390 * mode. 2391 * 2392 * Let's hope the user knows what he's doing. 2393 */ 2394 ret = 0; 2395 } 2396 if (ret < 0) { 2397 goto out; 2398 } 2399 2400 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2401 * in the case of cache=unsafe, so there are no useless flushes. 2402 */ 2403 flush_parent: 2404 ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0; 2405 out: 2406 tracked_request_end(&req); 2407 return ret; 2408 } 2409 2410 int bdrv_flush(BlockDriverState *bs) 2411 { 2412 Coroutine *co; 2413 RwCo rwco = { 2414 .bs = bs, 2415 .ret = NOT_DONE, 2416 }; 2417 2418 if (qemu_in_coroutine()) { 2419 /* Fast-path if already in coroutine context */ 2420 bdrv_flush_co_entry(&rwco); 2421 } else { 2422 AioContext *aio_context = bdrv_get_aio_context(bs); 2423 2424 co = qemu_coroutine_create(bdrv_flush_co_entry); 2425 qemu_coroutine_enter(co, &rwco); 2426 while (rwco.ret == NOT_DONE) { 2427 aio_poll(aio_context, true); 2428 } 2429 } 2430 2431 return rwco.ret; 2432 } 2433 2434 typedef struct DiscardCo { 2435 BlockDriverState *bs; 2436 int64_t sector_num; 2437 int nb_sectors; 2438 int ret; 2439 } DiscardCo; 2440 static void coroutine_fn bdrv_discard_co_entry(void *opaque) 2441 { 2442 DiscardCo *rwco = opaque; 2443 2444 rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors); 2445 } 2446 2447 int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, 2448 int nb_sectors) 2449 { 2450 BdrvTrackedRequest req; 2451 int max_discard, ret; 2452 2453 if (!bs->drv) { 2454 return -ENOMEDIUM; 2455 } 2456 2457 ret = bdrv_check_request(bs, sector_num, nb_sectors); 2458 if (ret < 0) { 2459 return ret; 2460 } else if (bs->read_only) { 2461 return -EPERM; 2462 } 2463 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 2464 2465 /* Do nothing if disabled. */ 2466 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2467 return 0; 2468 } 2469 2470 if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) { 2471 return 0; 2472 } 2473 2474 tracked_request_begin(&req, bs, sector_num, nb_sectors, 2475 BDRV_TRACKED_DISCARD); 2476 bdrv_set_dirty(bs, sector_num, nb_sectors); 2477 2478 max_discard = MIN_NON_ZERO(bs->bl.max_discard, BDRV_REQUEST_MAX_SECTORS); 2479 while (nb_sectors > 0) { 2480 int ret; 2481 int num = nb_sectors; 2482 2483 /* align request */ 2484 if (bs->bl.discard_alignment && 2485 num >= bs->bl.discard_alignment && 2486 sector_num % bs->bl.discard_alignment) { 2487 if (num > bs->bl.discard_alignment) { 2488 num = bs->bl.discard_alignment; 2489 } 2490 num -= sector_num % bs->bl.discard_alignment; 2491 } 2492 2493 /* limit request size */ 2494 if (num > max_discard) { 2495 num = max_discard; 2496 } 2497 2498 if (bs->drv->bdrv_co_discard) { 2499 ret = bs->drv->bdrv_co_discard(bs, sector_num, num); 2500 } else { 2501 BlockAIOCB *acb; 2502 CoroutineIOCompletion co = { 2503 .coroutine = qemu_coroutine_self(), 2504 }; 2505 2506 acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors, 2507 bdrv_co_io_em_complete, &co); 2508 if (acb == NULL) { 2509 ret = -EIO; 2510 goto out; 2511 } else { 2512 qemu_coroutine_yield(); 2513 ret = co.ret; 2514 } 2515 } 2516 if (ret && ret != -ENOTSUP) { 2517 goto out; 2518 } 2519 2520 sector_num += num; 2521 nb_sectors -= num; 2522 } 2523 ret = 0; 2524 out: 2525 tracked_request_end(&req); 2526 return ret; 2527 } 2528 2529 int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors) 2530 { 2531 Coroutine *co; 2532 DiscardCo rwco = { 2533 .bs = bs, 2534 .sector_num = sector_num, 2535 .nb_sectors = nb_sectors, 2536 .ret = NOT_DONE, 2537 }; 2538 2539 if (qemu_in_coroutine()) { 2540 /* Fast-path if already in coroutine context */ 2541 bdrv_discard_co_entry(&rwco); 2542 } else { 2543 AioContext *aio_context = bdrv_get_aio_context(bs); 2544 2545 co = qemu_coroutine_create(bdrv_discard_co_entry); 2546 qemu_coroutine_enter(co, &rwco); 2547 while (rwco.ret == NOT_DONE) { 2548 aio_poll(aio_context, true); 2549 } 2550 } 2551 2552 return rwco.ret; 2553 } 2554 2555 typedef struct { 2556 CoroutineIOCompletion *co; 2557 QEMUBH *bh; 2558 } BdrvIoctlCompletionData; 2559 2560 static void bdrv_ioctl_bh_cb(void *opaque) 2561 { 2562 BdrvIoctlCompletionData *data = opaque; 2563 2564 bdrv_co_io_em_complete(data->co, -ENOTSUP); 2565 qemu_bh_delete(data->bh); 2566 } 2567 2568 static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf) 2569 { 2570 BlockDriver *drv = bs->drv; 2571 BdrvTrackedRequest tracked_req; 2572 CoroutineIOCompletion co = { 2573 .coroutine = qemu_coroutine_self(), 2574 }; 2575 BlockAIOCB *acb; 2576 2577 tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL); 2578 if (!drv || !drv->bdrv_aio_ioctl) { 2579 co.ret = -ENOTSUP; 2580 goto out; 2581 } 2582 2583 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 2584 if (!acb) { 2585 BdrvIoctlCompletionData *data = g_new(BdrvIoctlCompletionData, 1); 2586 data->bh = aio_bh_new(bdrv_get_aio_context(bs), 2587 bdrv_ioctl_bh_cb, data); 2588 data->co = &co; 2589 qemu_bh_schedule(data->bh); 2590 } 2591 qemu_coroutine_yield(); 2592 out: 2593 tracked_request_end(&tracked_req); 2594 return co.ret; 2595 } 2596 2597 typedef struct { 2598 BlockDriverState *bs; 2599 int req; 2600 void *buf; 2601 int ret; 2602 } BdrvIoctlCoData; 2603 2604 static void coroutine_fn bdrv_co_ioctl_entry(void *opaque) 2605 { 2606 BdrvIoctlCoData *data = opaque; 2607 data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf); 2608 } 2609 2610 /* needed for generic scsi interface */ 2611 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf) 2612 { 2613 BdrvIoctlCoData data = { 2614 .bs = bs, 2615 .req = req, 2616 .buf = buf, 2617 .ret = -EINPROGRESS, 2618 }; 2619 2620 if (qemu_in_coroutine()) { 2621 /* Fast-path if already in coroutine context */ 2622 bdrv_co_ioctl_entry(&data); 2623 } else { 2624 Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry); 2625 2626 qemu_coroutine_enter(co, &data); 2627 while (data.ret == -EINPROGRESS) { 2628 aio_poll(bdrv_get_aio_context(bs), true); 2629 } 2630 } 2631 return data.ret; 2632 } 2633 2634 static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque) 2635 { 2636 BlockAIOCBCoroutine *acb = opaque; 2637 acb->req.error = bdrv_co_do_ioctl(acb->common.bs, 2638 acb->req.req, acb->req.buf); 2639 bdrv_co_complete(acb); 2640 } 2641 2642 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs, 2643 unsigned long int req, void *buf, 2644 BlockCompletionFunc *cb, void *opaque) 2645 { 2646 BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info, 2647 bs, cb, opaque); 2648 Coroutine *co; 2649 2650 acb->need_bh = true; 2651 acb->req.error = -EINPROGRESS; 2652 acb->req.req = req; 2653 acb->req.buf = buf; 2654 co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry); 2655 qemu_coroutine_enter(co, acb); 2656 2657 bdrv_co_maybe_schedule_bh(acb); 2658 return &acb->common; 2659 } 2660 2661 void *qemu_blockalign(BlockDriverState *bs, size_t size) 2662 { 2663 return qemu_memalign(bdrv_opt_mem_align(bs), size); 2664 } 2665 2666 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 2667 { 2668 return memset(qemu_blockalign(bs, size), 0, size); 2669 } 2670 2671 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 2672 { 2673 size_t align = bdrv_opt_mem_align(bs); 2674 2675 /* Ensure that NULL is never returned on success */ 2676 assert(align > 0); 2677 if (size == 0) { 2678 size = align; 2679 } 2680 2681 return qemu_try_memalign(align, size); 2682 } 2683 2684 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 2685 { 2686 void *mem = qemu_try_blockalign(bs, size); 2687 2688 if (mem) { 2689 memset(mem, 0, size); 2690 } 2691 2692 return mem; 2693 } 2694 2695 /* 2696 * Check if all memory in this vector is sector aligned. 2697 */ 2698 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 2699 { 2700 int i; 2701 size_t alignment = bdrv_min_mem_align(bs); 2702 2703 for (i = 0; i < qiov->niov; i++) { 2704 if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 2705 return false; 2706 } 2707 if (qiov->iov[i].iov_len % alignment) { 2708 return false; 2709 } 2710 } 2711 2712 return true; 2713 } 2714 2715 void bdrv_add_before_write_notifier(BlockDriverState *bs, 2716 NotifierWithReturn *notifier) 2717 { 2718 notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 2719 } 2720 2721 void bdrv_io_plug(BlockDriverState *bs) 2722 { 2723 BdrvChild *child; 2724 2725 QLIST_FOREACH(child, &bs->children, next) { 2726 bdrv_io_plug(child->bs); 2727 } 2728 2729 if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) { 2730 BlockDriver *drv = bs->drv; 2731 if (drv && drv->bdrv_io_plug) { 2732 drv->bdrv_io_plug(bs); 2733 } 2734 } 2735 } 2736 2737 void bdrv_io_unplug(BlockDriverState *bs) 2738 { 2739 BdrvChild *child; 2740 2741 assert(bs->io_plugged); 2742 if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) { 2743 BlockDriver *drv = bs->drv; 2744 if (drv && drv->bdrv_io_unplug) { 2745 drv->bdrv_io_unplug(bs); 2746 } 2747 } 2748 2749 QLIST_FOREACH(child, &bs->children, next) { 2750 bdrv_io_unplug(child->bs); 2751 } 2752 } 2753 2754 void bdrv_io_unplugged_begin(BlockDriverState *bs) 2755 { 2756 BdrvChild *child; 2757 2758 if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) { 2759 BlockDriver *drv = bs->drv; 2760 if (drv && drv->bdrv_io_unplug) { 2761 drv->bdrv_io_unplug(bs); 2762 } 2763 } 2764 2765 QLIST_FOREACH(child, &bs->children, next) { 2766 bdrv_io_unplugged_begin(child->bs); 2767 } 2768 } 2769 2770 void bdrv_io_unplugged_end(BlockDriverState *bs) 2771 { 2772 BdrvChild *child; 2773 2774 assert(bs->io_plug_disabled); 2775 QLIST_FOREACH(child, &bs->children, next) { 2776 bdrv_io_unplugged_end(child->bs); 2777 } 2778 2779 if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) { 2780 BlockDriver *drv = bs->drv; 2781 if (drv && drv->bdrv_io_plug) { 2782 drv->bdrv_io_plug(bs); 2783 } 2784 } 2785 } 2786 2787 void bdrv_drained_begin(BlockDriverState *bs) 2788 { 2789 if (!bs->quiesce_counter++) { 2790 aio_disable_external(bdrv_get_aio_context(bs)); 2791 } 2792 bdrv_drain(bs); 2793 } 2794 2795 void bdrv_drained_end(BlockDriverState *bs) 2796 { 2797 assert(bs->quiesce_counter > 0); 2798 if (--bs->quiesce_counter > 0) { 2799 return; 2800 } 2801 aio_enable_external(bdrv_get_aio_context(bs)); 2802 } 2803