1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/aio-wait.h" 29 #include "block/blockjob.h" 30 #include "block/blockjob_int.h" 31 #include "block/block_int.h" 32 #include "block/coroutines.h" 33 #include "block/dirty-bitmap.h" 34 #include "block/write-threshold.h" 35 #include "qemu/cutils.h" 36 #include "qemu/memalign.h" 37 #include "qapi/error.h" 38 #include "qemu/error-report.h" 39 #include "qemu/main-loop.h" 40 #include "sysemu/replay.h" 41 42 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 43 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 44 45 static void coroutine_fn GRAPH_RDLOCK 46 bdrv_parent_cb_resize(BlockDriverState *bs); 47 48 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 49 int64_t offset, int64_t bytes, BdrvRequestFlags flags); 50 51 static void GRAPH_RDLOCK 52 bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore) 53 { 54 BdrvChild *c, *next; 55 IO_OR_GS_CODE(); 56 assert_bdrv_graph_readable(); 57 58 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 59 if (c == ignore) { 60 continue; 61 } 62 bdrv_parent_drained_begin_single(c); 63 } 64 } 65 66 void bdrv_parent_drained_end_single(BdrvChild *c) 67 { 68 GLOBAL_STATE_CODE(); 69 70 assert(c->quiesced_parent); 71 c->quiesced_parent = false; 72 73 if (c->klass->drained_end) { 74 c->klass->drained_end(c); 75 } 76 } 77 78 static void GRAPH_RDLOCK 79 bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore) 80 { 81 BdrvChild *c; 82 IO_OR_GS_CODE(); 83 assert_bdrv_graph_readable(); 84 85 QLIST_FOREACH(c, &bs->parents, next_parent) { 86 if (c == ignore) { 87 continue; 88 } 89 bdrv_parent_drained_end_single(c); 90 } 91 } 92 93 bool bdrv_parent_drained_poll_single(BdrvChild *c) 94 { 95 IO_OR_GS_CODE(); 96 97 if (c->klass->drained_poll) { 98 return c->klass->drained_poll(c); 99 } 100 return false; 101 } 102 103 static bool GRAPH_RDLOCK 104 bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 105 bool ignore_bds_parents) 106 { 107 BdrvChild *c, *next; 108 bool busy = false; 109 IO_OR_GS_CODE(); 110 assert_bdrv_graph_readable(); 111 112 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 113 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 114 continue; 115 } 116 busy |= bdrv_parent_drained_poll_single(c); 117 } 118 119 return busy; 120 } 121 122 void bdrv_parent_drained_begin_single(BdrvChild *c) 123 { 124 GLOBAL_STATE_CODE(); 125 126 assert(!c->quiesced_parent); 127 c->quiesced_parent = true; 128 129 if (c->klass->drained_begin) { 130 /* called with rdlock taken, but it doesn't really need it. */ 131 c->klass->drained_begin(c); 132 } 133 } 134 135 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 136 { 137 dst->pdiscard_alignment = MAX(dst->pdiscard_alignment, 138 src->pdiscard_alignment); 139 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 140 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 141 dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer, 142 src->max_hw_transfer); 143 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 144 src->opt_mem_alignment); 145 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 146 src->min_mem_alignment); 147 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 148 dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov); 149 } 150 151 typedef struct BdrvRefreshLimitsState { 152 BlockDriverState *bs; 153 BlockLimits old_bl; 154 } BdrvRefreshLimitsState; 155 156 static void bdrv_refresh_limits_abort(void *opaque) 157 { 158 BdrvRefreshLimitsState *s = opaque; 159 160 s->bs->bl = s->old_bl; 161 } 162 163 static TransactionActionDrv bdrv_refresh_limits_drv = { 164 .abort = bdrv_refresh_limits_abort, 165 .clean = g_free, 166 }; 167 168 /* @tran is allowed to be NULL, in this case no rollback is possible. */ 169 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp) 170 { 171 ERRP_GUARD(); 172 BlockDriver *drv = bs->drv; 173 BdrvChild *c; 174 bool have_limits; 175 176 GLOBAL_STATE_CODE(); 177 178 if (tran) { 179 BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1); 180 *s = (BdrvRefreshLimitsState) { 181 .bs = bs, 182 .old_bl = bs->bl, 183 }; 184 tran_add(tran, &bdrv_refresh_limits_drv, s); 185 } 186 187 memset(&bs->bl, 0, sizeof(bs->bl)); 188 189 if (!drv) { 190 return; 191 } 192 193 /* Default alignment based on whether driver has byte interface */ 194 bs->bl.request_alignment = (drv->bdrv_co_preadv || 195 drv->bdrv_aio_preadv || 196 drv->bdrv_co_preadv_part) ? 1 : 512; 197 198 /* Take some limits from the children as a default */ 199 have_limits = false; 200 QLIST_FOREACH(c, &bs->children, next) { 201 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 202 { 203 bdrv_merge_limits(&bs->bl, &c->bs->bl); 204 have_limits = true; 205 } 206 207 if (c->role & BDRV_CHILD_FILTERED) { 208 bs->bl.has_variable_length |= c->bs->bl.has_variable_length; 209 } 210 } 211 212 if (!have_limits) { 213 bs->bl.min_mem_alignment = 512; 214 bs->bl.opt_mem_alignment = qemu_real_host_page_size(); 215 216 /* Safe default since most protocols use readv()/writev()/etc */ 217 bs->bl.max_iov = IOV_MAX; 218 } 219 220 /* Then let the driver override it */ 221 if (drv->bdrv_refresh_limits) { 222 drv->bdrv_refresh_limits(bs, errp); 223 if (*errp) { 224 return; 225 } 226 } 227 228 if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) { 229 error_setg(errp, "Driver requires too large request alignment"); 230 } 231 } 232 233 /** 234 * The copy-on-read flag is actually a reference count so multiple users may 235 * use the feature without worrying about clobbering its previous state. 236 * Copy-on-read stays enabled until all users have called to disable it. 237 */ 238 void bdrv_enable_copy_on_read(BlockDriverState *bs) 239 { 240 IO_CODE(); 241 qatomic_inc(&bs->copy_on_read); 242 } 243 244 void bdrv_disable_copy_on_read(BlockDriverState *bs) 245 { 246 int old = qatomic_fetch_dec(&bs->copy_on_read); 247 IO_CODE(); 248 assert(old >= 1); 249 } 250 251 typedef struct { 252 Coroutine *co; 253 BlockDriverState *bs; 254 bool done; 255 bool begin; 256 bool poll; 257 BdrvChild *parent; 258 } BdrvCoDrainData; 259 260 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 261 bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent, 262 bool ignore_bds_parents) 263 { 264 GLOBAL_STATE_CODE(); 265 266 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 267 return true; 268 } 269 270 if (qatomic_read(&bs->in_flight)) { 271 return true; 272 } 273 274 return false; 275 } 276 277 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, 278 BdrvChild *ignore_parent) 279 { 280 GLOBAL_STATE_CODE(); 281 GRAPH_RDLOCK_GUARD_MAINLOOP(); 282 283 return bdrv_drain_poll(bs, ignore_parent, false); 284 } 285 286 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent, 287 bool poll); 288 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent); 289 290 static void bdrv_co_drain_bh_cb(void *opaque) 291 { 292 BdrvCoDrainData *data = opaque; 293 Coroutine *co = data->co; 294 BlockDriverState *bs = data->bs; 295 296 if (bs) { 297 AioContext *ctx = bdrv_get_aio_context(bs); 298 aio_context_acquire(ctx); 299 bdrv_dec_in_flight(bs); 300 if (data->begin) { 301 bdrv_do_drained_begin(bs, data->parent, data->poll); 302 } else { 303 assert(!data->poll); 304 bdrv_do_drained_end(bs, data->parent); 305 } 306 aio_context_release(ctx); 307 } else { 308 assert(data->begin); 309 bdrv_drain_all_begin(); 310 } 311 312 data->done = true; 313 aio_co_wake(co); 314 } 315 316 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 317 bool begin, 318 BdrvChild *parent, 319 bool poll) 320 { 321 BdrvCoDrainData data; 322 Coroutine *self = qemu_coroutine_self(); 323 AioContext *ctx = bdrv_get_aio_context(bs); 324 AioContext *co_ctx = qemu_coroutine_get_aio_context(self); 325 326 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 327 * other coroutines run if they were queued by aio_co_enter(). */ 328 329 assert(qemu_in_coroutine()); 330 data = (BdrvCoDrainData) { 331 .co = self, 332 .bs = bs, 333 .done = false, 334 .begin = begin, 335 .parent = parent, 336 .poll = poll, 337 }; 338 339 if (bs) { 340 bdrv_inc_in_flight(bs); 341 } 342 343 /* 344 * Temporarily drop the lock across yield or we would get deadlocks. 345 * bdrv_co_drain_bh_cb() reaquires the lock as needed. 346 * 347 * When we yield below, the lock for the current context will be 348 * released, so if this is actually the lock that protects bs, don't drop 349 * it a second time. 350 */ 351 if (ctx != co_ctx) { 352 aio_context_release(ctx); 353 } 354 replay_bh_schedule_oneshot_event(qemu_get_aio_context(), 355 bdrv_co_drain_bh_cb, &data); 356 357 qemu_coroutine_yield(); 358 /* If we are resumed from some other event (such as an aio completion or a 359 * timer callback), it is a bug in the caller that should be fixed. */ 360 assert(data.done); 361 362 /* Reacquire the AioContext of bs if we dropped it */ 363 if (ctx != co_ctx) { 364 aio_context_acquire(ctx); 365 } 366 } 367 368 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent, 369 bool poll) 370 { 371 IO_OR_GS_CODE(); 372 373 if (qemu_in_coroutine()) { 374 bdrv_co_yield_to_drain(bs, true, parent, poll); 375 return; 376 } 377 378 GLOBAL_STATE_CODE(); 379 380 /* Stop things in parent-to-child order */ 381 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 382 GRAPH_RDLOCK_GUARD_MAINLOOP(); 383 bdrv_parent_drained_begin(bs, parent); 384 if (bs->drv && bs->drv->bdrv_drain_begin) { 385 bs->drv->bdrv_drain_begin(bs); 386 } 387 } 388 389 /* 390 * Wait for drained requests to finish. 391 * 392 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 393 * call is needed so things in this AioContext can make progress even 394 * though we don't return to the main AioContext loop - this automatically 395 * includes other nodes in the same AioContext and therefore all child 396 * nodes. 397 */ 398 if (poll) { 399 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent)); 400 } 401 } 402 403 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, BdrvChild *parent) 404 { 405 bdrv_do_drained_begin(bs, parent, false); 406 } 407 408 void coroutine_mixed_fn 409 bdrv_drained_begin(BlockDriverState *bs) 410 { 411 IO_OR_GS_CODE(); 412 bdrv_do_drained_begin(bs, NULL, true); 413 } 414 415 /** 416 * This function does not poll, nor must any of its recursively called 417 * functions. 418 */ 419 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent) 420 { 421 int old_quiesce_counter; 422 423 IO_OR_GS_CODE(); 424 425 if (qemu_in_coroutine()) { 426 bdrv_co_yield_to_drain(bs, false, parent, false); 427 return; 428 } 429 430 /* At this point, we should be always running in the main loop. */ 431 GLOBAL_STATE_CODE(); 432 assert(bs->quiesce_counter > 0); 433 GLOBAL_STATE_CODE(); 434 435 /* Re-enable things in child-to-parent order */ 436 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 437 if (old_quiesce_counter == 1) { 438 GRAPH_RDLOCK_GUARD_MAINLOOP(); 439 if (bs->drv && bs->drv->bdrv_drain_end) { 440 bs->drv->bdrv_drain_end(bs); 441 } 442 bdrv_parent_drained_end(bs, parent); 443 } 444 } 445 446 void bdrv_drained_end(BlockDriverState *bs) 447 { 448 IO_OR_GS_CODE(); 449 bdrv_do_drained_end(bs, NULL); 450 } 451 452 void bdrv_drain(BlockDriverState *bs) 453 { 454 IO_OR_GS_CODE(); 455 bdrv_drained_begin(bs); 456 bdrv_drained_end(bs); 457 } 458 459 static void bdrv_drain_assert_idle(BlockDriverState *bs) 460 { 461 BdrvChild *child, *next; 462 GLOBAL_STATE_CODE(); 463 GRAPH_RDLOCK_GUARD_MAINLOOP(); 464 465 assert(qatomic_read(&bs->in_flight) == 0); 466 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 467 bdrv_drain_assert_idle(child->bs); 468 } 469 } 470 471 unsigned int bdrv_drain_all_count = 0; 472 473 static bool bdrv_drain_all_poll(void) 474 { 475 BlockDriverState *bs = NULL; 476 bool result = false; 477 478 GLOBAL_STATE_CODE(); 479 GRAPH_RDLOCK_GUARD_MAINLOOP(); 480 481 /* bdrv_drain_poll() can't make changes to the graph and we are holding the 482 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 483 while ((bs = bdrv_next_all_states(bs))) { 484 AioContext *aio_context = bdrv_get_aio_context(bs); 485 aio_context_acquire(aio_context); 486 result |= bdrv_drain_poll(bs, NULL, true); 487 aio_context_release(aio_context); 488 } 489 490 return result; 491 } 492 493 /* 494 * Wait for pending requests to complete across all BlockDriverStates 495 * 496 * This function does not flush data to disk, use bdrv_flush_all() for that 497 * after calling this function. 498 * 499 * This pauses all block jobs and disables external clients. It must 500 * be paired with bdrv_drain_all_end(). 501 * 502 * NOTE: no new block jobs or BlockDriverStates can be created between 503 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 504 */ 505 void bdrv_drain_all_begin_nopoll(void) 506 { 507 BlockDriverState *bs = NULL; 508 GLOBAL_STATE_CODE(); 509 510 /* 511 * bdrv queue is managed by record/replay, 512 * waiting for finishing the I/O requests may 513 * be infinite 514 */ 515 if (replay_events_enabled()) { 516 return; 517 } 518 519 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 520 * loop AioContext, so make sure we're in the main context. */ 521 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 522 assert(bdrv_drain_all_count < INT_MAX); 523 bdrv_drain_all_count++; 524 525 /* Quiesce all nodes, without polling in-flight requests yet. The graph 526 * cannot change during this loop. */ 527 while ((bs = bdrv_next_all_states(bs))) { 528 AioContext *aio_context = bdrv_get_aio_context(bs); 529 530 aio_context_acquire(aio_context); 531 bdrv_do_drained_begin(bs, NULL, false); 532 aio_context_release(aio_context); 533 } 534 } 535 536 void coroutine_mixed_fn bdrv_drain_all_begin(void) 537 { 538 BlockDriverState *bs = NULL; 539 540 if (qemu_in_coroutine()) { 541 bdrv_co_yield_to_drain(NULL, true, NULL, true); 542 return; 543 } 544 545 /* 546 * bdrv queue is managed by record/replay, 547 * waiting for finishing the I/O requests may 548 * be infinite 549 */ 550 if (replay_events_enabled()) { 551 return; 552 } 553 554 bdrv_drain_all_begin_nopoll(); 555 556 /* Now poll the in-flight requests */ 557 AIO_WAIT_WHILE_UNLOCKED(NULL, bdrv_drain_all_poll()); 558 559 while ((bs = bdrv_next_all_states(bs))) { 560 bdrv_drain_assert_idle(bs); 561 } 562 } 563 564 void bdrv_drain_all_end_quiesce(BlockDriverState *bs) 565 { 566 GLOBAL_STATE_CODE(); 567 568 g_assert(bs->quiesce_counter > 0); 569 g_assert(!bs->refcnt); 570 571 while (bs->quiesce_counter) { 572 bdrv_do_drained_end(bs, NULL); 573 } 574 } 575 576 void bdrv_drain_all_end(void) 577 { 578 BlockDriverState *bs = NULL; 579 GLOBAL_STATE_CODE(); 580 581 /* 582 * bdrv queue is managed by record/replay, 583 * waiting for finishing the I/O requests may 584 * be endless 585 */ 586 if (replay_events_enabled()) { 587 return; 588 } 589 590 while ((bs = bdrv_next_all_states(bs))) { 591 AioContext *aio_context = bdrv_get_aio_context(bs); 592 593 aio_context_acquire(aio_context); 594 bdrv_do_drained_end(bs, NULL); 595 aio_context_release(aio_context); 596 } 597 598 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 599 assert(bdrv_drain_all_count > 0); 600 bdrv_drain_all_count--; 601 } 602 603 void bdrv_drain_all(void) 604 { 605 GLOBAL_STATE_CODE(); 606 bdrv_drain_all_begin(); 607 bdrv_drain_all_end(); 608 } 609 610 /** 611 * Remove an active request from the tracked requests list 612 * 613 * This function should be called when a tracked request is completing. 614 */ 615 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req) 616 { 617 if (req->serialising) { 618 qatomic_dec(&req->bs->serialising_in_flight); 619 } 620 621 qemu_mutex_lock(&req->bs->reqs_lock); 622 QLIST_REMOVE(req, list); 623 qemu_mutex_unlock(&req->bs->reqs_lock); 624 625 /* 626 * At this point qemu_co_queue_wait(&req->wait_queue, ...) won't be called 627 * anymore because the request has been removed from the list, so it's safe 628 * to restart the queue outside reqs_lock to minimize the critical section. 629 */ 630 qemu_co_queue_restart_all(&req->wait_queue); 631 } 632 633 /** 634 * Add an active request to the tracked requests list 635 */ 636 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req, 637 BlockDriverState *bs, 638 int64_t offset, 639 int64_t bytes, 640 enum BdrvTrackedRequestType type) 641 { 642 bdrv_check_request(offset, bytes, &error_abort); 643 644 *req = (BdrvTrackedRequest){ 645 .bs = bs, 646 .offset = offset, 647 .bytes = bytes, 648 .type = type, 649 .co = qemu_coroutine_self(), 650 .serialising = false, 651 .overlap_offset = offset, 652 .overlap_bytes = bytes, 653 }; 654 655 qemu_co_queue_init(&req->wait_queue); 656 657 qemu_mutex_lock(&bs->reqs_lock); 658 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 659 qemu_mutex_unlock(&bs->reqs_lock); 660 } 661 662 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 663 int64_t offset, int64_t bytes) 664 { 665 bdrv_check_request(offset, bytes, &error_abort); 666 667 /* aaaa bbbb */ 668 if (offset >= req->overlap_offset + req->overlap_bytes) { 669 return false; 670 } 671 /* bbbb aaaa */ 672 if (req->overlap_offset >= offset + bytes) { 673 return false; 674 } 675 return true; 676 } 677 678 /* Called with self->bs->reqs_lock held */ 679 static coroutine_fn BdrvTrackedRequest * 680 bdrv_find_conflicting_request(BdrvTrackedRequest *self) 681 { 682 BdrvTrackedRequest *req; 683 684 QLIST_FOREACH(req, &self->bs->tracked_requests, list) { 685 if (req == self || (!req->serialising && !self->serialising)) { 686 continue; 687 } 688 if (tracked_request_overlaps(req, self->overlap_offset, 689 self->overlap_bytes)) 690 { 691 /* 692 * Hitting this means there was a reentrant request, for 693 * example, a block driver issuing nested requests. This must 694 * never happen since it means deadlock. 695 */ 696 assert(qemu_coroutine_self() != req->co); 697 698 /* 699 * If the request is already (indirectly) waiting for us, or 700 * will wait for us as soon as it wakes up, then just go on 701 * (instead of producing a deadlock in the former case). 702 */ 703 if (!req->waiting_for) { 704 return req; 705 } 706 } 707 } 708 709 return NULL; 710 } 711 712 /* Called with self->bs->reqs_lock held */ 713 static void coroutine_fn 714 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self) 715 { 716 BdrvTrackedRequest *req; 717 718 while ((req = bdrv_find_conflicting_request(self))) { 719 self->waiting_for = req; 720 qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock); 721 self->waiting_for = NULL; 722 } 723 } 724 725 /* Called with req->bs->reqs_lock held */ 726 static void tracked_request_set_serialising(BdrvTrackedRequest *req, 727 uint64_t align) 728 { 729 int64_t overlap_offset = req->offset & ~(align - 1); 730 int64_t overlap_bytes = 731 ROUND_UP(req->offset + req->bytes, align) - overlap_offset; 732 733 bdrv_check_request(req->offset, req->bytes, &error_abort); 734 735 if (!req->serialising) { 736 qatomic_inc(&req->bs->serialising_in_flight); 737 req->serialising = true; 738 } 739 740 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 741 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 742 } 743 744 /** 745 * Return the tracked request on @bs for the current coroutine, or 746 * NULL if there is none. 747 */ 748 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 749 { 750 BdrvTrackedRequest *req; 751 Coroutine *self = qemu_coroutine_self(); 752 IO_CODE(); 753 754 QLIST_FOREACH(req, &bs->tracked_requests, list) { 755 if (req->co == self) { 756 return req; 757 } 758 } 759 760 return NULL; 761 } 762 763 /** 764 * Round a region to subcluster (if supported) or cluster boundaries 765 */ 766 void coroutine_fn GRAPH_RDLOCK 767 bdrv_round_to_subclusters(BlockDriverState *bs, int64_t offset, int64_t bytes, 768 int64_t *align_offset, int64_t *align_bytes) 769 { 770 BlockDriverInfo bdi; 771 IO_CODE(); 772 if (bdrv_co_get_info(bs, &bdi) < 0 || bdi.subcluster_size == 0) { 773 *align_offset = offset; 774 *align_bytes = bytes; 775 } else { 776 int64_t c = bdi.subcluster_size; 777 *align_offset = QEMU_ALIGN_DOWN(offset, c); 778 *align_bytes = QEMU_ALIGN_UP(offset - *align_offset + bytes, c); 779 } 780 } 781 782 static int coroutine_fn GRAPH_RDLOCK bdrv_get_cluster_size(BlockDriverState *bs) 783 { 784 BlockDriverInfo bdi; 785 int ret; 786 787 ret = bdrv_co_get_info(bs, &bdi); 788 if (ret < 0 || bdi.cluster_size == 0) { 789 return bs->bl.request_alignment; 790 } else { 791 return bdi.cluster_size; 792 } 793 } 794 795 void bdrv_inc_in_flight(BlockDriverState *bs) 796 { 797 IO_CODE(); 798 qatomic_inc(&bs->in_flight); 799 } 800 801 void bdrv_wakeup(BlockDriverState *bs) 802 { 803 IO_CODE(); 804 aio_wait_kick(); 805 } 806 807 void bdrv_dec_in_flight(BlockDriverState *bs) 808 { 809 IO_CODE(); 810 qatomic_dec(&bs->in_flight); 811 bdrv_wakeup(bs); 812 } 813 814 static void coroutine_fn 815 bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 816 { 817 BlockDriverState *bs = self->bs; 818 819 if (!qatomic_read(&bs->serialising_in_flight)) { 820 return; 821 } 822 823 qemu_mutex_lock(&bs->reqs_lock); 824 bdrv_wait_serialising_requests_locked(self); 825 qemu_mutex_unlock(&bs->reqs_lock); 826 } 827 828 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req, 829 uint64_t align) 830 { 831 IO_CODE(); 832 833 qemu_mutex_lock(&req->bs->reqs_lock); 834 835 tracked_request_set_serialising(req, align); 836 bdrv_wait_serialising_requests_locked(req); 837 838 qemu_mutex_unlock(&req->bs->reqs_lock); 839 } 840 841 int bdrv_check_qiov_request(int64_t offset, int64_t bytes, 842 QEMUIOVector *qiov, size_t qiov_offset, 843 Error **errp) 844 { 845 /* 846 * Check generic offset/bytes correctness 847 */ 848 849 if (offset < 0) { 850 error_setg(errp, "offset is negative: %" PRIi64, offset); 851 return -EIO; 852 } 853 854 if (bytes < 0) { 855 error_setg(errp, "bytes is negative: %" PRIi64, bytes); 856 return -EIO; 857 } 858 859 if (bytes > BDRV_MAX_LENGTH) { 860 error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 861 bytes, BDRV_MAX_LENGTH); 862 return -EIO; 863 } 864 865 if (offset > BDRV_MAX_LENGTH) { 866 error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 867 offset, BDRV_MAX_LENGTH); 868 return -EIO; 869 } 870 871 if (offset > BDRV_MAX_LENGTH - bytes) { 872 error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") " 873 "exceeds maximum(%" PRIi64 ")", offset, bytes, 874 BDRV_MAX_LENGTH); 875 return -EIO; 876 } 877 878 if (!qiov) { 879 return 0; 880 } 881 882 /* 883 * Check qiov and qiov_offset 884 */ 885 886 if (qiov_offset > qiov->size) { 887 error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)", 888 qiov_offset, qiov->size); 889 return -EIO; 890 } 891 892 if (bytes > qiov->size - qiov_offset) { 893 error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io " 894 "vector size(%zu)", bytes, qiov_offset, qiov->size); 895 return -EIO; 896 } 897 898 return 0; 899 } 900 901 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp) 902 { 903 return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp); 904 } 905 906 static int bdrv_check_request32(int64_t offset, int64_t bytes, 907 QEMUIOVector *qiov, size_t qiov_offset) 908 { 909 int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 910 if (ret < 0) { 911 return ret; 912 } 913 914 if (bytes > BDRV_REQUEST_MAX_BYTES) { 915 return -EIO; 916 } 917 918 return 0; 919 } 920 921 /* 922 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 923 * The operation is sped up by checking the block status and only writing 924 * zeroes to the device if they currently do not return zeroes. Optional 925 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 926 * BDRV_REQ_FUA). 927 * 928 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 929 */ 930 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 931 { 932 int ret; 933 int64_t target_size, bytes, offset = 0; 934 BlockDriverState *bs = child->bs; 935 IO_CODE(); 936 937 target_size = bdrv_getlength(bs); 938 if (target_size < 0) { 939 return target_size; 940 } 941 942 for (;;) { 943 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 944 if (bytes <= 0) { 945 return 0; 946 } 947 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 948 if (ret < 0) { 949 return ret; 950 } 951 if (ret & BDRV_BLOCK_ZERO) { 952 offset += bytes; 953 continue; 954 } 955 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 956 if (ret < 0) { 957 return ret; 958 } 959 offset += bytes; 960 } 961 } 962 963 /* 964 * Writes to the file and ensures that no writes are reordered across this 965 * request (acts as a barrier) 966 * 967 * Returns 0 on success, -errno in error cases. 968 */ 969 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset, 970 int64_t bytes, const void *buf, 971 BdrvRequestFlags flags) 972 { 973 int ret; 974 IO_CODE(); 975 assert_bdrv_graph_readable(); 976 977 ret = bdrv_co_pwrite(child, offset, bytes, buf, flags); 978 if (ret < 0) { 979 return ret; 980 } 981 982 ret = bdrv_co_flush(child->bs); 983 if (ret < 0) { 984 return ret; 985 } 986 987 return 0; 988 } 989 990 typedef struct CoroutineIOCompletion { 991 Coroutine *coroutine; 992 int ret; 993 } CoroutineIOCompletion; 994 995 static void bdrv_co_io_em_complete(void *opaque, int ret) 996 { 997 CoroutineIOCompletion *co = opaque; 998 999 co->ret = ret; 1000 aio_co_wake(co->coroutine); 1001 } 1002 1003 static int coroutine_fn GRAPH_RDLOCK 1004 bdrv_driver_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes, 1005 QEMUIOVector *qiov, size_t qiov_offset, int flags) 1006 { 1007 BlockDriver *drv = bs->drv; 1008 int64_t sector_num; 1009 unsigned int nb_sectors; 1010 QEMUIOVector local_qiov; 1011 int ret; 1012 assert_bdrv_graph_readable(); 1013 1014 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1015 assert(!(flags & ~bs->supported_read_flags)); 1016 1017 if (!drv) { 1018 return -ENOMEDIUM; 1019 } 1020 1021 if (drv->bdrv_co_preadv_part) { 1022 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 1023 flags); 1024 } 1025 1026 if (qiov_offset > 0 || bytes != qiov->size) { 1027 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1028 qiov = &local_qiov; 1029 } 1030 1031 if (drv->bdrv_co_preadv) { 1032 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 1033 goto out; 1034 } 1035 1036 if (drv->bdrv_aio_preadv) { 1037 BlockAIOCB *acb; 1038 CoroutineIOCompletion co = { 1039 .coroutine = qemu_coroutine_self(), 1040 }; 1041 1042 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 1043 bdrv_co_io_em_complete, &co); 1044 if (acb == NULL) { 1045 ret = -EIO; 1046 goto out; 1047 } else { 1048 qemu_coroutine_yield(); 1049 ret = co.ret; 1050 goto out; 1051 } 1052 } 1053 1054 sector_num = offset >> BDRV_SECTOR_BITS; 1055 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1056 1057 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1058 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1059 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1060 assert(drv->bdrv_co_readv); 1061 1062 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1063 1064 out: 1065 if (qiov == &local_qiov) { 1066 qemu_iovec_destroy(&local_qiov); 1067 } 1068 1069 return ret; 1070 } 1071 1072 static int coroutine_fn GRAPH_RDLOCK 1073 bdrv_driver_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes, 1074 QEMUIOVector *qiov, size_t qiov_offset, 1075 BdrvRequestFlags flags) 1076 { 1077 BlockDriver *drv = bs->drv; 1078 bool emulate_fua = false; 1079 int64_t sector_num; 1080 unsigned int nb_sectors; 1081 QEMUIOVector local_qiov; 1082 int ret; 1083 assert_bdrv_graph_readable(); 1084 1085 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1086 1087 if (!drv) { 1088 return -ENOMEDIUM; 1089 } 1090 1091 if ((flags & BDRV_REQ_FUA) && 1092 (~bs->supported_write_flags & BDRV_REQ_FUA)) { 1093 flags &= ~BDRV_REQ_FUA; 1094 emulate_fua = true; 1095 } 1096 1097 flags &= bs->supported_write_flags; 1098 1099 if (drv->bdrv_co_pwritev_part) { 1100 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1101 flags); 1102 goto emulate_flags; 1103 } 1104 1105 if (qiov_offset > 0 || bytes != qiov->size) { 1106 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1107 qiov = &local_qiov; 1108 } 1109 1110 if (drv->bdrv_co_pwritev) { 1111 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags); 1112 goto emulate_flags; 1113 } 1114 1115 if (drv->bdrv_aio_pwritev) { 1116 BlockAIOCB *acb; 1117 CoroutineIOCompletion co = { 1118 .coroutine = qemu_coroutine_self(), 1119 }; 1120 1121 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags, 1122 bdrv_co_io_em_complete, &co); 1123 if (acb == NULL) { 1124 ret = -EIO; 1125 } else { 1126 qemu_coroutine_yield(); 1127 ret = co.ret; 1128 } 1129 goto emulate_flags; 1130 } 1131 1132 sector_num = offset >> BDRV_SECTOR_BITS; 1133 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1134 1135 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1136 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1137 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1138 1139 assert(drv->bdrv_co_writev); 1140 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags); 1141 1142 emulate_flags: 1143 if (ret == 0 && emulate_fua) { 1144 ret = bdrv_co_flush(bs); 1145 } 1146 1147 if (qiov == &local_qiov) { 1148 qemu_iovec_destroy(&local_qiov); 1149 } 1150 1151 return ret; 1152 } 1153 1154 static int coroutine_fn GRAPH_RDLOCK 1155 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset, 1156 int64_t bytes, QEMUIOVector *qiov, 1157 size_t qiov_offset) 1158 { 1159 BlockDriver *drv = bs->drv; 1160 QEMUIOVector local_qiov; 1161 int ret; 1162 assert_bdrv_graph_readable(); 1163 1164 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1165 1166 if (!drv) { 1167 return -ENOMEDIUM; 1168 } 1169 1170 if (!block_driver_can_compress(drv)) { 1171 return -ENOTSUP; 1172 } 1173 1174 if (drv->bdrv_co_pwritev_compressed_part) { 1175 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1176 qiov, qiov_offset); 1177 } 1178 1179 if (qiov_offset == 0) { 1180 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 1181 } 1182 1183 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1184 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1185 qemu_iovec_destroy(&local_qiov); 1186 1187 return ret; 1188 } 1189 1190 static int coroutine_fn GRAPH_RDLOCK 1191 bdrv_co_do_copy_on_readv(BdrvChild *child, int64_t offset, int64_t bytes, 1192 QEMUIOVector *qiov, size_t qiov_offset, int flags) 1193 { 1194 BlockDriverState *bs = child->bs; 1195 1196 /* Perform I/O through a temporary buffer so that users who scribble over 1197 * their read buffer while the operation is in progress do not end up 1198 * modifying the image file. This is critical for zero-copy guest I/O 1199 * where anything might happen inside guest memory. 1200 */ 1201 void *bounce_buffer = NULL; 1202 1203 BlockDriver *drv = bs->drv; 1204 int64_t align_offset; 1205 int64_t align_bytes; 1206 int64_t skip_bytes; 1207 int ret; 1208 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1209 BDRV_REQUEST_MAX_BYTES); 1210 int64_t progress = 0; 1211 bool skip_write; 1212 1213 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1214 1215 if (!drv) { 1216 return -ENOMEDIUM; 1217 } 1218 1219 /* 1220 * Do not write anything when the BDS is inactive. That is not 1221 * allowed, and it would not help. 1222 */ 1223 skip_write = (bs->open_flags & BDRV_O_INACTIVE); 1224 1225 /* FIXME We cannot require callers to have write permissions when all they 1226 * are doing is a read request. If we did things right, write permissions 1227 * would be obtained anyway, but internally by the copy-on-read code. As 1228 * long as it is implemented here rather than in a separate filter driver, 1229 * the copy-on-read code doesn't have its own BdrvChild, however, for which 1230 * it could request permissions. Therefore we have to bypass the permission 1231 * system for the moment. */ 1232 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1233 1234 /* Cover entire cluster so no additional backing file I/O is required when 1235 * allocating cluster in the image file. Note that this value may exceed 1236 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1237 * is one reason we loop rather than doing it all at once. 1238 */ 1239 bdrv_round_to_subclusters(bs, offset, bytes, &align_offset, &align_bytes); 1240 skip_bytes = offset - align_offset; 1241 1242 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1243 align_offset, align_bytes); 1244 1245 while (align_bytes) { 1246 int64_t pnum; 1247 1248 if (skip_write) { 1249 ret = 1; /* "already allocated", so nothing will be copied */ 1250 pnum = MIN(align_bytes, max_transfer); 1251 } else { 1252 ret = bdrv_co_is_allocated(bs, align_offset, 1253 MIN(align_bytes, max_transfer), &pnum); 1254 if (ret < 0) { 1255 /* 1256 * Safe to treat errors in querying allocation as if 1257 * unallocated; we'll probably fail again soon on the 1258 * read, but at least that will set a decent errno. 1259 */ 1260 pnum = MIN(align_bytes, max_transfer); 1261 } 1262 1263 /* Stop at EOF if the image ends in the middle of the cluster */ 1264 if (ret == 0 && pnum == 0) { 1265 assert(progress >= bytes); 1266 break; 1267 } 1268 1269 assert(skip_bytes < pnum); 1270 } 1271 1272 if (ret <= 0) { 1273 QEMUIOVector local_qiov; 1274 1275 /* Must copy-on-read; use the bounce buffer */ 1276 pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1277 if (!bounce_buffer) { 1278 int64_t max_we_need = MAX(pnum, align_bytes - pnum); 1279 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 1280 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 1281 1282 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 1283 if (!bounce_buffer) { 1284 ret = -ENOMEM; 1285 goto err; 1286 } 1287 } 1288 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1289 1290 ret = bdrv_driver_preadv(bs, align_offset, pnum, 1291 &local_qiov, 0, 0); 1292 if (ret < 0) { 1293 goto err; 1294 } 1295 1296 bdrv_co_debug_event(bs, BLKDBG_COR_WRITE); 1297 if (drv->bdrv_co_pwrite_zeroes && 1298 buffer_is_zero(bounce_buffer, pnum)) { 1299 /* FIXME: Should we (perhaps conditionally) be setting 1300 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1301 * that still correctly reads as zero? */ 1302 ret = bdrv_co_do_pwrite_zeroes(bs, align_offset, pnum, 1303 BDRV_REQ_WRITE_UNCHANGED); 1304 } else { 1305 /* This does not change the data on the disk, it is not 1306 * necessary to flush even in cache=writethrough mode. 1307 */ 1308 ret = bdrv_driver_pwritev(bs, align_offset, pnum, 1309 &local_qiov, 0, 1310 BDRV_REQ_WRITE_UNCHANGED); 1311 } 1312 1313 if (ret < 0) { 1314 /* It might be okay to ignore write errors for guest 1315 * requests. If this is a deliberate copy-on-read 1316 * then we don't want to ignore the error. Simply 1317 * report it in all cases. 1318 */ 1319 goto err; 1320 } 1321 1322 if (!(flags & BDRV_REQ_PREFETCH)) { 1323 qemu_iovec_from_buf(qiov, qiov_offset + progress, 1324 bounce_buffer + skip_bytes, 1325 MIN(pnum - skip_bytes, bytes - progress)); 1326 } 1327 } else if (!(flags & BDRV_REQ_PREFETCH)) { 1328 /* Read directly into the destination */ 1329 ret = bdrv_driver_preadv(bs, offset + progress, 1330 MIN(pnum - skip_bytes, bytes - progress), 1331 qiov, qiov_offset + progress, 0); 1332 if (ret < 0) { 1333 goto err; 1334 } 1335 } 1336 1337 align_offset += pnum; 1338 align_bytes -= pnum; 1339 progress += pnum - skip_bytes; 1340 skip_bytes = 0; 1341 } 1342 ret = 0; 1343 1344 err: 1345 qemu_vfree(bounce_buffer); 1346 return ret; 1347 } 1348 1349 /* 1350 * Forwards an already correctly aligned request to the BlockDriver. This 1351 * handles copy on read, zeroing after EOF, and fragmentation of large 1352 * reads; any other features must be implemented by the caller. 1353 */ 1354 static int coroutine_fn GRAPH_RDLOCK 1355 bdrv_aligned_preadv(BdrvChild *child, BdrvTrackedRequest *req, 1356 int64_t offset, int64_t bytes, int64_t align, 1357 QEMUIOVector *qiov, size_t qiov_offset, int flags) 1358 { 1359 BlockDriverState *bs = child->bs; 1360 int64_t total_bytes, max_bytes; 1361 int ret = 0; 1362 int64_t bytes_remaining = bytes; 1363 int max_transfer; 1364 1365 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1366 assert(is_power_of_2(align)); 1367 assert((offset & (align - 1)) == 0); 1368 assert((bytes & (align - 1)) == 0); 1369 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1370 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1371 align); 1372 1373 /* 1374 * TODO: We would need a per-BDS .supported_read_flags and 1375 * potential fallback support, if we ever implement any read flags 1376 * to pass through to drivers. For now, there aren't any 1377 * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint. 1378 */ 1379 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH | 1380 BDRV_REQ_REGISTERED_BUF))); 1381 1382 /* Handle Copy on Read and associated serialisation */ 1383 if (flags & BDRV_REQ_COPY_ON_READ) { 1384 /* If we touch the same cluster it counts as an overlap. This 1385 * guarantees that allocating writes will be serialized and not race 1386 * with each other for the same cluster. For example, in copy-on-read 1387 * it ensures that the CoR read and write operations are atomic and 1388 * guest writes cannot interleave between them. */ 1389 bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs)); 1390 } else { 1391 bdrv_wait_serialising_requests(req); 1392 } 1393 1394 if (flags & BDRV_REQ_COPY_ON_READ) { 1395 int64_t pnum; 1396 1397 /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */ 1398 flags &= ~BDRV_REQ_COPY_ON_READ; 1399 1400 ret = bdrv_co_is_allocated(bs, offset, bytes, &pnum); 1401 if (ret < 0) { 1402 goto out; 1403 } 1404 1405 if (!ret || pnum != bytes) { 1406 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 1407 qiov, qiov_offset, flags); 1408 goto out; 1409 } else if (flags & BDRV_REQ_PREFETCH) { 1410 goto out; 1411 } 1412 } 1413 1414 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1415 total_bytes = bdrv_co_getlength(bs); 1416 if (total_bytes < 0) { 1417 ret = total_bytes; 1418 goto out; 1419 } 1420 1421 assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF))); 1422 1423 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1424 if (bytes <= max_bytes && bytes <= max_transfer) { 1425 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags); 1426 goto out; 1427 } 1428 1429 while (bytes_remaining) { 1430 int64_t num; 1431 1432 if (max_bytes) { 1433 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1434 assert(num); 1435 1436 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1437 num, qiov, 1438 qiov_offset + bytes - bytes_remaining, 1439 flags); 1440 max_bytes -= num; 1441 } else { 1442 num = bytes_remaining; 1443 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1444 0, bytes_remaining); 1445 } 1446 if (ret < 0) { 1447 goto out; 1448 } 1449 bytes_remaining -= num; 1450 } 1451 1452 out: 1453 return ret < 0 ? ret : 0; 1454 } 1455 1456 /* 1457 * Request padding 1458 * 1459 * |<---- align ----->| |<----- align ---->| 1460 * |<- head ->|<------------- bytes ------------->|<-- tail -->| 1461 * | | | | | | 1462 * -*----------$-------*-------- ... --------*-----$------------*--- 1463 * | | | | | | 1464 * | offset | | end | 1465 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 1466 * [buf ... ) [tail_buf ) 1467 * 1468 * @buf is an aligned allocation needed to store @head and @tail paddings. @head 1469 * is placed at the beginning of @buf and @tail at the @end. 1470 * 1471 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 1472 * around tail, if tail exists. 1473 * 1474 * @merge_reads is true for small requests, 1475 * if @buf_len == @head + bytes + @tail. In this case it is possible that both 1476 * head and tail exist but @buf_len == align and @tail_buf == @buf. 1477 * 1478 * @write is true for write requests, false for read requests. 1479 * 1480 * If padding makes the vector too long (exceeding IOV_MAX), then we need to 1481 * merge existing vector elements into a single one. @collapse_bounce_buf acts 1482 * as the bounce buffer in such cases. @pre_collapse_qiov has the pre-collapse 1483 * I/O vector elements so for read requests, the data can be copied back after 1484 * the read is done. 1485 */ 1486 typedef struct BdrvRequestPadding { 1487 uint8_t *buf; 1488 size_t buf_len; 1489 uint8_t *tail_buf; 1490 size_t head; 1491 size_t tail; 1492 bool merge_reads; 1493 bool write; 1494 QEMUIOVector local_qiov; 1495 1496 uint8_t *collapse_bounce_buf; 1497 size_t collapse_len; 1498 QEMUIOVector pre_collapse_qiov; 1499 } BdrvRequestPadding; 1500 1501 static bool bdrv_init_padding(BlockDriverState *bs, 1502 int64_t offset, int64_t bytes, 1503 bool write, 1504 BdrvRequestPadding *pad) 1505 { 1506 int64_t align = bs->bl.request_alignment; 1507 int64_t sum; 1508 1509 bdrv_check_request(offset, bytes, &error_abort); 1510 assert(align <= INT_MAX); /* documented in block/block_int.h */ 1511 assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */ 1512 1513 memset(pad, 0, sizeof(*pad)); 1514 1515 pad->head = offset & (align - 1); 1516 pad->tail = ((offset + bytes) & (align - 1)); 1517 if (pad->tail) { 1518 pad->tail = align - pad->tail; 1519 } 1520 1521 if (!pad->head && !pad->tail) { 1522 return false; 1523 } 1524 1525 assert(bytes); /* Nothing good in aligning zero-length requests */ 1526 1527 sum = pad->head + bytes + pad->tail; 1528 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 1529 pad->buf = qemu_blockalign(bs, pad->buf_len); 1530 pad->merge_reads = sum == pad->buf_len; 1531 if (pad->tail) { 1532 pad->tail_buf = pad->buf + pad->buf_len - align; 1533 } 1534 1535 pad->write = write; 1536 1537 return true; 1538 } 1539 1540 static int coroutine_fn GRAPH_RDLOCK 1541 bdrv_padding_rmw_read(BdrvChild *child, BdrvTrackedRequest *req, 1542 BdrvRequestPadding *pad, bool zero_middle) 1543 { 1544 QEMUIOVector local_qiov; 1545 BlockDriverState *bs = child->bs; 1546 uint64_t align = bs->bl.request_alignment; 1547 int ret; 1548 1549 assert(req->serialising && pad->buf); 1550 1551 if (pad->head || pad->merge_reads) { 1552 int64_t bytes = pad->merge_reads ? pad->buf_len : align; 1553 1554 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 1555 1556 if (pad->head) { 1557 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1558 } 1559 if (pad->merge_reads && pad->tail) { 1560 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1561 } 1562 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 1563 align, &local_qiov, 0, 0); 1564 if (ret < 0) { 1565 return ret; 1566 } 1567 if (pad->head) { 1568 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1569 } 1570 if (pad->merge_reads && pad->tail) { 1571 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1572 } 1573 1574 if (pad->merge_reads) { 1575 goto zero_mem; 1576 } 1577 } 1578 1579 if (pad->tail) { 1580 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 1581 1582 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1583 ret = bdrv_aligned_preadv( 1584 child, req, 1585 req->overlap_offset + req->overlap_bytes - align, 1586 align, align, &local_qiov, 0, 0); 1587 if (ret < 0) { 1588 return ret; 1589 } 1590 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1591 } 1592 1593 zero_mem: 1594 if (zero_middle) { 1595 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 1596 } 1597 1598 return 0; 1599 } 1600 1601 /** 1602 * Free *pad's associated buffers, and perform any necessary finalization steps. 1603 */ 1604 static void bdrv_padding_finalize(BdrvRequestPadding *pad) 1605 { 1606 if (pad->collapse_bounce_buf) { 1607 if (!pad->write) { 1608 /* 1609 * If padding required elements in the vector to be collapsed into a 1610 * bounce buffer, copy the bounce buffer content back 1611 */ 1612 qemu_iovec_from_buf(&pad->pre_collapse_qiov, 0, 1613 pad->collapse_bounce_buf, pad->collapse_len); 1614 } 1615 qemu_vfree(pad->collapse_bounce_buf); 1616 qemu_iovec_destroy(&pad->pre_collapse_qiov); 1617 } 1618 if (pad->buf) { 1619 qemu_vfree(pad->buf); 1620 qemu_iovec_destroy(&pad->local_qiov); 1621 } 1622 memset(pad, 0, sizeof(*pad)); 1623 } 1624 1625 /* 1626 * Create pad->local_qiov by wrapping @iov in the padding head and tail, while 1627 * ensuring that the resulting vector will not exceed IOV_MAX elements. 1628 * 1629 * To ensure this, when necessary, the first two or three elements of @iov are 1630 * merged into pad->collapse_bounce_buf and replaced by a reference to that 1631 * bounce buffer in pad->local_qiov. 1632 * 1633 * After performing a read request, the data from the bounce buffer must be 1634 * copied back into pad->pre_collapse_qiov (e.g. by bdrv_padding_finalize()). 1635 */ 1636 static int bdrv_create_padded_qiov(BlockDriverState *bs, 1637 BdrvRequestPadding *pad, 1638 struct iovec *iov, int niov, 1639 size_t iov_offset, size_t bytes) 1640 { 1641 int padded_niov, surplus_count, collapse_count; 1642 1643 /* Assert this invariant */ 1644 assert(niov <= IOV_MAX); 1645 1646 /* 1647 * Cannot pad if resulting length would exceed SIZE_MAX. Returning an error 1648 * to the guest is not ideal, but there is little else we can do. At least 1649 * this will practically never happen on 64-bit systems. 1650 */ 1651 if (SIZE_MAX - pad->head < bytes || 1652 SIZE_MAX - pad->head - bytes < pad->tail) 1653 { 1654 return -EINVAL; 1655 } 1656 1657 /* Length of the resulting IOV if we just concatenated everything */ 1658 padded_niov = !!pad->head + niov + !!pad->tail; 1659 1660 qemu_iovec_init(&pad->local_qiov, MIN(padded_niov, IOV_MAX)); 1661 1662 if (pad->head) { 1663 qemu_iovec_add(&pad->local_qiov, pad->buf, pad->head); 1664 } 1665 1666 /* 1667 * If padded_niov > IOV_MAX, we cannot just concatenate everything. 1668 * Instead, merge the first two or three elements of @iov to reduce the 1669 * number of vector elements as necessary. 1670 */ 1671 if (padded_niov > IOV_MAX) { 1672 /* 1673 * Only head and tail can have lead to the number of entries exceeding 1674 * IOV_MAX, so we can exceed it by the head and tail at most. We need 1675 * to reduce the number of elements by `surplus_count`, so we merge that 1676 * many elements plus one into one element. 1677 */ 1678 surplus_count = padded_niov - IOV_MAX; 1679 assert(surplus_count <= !!pad->head + !!pad->tail); 1680 collapse_count = surplus_count + 1; 1681 1682 /* 1683 * Move the elements to collapse into `pad->pre_collapse_qiov`, then 1684 * advance `iov` (and associated variables) by those elements. 1685 */ 1686 qemu_iovec_init(&pad->pre_collapse_qiov, collapse_count); 1687 qemu_iovec_concat_iov(&pad->pre_collapse_qiov, iov, 1688 collapse_count, iov_offset, SIZE_MAX); 1689 iov += collapse_count; 1690 iov_offset = 0; 1691 niov -= collapse_count; 1692 bytes -= pad->pre_collapse_qiov.size; 1693 1694 /* 1695 * Construct the bounce buffer to match the length of the to-collapse 1696 * vector elements, and for write requests, initialize it with the data 1697 * from those elements. Then add it to `pad->local_qiov`. 1698 */ 1699 pad->collapse_len = pad->pre_collapse_qiov.size; 1700 pad->collapse_bounce_buf = qemu_blockalign(bs, pad->collapse_len); 1701 if (pad->write) { 1702 qemu_iovec_to_buf(&pad->pre_collapse_qiov, 0, 1703 pad->collapse_bounce_buf, pad->collapse_len); 1704 } 1705 qemu_iovec_add(&pad->local_qiov, 1706 pad->collapse_bounce_buf, pad->collapse_len); 1707 } 1708 1709 qemu_iovec_concat_iov(&pad->local_qiov, iov, niov, iov_offset, bytes); 1710 1711 if (pad->tail) { 1712 qemu_iovec_add(&pad->local_qiov, 1713 pad->buf + pad->buf_len - pad->tail, pad->tail); 1714 } 1715 1716 assert(pad->local_qiov.niov == MIN(padded_niov, IOV_MAX)); 1717 return 0; 1718 } 1719 1720 /* 1721 * bdrv_pad_request 1722 * 1723 * Exchange request parameters with padded request if needed. Don't include RMW 1724 * read of padding, bdrv_padding_rmw_read() should be called separately if 1725 * needed. 1726 * 1727 * @write is true for write requests, false for read requests. 1728 * 1729 * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out: 1730 * - on function start they represent original request 1731 * - on failure or when padding is not needed they are unchanged 1732 * - on success when padding is needed they represent padded request 1733 */ 1734 static int bdrv_pad_request(BlockDriverState *bs, 1735 QEMUIOVector **qiov, size_t *qiov_offset, 1736 int64_t *offset, int64_t *bytes, 1737 bool write, 1738 BdrvRequestPadding *pad, bool *padded, 1739 BdrvRequestFlags *flags) 1740 { 1741 int ret; 1742 struct iovec *sliced_iov; 1743 int sliced_niov; 1744 size_t sliced_head, sliced_tail; 1745 1746 /* Should have been checked by the caller already */ 1747 ret = bdrv_check_request32(*offset, *bytes, *qiov, *qiov_offset); 1748 if (ret < 0) { 1749 return ret; 1750 } 1751 1752 if (!bdrv_init_padding(bs, *offset, *bytes, write, pad)) { 1753 if (padded) { 1754 *padded = false; 1755 } 1756 return 0; 1757 } 1758 1759 sliced_iov = qemu_iovec_slice(*qiov, *qiov_offset, *bytes, 1760 &sliced_head, &sliced_tail, 1761 &sliced_niov); 1762 1763 /* Guaranteed by bdrv_check_request32() */ 1764 assert(*bytes <= SIZE_MAX); 1765 ret = bdrv_create_padded_qiov(bs, pad, sliced_iov, sliced_niov, 1766 sliced_head, *bytes); 1767 if (ret < 0) { 1768 bdrv_padding_finalize(pad); 1769 return ret; 1770 } 1771 *bytes += pad->head + pad->tail; 1772 *offset -= pad->head; 1773 *qiov = &pad->local_qiov; 1774 *qiov_offset = 0; 1775 if (padded) { 1776 *padded = true; 1777 } 1778 if (flags) { 1779 /* Can't use optimization hint with bounce buffer */ 1780 *flags &= ~BDRV_REQ_REGISTERED_BUF; 1781 } 1782 1783 return 0; 1784 } 1785 1786 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1787 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 1788 BdrvRequestFlags flags) 1789 { 1790 IO_CODE(); 1791 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 1792 } 1793 1794 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 1795 int64_t offset, int64_t bytes, 1796 QEMUIOVector *qiov, size_t qiov_offset, 1797 BdrvRequestFlags flags) 1798 { 1799 BlockDriverState *bs = child->bs; 1800 BdrvTrackedRequest req; 1801 BdrvRequestPadding pad; 1802 int ret; 1803 IO_CODE(); 1804 1805 trace_bdrv_co_preadv_part(bs, offset, bytes, flags); 1806 1807 if (!bdrv_co_is_inserted(bs)) { 1808 return -ENOMEDIUM; 1809 } 1810 1811 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 1812 if (ret < 0) { 1813 return ret; 1814 } 1815 1816 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1817 /* 1818 * Aligning zero request is nonsense. Even if driver has special meaning 1819 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1820 * it to driver due to request_alignment. 1821 * 1822 * Still, no reason to return an error if someone do unaligned 1823 * zero-length read occasionally. 1824 */ 1825 return 0; 1826 } 1827 1828 bdrv_inc_in_flight(bs); 1829 1830 /* Don't do copy-on-read if we read data before write operation */ 1831 if (qatomic_read(&bs->copy_on_read)) { 1832 flags |= BDRV_REQ_COPY_ON_READ; 1833 } 1834 1835 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, false, 1836 &pad, NULL, &flags); 1837 if (ret < 0) { 1838 goto fail; 1839 } 1840 1841 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1842 ret = bdrv_aligned_preadv(child, &req, offset, bytes, 1843 bs->bl.request_alignment, 1844 qiov, qiov_offset, flags); 1845 tracked_request_end(&req); 1846 bdrv_padding_finalize(&pad); 1847 1848 fail: 1849 bdrv_dec_in_flight(bs); 1850 1851 return ret; 1852 } 1853 1854 static int coroutine_fn GRAPH_RDLOCK 1855 bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int64_t bytes, 1856 BdrvRequestFlags flags) 1857 { 1858 BlockDriver *drv = bs->drv; 1859 QEMUIOVector qiov; 1860 void *buf = NULL; 1861 int ret = 0; 1862 bool need_flush = false; 1863 int head = 0; 1864 int tail = 0; 1865 1866 int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, 1867 INT64_MAX); 1868 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1869 bs->bl.request_alignment); 1870 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1871 1872 assert_bdrv_graph_readable(); 1873 bdrv_check_request(offset, bytes, &error_abort); 1874 1875 if (!drv) { 1876 return -ENOMEDIUM; 1877 } 1878 1879 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1880 return -ENOTSUP; 1881 } 1882 1883 /* By definition there is no user buffer so this flag doesn't make sense */ 1884 if (flags & BDRV_REQ_REGISTERED_BUF) { 1885 return -EINVAL; 1886 } 1887 1888 /* Invalidate the cached block-status data range if this write overlaps */ 1889 bdrv_bsc_invalidate_range(bs, offset, bytes); 1890 1891 assert(alignment % bs->bl.request_alignment == 0); 1892 head = offset % alignment; 1893 tail = (offset + bytes) % alignment; 1894 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1895 assert(max_write_zeroes >= bs->bl.request_alignment); 1896 1897 while (bytes > 0 && !ret) { 1898 int64_t num = bytes; 1899 1900 /* Align request. Block drivers can expect the "bulk" of the request 1901 * to be aligned, and that unaligned requests do not cross cluster 1902 * boundaries. 1903 */ 1904 if (head) { 1905 /* Make a small request up to the first aligned sector. For 1906 * convenience, limit this request to max_transfer even if 1907 * we don't need to fall back to writes. */ 1908 num = MIN(MIN(bytes, max_transfer), alignment - head); 1909 head = (head + num) % alignment; 1910 assert(num < max_write_zeroes); 1911 } else if (tail && num > alignment) { 1912 /* Shorten the request to the last aligned sector. */ 1913 num -= tail; 1914 } 1915 1916 /* limit request size */ 1917 if (num > max_write_zeroes) { 1918 num = max_write_zeroes; 1919 } 1920 1921 ret = -ENOTSUP; 1922 /* First try the efficient write zeroes operation */ 1923 if (drv->bdrv_co_pwrite_zeroes) { 1924 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1925 flags & bs->supported_zero_flags); 1926 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1927 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1928 need_flush = true; 1929 } 1930 } else { 1931 assert(!bs->supported_zero_flags); 1932 } 1933 1934 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 1935 /* Fall back to bounce buffer if write zeroes is unsupported */ 1936 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1937 1938 if ((flags & BDRV_REQ_FUA) && 1939 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1940 /* No need for bdrv_driver_pwrite() to do a fallback 1941 * flush on each chunk; use just one at the end */ 1942 write_flags &= ~BDRV_REQ_FUA; 1943 need_flush = true; 1944 } 1945 num = MIN(num, max_transfer); 1946 if (buf == NULL) { 1947 buf = qemu_try_blockalign0(bs, num); 1948 if (buf == NULL) { 1949 ret = -ENOMEM; 1950 goto fail; 1951 } 1952 } 1953 qemu_iovec_init_buf(&qiov, buf, num); 1954 1955 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 1956 1957 /* Keep bounce buffer around if it is big enough for all 1958 * all future requests. 1959 */ 1960 if (num < max_transfer) { 1961 qemu_vfree(buf); 1962 buf = NULL; 1963 } 1964 } 1965 1966 offset += num; 1967 bytes -= num; 1968 } 1969 1970 fail: 1971 if (ret == 0 && need_flush) { 1972 ret = bdrv_co_flush(bs); 1973 } 1974 qemu_vfree(buf); 1975 return ret; 1976 } 1977 1978 static inline int coroutine_fn GRAPH_RDLOCK 1979 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes, 1980 BdrvTrackedRequest *req, int flags) 1981 { 1982 BlockDriverState *bs = child->bs; 1983 1984 bdrv_check_request(offset, bytes, &error_abort); 1985 1986 if (bdrv_is_read_only(bs)) { 1987 return -EPERM; 1988 } 1989 1990 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1991 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1992 assert(!(flags & ~BDRV_REQ_MASK)); 1993 assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING))); 1994 1995 if (flags & BDRV_REQ_SERIALISING) { 1996 QEMU_LOCK_GUARD(&bs->reqs_lock); 1997 1998 tracked_request_set_serialising(req, bdrv_get_cluster_size(bs)); 1999 2000 if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) { 2001 return -EBUSY; 2002 } 2003 2004 bdrv_wait_serialising_requests_locked(req); 2005 } else { 2006 bdrv_wait_serialising_requests(req); 2007 } 2008 2009 assert(req->overlap_offset <= offset); 2010 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 2011 assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE || 2012 child->perm & BLK_PERM_RESIZE); 2013 2014 switch (req->type) { 2015 case BDRV_TRACKED_WRITE: 2016 case BDRV_TRACKED_DISCARD: 2017 if (flags & BDRV_REQ_WRITE_UNCHANGED) { 2018 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 2019 } else { 2020 assert(child->perm & BLK_PERM_WRITE); 2021 } 2022 bdrv_write_threshold_check_write(bs, offset, bytes); 2023 return 0; 2024 case BDRV_TRACKED_TRUNCATE: 2025 assert(child->perm & BLK_PERM_RESIZE); 2026 return 0; 2027 default: 2028 abort(); 2029 } 2030 } 2031 2032 static inline void coroutine_fn GRAPH_RDLOCK 2033 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes, 2034 BdrvTrackedRequest *req, int ret) 2035 { 2036 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 2037 BlockDriverState *bs = child->bs; 2038 2039 bdrv_check_request(offset, bytes, &error_abort); 2040 2041 qatomic_inc(&bs->write_gen); 2042 2043 /* 2044 * Discard cannot extend the image, but in error handling cases, such as 2045 * when reverting a qcow2 cluster allocation, the discarded range can pass 2046 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 2047 * here. Instead, just skip it, since semantically a discard request 2048 * beyond EOF cannot expand the image anyway. 2049 */ 2050 if (ret == 0 && 2051 (req->type == BDRV_TRACKED_TRUNCATE || 2052 end_sector > bs->total_sectors) && 2053 req->type != BDRV_TRACKED_DISCARD) { 2054 bs->total_sectors = end_sector; 2055 bdrv_parent_cb_resize(bs); 2056 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 2057 } 2058 if (req->bytes) { 2059 switch (req->type) { 2060 case BDRV_TRACKED_WRITE: 2061 stat64_max(&bs->wr_highest_offset, offset + bytes); 2062 /* fall through, to set dirty bits */ 2063 case BDRV_TRACKED_DISCARD: 2064 bdrv_set_dirty(bs, offset, bytes); 2065 break; 2066 default: 2067 break; 2068 } 2069 } 2070 } 2071 2072 /* 2073 * Forwards an already correctly aligned write request to the BlockDriver, 2074 * after possibly fragmenting it. 2075 */ 2076 static int coroutine_fn GRAPH_RDLOCK 2077 bdrv_aligned_pwritev(BdrvChild *child, BdrvTrackedRequest *req, 2078 int64_t offset, int64_t bytes, int64_t align, 2079 QEMUIOVector *qiov, size_t qiov_offset, 2080 BdrvRequestFlags flags) 2081 { 2082 BlockDriverState *bs = child->bs; 2083 BlockDriver *drv = bs->drv; 2084 int ret; 2085 2086 int64_t bytes_remaining = bytes; 2087 int max_transfer; 2088 2089 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 2090 2091 if (!drv) { 2092 return -ENOMEDIUM; 2093 } 2094 2095 if (bdrv_has_readonly_bitmaps(bs)) { 2096 return -EPERM; 2097 } 2098 2099 assert(is_power_of_2(align)); 2100 assert((offset & (align - 1)) == 0); 2101 assert((bytes & (align - 1)) == 0); 2102 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 2103 align); 2104 2105 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 2106 2107 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 2108 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 2109 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 2110 flags |= BDRV_REQ_ZERO_WRITE; 2111 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 2112 flags |= BDRV_REQ_MAY_UNMAP; 2113 } 2114 2115 /* Can't use optimization hint with bufferless zero write */ 2116 flags &= ~BDRV_REQ_REGISTERED_BUF; 2117 } 2118 2119 if (ret < 0) { 2120 /* Do nothing, write notifier decided to fail this request */ 2121 } else if (flags & BDRV_REQ_ZERO_WRITE) { 2122 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_ZERO); 2123 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 2124 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 2125 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 2126 qiov, qiov_offset); 2127 } else if (bytes <= max_transfer) { 2128 bdrv_co_debug_event(bs, BLKDBG_PWRITEV); 2129 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 2130 } else { 2131 bdrv_co_debug_event(bs, BLKDBG_PWRITEV); 2132 while (bytes_remaining) { 2133 int num = MIN(bytes_remaining, max_transfer); 2134 int local_flags = flags; 2135 2136 assert(num); 2137 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 2138 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 2139 /* If FUA is going to be emulated by flush, we only 2140 * need to flush on the last iteration */ 2141 local_flags &= ~BDRV_REQ_FUA; 2142 } 2143 2144 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 2145 num, qiov, 2146 qiov_offset + bytes - bytes_remaining, 2147 local_flags); 2148 if (ret < 0) { 2149 break; 2150 } 2151 bytes_remaining -= num; 2152 } 2153 } 2154 bdrv_co_debug_event(bs, BLKDBG_PWRITEV_DONE); 2155 2156 if (ret >= 0) { 2157 ret = 0; 2158 } 2159 bdrv_co_write_req_finish(child, offset, bytes, req, ret); 2160 2161 return ret; 2162 } 2163 2164 static int coroutine_fn GRAPH_RDLOCK 2165 bdrv_co_do_zero_pwritev(BdrvChild *child, int64_t offset, int64_t bytes, 2166 BdrvRequestFlags flags, BdrvTrackedRequest *req) 2167 { 2168 BlockDriverState *bs = child->bs; 2169 QEMUIOVector local_qiov; 2170 uint64_t align = bs->bl.request_alignment; 2171 int ret = 0; 2172 bool padding; 2173 BdrvRequestPadding pad; 2174 2175 /* This flag doesn't make sense for padding or zero writes */ 2176 flags &= ~BDRV_REQ_REGISTERED_BUF; 2177 2178 padding = bdrv_init_padding(bs, offset, bytes, true, &pad); 2179 if (padding) { 2180 assert(!(flags & BDRV_REQ_NO_WAIT)); 2181 bdrv_make_request_serialising(req, align); 2182 2183 bdrv_padding_rmw_read(child, req, &pad, true); 2184 2185 if (pad.head || pad.merge_reads) { 2186 int64_t aligned_offset = offset & ~(align - 1); 2187 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 2188 2189 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 2190 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 2191 align, &local_qiov, 0, 2192 flags & ~BDRV_REQ_ZERO_WRITE); 2193 if (ret < 0 || pad.merge_reads) { 2194 /* Error or all work is done */ 2195 goto out; 2196 } 2197 offset += write_bytes - pad.head; 2198 bytes -= write_bytes - pad.head; 2199 } 2200 } 2201 2202 assert(!bytes || (offset & (align - 1)) == 0); 2203 if (bytes >= align) { 2204 /* Write the aligned part in the middle. */ 2205 int64_t aligned_bytes = bytes & ~(align - 1); 2206 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 2207 NULL, 0, flags); 2208 if (ret < 0) { 2209 goto out; 2210 } 2211 bytes -= aligned_bytes; 2212 offset += aligned_bytes; 2213 } 2214 2215 assert(!bytes || (offset & (align - 1)) == 0); 2216 if (bytes) { 2217 assert(align == pad.tail + bytes); 2218 2219 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 2220 ret = bdrv_aligned_pwritev(child, req, offset, align, align, 2221 &local_qiov, 0, 2222 flags & ~BDRV_REQ_ZERO_WRITE); 2223 } 2224 2225 out: 2226 bdrv_padding_finalize(&pad); 2227 2228 return ret; 2229 } 2230 2231 /* 2232 * Handle a write request in coroutine context 2233 */ 2234 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 2235 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 2236 BdrvRequestFlags flags) 2237 { 2238 IO_CODE(); 2239 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 2240 } 2241 2242 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 2243 int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset, 2244 BdrvRequestFlags flags) 2245 { 2246 BlockDriverState *bs = child->bs; 2247 BdrvTrackedRequest req; 2248 uint64_t align = bs->bl.request_alignment; 2249 BdrvRequestPadding pad; 2250 int ret; 2251 bool padded = false; 2252 IO_CODE(); 2253 2254 trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags); 2255 2256 if (!bdrv_co_is_inserted(bs)) { 2257 return -ENOMEDIUM; 2258 } 2259 2260 if (flags & BDRV_REQ_ZERO_WRITE) { 2261 ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 2262 } else { 2263 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 2264 } 2265 if (ret < 0) { 2266 return ret; 2267 } 2268 2269 /* If the request is misaligned then we can't make it efficient */ 2270 if ((flags & BDRV_REQ_NO_FALLBACK) && 2271 !QEMU_IS_ALIGNED(offset | bytes, align)) 2272 { 2273 return -ENOTSUP; 2274 } 2275 2276 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2277 /* 2278 * Aligning zero request is nonsense. Even if driver has special meaning 2279 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2280 * it to driver due to request_alignment. 2281 * 2282 * Still, no reason to return an error if someone do unaligned 2283 * zero-length write occasionally. 2284 */ 2285 return 0; 2286 } 2287 2288 if (!(flags & BDRV_REQ_ZERO_WRITE)) { 2289 /* 2290 * Pad request for following read-modify-write cycle. 2291 * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do 2292 * alignment only if there is no ZERO flag. 2293 */ 2294 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, true, 2295 &pad, &padded, &flags); 2296 if (ret < 0) { 2297 return ret; 2298 } 2299 } 2300 2301 bdrv_inc_in_flight(bs); 2302 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 2303 2304 if (flags & BDRV_REQ_ZERO_WRITE) { 2305 assert(!padded); 2306 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 2307 goto out; 2308 } 2309 2310 if (padded) { 2311 /* 2312 * Request was unaligned to request_alignment and therefore 2313 * padded. We are going to do read-modify-write, and must 2314 * serialize the request to prevent interactions of the 2315 * widened region with other transactions. 2316 */ 2317 assert(!(flags & BDRV_REQ_NO_WAIT)); 2318 bdrv_make_request_serialising(&req, align); 2319 bdrv_padding_rmw_read(child, &req, &pad, false); 2320 } 2321 2322 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 2323 qiov, qiov_offset, flags); 2324 2325 bdrv_padding_finalize(&pad); 2326 2327 out: 2328 tracked_request_end(&req); 2329 bdrv_dec_in_flight(bs); 2330 2331 return ret; 2332 } 2333 2334 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2335 int64_t bytes, BdrvRequestFlags flags) 2336 { 2337 IO_CODE(); 2338 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 2339 assert_bdrv_graph_readable(); 2340 2341 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 2342 flags &= ~BDRV_REQ_MAY_UNMAP; 2343 } 2344 2345 return bdrv_co_pwritev(child, offset, bytes, NULL, 2346 BDRV_REQ_ZERO_WRITE | flags); 2347 } 2348 2349 /* 2350 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 2351 */ 2352 int bdrv_flush_all(void) 2353 { 2354 BdrvNextIterator it; 2355 BlockDriverState *bs = NULL; 2356 int result = 0; 2357 2358 GLOBAL_STATE_CODE(); 2359 GRAPH_RDLOCK_GUARD_MAINLOOP(); 2360 2361 /* 2362 * bdrv queue is managed by record/replay, 2363 * creating new flush request for stopping 2364 * the VM may break the determinism 2365 */ 2366 if (replay_events_enabled()) { 2367 return result; 2368 } 2369 2370 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 2371 AioContext *aio_context = bdrv_get_aio_context(bs); 2372 int ret; 2373 2374 aio_context_acquire(aio_context); 2375 ret = bdrv_flush(bs); 2376 if (ret < 0 && !result) { 2377 result = ret; 2378 } 2379 aio_context_release(aio_context); 2380 } 2381 2382 return result; 2383 } 2384 2385 /* 2386 * Returns the allocation status of the specified sectors. 2387 * Drivers not implementing the functionality are assumed to not support 2388 * backing files, hence all their sectors are reported as allocated. 2389 * 2390 * If 'want_zero' is true, the caller is querying for mapping 2391 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 2392 * _ZERO where possible; otherwise, the result favors larger 'pnum', 2393 * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2394 * 2395 * If 'offset' is beyond the end of the disk image the return value is 2396 * BDRV_BLOCK_EOF and 'pnum' is set to 0. 2397 * 2398 * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2399 * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2400 * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 2401 * 2402 * 'pnum' is set to the number of bytes (including and immediately 2403 * following the specified offset) that are easily known to be in the 2404 * same allocated/unallocated state. Note that a second call starting 2405 * at the original offset plus returned pnum may have the same status. 2406 * The returned value is non-zero on success except at end-of-file. 2407 * 2408 * Returns negative errno on failure. Otherwise, if the 2409 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 2410 * set to the host mapping and BDS corresponding to the guest offset. 2411 */ 2412 static int coroutine_fn GRAPH_RDLOCK 2413 bdrv_co_do_block_status(BlockDriverState *bs, bool want_zero, 2414 int64_t offset, int64_t bytes, 2415 int64_t *pnum, int64_t *map, BlockDriverState **file) 2416 { 2417 int64_t total_size; 2418 int64_t n; /* bytes */ 2419 int ret; 2420 int64_t local_map = 0; 2421 BlockDriverState *local_file = NULL; 2422 int64_t aligned_offset, aligned_bytes; 2423 uint32_t align; 2424 bool has_filtered_child; 2425 2426 assert(pnum); 2427 assert_bdrv_graph_readable(); 2428 *pnum = 0; 2429 total_size = bdrv_co_getlength(bs); 2430 if (total_size < 0) { 2431 ret = total_size; 2432 goto early_out; 2433 } 2434 2435 if (offset >= total_size) { 2436 ret = BDRV_BLOCK_EOF; 2437 goto early_out; 2438 } 2439 if (!bytes) { 2440 ret = 0; 2441 goto early_out; 2442 } 2443 2444 n = total_size - offset; 2445 if (n < bytes) { 2446 bytes = n; 2447 } 2448 2449 /* Must be non-NULL or bdrv_co_getlength() would have failed */ 2450 assert(bs->drv); 2451 has_filtered_child = bdrv_filter_child(bs); 2452 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 2453 *pnum = bytes; 2454 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 2455 if (offset + bytes == total_size) { 2456 ret |= BDRV_BLOCK_EOF; 2457 } 2458 if (bs->drv->protocol_name) { 2459 ret |= BDRV_BLOCK_OFFSET_VALID; 2460 local_map = offset; 2461 local_file = bs; 2462 } 2463 goto early_out; 2464 } 2465 2466 bdrv_inc_in_flight(bs); 2467 2468 /* Round out to request_alignment boundaries */ 2469 align = bs->bl.request_alignment; 2470 aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2471 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2472 2473 if (bs->drv->bdrv_co_block_status) { 2474 /* 2475 * Use the block-status cache only for protocol nodes: Format 2476 * drivers are generally quick to inquire the status, but protocol 2477 * drivers often need to get information from outside of qemu, so 2478 * we do not have control over the actual implementation. There 2479 * have been cases where inquiring the status took an unreasonably 2480 * long time, and we can do nothing in qemu to fix it. 2481 * This is especially problematic for images with large data areas, 2482 * because finding the few holes in them and giving them special 2483 * treatment does not gain much performance. Therefore, we try to 2484 * cache the last-identified data region. 2485 * 2486 * Second, limiting ourselves to protocol nodes allows us to assume 2487 * the block status for data regions to be DATA | OFFSET_VALID, and 2488 * that the host offset is the same as the guest offset. 2489 * 2490 * Note that it is possible that external writers zero parts of 2491 * the cached regions without the cache being invalidated, and so 2492 * we may report zeroes as data. This is not catastrophic, 2493 * however, because reporting zeroes as data is fine. 2494 */ 2495 if (QLIST_EMPTY(&bs->children) && 2496 bdrv_bsc_is_data(bs, aligned_offset, pnum)) 2497 { 2498 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID; 2499 local_file = bs; 2500 local_map = aligned_offset; 2501 } else { 2502 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 2503 aligned_bytes, pnum, &local_map, 2504 &local_file); 2505 2506 /* 2507 * Note that checking QLIST_EMPTY(&bs->children) is also done when 2508 * the cache is queried above. Technically, we do not need to check 2509 * it here; the worst that can happen is that we fill the cache for 2510 * non-protocol nodes, and then it is never used. However, filling 2511 * the cache requires an RCU update, so double check here to avoid 2512 * such an update if possible. 2513 * 2514 * Check want_zero, because we only want to update the cache when we 2515 * have accurate information about what is zero and what is data. 2516 */ 2517 if (want_zero && 2518 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) && 2519 QLIST_EMPTY(&bs->children)) 2520 { 2521 /* 2522 * When a protocol driver reports BLOCK_OFFSET_VALID, the 2523 * returned local_map value must be the same as the offset we 2524 * have passed (aligned_offset), and local_bs must be the node 2525 * itself. 2526 * Assert this, because we follow this rule when reading from 2527 * the cache (see the `local_file = bs` and 2528 * `local_map = aligned_offset` assignments above), and the 2529 * result the cache delivers must be the same as the driver 2530 * would deliver. 2531 */ 2532 assert(local_file == bs); 2533 assert(local_map == aligned_offset); 2534 bdrv_bsc_fill(bs, aligned_offset, *pnum); 2535 } 2536 } 2537 } else { 2538 /* Default code for filters */ 2539 2540 local_file = bdrv_filter_bs(bs); 2541 assert(local_file); 2542 2543 *pnum = aligned_bytes; 2544 local_map = aligned_offset; 2545 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2546 } 2547 if (ret < 0) { 2548 *pnum = 0; 2549 goto out; 2550 } 2551 2552 /* 2553 * The driver's result must be a non-zero multiple of request_alignment. 2554 * Clamp pnum and adjust map to original request. 2555 */ 2556 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2557 align > offset - aligned_offset); 2558 if (ret & BDRV_BLOCK_RECURSE) { 2559 assert(ret & BDRV_BLOCK_DATA); 2560 assert(ret & BDRV_BLOCK_OFFSET_VALID); 2561 assert(!(ret & BDRV_BLOCK_ZERO)); 2562 } 2563 2564 *pnum -= offset - aligned_offset; 2565 if (*pnum > bytes) { 2566 *pnum = bytes; 2567 } 2568 if (ret & BDRV_BLOCK_OFFSET_VALID) { 2569 local_map += offset - aligned_offset; 2570 } 2571 2572 if (ret & BDRV_BLOCK_RAW) { 2573 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 2574 ret = bdrv_co_do_block_status(local_file, want_zero, local_map, 2575 *pnum, pnum, &local_map, &local_file); 2576 goto out; 2577 } 2578 2579 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 2580 ret |= BDRV_BLOCK_ALLOCATED; 2581 } else if (bs->drv->supports_backing) { 2582 BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2583 2584 if (!cow_bs) { 2585 ret |= BDRV_BLOCK_ZERO; 2586 } else if (want_zero) { 2587 int64_t size2 = bdrv_co_getlength(cow_bs); 2588 2589 if (size2 >= 0 && offset >= size2) { 2590 ret |= BDRV_BLOCK_ZERO; 2591 } 2592 } 2593 } 2594 2595 if (want_zero && ret & BDRV_BLOCK_RECURSE && 2596 local_file && local_file != bs && 2597 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 2598 (ret & BDRV_BLOCK_OFFSET_VALID)) { 2599 int64_t file_pnum; 2600 int ret2; 2601 2602 ret2 = bdrv_co_do_block_status(local_file, want_zero, local_map, 2603 *pnum, &file_pnum, NULL, NULL); 2604 if (ret2 >= 0) { 2605 /* Ignore errors. This is just providing extra information, it 2606 * is useful but not necessary. 2607 */ 2608 if (ret2 & BDRV_BLOCK_EOF && 2609 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2610 /* 2611 * It is valid for the format block driver to read 2612 * beyond the end of the underlying file's current 2613 * size; such areas read as zero. 2614 */ 2615 ret |= BDRV_BLOCK_ZERO; 2616 } else { 2617 /* Limit request to the range reported by the protocol driver */ 2618 *pnum = file_pnum; 2619 ret |= (ret2 & BDRV_BLOCK_ZERO); 2620 } 2621 } 2622 2623 /* 2624 * Now that the recursive search was done, clear the flag. Otherwise, 2625 * with more complicated block graphs like snapshot-access -> 2626 * copy-before-write -> qcow2, where the return value will be propagated 2627 * further up to a parent bdrv_co_do_block_status() call, both the 2628 * BDRV_BLOCK_RECURSE and BDRV_BLOCK_ZERO flags would be set, which is 2629 * not allowed. 2630 */ 2631 ret &= ~BDRV_BLOCK_RECURSE; 2632 } 2633 2634 out: 2635 bdrv_dec_in_flight(bs); 2636 if (ret >= 0 && offset + *pnum == total_size) { 2637 ret |= BDRV_BLOCK_EOF; 2638 } 2639 early_out: 2640 if (file) { 2641 *file = local_file; 2642 } 2643 if (map) { 2644 *map = local_map; 2645 } 2646 return ret; 2647 } 2648 2649 int coroutine_fn 2650 bdrv_co_common_block_status_above(BlockDriverState *bs, 2651 BlockDriverState *base, 2652 bool include_base, 2653 bool want_zero, 2654 int64_t offset, 2655 int64_t bytes, 2656 int64_t *pnum, 2657 int64_t *map, 2658 BlockDriverState **file, 2659 int *depth) 2660 { 2661 int ret; 2662 BlockDriverState *p; 2663 int64_t eof = 0; 2664 int dummy; 2665 IO_CODE(); 2666 2667 assert(!include_base || base); /* Can't include NULL base */ 2668 assert_bdrv_graph_readable(); 2669 2670 if (!depth) { 2671 depth = &dummy; 2672 } 2673 *depth = 0; 2674 2675 if (!include_base && bs == base) { 2676 *pnum = bytes; 2677 return 0; 2678 } 2679 2680 ret = bdrv_co_do_block_status(bs, want_zero, offset, bytes, pnum, 2681 map, file); 2682 ++*depth; 2683 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 2684 return ret; 2685 } 2686 2687 if (ret & BDRV_BLOCK_EOF) { 2688 eof = offset + *pnum; 2689 } 2690 2691 assert(*pnum <= bytes); 2692 bytes = *pnum; 2693 2694 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 2695 p = bdrv_filter_or_cow_bs(p)) 2696 { 2697 ret = bdrv_co_do_block_status(p, want_zero, offset, bytes, pnum, 2698 map, file); 2699 ++*depth; 2700 if (ret < 0) { 2701 return ret; 2702 } 2703 if (*pnum == 0) { 2704 /* 2705 * The top layer deferred to this layer, and because this layer is 2706 * short, any zeroes that we synthesize beyond EOF behave as if they 2707 * were allocated at this layer. 2708 * 2709 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 2710 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2711 * below. 2712 */ 2713 assert(ret & BDRV_BLOCK_EOF); 2714 *pnum = bytes; 2715 if (file) { 2716 *file = p; 2717 } 2718 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2719 break; 2720 } 2721 if (ret & BDRV_BLOCK_ALLOCATED) { 2722 /* 2723 * We've found the node and the status, we must break. 2724 * 2725 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 2726 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2727 * below. 2728 */ 2729 ret &= ~BDRV_BLOCK_EOF; 2730 break; 2731 } 2732 2733 if (p == base) { 2734 assert(include_base); 2735 break; 2736 } 2737 2738 /* 2739 * OK, [offset, offset + *pnum) region is unallocated on this layer, 2740 * let's continue the diving. 2741 */ 2742 assert(*pnum <= bytes); 2743 bytes = *pnum; 2744 } 2745 2746 if (offset + *pnum == eof) { 2747 ret |= BDRV_BLOCK_EOF; 2748 } 2749 2750 return ret; 2751 } 2752 2753 int coroutine_fn bdrv_co_block_status_above(BlockDriverState *bs, 2754 BlockDriverState *base, 2755 int64_t offset, int64_t bytes, 2756 int64_t *pnum, int64_t *map, 2757 BlockDriverState **file) 2758 { 2759 IO_CODE(); 2760 return bdrv_co_common_block_status_above(bs, base, false, true, offset, 2761 bytes, pnum, map, file, NULL); 2762 } 2763 2764 int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, int64_t offset, 2765 int64_t bytes, int64_t *pnum, 2766 int64_t *map, BlockDriverState **file) 2767 { 2768 IO_CODE(); 2769 return bdrv_co_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 2770 offset, bytes, pnum, map, file); 2771 } 2772 2773 /* 2774 * Check @bs (and its backing chain) to see if the range defined 2775 * by @offset and @bytes is known to read as zeroes. 2776 * Return 1 if that is the case, 0 otherwise and -errno on error. 2777 * This test is meant to be fast rather than accurate so returning 0 2778 * does not guarantee non-zero data. 2779 */ 2780 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset, 2781 int64_t bytes) 2782 { 2783 int ret; 2784 int64_t pnum = bytes; 2785 IO_CODE(); 2786 2787 if (!bytes) { 2788 return 1; 2789 } 2790 2791 ret = bdrv_co_common_block_status_above(bs, NULL, false, false, offset, 2792 bytes, &pnum, NULL, NULL, NULL); 2793 2794 if (ret < 0) { 2795 return ret; 2796 } 2797 2798 return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO); 2799 } 2800 2801 int coroutine_fn bdrv_co_is_allocated(BlockDriverState *bs, int64_t offset, 2802 int64_t bytes, int64_t *pnum) 2803 { 2804 int ret; 2805 int64_t dummy; 2806 IO_CODE(); 2807 2808 ret = bdrv_co_common_block_status_above(bs, bs, true, false, offset, 2809 bytes, pnum ? pnum : &dummy, NULL, 2810 NULL, NULL); 2811 if (ret < 0) { 2812 return ret; 2813 } 2814 return !!(ret & BDRV_BLOCK_ALLOCATED); 2815 } 2816 2817 /* 2818 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 2819 * 2820 * Return a positive depth if (a prefix of) the given range is allocated 2821 * in any image between BASE and TOP (BASE is only included if include_base 2822 * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth. 2823 * BASE can be NULL to check if the given offset is allocated in any 2824 * image of the chain. Return 0 otherwise, or negative errno on 2825 * failure. 2826 * 2827 * 'pnum' is set to the number of bytes (including and immediately 2828 * following the specified offset) that are known to be in the same 2829 * allocated/unallocated state. Note that a subsequent call starting 2830 * at 'offset + *pnum' may return the same allocation status (in other 2831 * words, the result is not necessarily the maximum possible range); 2832 * but 'pnum' will only be 0 when end of file is reached. 2833 */ 2834 int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *bs, 2835 BlockDriverState *base, 2836 bool include_base, int64_t offset, 2837 int64_t bytes, int64_t *pnum) 2838 { 2839 int depth; 2840 int ret; 2841 IO_CODE(); 2842 2843 ret = bdrv_co_common_block_status_above(bs, base, include_base, false, 2844 offset, bytes, pnum, NULL, NULL, 2845 &depth); 2846 if (ret < 0) { 2847 return ret; 2848 } 2849 2850 if (ret & BDRV_BLOCK_ALLOCATED) { 2851 return depth; 2852 } 2853 return 0; 2854 } 2855 2856 int coroutine_fn 2857 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2858 { 2859 BlockDriver *drv = bs->drv; 2860 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2861 int ret; 2862 IO_CODE(); 2863 assert_bdrv_graph_readable(); 2864 2865 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2866 if (ret < 0) { 2867 return ret; 2868 } 2869 2870 if (!drv) { 2871 return -ENOMEDIUM; 2872 } 2873 2874 bdrv_inc_in_flight(bs); 2875 2876 if (drv->bdrv_co_load_vmstate) { 2877 ret = drv->bdrv_co_load_vmstate(bs, qiov, pos); 2878 } else if (child_bs) { 2879 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 2880 } else { 2881 ret = -ENOTSUP; 2882 } 2883 2884 bdrv_dec_in_flight(bs); 2885 2886 return ret; 2887 } 2888 2889 int coroutine_fn 2890 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2891 { 2892 BlockDriver *drv = bs->drv; 2893 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2894 int ret; 2895 IO_CODE(); 2896 assert_bdrv_graph_readable(); 2897 2898 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2899 if (ret < 0) { 2900 return ret; 2901 } 2902 2903 if (!drv) { 2904 return -ENOMEDIUM; 2905 } 2906 2907 bdrv_inc_in_flight(bs); 2908 2909 if (drv->bdrv_co_save_vmstate) { 2910 ret = drv->bdrv_co_save_vmstate(bs, qiov, pos); 2911 } else if (child_bs) { 2912 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2913 } else { 2914 ret = -ENOTSUP; 2915 } 2916 2917 bdrv_dec_in_flight(bs); 2918 2919 return ret; 2920 } 2921 2922 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 2923 int64_t pos, int size) 2924 { 2925 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2926 int ret = bdrv_writev_vmstate(bs, &qiov, pos); 2927 IO_CODE(); 2928 2929 return ret < 0 ? ret : size; 2930 } 2931 2932 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 2933 int64_t pos, int size) 2934 { 2935 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2936 int ret = bdrv_readv_vmstate(bs, &qiov, pos); 2937 IO_CODE(); 2938 2939 return ret < 0 ? ret : size; 2940 } 2941 2942 /**************************************************************/ 2943 /* async I/Os */ 2944 2945 /** 2946 * Synchronously cancels an acb. Must be called with the BQL held and the acb 2947 * must be processed with the BQL held too (IOThreads are not allowed). 2948 * 2949 * Use bdrv_aio_cancel_async() instead when possible. 2950 */ 2951 void bdrv_aio_cancel(BlockAIOCB *acb) 2952 { 2953 GLOBAL_STATE_CODE(); 2954 qemu_aio_ref(acb); 2955 bdrv_aio_cancel_async(acb); 2956 AIO_WAIT_WHILE_UNLOCKED(NULL, acb->refcnt > 1); 2957 qemu_aio_unref(acb); 2958 } 2959 2960 /* Async version of aio cancel. The caller is not blocked if the acb implements 2961 * cancel_async, otherwise we do nothing and let the request normally complete. 2962 * In either case the completion callback must be called. */ 2963 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2964 { 2965 IO_CODE(); 2966 if (acb->aiocb_info->cancel_async) { 2967 acb->aiocb_info->cancel_async(acb); 2968 } 2969 } 2970 2971 /**************************************************************/ 2972 /* Coroutine block device emulation */ 2973 2974 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2975 { 2976 BdrvChild *primary_child = bdrv_primary_child(bs); 2977 BdrvChild *child; 2978 int current_gen; 2979 int ret = 0; 2980 IO_CODE(); 2981 2982 assert_bdrv_graph_readable(); 2983 bdrv_inc_in_flight(bs); 2984 2985 if (!bdrv_co_is_inserted(bs) || bdrv_is_read_only(bs) || 2986 bdrv_is_sg(bs)) { 2987 goto early_exit; 2988 } 2989 2990 qemu_mutex_lock(&bs->reqs_lock); 2991 current_gen = qatomic_read(&bs->write_gen); 2992 2993 /* Wait until any previous flushes are completed */ 2994 while (bs->active_flush_req) { 2995 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 2996 } 2997 2998 /* Flushes reach this point in nondecreasing current_gen order. */ 2999 bs->active_flush_req = true; 3000 qemu_mutex_unlock(&bs->reqs_lock); 3001 3002 /* Write back all layers by calling one driver function */ 3003 if (bs->drv->bdrv_co_flush) { 3004 ret = bs->drv->bdrv_co_flush(bs); 3005 goto out; 3006 } 3007 3008 /* Write back cached data to the OS even with cache=unsafe */ 3009 BLKDBG_CO_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 3010 if (bs->drv->bdrv_co_flush_to_os) { 3011 ret = bs->drv->bdrv_co_flush_to_os(bs); 3012 if (ret < 0) { 3013 goto out; 3014 } 3015 } 3016 3017 /* But don't actually force it to the disk with cache=unsafe */ 3018 if (bs->open_flags & BDRV_O_NO_FLUSH) { 3019 goto flush_children; 3020 } 3021 3022 /* Check if we really need to flush anything */ 3023 if (bs->flushed_gen == current_gen) { 3024 goto flush_children; 3025 } 3026 3027 BLKDBG_CO_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 3028 if (!bs->drv) { 3029 /* bs->drv->bdrv_co_flush() might have ejected the BDS 3030 * (even in case of apparent success) */ 3031 ret = -ENOMEDIUM; 3032 goto out; 3033 } 3034 if (bs->drv->bdrv_co_flush_to_disk) { 3035 ret = bs->drv->bdrv_co_flush_to_disk(bs); 3036 } else if (bs->drv->bdrv_aio_flush) { 3037 BlockAIOCB *acb; 3038 CoroutineIOCompletion co = { 3039 .coroutine = qemu_coroutine_self(), 3040 }; 3041 3042 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 3043 if (acb == NULL) { 3044 ret = -EIO; 3045 } else { 3046 qemu_coroutine_yield(); 3047 ret = co.ret; 3048 } 3049 } else { 3050 /* 3051 * Some block drivers always operate in either writethrough or unsafe 3052 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 3053 * know how the server works (because the behaviour is hardcoded or 3054 * depends on server-side configuration), so we can't ensure that 3055 * everything is safe on disk. Returning an error doesn't work because 3056 * that would break guests even if the server operates in writethrough 3057 * mode. 3058 * 3059 * Let's hope the user knows what he's doing. 3060 */ 3061 ret = 0; 3062 } 3063 3064 if (ret < 0) { 3065 goto out; 3066 } 3067 3068 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 3069 * in the case of cache=unsafe, so there are no useless flushes. 3070 */ 3071 flush_children: 3072 ret = 0; 3073 QLIST_FOREACH(child, &bs->children, next) { 3074 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 3075 int this_child_ret = bdrv_co_flush(child->bs); 3076 if (!ret) { 3077 ret = this_child_ret; 3078 } 3079 } 3080 } 3081 3082 out: 3083 /* Notify any pending flushes that we have completed */ 3084 if (ret == 0) { 3085 bs->flushed_gen = current_gen; 3086 } 3087 3088 qemu_mutex_lock(&bs->reqs_lock); 3089 bs->active_flush_req = false; 3090 /* Return value is ignored - it's ok if wait queue is empty */ 3091 qemu_co_queue_next(&bs->flush_queue); 3092 qemu_mutex_unlock(&bs->reqs_lock); 3093 3094 early_exit: 3095 bdrv_dec_in_flight(bs); 3096 return ret; 3097 } 3098 3099 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 3100 int64_t bytes) 3101 { 3102 BdrvTrackedRequest req; 3103 int ret; 3104 int64_t max_pdiscard; 3105 int head, tail, align; 3106 BlockDriverState *bs = child->bs; 3107 IO_CODE(); 3108 assert_bdrv_graph_readable(); 3109 3110 if (!bs || !bs->drv || !bdrv_co_is_inserted(bs)) { 3111 return -ENOMEDIUM; 3112 } 3113 3114 if (bdrv_has_readonly_bitmaps(bs)) { 3115 return -EPERM; 3116 } 3117 3118 ret = bdrv_check_request(offset, bytes, NULL); 3119 if (ret < 0) { 3120 return ret; 3121 } 3122 3123 /* Do nothing if disabled. */ 3124 if (!(bs->open_flags & BDRV_O_UNMAP)) { 3125 return 0; 3126 } 3127 3128 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 3129 return 0; 3130 } 3131 3132 /* Invalidate the cached block-status data range if this discard overlaps */ 3133 bdrv_bsc_invalidate_range(bs, offset, bytes); 3134 3135 /* Discard is advisory, but some devices track and coalesce 3136 * unaligned requests, so we must pass everything down rather than 3137 * round here. Still, most devices will just silently ignore 3138 * unaligned requests (by returning -ENOTSUP), so we must fragment 3139 * the request accordingly. */ 3140 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 3141 assert(align % bs->bl.request_alignment == 0); 3142 head = offset % align; 3143 tail = (offset + bytes) % align; 3144 3145 bdrv_inc_in_flight(bs); 3146 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 3147 3148 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 3149 if (ret < 0) { 3150 goto out; 3151 } 3152 3153 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX), 3154 align); 3155 assert(max_pdiscard >= bs->bl.request_alignment); 3156 3157 while (bytes > 0) { 3158 int64_t num = bytes; 3159 3160 if (head) { 3161 /* Make small requests to get to alignment boundaries. */ 3162 num = MIN(bytes, align - head); 3163 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 3164 num %= bs->bl.request_alignment; 3165 } 3166 head = (head + num) % align; 3167 assert(num < max_pdiscard); 3168 } else if (tail) { 3169 if (num > align) { 3170 /* Shorten the request to the last aligned cluster. */ 3171 num -= tail; 3172 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 3173 tail > bs->bl.request_alignment) { 3174 tail %= bs->bl.request_alignment; 3175 num -= tail; 3176 } 3177 } 3178 /* limit request size */ 3179 if (num > max_pdiscard) { 3180 num = max_pdiscard; 3181 } 3182 3183 if (!bs->drv) { 3184 ret = -ENOMEDIUM; 3185 goto out; 3186 } 3187 if (bs->drv->bdrv_co_pdiscard) { 3188 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 3189 } else { 3190 BlockAIOCB *acb; 3191 CoroutineIOCompletion co = { 3192 .coroutine = qemu_coroutine_self(), 3193 }; 3194 3195 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 3196 bdrv_co_io_em_complete, &co); 3197 if (acb == NULL) { 3198 ret = -EIO; 3199 goto out; 3200 } else { 3201 qemu_coroutine_yield(); 3202 ret = co.ret; 3203 } 3204 } 3205 if (ret && ret != -ENOTSUP) { 3206 goto out; 3207 } 3208 3209 offset += num; 3210 bytes -= num; 3211 } 3212 ret = 0; 3213 out: 3214 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 3215 tracked_request_end(&req); 3216 bdrv_dec_in_flight(bs); 3217 return ret; 3218 } 3219 3220 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 3221 { 3222 BlockDriver *drv = bs->drv; 3223 CoroutineIOCompletion co = { 3224 .coroutine = qemu_coroutine_self(), 3225 }; 3226 BlockAIOCB *acb; 3227 IO_CODE(); 3228 assert_bdrv_graph_readable(); 3229 3230 bdrv_inc_in_flight(bs); 3231 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 3232 co.ret = -ENOTSUP; 3233 goto out; 3234 } 3235 3236 if (drv->bdrv_co_ioctl) { 3237 co.ret = drv->bdrv_co_ioctl(bs, req, buf); 3238 } else { 3239 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 3240 if (!acb) { 3241 co.ret = -ENOTSUP; 3242 goto out; 3243 } 3244 qemu_coroutine_yield(); 3245 } 3246 out: 3247 bdrv_dec_in_flight(bs); 3248 return co.ret; 3249 } 3250 3251 int coroutine_fn bdrv_co_zone_report(BlockDriverState *bs, int64_t offset, 3252 unsigned int *nr_zones, 3253 BlockZoneDescriptor *zones) 3254 { 3255 BlockDriver *drv = bs->drv; 3256 CoroutineIOCompletion co = { 3257 .coroutine = qemu_coroutine_self(), 3258 }; 3259 IO_CODE(); 3260 3261 bdrv_inc_in_flight(bs); 3262 if (!drv || !drv->bdrv_co_zone_report || bs->bl.zoned == BLK_Z_NONE) { 3263 co.ret = -ENOTSUP; 3264 goto out; 3265 } 3266 co.ret = drv->bdrv_co_zone_report(bs, offset, nr_zones, zones); 3267 out: 3268 bdrv_dec_in_flight(bs); 3269 return co.ret; 3270 } 3271 3272 int coroutine_fn bdrv_co_zone_mgmt(BlockDriverState *bs, BlockZoneOp op, 3273 int64_t offset, int64_t len) 3274 { 3275 BlockDriver *drv = bs->drv; 3276 CoroutineIOCompletion co = { 3277 .coroutine = qemu_coroutine_self(), 3278 }; 3279 IO_CODE(); 3280 3281 bdrv_inc_in_flight(bs); 3282 if (!drv || !drv->bdrv_co_zone_mgmt || bs->bl.zoned == BLK_Z_NONE) { 3283 co.ret = -ENOTSUP; 3284 goto out; 3285 } 3286 co.ret = drv->bdrv_co_zone_mgmt(bs, op, offset, len); 3287 out: 3288 bdrv_dec_in_flight(bs); 3289 return co.ret; 3290 } 3291 3292 int coroutine_fn bdrv_co_zone_append(BlockDriverState *bs, int64_t *offset, 3293 QEMUIOVector *qiov, 3294 BdrvRequestFlags flags) 3295 { 3296 int ret; 3297 BlockDriver *drv = bs->drv; 3298 CoroutineIOCompletion co = { 3299 .coroutine = qemu_coroutine_self(), 3300 }; 3301 IO_CODE(); 3302 3303 ret = bdrv_check_qiov_request(*offset, qiov->size, qiov, 0, NULL); 3304 if (ret < 0) { 3305 return ret; 3306 } 3307 3308 bdrv_inc_in_flight(bs); 3309 if (!drv || !drv->bdrv_co_zone_append || bs->bl.zoned == BLK_Z_NONE) { 3310 co.ret = -ENOTSUP; 3311 goto out; 3312 } 3313 co.ret = drv->bdrv_co_zone_append(bs, offset, qiov, flags); 3314 out: 3315 bdrv_dec_in_flight(bs); 3316 return co.ret; 3317 } 3318 3319 void *qemu_blockalign(BlockDriverState *bs, size_t size) 3320 { 3321 IO_CODE(); 3322 return qemu_memalign(bdrv_opt_mem_align(bs), size); 3323 } 3324 3325 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 3326 { 3327 IO_CODE(); 3328 return memset(qemu_blockalign(bs, size), 0, size); 3329 } 3330 3331 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 3332 { 3333 size_t align = bdrv_opt_mem_align(bs); 3334 IO_CODE(); 3335 3336 /* Ensure that NULL is never returned on success */ 3337 assert(align > 0); 3338 if (size == 0) { 3339 size = align; 3340 } 3341 3342 return qemu_try_memalign(align, size); 3343 } 3344 3345 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 3346 { 3347 void *mem = qemu_try_blockalign(bs, size); 3348 IO_CODE(); 3349 3350 if (mem) { 3351 memset(mem, 0, size); 3352 } 3353 3354 return mem; 3355 } 3356 3357 /* Helper that undoes bdrv_register_buf() when it fails partway through */ 3358 static void GRAPH_RDLOCK 3359 bdrv_register_buf_rollback(BlockDriverState *bs, void *host, size_t size, 3360 BdrvChild *final_child) 3361 { 3362 BdrvChild *child; 3363 3364 GLOBAL_STATE_CODE(); 3365 assert_bdrv_graph_readable(); 3366 3367 QLIST_FOREACH(child, &bs->children, next) { 3368 if (child == final_child) { 3369 break; 3370 } 3371 3372 bdrv_unregister_buf(child->bs, host, size); 3373 } 3374 3375 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3376 bs->drv->bdrv_unregister_buf(bs, host, size); 3377 } 3378 } 3379 3380 bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size, 3381 Error **errp) 3382 { 3383 BdrvChild *child; 3384 3385 GLOBAL_STATE_CODE(); 3386 GRAPH_RDLOCK_GUARD_MAINLOOP(); 3387 3388 if (bs->drv && bs->drv->bdrv_register_buf) { 3389 if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) { 3390 return false; 3391 } 3392 } 3393 QLIST_FOREACH(child, &bs->children, next) { 3394 if (!bdrv_register_buf(child->bs, host, size, errp)) { 3395 bdrv_register_buf_rollback(bs, host, size, child); 3396 return false; 3397 } 3398 } 3399 return true; 3400 } 3401 3402 void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size) 3403 { 3404 BdrvChild *child; 3405 3406 GLOBAL_STATE_CODE(); 3407 GRAPH_RDLOCK_GUARD_MAINLOOP(); 3408 3409 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3410 bs->drv->bdrv_unregister_buf(bs, host, size); 3411 } 3412 QLIST_FOREACH(child, &bs->children, next) { 3413 bdrv_unregister_buf(child->bs, host, size); 3414 } 3415 } 3416 3417 static int coroutine_fn GRAPH_RDLOCK bdrv_co_copy_range_internal( 3418 BdrvChild *src, int64_t src_offset, BdrvChild *dst, 3419 int64_t dst_offset, int64_t bytes, 3420 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 3421 bool recurse_src) 3422 { 3423 BdrvTrackedRequest req; 3424 int ret; 3425 assert_bdrv_graph_readable(); 3426 3427 /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 3428 assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 3429 assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 3430 assert(!(read_flags & BDRV_REQ_NO_WAIT)); 3431 assert(!(write_flags & BDRV_REQ_NO_WAIT)); 3432 3433 if (!dst || !dst->bs || !bdrv_co_is_inserted(dst->bs)) { 3434 return -ENOMEDIUM; 3435 } 3436 ret = bdrv_check_request32(dst_offset, bytes, NULL, 0); 3437 if (ret) { 3438 return ret; 3439 } 3440 if (write_flags & BDRV_REQ_ZERO_WRITE) { 3441 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3442 } 3443 3444 if (!src || !src->bs || !bdrv_co_is_inserted(src->bs)) { 3445 return -ENOMEDIUM; 3446 } 3447 ret = bdrv_check_request32(src_offset, bytes, NULL, 0); 3448 if (ret) { 3449 return ret; 3450 } 3451 3452 if (!src->bs->drv->bdrv_co_copy_range_from 3453 || !dst->bs->drv->bdrv_co_copy_range_to 3454 || src->bs->encrypted || dst->bs->encrypted) { 3455 return -ENOTSUP; 3456 } 3457 3458 if (recurse_src) { 3459 bdrv_inc_in_flight(src->bs); 3460 tracked_request_begin(&req, src->bs, src_offset, bytes, 3461 BDRV_TRACKED_READ); 3462 3463 /* BDRV_REQ_SERIALISING is only for write operation */ 3464 assert(!(read_flags & BDRV_REQ_SERIALISING)); 3465 bdrv_wait_serialising_requests(&req); 3466 3467 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3468 src, src_offset, 3469 dst, dst_offset, 3470 bytes, 3471 read_flags, write_flags); 3472 3473 tracked_request_end(&req); 3474 bdrv_dec_in_flight(src->bs); 3475 } else { 3476 bdrv_inc_in_flight(dst->bs); 3477 tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3478 BDRV_TRACKED_WRITE); 3479 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 3480 write_flags); 3481 if (!ret) { 3482 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3483 src, src_offset, 3484 dst, dst_offset, 3485 bytes, 3486 read_flags, write_flags); 3487 } 3488 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3489 tracked_request_end(&req); 3490 bdrv_dec_in_flight(dst->bs); 3491 } 3492 3493 return ret; 3494 } 3495 3496 /* Copy range from @src to @dst. 3497 * 3498 * See the comment of bdrv_co_copy_range for the parameter and return value 3499 * semantics. */ 3500 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset, 3501 BdrvChild *dst, int64_t dst_offset, 3502 int64_t bytes, 3503 BdrvRequestFlags read_flags, 3504 BdrvRequestFlags write_flags) 3505 { 3506 IO_CODE(); 3507 assert_bdrv_graph_readable(); 3508 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3509 read_flags, write_flags); 3510 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3511 bytes, read_flags, write_flags, true); 3512 } 3513 3514 /* Copy range from @src to @dst. 3515 * 3516 * See the comment of bdrv_co_copy_range for the parameter and return value 3517 * semantics. */ 3518 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset, 3519 BdrvChild *dst, int64_t dst_offset, 3520 int64_t bytes, 3521 BdrvRequestFlags read_flags, 3522 BdrvRequestFlags write_flags) 3523 { 3524 IO_CODE(); 3525 assert_bdrv_graph_readable(); 3526 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3527 read_flags, write_flags); 3528 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3529 bytes, read_flags, write_flags, false); 3530 } 3531 3532 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset, 3533 BdrvChild *dst, int64_t dst_offset, 3534 int64_t bytes, BdrvRequestFlags read_flags, 3535 BdrvRequestFlags write_flags) 3536 { 3537 IO_CODE(); 3538 assert_bdrv_graph_readable(); 3539 3540 return bdrv_co_copy_range_from(src, src_offset, 3541 dst, dst_offset, 3542 bytes, read_flags, write_flags); 3543 } 3544 3545 static void coroutine_fn GRAPH_RDLOCK 3546 bdrv_parent_cb_resize(BlockDriverState *bs) 3547 { 3548 BdrvChild *c; 3549 3550 assert_bdrv_graph_readable(); 3551 3552 QLIST_FOREACH(c, &bs->parents, next_parent) { 3553 if (c->klass->resize) { 3554 c->klass->resize(c); 3555 } 3556 } 3557 } 3558 3559 /** 3560 * Truncate file to 'offset' bytes (needed only for file protocols) 3561 * 3562 * If 'exact' is true, the file must be resized to exactly the given 3563 * 'offset'. Otherwise, it is sufficient for the node to be at least 3564 * 'offset' bytes in length. 3565 */ 3566 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 3567 PreallocMode prealloc, BdrvRequestFlags flags, 3568 Error **errp) 3569 { 3570 BlockDriverState *bs = child->bs; 3571 BdrvChild *filtered, *backing; 3572 BlockDriver *drv = bs->drv; 3573 BdrvTrackedRequest req; 3574 int64_t old_size, new_bytes; 3575 int ret; 3576 IO_CODE(); 3577 assert_bdrv_graph_readable(); 3578 3579 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 3580 if (!drv) { 3581 error_setg(errp, "No medium inserted"); 3582 return -ENOMEDIUM; 3583 } 3584 if (offset < 0) { 3585 error_setg(errp, "Image size cannot be negative"); 3586 return -EINVAL; 3587 } 3588 3589 ret = bdrv_check_request(offset, 0, errp); 3590 if (ret < 0) { 3591 return ret; 3592 } 3593 3594 old_size = bdrv_co_getlength(bs); 3595 if (old_size < 0) { 3596 error_setg_errno(errp, -old_size, "Failed to get old image size"); 3597 return old_size; 3598 } 3599 3600 if (bdrv_is_read_only(bs)) { 3601 error_setg(errp, "Image is read-only"); 3602 return -EACCES; 3603 } 3604 3605 if (offset > old_size) { 3606 new_bytes = offset - old_size; 3607 } else { 3608 new_bytes = 0; 3609 } 3610 3611 bdrv_inc_in_flight(bs); 3612 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 3613 BDRV_TRACKED_TRUNCATE); 3614 3615 /* If we are growing the image and potentially using preallocation for the 3616 * new area, we need to make sure that no write requests are made to it 3617 * concurrently or they might be overwritten by preallocation. */ 3618 if (new_bytes) { 3619 bdrv_make_request_serialising(&req, 1); 3620 } 3621 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3622 0); 3623 if (ret < 0) { 3624 error_setg_errno(errp, -ret, 3625 "Failed to prepare request for truncation"); 3626 goto out; 3627 } 3628 3629 filtered = bdrv_filter_child(bs); 3630 backing = bdrv_cow_child(bs); 3631 3632 /* 3633 * If the image has a backing file that is large enough that it would 3634 * provide data for the new area, we cannot leave it unallocated because 3635 * then the backing file content would become visible. Instead, zero-fill 3636 * the new area. 3637 * 3638 * Note that if the image has a backing file, but was opened without the 3639 * backing file, taking care of keeping things consistent with that backing 3640 * file is the user's responsibility. 3641 */ 3642 if (new_bytes && backing) { 3643 int64_t backing_len; 3644 3645 backing_len = bdrv_co_getlength(backing->bs); 3646 if (backing_len < 0) { 3647 ret = backing_len; 3648 error_setg_errno(errp, -ret, "Could not get backing file size"); 3649 goto out; 3650 } 3651 3652 if (backing_len > old_size) { 3653 flags |= BDRV_REQ_ZERO_WRITE; 3654 } 3655 } 3656 3657 if (drv->bdrv_co_truncate) { 3658 if (flags & ~bs->supported_truncate_flags) { 3659 error_setg(errp, "Block driver does not support requested flags"); 3660 ret = -ENOTSUP; 3661 goto out; 3662 } 3663 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 3664 } else if (filtered) { 3665 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 3666 } else { 3667 error_setg(errp, "Image format driver does not support resize"); 3668 ret = -ENOTSUP; 3669 goto out; 3670 } 3671 if (ret < 0) { 3672 goto out; 3673 } 3674 3675 ret = bdrv_co_refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 3676 if (ret < 0) { 3677 error_setg_errno(errp, -ret, "Could not refresh total sector count"); 3678 } else { 3679 offset = bs->total_sectors * BDRV_SECTOR_SIZE; 3680 } 3681 /* 3682 * It's possible that truncation succeeded but bdrv_refresh_total_sectors 3683 * failed, but the latter doesn't affect how we should finish the request. 3684 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. 3685 */ 3686 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 3687 3688 out: 3689 tracked_request_end(&req); 3690 bdrv_dec_in_flight(bs); 3691 3692 return ret; 3693 } 3694 3695 void bdrv_cancel_in_flight(BlockDriverState *bs) 3696 { 3697 GLOBAL_STATE_CODE(); 3698 GRAPH_RDLOCK_GUARD_MAINLOOP(); 3699 3700 if (!bs || !bs->drv) { 3701 return; 3702 } 3703 3704 if (bs->drv->bdrv_cancel_in_flight) { 3705 bs->drv->bdrv_cancel_in_flight(bs); 3706 } 3707 } 3708 3709 int coroutine_fn 3710 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes, 3711 QEMUIOVector *qiov, size_t qiov_offset) 3712 { 3713 BlockDriverState *bs = child->bs; 3714 BlockDriver *drv = bs->drv; 3715 int ret; 3716 IO_CODE(); 3717 assert_bdrv_graph_readable(); 3718 3719 if (!drv) { 3720 return -ENOMEDIUM; 3721 } 3722 3723 if (!drv->bdrv_co_preadv_snapshot) { 3724 return -ENOTSUP; 3725 } 3726 3727 bdrv_inc_in_flight(bs); 3728 ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset); 3729 bdrv_dec_in_flight(bs); 3730 3731 return ret; 3732 } 3733 3734 int coroutine_fn 3735 bdrv_co_snapshot_block_status(BlockDriverState *bs, 3736 bool want_zero, int64_t offset, int64_t bytes, 3737 int64_t *pnum, int64_t *map, 3738 BlockDriverState **file) 3739 { 3740 BlockDriver *drv = bs->drv; 3741 int ret; 3742 IO_CODE(); 3743 assert_bdrv_graph_readable(); 3744 3745 if (!drv) { 3746 return -ENOMEDIUM; 3747 } 3748 3749 if (!drv->bdrv_co_snapshot_block_status) { 3750 return -ENOTSUP; 3751 } 3752 3753 bdrv_inc_in_flight(bs); 3754 ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes, 3755 pnum, map, file); 3756 bdrv_dec_in_flight(bs); 3757 3758 return ret; 3759 } 3760 3761 int coroutine_fn 3762 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes) 3763 { 3764 BlockDriver *drv = bs->drv; 3765 int ret; 3766 IO_CODE(); 3767 assert_bdrv_graph_readable(); 3768 3769 if (!drv) { 3770 return -ENOMEDIUM; 3771 } 3772 3773 if (!drv->bdrv_co_pdiscard_snapshot) { 3774 return -ENOTSUP; 3775 } 3776 3777 bdrv_inc_in_flight(bs); 3778 ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes); 3779 bdrv_dec_in_flight(bs); 3780 3781 return ret; 3782 } 3783