1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/aio-wait.h" 29 #include "block/blockjob.h" 30 #include "block/blockjob_int.h" 31 #include "block/block_int.h" 32 #include "block/coroutines.h" 33 #include "block/write-threshold.h" 34 #include "qemu/cutils.h" 35 #include "qemu/memalign.h" 36 #include "qapi/error.h" 37 #include "qemu/error-report.h" 38 #include "qemu/main-loop.h" 39 #include "sysemu/replay.h" 40 41 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 42 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 43 44 static void bdrv_parent_cb_resize(BlockDriverState *bs); 45 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 46 int64_t offset, int64_t bytes, BdrvRequestFlags flags); 47 48 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore, 49 bool ignore_bds_parents) 50 { 51 BdrvChild *c, *next; 52 53 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 54 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 55 continue; 56 } 57 bdrv_parent_drained_begin_single(c, false); 58 } 59 } 60 61 static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c, 62 int *drained_end_counter) 63 { 64 assert(c->parent_quiesce_counter > 0); 65 c->parent_quiesce_counter--; 66 if (c->klass->drained_end) { 67 c->klass->drained_end(c, drained_end_counter); 68 } 69 } 70 71 void bdrv_parent_drained_end_single(BdrvChild *c) 72 { 73 int drained_end_counter = 0; 74 IO_OR_GS_CODE(); 75 bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter); 76 BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0); 77 } 78 79 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore, 80 bool ignore_bds_parents, 81 int *drained_end_counter) 82 { 83 BdrvChild *c; 84 85 QLIST_FOREACH(c, &bs->parents, next_parent) { 86 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 87 continue; 88 } 89 bdrv_parent_drained_end_single_no_poll(c, drained_end_counter); 90 } 91 } 92 93 static bool bdrv_parent_drained_poll_single(BdrvChild *c) 94 { 95 if (c->klass->drained_poll) { 96 return c->klass->drained_poll(c); 97 } 98 return false; 99 } 100 101 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 102 bool ignore_bds_parents) 103 { 104 BdrvChild *c, *next; 105 bool busy = false; 106 107 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 108 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 109 continue; 110 } 111 busy |= bdrv_parent_drained_poll_single(c); 112 } 113 114 return busy; 115 } 116 117 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll) 118 { 119 IO_OR_GS_CODE(); 120 c->parent_quiesce_counter++; 121 if (c->klass->drained_begin) { 122 c->klass->drained_begin(c); 123 } 124 if (poll) { 125 BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c)); 126 } 127 } 128 129 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 130 { 131 dst->pdiscard_alignment = MAX(dst->pdiscard_alignment, 132 src->pdiscard_alignment); 133 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 134 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 135 dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer, 136 src->max_hw_transfer); 137 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 138 src->opt_mem_alignment); 139 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 140 src->min_mem_alignment); 141 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 142 dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov); 143 } 144 145 typedef struct BdrvRefreshLimitsState { 146 BlockDriverState *bs; 147 BlockLimits old_bl; 148 } BdrvRefreshLimitsState; 149 150 static void bdrv_refresh_limits_abort(void *opaque) 151 { 152 BdrvRefreshLimitsState *s = opaque; 153 154 s->bs->bl = s->old_bl; 155 } 156 157 static TransactionActionDrv bdrv_refresh_limits_drv = { 158 .abort = bdrv_refresh_limits_abort, 159 .clean = g_free, 160 }; 161 162 /* @tran is allowed to be NULL, in this case no rollback is possible. */ 163 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp) 164 { 165 ERRP_GUARD(); 166 BlockDriver *drv = bs->drv; 167 BdrvChild *c; 168 bool have_limits; 169 170 GLOBAL_STATE_CODE(); 171 172 if (tran) { 173 BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1); 174 *s = (BdrvRefreshLimitsState) { 175 .bs = bs, 176 .old_bl = bs->bl, 177 }; 178 tran_add(tran, &bdrv_refresh_limits_drv, s); 179 } 180 181 memset(&bs->bl, 0, sizeof(bs->bl)); 182 183 if (!drv) { 184 return; 185 } 186 187 /* Default alignment based on whether driver has byte interface */ 188 bs->bl.request_alignment = (drv->bdrv_co_preadv || 189 drv->bdrv_aio_preadv || 190 drv->bdrv_co_preadv_part) ? 1 : 512; 191 192 /* Take some limits from the children as a default */ 193 have_limits = false; 194 QLIST_FOREACH(c, &bs->children, next) { 195 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 196 { 197 bdrv_merge_limits(&bs->bl, &c->bs->bl); 198 have_limits = true; 199 } 200 } 201 202 if (!have_limits) { 203 bs->bl.min_mem_alignment = 512; 204 bs->bl.opt_mem_alignment = qemu_real_host_page_size(); 205 206 /* Safe default since most protocols use readv()/writev()/etc */ 207 bs->bl.max_iov = IOV_MAX; 208 } 209 210 /* Then let the driver override it */ 211 if (drv->bdrv_refresh_limits) { 212 drv->bdrv_refresh_limits(bs, errp); 213 if (*errp) { 214 return; 215 } 216 } 217 218 if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) { 219 error_setg(errp, "Driver requires too large request alignment"); 220 } 221 } 222 223 /** 224 * The copy-on-read flag is actually a reference count so multiple users may 225 * use the feature without worrying about clobbering its previous state. 226 * Copy-on-read stays enabled until all users have called to disable it. 227 */ 228 void bdrv_enable_copy_on_read(BlockDriverState *bs) 229 { 230 IO_CODE(); 231 qatomic_inc(&bs->copy_on_read); 232 } 233 234 void bdrv_disable_copy_on_read(BlockDriverState *bs) 235 { 236 int old = qatomic_fetch_dec(&bs->copy_on_read); 237 IO_CODE(); 238 assert(old >= 1); 239 } 240 241 typedef struct { 242 Coroutine *co; 243 BlockDriverState *bs; 244 bool done; 245 bool begin; 246 bool recursive; 247 bool poll; 248 BdrvChild *parent; 249 bool ignore_bds_parents; 250 int *drained_end_counter; 251 } BdrvCoDrainData; 252 253 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 254 { 255 BdrvCoDrainData *data = opaque; 256 BlockDriverState *bs = data->bs; 257 258 if (data->begin) { 259 bs->drv->bdrv_co_drain_begin(bs); 260 } else { 261 bs->drv->bdrv_co_drain_end(bs); 262 } 263 264 /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */ 265 qatomic_mb_set(&data->done, true); 266 if (!data->begin) { 267 qatomic_dec(data->drained_end_counter); 268 } 269 bdrv_dec_in_flight(bs); 270 271 g_free(data); 272 } 273 274 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */ 275 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, 276 int *drained_end_counter) 277 { 278 BdrvCoDrainData *data; 279 280 if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 281 (!begin && !bs->drv->bdrv_co_drain_end)) { 282 return; 283 } 284 285 data = g_new(BdrvCoDrainData, 1); 286 *data = (BdrvCoDrainData) { 287 .bs = bs, 288 .done = false, 289 .begin = begin, 290 .drained_end_counter = drained_end_counter, 291 }; 292 293 if (!begin) { 294 qatomic_inc(drained_end_counter); 295 } 296 297 /* Make sure the driver callback completes during the polling phase for 298 * drain_begin. */ 299 bdrv_inc_in_flight(bs); 300 data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data); 301 aio_co_schedule(bdrv_get_aio_context(bs), data->co); 302 } 303 304 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 305 bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, 306 BdrvChild *ignore_parent, bool ignore_bds_parents) 307 { 308 BdrvChild *child, *next; 309 IO_OR_GS_CODE(); 310 311 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 312 return true; 313 } 314 315 if (qatomic_read(&bs->in_flight)) { 316 return true; 317 } 318 319 if (recursive) { 320 assert(!ignore_bds_parents); 321 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 322 if (bdrv_drain_poll(child->bs, recursive, child, false)) { 323 return true; 324 } 325 } 326 } 327 328 return false; 329 } 330 331 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive, 332 BdrvChild *ignore_parent) 333 { 334 return bdrv_drain_poll(bs, recursive, ignore_parent, false); 335 } 336 337 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 338 BdrvChild *parent, bool ignore_bds_parents, 339 bool poll); 340 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 341 BdrvChild *parent, bool ignore_bds_parents, 342 int *drained_end_counter); 343 344 static void bdrv_co_drain_bh_cb(void *opaque) 345 { 346 BdrvCoDrainData *data = opaque; 347 Coroutine *co = data->co; 348 BlockDriverState *bs = data->bs; 349 350 if (bs) { 351 AioContext *ctx = bdrv_get_aio_context(bs); 352 aio_context_acquire(ctx); 353 bdrv_dec_in_flight(bs); 354 if (data->begin) { 355 assert(!data->drained_end_counter); 356 bdrv_do_drained_begin(bs, data->recursive, data->parent, 357 data->ignore_bds_parents, data->poll); 358 } else { 359 assert(!data->poll); 360 bdrv_do_drained_end(bs, data->recursive, data->parent, 361 data->ignore_bds_parents, 362 data->drained_end_counter); 363 } 364 aio_context_release(ctx); 365 } else { 366 assert(data->begin); 367 bdrv_drain_all_begin(); 368 } 369 370 data->done = true; 371 aio_co_wake(co); 372 } 373 374 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 375 bool begin, bool recursive, 376 BdrvChild *parent, 377 bool ignore_bds_parents, 378 bool poll, 379 int *drained_end_counter) 380 { 381 BdrvCoDrainData data; 382 Coroutine *self = qemu_coroutine_self(); 383 AioContext *ctx = bdrv_get_aio_context(bs); 384 AioContext *co_ctx = qemu_coroutine_get_aio_context(self); 385 386 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 387 * other coroutines run if they were queued by aio_co_enter(). */ 388 389 assert(qemu_in_coroutine()); 390 data = (BdrvCoDrainData) { 391 .co = self, 392 .bs = bs, 393 .done = false, 394 .begin = begin, 395 .recursive = recursive, 396 .parent = parent, 397 .ignore_bds_parents = ignore_bds_parents, 398 .poll = poll, 399 .drained_end_counter = drained_end_counter, 400 }; 401 402 if (bs) { 403 bdrv_inc_in_flight(bs); 404 } 405 406 /* 407 * Temporarily drop the lock across yield or we would get deadlocks. 408 * bdrv_co_drain_bh_cb() reaquires the lock as needed. 409 * 410 * When we yield below, the lock for the current context will be 411 * released, so if this is actually the lock that protects bs, don't drop 412 * it a second time. 413 */ 414 if (ctx != co_ctx) { 415 aio_context_release(ctx); 416 } 417 replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data); 418 419 qemu_coroutine_yield(); 420 /* If we are resumed from some other event (such as an aio completion or a 421 * timer callback), it is a bug in the caller that should be fixed. */ 422 assert(data.done); 423 424 /* Reaquire the AioContext of bs if we dropped it */ 425 if (ctx != co_ctx) { 426 aio_context_acquire(ctx); 427 } 428 } 429 430 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, 431 BdrvChild *parent, bool ignore_bds_parents) 432 { 433 IO_OR_GS_CODE(); 434 assert(!qemu_in_coroutine()); 435 436 /* Stop things in parent-to-child order */ 437 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 438 aio_disable_external(bdrv_get_aio_context(bs)); 439 } 440 441 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents); 442 bdrv_drain_invoke(bs, true, NULL); 443 } 444 445 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 446 BdrvChild *parent, bool ignore_bds_parents, 447 bool poll) 448 { 449 BdrvChild *child, *next; 450 451 if (qemu_in_coroutine()) { 452 bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents, 453 poll, NULL); 454 return; 455 } 456 457 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents); 458 459 if (recursive) { 460 assert(!ignore_bds_parents); 461 bs->recursive_quiesce_counter++; 462 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 463 bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents, 464 false); 465 } 466 } 467 468 /* 469 * Wait for drained requests to finish. 470 * 471 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 472 * call is needed so things in this AioContext can make progress even 473 * though we don't return to the main AioContext loop - this automatically 474 * includes other nodes in the same AioContext and therefore all child 475 * nodes. 476 */ 477 if (poll) { 478 assert(!ignore_bds_parents); 479 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent)); 480 } 481 } 482 483 void bdrv_drained_begin(BlockDriverState *bs) 484 { 485 IO_OR_GS_CODE(); 486 bdrv_do_drained_begin(bs, false, NULL, false, true); 487 } 488 489 void bdrv_subtree_drained_begin(BlockDriverState *bs) 490 { 491 IO_OR_GS_CODE(); 492 bdrv_do_drained_begin(bs, true, NULL, false, true); 493 } 494 495 /** 496 * This function does not poll, nor must any of its recursively called 497 * functions. The *drained_end_counter pointee will be incremented 498 * once for every background operation scheduled, and decremented once 499 * the operation settles. Therefore, the pointer must remain valid 500 * until the pointee reaches 0. That implies that whoever sets up the 501 * pointee has to poll until it is 0. 502 * 503 * We use atomic operations to access *drained_end_counter, because 504 * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of 505 * @bs may contain nodes in different AioContexts, 506 * (2) bdrv_drain_all_end() uses the same counter for all nodes, 507 * regardless of which AioContext they are in. 508 */ 509 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 510 BdrvChild *parent, bool ignore_bds_parents, 511 int *drained_end_counter) 512 { 513 BdrvChild *child; 514 int old_quiesce_counter; 515 516 assert(drained_end_counter != NULL); 517 518 if (qemu_in_coroutine()) { 519 bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents, 520 false, drained_end_counter); 521 return; 522 } 523 assert(bs->quiesce_counter > 0); 524 525 /* Re-enable things in child-to-parent order */ 526 bdrv_drain_invoke(bs, false, drained_end_counter); 527 bdrv_parent_drained_end(bs, parent, ignore_bds_parents, 528 drained_end_counter); 529 530 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 531 if (old_quiesce_counter == 1) { 532 aio_enable_external(bdrv_get_aio_context(bs)); 533 } 534 535 if (recursive) { 536 assert(!ignore_bds_parents); 537 bs->recursive_quiesce_counter--; 538 QLIST_FOREACH(child, &bs->children, next) { 539 bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents, 540 drained_end_counter); 541 } 542 } 543 } 544 545 void bdrv_drained_end(BlockDriverState *bs) 546 { 547 int drained_end_counter = 0; 548 IO_OR_GS_CODE(); 549 bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter); 550 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 551 } 552 553 void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter) 554 { 555 IO_CODE(); 556 bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter); 557 } 558 559 void bdrv_subtree_drained_end(BlockDriverState *bs) 560 { 561 int drained_end_counter = 0; 562 IO_OR_GS_CODE(); 563 bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter); 564 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 565 } 566 567 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent) 568 { 569 int i; 570 IO_OR_GS_CODE(); 571 572 for (i = 0; i < new_parent->recursive_quiesce_counter; i++) { 573 bdrv_do_drained_begin(child->bs, true, child, false, true); 574 } 575 } 576 577 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent) 578 { 579 int drained_end_counter = 0; 580 int i; 581 IO_OR_GS_CODE(); 582 583 for (i = 0; i < old_parent->recursive_quiesce_counter; i++) { 584 bdrv_do_drained_end(child->bs, true, child, false, 585 &drained_end_counter); 586 } 587 588 BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0); 589 } 590 591 void bdrv_drain(BlockDriverState *bs) 592 { 593 IO_OR_GS_CODE(); 594 bdrv_drained_begin(bs); 595 bdrv_drained_end(bs); 596 } 597 598 static void bdrv_drain_assert_idle(BlockDriverState *bs) 599 { 600 BdrvChild *child, *next; 601 602 assert(qatomic_read(&bs->in_flight) == 0); 603 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 604 bdrv_drain_assert_idle(child->bs); 605 } 606 } 607 608 unsigned int bdrv_drain_all_count = 0; 609 610 static bool bdrv_drain_all_poll(void) 611 { 612 BlockDriverState *bs = NULL; 613 bool result = false; 614 GLOBAL_STATE_CODE(); 615 616 /* bdrv_drain_poll() can't make changes to the graph and we are holding the 617 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 618 while ((bs = bdrv_next_all_states(bs))) { 619 AioContext *aio_context = bdrv_get_aio_context(bs); 620 aio_context_acquire(aio_context); 621 result |= bdrv_drain_poll(bs, false, NULL, true); 622 aio_context_release(aio_context); 623 } 624 625 return result; 626 } 627 628 /* 629 * Wait for pending requests to complete across all BlockDriverStates 630 * 631 * This function does not flush data to disk, use bdrv_flush_all() for that 632 * after calling this function. 633 * 634 * This pauses all block jobs and disables external clients. It must 635 * be paired with bdrv_drain_all_end(). 636 * 637 * NOTE: no new block jobs or BlockDriverStates can be created between 638 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 639 */ 640 void bdrv_drain_all_begin(void) 641 { 642 BlockDriverState *bs = NULL; 643 GLOBAL_STATE_CODE(); 644 645 if (qemu_in_coroutine()) { 646 bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL); 647 return; 648 } 649 650 /* 651 * bdrv queue is managed by record/replay, 652 * waiting for finishing the I/O requests may 653 * be infinite 654 */ 655 if (replay_events_enabled()) { 656 return; 657 } 658 659 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 660 * loop AioContext, so make sure we're in the main context. */ 661 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 662 assert(bdrv_drain_all_count < INT_MAX); 663 bdrv_drain_all_count++; 664 665 /* Quiesce all nodes, without polling in-flight requests yet. The graph 666 * cannot change during this loop. */ 667 while ((bs = bdrv_next_all_states(bs))) { 668 AioContext *aio_context = bdrv_get_aio_context(bs); 669 670 aio_context_acquire(aio_context); 671 bdrv_do_drained_begin(bs, false, NULL, true, false); 672 aio_context_release(aio_context); 673 } 674 675 /* Now poll the in-flight requests */ 676 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 677 678 while ((bs = bdrv_next_all_states(bs))) { 679 bdrv_drain_assert_idle(bs); 680 } 681 } 682 683 void bdrv_drain_all_end_quiesce(BlockDriverState *bs) 684 { 685 int drained_end_counter = 0; 686 GLOBAL_STATE_CODE(); 687 688 g_assert(bs->quiesce_counter > 0); 689 g_assert(!bs->refcnt); 690 691 while (bs->quiesce_counter) { 692 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 693 } 694 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 695 } 696 697 void bdrv_drain_all_end(void) 698 { 699 BlockDriverState *bs = NULL; 700 int drained_end_counter = 0; 701 GLOBAL_STATE_CODE(); 702 703 /* 704 * bdrv queue is managed by record/replay, 705 * waiting for finishing the I/O requests may 706 * be endless 707 */ 708 if (replay_events_enabled()) { 709 return; 710 } 711 712 while ((bs = bdrv_next_all_states(bs))) { 713 AioContext *aio_context = bdrv_get_aio_context(bs); 714 715 aio_context_acquire(aio_context); 716 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 717 aio_context_release(aio_context); 718 } 719 720 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 721 AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0); 722 723 assert(bdrv_drain_all_count > 0); 724 bdrv_drain_all_count--; 725 } 726 727 void bdrv_drain_all(void) 728 { 729 GLOBAL_STATE_CODE(); 730 bdrv_drain_all_begin(); 731 bdrv_drain_all_end(); 732 } 733 734 /** 735 * Remove an active request from the tracked requests list 736 * 737 * This function should be called when a tracked request is completing. 738 */ 739 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req) 740 { 741 if (req->serialising) { 742 qatomic_dec(&req->bs->serialising_in_flight); 743 } 744 745 qemu_co_mutex_lock(&req->bs->reqs_lock); 746 QLIST_REMOVE(req, list); 747 qemu_co_queue_restart_all(&req->wait_queue); 748 qemu_co_mutex_unlock(&req->bs->reqs_lock); 749 } 750 751 /** 752 * Add an active request to the tracked requests list 753 */ 754 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req, 755 BlockDriverState *bs, 756 int64_t offset, 757 int64_t bytes, 758 enum BdrvTrackedRequestType type) 759 { 760 bdrv_check_request(offset, bytes, &error_abort); 761 762 *req = (BdrvTrackedRequest){ 763 .bs = bs, 764 .offset = offset, 765 .bytes = bytes, 766 .type = type, 767 .co = qemu_coroutine_self(), 768 .serialising = false, 769 .overlap_offset = offset, 770 .overlap_bytes = bytes, 771 }; 772 773 qemu_co_queue_init(&req->wait_queue); 774 775 qemu_co_mutex_lock(&bs->reqs_lock); 776 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 777 qemu_co_mutex_unlock(&bs->reqs_lock); 778 } 779 780 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 781 int64_t offset, int64_t bytes) 782 { 783 bdrv_check_request(offset, bytes, &error_abort); 784 785 /* aaaa bbbb */ 786 if (offset >= req->overlap_offset + req->overlap_bytes) { 787 return false; 788 } 789 /* bbbb aaaa */ 790 if (req->overlap_offset >= offset + bytes) { 791 return false; 792 } 793 return true; 794 } 795 796 /* Called with self->bs->reqs_lock held */ 797 static coroutine_fn BdrvTrackedRequest * 798 bdrv_find_conflicting_request(BdrvTrackedRequest *self) 799 { 800 BdrvTrackedRequest *req; 801 802 QLIST_FOREACH(req, &self->bs->tracked_requests, list) { 803 if (req == self || (!req->serialising && !self->serialising)) { 804 continue; 805 } 806 if (tracked_request_overlaps(req, self->overlap_offset, 807 self->overlap_bytes)) 808 { 809 /* 810 * Hitting this means there was a reentrant request, for 811 * example, a block driver issuing nested requests. This must 812 * never happen since it means deadlock. 813 */ 814 assert(qemu_coroutine_self() != req->co); 815 816 /* 817 * If the request is already (indirectly) waiting for us, or 818 * will wait for us as soon as it wakes up, then just go on 819 * (instead of producing a deadlock in the former case). 820 */ 821 if (!req->waiting_for) { 822 return req; 823 } 824 } 825 } 826 827 return NULL; 828 } 829 830 /* Called with self->bs->reqs_lock held */ 831 static void coroutine_fn 832 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self) 833 { 834 BdrvTrackedRequest *req; 835 836 while ((req = bdrv_find_conflicting_request(self))) { 837 self->waiting_for = req; 838 qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock); 839 self->waiting_for = NULL; 840 } 841 } 842 843 /* Called with req->bs->reqs_lock held */ 844 static void tracked_request_set_serialising(BdrvTrackedRequest *req, 845 uint64_t align) 846 { 847 int64_t overlap_offset = req->offset & ~(align - 1); 848 int64_t overlap_bytes = 849 ROUND_UP(req->offset + req->bytes, align) - overlap_offset; 850 851 bdrv_check_request(req->offset, req->bytes, &error_abort); 852 853 if (!req->serialising) { 854 qatomic_inc(&req->bs->serialising_in_flight); 855 req->serialising = true; 856 } 857 858 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 859 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 860 } 861 862 /** 863 * Return the tracked request on @bs for the current coroutine, or 864 * NULL if there is none. 865 */ 866 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 867 { 868 BdrvTrackedRequest *req; 869 Coroutine *self = qemu_coroutine_self(); 870 IO_CODE(); 871 872 QLIST_FOREACH(req, &bs->tracked_requests, list) { 873 if (req->co == self) { 874 return req; 875 } 876 } 877 878 return NULL; 879 } 880 881 /** 882 * Round a region to cluster boundaries 883 */ 884 void bdrv_round_to_clusters(BlockDriverState *bs, 885 int64_t offset, int64_t bytes, 886 int64_t *cluster_offset, 887 int64_t *cluster_bytes) 888 { 889 BlockDriverInfo bdi; 890 IO_CODE(); 891 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 892 *cluster_offset = offset; 893 *cluster_bytes = bytes; 894 } else { 895 int64_t c = bdi.cluster_size; 896 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 897 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 898 } 899 } 900 901 static int bdrv_get_cluster_size(BlockDriverState *bs) 902 { 903 BlockDriverInfo bdi; 904 int ret; 905 906 ret = bdrv_get_info(bs, &bdi); 907 if (ret < 0 || bdi.cluster_size == 0) { 908 return bs->bl.request_alignment; 909 } else { 910 return bdi.cluster_size; 911 } 912 } 913 914 void bdrv_inc_in_flight(BlockDriverState *bs) 915 { 916 IO_CODE(); 917 qatomic_inc(&bs->in_flight); 918 } 919 920 void bdrv_wakeup(BlockDriverState *bs) 921 { 922 IO_CODE(); 923 aio_wait_kick(); 924 } 925 926 void bdrv_dec_in_flight(BlockDriverState *bs) 927 { 928 IO_CODE(); 929 qatomic_dec(&bs->in_flight); 930 bdrv_wakeup(bs); 931 } 932 933 static void coroutine_fn 934 bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 935 { 936 BlockDriverState *bs = self->bs; 937 938 if (!qatomic_read(&bs->serialising_in_flight)) { 939 return; 940 } 941 942 qemu_co_mutex_lock(&bs->reqs_lock); 943 bdrv_wait_serialising_requests_locked(self); 944 qemu_co_mutex_unlock(&bs->reqs_lock); 945 } 946 947 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req, 948 uint64_t align) 949 { 950 IO_CODE(); 951 952 qemu_co_mutex_lock(&req->bs->reqs_lock); 953 954 tracked_request_set_serialising(req, align); 955 bdrv_wait_serialising_requests_locked(req); 956 957 qemu_co_mutex_unlock(&req->bs->reqs_lock); 958 } 959 960 int bdrv_check_qiov_request(int64_t offset, int64_t bytes, 961 QEMUIOVector *qiov, size_t qiov_offset, 962 Error **errp) 963 { 964 /* 965 * Check generic offset/bytes correctness 966 */ 967 968 if (offset < 0) { 969 error_setg(errp, "offset is negative: %" PRIi64, offset); 970 return -EIO; 971 } 972 973 if (bytes < 0) { 974 error_setg(errp, "bytes is negative: %" PRIi64, bytes); 975 return -EIO; 976 } 977 978 if (bytes > BDRV_MAX_LENGTH) { 979 error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 980 bytes, BDRV_MAX_LENGTH); 981 return -EIO; 982 } 983 984 if (offset > BDRV_MAX_LENGTH) { 985 error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")", 986 offset, BDRV_MAX_LENGTH); 987 return -EIO; 988 } 989 990 if (offset > BDRV_MAX_LENGTH - bytes) { 991 error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") " 992 "exceeds maximum(%" PRIi64 ")", offset, bytes, 993 BDRV_MAX_LENGTH); 994 return -EIO; 995 } 996 997 if (!qiov) { 998 return 0; 999 } 1000 1001 /* 1002 * Check qiov and qiov_offset 1003 */ 1004 1005 if (qiov_offset > qiov->size) { 1006 error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)", 1007 qiov_offset, qiov->size); 1008 return -EIO; 1009 } 1010 1011 if (bytes > qiov->size - qiov_offset) { 1012 error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io " 1013 "vector size(%zu)", bytes, qiov_offset, qiov->size); 1014 return -EIO; 1015 } 1016 1017 return 0; 1018 } 1019 1020 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp) 1021 { 1022 return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp); 1023 } 1024 1025 static int bdrv_check_request32(int64_t offset, int64_t bytes, 1026 QEMUIOVector *qiov, size_t qiov_offset) 1027 { 1028 int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 1029 if (ret < 0) { 1030 return ret; 1031 } 1032 1033 if (bytes > BDRV_REQUEST_MAX_BYTES) { 1034 return -EIO; 1035 } 1036 1037 return 0; 1038 } 1039 1040 /* 1041 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 1042 * The operation is sped up by checking the block status and only writing 1043 * zeroes to the device if they currently do not return zeroes. Optional 1044 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 1045 * BDRV_REQ_FUA). 1046 * 1047 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 1048 */ 1049 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 1050 { 1051 int ret; 1052 int64_t target_size, bytes, offset = 0; 1053 BlockDriverState *bs = child->bs; 1054 IO_CODE(); 1055 1056 target_size = bdrv_getlength(bs); 1057 if (target_size < 0) { 1058 return target_size; 1059 } 1060 1061 for (;;) { 1062 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 1063 if (bytes <= 0) { 1064 return 0; 1065 } 1066 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 1067 if (ret < 0) { 1068 return ret; 1069 } 1070 if (ret & BDRV_BLOCK_ZERO) { 1071 offset += bytes; 1072 continue; 1073 } 1074 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 1075 if (ret < 0) { 1076 return ret; 1077 } 1078 offset += bytes; 1079 } 1080 } 1081 1082 /* 1083 * Writes to the file and ensures that no writes are reordered across this 1084 * request (acts as a barrier) 1085 * 1086 * Returns 0 on success, -errno in error cases. 1087 */ 1088 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset, 1089 int64_t bytes, const void *buf, 1090 BdrvRequestFlags flags) 1091 { 1092 int ret; 1093 IO_CODE(); 1094 1095 ret = bdrv_co_pwrite(child, offset, bytes, buf, flags); 1096 if (ret < 0) { 1097 return ret; 1098 } 1099 1100 ret = bdrv_co_flush(child->bs); 1101 if (ret < 0) { 1102 return ret; 1103 } 1104 1105 return 0; 1106 } 1107 1108 typedef struct CoroutineIOCompletion { 1109 Coroutine *coroutine; 1110 int ret; 1111 } CoroutineIOCompletion; 1112 1113 static void bdrv_co_io_em_complete(void *opaque, int ret) 1114 { 1115 CoroutineIOCompletion *co = opaque; 1116 1117 co->ret = ret; 1118 aio_co_wake(co->coroutine); 1119 } 1120 1121 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 1122 int64_t offset, int64_t bytes, 1123 QEMUIOVector *qiov, 1124 size_t qiov_offset, int flags) 1125 { 1126 BlockDriver *drv = bs->drv; 1127 int64_t sector_num; 1128 unsigned int nb_sectors; 1129 QEMUIOVector local_qiov; 1130 int ret; 1131 1132 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1133 assert(!(flags & ~bs->supported_read_flags)); 1134 1135 if (!drv) { 1136 return -ENOMEDIUM; 1137 } 1138 1139 if (drv->bdrv_co_preadv_part) { 1140 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 1141 flags); 1142 } 1143 1144 if (qiov_offset > 0 || bytes != qiov->size) { 1145 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1146 qiov = &local_qiov; 1147 } 1148 1149 if (drv->bdrv_co_preadv) { 1150 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 1151 goto out; 1152 } 1153 1154 if (drv->bdrv_aio_preadv) { 1155 BlockAIOCB *acb; 1156 CoroutineIOCompletion co = { 1157 .coroutine = qemu_coroutine_self(), 1158 }; 1159 1160 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 1161 bdrv_co_io_em_complete, &co); 1162 if (acb == NULL) { 1163 ret = -EIO; 1164 goto out; 1165 } else { 1166 qemu_coroutine_yield(); 1167 ret = co.ret; 1168 goto out; 1169 } 1170 } 1171 1172 sector_num = offset >> BDRV_SECTOR_BITS; 1173 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1174 1175 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1176 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1177 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1178 assert(drv->bdrv_co_readv); 1179 1180 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1181 1182 out: 1183 if (qiov == &local_qiov) { 1184 qemu_iovec_destroy(&local_qiov); 1185 } 1186 1187 return ret; 1188 } 1189 1190 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 1191 int64_t offset, int64_t bytes, 1192 QEMUIOVector *qiov, 1193 size_t qiov_offset, 1194 BdrvRequestFlags flags) 1195 { 1196 BlockDriver *drv = bs->drv; 1197 bool emulate_fua = false; 1198 int64_t sector_num; 1199 unsigned int nb_sectors; 1200 QEMUIOVector local_qiov; 1201 int ret; 1202 1203 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1204 1205 if (!drv) { 1206 return -ENOMEDIUM; 1207 } 1208 1209 if ((flags & BDRV_REQ_FUA) && 1210 (~bs->supported_write_flags & BDRV_REQ_FUA)) { 1211 flags &= ~BDRV_REQ_FUA; 1212 emulate_fua = true; 1213 } 1214 1215 flags &= bs->supported_write_flags; 1216 1217 if (drv->bdrv_co_pwritev_part) { 1218 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1219 flags); 1220 goto emulate_flags; 1221 } 1222 1223 if (qiov_offset > 0 || bytes != qiov->size) { 1224 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1225 qiov = &local_qiov; 1226 } 1227 1228 if (drv->bdrv_co_pwritev) { 1229 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags); 1230 goto emulate_flags; 1231 } 1232 1233 if (drv->bdrv_aio_pwritev) { 1234 BlockAIOCB *acb; 1235 CoroutineIOCompletion co = { 1236 .coroutine = qemu_coroutine_self(), 1237 }; 1238 1239 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags, 1240 bdrv_co_io_em_complete, &co); 1241 if (acb == NULL) { 1242 ret = -EIO; 1243 } else { 1244 qemu_coroutine_yield(); 1245 ret = co.ret; 1246 } 1247 goto emulate_flags; 1248 } 1249 1250 sector_num = offset >> BDRV_SECTOR_BITS; 1251 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1252 1253 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1254 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1255 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1256 1257 assert(drv->bdrv_co_writev); 1258 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags); 1259 1260 emulate_flags: 1261 if (ret == 0 && emulate_fua) { 1262 ret = bdrv_co_flush(bs); 1263 } 1264 1265 if (qiov == &local_qiov) { 1266 qemu_iovec_destroy(&local_qiov); 1267 } 1268 1269 return ret; 1270 } 1271 1272 static int coroutine_fn 1273 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset, 1274 int64_t bytes, QEMUIOVector *qiov, 1275 size_t qiov_offset) 1276 { 1277 BlockDriver *drv = bs->drv; 1278 QEMUIOVector local_qiov; 1279 int ret; 1280 1281 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1282 1283 if (!drv) { 1284 return -ENOMEDIUM; 1285 } 1286 1287 if (!block_driver_can_compress(drv)) { 1288 return -ENOTSUP; 1289 } 1290 1291 if (drv->bdrv_co_pwritev_compressed_part) { 1292 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1293 qiov, qiov_offset); 1294 } 1295 1296 if (qiov_offset == 0) { 1297 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 1298 } 1299 1300 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1301 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1302 qemu_iovec_destroy(&local_qiov); 1303 1304 return ret; 1305 } 1306 1307 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 1308 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 1309 size_t qiov_offset, int flags) 1310 { 1311 BlockDriverState *bs = child->bs; 1312 1313 /* Perform I/O through a temporary buffer so that users who scribble over 1314 * their read buffer while the operation is in progress do not end up 1315 * modifying the image file. This is critical for zero-copy guest I/O 1316 * where anything might happen inside guest memory. 1317 */ 1318 void *bounce_buffer = NULL; 1319 1320 BlockDriver *drv = bs->drv; 1321 int64_t cluster_offset; 1322 int64_t cluster_bytes; 1323 int64_t skip_bytes; 1324 int ret; 1325 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1326 BDRV_REQUEST_MAX_BYTES); 1327 int64_t progress = 0; 1328 bool skip_write; 1329 1330 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1331 1332 if (!drv) { 1333 return -ENOMEDIUM; 1334 } 1335 1336 /* 1337 * Do not write anything when the BDS is inactive. That is not 1338 * allowed, and it would not help. 1339 */ 1340 skip_write = (bs->open_flags & BDRV_O_INACTIVE); 1341 1342 /* FIXME We cannot require callers to have write permissions when all they 1343 * are doing is a read request. If we did things right, write permissions 1344 * would be obtained anyway, but internally by the copy-on-read code. As 1345 * long as it is implemented here rather than in a separate filter driver, 1346 * the copy-on-read code doesn't have its own BdrvChild, however, for which 1347 * it could request permissions. Therefore we have to bypass the permission 1348 * system for the moment. */ 1349 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1350 1351 /* Cover entire cluster so no additional backing file I/O is required when 1352 * allocating cluster in the image file. Note that this value may exceed 1353 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1354 * is one reason we loop rather than doing it all at once. 1355 */ 1356 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1357 skip_bytes = offset - cluster_offset; 1358 1359 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1360 cluster_offset, cluster_bytes); 1361 1362 while (cluster_bytes) { 1363 int64_t pnum; 1364 1365 if (skip_write) { 1366 ret = 1; /* "already allocated", so nothing will be copied */ 1367 pnum = MIN(cluster_bytes, max_transfer); 1368 } else { 1369 ret = bdrv_is_allocated(bs, cluster_offset, 1370 MIN(cluster_bytes, max_transfer), &pnum); 1371 if (ret < 0) { 1372 /* 1373 * Safe to treat errors in querying allocation as if 1374 * unallocated; we'll probably fail again soon on the 1375 * read, but at least that will set a decent errno. 1376 */ 1377 pnum = MIN(cluster_bytes, max_transfer); 1378 } 1379 1380 /* Stop at EOF if the image ends in the middle of the cluster */ 1381 if (ret == 0 && pnum == 0) { 1382 assert(progress >= bytes); 1383 break; 1384 } 1385 1386 assert(skip_bytes < pnum); 1387 } 1388 1389 if (ret <= 0) { 1390 QEMUIOVector local_qiov; 1391 1392 /* Must copy-on-read; use the bounce buffer */ 1393 pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1394 if (!bounce_buffer) { 1395 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum); 1396 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 1397 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 1398 1399 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 1400 if (!bounce_buffer) { 1401 ret = -ENOMEM; 1402 goto err; 1403 } 1404 } 1405 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1406 1407 ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1408 &local_qiov, 0, 0); 1409 if (ret < 0) { 1410 goto err; 1411 } 1412 1413 bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1414 if (drv->bdrv_co_pwrite_zeroes && 1415 buffer_is_zero(bounce_buffer, pnum)) { 1416 /* FIXME: Should we (perhaps conditionally) be setting 1417 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1418 * that still correctly reads as zero? */ 1419 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 1420 BDRV_REQ_WRITE_UNCHANGED); 1421 } else { 1422 /* This does not change the data on the disk, it is not 1423 * necessary to flush even in cache=writethrough mode. 1424 */ 1425 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1426 &local_qiov, 0, 1427 BDRV_REQ_WRITE_UNCHANGED); 1428 } 1429 1430 if (ret < 0) { 1431 /* It might be okay to ignore write errors for guest 1432 * requests. If this is a deliberate copy-on-read 1433 * then we don't want to ignore the error. Simply 1434 * report it in all cases. 1435 */ 1436 goto err; 1437 } 1438 1439 if (!(flags & BDRV_REQ_PREFETCH)) { 1440 qemu_iovec_from_buf(qiov, qiov_offset + progress, 1441 bounce_buffer + skip_bytes, 1442 MIN(pnum - skip_bytes, bytes - progress)); 1443 } 1444 } else if (!(flags & BDRV_REQ_PREFETCH)) { 1445 /* Read directly into the destination */ 1446 ret = bdrv_driver_preadv(bs, offset + progress, 1447 MIN(pnum - skip_bytes, bytes - progress), 1448 qiov, qiov_offset + progress, 0); 1449 if (ret < 0) { 1450 goto err; 1451 } 1452 } 1453 1454 cluster_offset += pnum; 1455 cluster_bytes -= pnum; 1456 progress += pnum - skip_bytes; 1457 skip_bytes = 0; 1458 } 1459 ret = 0; 1460 1461 err: 1462 qemu_vfree(bounce_buffer); 1463 return ret; 1464 } 1465 1466 /* 1467 * Forwards an already correctly aligned request to the BlockDriver. This 1468 * handles copy on read, zeroing after EOF, and fragmentation of large 1469 * reads; any other features must be implemented by the caller. 1470 */ 1471 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 1472 BdrvTrackedRequest *req, int64_t offset, int64_t bytes, 1473 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 1474 { 1475 BlockDriverState *bs = child->bs; 1476 int64_t total_bytes, max_bytes; 1477 int ret = 0; 1478 int64_t bytes_remaining = bytes; 1479 int max_transfer; 1480 1481 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 1482 assert(is_power_of_2(align)); 1483 assert((offset & (align - 1)) == 0); 1484 assert((bytes & (align - 1)) == 0); 1485 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1486 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1487 align); 1488 1489 /* 1490 * TODO: We would need a per-BDS .supported_read_flags and 1491 * potential fallback support, if we ever implement any read flags 1492 * to pass through to drivers. For now, there aren't any 1493 * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint. 1494 */ 1495 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH | 1496 BDRV_REQ_REGISTERED_BUF))); 1497 1498 /* Handle Copy on Read and associated serialisation */ 1499 if (flags & BDRV_REQ_COPY_ON_READ) { 1500 /* If we touch the same cluster it counts as an overlap. This 1501 * guarantees that allocating writes will be serialized and not race 1502 * with each other for the same cluster. For example, in copy-on-read 1503 * it ensures that the CoR read and write operations are atomic and 1504 * guest writes cannot interleave between them. */ 1505 bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs)); 1506 } else { 1507 bdrv_wait_serialising_requests(req); 1508 } 1509 1510 if (flags & BDRV_REQ_COPY_ON_READ) { 1511 int64_t pnum; 1512 1513 /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */ 1514 flags &= ~BDRV_REQ_COPY_ON_READ; 1515 1516 ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 1517 if (ret < 0) { 1518 goto out; 1519 } 1520 1521 if (!ret || pnum != bytes) { 1522 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 1523 qiov, qiov_offset, flags); 1524 goto out; 1525 } else if (flags & BDRV_REQ_PREFETCH) { 1526 goto out; 1527 } 1528 } 1529 1530 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1531 total_bytes = bdrv_getlength(bs); 1532 if (total_bytes < 0) { 1533 ret = total_bytes; 1534 goto out; 1535 } 1536 1537 assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF))); 1538 1539 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1540 if (bytes <= max_bytes && bytes <= max_transfer) { 1541 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags); 1542 goto out; 1543 } 1544 1545 while (bytes_remaining) { 1546 int64_t num; 1547 1548 if (max_bytes) { 1549 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1550 assert(num); 1551 1552 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1553 num, qiov, 1554 qiov_offset + bytes - bytes_remaining, 1555 flags); 1556 max_bytes -= num; 1557 } else { 1558 num = bytes_remaining; 1559 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1560 0, bytes_remaining); 1561 } 1562 if (ret < 0) { 1563 goto out; 1564 } 1565 bytes_remaining -= num; 1566 } 1567 1568 out: 1569 return ret < 0 ? ret : 0; 1570 } 1571 1572 /* 1573 * Request padding 1574 * 1575 * |<---- align ----->| |<----- align ---->| 1576 * |<- head ->|<------------- bytes ------------->|<-- tail -->| 1577 * | | | | | | 1578 * -*----------$-------*-------- ... --------*-----$------------*--- 1579 * | | | | | | 1580 * | offset | | end | 1581 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 1582 * [buf ... ) [tail_buf ) 1583 * 1584 * @buf is an aligned allocation needed to store @head and @tail paddings. @head 1585 * is placed at the beginning of @buf and @tail at the @end. 1586 * 1587 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 1588 * around tail, if tail exists. 1589 * 1590 * @merge_reads is true for small requests, 1591 * if @buf_len == @head + bytes + @tail. In this case it is possible that both 1592 * head and tail exist but @buf_len == align and @tail_buf == @buf. 1593 */ 1594 typedef struct BdrvRequestPadding { 1595 uint8_t *buf; 1596 size_t buf_len; 1597 uint8_t *tail_buf; 1598 size_t head; 1599 size_t tail; 1600 bool merge_reads; 1601 QEMUIOVector local_qiov; 1602 } BdrvRequestPadding; 1603 1604 static bool bdrv_init_padding(BlockDriverState *bs, 1605 int64_t offset, int64_t bytes, 1606 BdrvRequestPadding *pad) 1607 { 1608 int64_t align = bs->bl.request_alignment; 1609 int64_t sum; 1610 1611 bdrv_check_request(offset, bytes, &error_abort); 1612 assert(align <= INT_MAX); /* documented in block/block_int.h */ 1613 assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */ 1614 1615 memset(pad, 0, sizeof(*pad)); 1616 1617 pad->head = offset & (align - 1); 1618 pad->tail = ((offset + bytes) & (align - 1)); 1619 if (pad->tail) { 1620 pad->tail = align - pad->tail; 1621 } 1622 1623 if (!pad->head && !pad->tail) { 1624 return false; 1625 } 1626 1627 assert(bytes); /* Nothing good in aligning zero-length requests */ 1628 1629 sum = pad->head + bytes + pad->tail; 1630 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 1631 pad->buf = qemu_blockalign(bs, pad->buf_len); 1632 pad->merge_reads = sum == pad->buf_len; 1633 if (pad->tail) { 1634 pad->tail_buf = pad->buf + pad->buf_len - align; 1635 } 1636 1637 return true; 1638 } 1639 1640 static coroutine_fn int bdrv_padding_rmw_read(BdrvChild *child, 1641 BdrvTrackedRequest *req, 1642 BdrvRequestPadding *pad, 1643 bool zero_middle) 1644 { 1645 QEMUIOVector local_qiov; 1646 BlockDriverState *bs = child->bs; 1647 uint64_t align = bs->bl.request_alignment; 1648 int ret; 1649 1650 assert(req->serialising && pad->buf); 1651 1652 if (pad->head || pad->merge_reads) { 1653 int64_t bytes = pad->merge_reads ? pad->buf_len : align; 1654 1655 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 1656 1657 if (pad->head) { 1658 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1659 } 1660 if (pad->merge_reads && pad->tail) { 1661 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1662 } 1663 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 1664 align, &local_qiov, 0, 0); 1665 if (ret < 0) { 1666 return ret; 1667 } 1668 if (pad->head) { 1669 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1670 } 1671 if (pad->merge_reads && pad->tail) { 1672 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1673 } 1674 1675 if (pad->merge_reads) { 1676 goto zero_mem; 1677 } 1678 } 1679 1680 if (pad->tail) { 1681 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 1682 1683 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1684 ret = bdrv_aligned_preadv( 1685 child, req, 1686 req->overlap_offset + req->overlap_bytes - align, 1687 align, align, &local_qiov, 0, 0); 1688 if (ret < 0) { 1689 return ret; 1690 } 1691 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1692 } 1693 1694 zero_mem: 1695 if (zero_middle) { 1696 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 1697 } 1698 1699 return 0; 1700 } 1701 1702 static void bdrv_padding_destroy(BdrvRequestPadding *pad) 1703 { 1704 if (pad->buf) { 1705 qemu_vfree(pad->buf); 1706 qemu_iovec_destroy(&pad->local_qiov); 1707 } 1708 memset(pad, 0, sizeof(*pad)); 1709 } 1710 1711 /* 1712 * bdrv_pad_request 1713 * 1714 * Exchange request parameters with padded request if needed. Don't include RMW 1715 * read of padding, bdrv_padding_rmw_read() should be called separately if 1716 * needed. 1717 * 1718 * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out: 1719 * - on function start they represent original request 1720 * - on failure or when padding is not needed they are unchanged 1721 * - on success when padding is needed they represent padded request 1722 */ 1723 static int bdrv_pad_request(BlockDriverState *bs, 1724 QEMUIOVector **qiov, size_t *qiov_offset, 1725 int64_t *offset, int64_t *bytes, 1726 BdrvRequestPadding *pad, bool *padded, 1727 BdrvRequestFlags *flags) 1728 { 1729 int ret; 1730 1731 bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort); 1732 1733 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) { 1734 if (padded) { 1735 *padded = false; 1736 } 1737 return 0; 1738 } 1739 1740 ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head, 1741 *qiov, *qiov_offset, *bytes, 1742 pad->buf + pad->buf_len - pad->tail, 1743 pad->tail); 1744 if (ret < 0) { 1745 bdrv_padding_destroy(pad); 1746 return ret; 1747 } 1748 *bytes += pad->head + pad->tail; 1749 *offset -= pad->head; 1750 *qiov = &pad->local_qiov; 1751 *qiov_offset = 0; 1752 if (padded) { 1753 *padded = true; 1754 } 1755 if (flags) { 1756 /* Can't use optimization hint with bounce buffer */ 1757 *flags &= ~BDRV_REQ_REGISTERED_BUF; 1758 } 1759 1760 return 0; 1761 } 1762 1763 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1764 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 1765 BdrvRequestFlags flags) 1766 { 1767 IO_CODE(); 1768 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 1769 } 1770 1771 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 1772 int64_t offset, int64_t bytes, 1773 QEMUIOVector *qiov, size_t qiov_offset, 1774 BdrvRequestFlags flags) 1775 { 1776 BlockDriverState *bs = child->bs; 1777 BdrvTrackedRequest req; 1778 BdrvRequestPadding pad; 1779 int ret; 1780 IO_CODE(); 1781 1782 trace_bdrv_co_preadv_part(bs, offset, bytes, flags); 1783 1784 if (!bdrv_is_inserted(bs)) { 1785 return -ENOMEDIUM; 1786 } 1787 1788 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 1789 if (ret < 0) { 1790 return ret; 1791 } 1792 1793 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1794 /* 1795 * Aligning zero request is nonsense. Even if driver has special meaning 1796 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1797 * it to driver due to request_alignment. 1798 * 1799 * Still, no reason to return an error if someone do unaligned 1800 * zero-length read occasionally. 1801 */ 1802 return 0; 1803 } 1804 1805 bdrv_inc_in_flight(bs); 1806 1807 /* Don't do copy-on-read if we read data before write operation */ 1808 if (qatomic_read(&bs->copy_on_read)) { 1809 flags |= BDRV_REQ_COPY_ON_READ; 1810 } 1811 1812 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 1813 NULL, &flags); 1814 if (ret < 0) { 1815 goto fail; 1816 } 1817 1818 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1819 ret = bdrv_aligned_preadv(child, &req, offset, bytes, 1820 bs->bl.request_alignment, 1821 qiov, qiov_offset, flags); 1822 tracked_request_end(&req); 1823 bdrv_padding_destroy(&pad); 1824 1825 fail: 1826 bdrv_dec_in_flight(bs); 1827 1828 return ret; 1829 } 1830 1831 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1832 int64_t offset, int64_t bytes, BdrvRequestFlags flags) 1833 { 1834 BlockDriver *drv = bs->drv; 1835 QEMUIOVector qiov; 1836 void *buf = NULL; 1837 int ret = 0; 1838 bool need_flush = false; 1839 int head = 0; 1840 int tail = 0; 1841 1842 int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, 1843 INT64_MAX); 1844 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1845 bs->bl.request_alignment); 1846 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1847 1848 bdrv_check_request(offset, bytes, &error_abort); 1849 1850 if (!drv) { 1851 return -ENOMEDIUM; 1852 } 1853 1854 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1855 return -ENOTSUP; 1856 } 1857 1858 /* By definition there is no user buffer so this flag doesn't make sense */ 1859 if (flags & BDRV_REQ_REGISTERED_BUF) { 1860 return -EINVAL; 1861 } 1862 1863 /* Invalidate the cached block-status data range if this write overlaps */ 1864 bdrv_bsc_invalidate_range(bs, offset, bytes); 1865 1866 assert(alignment % bs->bl.request_alignment == 0); 1867 head = offset % alignment; 1868 tail = (offset + bytes) % alignment; 1869 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1870 assert(max_write_zeroes >= bs->bl.request_alignment); 1871 1872 while (bytes > 0 && !ret) { 1873 int64_t num = bytes; 1874 1875 /* Align request. Block drivers can expect the "bulk" of the request 1876 * to be aligned, and that unaligned requests do not cross cluster 1877 * boundaries. 1878 */ 1879 if (head) { 1880 /* Make a small request up to the first aligned sector. For 1881 * convenience, limit this request to max_transfer even if 1882 * we don't need to fall back to writes. */ 1883 num = MIN(MIN(bytes, max_transfer), alignment - head); 1884 head = (head + num) % alignment; 1885 assert(num < max_write_zeroes); 1886 } else if (tail && num > alignment) { 1887 /* Shorten the request to the last aligned sector. */ 1888 num -= tail; 1889 } 1890 1891 /* limit request size */ 1892 if (num > max_write_zeroes) { 1893 num = max_write_zeroes; 1894 } 1895 1896 ret = -ENOTSUP; 1897 /* First try the efficient write zeroes operation */ 1898 if (drv->bdrv_co_pwrite_zeroes) { 1899 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1900 flags & bs->supported_zero_flags); 1901 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1902 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1903 need_flush = true; 1904 } 1905 } else { 1906 assert(!bs->supported_zero_flags); 1907 } 1908 1909 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 1910 /* Fall back to bounce buffer if write zeroes is unsupported */ 1911 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1912 1913 if ((flags & BDRV_REQ_FUA) && 1914 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1915 /* No need for bdrv_driver_pwrite() to do a fallback 1916 * flush on each chunk; use just one at the end */ 1917 write_flags &= ~BDRV_REQ_FUA; 1918 need_flush = true; 1919 } 1920 num = MIN(num, max_transfer); 1921 if (buf == NULL) { 1922 buf = qemu_try_blockalign0(bs, num); 1923 if (buf == NULL) { 1924 ret = -ENOMEM; 1925 goto fail; 1926 } 1927 } 1928 qemu_iovec_init_buf(&qiov, buf, num); 1929 1930 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 1931 1932 /* Keep bounce buffer around if it is big enough for all 1933 * all future requests. 1934 */ 1935 if (num < max_transfer) { 1936 qemu_vfree(buf); 1937 buf = NULL; 1938 } 1939 } 1940 1941 offset += num; 1942 bytes -= num; 1943 } 1944 1945 fail: 1946 if (ret == 0 && need_flush) { 1947 ret = bdrv_co_flush(bs); 1948 } 1949 qemu_vfree(buf); 1950 return ret; 1951 } 1952 1953 static inline int coroutine_fn 1954 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes, 1955 BdrvTrackedRequest *req, int flags) 1956 { 1957 BlockDriverState *bs = child->bs; 1958 1959 bdrv_check_request(offset, bytes, &error_abort); 1960 1961 if (bdrv_is_read_only(bs)) { 1962 return -EPERM; 1963 } 1964 1965 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1966 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1967 assert(!(flags & ~BDRV_REQ_MASK)); 1968 assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING))); 1969 1970 if (flags & BDRV_REQ_SERIALISING) { 1971 QEMU_LOCK_GUARD(&bs->reqs_lock); 1972 1973 tracked_request_set_serialising(req, bdrv_get_cluster_size(bs)); 1974 1975 if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) { 1976 return -EBUSY; 1977 } 1978 1979 bdrv_wait_serialising_requests_locked(req); 1980 } else { 1981 bdrv_wait_serialising_requests(req); 1982 } 1983 1984 assert(req->overlap_offset <= offset); 1985 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1986 assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE || 1987 child->perm & BLK_PERM_RESIZE); 1988 1989 switch (req->type) { 1990 case BDRV_TRACKED_WRITE: 1991 case BDRV_TRACKED_DISCARD: 1992 if (flags & BDRV_REQ_WRITE_UNCHANGED) { 1993 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1994 } else { 1995 assert(child->perm & BLK_PERM_WRITE); 1996 } 1997 bdrv_write_threshold_check_write(bs, offset, bytes); 1998 return 0; 1999 case BDRV_TRACKED_TRUNCATE: 2000 assert(child->perm & BLK_PERM_RESIZE); 2001 return 0; 2002 default: 2003 abort(); 2004 } 2005 } 2006 2007 static inline void coroutine_fn 2008 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes, 2009 BdrvTrackedRequest *req, int ret) 2010 { 2011 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 2012 BlockDriverState *bs = child->bs; 2013 2014 bdrv_check_request(offset, bytes, &error_abort); 2015 2016 qatomic_inc(&bs->write_gen); 2017 2018 /* 2019 * Discard cannot extend the image, but in error handling cases, such as 2020 * when reverting a qcow2 cluster allocation, the discarded range can pass 2021 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 2022 * here. Instead, just skip it, since semantically a discard request 2023 * beyond EOF cannot expand the image anyway. 2024 */ 2025 if (ret == 0 && 2026 (req->type == BDRV_TRACKED_TRUNCATE || 2027 end_sector > bs->total_sectors) && 2028 req->type != BDRV_TRACKED_DISCARD) { 2029 bs->total_sectors = end_sector; 2030 bdrv_parent_cb_resize(bs); 2031 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 2032 } 2033 if (req->bytes) { 2034 switch (req->type) { 2035 case BDRV_TRACKED_WRITE: 2036 stat64_max(&bs->wr_highest_offset, offset + bytes); 2037 /* fall through, to set dirty bits */ 2038 case BDRV_TRACKED_DISCARD: 2039 bdrv_set_dirty(bs, offset, bytes); 2040 break; 2041 default: 2042 break; 2043 } 2044 } 2045 } 2046 2047 /* 2048 * Forwards an already correctly aligned write request to the BlockDriver, 2049 * after possibly fragmenting it. 2050 */ 2051 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 2052 BdrvTrackedRequest *req, int64_t offset, int64_t bytes, 2053 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, 2054 BdrvRequestFlags flags) 2055 { 2056 BlockDriverState *bs = child->bs; 2057 BlockDriver *drv = bs->drv; 2058 int ret; 2059 2060 int64_t bytes_remaining = bytes; 2061 int max_transfer; 2062 2063 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort); 2064 2065 if (!drv) { 2066 return -ENOMEDIUM; 2067 } 2068 2069 if (bdrv_has_readonly_bitmaps(bs)) { 2070 return -EPERM; 2071 } 2072 2073 assert(is_power_of_2(align)); 2074 assert((offset & (align - 1)) == 0); 2075 assert((bytes & (align - 1)) == 0); 2076 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 2077 align); 2078 2079 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 2080 2081 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 2082 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 2083 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 2084 flags |= BDRV_REQ_ZERO_WRITE; 2085 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 2086 flags |= BDRV_REQ_MAY_UNMAP; 2087 } 2088 } 2089 2090 if (ret < 0) { 2091 /* Do nothing, write notifier decided to fail this request */ 2092 } else if (flags & BDRV_REQ_ZERO_WRITE) { 2093 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 2094 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 2095 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 2096 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 2097 qiov, qiov_offset); 2098 } else if (bytes <= max_transfer) { 2099 bdrv_debug_event(bs, BLKDBG_PWRITEV); 2100 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 2101 } else { 2102 bdrv_debug_event(bs, BLKDBG_PWRITEV); 2103 while (bytes_remaining) { 2104 int num = MIN(bytes_remaining, max_transfer); 2105 int local_flags = flags; 2106 2107 assert(num); 2108 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 2109 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 2110 /* If FUA is going to be emulated by flush, we only 2111 * need to flush on the last iteration */ 2112 local_flags &= ~BDRV_REQ_FUA; 2113 } 2114 2115 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 2116 num, qiov, 2117 qiov_offset + bytes - bytes_remaining, 2118 local_flags); 2119 if (ret < 0) { 2120 break; 2121 } 2122 bytes_remaining -= num; 2123 } 2124 } 2125 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 2126 2127 if (ret >= 0) { 2128 ret = 0; 2129 } 2130 bdrv_co_write_req_finish(child, offset, bytes, req, ret); 2131 2132 return ret; 2133 } 2134 2135 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 2136 int64_t offset, 2137 int64_t bytes, 2138 BdrvRequestFlags flags, 2139 BdrvTrackedRequest *req) 2140 { 2141 BlockDriverState *bs = child->bs; 2142 QEMUIOVector local_qiov; 2143 uint64_t align = bs->bl.request_alignment; 2144 int ret = 0; 2145 bool padding; 2146 BdrvRequestPadding pad; 2147 2148 /* This flag doesn't make sense for padding or zero writes */ 2149 flags &= ~BDRV_REQ_REGISTERED_BUF; 2150 2151 padding = bdrv_init_padding(bs, offset, bytes, &pad); 2152 if (padding) { 2153 assert(!(flags & BDRV_REQ_NO_WAIT)); 2154 bdrv_make_request_serialising(req, align); 2155 2156 bdrv_padding_rmw_read(child, req, &pad, true); 2157 2158 if (pad.head || pad.merge_reads) { 2159 int64_t aligned_offset = offset & ~(align - 1); 2160 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 2161 2162 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 2163 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 2164 align, &local_qiov, 0, 2165 flags & ~BDRV_REQ_ZERO_WRITE); 2166 if (ret < 0 || pad.merge_reads) { 2167 /* Error or all work is done */ 2168 goto out; 2169 } 2170 offset += write_bytes - pad.head; 2171 bytes -= write_bytes - pad.head; 2172 } 2173 } 2174 2175 assert(!bytes || (offset & (align - 1)) == 0); 2176 if (bytes >= align) { 2177 /* Write the aligned part in the middle. */ 2178 int64_t aligned_bytes = bytes & ~(align - 1); 2179 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 2180 NULL, 0, flags); 2181 if (ret < 0) { 2182 goto out; 2183 } 2184 bytes -= aligned_bytes; 2185 offset += aligned_bytes; 2186 } 2187 2188 assert(!bytes || (offset & (align - 1)) == 0); 2189 if (bytes) { 2190 assert(align == pad.tail + bytes); 2191 2192 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 2193 ret = bdrv_aligned_pwritev(child, req, offset, align, align, 2194 &local_qiov, 0, 2195 flags & ~BDRV_REQ_ZERO_WRITE); 2196 } 2197 2198 out: 2199 bdrv_padding_destroy(&pad); 2200 2201 return ret; 2202 } 2203 2204 /* 2205 * Handle a write request in coroutine context 2206 */ 2207 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 2208 int64_t offset, int64_t bytes, QEMUIOVector *qiov, 2209 BdrvRequestFlags flags) 2210 { 2211 IO_CODE(); 2212 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 2213 } 2214 2215 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 2216 int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset, 2217 BdrvRequestFlags flags) 2218 { 2219 BlockDriverState *bs = child->bs; 2220 BdrvTrackedRequest req; 2221 uint64_t align = bs->bl.request_alignment; 2222 BdrvRequestPadding pad; 2223 int ret; 2224 bool padded = false; 2225 IO_CODE(); 2226 2227 trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags); 2228 2229 if (!bdrv_is_inserted(bs)) { 2230 return -ENOMEDIUM; 2231 } 2232 2233 if (flags & BDRV_REQ_ZERO_WRITE) { 2234 ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL); 2235 } else { 2236 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset); 2237 } 2238 if (ret < 0) { 2239 return ret; 2240 } 2241 2242 /* If the request is misaligned then we can't make it efficient */ 2243 if ((flags & BDRV_REQ_NO_FALLBACK) && 2244 !QEMU_IS_ALIGNED(offset | bytes, align)) 2245 { 2246 return -ENOTSUP; 2247 } 2248 2249 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2250 /* 2251 * Aligning zero request is nonsense. Even if driver has special meaning 2252 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2253 * it to driver due to request_alignment. 2254 * 2255 * Still, no reason to return an error if someone do unaligned 2256 * zero-length write occasionally. 2257 */ 2258 return 0; 2259 } 2260 2261 if (!(flags & BDRV_REQ_ZERO_WRITE)) { 2262 /* 2263 * Pad request for following read-modify-write cycle. 2264 * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do 2265 * alignment only if there is no ZERO flag. 2266 */ 2267 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad, 2268 &padded, &flags); 2269 if (ret < 0) { 2270 return ret; 2271 } 2272 } 2273 2274 bdrv_inc_in_flight(bs); 2275 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 2276 2277 if (flags & BDRV_REQ_ZERO_WRITE) { 2278 assert(!padded); 2279 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 2280 goto out; 2281 } 2282 2283 if (padded) { 2284 /* 2285 * Request was unaligned to request_alignment and therefore 2286 * padded. We are going to do read-modify-write, and must 2287 * serialize the request to prevent interactions of the 2288 * widened region with other transactions. 2289 */ 2290 assert(!(flags & BDRV_REQ_NO_WAIT)); 2291 bdrv_make_request_serialising(&req, align); 2292 bdrv_padding_rmw_read(child, &req, &pad, false); 2293 } 2294 2295 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 2296 qiov, qiov_offset, flags); 2297 2298 bdrv_padding_destroy(&pad); 2299 2300 out: 2301 tracked_request_end(&req); 2302 bdrv_dec_in_flight(bs); 2303 2304 return ret; 2305 } 2306 2307 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2308 int64_t bytes, BdrvRequestFlags flags) 2309 { 2310 IO_CODE(); 2311 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 2312 2313 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 2314 flags &= ~BDRV_REQ_MAY_UNMAP; 2315 } 2316 2317 return bdrv_co_pwritev(child, offset, bytes, NULL, 2318 BDRV_REQ_ZERO_WRITE | flags); 2319 } 2320 2321 /* 2322 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 2323 */ 2324 int bdrv_flush_all(void) 2325 { 2326 BdrvNextIterator it; 2327 BlockDriverState *bs = NULL; 2328 int result = 0; 2329 2330 GLOBAL_STATE_CODE(); 2331 2332 /* 2333 * bdrv queue is managed by record/replay, 2334 * creating new flush request for stopping 2335 * the VM may break the determinism 2336 */ 2337 if (replay_events_enabled()) { 2338 return result; 2339 } 2340 2341 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 2342 AioContext *aio_context = bdrv_get_aio_context(bs); 2343 int ret; 2344 2345 aio_context_acquire(aio_context); 2346 ret = bdrv_flush(bs); 2347 if (ret < 0 && !result) { 2348 result = ret; 2349 } 2350 aio_context_release(aio_context); 2351 } 2352 2353 return result; 2354 } 2355 2356 /* 2357 * Returns the allocation status of the specified sectors. 2358 * Drivers not implementing the functionality are assumed to not support 2359 * backing files, hence all their sectors are reported as allocated. 2360 * 2361 * If 'want_zero' is true, the caller is querying for mapping 2362 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 2363 * _ZERO where possible; otherwise, the result favors larger 'pnum', 2364 * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2365 * 2366 * If 'offset' is beyond the end of the disk image the return value is 2367 * BDRV_BLOCK_EOF and 'pnum' is set to 0. 2368 * 2369 * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2370 * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2371 * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 2372 * 2373 * 'pnum' is set to the number of bytes (including and immediately 2374 * following the specified offset) that are easily known to be in the 2375 * same allocated/unallocated state. Note that a second call starting 2376 * at the original offset plus returned pnum may have the same status. 2377 * The returned value is non-zero on success except at end-of-file. 2378 * 2379 * Returns negative errno on failure. Otherwise, if the 2380 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 2381 * set to the host mapping and BDS corresponding to the guest offset. 2382 */ 2383 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, 2384 bool want_zero, 2385 int64_t offset, int64_t bytes, 2386 int64_t *pnum, int64_t *map, 2387 BlockDriverState **file) 2388 { 2389 int64_t total_size; 2390 int64_t n; /* bytes */ 2391 int ret; 2392 int64_t local_map = 0; 2393 BlockDriverState *local_file = NULL; 2394 int64_t aligned_offset, aligned_bytes; 2395 uint32_t align; 2396 bool has_filtered_child; 2397 2398 assert(pnum); 2399 *pnum = 0; 2400 total_size = bdrv_getlength(bs); 2401 if (total_size < 0) { 2402 ret = total_size; 2403 goto early_out; 2404 } 2405 2406 if (offset >= total_size) { 2407 ret = BDRV_BLOCK_EOF; 2408 goto early_out; 2409 } 2410 if (!bytes) { 2411 ret = 0; 2412 goto early_out; 2413 } 2414 2415 n = total_size - offset; 2416 if (n < bytes) { 2417 bytes = n; 2418 } 2419 2420 /* Must be non-NULL or bdrv_getlength() would have failed */ 2421 assert(bs->drv); 2422 has_filtered_child = bdrv_filter_child(bs); 2423 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 2424 *pnum = bytes; 2425 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 2426 if (offset + bytes == total_size) { 2427 ret |= BDRV_BLOCK_EOF; 2428 } 2429 if (bs->drv->protocol_name) { 2430 ret |= BDRV_BLOCK_OFFSET_VALID; 2431 local_map = offset; 2432 local_file = bs; 2433 } 2434 goto early_out; 2435 } 2436 2437 bdrv_inc_in_flight(bs); 2438 2439 /* Round out to request_alignment boundaries */ 2440 align = bs->bl.request_alignment; 2441 aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2442 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2443 2444 if (bs->drv->bdrv_co_block_status) { 2445 /* 2446 * Use the block-status cache only for protocol nodes: Format 2447 * drivers are generally quick to inquire the status, but protocol 2448 * drivers often need to get information from outside of qemu, so 2449 * we do not have control over the actual implementation. There 2450 * have been cases where inquiring the status took an unreasonably 2451 * long time, and we can do nothing in qemu to fix it. 2452 * This is especially problematic for images with large data areas, 2453 * because finding the few holes in them and giving them special 2454 * treatment does not gain much performance. Therefore, we try to 2455 * cache the last-identified data region. 2456 * 2457 * Second, limiting ourselves to protocol nodes allows us to assume 2458 * the block status for data regions to be DATA | OFFSET_VALID, and 2459 * that the host offset is the same as the guest offset. 2460 * 2461 * Note that it is possible that external writers zero parts of 2462 * the cached regions without the cache being invalidated, and so 2463 * we may report zeroes as data. This is not catastrophic, 2464 * however, because reporting zeroes as data is fine. 2465 */ 2466 if (QLIST_EMPTY(&bs->children) && 2467 bdrv_bsc_is_data(bs, aligned_offset, pnum)) 2468 { 2469 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID; 2470 local_file = bs; 2471 local_map = aligned_offset; 2472 } else { 2473 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 2474 aligned_bytes, pnum, &local_map, 2475 &local_file); 2476 2477 /* 2478 * Note that checking QLIST_EMPTY(&bs->children) is also done when 2479 * the cache is queried above. Technically, we do not need to check 2480 * it here; the worst that can happen is that we fill the cache for 2481 * non-protocol nodes, and then it is never used. However, filling 2482 * the cache requires an RCU update, so double check here to avoid 2483 * such an update if possible. 2484 * 2485 * Check want_zero, because we only want to update the cache when we 2486 * have accurate information about what is zero and what is data. 2487 */ 2488 if (want_zero && 2489 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) && 2490 QLIST_EMPTY(&bs->children)) 2491 { 2492 /* 2493 * When a protocol driver reports BLOCK_OFFSET_VALID, the 2494 * returned local_map value must be the same as the offset we 2495 * have passed (aligned_offset), and local_bs must be the node 2496 * itself. 2497 * Assert this, because we follow this rule when reading from 2498 * the cache (see the `local_file = bs` and 2499 * `local_map = aligned_offset` assignments above), and the 2500 * result the cache delivers must be the same as the driver 2501 * would deliver. 2502 */ 2503 assert(local_file == bs); 2504 assert(local_map == aligned_offset); 2505 bdrv_bsc_fill(bs, aligned_offset, *pnum); 2506 } 2507 } 2508 } else { 2509 /* Default code for filters */ 2510 2511 local_file = bdrv_filter_bs(bs); 2512 assert(local_file); 2513 2514 *pnum = aligned_bytes; 2515 local_map = aligned_offset; 2516 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2517 } 2518 if (ret < 0) { 2519 *pnum = 0; 2520 goto out; 2521 } 2522 2523 /* 2524 * The driver's result must be a non-zero multiple of request_alignment. 2525 * Clamp pnum and adjust map to original request. 2526 */ 2527 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2528 align > offset - aligned_offset); 2529 if (ret & BDRV_BLOCK_RECURSE) { 2530 assert(ret & BDRV_BLOCK_DATA); 2531 assert(ret & BDRV_BLOCK_OFFSET_VALID); 2532 assert(!(ret & BDRV_BLOCK_ZERO)); 2533 } 2534 2535 *pnum -= offset - aligned_offset; 2536 if (*pnum > bytes) { 2537 *pnum = bytes; 2538 } 2539 if (ret & BDRV_BLOCK_OFFSET_VALID) { 2540 local_map += offset - aligned_offset; 2541 } 2542 2543 if (ret & BDRV_BLOCK_RAW) { 2544 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 2545 ret = bdrv_co_block_status(local_file, want_zero, local_map, 2546 *pnum, pnum, &local_map, &local_file); 2547 goto out; 2548 } 2549 2550 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 2551 ret |= BDRV_BLOCK_ALLOCATED; 2552 } else if (bs->drv->supports_backing) { 2553 BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2554 2555 if (!cow_bs) { 2556 ret |= BDRV_BLOCK_ZERO; 2557 } else if (want_zero) { 2558 int64_t size2 = bdrv_getlength(cow_bs); 2559 2560 if (size2 >= 0 && offset >= size2) { 2561 ret |= BDRV_BLOCK_ZERO; 2562 } 2563 } 2564 } 2565 2566 if (want_zero && ret & BDRV_BLOCK_RECURSE && 2567 local_file && local_file != bs && 2568 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 2569 (ret & BDRV_BLOCK_OFFSET_VALID)) { 2570 int64_t file_pnum; 2571 int ret2; 2572 2573 ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 2574 *pnum, &file_pnum, NULL, NULL); 2575 if (ret2 >= 0) { 2576 /* Ignore errors. This is just providing extra information, it 2577 * is useful but not necessary. 2578 */ 2579 if (ret2 & BDRV_BLOCK_EOF && 2580 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2581 /* 2582 * It is valid for the format block driver to read 2583 * beyond the end of the underlying file's current 2584 * size; such areas read as zero. 2585 */ 2586 ret |= BDRV_BLOCK_ZERO; 2587 } else { 2588 /* Limit request to the range reported by the protocol driver */ 2589 *pnum = file_pnum; 2590 ret |= (ret2 & BDRV_BLOCK_ZERO); 2591 } 2592 } 2593 } 2594 2595 out: 2596 bdrv_dec_in_flight(bs); 2597 if (ret >= 0 && offset + *pnum == total_size) { 2598 ret |= BDRV_BLOCK_EOF; 2599 } 2600 early_out: 2601 if (file) { 2602 *file = local_file; 2603 } 2604 if (map) { 2605 *map = local_map; 2606 } 2607 return ret; 2608 } 2609 2610 int coroutine_fn 2611 bdrv_co_common_block_status_above(BlockDriverState *bs, 2612 BlockDriverState *base, 2613 bool include_base, 2614 bool want_zero, 2615 int64_t offset, 2616 int64_t bytes, 2617 int64_t *pnum, 2618 int64_t *map, 2619 BlockDriverState **file, 2620 int *depth) 2621 { 2622 int ret; 2623 BlockDriverState *p; 2624 int64_t eof = 0; 2625 int dummy; 2626 IO_CODE(); 2627 2628 assert(!include_base || base); /* Can't include NULL base */ 2629 2630 if (!depth) { 2631 depth = &dummy; 2632 } 2633 *depth = 0; 2634 2635 if (!include_base && bs == base) { 2636 *pnum = bytes; 2637 return 0; 2638 } 2639 2640 ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file); 2641 ++*depth; 2642 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 2643 return ret; 2644 } 2645 2646 if (ret & BDRV_BLOCK_EOF) { 2647 eof = offset + *pnum; 2648 } 2649 2650 assert(*pnum <= bytes); 2651 bytes = *pnum; 2652 2653 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 2654 p = bdrv_filter_or_cow_bs(p)) 2655 { 2656 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 2657 file); 2658 ++*depth; 2659 if (ret < 0) { 2660 return ret; 2661 } 2662 if (*pnum == 0) { 2663 /* 2664 * The top layer deferred to this layer, and because this layer is 2665 * short, any zeroes that we synthesize beyond EOF behave as if they 2666 * were allocated at this layer. 2667 * 2668 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 2669 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2670 * below. 2671 */ 2672 assert(ret & BDRV_BLOCK_EOF); 2673 *pnum = bytes; 2674 if (file) { 2675 *file = p; 2676 } 2677 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2678 break; 2679 } 2680 if (ret & BDRV_BLOCK_ALLOCATED) { 2681 /* 2682 * We've found the node and the status, we must break. 2683 * 2684 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 2685 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2686 * below. 2687 */ 2688 ret &= ~BDRV_BLOCK_EOF; 2689 break; 2690 } 2691 2692 if (p == base) { 2693 assert(include_base); 2694 break; 2695 } 2696 2697 /* 2698 * OK, [offset, offset + *pnum) region is unallocated on this layer, 2699 * let's continue the diving. 2700 */ 2701 assert(*pnum <= bytes); 2702 bytes = *pnum; 2703 } 2704 2705 if (offset + *pnum == eof) { 2706 ret |= BDRV_BLOCK_EOF; 2707 } 2708 2709 return ret; 2710 } 2711 2712 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 2713 int64_t offset, int64_t bytes, int64_t *pnum, 2714 int64_t *map, BlockDriverState **file) 2715 { 2716 IO_CODE(); 2717 return bdrv_common_block_status_above(bs, base, false, true, offset, bytes, 2718 pnum, map, file, NULL); 2719 } 2720 2721 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2722 int64_t *pnum, int64_t *map, BlockDriverState **file) 2723 { 2724 IO_CODE(); 2725 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 2726 offset, bytes, pnum, map, file); 2727 } 2728 2729 /* 2730 * Check @bs (and its backing chain) to see if the range defined 2731 * by @offset and @bytes is known to read as zeroes. 2732 * Return 1 if that is the case, 0 otherwise and -errno on error. 2733 * This test is meant to be fast rather than accurate so returning 0 2734 * does not guarantee non-zero data. 2735 */ 2736 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset, 2737 int64_t bytes) 2738 { 2739 int ret; 2740 int64_t pnum = bytes; 2741 IO_CODE(); 2742 2743 if (!bytes) { 2744 return 1; 2745 } 2746 2747 ret = bdrv_common_block_status_above(bs, NULL, false, false, offset, 2748 bytes, &pnum, NULL, NULL, NULL); 2749 2750 if (ret < 0) { 2751 return ret; 2752 } 2753 2754 return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO); 2755 } 2756 2757 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2758 int64_t bytes, int64_t *pnum) 2759 { 2760 int ret; 2761 int64_t dummy; 2762 IO_CODE(); 2763 2764 ret = bdrv_common_block_status_above(bs, bs, true, false, offset, 2765 bytes, pnum ? pnum : &dummy, NULL, 2766 NULL, NULL); 2767 if (ret < 0) { 2768 return ret; 2769 } 2770 return !!(ret & BDRV_BLOCK_ALLOCATED); 2771 } 2772 2773 /* 2774 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 2775 * 2776 * Return a positive depth if (a prefix of) the given range is allocated 2777 * in any image between BASE and TOP (BASE is only included if include_base 2778 * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth. 2779 * BASE can be NULL to check if the given offset is allocated in any 2780 * image of the chain. Return 0 otherwise, or negative errno on 2781 * failure. 2782 * 2783 * 'pnum' is set to the number of bytes (including and immediately 2784 * following the specified offset) that are known to be in the same 2785 * allocated/unallocated state. Note that a subsequent call starting 2786 * at 'offset + *pnum' may return the same allocation status (in other 2787 * words, the result is not necessarily the maximum possible range); 2788 * but 'pnum' will only be 0 when end of file is reached. 2789 */ 2790 int bdrv_is_allocated_above(BlockDriverState *top, 2791 BlockDriverState *base, 2792 bool include_base, int64_t offset, 2793 int64_t bytes, int64_t *pnum) 2794 { 2795 int depth; 2796 int ret = bdrv_common_block_status_above(top, base, include_base, false, 2797 offset, bytes, pnum, NULL, NULL, 2798 &depth); 2799 IO_CODE(); 2800 if (ret < 0) { 2801 return ret; 2802 } 2803 2804 if (ret & BDRV_BLOCK_ALLOCATED) { 2805 return depth; 2806 } 2807 return 0; 2808 } 2809 2810 int coroutine_fn 2811 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2812 { 2813 BlockDriver *drv = bs->drv; 2814 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2815 int ret; 2816 IO_CODE(); 2817 2818 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2819 if (ret < 0) { 2820 return ret; 2821 } 2822 2823 if (!drv) { 2824 return -ENOMEDIUM; 2825 } 2826 2827 bdrv_inc_in_flight(bs); 2828 2829 if (drv->bdrv_load_vmstate) { 2830 ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2831 } else if (child_bs) { 2832 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 2833 } else { 2834 ret = -ENOTSUP; 2835 } 2836 2837 bdrv_dec_in_flight(bs); 2838 2839 return ret; 2840 } 2841 2842 int coroutine_fn 2843 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2844 { 2845 BlockDriver *drv = bs->drv; 2846 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2847 int ret; 2848 IO_CODE(); 2849 2850 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL); 2851 if (ret < 0) { 2852 return ret; 2853 } 2854 2855 if (!drv) { 2856 return -ENOMEDIUM; 2857 } 2858 2859 bdrv_inc_in_flight(bs); 2860 2861 if (drv->bdrv_save_vmstate) { 2862 ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2863 } else if (child_bs) { 2864 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2865 } else { 2866 ret = -ENOTSUP; 2867 } 2868 2869 bdrv_dec_in_flight(bs); 2870 2871 return ret; 2872 } 2873 2874 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 2875 int64_t pos, int size) 2876 { 2877 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2878 int ret = bdrv_writev_vmstate(bs, &qiov, pos); 2879 IO_CODE(); 2880 2881 return ret < 0 ? ret : size; 2882 } 2883 2884 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 2885 int64_t pos, int size) 2886 { 2887 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2888 int ret = bdrv_readv_vmstate(bs, &qiov, pos); 2889 IO_CODE(); 2890 2891 return ret < 0 ? ret : size; 2892 } 2893 2894 /**************************************************************/ 2895 /* async I/Os */ 2896 2897 void bdrv_aio_cancel(BlockAIOCB *acb) 2898 { 2899 IO_CODE(); 2900 qemu_aio_ref(acb); 2901 bdrv_aio_cancel_async(acb); 2902 while (acb->refcnt > 1) { 2903 if (acb->aiocb_info->get_aio_context) { 2904 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2905 } else if (acb->bs) { 2906 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 2907 * assert that we're not using an I/O thread. Thread-safe 2908 * code should use bdrv_aio_cancel_async exclusively. 2909 */ 2910 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 2911 aio_poll(bdrv_get_aio_context(acb->bs), true); 2912 } else { 2913 abort(); 2914 } 2915 } 2916 qemu_aio_unref(acb); 2917 } 2918 2919 /* Async version of aio cancel. The caller is not blocked if the acb implements 2920 * cancel_async, otherwise we do nothing and let the request normally complete. 2921 * In either case the completion callback must be called. */ 2922 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2923 { 2924 IO_CODE(); 2925 if (acb->aiocb_info->cancel_async) { 2926 acb->aiocb_info->cancel_async(acb); 2927 } 2928 } 2929 2930 /**************************************************************/ 2931 /* Coroutine block device emulation */ 2932 2933 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2934 { 2935 BdrvChild *primary_child = bdrv_primary_child(bs); 2936 BdrvChild *child; 2937 int current_gen; 2938 int ret = 0; 2939 IO_CODE(); 2940 2941 bdrv_inc_in_flight(bs); 2942 2943 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2944 bdrv_is_sg(bs)) { 2945 goto early_exit; 2946 } 2947 2948 qemu_co_mutex_lock(&bs->reqs_lock); 2949 current_gen = qatomic_read(&bs->write_gen); 2950 2951 /* Wait until any previous flushes are completed */ 2952 while (bs->active_flush_req) { 2953 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 2954 } 2955 2956 /* Flushes reach this point in nondecreasing current_gen order. */ 2957 bs->active_flush_req = true; 2958 qemu_co_mutex_unlock(&bs->reqs_lock); 2959 2960 /* Write back all layers by calling one driver function */ 2961 if (bs->drv->bdrv_co_flush) { 2962 ret = bs->drv->bdrv_co_flush(bs); 2963 goto out; 2964 } 2965 2966 /* Write back cached data to the OS even with cache=unsafe */ 2967 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 2968 if (bs->drv->bdrv_co_flush_to_os) { 2969 ret = bs->drv->bdrv_co_flush_to_os(bs); 2970 if (ret < 0) { 2971 goto out; 2972 } 2973 } 2974 2975 /* But don't actually force it to the disk with cache=unsafe */ 2976 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2977 goto flush_children; 2978 } 2979 2980 /* Check if we really need to flush anything */ 2981 if (bs->flushed_gen == current_gen) { 2982 goto flush_children; 2983 } 2984 2985 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 2986 if (!bs->drv) { 2987 /* bs->drv->bdrv_co_flush() might have ejected the BDS 2988 * (even in case of apparent success) */ 2989 ret = -ENOMEDIUM; 2990 goto out; 2991 } 2992 if (bs->drv->bdrv_co_flush_to_disk) { 2993 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2994 } else if (bs->drv->bdrv_aio_flush) { 2995 BlockAIOCB *acb; 2996 CoroutineIOCompletion co = { 2997 .coroutine = qemu_coroutine_self(), 2998 }; 2999 3000 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 3001 if (acb == NULL) { 3002 ret = -EIO; 3003 } else { 3004 qemu_coroutine_yield(); 3005 ret = co.ret; 3006 } 3007 } else { 3008 /* 3009 * Some block drivers always operate in either writethrough or unsafe 3010 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 3011 * know how the server works (because the behaviour is hardcoded or 3012 * depends on server-side configuration), so we can't ensure that 3013 * everything is safe on disk. Returning an error doesn't work because 3014 * that would break guests even if the server operates in writethrough 3015 * mode. 3016 * 3017 * Let's hope the user knows what he's doing. 3018 */ 3019 ret = 0; 3020 } 3021 3022 if (ret < 0) { 3023 goto out; 3024 } 3025 3026 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 3027 * in the case of cache=unsafe, so there are no useless flushes. 3028 */ 3029 flush_children: 3030 ret = 0; 3031 QLIST_FOREACH(child, &bs->children, next) { 3032 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 3033 int this_child_ret = bdrv_co_flush(child->bs); 3034 if (!ret) { 3035 ret = this_child_ret; 3036 } 3037 } 3038 } 3039 3040 out: 3041 /* Notify any pending flushes that we have completed */ 3042 if (ret == 0) { 3043 bs->flushed_gen = current_gen; 3044 } 3045 3046 qemu_co_mutex_lock(&bs->reqs_lock); 3047 bs->active_flush_req = false; 3048 /* Return value is ignored - it's ok if wait queue is empty */ 3049 qemu_co_queue_next(&bs->flush_queue); 3050 qemu_co_mutex_unlock(&bs->reqs_lock); 3051 3052 early_exit: 3053 bdrv_dec_in_flight(bs); 3054 return ret; 3055 } 3056 3057 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 3058 int64_t bytes) 3059 { 3060 BdrvTrackedRequest req; 3061 int ret; 3062 int64_t max_pdiscard; 3063 int head, tail, align; 3064 BlockDriverState *bs = child->bs; 3065 IO_CODE(); 3066 3067 if (!bs || !bs->drv || !bdrv_is_inserted(bs)) { 3068 return -ENOMEDIUM; 3069 } 3070 3071 if (bdrv_has_readonly_bitmaps(bs)) { 3072 return -EPERM; 3073 } 3074 3075 ret = bdrv_check_request(offset, bytes, NULL); 3076 if (ret < 0) { 3077 return ret; 3078 } 3079 3080 /* Do nothing if disabled. */ 3081 if (!(bs->open_flags & BDRV_O_UNMAP)) { 3082 return 0; 3083 } 3084 3085 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 3086 return 0; 3087 } 3088 3089 /* Invalidate the cached block-status data range if this discard overlaps */ 3090 bdrv_bsc_invalidate_range(bs, offset, bytes); 3091 3092 /* Discard is advisory, but some devices track and coalesce 3093 * unaligned requests, so we must pass everything down rather than 3094 * round here. Still, most devices will just silently ignore 3095 * unaligned requests (by returning -ENOTSUP), so we must fragment 3096 * the request accordingly. */ 3097 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 3098 assert(align % bs->bl.request_alignment == 0); 3099 head = offset % align; 3100 tail = (offset + bytes) % align; 3101 3102 bdrv_inc_in_flight(bs); 3103 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 3104 3105 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 3106 if (ret < 0) { 3107 goto out; 3108 } 3109 3110 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX), 3111 align); 3112 assert(max_pdiscard >= bs->bl.request_alignment); 3113 3114 while (bytes > 0) { 3115 int64_t num = bytes; 3116 3117 if (head) { 3118 /* Make small requests to get to alignment boundaries. */ 3119 num = MIN(bytes, align - head); 3120 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 3121 num %= bs->bl.request_alignment; 3122 } 3123 head = (head + num) % align; 3124 assert(num < max_pdiscard); 3125 } else if (tail) { 3126 if (num > align) { 3127 /* Shorten the request to the last aligned cluster. */ 3128 num -= tail; 3129 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 3130 tail > bs->bl.request_alignment) { 3131 tail %= bs->bl.request_alignment; 3132 num -= tail; 3133 } 3134 } 3135 /* limit request size */ 3136 if (num > max_pdiscard) { 3137 num = max_pdiscard; 3138 } 3139 3140 if (!bs->drv) { 3141 ret = -ENOMEDIUM; 3142 goto out; 3143 } 3144 if (bs->drv->bdrv_co_pdiscard) { 3145 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 3146 } else { 3147 BlockAIOCB *acb; 3148 CoroutineIOCompletion co = { 3149 .coroutine = qemu_coroutine_self(), 3150 }; 3151 3152 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 3153 bdrv_co_io_em_complete, &co); 3154 if (acb == NULL) { 3155 ret = -EIO; 3156 goto out; 3157 } else { 3158 qemu_coroutine_yield(); 3159 ret = co.ret; 3160 } 3161 } 3162 if (ret && ret != -ENOTSUP) { 3163 goto out; 3164 } 3165 3166 offset += num; 3167 bytes -= num; 3168 } 3169 ret = 0; 3170 out: 3171 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 3172 tracked_request_end(&req); 3173 bdrv_dec_in_flight(bs); 3174 return ret; 3175 } 3176 3177 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 3178 { 3179 BlockDriver *drv = bs->drv; 3180 CoroutineIOCompletion co = { 3181 .coroutine = qemu_coroutine_self(), 3182 }; 3183 BlockAIOCB *acb; 3184 IO_CODE(); 3185 3186 bdrv_inc_in_flight(bs); 3187 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 3188 co.ret = -ENOTSUP; 3189 goto out; 3190 } 3191 3192 if (drv->bdrv_co_ioctl) { 3193 co.ret = drv->bdrv_co_ioctl(bs, req, buf); 3194 } else { 3195 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 3196 if (!acb) { 3197 co.ret = -ENOTSUP; 3198 goto out; 3199 } 3200 qemu_coroutine_yield(); 3201 } 3202 out: 3203 bdrv_dec_in_flight(bs); 3204 return co.ret; 3205 } 3206 3207 void *qemu_blockalign(BlockDriverState *bs, size_t size) 3208 { 3209 IO_CODE(); 3210 return qemu_memalign(bdrv_opt_mem_align(bs), size); 3211 } 3212 3213 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 3214 { 3215 IO_CODE(); 3216 return memset(qemu_blockalign(bs, size), 0, size); 3217 } 3218 3219 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 3220 { 3221 size_t align = bdrv_opt_mem_align(bs); 3222 IO_CODE(); 3223 3224 /* Ensure that NULL is never returned on success */ 3225 assert(align > 0); 3226 if (size == 0) { 3227 size = align; 3228 } 3229 3230 return qemu_try_memalign(align, size); 3231 } 3232 3233 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 3234 { 3235 void *mem = qemu_try_blockalign(bs, size); 3236 IO_CODE(); 3237 3238 if (mem) { 3239 memset(mem, 0, size); 3240 } 3241 3242 return mem; 3243 } 3244 3245 void bdrv_io_plug(BlockDriverState *bs) 3246 { 3247 BdrvChild *child; 3248 IO_CODE(); 3249 3250 QLIST_FOREACH(child, &bs->children, next) { 3251 bdrv_io_plug(child->bs); 3252 } 3253 3254 if (qatomic_fetch_inc(&bs->io_plugged) == 0) { 3255 BlockDriver *drv = bs->drv; 3256 if (drv && drv->bdrv_io_plug) { 3257 drv->bdrv_io_plug(bs); 3258 } 3259 } 3260 } 3261 3262 void bdrv_io_unplug(BlockDriverState *bs) 3263 { 3264 BdrvChild *child; 3265 IO_CODE(); 3266 3267 assert(bs->io_plugged); 3268 if (qatomic_fetch_dec(&bs->io_plugged) == 1) { 3269 BlockDriver *drv = bs->drv; 3270 if (drv && drv->bdrv_io_unplug) { 3271 drv->bdrv_io_unplug(bs); 3272 } 3273 } 3274 3275 QLIST_FOREACH(child, &bs->children, next) { 3276 bdrv_io_unplug(child->bs); 3277 } 3278 } 3279 3280 /* Helper that undoes bdrv_register_buf() when it fails partway through */ 3281 static void bdrv_register_buf_rollback(BlockDriverState *bs, 3282 void *host, 3283 size_t size, 3284 BdrvChild *final_child) 3285 { 3286 BdrvChild *child; 3287 3288 QLIST_FOREACH(child, &bs->children, next) { 3289 if (child == final_child) { 3290 break; 3291 } 3292 3293 bdrv_unregister_buf(child->bs, host, size); 3294 } 3295 3296 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3297 bs->drv->bdrv_unregister_buf(bs, host, size); 3298 } 3299 } 3300 3301 bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size, 3302 Error **errp) 3303 { 3304 BdrvChild *child; 3305 3306 GLOBAL_STATE_CODE(); 3307 if (bs->drv && bs->drv->bdrv_register_buf) { 3308 if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) { 3309 return false; 3310 } 3311 } 3312 QLIST_FOREACH(child, &bs->children, next) { 3313 if (!bdrv_register_buf(child->bs, host, size, errp)) { 3314 bdrv_register_buf_rollback(bs, host, size, child); 3315 return false; 3316 } 3317 } 3318 return true; 3319 } 3320 3321 void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size) 3322 { 3323 BdrvChild *child; 3324 3325 GLOBAL_STATE_CODE(); 3326 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3327 bs->drv->bdrv_unregister_buf(bs, host, size); 3328 } 3329 QLIST_FOREACH(child, &bs->children, next) { 3330 bdrv_unregister_buf(child->bs, host, size); 3331 } 3332 } 3333 3334 static int coroutine_fn bdrv_co_copy_range_internal( 3335 BdrvChild *src, int64_t src_offset, BdrvChild *dst, 3336 int64_t dst_offset, int64_t bytes, 3337 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 3338 bool recurse_src) 3339 { 3340 BdrvTrackedRequest req; 3341 int ret; 3342 3343 /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 3344 assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 3345 assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 3346 assert(!(read_flags & BDRV_REQ_NO_WAIT)); 3347 assert(!(write_flags & BDRV_REQ_NO_WAIT)); 3348 3349 if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) { 3350 return -ENOMEDIUM; 3351 } 3352 ret = bdrv_check_request32(dst_offset, bytes, NULL, 0); 3353 if (ret) { 3354 return ret; 3355 } 3356 if (write_flags & BDRV_REQ_ZERO_WRITE) { 3357 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3358 } 3359 3360 if (!src || !src->bs || !bdrv_is_inserted(src->bs)) { 3361 return -ENOMEDIUM; 3362 } 3363 ret = bdrv_check_request32(src_offset, bytes, NULL, 0); 3364 if (ret) { 3365 return ret; 3366 } 3367 3368 if (!src->bs->drv->bdrv_co_copy_range_from 3369 || !dst->bs->drv->bdrv_co_copy_range_to 3370 || src->bs->encrypted || dst->bs->encrypted) { 3371 return -ENOTSUP; 3372 } 3373 3374 if (recurse_src) { 3375 bdrv_inc_in_flight(src->bs); 3376 tracked_request_begin(&req, src->bs, src_offset, bytes, 3377 BDRV_TRACKED_READ); 3378 3379 /* BDRV_REQ_SERIALISING is only for write operation */ 3380 assert(!(read_flags & BDRV_REQ_SERIALISING)); 3381 bdrv_wait_serialising_requests(&req); 3382 3383 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3384 src, src_offset, 3385 dst, dst_offset, 3386 bytes, 3387 read_flags, write_flags); 3388 3389 tracked_request_end(&req); 3390 bdrv_dec_in_flight(src->bs); 3391 } else { 3392 bdrv_inc_in_flight(dst->bs); 3393 tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3394 BDRV_TRACKED_WRITE); 3395 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 3396 write_flags); 3397 if (!ret) { 3398 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3399 src, src_offset, 3400 dst, dst_offset, 3401 bytes, 3402 read_flags, write_flags); 3403 } 3404 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3405 tracked_request_end(&req); 3406 bdrv_dec_in_flight(dst->bs); 3407 } 3408 3409 return ret; 3410 } 3411 3412 /* Copy range from @src to @dst. 3413 * 3414 * See the comment of bdrv_co_copy_range for the parameter and return value 3415 * semantics. */ 3416 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset, 3417 BdrvChild *dst, int64_t dst_offset, 3418 int64_t bytes, 3419 BdrvRequestFlags read_flags, 3420 BdrvRequestFlags write_flags) 3421 { 3422 IO_CODE(); 3423 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3424 read_flags, write_flags); 3425 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3426 bytes, read_flags, write_flags, true); 3427 } 3428 3429 /* Copy range from @src to @dst. 3430 * 3431 * See the comment of bdrv_co_copy_range for the parameter and return value 3432 * semantics. */ 3433 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset, 3434 BdrvChild *dst, int64_t dst_offset, 3435 int64_t bytes, 3436 BdrvRequestFlags read_flags, 3437 BdrvRequestFlags write_flags) 3438 { 3439 IO_CODE(); 3440 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3441 read_flags, write_flags); 3442 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3443 bytes, read_flags, write_flags, false); 3444 } 3445 3446 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset, 3447 BdrvChild *dst, int64_t dst_offset, 3448 int64_t bytes, BdrvRequestFlags read_flags, 3449 BdrvRequestFlags write_flags) 3450 { 3451 IO_CODE(); 3452 return bdrv_co_copy_range_from(src, src_offset, 3453 dst, dst_offset, 3454 bytes, read_flags, write_flags); 3455 } 3456 3457 static void bdrv_parent_cb_resize(BlockDriverState *bs) 3458 { 3459 BdrvChild *c; 3460 QLIST_FOREACH(c, &bs->parents, next_parent) { 3461 if (c->klass->resize) { 3462 c->klass->resize(c); 3463 } 3464 } 3465 } 3466 3467 /** 3468 * Truncate file to 'offset' bytes (needed only for file protocols) 3469 * 3470 * If 'exact' is true, the file must be resized to exactly the given 3471 * 'offset'. Otherwise, it is sufficient for the node to be at least 3472 * 'offset' bytes in length. 3473 */ 3474 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 3475 PreallocMode prealloc, BdrvRequestFlags flags, 3476 Error **errp) 3477 { 3478 BlockDriverState *bs = child->bs; 3479 BdrvChild *filtered, *backing; 3480 BlockDriver *drv = bs->drv; 3481 BdrvTrackedRequest req; 3482 int64_t old_size, new_bytes; 3483 int ret; 3484 IO_CODE(); 3485 3486 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 3487 if (!drv) { 3488 error_setg(errp, "No medium inserted"); 3489 return -ENOMEDIUM; 3490 } 3491 if (offset < 0) { 3492 error_setg(errp, "Image size cannot be negative"); 3493 return -EINVAL; 3494 } 3495 3496 ret = bdrv_check_request(offset, 0, errp); 3497 if (ret < 0) { 3498 return ret; 3499 } 3500 3501 old_size = bdrv_getlength(bs); 3502 if (old_size < 0) { 3503 error_setg_errno(errp, -old_size, "Failed to get old image size"); 3504 return old_size; 3505 } 3506 3507 if (bdrv_is_read_only(bs)) { 3508 error_setg(errp, "Image is read-only"); 3509 return -EACCES; 3510 } 3511 3512 if (offset > old_size) { 3513 new_bytes = offset - old_size; 3514 } else { 3515 new_bytes = 0; 3516 } 3517 3518 bdrv_inc_in_flight(bs); 3519 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 3520 BDRV_TRACKED_TRUNCATE); 3521 3522 /* If we are growing the image and potentially using preallocation for the 3523 * new area, we need to make sure that no write requests are made to it 3524 * concurrently or they might be overwritten by preallocation. */ 3525 if (new_bytes) { 3526 bdrv_make_request_serialising(&req, 1); 3527 } 3528 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3529 0); 3530 if (ret < 0) { 3531 error_setg_errno(errp, -ret, 3532 "Failed to prepare request for truncation"); 3533 goto out; 3534 } 3535 3536 filtered = bdrv_filter_child(bs); 3537 backing = bdrv_cow_child(bs); 3538 3539 /* 3540 * If the image has a backing file that is large enough that it would 3541 * provide data for the new area, we cannot leave it unallocated because 3542 * then the backing file content would become visible. Instead, zero-fill 3543 * the new area. 3544 * 3545 * Note that if the image has a backing file, but was opened without the 3546 * backing file, taking care of keeping things consistent with that backing 3547 * file is the user's responsibility. 3548 */ 3549 if (new_bytes && backing) { 3550 int64_t backing_len; 3551 3552 backing_len = bdrv_getlength(backing->bs); 3553 if (backing_len < 0) { 3554 ret = backing_len; 3555 error_setg_errno(errp, -ret, "Could not get backing file size"); 3556 goto out; 3557 } 3558 3559 if (backing_len > old_size) { 3560 flags |= BDRV_REQ_ZERO_WRITE; 3561 } 3562 } 3563 3564 if (drv->bdrv_co_truncate) { 3565 if (flags & ~bs->supported_truncate_flags) { 3566 error_setg(errp, "Block driver does not support requested flags"); 3567 ret = -ENOTSUP; 3568 goto out; 3569 } 3570 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 3571 } else if (filtered) { 3572 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 3573 } else { 3574 error_setg(errp, "Image format driver does not support resize"); 3575 ret = -ENOTSUP; 3576 goto out; 3577 } 3578 if (ret < 0) { 3579 goto out; 3580 } 3581 3582 ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 3583 if (ret < 0) { 3584 error_setg_errno(errp, -ret, "Could not refresh total sector count"); 3585 } else { 3586 offset = bs->total_sectors * BDRV_SECTOR_SIZE; 3587 } 3588 /* It's possible that truncation succeeded but refresh_total_sectors 3589 * failed, but the latter doesn't affect how we should finish the request. 3590 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */ 3591 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 3592 3593 out: 3594 tracked_request_end(&req); 3595 bdrv_dec_in_flight(bs); 3596 3597 return ret; 3598 } 3599 3600 void bdrv_cancel_in_flight(BlockDriverState *bs) 3601 { 3602 GLOBAL_STATE_CODE(); 3603 if (!bs || !bs->drv) { 3604 return; 3605 } 3606 3607 if (bs->drv->bdrv_cancel_in_flight) { 3608 bs->drv->bdrv_cancel_in_flight(bs); 3609 } 3610 } 3611 3612 int coroutine_fn 3613 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes, 3614 QEMUIOVector *qiov, size_t qiov_offset) 3615 { 3616 BlockDriverState *bs = child->bs; 3617 BlockDriver *drv = bs->drv; 3618 int ret; 3619 IO_CODE(); 3620 3621 if (!drv) { 3622 return -ENOMEDIUM; 3623 } 3624 3625 if (!drv->bdrv_co_preadv_snapshot) { 3626 return -ENOTSUP; 3627 } 3628 3629 bdrv_inc_in_flight(bs); 3630 ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset); 3631 bdrv_dec_in_flight(bs); 3632 3633 return ret; 3634 } 3635 3636 int coroutine_fn 3637 bdrv_co_snapshot_block_status(BlockDriverState *bs, 3638 bool want_zero, int64_t offset, int64_t bytes, 3639 int64_t *pnum, int64_t *map, 3640 BlockDriverState **file) 3641 { 3642 BlockDriver *drv = bs->drv; 3643 int ret; 3644 IO_CODE(); 3645 3646 if (!drv) { 3647 return -ENOMEDIUM; 3648 } 3649 3650 if (!drv->bdrv_co_snapshot_block_status) { 3651 return -ENOTSUP; 3652 } 3653 3654 bdrv_inc_in_flight(bs); 3655 ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes, 3656 pnum, map, file); 3657 bdrv_dec_in_flight(bs); 3658 3659 return ret; 3660 } 3661 3662 int coroutine_fn 3663 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes) 3664 { 3665 BlockDriver *drv = bs->drv; 3666 int ret; 3667 IO_CODE(); 3668 3669 if (!drv) { 3670 return -ENOMEDIUM; 3671 } 3672 3673 if (!drv->bdrv_co_pdiscard_snapshot) { 3674 return -ENOTSUP; 3675 } 3676 3677 bdrv_inc_in_flight(bs); 3678 ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes); 3679 bdrv_dec_in_flight(bs); 3680 3681 return ret; 3682 } 3683