1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/aio-wait.h" 29 #include "block/blockjob.h" 30 #include "block/blockjob_int.h" 31 #include "block/block_int.h" 32 #include "block/coroutines.h" 33 #include "qemu/cutils.h" 34 #include "qapi/error.h" 35 #include "qemu/error-report.h" 36 #include "qemu/main-loop.h" 37 #include "sysemu/replay.h" 38 39 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 40 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 41 42 static void bdrv_parent_cb_resize(BlockDriverState *bs); 43 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 44 int64_t offset, int bytes, BdrvRequestFlags flags); 45 46 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore, 47 bool ignore_bds_parents) 48 { 49 BdrvChild *c, *next; 50 51 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 52 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 53 continue; 54 } 55 bdrv_parent_drained_begin_single(c, false); 56 } 57 } 58 59 static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c, 60 int *drained_end_counter) 61 { 62 assert(c->parent_quiesce_counter > 0); 63 c->parent_quiesce_counter--; 64 if (c->klass->drained_end) { 65 c->klass->drained_end(c, drained_end_counter); 66 } 67 } 68 69 void bdrv_parent_drained_end_single(BdrvChild *c) 70 { 71 int drained_end_counter = 0; 72 bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter); 73 BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0); 74 } 75 76 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore, 77 bool ignore_bds_parents, 78 int *drained_end_counter) 79 { 80 BdrvChild *c; 81 82 QLIST_FOREACH(c, &bs->parents, next_parent) { 83 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 84 continue; 85 } 86 bdrv_parent_drained_end_single_no_poll(c, drained_end_counter); 87 } 88 } 89 90 static bool bdrv_parent_drained_poll_single(BdrvChild *c) 91 { 92 if (c->klass->drained_poll) { 93 return c->klass->drained_poll(c); 94 } 95 return false; 96 } 97 98 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 99 bool ignore_bds_parents) 100 { 101 BdrvChild *c, *next; 102 bool busy = false; 103 104 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 105 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 106 continue; 107 } 108 busy |= bdrv_parent_drained_poll_single(c); 109 } 110 111 return busy; 112 } 113 114 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll) 115 { 116 c->parent_quiesce_counter++; 117 if (c->klass->drained_begin) { 118 c->klass->drained_begin(c); 119 } 120 if (poll) { 121 BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c)); 122 } 123 } 124 125 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 126 { 127 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 128 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 129 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 130 src->opt_mem_alignment); 131 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 132 src->min_mem_alignment); 133 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 134 } 135 136 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 137 { 138 ERRP_GUARD(); 139 BlockDriver *drv = bs->drv; 140 BdrvChild *c; 141 bool have_limits; 142 143 memset(&bs->bl, 0, sizeof(bs->bl)); 144 145 if (!drv) { 146 return; 147 } 148 149 /* Default alignment based on whether driver has byte interface */ 150 bs->bl.request_alignment = (drv->bdrv_co_preadv || 151 drv->bdrv_aio_preadv || 152 drv->bdrv_co_preadv_part) ? 1 : 512; 153 154 /* Take some limits from the children as a default */ 155 have_limits = false; 156 QLIST_FOREACH(c, &bs->children, next) { 157 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 158 { 159 bdrv_refresh_limits(c->bs, errp); 160 if (*errp) { 161 return; 162 } 163 bdrv_merge_limits(&bs->bl, &c->bs->bl); 164 have_limits = true; 165 } 166 } 167 168 if (!have_limits) { 169 bs->bl.min_mem_alignment = 512; 170 bs->bl.opt_mem_alignment = qemu_real_host_page_size; 171 172 /* Safe default since most protocols use readv()/writev()/etc */ 173 bs->bl.max_iov = IOV_MAX; 174 } 175 176 /* Then let the driver override it */ 177 if (drv->bdrv_refresh_limits) { 178 drv->bdrv_refresh_limits(bs, errp); 179 if (*errp) { 180 return; 181 } 182 } 183 184 if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) { 185 error_setg(errp, "Driver requires too large request alignment"); 186 } 187 } 188 189 /** 190 * The copy-on-read flag is actually a reference count so multiple users may 191 * use the feature without worrying about clobbering its previous state. 192 * Copy-on-read stays enabled until all users have called to disable it. 193 */ 194 void bdrv_enable_copy_on_read(BlockDriverState *bs) 195 { 196 qatomic_inc(&bs->copy_on_read); 197 } 198 199 void bdrv_disable_copy_on_read(BlockDriverState *bs) 200 { 201 int old = qatomic_fetch_dec(&bs->copy_on_read); 202 assert(old >= 1); 203 } 204 205 typedef struct { 206 Coroutine *co; 207 BlockDriverState *bs; 208 bool done; 209 bool begin; 210 bool recursive; 211 bool poll; 212 BdrvChild *parent; 213 bool ignore_bds_parents; 214 int *drained_end_counter; 215 } BdrvCoDrainData; 216 217 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 218 { 219 BdrvCoDrainData *data = opaque; 220 BlockDriverState *bs = data->bs; 221 222 if (data->begin) { 223 bs->drv->bdrv_co_drain_begin(bs); 224 } else { 225 bs->drv->bdrv_co_drain_end(bs); 226 } 227 228 /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */ 229 qatomic_mb_set(&data->done, true); 230 if (!data->begin) { 231 qatomic_dec(data->drained_end_counter); 232 } 233 bdrv_dec_in_flight(bs); 234 235 g_free(data); 236 } 237 238 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */ 239 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, 240 int *drained_end_counter) 241 { 242 BdrvCoDrainData *data; 243 244 if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 245 (!begin && !bs->drv->bdrv_co_drain_end)) { 246 return; 247 } 248 249 data = g_new(BdrvCoDrainData, 1); 250 *data = (BdrvCoDrainData) { 251 .bs = bs, 252 .done = false, 253 .begin = begin, 254 .drained_end_counter = drained_end_counter, 255 }; 256 257 if (!begin) { 258 qatomic_inc(drained_end_counter); 259 } 260 261 /* Make sure the driver callback completes during the polling phase for 262 * drain_begin. */ 263 bdrv_inc_in_flight(bs); 264 data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data); 265 aio_co_schedule(bdrv_get_aio_context(bs), data->co); 266 } 267 268 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 269 bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, 270 BdrvChild *ignore_parent, bool ignore_bds_parents) 271 { 272 BdrvChild *child, *next; 273 274 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 275 return true; 276 } 277 278 if (qatomic_read(&bs->in_flight)) { 279 return true; 280 } 281 282 if (recursive) { 283 assert(!ignore_bds_parents); 284 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 285 if (bdrv_drain_poll(child->bs, recursive, child, false)) { 286 return true; 287 } 288 } 289 } 290 291 return false; 292 } 293 294 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive, 295 BdrvChild *ignore_parent) 296 { 297 return bdrv_drain_poll(bs, recursive, ignore_parent, false); 298 } 299 300 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 301 BdrvChild *parent, bool ignore_bds_parents, 302 bool poll); 303 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 304 BdrvChild *parent, bool ignore_bds_parents, 305 int *drained_end_counter); 306 307 static void bdrv_co_drain_bh_cb(void *opaque) 308 { 309 BdrvCoDrainData *data = opaque; 310 Coroutine *co = data->co; 311 BlockDriverState *bs = data->bs; 312 313 if (bs) { 314 AioContext *ctx = bdrv_get_aio_context(bs); 315 aio_context_acquire(ctx); 316 bdrv_dec_in_flight(bs); 317 if (data->begin) { 318 assert(!data->drained_end_counter); 319 bdrv_do_drained_begin(bs, data->recursive, data->parent, 320 data->ignore_bds_parents, data->poll); 321 } else { 322 assert(!data->poll); 323 bdrv_do_drained_end(bs, data->recursive, data->parent, 324 data->ignore_bds_parents, 325 data->drained_end_counter); 326 } 327 aio_context_release(ctx); 328 } else { 329 assert(data->begin); 330 bdrv_drain_all_begin(); 331 } 332 333 data->done = true; 334 aio_co_wake(co); 335 } 336 337 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 338 bool begin, bool recursive, 339 BdrvChild *parent, 340 bool ignore_bds_parents, 341 bool poll, 342 int *drained_end_counter) 343 { 344 BdrvCoDrainData data; 345 Coroutine *self = qemu_coroutine_self(); 346 AioContext *ctx = bdrv_get_aio_context(bs); 347 AioContext *co_ctx = qemu_coroutine_get_aio_context(self); 348 349 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 350 * other coroutines run if they were queued by aio_co_enter(). */ 351 352 assert(qemu_in_coroutine()); 353 data = (BdrvCoDrainData) { 354 .co = self, 355 .bs = bs, 356 .done = false, 357 .begin = begin, 358 .recursive = recursive, 359 .parent = parent, 360 .ignore_bds_parents = ignore_bds_parents, 361 .poll = poll, 362 .drained_end_counter = drained_end_counter, 363 }; 364 365 if (bs) { 366 bdrv_inc_in_flight(bs); 367 } 368 369 /* 370 * Temporarily drop the lock across yield or we would get deadlocks. 371 * bdrv_co_drain_bh_cb() reaquires the lock as needed. 372 * 373 * When we yield below, the lock for the current context will be 374 * released, so if this is actually the lock that protects bs, don't drop 375 * it a second time. 376 */ 377 if (ctx != co_ctx) { 378 aio_context_release(ctx); 379 } 380 replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data); 381 382 qemu_coroutine_yield(); 383 /* If we are resumed from some other event (such as an aio completion or a 384 * timer callback), it is a bug in the caller that should be fixed. */ 385 assert(data.done); 386 387 /* Reaquire the AioContext of bs if we dropped it */ 388 if (ctx != co_ctx) { 389 aio_context_acquire(ctx); 390 } 391 } 392 393 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, 394 BdrvChild *parent, bool ignore_bds_parents) 395 { 396 assert(!qemu_in_coroutine()); 397 398 /* Stop things in parent-to-child order */ 399 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 400 aio_disable_external(bdrv_get_aio_context(bs)); 401 } 402 403 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents); 404 bdrv_drain_invoke(bs, true, NULL); 405 } 406 407 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 408 BdrvChild *parent, bool ignore_bds_parents, 409 bool poll) 410 { 411 BdrvChild *child, *next; 412 413 if (qemu_in_coroutine()) { 414 bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents, 415 poll, NULL); 416 return; 417 } 418 419 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents); 420 421 if (recursive) { 422 assert(!ignore_bds_parents); 423 bs->recursive_quiesce_counter++; 424 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 425 bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents, 426 false); 427 } 428 } 429 430 /* 431 * Wait for drained requests to finish. 432 * 433 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 434 * call is needed so things in this AioContext can make progress even 435 * though we don't return to the main AioContext loop - this automatically 436 * includes other nodes in the same AioContext and therefore all child 437 * nodes. 438 */ 439 if (poll) { 440 assert(!ignore_bds_parents); 441 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent)); 442 } 443 } 444 445 void bdrv_drained_begin(BlockDriverState *bs) 446 { 447 bdrv_do_drained_begin(bs, false, NULL, false, true); 448 } 449 450 void bdrv_subtree_drained_begin(BlockDriverState *bs) 451 { 452 bdrv_do_drained_begin(bs, true, NULL, false, true); 453 } 454 455 /** 456 * This function does not poll, nor must any of its recursively called 457 * functions. The *drained_end_counter pointee will be incremented 458 * once for every background operation scheduled, and decremented once 459 * the operation settles. Therefore, the pointer must remain valid 460 * until the pointee reaches 0. That implies that whoever sets up the 461 * pointee has to poll until it is 0. 462 * 463 * We use atomic operations to access *drained_end_counter, because 464 * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of 465 * @bs may contain nodes in different AioContexts, 466 * (2) bdrv_drain_all_end() uses the same counter for all nodes, 467 * regardless of which AioContext they are in. 468 */ 469 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 470 BdrvChild *parent, bool ignore_bds_parents, 471 int *drained_end_counter) 472 { 473 BdrvChild *child; 474 int old_quiesce_counter; 475 476 assert(drained_end_counter != NULL); 477 478 if (qemu_in_coroutine()) { 479 bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents, 480 false, drained_end_counter); 481 return; 482 } 483 assert(bs->quiesce_counter > 0); 484 485 /* Re-enable things in child-to-parent order */ 486 bdrv_drain_invoke(bs, false, drained_end_counter); 487 bdrv_parent_drained_end(bs, parent, ignore_bds_parents, 488 drained_end_counter); 489 490 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 491 if (old_quiesce_counter == 1) { 492 aio_enable_external(bdrv_get_aio_context(bs)); 493 } 494 495 if (recursive) { 496 assert(!ignore_bds_parents); 497 bs->recursive_quiesce_counter--; 498 QLIST_FOREACH(child, &bs->children, next) { 499 bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents, 500 drained_end_counter); 501 } 502 } 503 } 504 505 void bdrv_drained_end(BlockDriverState *bs) 506 { 507 int drained_end_counter = 0; 508 bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter); 509 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 510 } 511 512 void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter) 513 { 514 bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter); 515 } 516 517 void bdrv_subtree_drained_end(BlockDriverState *bs) 518 { 519 int drained_end_counter = 0; 520 bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter); 521 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 522 } 523 524 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent) 525 { 526 int i; 527 528 for (i = 0; i < new_parent->recursive_quiesce_counter; i++) { 529 bdrv_do_drained_begin(child->bs, true, child, false, true); 530 } 531 } 532 533 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent) 534 { 535 int drained_end_counter = 0; 536 int i; 537 538 for (i = 0; i < old_parent->recursive_quiesce_counter; i++) { 539 bdrv_do_drained_end(child->bs, true, child, false, 540 &drained_end_counter); 541 } 542 543 BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0); 544 } 545 546 /* 547 * Wait for pending requests to complete on a single BlockDriverState subtree, 548 * and suspend block driver's internal I/O until next request arrives. 549 * 550 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 551 * AioContext. 552 */ 553 void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 554 { 555 assert(qemu_in_coroutine()); 556 bdrv_drained_begin(bs); 557 bdrv_drained_end(bs); 558 } 559 560 void bdrv_drain(BlockDriverState *bs) 561 { 562 bdrv_drained_begin(bs); 563 bdrv_drained_end(bs); 564 } 565 566 static void bdrv_drain_assert_idle(BlockDriverState *bs) 567 { 568 BdrvChild *child, *next; 569 570 assert(qatomic_read(&bs->in_flight) == 0); 571 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 572 bdrv_drain_assert_idle(child->bs); 573 } 574 } 575 576 unsigned int bdrv_drain_all_count = 0; 577 578 static bool bdrv_drain_all_poll(void) 579 { 580 BlockDriverState *bs = NULL; 581 bool result = false; 582 583 /* bdrv_drain_poll() can't make changes to the graph and we are holding the 584 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 585 while ((bs = bdrv_next_all_states(bs))) { 586 AioContext *aio_context = bdrv_get_aio_context(bs); 587 aio_context_acquire(aio_context); 588 result |= bdrv_drain_poll(bs, false, NULL, true); 589 aio_context_release(aio_context); 590 } 591 592 return result; 593 } 594 595 /* 596 * Wait for pending requests to complete across all BlockDriverStates 597 * 598 * This function does not flush data to disk, use bdrv_flush_all() for that 599 * after calling this function. 600 * 601 * This pauses all block jobs and disables external clients. It must 602 * be paired with bdrv_drain_all_end(). 603 * 604 * NOTE: no new block jobs or BlockDriverStates can be created between 605 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 606 */ 607 void bdrv_drain_all_begin(void) 608 { 609 BlockDriverState *bs = NULL; 610 611 if (qemu_in_coroutine()) { 612 bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL); 613 return; 614 } 615 616 /* 617 * bdrv queue is managed by record/replay, 618 * waiting for finishing the I/O requests may 619 * be infinite 620 */ 621 if (replay_events_enabled()) { 622 return; 623 } 624 625 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 626 * loop AioContext, so make sure we're in the main context. */ 627 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 628 assert(bdrv_drain_all_count < INT_MAX); 629 bdrv_drain_all_count++; 630 631 /* Quiesce all nodes, without polling in-flight requests yet. The graph 632 * cannot change during this loop. */ 633 while ((bs = bdrv_next_all_states(bs))) { 634 AioContext *aio_context = bdrv_get_aio_context(bs); 635 636 aio_context_acquire(aio_context); 637 bdrv_do_drained_begin(bs, false, NULL, true, false); 638 aio_context_release(aio_context); 639 } 640 641 /* Now poll the in-flight requests */ 642 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 643 644 while ((bs = bdrv_next_all_states(bs))) { 645 bdrv_drain_assert_idle(bs); 646 } 647 } 648 649 void bdrv_drain_all_end_quiesce(BlockDriverState *bs) 650 { 651 int drained_end_counter = 0; 652 653 g_assert(bs->quiesce_counter > 0); 654 g_assert(!bs->refcnt); 655 656 while (bs->quiesce_counter) { 657 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 658 } 659 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 660 } 661 662 void bdrv_drain_all_end(void) 663 { 664 BlockDriverState *bs = NULL; 665 int drained_end_counter = 0; 666 667 /* 668 * bdrv queue is managed by record/replay, 669 * waiting for finishing the I/O requests may 670 * be endless 671 */ 672 if (replay_events_enabled()) { 673 return; 674 } 675 676 while ((bs = bdrv_next_all_states(bs))) { 677 AioContext *aio_context = bdrv_get_aio_context(bs); 678 679 aio_context_acquire(aio_context); 680 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 681 aio_context_release(aio_context); 682 } 683 684 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 685 AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0); 686 687 assert(bdrv_drain_all_count > 0); 688 bdrv_drain_all_count--; 689 } 690 691 void bdrv_drain_all(void) 692 { 693 bdrv_drain_all_begin(); 694 bdrv_drain_all_end(); 695 } 696 697 /** 698 * Remove an active request from the tracked requests list 699 * 700 * This function should be called when a tracked request is completing. 701 */ 702 static void tracked_request_end(BdrvTrackedRequest *req) 703 { 704 if (req->serialising) { 705 qatomic_dec(&req->bs->serialising_in_flight); 706 } 707 708 qemu_co_mutex_lock(&req->bs->reqs_lock); 709 QLIST_REMOVE(req, list); 710 qemu_co_queue_restart_all(&req->wait_queue); 711 qemu_co_mutex_unlock(&req->bs->reqs_lock); 712 } 713 714 /** 715 * Add an active request to the tracked requests list 716 */ 717 static void tracked_request_begin(BdrvTrackedRequest *req, 718 BlockDriverState *bs, 719 int64_t offset, 720 uint64_t bytes, 721 enum BdrvTrackedRequestType type) 722 { 723 assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes); 724 725 *req = (BdrvTrackedRequest){ 726 .bs = bs, 727 .offset = offset, 728 .bytes = bytes, 729 .type = type, 730 .co = qemu_coroutine_self(), 731 .serialising = false, 732 .overlap_offset = offset, 733 .overlap_bytes = bytes, 734 }; 735 736 qemu_co_queue_init(&req->wait_queue); 737 738 qemu_co_mutex_lock(&bs->reqs_lock); 739 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 740 qemu_co_mutex_unlock(&bs->reqs_lock); 741 } 742 743 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 744 int64_t offset, uint64_t bytes) 745 { 746 /* aaaa bbbb */ 747 if (offset >= req->overlap_offset + req->overlap_bytes) { 748 return false; 749 } 750 /* bbbb aaaa */ 751 if (req->overlap_offset >= offset + bytes) { 752 return false; 753 } 754 return true; 755 } 756 757 static bool coroutine_fn 758 bdrv_wait_serialising_requests_locked(BlockDriverState *bs, 759 BdrvTrackedRequest *self) 760 { 761 BdrvTrackedRequest *req; 762 bool retry; 763 bool waited = false; 764 765 do { 766 retry = false; 767 QLIST_FOREACH(req, &bs->tracked_requests, list) { 768 if (req == self || (!req->serialising && !self->serialising)) { 769 continue; 770 } 771 if (tracked_request_overlaps(req, self->overlap_offset, 772 self->overlap_bytes)) 773 { 774 /* Hitting this means there was a reentrant request, for 775 * example, a block driver issuing nested requests. This must 776 * never happen since it means deadlock. 777 */ 778 assert(qemu_coroutine_self() != req->co); 779 780 /* If the request is already (indirectly) waiting for us, or 781 * will wait for us as soon as it wakes up, then just go on 782 * (instead of producing a deadlock in the former case). */ 783 if (!req->waiting_for) { 784 self->waiting_for = req; 785 qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock); 786 self->waiting_for = NULL; 787 retry = true; 788 waited = true; 789 break; 790 } 791 } 792 } 793 } while (retry); 794 return waited; 795 } 796 797 bool bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 798 { 799 BlockDriverState *bs = req->bs; 800 int64_t overlap_offset = req->offset & ~(align - 1); 801 uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 802 - overlap_offset; 803 bool waited; 804 805 qemu_co_mutex_lock(&bs->reqs_lock); 806 if (!req->serialising) { 807 qatomic_inc(&req->bs->serialising_in_flight); 808 req->serialising = true; 809 } 810 811 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 812 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 813 waited = bdrv_wait_serialising_requests_locked(bs, req); 814 qemu_co_mutex_unlock(&bs->reqs_lock); 815 return waited; 816 } 817 818 /** 819 * Return the tracked request on @bs for the current coroutine, or 820 * NULL if there is none. 821 */ 822 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 823 { 824 BdrvTrackedRequest *req; 825 Coroutine *self = qemu_coroutine_self(); 826 827 QLIST_FOREACH(req, &bs->tracked_requests, list) { 828 if (req->co == self) { 829 return req; 830 } 831 } 832 833 return NULL; 834 } 835 836 /** 837 * Round a region to cluster boundaries 838 */ 839 void bdrv_round_to_clusters(BlockDriverState *bs, 840 int64_t offset, int64_t bytes, 841 int64_t *cluster_offset, 842 int64_t *cluster_bytes) 843 { 844 BlockDriverInfo bdi; 845 846 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 847 *cluster_offset = offset; 848 *cluster_bytes = bytes; 849 } else { 850 int64_t c = bdi.cluster_size; 851 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 852 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 853 } 854 } 855 856 static int bdrv_get_cluster_size(BlockDriverState *bs) 857 { 858 BlockDriverInfo bdi; 859 int ret; 860 861 ret = bdrv_get_info(bs, &bdi); 862 if (ret < 0 || bdi.cluster_size == 0) { 863 return bs->bl.request_alignment; 864 } else { 865 return bdi.cluster_size; 866 } 867 } 868 869 void bdrv_inc_in_flight(BlockDriverState *bs) 870 { 871 qatomic_inc(&bs->in_flight); 872 } 873 874 void bdrv_wakeup(BlockDriverState *bs) 875 { 876 aio_wait_kick(); 877 } 878 879 void bdrv_dec_in_flight(BlockDriverState *bs) 880 { 881 qatomic_dec(&bs->in_flight); 882 bdrv_wakeup(bs); 883 } 884 885 static bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 886 { 887 BlockDriverState *bs = self->bs; 888 bool waited = false; 889 890 if (!qatomic_read(&bs->serialising_in_flight)) { 891 return false; 892 } 893 894 qemu_co_mutex_lock(&bs->reqs_lock); 895 waited = bdrv_wait_serialising_requests_locked(bs, self); 896 qemu_co_mutex_unlock(&bs->reqs_lock); 897 898 return waited; 899 } 900 901 int bdrv_check_request(int64_t offset, int64_t bytes) 902 { 903 if (offset < 0 || bytes < 0) { 904 return -EIO; 905 } 906 907 if (bytes > BDRV_MAX_LENGTH) { 908 return -EIO; 909 } 910 911 if (offset > BDRV_MAX_LENGTH - bytes) { 912 return -EIO; 913 } 914 915 return 0; 916 } 917 918 static int bdrv_check_request32(int64_t offset, int64_t bytes) 919 { 920 int ret = bdrv_check_request(offset, bytes); 921 if (ret < 0) { 922 return ret; 923 } 924 925 if (bytes > BDRV_REQUEST_MAX_BYTES) { 926 return -EIO; 927 } 928 929 return 0; 930 } 931 932 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 933 int bytes, BdrvRequestFlags flags) 934 { 935 return bdrv_pwritev(child, offset, bytes, NULL, 936 BDRV_REQ_ZERO_WRITE | flags); 937 } 938 939 /* 940 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 941 * The operation is sped up by checking the block status and only writing 942 * zeroes to the device if they currently do not return zeroes. Optional 943 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 944 * BDRV_REQ_FUA). 945 * 946 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 947 */ 948 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 949 { 950 int ret; 951 int64_t target_size, bytes, offset = 0; 952 BlockDriverState *bs = child->bs; 953 954 target_size = bdrv_getlength(bs); 955 if (target_size < 0) { 956 return target_size; 957 } 958 959 for (;;) { 960 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 961 if (bytes <= 0) { 962 return 0; 963 } 964 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 965 if (ret < 0) { 966 return ret; 967 } 968 if (ret & BDRV_BLOCK_ZERO) { 969 offset += bytes; 970 continue; 971 } 972 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 973 if (ret < 0) { 974 return ret; 975 } 976 offset += bytes; 977 } 978 } 979 980 /* See bdrv_pwrite() for the return codes */ 981 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 982 { 983 int ret; 984 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 985 986 if (bytes < 0) { 987 return -EINVAL; 988 } 989 990 ret = bdrv_preadv(child, offset, bytes, &qiov, 0); 991 992 return ret < 0 ? ret : bytes; 993 } 994 995 /* Return no. of bytes on success or < 0 on error. Important errors are: 996 -EIO generic I/O error (may happen for all errors) 997 -ENOMEDIUM No media inserted. 998 -EINVAL Invalid offset or number of bytes 999 -EACCES Trying to write a read-only device 1000 */ 1001 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 1002 { 1003 int ret; 1004 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 1005 1006 if (bytes < 0) { 1007 return -EINVAL; 1008 } 1009 1010 ret = bdrv_pwritev(child, offset, bytes, &qiov, 0); 1011 1012 return ret < 0 ? ret : bytes; 1013 } 1014 1015 /* 1016 * Writes to the file and ensures that no writes are reordered across this 1017 * request (acts as a barrier) 1018 * 1019 * Returns 0 on success, -errno in error cases. 1020 */ 1021 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 1022 const void *buf, int count) 1023 { 1024 int ret; 1025 1026 ret = bdrv_pwrite(child, offset, buf, count); 1027 if (ret < 0) { 1028 return ret; 1029 } 1030 1031 ret = bdrv_flush(child->bs); 1032 if (ret < 0) { 1033 return ret; 1034 } 1035 1036 return 0; 1037 } 1038 1039 typedef struct CoroutineIOCompletion { 1040 Coroutine *coroutine; 1041 int ret; 1042 } CoroutineIOCompletion; 1043 1044 static void bdrv_co_io_em_complete(void *opaque, int ret) 1045 { 1046 CoroutineIOCompletion *co = opaque; 1047 1048 co->ret = ret; 1049 aio_co_wake(co->coroutine); 1050 } 1051 1052 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 1053 uint64_t offset, uint64_t bytes, 1054 QEMUIOVector *qiov, 1055 size_t qiov_offset, int flags) 1056 { 1057 BlockDriver *drv = bs->drv; 1058 int64_t sector_num; 1059 unsigned int nb_sectors; 1060 QEMUIOVector local_qiov; 1061 int ret; 1062 1063 assert(!(flags & ~BDRV_REQ_MASK)); 1064 assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1065 1066 if (!drv) { 1067 return -ENOMEDIUM; 1068 } 1069 1070 if (drv->bdrv_co_preadv_part) { 1071 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 1072 flags); 1073 } 1074 1075 if (qiov_offset > 0 || bytes != qiov->size) { 1076 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1077 qiov = &local_qiov; 1078 } 1079 1080 if (drv->bdrv_co_preadv) { 1081 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 1082 goto out; 1083 } 1084 1085 if (drv->bdrv_aio_preadv) { 1086 BlockAIOCB *acb; 1087 CoroutineIOCompletion co = { 1088 .coroutine = qemu_coroutine_self(), 1089 }; 1090 1091 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 1092 bdrv_co_io_em_complete, &co); 1093 if (acb == NULL) { 1094 ret = -EIO; 1095 goto out; 1096 } else { 1097 qemu_coroutine_yield(); 1098 ret = co.ret; 1099 goto out; 1100 } 1101 } 1102 1103 sector_num = offset >> BDRV_SECTOR_BITS; 1104 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1105 1106 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1107 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1108 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1109 assert(drv->bdrv_co_readv); 1110 1111 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1112 1113 out: 1114 if (qiov == &local_qiov) { 1115 qemu_iovec_destroy(&local_qiov); 1116 } 1117 1118 return ret; 1119 } 1120 1121 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 1122 uint64_t offset, uint64_t bytes, 1123 QEMUIOVector *qiov, 1124 size_t qiov_offset, int flags) 1125 { 1126 BlockDriver *drv = bs->drv; 1127 int64_t sector_num; 1128 unsigned int nb_sectors; 1129 QEMUIOVector local_qiov; 1130 int ret; 1131 1132 assert(!(flags & ~BDRV_REQ_MASK)); 1133 assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1134 1135 if (!drv) { 1136 return -ENOMEDIUM; 1137 } 1138 1139 if (drv->bdrv_co_pwritev_part) { 1140 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1141 flags & bs->supported_write_flags); 1142 flags &= ~bs->supported_write_flags; 1143 goto emulate_flags; 1144 } 1145 1146 if (qiov_offset > 0 || bytes != qiov->size) { 1147 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1148 qiov = &local_qiov; 1149 } 1150 1151 if (drv->bdrv_co_pwritev) { 1152 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 1153 flags & bs->supported_write_flags); 1154 flags &= ~bs->supported_write_flags; 1155 goto emulate_flags; 1156 } 1157 1158 if (drv->bdrv_aio_pwritev) { 1159 BlockAIOCB *acb; 1160 CoroutineIOCompletion co = { 1161 .coroutine = qemu_coroutine_self(), 1162 }; 1163 1164 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, 1165 flags & bs->supported_write_flags, 1166 bdrv_co_io_em_complete, &co); 1167 flags &= ~bs->supported_write_flags; 1168 if (acb == NULL) { 1169 ret = -EIO; 1170 } else { 1171 qemu_coroutine_yield(); 1172 ret = co.ret; 1173 } 1174 goto emulate_flags; 1175 } 1176 1177 sector_num = offset >> BDRV_SECTOR_BITS; 1178 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1179 1180 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1181 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1182 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1183 1184 assert(drv->bdrv_co_writev); 1185 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, 1186 flags & bs->supported_write_flags); 1187 flags &= ~bs->supported_write_flags; 1188 1189 emulate_flags: 1190 if (ret == 0 && (flags & BDRV_REQ_FUA)) { 1191 ret = bdrv_co_flush(bs); 1192 } 1193 1194 if (qiov == &local_qiov) { 1195 qemu_iovec_destroy(&local_qiov); 1196 } 1197 1198 return ret; 1199 } 1200 1201 static int coroutine_fn 1202 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 1203 uint64_t bytes, QEMUIOVector *qiov, 1204 size_t qiov_offset) 1205 { 1206 BlockDriver *drv = bs->drv; 1207 QEMUIOVector local_qiov; 1208 int ret; 1209 1210 if (!drv) { 1211 return -ENOMEDIUM; 1212 } 1213 1214 if (!block_driver_can_compress(drv)) { 1215 return -ENOTSUP; 1216 } 1217 1218 if (drv->bdrv_co_pwritev_compressed_part) { 1219 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1220 qiov, qiov_offset); 1221 } 1222 1223 if (qiov_offset == 0) { 1224 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 1225 } 1226 1227 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1228 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1229 qemu_iovec_destroy(&local_qiov); 1230 1231 return ret; 1232 } 1233 1234 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 1235 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1236 size_t qiov_offset, int flags) 1237 { 1238 BlockDriverState *bs = child->bs; 1239 1240 /* Perform I/O through a temporary buffer so that users who scribble over 1241 * their read buffer while the operation is in progress do not end up 1242 * modifying the image file. This is critical for zero-copy guest I/O 1243 * where anything might happen inside guest memory. 1244 */ 1245 void *bounce_buffer = NULL; 1246 1247 BlockDriver *drv = bs->drv; 1248 int64_t cluster_offset; 1249 int64_t cluster_bytes; 1250 size_t skip_bytes; 1251 int ret; 1252 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1253 BDRV_REQUEST_MAX_BYTES); 1254 unsigned int progress = 0; 1255 bool skip_write; 1256 1257 if (!drv) { 1258 return -ENOMEDIUM; 1259 } 1260 1261 /* 1262 * Do not write anything when the BDS is inactive. That is not 1263 * allowed, and it would not help. 1264 */ 1265 skip_write = (bs->open_flags & BDRV_O_INACTIVE); 1266 1267 /* FIXME We cannot require callers to have write permissions when all they 1268 * are doing is a read request. If we did things right, write permissions 1269 * would be obtained anyway, but internally by the copy-on-read code. As 1270 * long as it is implemented here rather than in a separate filter driver, 1271 * the copy-on-read code doesn't have its own BdrvChild, however, for which 1272 * it could request permissions. Therefore we have to bypass the permission 1273 * system for the moment. */ 1274 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1275 1276 /* Cover entire cluster so no additional backing file I/O is required when 1277 * allocating cluster in the image file. Note that this value may exceed 1278 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1279 * is one reason we loop rather than doing it all at once. 1280 */ 1281 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1282 skip_bytes = offset - cluster_offset; 1283 1284 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1285 cluster_offset, cluster_bytes); 1286 1287 while (cluster_bytes) { 1288 int64_t pnum; 1289 1290 if (skip_write) { 1291 ret = 1; /* "already allocated", so nothing will be copied */ 1292 pnum = MIN(cluster_bytes, max_transfer); 1293 } else { 1294 ret = bdrv_is_allocated(bs, cluster_offset, 1295 MIN(cluster_bytes, max_transfer), &pnum); 1296 if (ret < 0) { 1297 /* 1298 * Safe to treat errors in querying allocation as if 1299 * unallocated; we'll probably fail again soon on the 1300 * read, but at least that will set a decent errno. 1301 */ 1302 pnum = MIN(cluster_bytes, max_transfer); 1303 } 1304 1305 /* Stop at EOF if the image ends in the middle of the cluster */ 1306 if (ret == 0 && pnum == 0) { 1307 assert(progress >= bytes); 1308 break; 1309 } 1310 1311 assert(skip_bytes < pnum); 1312 } 1313 1314 if (ret <= 0) { 1315 QEMUIOVector local_qiov; 1316 1317 /* Must copy-on-read; use the bounce buffer */ 1318 pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1319 if (!bounce_buffer) { 1320 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum); 1321 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 1322 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 1323 1324 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 1325 if (!bounce_buffer) { 1326 ret = -ENOMEM; 1327 goto err; 1328 } 1329 } 1330 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1331 1332 ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1333 &local_qiov, 0, 0); 1334 if (ret < 0) { 1335 goto err; 1336 } 1337 1338 bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1339 if (drv->bdrv_co_pwrite_zeroes && 1340 buffer_is_zero(bounce_buffer, pnum)) { 1341 /* FIXME: Should we (perhaps conditionally) be setting 1342 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1343 * that still correctly reads as zero? */ 1344 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 1345 BDRV_REQ_WRITE_UNCHANGED); 1346 } else { 1347 /* This does not change the data on the disk, it is not 1348 * necessary to flush even in cache=writethrough mode. 1349 */ 1350 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1351 &local_qiov, 0, 1352 BDRV_REQ_WRITE_UNCHANGED); 1353 } 1354 1355 if (ret < 0) { 1356 /* It might be okay to ignore write errors for guest 1357 * requests. If this is a deliberate copy-on-read 1358 * then we don't want to ignore the error. Simply 1359 * report it in all cases. 1360 */ 1361 goto err; 1362 } 1363 1364 if (!(flags & BDRV_REQ_PREFETCH)) { 1365 qemu_iovec_from_buf(qiov, qiov_offset + progress, 1366 bounce_buffer + skip_bytes, 1367 MIN(pnum - skip_bytes, bytes - progress)); 1368 } 1369 } else if (!(flags & BDRV_REQ_PREFETCH)) { 1370 /* Read directly into the destination */ 1371 ret = bdrv_driver_preadv(bs, offset + progress, 1372 MIN(pnum - skip_bytes, bytes - progress), 1373 qiov, qiov_offset + progress, 0); 1374 if (ret < 0) { 1375 goto err; 1376 } 1377 } 1378 1379 cluster_offset += pnum; 1380 cluster_bytes -= pnum; 1381 progress += pnum - skip_bytes; 1382 skip_bytes = 0; 1383 } 1384 ret = 0; 1385 1386 err: 1387 qemu_vfree(bounce_buffer); 1388 return ret; 1389 } 1390 1391 /* 1392 * Forwards an already correctly aligned request to the BlockDriver. This 1393 * handles copy on read, zeroing after EOF, and fragmentation of large 1394 * reads; any other features must be implemented by the caller. 1395 */ 1396 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 1397 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1398 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 1399 { 1400 BlockDriverState *bs = child->bs; 1401 int64_t total_bytes, max_bytes; 1402 int ret = 0; 1403 uint64_t bytes_remaining = bytes; 1404 int max_transfer; 1405 1406 assert(is_power_of_2(align)); 1407 assert((offset & (align - 1)) == 0); 1408 assert((bytes & (align - 1)) == 0); 1409 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1410 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1411 align); 1412 1413 /* TODO: We would need a per-BDS .supported_read_flags and 1414 * potential fallback support, if we ever implement any read flags 1415 * to pass through to drivers. For now, there aren't any 1416 * passthrough flags. */ 1417 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH))); 1418 1419 /* Handle Copy on Read and associated serialisation */ 1420 if (flags & BDRV_REQ_COPY_ON_READ) { 1421 /* If we touch the same cluster it counts as an overlap. This 1422 * guarantees that allocating writes will be serialized and not race 1423 * with each other for the same cluster. For example, in copy-on-read 1424 * it ensures that the CoR read and write operations are atomic and 1425 * guest writes cannot interleave between them. */ 1426 bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs)); 1427 } else { 1428 bdrv_wait_serialising_requests(req); 1429 } 1430 1431 if (flags & BDRV_REQ_COPY_ON_READ) { 1432 int64_t pnum; 1433 1434 ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 1435 if (ret < 0) { 1436 goto out; 1437 } 1438 1439 if (!ret || pnum != bytes) { 1440 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 1441 qiov, qiov_offset, flags); 1442 goto out; 1443 } else if (flags & BDRV_REQ_PREFETCH) { 1444 goto out; 1445 } 1446 } 1447 1448 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1449 total_bytes = bdrv_getlength(bs); 1450 if (total_bytes < 0) { 1451 ret = total_bytes; 1452 goto out; 1453 } 1454 1455 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1456 if (bytes <= max_bytes && bytes <= max_transfer) { 1457 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0); 1458 goto out; 1459 } 1460 1461 while (bytes_remaining) { 1462 int num; 1463 1464 if (max_bytes) { 1465 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1466 assert(num); 1467 1468 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1469 num, qiov, 1470 qiov_offset + bytes - bytes_remaining, 0); 1471 max_bytes -= num; 1472 } else { 1473 num = bytes_remaining; 1474 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1475 0, bytes_remaining); 1476 } 1477 if (ret < 0) { 1478 goto out; 1479 } 1480 bytes_remaining -= num; 1481 } 1482 1483 out: 1484 return ret < 0 ? ret : 0; 1485 } 1486 1487 /* 1488 * Request padding 1489 * 1490 * |<---- align ----->| |<----- align ---->| 1491 * |<- head ->|<------------- bytes ------------->|<-- tail -->| 1492 * | | | | | | 1493 * -*----------$-------*-------- ... --------*-----$------------*--- 1494 * | | | | | | 1495 * | offset | | end | 1496 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 1497 * [buf ... ) [tail_buf ) 1498 * 1499 * @buf is an aligned allocation needed to store @head and @tail paddings. @head 1500 * is placed at the beginning of @buf and @tail at the @end. 1501 * 1502 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 1503 * around tail, if tail exists. 1504 * 1505 * @merge_reads is true for small requests, 1506 * if @buf_len == @head + bytes + @tail. In this case it is possible that both 1507 * head and tail exist but @buf_len == align and @tail_buf == @buf. 1508 */ 1509 typedef struct BdrvRequestPadding { 1510 uint8_t *buf; 1511 size_t buf_len; 1512 uint8_t *tail_buf; 1513 size_t head; 1514 size_t tail; 1515 bool merge_reads; 1516 QEMUIOVector local_qiov; 1517 } BdrvRequestPadding; 1518 1519 static bool bdrv_init_padding(BlockDriverState *bs, 1520 int64_t offset, int64_t bytes, 1521 BdrvRequestPadding *pad) 1522 { 1523 uint64_t align = bs->bl.request_alignment; 1524 size_t sum; 1525 1526 memset(pad, 0, sizeof(*pad)); 1527 1528 pad->head = offset & (align - 1); 1529 pad->tail = ((offset + bytes) & (align - 1)); 1530 if (pad->tail) { 1531 pad->tail = align - pad->tail; 1532 } 1533 1534 if (!pad->head && !pad->tail) { 1535 return false; 1536 } 1537 1538 assert(bytes); /* Nothing good in aligning zero-length requests */ 1539 1540 sum = pad->head + bytes + pad->tail; 1541 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 1542 pad->buf = qemu_blockalign(bs, pad->buf_len); 1543 pad->merge_reads = sum == pad->buf_len; 1544 if (pad->tail) { 1545 pad->tail_buf = pad->buf + pad->buf_len - align; 1546 } 1547 1548 return true; 1549 } 1550 1551 static int bdrv_padding_rmw_read(BdrvChild *child, 1552 BdrvTrackedRequest *req, 1553 BdrvRequestPadding *pad, 1554 bool zero_middle) 1555 { 1556 QEMUIOVector local_qiov; 1557 BlockDriverState *bs = child->bs; 1558 uint64_t align = bs->bl.request_alignment; 1559 int ret; 1560 1561 assert(req->serialising && pad->buf); 1562 1563 if (pad->head || pad->merge_reads) { 1564 uint64_t bytes = pad->merge_reads ? pad->buf_len : align; 1565 1566 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 1567 1568 if (pad->head) { 1569 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1570 } 1571 if (pad->merge_reads && pad->tail) { 1572 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1573 } 1574 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 1575 align, &local_qiov, 0, 0); 1576 if (ret < 0) { 1577 return ret; 1578 } 1579 if (pad->head) { 1580 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1581 } 1582 if (pad->merge_reads && pad->tail) { 1583 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1584 } 1585 1586 if (pad->merge_reads) { 1587 goto zero_mem; 1588 } 1589 } 1590 1591 if (pad->tail) { 1592 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 1593 1594 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1595 ret = bdrv_aligned_preadv( 1596 child, req, 1597 req->overlap_offset + req->overlap_bytes - align, 1598 align, align, &local_qiov, 0, 0); 1599 if (ret < 0) { 1600 return ret; 1601 } 1602 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1603 } 1604 1605 zero_mem: 1606 if (zero_middle) { 1607 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 1608 } 1609 1610 return 0; 1611 } 1612 1613 static void bdrv_padding_destroy(BdrvRequestPadding *pad) 1614 { 1615 if (pad->buf) { 1616 qemu_vfree(pad->buf); 1617 qemu_iovec_destroy(&pad->local_qiov); 1618 } 1619 } 1620 1621 /* 1622 * bdrv_pad_request 1623 * 1624 * Exchange request parameters with padded request if needed. Don't include RMW 1625 * read of padding, bdrv_padding_rmw_read() should be called separately if 1626 * needed. 1627 * 1628 * All parameters except @bs are in-out: they represent original request at 1629 * function call and padded (if padding needed) at function finish. 1630 * 1631 * Function always succeeds. 1632 */ 1633 static bool bdrv_pad_request(BlockDriverState *bs, 1634 QEMUIOVector **qiov, size_t *qiov_offset, 1635 int64_t *offset, unsigned int *bytes, 1636 BdrvRequestPadding *pad) 1637 { 1638 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) { 1639 return false; 1640 } 1641 1642 qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head, 1643 *qiov, *qiov_offset, *bytes, 1644 pad->buf + pad->buf_len - pad->tail, pad->tail); 1645 *bytes += pad->head + pad->tail; 1646 *offset -= pad->head; 1647 *qiov = &pad->local_qiov; 1648 *qiov_offset = 0; 1649 1650 return true; 1651 } 1652 1653 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1654 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1655 BdrvRequestFlags flags) 1656 { 1657 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 1658 } 1659 1660 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 1661 int64_t offset, unsigned int bytes, 1662 QEMUIOVector *qiov, size_t qiov_offset, 1663 BdrvRequestFlags flags) 1664 { 1665 BlockDriverState *bs = child->bs; 1666 BdrvTrackedRequest req; 1667 BdrvRequestPadding pad; 1668 int ret; 1669 1670 trace_bdrv_co_preadv(bs, offset, bytes, flags); 1671 1672 if (!bdrv_is_inserted(bs)) { 1673 return -ENOMEDIUM; 1674 } 1675 1676 ret = bdrv_check_request32(offset, bytes); 1677 if (ret < 0) { 1678 return ret; 1679 } 1680 1681 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1682 /* 1683 * Aligning zero request is nonsense. Even if driver has special meaning 1684 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1685 * it to driver due to request_alignment. 1686 * 1687 * Still, no reason to return an error if someone do unaligned 1688 * zero-length read occasionally. 1689 */ 1690 return 0; 1691 } 1692 1693 bdrv_inc_in_flight(bs); 1694 1695 /* Don't do copy-on-read if we read data before write operation */ 1696 if (qatomic_read(&bs->copy_on_read)) { 1697 flags |= BDRV_REQ_COPY_ON_READ; 1698 } 1699 1700 bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad); 1701 1702 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1703 ret = bdrv_aligned_preadv(child, &req, offset, bytes, 1704 bs->bl.request_alignment, 1705 qiov, qiov_offset, flags); 1706 tracked_request_end(&req); 1707 bdrv_dec_in_flight(bs); 1708 1709 bdrv_padding_destroy(&pad); 1710 1711 return ret; 1712 } 1713 1714 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1715 int64_t offset, int bytes, BdrvRequestFlags flags) 1716 { 1717 BlockDriver *drv = bs->drv; 1718 QEMUIOVector qiov; 1719 void *buf = NULL; 1720 int ret = 0; 1721 bool need_flush = false; 1722 int head = 0; 1723 int tail = 0; 1724 1725 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1726 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1727 bs->bl.request_alignment); 1728 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1729 1730 if (!drv) { 1731 return -ENOMEDIUM; 1732 } 1733 1734 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1735 return -ENOTSUP; 1736 } 1737 1738 assert(alignment % bs->bl.request_alignment == 0); 1739 head = offset % alignment; 1740 tail = (offset + bytes) % alignment; 1741 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1742 assert(max_write_zeroes >= bs->bl.request_alignment); 1743 1744 while (bytes > 0 && !ret) { 1745 int num = bytes; 1746 1747 /* Align request. Block drivers can expect the "bulk" of the request 1748 * to be aligned, and that unaligned requests do not cross cluster 1749 * boundaries. 1750 */ 1751 if (head) { 1752 /* Make a small request up to the first aligned sector. For 1753 * convenience, limit this request to max_transfer even if 1754 * we don't need to fall back to writes. */ 1755 num = MIN(MIN(bytes, max_transfer), alignment - head); 1756 head = (head + num) % alignment; 1757 assert(num < max_write_zeroes); 1758 } else if (tail && num > alignment) { 1759 /* Shorten the request to the last aligned sector. */ 1760 num -= tail; 1761 } 1762 1763 /* limit request size */ 1764 if (num > max_write_zeroes) { 1765 num = max_write_zeroes; 1766 } 1767 1768 ret = -ENOTSUP; 1769 /* First try the efficient write zeroes operation */ 1770 if (drv->bdrv_co_pwrite_zeroes) { 1771 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1772 flags & bs->supported_zero_flags); 1773 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1774 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1775 need_flush = true; 1776 } 1777 } else { 1778 assert(!bs->supported_zero_flags); 1779 } 1780 1781 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 1782 /* Fall back to bounce buffer if write zeroes is unsupported */ 1783 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1784 1785 if ((flags & BDRV_REQ_FUA) && 1786 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1787 /* No need for bdrv_driver_pwrite() to do a fallback 1788 * flush on each chunk; use just one at the end */ 1789 write_flags &= ~BDRV_REQ_FUA; 1790 need_flush = true; 1791 } 1792 num = MIN(num, max_transfer); 1793 if (buf == NULL) { 1794 buf = qemu_try_blockalign0(bs, num); 1795 if (buf == NULL) { 1796 ret = -ENOMEM; 1797 goto fail; 1798 } 1799 } 1800 qemu_iovec_init_buf(&qiov, buf, num); 1801 1802 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 1803 1804 /* Keep bounce buffer around if it is big enough for all 1805 * all future requests. 1806 */ 1807 if (num < max_transfer) { 1808 qemu_vfree(buf); 1809 buf = NULL; 1810 } 1811 } 1812 1813 offset += num; 1814 bytes -= num; 1815 } 1816 1817 fail: 1818 if (ret == 0 && need_flush) { 1819 ret = bdrv_co_flush(bs); 1820 } 1821 qemu_vfree(buf); 1822 return ret; 1823 } 1824 1825 static inline int coroutine_fn 1826 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes, 1827 BdrvTrackedRequest *req, int flags) 1828 { 1829 BlockDriverState *bs = child->bs; 1830 bool waited; 1831 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1832 1833 if (bs->read_only) { 1834 return -EPERM; 1835 } 1836 1837 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1838 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1839 assert(!(flags & ~BDRV_REQ_MASK)); 1840 1841 if (flags & BDRV_REQ_SERIALISING) { 1842 waited = bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs)); 1843 /* 1844 * For a misaligned request we should have already waited earlier, 1845 * because we come after bdrv_padding_rmw_read which must be called 1846 * with the request already marked as serialising. 1847 */ 1848 assert(!waited || 1849 (req->offset == req->overlap_offset && 1850 req->bytes == req->overlap_bytes)); 1851 } else { 1852 bdrv_wait_serialising_requests(req); 1853 } 1854 1855 assert(req->overlap_offset <= offset); 1856 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1857 assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE); 1858 1859 switch (req->type) { 1860 case BDRV_TRACKED_WRITE: 1861 case BDRV_TRACKED_DISCARD: 1862 if (flags & BDRV_REQ_WRITE_UNCHANGED) { 1863 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1864 } else { 1865 assert(child->perm & BLK_PERM_WRITE); 1866 } 1867 return notifier_with_return_list_notify(&bs->before_write_notifiers, 1868 req); 1869 case BDRV_TRACKED_TRUNCATE: 1870 assert(child->perm & BLK_PERM_RESIZE); 1871 return 0; 1872 default: 1873 abort(); 1874 } 1875 } 1876 1877 static inline void coroutine_fn 1878 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes, 1879 BdrvTrackedRequest *req, int ret) 1880 { 1881 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1882 BlockDriverState *bs = child->bs; 1883 1884 qatomic_inc(&bs->write_gen); 1885 1886 /* 1887 * Discard cannot extend the image, but in error handling cases, such as 1888 * when reverting a qcow2 cluster allocation, the discarded range can pass 1889 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 1890 * here. Instead, just skip it, since semantically a discard request 1891 * beyond EOF cannot expand the image anyway. 1892 */ 1893 if (ret == 0 && 1894 (req->type == BDRV_TRACKED_TRUNCATE || 1895 end_sector > bs->total_sectors) && 1896 req->type != BDRV_TRACKED_DISCARD) { 1897 bs->total_sectors = end_sector; 1898 bdrv_parent_cb_resize(bs); 1899 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 1900 } 1901 if (req->bytes) { 1902 switch (req->type) { 1903 case BDRV_TRACKED_WRITE: 1904 stat64_max(&bs->wr_highest_offset, offset + bytes); 1905 /* fall through, to set dirty bits */ 1906 case BDRV_TRACKED_DISCARD: 1907 bdrv_set_dirty(bs, offset, bytes); 1908 break; 1909 default: 1910 break; 1911 } 1912 } 1913 } 1914 1915 /* 1916 * Forwards an already correctly aligned write request to the BlockDriver, 1917 * after possibly fragmenting it. 1918 */ 1919 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 1920 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1921 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 1922 { 1923 BlockDriverState *bs = child->bs; 1924 BlockDriver *drv = bs->drv; 1925 int ret; 1926 1927 uint64_t bytes_remaining = bytes; 1928 int max_transfer; 1929 1930 if (!drv) { 1931 return -ENOMEDIUM; 1932 } 1933 1934 if (bdrv_has_readonly_bitmaps(bs)) { 1935 return -EPERM; 1936 } 1937 1938 assert(is_power_of_2(align)); 1939 assert((offset & (align - 1)) == 0); 1940 assert((bytes & (align - 1)) == 0); 1941 assert(!qiov || qiov_offset + bytes <= qiov->size); 1942 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1943 align); 1944 1945 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 1946 1947 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1948 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 1949 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 1950 flags |= BDRV_REQ_ZERO_WRITE; 1951 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1952 flags |= BDRV_REQ_MAY_UNMAP; 1953 } 1954 } 1955 1956 if (ret < 0) { 1957 /* Do nothing, write notifier decided to fail this request */ 1958 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1959 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1960 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 1961 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 1962 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 1963 qiov, qiov_offset); 1964 } else if (bytes <= max_transfer) { 1965 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1966 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 1967 } else { 1968 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1969 while (bytes_remaining) { 1970 int num = MIN(bytes_remaining, max_transfer); 1971 int local_flags = flags; 1972 1973 assert(num); 1974 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 1975 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1976 /* If FUA is going to be emulated by flush, we only 1977 * need to flush on the last iteration */ 1978 local_flags &= ~BDRV_REQ_FUA; 1979 } 1980 1981 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 1982 num, qiov, 1983 qiov_offset + bytes - bytes_remaining, 1984 local_flags); 1985 if (ret < 0) { 1986 break; 1987 } 1988 bytes_remaining -= num; 1989 } 1990 } 1991 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 1992 1993 if (ret >= 0) { 1994 ret = 0; 1995 } 1996 bdrv_co_write_req_finish(child, offset, bytes, req, ret); 1997 1998 return ret; 1999 } 2000 2001 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 2002 int64_t offset, 2003 unsigned int bytes, 2004 BdrvRequestFlags flags, 2005 BdrvTrackedRequest *req) 2006 { 2007 BlockDriverState *bs = child->bs; 2008 QEMUIOVector local_qiov; 2009 uint64_t align = bs->bl.request_alignment; 2010 int ret = 0; 2011 bool padding; 2012 BdrvRequestPadding pad; 2013 2014 padding = bdrv_init_padding(bs, offset, bytes, &pad); 2015 if (padding) { 2016 bdrv_mark_request_serialising(req, align); 2017 2018 bdrv_padding_rmw_read(child, req, &pad, true); 2019 2020 if (pad.head || pad.merge_reads) { 2021 int64_t aligned_offset = offset & ~(align - 1); 2022 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 2023 2024 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 2025 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 2026 align, &local_qiov, 0, 2027 flags & ~BDRV_REQ_ZERO_WRITE); 2028 if (ret < 0 || pad.merge_reads) { 2029 /* Error or all work is done */ 2030 goto out; 2031 } 2032 offset += write_bytes - pad.head; 2033 bytes -= write_bytes - pad.head; 2034 } 2035 } 2036 2037 assert(!bytes || (offset & (align - 1)) == 0); 2038 if (bytes >= align) { 2039 /* Write the aligned part in the middle. */ 2040 uint64_t aligned_bytes = bytes & ~(align - 1); 2041 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 2042 NULL, 0, flags); 2043 if (ret < 0) { 2044 goto out; 2045 } 2046 bytes -= aligned_bytes; 2047 offset += aligned_bytes; 2048 } 2049 2050 assert(!bytes || (offset & (align - 1)) == 0); 2051 if (bytes) { 2052 assert(align == pad.tail + bytes); 2053 2054 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 2055 ret = bdrv_aligned_pwritev(child, req, offset, align, align, 2056 &local_qiov, 0, 2057 flags & ~BDRV_REQ_ZERO_WRITE); 2058 } 2059 2060 out: 2061 bdrv_padding_destroy(&pad); 2062 2063 return ret; 2064 } 2065 2066 /* 2067 * Handle a write request in coroutine context 2068 */ 2069 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 2070 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 2071 BdrvRequestFlags flags) 2072 { 2073 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 2074 } 2075 2076 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 2077 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset, 2078 BdrvRequestFlags flags) 2079 { 2080 BlockDriverState *bs = child->bs; 2081 BdrvTrackedRequest req; 2082 uint64_t align = bs->bl.request_alignment; 2083 BdrvRequestPadding pad; 2084 int ret; 2085 2086 trace_bdrv_co_pwritev(child->bs, offset, bytes, flags); 2087 2088 if (!bdrv_is_inserted(bs)) { 2089 return -ENOMEDIUM; 2090 } 2091 2092 ret = bdrv_check_request32(offset, bytes); 2093 if (ret < 0) { 2094 return ret; 2095 } 2096 2097 /* If the request is misaligned then we can't make it efficient */ 2098 if ((flags & BDRV_REQ_NO_FALLBACK) && 2099 !QEMU_IS_ALIGNED(offset | bytes, align)) 2100 { 2101 return -ENOTSUP; 2102 } 2103 2104 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2105 /* 2106 * Aligning zero request is nonsense. Even if driver has special meaning 2107 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2108 * it to driver due to request_alignment. 2109 * 2110 * Still, no reason to return an error if someone do unaligned 2111 * zero-length write occasionally. 2112 */ 2113 return 0; 2114 } 2115 2116 bdrv_inc_in_flight(bs); 2117 /* 2118 * Align write if necessary by performing a read-modify-write cycle. 2119 * Pad qiov with the read parts and be sure to have a tracked request not 2120 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 2121 */ 2122 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 2123 2124 if (flags & BDRV_REQ_ZERO_WRITE) { 2125 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 2126 goto out; 2127 } 2128 2129 if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) { 2130 bdrv_mark_request_serialising(&req, align); 2131 bdrv_padding_rmw_read(child, &req, &pad, false); 2132 } 2133 2134 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 2135 qiov, qiov_offset, flags); 2136 2137 bdrv_padding_destroy(&pad); 2138 2139 out: 2140 tracked_request_end(&req); 2141 bdrv_dec_in_flight(bs); 2142 2143 return ret; 2144 } 2145 2146 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2147 int bytes, BdrvRequestFlags flags) 2148 { 2149 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 2150 2151 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 2152 flags &= ~BDRV_REQ_MAY_UNMAP; 2153 } 2154 2155 return bdrv_co_pwritev(child, offset, bytes, NULL, 2156 BDRV_REQ_ZERO_WRITE | flags); 2157 } 2158 2159 /* 2160 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 2161 */ 2162 int bdrv_flush_all(void) 2163 { 2164 BdrvNextIterator it; 2165 BlockDriverState *bs = NULL; 2166 int result = 0; 2167 2168 /* 2169 * bdrv queue is managed by record/replay, 2170 * creating new flush request for stopping 2171 * the VM may break the determinism 2172 */ 2173 if (replay_events_enabled()) { 2174 return result; 2175 } 2176 2177 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 2178 AioContext *aio_context = bdrv_get_aio_context(bs); 2179 int ret; 2180 2181 aio_context_acquire(aio_context); 2182 ret = bdrv_flush(bs); 2183 if (ret < 0 && !result) { 2184 result = ret; 2185 } 2186 aio_context_release(aio_context); 2187 } 2188 2189 return result; 2190 } 2191 2192 /* 2193 * Returns the allocation status of the specified sectors. 2194 * Drivers not implementing the functionality are assumed to not support 2195 * backing files, hence all their sectors are reported as allocated. 2196 * 2197 * If 'want_zero' is true, the caller is querying for mapping 2198 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 2199 * _ZERO where possible; otherwise, the result favors larger 'pnum', 2200 * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2201 * 2202 * If 'offset' is beyond the end of the disk image the return value is 2203 * BDRV_BLOCK_EOF and 'pnum' is set to 0. 2204 * 2205 * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2206 * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2207 * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 2208 * 2209 * 'pnum' is set to the number of bytes (including and immediately 2210 * following the specified offset) that are easily known to be in the 2211 * same allocated/unallocated state. Note that a second call starting 2212 * at the original offset plus returned pnum may have the same status. 2213 * The returned value is non-zero on success except at end-of-file. 2214 * 2215 * Returns negative errno on failure. Otherwise, if the 2216 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 2217 * set to the host mapping and BDS corresponding to the guest offset. 2218 */ 2219 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, 2220 bool want_zero, 2221 int64_t offset, int64_t bytes, 2222 int64_t *pnum, int64_t *map, 2223 BlockDriverState **file) 2224 { 2225 int64_t total_size; 2226 int64_t n; /* bytes */ 2227 int ret; 2228 int64_t local_map = 0; 2229 BlockDriverState *local_file = NULL; 2230 int64_t aligned_offset, aligned_bytes; 2231 uint32_t align; 2232 bool has_filtered_child; 2233 2234 assert(pnum); 2235 *pnum = 0; 2236 total_size = bdrv_getlength(bs); 2237 if (total_size < 0) { 2238 ret = total_size; 2239 goto early_out; 2240 } 2241 2242 if (offset >= total_size) { 2243 ret = BDRV_BLOCK_EOF; 2244 goto early_out; 2245 } 2246 if (!bytes) { 2247 ret = 0; 2248 goto early_out; 2249 } 2250 2251 n = total_size - offset; 2252 if (n < bytes) { 2253 bytes = n; 2254 } 2255 2256 /* Must be non-NULL or bdrv_getlength() would have failed */ 2257 assert(bs->drv); 2258 has_filtered_child = bdrv_filter_child(bs); 2259 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 2260 *pnum = bytes; 2261 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 2262 if (offset + bytes == total_size) { 2263 ret |= BDRV_BLOCK_EOF; 2264 } 2265 if (bs->drv->protocol_name) { 2266 ret |= BDRV_BLOCK_OFFSET_VALID; 2267 local_map = offset; 2268 local_file = bs; 2269 } 2270 goto early_out; 2271 } 2272 2273 bdrv_inc_in_flight(bs); 2274 2275 /* Round out to request_alignment boundaries */ 2276 align = bs->bl.request_alignment; 2277 aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2278 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2279 2280 if (bs->drv->bdrv_co_block_status) { 2281 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 2282 aligned_bytes, pnum, &local_map, 2283 &local_file); 2284 } else { 2285 /* Default code for filters */ 2286 2287 local_file = bdrv_filter_bs(bs); 2288 assert(local_file); 2289 2290 *pnum = aligned_bytes; 2291 local_map = aligned_offset; 2292 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2293 } 2294 if (ret < 0) { 2295 *pnum = 0; 2296 goto out; 2297 } 2298 2299 /* 2300 * The driver's result must be a non-zero multiple of request_alignment. 2301 * Clamp pnum and adjust map to original request. 2302 */ 2303 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2304 align > offset - aligned_offset); 2305 if (ret & BDRV_BLOCK_RECURSE) { 2306 assert(ret & BDRV_BLOCK_DATA); 2307 assert(ret & BDRV_BLOCK_OFFSET_VALID); 2308 assert(!(ret & BDRV_BLOCK_ZERO)); 2309 } 2310 2311 *pnum -= offset - aligned_offset; 2312 if (*pnum > bytes) { 2313 *pnum = bytes; 2314 } 2315 if (ret & BDRV_BLOCK_OFFSET_VALID) { 2316 local_map += offset - aligned_offset; 2317 } 2318 2319 if (ret & BDRV_BLOCK_RAW) { 2320 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 2321 ret = bdrv_co_block_status(local_file, want_zero, local_map, 2322 *pnum, pnum, &local_map, &local_file); 2323 goto out; 2324 } 2325 2326 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 2327 ret |= BDRV_BLOCK_ALLOCATED; 2328 } else if (bs->drv->supports_backing) { 2329 BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2330 2331 if (!cow_bs) { 2332 ret |= BDRV_BLOCK_ZERO; 2333 } else if (want_zero) { 2334 int64_t size2 = bdrv_getlength(cow_bs); 2335 2336 if (size2 >= 0 && offset >= size2) { 2337 ret |= BDRV_BLOCK_ZERO; 2338 } 2339 } 2340 } 2341 2342 if (want_zero && ret & BDRV_BLOCK_RECURSE && 2343 local_file && local_file != bs && 2344 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 2345 (ret & BDRV_BLOCK_OFFSET_VALID)) { 2346 int64_t file_pnum; 2347 int ret2; 2348 2349 ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 2350 *pnum, &file_pnum, NULL, NULL); 2351 if (ret2 >= 0) { 2352 /* Ignore errors. This is just providing extra information, it 2353 * is useful but not necessary. 2354 */ 2355 if (ret2 & BDRV_BLOCK_EOF && 2356 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2357 /* 2358 * It is valid for the format block driver to read 2359 * beyond the end of the underlying file's current 2360 * size; such areas read as zero. 2361 */ 2362 ret |= BDRV_BLOCK_ZERO; 2363 } else { 2364 /* Limit request to the range reported by the protocol driver */ 2365 *pnum = file_pnum; 2366 ret |= (ret2 & BDRV_BLOCK_ZERO); 2367 } 2368 } 2369 } 2370 2371 out: 2372 bdrv_dec_in_flight(bs); 2373 if (ret >= 0 && offset + *pnum == total_size) { 2374 ret |= BDRV_BLOCK_EOF; 2375 } 2376 early_out: 2377 if (file) { 2378 *file = local_file; 2379 } 2380 if (map) { 2381 *map = local_map; 2382 } 2383 return ret; 2384 } 2385 2386 int coroutine_fn 2387 bdrv_co_common_block_status_above(BlockDriverState *bs, 2388 BlockDriverState *base, 2389 bool include_base, 2390 bool want_zero, 2391 int64_t offset, 2392 int64_t bytes, 2393 int64_t *pnum, 2394 int64_t *map, 2395 BlockDriverState **file, 2396 int *depth) 2397 { 2398 int ret; 2399 BlockDriverState *p; 2400 int64_t eof = 0; 2401 int dummy; 2402 2403 assert(!include_base || base); /* Can't include NULL base */ 2404 2405 if (!depth) { 2406 depth = &dummy; 2407 } 2408 *depth = 0; 2409 2410 if (!include_base && bs == base) { 2411 *pnum = bytes; 2412 return 0; 2413 } 2414 2415 ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file); 2416 ++*depth; 2417 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 2418 return ret; 2419 } 2420 2421 if (ret & BDRV_BLOCK_EOF) { 2422 eof = offset + *pnum; 2423 } 2424 2425 assert(*pnum <= bytes); 2426 bytes = *pnum; 2427 2428 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 2429 p = bdrv_filter_or_cow_bs(p)) 2430 { 2431 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 2432 file); 2433 ++*depth; 2434 if (ret < 0) { 2435 return ret; 2436 } 2437 if (*pnum == 0) { 2438 /* 2439 * The top layer deferred to this layer, and because this layer is 2440 * short, any zeroes that we synthesize beyond EOF behave as if they 2441 * were allocated at this layer. 2442 * 2443 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 2444 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2445 * below. 2446 */ 2447 assert(ret & BDRV_BLOCK_EOF); 2448 *pnum = bytes; 2449 if (file) { 2450 *file = p; 2451 } 2452 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2453 break; 2454 } 2455 if (ret & BDRV_BLOCK_ALLOCATED) { 2456 /* 2457 * We've found the node and the status, we must break. 2458 * 2459 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 2460 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2461 * below. 2462 */ 2463 ret &= ~BDRV_BLOCK_EOF; 2464 break; 2465 } 2466 2467 if (p == base) { 2468 assert(include_base); 2469 break; 2470 } 2471 2472 /* 2473 * OK, [offset, offset + *pnum) region is unallocated on this layer, 2474 * let's continue the diving. 2475 */ 2476 assert(*pnum <= bytes); 2477 bytes = *pnum; 2478 } 2479 2480 if (offset + *pnum == eof) { 2481 ret |= BDRV_BLOCK_EOF; 2482 } 2483 2484 return ret; 2485 } 2486 2487 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 2488 int64_t offset, int64_t bytes, int64_t *pnum, 2489 int64_t *map, BlockDriverState **file) 2490 { 2491 return bdrv_common_block_status_above(bs, base, false, true, offset, bytes, 2492 pnum, map, file, NULL); 2493 } 2494 2495 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2496 int64_t *pnum, int64_t *map, BlockDriverState **file) 2497 { 2498 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 2499 offset, bytes, pnum, map, file); 2500 } 2501 2502 /* 2503 * Check @bs (and its backing chain) to see if the range defined 2504 * by @offset and @bytes is known to read as zeroes. 2505 * Return 1 if that is the case, 0 otherwise and -errno on error. 2506 * This test is meant to be fast rather than accurate so returning 0 2507 * does not guarantee non-zero data. 2508 */ 2509 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset, 2510 int64_t bytes) 2511 { 2512 int ret; 2513 int64_t pnum = bytes; 2514 2515 if (!bytes) { 2516 return 1; 2517 } 2518 2519 ret = bdrv_common_block_status_above(bs, NULL, false, false, offset, 2520 bytes, &pnum, NULL, NULL, NULL); 2521 2522 if (ret < 0) { 2523 return ret; 2524 } 2525 2526 return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO); 2527 } 2528 2529 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2530 int64_t bytes, int64_t *pnum) 2531 { 2532 int ret; 2533 int64_t dummy; 2534 2535 ret = bdrv_common_block_status_above(bs, bs, true, false, offset, 2536 bytes, pnum ? pnum : &dummy, NULL, 2537 NULL, NULL); 2538 if (ret < 0) { 2539 return ret; 2540 } 2541 return !!(ret & BDRV_BLOCK_ALLOCATED); 2542 } 2543 2544 /* 2545 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 2546 * 2547 * Return a positive depth if (a prefix of) the given range is allocated 2548 * in any image between BASE and TOP (BASE is only included if include_base 2549 * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth. 2550 * BASE can be NULL to check if the given offset is allocated in any 2551 * image of the chain. Return 0 otherwise, or negative errno on 2552 * failure. 2553 * 2554 * 'pnum' is set to the number of bytes (including and immediately 2555 * following the specified offset) that are known to be in the same 2556 * allocated/unallocated state. Note that a subsequent call starting 2557 * at 'offset + *pnum' may return the same allocation status (in other 2558 * words, the result is not necessarily the maximum possible range); 2559 * but 'pnum' will only be 0 when end of file is reached. 2560 */ 2561 int bdrv_is_allocated_above(BlockDriverState *top, 2562 BlockDriverState *base, 2563 bool include_base, int64_t offset, 2564 int64_t bytes, int64_t *pnum) 2565 { 2566 int depth; 2567 int ret = bdrv_common_block_status_above(top, base, include_base, false, 2568 offset, bytes, pnum, NULL, NULL, 2569 &depth); 2570 if (ret < 0) { 2571 return ret; 2572 } 2573 2574 if (ret & BDRV_BLOCK_ALLOCATED) { 2575 return depth; 2576 } 2577 return 0; 2578 } 2579 2580 int coroutine_fn 2581 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2582 { 2583 BlockDriver *drv = bs->drv; 2584 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2585 int ret = -ENOTSUP; 2586 2587 if (!drv) { 2588 return -ENOMEDIUM; 2589 } 2590 2591 bdrv_inc_in_flight(bs); 2592 2593 if (drv->bdrv_load_vmstate) { 2594 ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2595 } else if (child_bs) { 2596 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 2597 } 2598 2599 bdrv_dec_in_flight(bs); 2600 2601 return ret; 2602 } 2603 2604 int coroutine_fn 2605 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2606 { 2607 BlockDriver *drv = bs->drv; 2608 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2609 int ret = -ENOTSUP; 2610 2611 if (!drv) { 2612 return -ENOMEDIUM; 2613 } 2614 2615 bdrv_inc_in_flight(bs); 2616 2617 if (drv->bdrv_save_vmstate) { 2618 ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2619 } else if (child_bs) { 2620 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2621 } 2622 2623 bdrv_dec_in_flight(bs); 2624 2625 return ret; 2626 } 2627 2628 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 2629 int64_t pos, int size) 2630 { 2631 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2632 int ret = bdrv_writev_vmstate(bs, &qiov, pos); 2633 2634 return ret < 0 ? ret : size; 2635 } 2636 2637 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 2638 int64_t pos, int size) 2639 { 2640 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2641 int ret = bdrv_readv_vmstate(bs, &qiov, pos); 2642 2643 return ret < 0 ? ret : size; 2644 } 2645 2646 /**************************************************************/ 2647 /* async I/Os */ 2648 2649 void bdrv_aio_cancel(BlockAIOCB *acb) 2650 { 2651 qemu_aio_ref(acb); 2652 bdrv_aio_cancel_async(acb); 2653 while (acb->refcnt > 1) { 2654 if (acb->aiocb_info->get_aio_context) { 2655 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2656 } else if (acb->bs) { 2657 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 2658 * assert that we're not using an I/O thread. Thread-safe 2659 * code should use bdrv_aio_cancel_async exclusively. 2660 */ 2661 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 2662 aio_poll(bdrv_get_aio_context(acb->bs), true); 2663 } else { 2664 abort(); 2665 } 2666 } 2667 qemu_aio_unref(acb); 2668 } 2669 2670 /* Async version of aio cancel. The caller is not blocked if the acb implements 2671 * cancel_async, otherwise we do nothing and let the request normally complete. 2672 * In either case the completion callback must be called. */ 2673 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2674 { 2675 if (acb->aiocb_info->cancel_async) { 2676 acb->aiocb_info->cancel_async(acb); 2677 } 2678 } 2679 2680 /**************************************************************/ 2681 /* Coroutine block device emulation */ 2682 2683 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2684 { 2685 BdrvChild *primary_child = bdrv_primary_child(bs); 2686 BdrvChild *child; 2687 int current_gen; 2688 int ret = 0; 2689 2690 bdrv_inc_in_flight(bs); 2691 2692 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2693 bdrv_is_sg(bs)) { 2694 goto early_exit; 2695 } 2696 2697 qemu_co_mutex_lock(&bs->reqs_lock); 2698 current_gen = qatomic_read(&bs->write_gen); 2699 2700 /* Wait until any previous flushes are completed */ 2701 while (bs->active_flush_req) { 2702 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 2703 } 2704 2705 /* Flushes reach this point in nondecreasing current_gen order. */ 2706 bs->active_flush_req = true; 2707 qemu_co_mutex_unlock(&bs->reqs_lock); 2708 2709 /* Write back all layers by calling one driver function */ 2710 if (bs->drv->bdrv_co_flush) { 2711 ret = bs->drv->bdrv_co_flush(bs); 2712 goto out; 2713 } 2714 2715 /* Write back cached data to the OS even with cache=unsafe */ 2716 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 2717 if (bs->drv->bdrv_co_flush_to_os) { 2718 ret = bs->drv->bdrv_co_flush_to_os(bs); 2719 if (ret < 0) { 2720 goto out; 2721 } 2722 } 2723 2724 /* But don't actually force it to the disk with cache=unsafe */ 2725 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2726 goto flush_children; 2727 } 2728 2729 /* Check if we really need to flush anything */ 2730 if (bs->flushed_gen == current_gen) { 2731 goto flush_children; 2732 } 2733 2734 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 2735 if (!bs->drv) { 2736 /* bs->drv->bdrv_co_flush() might have ejected the BDS 2737 * (even in case of apparent success) */ 2738 ret = -ENOMEDIUM; 2739 goto out; 2740 } 2741 if (bs->drv->bdrv_co_flush_to_disk) { 2742 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2743 } else if (bs->drv->bdrv_aio_flush) { 2744 BlockAIOCB *acb; 2745 CoroutineIOCompletion co = { 2746 .coroutine = qemu_coroutine_self(), 2747 }; 2748 2749 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2750 if (acb == NULL) { 2751 ret = -EIO; 2752 } else { 2753 qemu_coroutine_yield(); 2754 ret = co.ret; 2755 } 2756 } else { 2757 /* 2758 * Some block drivers always operate in either writethrough or unsafe 2759 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2760 * know how the server works (because the behaviour is hardcoded or 2761 * depends on server-side configuration), so we can't ensure that 2762 * everything is safe on disk. Returning an error doesn't work because 2763 * that would break guests even if the server operates in writethrough 2764 * mode. 2765 * 2766 * Let's hope the user knows what he's doing. 2767 */ 2768 ret = 0; 2769 } 2770 2771 if (ret < 0) { 2772 goto out; 2773 } 2774 2775 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2776 * in the case of cache=unsafe, so there are no useless flushes. 2777 */ 2778 flush_children: 2779 ret = 0; 2780 QLIST_FOREACH(child, &bs->children, next) { 2781 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 2782 int this_child_ret = bdrv_co_flush(child->bs); 2783 if (!ret) { 2784 ret = this_child_ret; 2785 } 2786 } 2787 } 2788 2789 out: 2790 /* Notify any pending flushes that we have completed */ 2791 if (ret == 0) { 2792 bs->flushed_gen = current_gen; 2793 } 2794 2795 qemu_co_mutex_lock(&bs->reqs_lock); 2796 bs->active_flush_req = false; 2797 /* Return value is ignored - it's ok if wait queue is empty */ 2798 qemu_co_queue_next(&bs->flush_queue); 2799 qemu_co_mutex_unlock(&bs->reqs_lock); 2800 2801 early_exit: 2802 bdrv_dec_in_flight(bs); 2803 return ret; 2804 } 2805 2806 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 2807 int64_t bytes) 2808 { 2809 BdrvTrackedRequest req; 2810 int max_pdiscard, ret; 2811 int head, tail, align; 2812 BlockDriverState *bs = child->bs; 2813 2814 if (!bs || !bs->drv || !bdrv_is_inserted(bs)) { 2815 return -ENOMEDIUM; 2816 } 2817 2818 if (bdrv_has_readonly_bitmaps(bs)) { 2819 return -EPERM; 2820 } 2821 2822 ret = bdrv_check_request(offset, bytes); 2823 if (ret < 0) { 2824 return ret; 2825 } 2826 2827 /* Do nothing if disabled. */ 2828 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2829 return 0; 2830 } 2831 2832 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 2833 return 0; 2834 } 2835 2836 /* Discard is advisory, but some devices track and coalesce 2837 * unaligned requests, so we must pass everything down rather than 2838 * round here. Still, most devices will just silently ignore 2839 * unaligned requests (by returning -ENOTSUP), so we must fragment 2840 * the request accordingly. */ 2841 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2842 assert(align % bs->bl.request_alignment == 0); 2843 head = offset % align; 2844 tail = (offset + bytes) % align; 2845 2846 bdrv_inc_in_flight(bs); 2847 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 2848 2849 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 2850 if (ret < 0) { 2851 goto out; 2852 } 2853 2854 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 2855 align); 2856 assert(max_pdiscard >= bs->bl.request_alignment); 2857 2858 while (bytes > 0) { 2859 int64_t num = bytes; 2860 2861 if (head) { 2862 /* Make small requests to get to alignment boundaries. */ 2863 num = MIN(bytes, align - head); 2864 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 2865 num %= bs->bl.request_alignment; 2866 } 2867 head = (head + num) % align; 2868 assert(num < max_pdiscard); 2869 } else if (tail) { 2870 if (num > align) { 2871 /* Shorten the request to the last aligned cluster. */ 2872 num -= tail; 2873 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 2874 tail > bs->bl.request_alignment) { 2875 tail %= bs->bl.request_alignment; 2876 num -= tail; 2877 } 2878 } 2879 /* limit request size */ 2880 if (num > max_pdiscard) { 2881 num = max_pdiscard; 2882 } 2883 2884 if (!bs->drv) { 2885 ret = -ENOMEDIUM; 2886 goto out; 2887 } 2888 if (bs->drv->bdrv_co_pdiscard) { 2889 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 2890 } else { 2891 BlockAIOCB *acb; 2892 CoroutineIOCompletion co = { 2893 .coroutine = qemu_coroutine_self(), 2894 }; 2895 2896 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 2897 bdrv_co_io_em_complete, &co); 2898 if (acb == NULL) { 2899 ret = -EIO; 2900 goto out; 2901 } else { 2902 qemu_coroutine_yield(); 2903 ret = co.ret; 2904 } 2905 } 2906 if (ret && ret != -ENOTSUP) { 2907 goto out; 2908 } 2909 2910 offset += num; 2911 bytes -= num; 2912 } 2913 ret = 0; 2914 out: 2915 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 2916 tracked_request_end(&req); 2917 bdrv_dec_in_flight(bs); 2918 return ret; 2919 } 2920 2921 int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 2922 { 2923 BlockDriver *drv = bs->drv; 2924 CoroutineIOCompletion co = { 2925 .coroutine = qemu_coroutine_self(), 2926 }; 2927 BlockAIOCB *acb; 2928 2929 bdrv_inc_in_flight(bs); 2930 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 2931 co.ret = -ENOTSUP; 2932 goto out; 2933 } 2934 2935 if (drv->bdrv_co_ioctl) { 2936 co.ret = drv->bdrv_co_ioctl(bs, req, buf); 2937 } else { 2938 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 2939 if (!acb) { 2940 co.ret = -ENOTSUP; 2941 goto out; 2942 } 2943 qemu_coroutine_yield(); 2944 } 2945 out: 2946 bdrv_dec_in_flight(bs); 2947 return co.ret; 2948 } 2949 2950 void *qemu_blockalign(BlockDriverState *bs, size_t size) 2951 { 2952 return qemu_memalign(bdrv_opt_mem_align(bs), size); 2953 } 2954 2955 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 2956 { 2957 return memset(qemu_blockalign(bs, size), 0, size); 2958 } 2959 2960 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 2961 { 2962 size_t align = bdrv_opt_mem_align(bs); 2963 2964 /* Ensure that NULL is never returned on success */ 2965 assert(align > 0); 2966 if (size == 0) { 2967 size = align; 2968 } 2969 2970 return qemu_try_memalign(align, size); 2971 } 2972 2973 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 2974 { 2975 void *mem = qemu_try_blockalign(bs, size); 2976 2977 if (mem) { 2978 memset(mem, 0, size); 2979 } 2980 2981 return mem; 2982 } 2983 2984 /* 2985 * Check if all memory in this vector is sector aligned. 2986 */ 2987 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 2988 { 2989 int i; 2990 size_t alignment = bdrv_min_mem_align(bs); 2991 2992 for (i = 0; i < qiov->niov; i++) { 2993 if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 2994 return false; 2995 } 2996 if (qiov->iov[i].iov_len % alignment) { 2997 return false; 2998 } 2999 } 3000 3001 return true; 3002 } 3003 3004 void bdrv_add_before_write_notifier(BlockDriverState *bs, 3005 NotifierWithReturn *notifier) 3006 { 3007 notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 3008 } 3009 3010 void bdrv_io_plug(BlockDriverState *bs) 3011 { 3012 BdrvChild *child; 3013 3014 QLIST_FOREACH(child, &bs->children, next) { 3015 bdrv_io_plug(child->bs); 3016 } 3017 3018 if (qatomic_fetch_inc(&bs->io_plugged) == 0) { 3019 BlockDriver *drv = bs->drv; 3020 if (drv && drv->bdrv_io_plug) { 3021 drv->bdrv_io_plug(bs); 3022 } 3023 } 3024 } 3025 3026 void bdrv_io_unplug(BlockDriverState *bs) 3027 { 3028 BdrvChild *child; 3029 3030 assert(bs->io_plugged); 3031 if (qatomic_fetch_dec(&bs->io_plugged) == 1) { 3032 BlockDriver *drv = bs->drv; 3033 if (drv && drv->bdrv_io_unplug) { 3034 drv->bdrv_io_unplug(bs); 3035 } 3036 } 3037 3038 QLIST_FOREACH(child, &bs->children, next) { 3039 bdrv_io_unplug(child->bs); 3040 } 3041 } 3042 3043 void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size) 3044 { 3045 BdrvChild *child; 3046 3047 if (bs->drv && bs->drv->bdrv_register_buf) { 3048 bs->drv->bdrv_register_buf(bs, host, size); 3049 } 3050 QLIST_FOREACH(child, &bs->children, next) { 3051 bdrv_register_buf(child->bs, host, size); 3052 } 3053 } 3054 3055 void bdrv_unregister_buf(BlockDriverState *bs, void *host) 3056 { 3057 BdrvChild *child; 3058 3059 if (bs->drv && bs->drv->bdrv_unregister_buf) { 3060 bs->drv->bdrv_unregister_buf(bs, host); 3061 } 3062 QLIST_FOREACH(child, &bs->children, next) { 3063 bdrv_unregister_buf(child->bs, host); 3064 } 3065 } 3066 3067 static int coroutine_fn bdrv_co_copy_range_internal( 3068 BdrvChild *src, uint64_t src_offset, BdrvChild *dst, 3069 uint64_t dst_offset, uint64_t bytes, 3070 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 3071 bool recurse_src) 3072 { 3073 BdrvTrackedRequest req; 3074 int ret; 3075 3076 /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 3077 assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 3078 assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 3079 3080 if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) { 3081 return -ENOMEDIUM; 3082 } 3083 ret = bdrv_check_request32(dst_offset, bytes); 3084 if (ret) { 3085 return ret; 3086 } 3087 if (write_flags & BDRV_REQ_ZERO_WRITE) { 3088 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3089 } 3090 3091 if (!src || !src->bs || !bdrv_is_inserted(src->bs)) { 3092 return -ENOMEDIUM; 3093 } 3094 ret = bdrv_check_request32(src_offset, bytes); 3095 if (ret) { 3096 return ret; 3097 } 3098 3099 if (!src->bs->drv->bdrv_co_copy_range_from 3100 || !dst->bs->drv->bdrv_co_copy_range_to 3101 || src->bs->encrypted || dst->bs->encrypted) { 3102 return -ENOTSUP; 3103 } 3104 3105 if (recurse_src) { 3106 bdrv_inc_in_flight(src->bs); 3107 tracked_request_begin(&req, src->bs, src_offset, bytes, 3108 BDRV_TRACKED_READ); 3109 3110 /* BDRV_REQ_SERIALISING is only for write operation */ 3111 assert(!(read_flags & BDRV_REQ_SERIALISING)); 3112 bdrv_wait_serialising_requests(&req); 3113 3114 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3115 src, src_offset, 3116 dst, dst_offset, 3117 bytes, 3118 read_flags, write_flags); 3119 3120 tracked_request_end(&req); 3121 bdrv_dec_in_flight(src->bs); 3122 } else { 3123 bdrv_inc_in_flight(dst->bs); 3124 tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3125 BDRV_TRACKED_WRITE); 3126 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 3127 write_flags); 3128 if (!ret) { 3129 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3130 src, src_offset, 3131 dst, dst_offset, 3132 bytes, 3133 read_flags, write_flags); 3134 } 3135 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3136 tracked_request_end(&req); 3137 bdrv_dec_in_flight(dst->bs); 3138 } 3139 3140 return ret; 3141 } 3142 3143 /* Copy range from @src to @dst. 3144 * 3145 * See the comment of bdrv_co_copy_range for the parameter and return value 3146 * semantics. */ 3147 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset, 3148 BdrvChild *dst, uint64_t dst_offset, 3149 uint64_t bytes, 3150 BdrvRequestFlags read_flags, 3151 BdrvRequestFlags write_flags) 3152 { 3153 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3154 read_flags, write_flags); 3155 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3156 bytes, read_flags, write_flags, true); 3157 } 3158 3159 /* Copy range from @src to @dst. 3160 * 3161 * See the comment of bdrv_co_copy_range for the parameter and return value 3162 * semantics. */ 3163 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset, 3164 BdrvChild *dst, uint64_t dst_offset, 3165 uint64_t bytes, 3166 BdrvRequestFlags read_flags, 3167 BdrvRequestFlags write_flags) 3168 { 3169 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3170 read_flags, write_flags); 3171 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3172 bytes, read_flags, write_flags, false); 3173 } 3174 3175 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset, 3176 BdrvChild *dst, uint64_t dst_offset, 3177 uint64_t bytes, BdrvRequestFlags read_flags, 3178 BdrvRequestFlags write_flags) 3179 { 3180 return bdrv_co_copy_range_from(src, src_offset, 3181 dst, dst_offset, 3182 bytes, read_flags, write_flags); 3183 } 3184 3185 static void bdrv_parent_cb_resize(BlockDriverState *bs) 3186 { 3187 BdrvChild *c; 3188 QLIST_FOREACH(c, &bs->parents, next_parent) { 3189 if (c->klass->resize) { 3190 c->klass->resize(c); 3191 } 3192 } 3193 } 3194 3195 /** 3196 * Truncate file to 'offset' bytes (needed only for file protocols) 3197 * 3198 * If 'exact' is true, the file must be resized to exactly the given 3199 * 'offset'. Otherwise, it is sufficient for the node to be at least 3200 * 'offset' bytes in length. 3201 */ 3202 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 3203 PreallocMode prealloc, BdrvRequestFlags flags, 3204 Error **errp) 3205 { 3206 BlockDriverState *bs = child->bs; 3207 BdrvChild *filtered, *backing; 3208 BlockDriver *drv = bs->drv; 3209 BdrvTrackedRequest req; 3210 int64_t old_size, new_bytes; 3211 int ret; 3212 3213 3214 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 3215 if (!drv) { 3216 error_setg(errp, "No medium inserted"); 3217 return -ENOMEDIUM; 3218 } 3219 if (offset < 0) { 3220 error_setg(errp, "Image size cannot be negative"); 3221 return -EINVAL; 3222 } 3223 3224 ret = bdrv_check_request(offset, 0); 3225 if (ret < 0) { 3226 error_setg(errp, "Required too big image size, it must be not greater " 3227 "than %" PRId64, BDRV_MAX_LENGTH); 3228 return ret; 3229 } 3230 3231 old_size = bdrv_getlength(bs); 3232 if (old_size < 0) { 3233 error_setg_errno(errp, -old_size, "Failed to get old image size"); 3234 return old_size; 3235 } 3236 3237 if (offset > old_size) { 3238 new_bytes = offset - old_size; 3239 } else { 3240 new_bytes = 0; 3241 } 3242 3243 bdrv_inc_in_flight(bs); 3244 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 3245 BDRV_TRACKED_TRUNCATE); 3246 3247 /* If we are growing the image and potentially using preallocation for the 3248 * new area, we need to make sure that no write requests are made to it 3249 * concurrently or they might be overwritten by preallocation. */ 3250 if (new_bytes) { 3251 bdrv_mark_request_serialising(&req, 1); 3252 } 3253 if (bs->read_only) { 3254 error_setg(errp, "Image is read-only"); 3255 ret = -EACCES; 3256 goto out; 3257 } 3258 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3259 0); 3260 if (ret < 0) { 3261 error_setg_errno(errp, -ret, 3262 "Failed to prepare request for truncation"); 3263 goto out; 3264 } 3265 3266 filtered = bdrv_filter_child(bs); 3267 backing = bdrv_cow_child(bs); 3268 3269 /* 3270 * If the image has a backing file that is large enough that it would 3271 * provide data for the new area, we cannot leave it unallocated because 3272 * then the backing file content would become visible. Instead, zero-fill 3273 * the new area. 3274 * 3275 * Note that if the image has a backing file, but was opened without the 3276 * backing file, taking care of keeping things consistent with that backing 3277 * file is the user's responsibility. 3278 */ 3279 if (new_bytes && backing) { 3280 int64_t backing_len; 3281 3282 backing_len = bdrv_getlength(backing->bs); 3283 if (backing_len < 0) { 3284 ret = backing_len; 3285 error_setg_errno(errp, -ret, "Could not get backing file size"); 3286 goto out; 3287 } 3288 3289 if (backing_len > old_size) { 3290 flags |= BDRV_REQ_ZERO_WRITE; 3291 } 3292 } 3293 3294 if (drv->bdrv_co_truncate) { 3295 if (flags & ~bs->supported_truncate_flags) { 3296 error_setg(errp, "Block driver does not support requested flags"); 3297 ret = -ENOTSUP; 3298 goto out; 3299 } 3300 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 3301 } else if (filtered) { 3302 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 3303 } else { 3304 error_setg(errp, "Image format driver does not support resize"); 3305 ret = -ENOTSUP; 3306 goto out; 3307 } 3308 if (ret < 0) { 3309 goto out; 3310 } 3311 3312 ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 3313 if (ret < 0) { 3314 error_setg_errno(errp, -ret, "Could not refresh total sector count"); 3315 } else { 3316 offset = bs->total_sectors * BDRV_SECTOR_SIZE; 3317 } 3318 /* It's possible that truncation succeeded but refresh_total_sectors 3319 * failed, but the latter doesn't affect how we should finish the request. 3320 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */ 3321 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 3322 3323 out: 3324 tracked_request_end(&req); 3325 bdrv_dec_in_flight(bs); 3326 3327 return ret; 3328 } 3329