1 /* 2 * Block layer I/O functions 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "trace.h" 27 #include "sysemu/block-backend.h" 28 #include "block/aio-wait.h" 29 #include "block/blockjob.h" 30 #include "block/blockjob_int.h" 31 #include "block/block_int.h" 32 #include "block/coroutines.h" 33 #include "qemu/cutils.h" 34 #include "qapi/error.h" 35 #include "qemu/error-report.h" 36 #include "qemu/main-loop.h" 37 #include "sysemu/replay.h" 38 39 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */ 40 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS) 41 42 static void bdrv_parent_cb_resize(BlockDriverState *bs); 43 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 44 int64_t offset, int bytes, BdrvRequestFlags flags); 45 46 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore, 47 bool ignore_bds_parents) 48 { 49 BdrvChild *c, *next; 50 51 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 52 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 53 continue; 54 } 55 bdrv_parent_drained_begin_single(c, false); 56 } 57 } 58 59 static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c, 60 int *drained_end_counter) 61 { 62 assert(c->parent_quiesce_counter > 0); 63 c->parent_quiesce_counter--; 64 if (c->klass->drained_end) { 65 c->klass->drained_end(c, drained_end_counter); 66 } 67 } 68 69 void bdrv_parent_drained_end_single(BdrvChild *c) 70 { 71 int drained_end_counter = 0; 72 bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter); 73 BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0); 74 } 75 76 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore, 77 bool ignore_bds_parents, 78 int *drained_end_counter) 79 { 80 BdrvChild *c; 81 82 QLIST_FOREACH(c, &bs->parents, next_parent) { 83 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 84 continue; 85 } 86 bdrv_parent_drained_end_single_no_poll(c, drained_end_counter); 87 } 88 } 89 90 static bool bdrv_parent_drained_poll_single(BdrvChild *c) 91 { 92 if (c->klass->drained_poll) { 93 return c->klass->drained_poll(c); 94 } 95 return false; 96 } 97 98 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore, 99 bool ignore_bds_parents) 100 { 101 BdrvChild *c, *next; 102 bool busy = false; 103 104 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) { 105 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) { 106 continue; 107 } 108 busy |= bdrv_parent_drained_poll_single(c); 109 } 110 111 return busy; 112 } 113 114 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll) 115 { 116 c->parent_quiesce_counter++; 117 if (c->klass->drained_begin) { 118 c->klass->drained_begin(c); 119 } 120 if (poll) { 121 BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c)); 122 } 123 } 124 125 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src) 126 { 127 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer); 128 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer); 129 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment, 130 src->opt_mem_alignment); 131 dst->min_mem_alignment = MAX(dst->min_mem_alignment, 132 src->min_mem_alignment); 133 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov); 134 } 135 136 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp) 137 { 138 BlockDriver *drv = bs->drv; 139 BdrvChild *c; 140 bool have_limits; 141 Error *local_err = NULL; 142 143 memset(&bs->bl, 0, sizeof(bs->bl)); 144 145 if (!drv) { 146 return; 147 } 148 149 /* Default alignment based on whether driver has byte interface */ 150 bs->bl.request_alignment = (drv->bdrv_co_preadv || 151 drv->bdrv_aio_preadv || 152 drv->bdrv_co_preadv_part) ? 1 : 512; 153 154 /* Take some limits from the children as a default */ 155 have_limits = false; 156 QLIST_FOREACH(c, &bs->children, next) { 157 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW)) 158 { 159 bdrv_refresh_limits(c->bs, &local_err); 160 if (local_err) { 161 error_propagate(errp, local_err); 162 return; 163 } 164 bdrv_merge_limits(&bs->bl, &c->bs->bl); 165 have_limits = true; 166 } 167 } 168 169 if (!have_limits) { 170 bs->bl.min_mem_alignment = 512; 171 bs->bl.opt_mem_alignment = qemu_real_host_page_size; 172 173 /* Safe default since most protocols use readv()/writev()/etc */ 174 bs->bl.max_iov = IOV_MAX; 175 } 176 177 /* Then let the driver override it */ 178 if (drv->bdrv_refresh_limits) { 179 drv->bdrv_refresh_limits(bs, errp); 180 } 181 } 182 183 /** 184 * The copy-on-read flag is actually a reference count so multiple users may 185 * use the feature without worrying about clobbering its previous state. 186 * Copy-on-read stays enabled until all users have called to disable it. 187 */ 188 void bdrv_enable_copy_on_read(BlockDriverState *bs) 189 { 190 qatomic_inc(&bs->copy_on_read); 191 } 192 193 void bdrv_disable_copy_on_read(BlockDriverState *bs) 194 { 195 int old = qatomic_fetch_dec(&bs->copy_on_read); 196 assert(old >= 1); 197 } 198 199 typedef struct { 200 Coroutine *co; 201 BlockDriverState *bs; 202 bool done; 203 bool begin; 204 bool recursive; 205 bool poll; 206 BdrvChild *parent; 207 bool ignore_bds_parents; 208 int *drained_end_counter; 209 } BdrvCoDrainData; 210 211 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque) 212 { 213 BdrvCoDrainData *data = opaque; 214 BlockDriverState *bs = data->bs; 215 216 if (data->begin) { 217 bs->drv->bdrv_co_drain_begin(bs); 218 } else { 219 bs->drv->bdrv_co_drain_end(bs); 220 } 221 222 /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */ 223 qatomic_mb_set(&data->done, true); 224 if (!data->begin) { 225 qatomic_dec(data->drained_end_counter); 226 } 227 bdrv_dec_in_flight(bs); 228 229 g_free(data); 230 } 231 232 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */ 233 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin, 234 int *drained_end_counter) 235 { 236 BdrvCoDrainData *data; 237 238 if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) || 239 (!begin && !bs->drv->bdrv_co_drain_end)) { 240 return; 241 } 242 243 data = g_new(BdrvCoDrainData, 1); 244 *data = (BdrvCoDrainData) { 245 .bs = bs, 246 .done = false, 247 .begin = begin, 248 .drained_end_counter = drained_end_counter, 249 }; 250 251 if (!begin) { 252 qatomic_inc(drained_end_counter); 253 } 254 255 /* Make sure the driver callback completes during the polling phase for 256 * drain_begin. */ 257 bdrv_inc_in_flight(bs); 258 data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data); 259 aio_co_schedule(bdrv_get_aio_context(bs), data->co); 260 } 261 262 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */ 263 bool bdrv_drain_poll(BlockDriverState *bs, bool recursive, 264 BdrvChild *ignore_parent, bool ignore_bds_parents) 265 { 266 BdrvChild *child, *next; 267 268 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) { 269 return true; 270 } 271 272 if (qatomic_read(&bs->in_flight)) { 273 return true; 274 } 275 276 if (recursive) { 277 assert(!ignore_bds_parents); 278 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 279 if (bdrv_drain_poll(child->bs, recursive, child, false)) { 280 return true; 281 } 282 } 283 } 284 285 return false; 286 } 287 288 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive, 289 BdrvChild *ignore_parent) 290 { 291 return bdrv_drain_poll(bs, recursive, ignore_parent, false); 292 } 293 294 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 295 BdrvChild *parent, bool ignore_bds_parents, 296 bool poll); 297 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 298 BdrvChild *parent, bool ignore_bds_parents, 299 int *drained_end_counter); 300 301 static void bdrv_co_drain_bh_cb(void *opaque) 302 { 303 BdrvCoDrainData *data = opaque; 304 Coroutine *co = data->co; 305 BlockDriverState *bs = data->bs; 306 307 if (bs) { 308 AioContext *ctx = bdrv_get_aio_context(bs); 309 AioContext *co_ctx = qemu_coroutine_get_aio_context(co); 310 311 /* 312 * When the coroutine yielded, the lock for its home context was 313 * released, so we need to re-acquire it here. If it explicitly 314 * acquired a different context, the lock is still held and we don't 315 * want to lock it a second time (or AIO_WAIT_WHILE() would hang). 316 */ 317 if (ctx == co_ctx) { 318 aio_context_acquire(ctx); 319 } 320 bdrv_dec_in_flight(bs); 321 if (data->begin) { 322 assert(!data->drained_end_counter); 323 bdrv_do_drained_begin(bs, data->recursive, data->parent, 324 data->ignore_bds_parents, data->poll); 325 } else { 326 assert(!data->poll); 327 bdrv_do_drained_end(bs, data->recursive, data->parent, 328 data->ignore_bds_parents, 329 data->drained_end_counter); 330 } 331 if (ctx == co_ctx) { 332 aio_context_release(ctx); 333 } 334 } else { 335 assert(data->begin); 336 bdrv_drain_all_begin(); 337 } 338 339 data->done = true; 340 aio_co_wake(co); 341 } 342 343 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs, 344 bool begin, bool recursive, 345 BdrvChild *parent, 346 bool ignore_bds_parents, 347 bool poll, 348 int *drained_end_counter) 349 { 350 BdrvCoDrainData data; 351 352 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and 353 * other coroutines run if they were queued by aio_co_enter(). */ 354 355 assert(qemu_in_coroutine()); 356 data = (BdrvCoDrainData) { 357 .co = qemu_coroutine_self(), 358 .bs = bs, 359 .done = false, 360 .begin = begin, 361 .recursive = recursive, 362 .parent = parent, 363 .ignore_bds_parents = ignore_bds_parents, 364 .poll = poll, 365 .drained_end_counter = drained_end_counter, 366 }; 367 368 if (bs) { 369 bdrv_inc_in_flight(bs); 370 } 371 replay_bh_schedule_oneshot_event(bdrv_get_aio_context(bs), 372 bdrv_co_drain_bh_cb, &data); 373 374 qemu_coroutine_yield(); 375 /* If we are resumed from some other event (such as an aio completion or a 376 * timer callback), it is a bug in the caller that should be fixed. */ 377 assert(data.done); 378 } 379 380 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs, 381 BdrvChild *parent, bool ignore_bds_parents) 382 { 383 assert(!qemu_in_coroutine()); 384 385 /* Stop things in parent-to-child order */ 386 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) { 387 aio_disable_external(bdrv_get_aio_context(bs)); 388 } 389 390 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents); 391 bdrv_drain_invoke(bs, true, NULL); 392 } 393 394 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive, 395 BdrvChild *parent, bool ignore_bds_parents, 396 bool poll) 397 { 398 BdrvChild *child, *next; 399 400 if (qemu_in_coroutine()) { 401 bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents, 402 poll, NULL); 403 return; 404 } 405 406 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents); 407 408 if (recursive) { 409 assert(!ignore_bds_parents); 410 bs->recursive_quiesce_counter++; 411 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 412 bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents, 413 false); 414 } 415 } 416 417 /* 418 * Wait for drained requests to finish. 419 * 420 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The 421 * call is needed so things in this AioContext can make progress even 422 * though we don't return to the main AioContext loop - this automatically 423 * includes other nodes in the same AioContext and therefore all child 424 * nodes. 425 */ 426 if (poll) { 427 assert(!ignore_bds_parents); 428 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent)); 429 } 430 } 431 432 void bdrv_drained_begin(BlockDriverState *bs) 433 { 434 bdrv_do_drained_begin(bs, false, NULL, false, true); 435 } 436 437 void bdrv_subtree_drained_begin(BlockDriverState *bs) 438 { 439 bdrv_do_drained_begin(bs, true, NULL, false, true); 440 } 441 442 /** 443 * This function does not poll, nor must any of its recursively called 444 * functions. The *drained_end_counter pointee will be incremented 445 * once for every background operation scheduled, and decremented once 446 * the operation settles. Therefore, the pointer must remain valid 447 * until the pointee reaches 0. That implies that whoever sets up the 448 * pointee has to poll until it is 0. 449 * 450 * We use atomic operations to access *drained_end_counter, because 451 * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of 452 * @bs may contain nodes in different AioContexts, 453 * (2) bdrv_drain_all_end() uses the same counter for all nodes, 454 * regardless of which AioContext they are in. 455 */ 456 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive, 457 BdrvChild *parent, bool ignore_bds_parents, 458 int *drained_end_counter) 459 { 460 BdrvChild *child; 461 int old_quiesce_counter; 462 463 assert(drained_end_counter != NULL); 464 465 if (qemu_in_coroutine()) { 466 bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents, 467 false, drained_end_counter); 468 return; 469 } 470 assert(bs->quiesce_counter > 0); 471 472 /* Re-enable things in child-to-parent order */ 473 bdrv_drain_invoke(bs, false, drained_end_counter); 474 bdrv_parent_drained_end(bs, parent, ignore_bds_parents, 475 drained_end_counter); 476 477 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter); 478 if (old_quiesce_counter == 1) { 479 aio_enable_external(bdrv_get_aio_context(bs)); 480 } 481 482 if (recursive) { 483 assert(!ignore_bds_parents); 484 bs->recursive_quiesce_counter--; 485 QLIST_FOREACH(child, &bs->children, next) { 486 bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents, 487 drained_end_counter); 488 } 489 } 490 } 491 492 void bdrv_drained_end(BlockDriverState *bs) 493 { 494 int drained_end_counter = 0; 495 bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter); 496 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 497 } 498 499 void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter) 500 { 501 bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter); 502 } 503 504 void bdrv_subtree_drained_end(BlockDriverState *bs) 505 { 506 int drained_end_counter = 0; 507 bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter); 508 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0); 509 } 510 511 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent) 512 { 513 int i; 514 515 for (i = 0; i < new_parent->recursive_quiesce_counter; i++) { 516 bdrv_do_drained_begin(child->bs, true, child, false, true); 517 } 518 } 519 520 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent) 521 { 522 int drained_end_counter = 0; 523 int i; 524 525 for (i = 0; i < old_parent->recursive_quiesce_counter; i++) { 526 bdrv_do_drained_end(child->bs, true, child, false, 527 &drained_end_counter); 528 } 529 530 BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0); 531 } 532 533 /* 534 * Wait for pending requests to complete on a single BlockDriverState subtree, 535 * and suspend block driver's internal I/O until next request arrives. 536 * 537 * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState 538 * AioContext. 539 */ 540 void coroutine_fn bdrv_co_drain(BlockDriverState *bs) 541 { 542 assert(qemu_in_coroutine()); 543 bdrv_drained_begin(bs); 544 bdrv_drained_end(bs); 545 } 546 547 void bdrv_drain(BlockDriverState *bs) 548 { 549 bdrv_drained_begin(bs); 550 bdrv_drained_end(bs); 551 } 552 553 static void bdrv_drain_assert_idle(BlockDriverState *bs) 554 { 555 BdrvChild *child, *next; 556 557 assert(qatomic_read(&bs->in_flight) == 0); 558 QLIST_FOREACH_SAFE(child, &bs->children, next, next) { 559 bdrv_drain_assert_idle(child->bs); 560 } 561 } 562 563 unsigned int bdrv_drain_all_count = 0; 564 565 static bool bdrv_drain_all_poll(void) 566 { 567 BlockDriverState *bs = NULL; 568 bool result = false; 569 570 /* bdrv_drain_poll() can't make changes to the graph and we are holding the 571 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */ 572 while ((bs = bdrv_next_all_states(bs))) { 573 AioContext *aio_context = bdrv_get_aio_context(bs); 574 aio_context_acquire(aio_context); 575 result |= bdrv_drain_poll(bs, false, NULL, true); 576 aio_context_release(aio_context); 577 } 578 579 return result; 580 } 581 582 /* 583 * Wait for pending requests to complete across all BlockDriverStates 584 * 585 * This function does not flush data to disk, use bdrv_flush_all() for that 586 * after calling this function. 587 * 588 * This pauses all block jobs and disables external clients. It must 589 * be paired with bdrv_drain_all_end(). 590 * 591 * NOTE: no new block jobs or BlockDriverStates can be created between 592 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls. 593 */ 594 void bdrv_drain_all_begin(void) 595 { 596 BlockDriverState *bs = NULL; 597 598 if (qemu_in_coroutine()) { 599 bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL); 600 return; 601 } 602 603 /* 604 * bdrv queue is managed by record/replay, 605 * waiting for finishing the I/O requests may 606 * be infinite 607 */ 608 if (replay_events_enabled()) { 609 return; 610 } 611 612 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main 613 * loop AioContext, so make sure we're in the main context. */ 614 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 615 assert(bdrv_drain_all_count < INT_MAX); 616 bdrv_drain_all_count++; 617 618 /* Quiesce all nodes, without polling in-flight requests yet. The graph 619 * cannot change during this loop. */ 620 while ((bs = bdrv_next_all_states(bs))) { 621 AioContext *aio_context = bdrv_get_aio_context(bs); 622 623 aio_context_acquire(aio_context); 624 bdrv_do_drained_begin(bs, false, NULL, true, false); 625 aio_context_release(aio_context); 626 } 627 628 /* Now poll the in-flight requests */ 629 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll()); 630 631 while ((bs = bdrv_next_all_states(bs))) { 632 bdrv_drain_assert_idle(bs); 633 } 634 } 635 636 void bdrv_drain_all_end(void) 637 { 638 BlockDriverState *bs = NULL; 639 int drained_end_counter = 0; 640 641 /* 642 * bdrv queue is managed by record/replay, 643 * waiting for finishing the I/O requests may 644 * be endless 645 */ 646 if (replay_events_enabled()) { 647 return; 648 } 649 650 while ((bs = bdrv_next_all_states(bs))) { 651 AioContext *aio_context = bdrv_get_aio_context(bs); 652 653 aio_context_acquire(aio_context); 654 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter); 655 aio_context_release(aio_context); 656 } 657 658 assert(qemu_get_current_aio_context() == qemu_get_aio_context()); 659 AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0); 660 661 assert(bdrv_drain_all_count > 0); 662 bdrv_drain_all_count--; 663 } 664 665 void bdrv_drain_all(void) 666 { 667 bdrv_drain_all_begin(); 668 bdrv_drain_all_end(); 669 } 670 671 /** 672 * Remove an active request from the tracked requests list 673 * 674 * This function should be called when a tracked request is completing. 675 */ 676 static void tracked_request_end(BdrvTrackedRequest *req) 677 { 678 if (req->serialising) { 679 qatomic_dec(&req->bs->serialising_in_flight); 680 } 681 682 qemu_co_mutex_lock(&req->bs->reqs_lock); 683 QLIST_REMOVE(req, list); 684 qemu_co_queue_restart_all(&req->wait_queue); 685 qemu_co_mutex_unlock(&req->bs->reqs_lock); 686 } 687 688 /** 689 * Add an active request to the tracked requests list 690 */ 691 static void tracked_request_begin(BdrvTrackedRequest *req, 692 BlockDriverState *bs, 693 int64_t offset, 694 uint64_t bytes, 695 enum BdrvTrackedRequestType type) 696 { 697 assert(bytes <= INT64_MAX && offset <= INT64_MAX - bytes); 698 699 *req = (BdrvTrackedRequest){ 700 .bs = bs, 701 .offset = offset, 702 .bytes = bytes, 703 .type = type, 704 .co = qemu_coroutine_self(), 705 .serialising = false, 706 .overlap_offset = offset, 707 .overlap_bytes = bytes, 708 }; 709 710 qemu_co_queue_init(&req->wait_queue); 711 712 qemu_co_mutex_lock(&bs->reqs_lock); 713 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list); 714 qemu_co_mutex_unlock(&bs->reqs_lock); 715 } 716 717 static bool tracked_request_overlaps(BdrvTrackedRequest *req, 718 int64_t offset, uint64_t bytes) 719 { 720 /* aaaa bbbb */ 721 if (offset >= req->overlap_offset + req->overlap_bytes) { 722 return false; 723 } 724 /* bbbb aaaa */ 725 if (req->overlap_offset >= offset + bytes) { 726 return false; 727 } 728 return true; 729 } 730 731 static bool coroutine_fn 732 bdrv_wait_serialising_requests_locked(BlockDriverState *bs, 733 BdrvTrackedRequest *self) 734 { 735 BdrvTrackedRequest *req; 736 bool retry; 737 bool waited = false; 738 739 do { 740 retry = false; 741 QLIST_FOREACH(req, &bs->tracked_requests, list) { 742 if (req == self || (!req->serialising && !self->serialising)) { 743 continue; 744 } 745 if (tracked_request_overlaps(req, self->overlap_offset, 746 self->overlap_bytes)) 747 { 748 /* Hitting this means there was a reentrant request, for 749 * example, a block driver issuing nested requests. This must 750 * never happen since it means deadlock. 751 */ 752 assert(qemu_coroutine_self() != req->co); 753 754 /* If the request is already (indirectly) waiting for us, or 755 * will wait for us as soon as it wakes up, then just go on 756 * (instead of producing a deadlock in the former case). */ 757 if (!req->waiting_for) { 758 self->waiting_for = req; 759 qemu_co_queue_wait(&req->wait_queue, &bs->reqs_lock); 760 self->waiting_for = NULL; 761 retry = true; 762 waited = true; 763 break; 764 } 765 } 766 } 767 } while (retry); 768 return waited; 769 } 770 771 bool bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align) 772 { 773 BlockDriverState *bs = req->bs; 774 int64_t overlap_offset = req->offset & ~(align - 1); 775 uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align) 776 - overlap_offset; 777 bool waited; 778 779 qemu_co_mutex_lock(&bs->reqs_lock); 780 if (!req->serialising) { 781 qatomic_inc(&req->bs->serialising_in_flight); 782 req->serialising = true; 783 } 784 785 req->overlap_offset = MIN(req->overlap_offset, overlap_offset); 786 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes); 787 waited = bdrv_wait_serialising_requests_locked(bs, req); 788 qemu_co_mutex_unlock(&bs->reqs_lock); 789 return waited; 790 } 791 792 /** 793 * Return the tracked request on @bs for the current coroutine, or 794 * NULL if there is none. 795 */ 796 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs) 797 { 798 BdrvTrackedRequest *req; 799 Coroutine *self = qemu_coroutine_self(); 800 801 QLIST_FOREACH(req, &bs->tracked_requests, list) { 802 if (req->co == self) { 803 return req; 804 } 805 } 806 807 return NULL; 808 } 809 810 /** 811 * Round a region to cluster boundaries 812 */ 813 void bdrv_round_to_clusters(BlockDriverState *bs, 814 int64_t offset, int64_t bytes, 815 int64_t *cluster_offset, 816 int64_t *cluster_bytes) 817 { 818 BlockDriverInfo bdi; 819 820 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) { 821 *cluster_offset = offset; 822 *cluster_bytes = bytes; 823 } else { 824 int64_t c = bdi.cluster_size; 825 *cluster_offset = QEMU_ALIGN_DOWN(offset, c); 826 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c); 827 } 828 } 829 830 static int bdrv_get_cluster_size(BlockDriverState *bs) 831 { 832 BlockDriverInfo bdi; 833 int ret; 834 835 ret = bdrv_get_info(bs, &bdi); 836 if (ret < 0 || bdi.cluster_size == 0) { 837 return bs->bl.request_alignment; 838 } else { 839 return bdi.cluster_size; 840 } 841 } 842 843 void bdrv_inc_in_flight(BlockDriverState *bs) 844 { 845 qatomic_inc(&bs->in_flight); 846 } 847 848 void bdrv_wakeup(BlockDriverState *bs) 849 { 850 aio_wait_kick(); 851 } 852 853 void bdrv_dec_in_flight(BlockDriverState *bs) 854 { 855 qatomic_dec(&bs->in_flight); 856 bdrv_wakeup(bs); 857 } 858 859 static bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self) 860 { 861 BlockDriverState *bs = self->bs; 862 bool waited = false; 863 864 if (!qatomic_read(&bs->serialising_in_flight)) { 865 return false; 866 } 867 868 qemu_co_mutex_lock(&bs->reqs_lock); 869 waited = bdrv_wait_serialising_requests_locked(bs, self); 870 qemu_co_mutex_unlock(&bs->reqs_lock); 871 872 return waited; 873 } 874 875 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset, 876 size_t size) 877 { 878 if (size > BDRV_REQUEST_MAX_BYTES) { 879 return -EIO; 880 } 881 882 if (!bdrv_is_inserted(bs)) { 883 return -ENOMEDIUM; 884 } 885 886 if (offset < 0) { 887 return -EIO; 888 } 889 890 return 0; 891 } 892 893 int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset, 894 int bytes, BdrvRequestFlags flags) 895 { 896 return bdrv_pwritev(child, offset, bytes, NULL, 897 BDRV_REQ_ZERO_WRITE | flags); 898 } 899 900 /* 901 * Completely zero out a block device with the help of bdrv_pwrite_zeroes. 902 * The operation is sped up by checking the block status and only writing 903 * zeroes to the device if they currently do not return zeroes. Optional 904 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP, 905 * BDRV_REQ_FUA). 906 * 907 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite(). 908 */ 909 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags) 910 { 911 int ret; 912 int64_t target_size, bytes, offset = 0; 913 BlockDriverState *bs = child->bs; 914 915 target_size = bdrv_getlength(bs); 916 if (target_size < 0) { 917 return target_size; 918 } 919 920 for (;;) { 921 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES); 922 if (bytes <= 0) { 923 return 0; 924 } 925 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL); 926 if (ret < 0) { 927 return ret; 928 } 929 if (ret & BDRV_BLOCK_ZERO) { 930 offset += bytes; 931 continue; 932 } 933 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags); 934 if (ret < 0) { 935 return ret; 936 } 937 offset += bytes; 938 } 939 } 940 941 /* See bdrv_pwrite() for the return codes */ 942 int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes) 943 { 944 int ret; 945 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 946 947 if (bytes < 0) { 948 return -EINVAL; 949 } 950 951 ret = bdrv_preadv(child, offset, bytes, &qiov, 0); 952 953 return ret < 0 ? ret : bytes; 954 } 955 956 /* Return no. of bytes on success or < 0 on error. Important errors are: 957 -EIO generic I/O error (may happen for all errors) 958 -ENOMEDIUM No media inserted. 959 -EINVAL Invalid offset or number of bytes 960 -EACCES Trying to write a read-only device 961 */ 962 int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes) 963 { 964 int ret; 965 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, bytes); 966 967 if (bytes < 0) { 968 return -EINVAL; 969 } 970 971 ret = bdrv_pwritev(child, offset, bytes, &qiov, 0); 972 973 return ret < 0 ? ret : bytes; 974 } 975 976 /* 977 * Writes to the file and ensures that no writes are reordered across this 978 * request (acts as a barrier) 979 * 980 * Returns 0 on success, -errno in error cases. 981 */ 982 int bdrv_pwrite_sync(BdrvChild *child, int64_t offset, 983 const void *buf, int count) 984 { 985 int ret; 986 987 ret = bdrv_pwrite(child, offset, buf, count); 988 if (ret < 0) { 989 return ret; 990 } 991 992 ret = bdrv_flush(child->bs); 993 if (ret < 0) { 994 return ret; 995 } 996 997 return 0; 998 } 999 1000 typedef struct CoroutineIOCompletion { 1001 Coroutine *coroutine; 1002 int ret; 1003 } CoroutineIOCompletion; 1004 1005 static void bdrv_co_io_em_complete(void *opaque, int ret) 1006 { 1007 CoroutineIOCompletion *co = opaque; 1008 1009 co->ret = ret; 1010 aio_co_wake(co->coroutine); 1011 } 1012 1013 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs, 1014 uint64_t offset, uint64_t bytes, 1015 QEMUIOVector *qiov, 1016 size_t qiov_offset, int flags) 1017 { 1018 BlockDriver *drv = bs->drv; 1019 int64_t sector_num; 1020 unsigned int nb_sectors; 1021 QEMUIOVector local_qiov; 1022 int ret; 1023 1024 assert(!(flags & ~BDRV_REQ_MASK)); 1025 assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1026 1027 if (!drv) { 1028 return -ENOMEDIUM; 1029 } 1030 1031 if (drv->bdrv_co_preadv_part) { 1032 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset, 1033 flags); 1034 } 1035 1036 if (qiov_offset > 0 || bytes != qiov->size) { 1037 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1038 qiov = &local_qiov; 1039 } 1040 1041 if (drv->bdrv_co_preadv) { 1042 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags); 1043 goto out; 1044 } 1045 1046 if (drv->bdrv_aio_preadv) { 1047 BlockAIOCB *acb; 1048 CoroutineIOCompletion co = { 1049 .coroutine = qemu_coroutine_self(), 1050 }; 1051 1052 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags, 1053 bdrv_co_io_em_complete, &co); 1054 if (acb == NULL) { 1055 ret = -EIO; 1056 goto out; 1057 } else { 1058 qemu_coroutine_yield(); 1059 ret = co.ret; 1060 goto out; 1061 } 1062 } 1063 1064 sector_num = offset >> BDRV_SECTOR_BITS; 1065 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1066 1067 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1068 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1069 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1070 assert(drv->bdrv_co_readv); 1071 1072 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov); 1073 1074 out: 1075 if (qiov == &local_qiov) { 1076 qemu_iovec_destroy(&local_qiov); 1077 } 1078 1079 return ret; 1080 } 1081 1082 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs, 1083 uint64_t offset, uint64_t bytes, 1084 QEMUIOVector *qiov, 1085 size_t qiov_offset, int flags) 1086 { 1087 BlockDriver *drv = bs->drv; 1088 int64_t sector_num; 1089 unsigned int nb_sectors; 1090 QEMUIOVector local_qiov; 1091 int ret; 1092 1093 assert(!(flags & ~BDRV_REQ_MASK)); 1094 assert(!(flags & BDRV_REQ_NO_FALLBACK)); 1095 1096 if (!drv) { 1097 return -ENOMEDIUM; 1098 } 1099 1100 if (drv->bdrv_co_pwritev_part) { 1101 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset, 1102 flags & bs->supported_write_flags); 1103 flags &= ~bs->supported_write_flags; 1104 goto emulate_flags; 1105 } 1106 1107 if (qiov_offset > 0 || bytes != qiov->size) { 1108 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1109 qiov = &local_qiov; 1110 } 1111 1112 if (drv->bdrv_co_pwritev) { 1113 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, 1114 flags & bs->supported_write_flags); 1115 flags &= ~bs->supported_write_flags; 1116 goto emulate_flags; 1117 } 1118 1119 if (drv->bdrv_aio_pwritev) { 1120 BlockAIOCB *acb; 1121 CoroutineIOCompletion co = { 1122 .coroutine = qemu_coroutine_self(), 1123 }; 1124 1125 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, 1126 flags & bs->supported_write_flags, 1127 bdrv_co_io_em_complete, &co); 1128 flags &= ~bs->supported_write_flags; 1129 if (acb == NULL) { 1130 ret = -EIO; 1131 } else { 1132 qemu_coroutine_yield(); 1133 ret = co.ret; 1134 } 1135 goto emulate_flags; 1136 } 1137 1138 sector_num = offset >> BDRV_SECTOR_BITS; 1139 nb_sectors = bytes >> BDRV_SECTOR_BITS; 1140 1141 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE)); 1142 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE)); 1143 assert(bytes <= BDRV_REQUEST_MAX_BYTES); 1144 1145 assert(drv->bdrv_co_writev); 1146 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, 1147 flags & bs->supported_write_flags); 1148 flags &= ~bs->supported_write_flags; 1149 1150 emulate_flags: 1151 if (ret == 0 && (flags & BDRV_REQ_FUA)) { 1152 ret = bdrv_co_flush(bs); 1153 } 1154 1155 if (qiov == &local_qiov) { 1156 qemu_iovec_destroy(&local_qiov); 1157 } 1158 1159 return ret; 1160 } 1161 1162 static int coroutine_fn 1163 bdrv_driver_pwritev_compressed(BlockDriverState *bs, uint64_t offset, 1164 uint64_t bytes, QEMUIOVector *qiov, 1165 size_t qiov_offset) 1166 { 1167 BlockDriver *drv = bs->drv; 1168 QEMUIOVector local_qiov; 1169 int ret; 1170 1171 if (!drv) { 1172 return -ENOMEDIUM; 1173 } 1174 1175 if (!block_driver_can_compress(drv)) { 1176 return -ENOTSUP; 1177 } 1178 1179 if (drv->bdrv_co_pwritev_compressed_part) { 1180 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes, 1181 qiov, qiov_offset); 1182 } 1183 1184 if (qiov_offset == 0) { 1185 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov); 1186 } 1187 1188 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes); 1189 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov); 1190 qemu_iovec_destroy(&local_qiov); 1191 1192 return ret; 1193 } 1194 1195 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child, 1196 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1197 size_t qiov_offset, int flags) 1198 { 1199 BlockDriverState *bs = child->bs; 1200 1201 /* Perform I/O through a temporary buffer so that users who scribble over 1202 * their read buffer while the operation is in progress do not end up 1203 * modifying the image file. This is critical for zero-copy guest I/O 1204 * where anything might happen inside guest memory. 1205 */ 1206 void *bounce_buffer = NULL; 1207 1208 BlockDriver *drv = bs->drv; 1209 int64_t cluster_offset; 1210 int64_t cluster_bytes; 1211 size_t skip_bytes; 1212 int ret; 1213 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, 1214 BDRV_REQUEST_MAX_BYTES); 1215 unsigned int progress = 0; 1216 bool skip_write; 1217 1218 if (!drv) { 1219 return -ENOMEDIUM; 1220 } 1221 1222 /* 1223 * Do not write anything when the BDS is inactive. That is not 1224 * allowed, and it would not help. 1225 */ 1226 skip_write = (bs->open_flags & BDRV_O_INACTIVE); 1227 1228 /* FIXME We cannot require callers to have write permissions when all they 1229 * are doing is a read request. If we did things right, write permissions 1230 * would be obtained anyway, but internally by the copy-on-read code. As 1231 * long as it is implemented here rather than in a separate filter driver, 1232 * the copy-on-read code doesn't have its own BdrvChild, however, for which 1233 * it could request permissions. Therefore we have to bypass the permission 1234 * system for the moment. */ 1235 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1236 1237 /* Cover entire cluster so no additional backing file I/O is required when 1238 * allocating cluster in the image file. Note that this value may exceed 1239 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which 1240 * is one reason we loop rather than doing it all at once. 1241 */ 1242 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes); 1243 skip_bytes = offset - cluster_offset; 1244 1245 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes, 1246 cluster_offset, cluster_bytes); 1247 1248 while (cluster_bytes) { 1249 int64_t pnum; 1250 1251 if (skip_write) { 1252 ret = 1; /* "already allocated", so nothing will be copied */ 1253 pnum = MIN(cluster_bytes, max_transfer); 1254 } else { 1255 ret = bdrv_is_allocated(bs, cluster_offset, 1256 MIN(cluster_bytes, max_transfer), &pnum); 1257 if (ret < 0) { 1258 /* 1259 * Safe to treat errors in querying allocation as if 1260 * unallocated; we'll probably fail again soon on the 1261 * read, but at least that will set a decent errno. 1262 */ 1263 pnum = MIN(cluster_bytes, max_transfer); 1264 } 1265 1266 /* Stop at EOF if the image ends in the middle of the cluster */ 1267 if (ret == 0 && pnum == 0) { 1268 assert(progress >= bytes); 1269 break; 1270 } 1271 1272 assert(skip_bytes < pnum); 1273 } 1274 1275 if (ret <= 0) { 1276 QEMUIOVector local_qiov; 1277 1278 /* Must copy-on-read; use the bounce buffer */ 1279 pnum = MIN(pnum, MAX_BOUNCE_BUFFER); 1280 if (!bounce_buffer) { 1281 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum); 1282 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER); 1283 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed); 1284 1285 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len); 1286 if (!bounce_buffer) { 1287 ret = -ENOMEM; 1288 goto err; 1289 } 1290 } 1291 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum); 1292 1293 ret = bdrv_driver_preadv(bs, cluster_offset, pnum, 1294 &local_qiov, 0, 0); 1295 if (ret < 0) { 1296 goto err; 1297 } 1298 1299 bdrv_debug_event(bs, BLKDBG_COR_WRITE); 1300 if (drv->bdrv_co_pwrite_zeroes && 1301 buffer_is_zero(bounce_buffer, pnum)) { 1302 /* FIXME: Should we (perhaps conditionally) be setting 1303 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy 1304 * that still correctly reads as zero? */ 1305 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum, 1306 BDRV_REQ_WRITE_UNCHANGED); 1307 } else { 1308 /* This does not change the data on the disk, it is not 1309 * necessary to flush even in cache=writethrough mode. 1310 */ 1311 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum, 1312 &local_qiov, 0, 1313 BDRV_REQ_WRITE_UNCHANGED); 1314 } 1315 1316 if (ret < 0) { 1317 /* It might be okay to ignore write errors for guest 1318 * requests. If this is a deliberate copy-on-read 1319 * then we don't want to ignore the error. Simply 1320 * report it in all cases. 1321 */ 1322 goto err; 1323 } 1324 1325 if (!(flags & BDRV_REQ_PREFETCH)) { 1326 qemu_iovec_from_buf(qiov, qiov_offset + progress, 1327 bounce_buffer + skip_bytes, 1328 MIN(pnum - skip_bytes, bytes - progress)); 1329 } 1330 } else if (!(flags & BDRV_REQ_PREFETCH)) { 1331 /* Read directly into the destination */ 1332 ret = bdrv_driver_preadv(bs, offset + progress, 1333 MIN(pnum - skip_bytes, bytes - progress), 1334 qiov, qiov_offset + progress, 0); 1335 if (ret < 0) { 1336 goto err; 1337 } 1338 } 1339 1340 cluster_offset += pnum; 1341 cluster_bytes -= pnum; 1342 progress += pnum - skip_bytes; 1343 skip_bytes = 0; 1344 } 1345 ret = 0; 1346 1347 err: 1348 qemu_vfree(bounce_buffer); 1349 return ret; 1350 } 1351 1352 /* 1353 * Forwards an already correctly aligned request to the BlockDriver. This 1354 * handles copy on read, zeroing after EOF, and fragmentation of large 1355 * reads; any other features must be implemented by the caller. 1356 */ 1357 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child, 1358 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1359 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 1360 { 1361 BlockDriverState *bs = child->bs; 1362 int64_t total_bytes, max_bytes; 1363 int ret = 0; 1364 uint64_t bytes_remaining = bytes; 1365 int max_transfer; 1366 1367 assert(is_power_of_2(align)); 1368 assert((offset & (align - 1)) == 0); 1369 assert((bytes & (align - 1)) == 0); 1370 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1371 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1372 align); 1373 1374 /* TODO: We would need a per-BDS .supported_read_flags and 1375 * potential fallback support, if we ever implement any read flags 1376 * to pass through to drivers. For now, there aren't any 1377 * passthrough flags. */ 1378 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH))); 1379 1380 /* Handle Copy on Read and associated serialisation */ 1381 if (flags & BDRV_REQ_COPY_ON_READ) { 1382 /* If we touch the same cluster it counts as an overlap. This 1383 * guarantees that allocating writes will be serialized and not race 1384 * with each other for the same cluster. For example, in copy-on-read 1385 * it ensures that the CoR read and write operations are atomic and 1386 * guest writes cannot interleave between them. */ 1387 bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs)); 1388 } else { 1389 bdrv_wait_serialising_requests(req); 1390 } 1391 1392 if (flags & BDRV_REQ_COPY_ON_READ) { 1393 int64_t pnum; 1394 1395 ret = bdrv_is_allocated(bs, offset, bytes, &pnum); 1396 if (ret < 0) { 1397 goto out; 1398 } 1399 1400 if (!ret || pnum != bytes) { 1401 ret = bdrv_co_do_copy_on_readv(child, offset, bytes, 1402 qiov, qiov_offset, flags); 1403 goto out; 1404 } else if (flags & BDRV_REQ_PREFETCH) { 1405 goto out; 1406 } 1407 } 1408 1409 /* Forward the request to the BlockDriver, possibly fragmenting it */ 1410 total_bytes = bdrv_getlength(bs); 1411 if (total_bytes < 0) { 1412 ret = total_bytes; 1413 goto out; 1414 } 1415 1416 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align); 1417 if (bytes <= max_bytes && bytes <= max_transfer) { 1418 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0); 1419 goto out; 1420 } 1421 1422 while (bytes_remaining) { 1423 int num; 1424 1425 if (max_bytes) { 1426 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer)); 1427 assert(num); 1428 1429 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining, 1430 num, qiov, 1431 qiov_offset + bytes - bytes_remaining, 0); 1432 max_bytes -= num; 1433 } else { 1434 num = bytes_remaining; 1435 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining, 1436 0, bytes_remaining); 1437 } 1438 if (ret < 0) { 1439 goto out; 1440 } 1441 bytes_remaining -= num; 1442 } 1443 1444 out: 1445 return ret < 0 ? ret : 0; 1446 } 1447 1448 /* 1449 * Request padding 1450 * 1451 * |<---- align ----->| |<----- align ---->| 1452 * |<- head ->|<------------- bytes ------------->|<-- tail -->| 1453 * | | | | | | 1454 * -*----------$-------*-------- ... --------*-----$------------*--- 1455 * | | | | | | 1456 * | offset | | end | 1457 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end) 1458 * [buf ... ) [tail_buf ) 1459 * 1460 * @buf is an aligned allocation needed to store @head and @tail paddings. @head 1461 * is placed at the beginning of @buf and @tail at the @end. 1462 * 1463 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk 1464 * around tail, if tail exists. 1465 * 1466 * @merge_reads is true for small requests, 1467 * if @buf_len == @head + bytes + @tail. In this case it is possible that both 1468 * head and tail exist but @buf_len == align and @tail_buf == @buf. 1469 */ 1470 typedef struct BdrvRequestPadding { 1471 uint8_t *buf; 1472 size_t buf_len; 1473 uint8_t *tail_buf; 1474 size_t head; 1475 size_t tail; 1476 bool merge_reads; 1477 QEMUIOVector local_qiov; 1478 } BdrvRequestPadding; 1479 1480 static bool bdrv_init_padding(BlockDriverState *bs, 1481 int64_t offset, int64_t bytes, 1482 BdrvRequestPadding *pad) 1483 { 1484 uint64_t align = bs->bl.request_alignment; 1485 size_t sum; 1486 1487 memset(pad, 0, sizeof(*pad)); 1488 1489 pad->head = offset & (align - 1); 1490 pad->tail = ((offset + bytes) & (align - 1)); 1491 if (pad->tail) { 1492 pad->tail = align - pad->tail; 1493 } 1494 1495 if (!pad->head && !pad->tail) { 1496 return false; 1497 } 1498 1499 assert(bytes); /* Nothing good in aligning zero-length requests */ 1500 1501 sum = pad->head + bytes + pad->tail; 1502 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align; 1503 pad->buf = qemu_blockalign(bs, pad->buf_len); 1504 pad->merge_reads = sum == pad->buf_len; 1505 if (pad->tail) { 1506 pad->tail_buf = pad->buf + pad->buf_len - align; 1507 } 1508 1509 return true; 1510 } 1511 1512 static int bdrv_padding_rmw_read(BdrvChild *child, 1513 BdrvTrackedRequest *req, 1514 BdrvRequestPadding *pad, 1515 bool zero_middle) 1516 { 1517 QEMUIOVector local_qiov; 1518 BlockDriverState *bs = child->bs; 1519 uint64_t align = bs->bl.request_alignment; 1520 int ret; 1521 1522 assert(req->serialising && pad->buf); 1523 1524 if (pad->head || pad->merge_reads) { 1525 uint64_t bytes = pad->merge_reads ? pad->buf_len : align; 1526 1527 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes); 1528 1529 if (pad->head) { 1530 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD); 1531 } 1532 if (pad->merge_reads && pad->tail) { 1533 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1534 } 1535 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes, 1536 align, &local_qiov, 0, 0); 1537 if (ret < 0) { 1538 return ret; 1539 } 1540 if (pad->head) { 1541 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD); 1542 } 1543 if (pad->merge_reads && pad->tail) { 1544 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1545 } 1546 1547 if (pad->merge_reads) { 1548 goto zero_mem; 1549 } 1550 } 1551 1552 if (pad->tail) { 1553 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align); 1554 1555 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL); 1556 ret = bdrv_aligned_preadv( 1557 child, req, 1558 req->overlap_offset + req->overlap_bytes - align, 1559 align, align, &local_qiov, 0, 0); 1560 if (ret < 0) { 1561 return ret; 1562 } 1563 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL); 1564 } 1565 1566 zero_mem: 1567 if (zero_middle) { 1568 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail); 1569 } 1570 1571 return 0; 1572 } 1573 1574 static void bdrv_padding_destroy(BdrvRequestPadding *pad) 1575 { 1576 if (pad->buf) { 1577 qemu_vfree(pad->buf); 1578 qemu_iovec_destroy(&pad->local_qiov); 1579 } 1580 } 1581 1582 /* 1583 * bdrv_pad_request 1584 * 1585 * Exchange request parameters with padded request if needed. Don't include RMW 1586 * read of padding, bdrv_padding_rmw_read() should be called separately if 1587 * needed. 1588 * 1589 * All parameters except @bs are in-out: they represent original request at 1590 * function call and padded (if padding needed) at function finish. 1591 * 1592 * Function always succeeds. 1593 */ 1594 static bool bdrv_pad_request(BlockDriverState *bs, 1595 QEMUIOVector **qiov, size_t *qiov_offset, 1596 int64_t *offset, unsigned int *bytes, 1597 BdrvRequestPadding *pad) 1598 { 1599 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) { 1600 return false; 1601 } 1602 1603 qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head, 1604 *qiov, *qiov_offset, *bytes, 1605 pad->buf + pad->buf_len - pad->tail, pad->tail); 1606 *bytes += pad->head + pad->tail; 1607 *offset -= pad->head; 1608 *qiov = &pad->local_qiov; 1609 *qiov_offset = 0; 1610 1611 return true; 1612 } 1613 1614 int coroutine_fn bdrv_co_preadv(BdrvChild *child, 1615 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 1616 BdrvRequestFlags flags) 1617 { 1618 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags); 1619 } 1620 1621 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child, 1622 int64_t offset, unsigned int bytes, 1623 QEMUIOVector *qiov, size_t qiov_offset, 1624 BdrvRequestFlags flags) 1625 { 1626 BlockDriverState *bs = child->bs; 1627 BdrvTrackedRequest req; 1628 BdrvRequestPadding pad; 1629 int ret; 1630 1631 trace_bdrv_co_preadv(bs, offset, bytes, flags); 1632 1633 ret = bdrv_check_byte_request(bs, offset, bytes); 1634 if (ret < 0) { 1635 return ret; 1636 } 1637 1638 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 1639 /* 1640 * Aligning zero request is nonsense. Even if driver has special meaning 1641 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 1642 * it to driver due to request_alignment. 1643 * 1644 * Still, no reason to return an error if someone do unaligned 1645 * zero-length read occasionally. 1646 */ 1647 return 0; 1648 } 1649 1650 bdrv_inc_in_flight(bs); 1651 1652 /* Don't do copy-on-read if we read data before write operation */ 1653 if (qatomic_read(&bs->copy_on_read)) { 1654 flags |= BDRV_REQ_COPY_ON_READ; 1655 } 1656 1657 bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad); 1658 1659 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ); 1660 ret = bdrv_aligned_preadv(child, &req, offset, bytes, 1661 bs->bl.request_alignment, 1662 qiov, qiov_offset, flags); 1663 tracked_request_end(&req); 1664 bdrv_dec_in_flight(bs); 1665 1666 bdrv_padding_destroy(&pad); 1667 1668 return ret; 1669 } 1670 1671 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs, 1672 int64_t offset, int bytes, BdrvRequestFlags flags) 1673 { 1674 BlockDriver *drv = bs->drv; 1675 QEMUIOVector qiov; 1676 void *buf = NULL; 1677 int ret = 0; 1678 bool need_flush = false; 1679 int head = 0; 1680 int tail = 0; 1681 1682 int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX); 1683 int alignment = MAX(bs->bl.pwrite_zeroes_alignment, 1684 bs->bl.request_alignment); 1685 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER); 1686 1687 if (!drv) { 1688 return -ENOMEDIUM; 1689 } 1690 1691 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) { 1692 return -ENOTSUP; 1693 } 1694 1695 assert(alignment % bs->bl.request_alignment == 0); 1696 head = offset % alignment; 1697 tail = (offset + bytes) % alignment; 1698 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment); 1699 assert(max_write_zeroes >= bs->bl.request_alignment); 1700 1701 while (bytes > 0 && !ret) { 1702 int num = bytes; 1703 1704 /* Align request. Block drivers can expect the "bulk" of the request 1705 * to be aligned, and that unaligned requests do not cross cluster 1706 * boundaries. 1707 */ 1708 if (head) { 1709 /* Make a small request up to the first aligned sector. For 1710 * convenience, limit this request to max_transfer even if 1711 * we don't need to fall back to writes. */ 1712 num = MIN(MIN(bytes, max_transfer), alignment - head); 1713 head = (head + num) % alignment; 1714 assert(num < max_write_zeroes); 1715 } else if (tail && num > alignment) { 1716 /* Shorten the request to the last aligned sector. */ 1717 num -= tail; 1718 } 1719 1720 /* limit request size */ 1721 if (num > max_write_zeroes) { 1722 num = max_write_zeroes; 1723 } 1724 1725 ret = -ENOTSUP; 1726 /* First try the efficient write zeroes operation */ 1727 if (drv->bdrv_co_pwrite_zeroes) { 1728 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num, 1729 flags & bs->supported_zero_flags); 1730 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) && 1731 !(bs->supported_zero_flags & BDRV_REQ_FUA)) { 1732 need_flush = true; 1733 } 1734 } else { 1735 assert(!bs->supported_zero_flags); 1736 } 1737 1738 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) { 1739 /* Fall back to bounce buffer if write zeroes is unsupported */ 1740 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE; 1741 1742 if ((flags & BDRV_REQ_FUA) && 1743 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1744 /* No need for bdrv_driver_pwrite() to do a fallback 1745 * flush on each chunk; use just one at the end */ 1746 write_flags &= ~BDRV_REQ_FUA; 1747 need_flush = true; 1748 } 1749 num = MIN(num, max_transfer); 1750 if (buf == NULL) { 1751 buf = qemu_try_blockalign0(bs, num); 1752 if (buf == NULL) { 1753 ret = -ENOMEM; 1754 goto fail; 1755 } 1756 } 1757 qemu_iovec_init_buf(&qiov, buf, num); 1758 1759 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags); 1760 1761 /* Keep bounce buffer around if it is big enough for all 1762 * all future requests. 1763 */ 1764 if (num < max_transfer) { 1765 qemu_vfree(buf); 1766 buf = NULL; 1767 } 1768 } 1769 1770 offset += num; 1771 bytes -= num; 1772 } 1773 1774 fail: 1775 if (ret == 0 && need_flush) { 1776 ret = bdrv_co_flush(bs); 1777 } 1778 qemu_vfree(buf); 1779 return ret; 1780 } 1781 1782 static inline int coroutine_fn 1783 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes, 1784 BdrvTrackedRequest *req, int flags) 1785 { 1786 BlockDriverState *bs = child->bs; 1787 bool waited; 1788 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1789 1790 if (bs->read_only) { 1791 return -EPERM; 1792 } 1793 1794 assert(!(bs->open_flags & BDRV_O_INACTIVE)); 1795 assert((bs->open_flags & BDRV_O_NO_IO) == 0); 1796 assert(!(flags & ~BDRV_REQ_MASK)); 1797 1798 if (flags & BDRV_REQ_SERIALISING) { 1799 waited = bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs)); 1800 /* 1801 * For a misaligned request we should have already waited earlier, 1802 * because we come after bdrv_padding_rmw_read which must be called 1803 * with the request already marked as serialising. 1804 */ 1805 assert(!waited || 1806 (req->offset == req->overlap_offset && 1807 req->bytes == req->overlap_bytes)); 1808 } else { 1809 bdrv_wait_serialising_requests(req); 1810 } 1811 1812 assert(req->overlap_offset <= offset); 1813 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes); 1814 assert(end_sector <= bs->total_sectors || child->perm & BLK_PERM_RESIZE); 1815 1816 switch (req->type) { 1817 case BDRV_TRACKED_WRITE: 1818 case BDRV_TRACKED_DISCARD: 1819 if (flags & BDRV_REQ_WRITE_UNCHANGED) { 1820 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE)); 1821 } else { 1822 assert(child->perm & BLK_PERM_WRITE); 1823 } 1824 return notifier_with_return_list_notify(&bs->before_write_notifiers, 1825 req); 1826 case BDRV_TRACKED_TRUNCATE: 1827 assert(child->perm & BLK_PERM_RESIZE); 1828 return 0; 1829 default: 1830 abort(); 1831 } 1832 } 1833 1834 static inline void coroutine_fn 1835 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, uint64_t bytes, 1836 BdrvTrackedRequest *req, int ret) 1837 { 1838 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE); 1839 BlockDriverState *bs = child->bs; 1840 1841 qatomic_inc(&bs->write_gen); 1842 1843 /* 1844 * Discard cannot extend the image, but in error handling cases, such as 1845 * when reverting a qcow2 cluster allocation, the discarded range can pass 1846 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD 1847 * here. Instead, just skip it, since semantically a discard request 1848 * beyond EOF cannot expand the image anyway. 1849 */ 1850 if (ret == 0 && 1851 (req->type == BDRV_TRACKED_TRUNCATE || 1852 end_sector > bs->total_sectors) && 1853 req->type != BDRV_TRACKED_DISCARD) { 1854 bs->total_sectors = end_sector; 1855 bdrv_parent_cb_resize(bs); 1856 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS); 1857 } 1858 if (req->bytes) { 1859 switch (req->type) { 1860 case BDRV_TRACKED_WRITE: 1861 stat64_max(&bs->wr_highest_offset, offset + bytes); 1862 /* fall through, to set dirty bits */ 1863 case BDRV_TRACKED_DISCARD: 1864 bdrv_set_dirty(bs, offset, bytes); 1865 break; 1866 default: 1867 break; 1868 } 1869 } 1870 } 1871 1872 /* 1873 * Forwards an already correctly aligned write request to the BlockDriver, 1874 * after possibly fragmenting it. 1875 */ 1876 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child, 1877 BdrvTrackedRequest *req, int64_t offset, unsigned int bytes, 1878 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags) 1879 { 1880 BlockDriverState *bs = child->bs; 1881 BlockDriver *drv = bs->drv; 1882 int ret; 1883 1884 uint64_t bytes_remaining = bytes; 1885 int max_transfer; 1886 1887 if (!drv) { 1888 return -ENOMEDIUM; 1889 } 1890 1891 if (bdrv_has_readonly_bitmaps(bs)) { 1892 return -EPERM; 1893 } 1894 1895 assert(is_power_of_2(align)); 1896 assert((offset & (align - 1)) == 0); 1897 assert((bytes & (align - 1)) == 0); 1898 assert(!qiov || qiov_offset + bytes <= qiov->size); 1899 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX), 1900 align); 1901 1902 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags); 1903 1904 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF && 1905 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes && 1906 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) { 1907 flags |= BDRV_REQ_ZERO_WRITE; 1908 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) { 1909 flags |= BDRV_REQ_MAY_UNMAP; 1910 } 1911 } 1912 1913 if (ret < 0) { 1914 /* Do nothing, write notifier decided to fail this request */ 1915 } else if (flags & BDRV_REQ_ZERO_WRITE) { 1916 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO); 1917 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags); 1918 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) { 1919 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes, 1920 qiov, qiov_offset); 1921 } else if (bytes <= max_transfer) { 1922 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1923 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags); 1924 } else { 1925 bdrv_debug_event(bs, BLKDBG_PWRITEV); 1926 while (bytes_remaining) { 1927 int num = MIN(bytes_remaining, max_transfer); 1928 int local_flags = flags; 1929 1930 assert(num); 1931 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) && 1932 !(bs->supported_write_flags & BDRV_REQ_FUA)) { 1933 /* If FUA is going to be emulated by flush, we only 1934 * need to flush on the last iteration */ 1935 local_flags &= ~BDRV_REQ_FUA; 1936 } 1937 1938 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining, 1939 num, qiov, 1940 qiov_offset + bytes - bytes_remaining, 1941 local_flags); 1942 if (ret < 0) { 1943 break; 1944 } 1945 bytes_remaining -= num; 1946 } 1947 } 1948 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE); 1949 1950 if (ret >= 0) { 1951 ret = 0; 1952 } 1953 bdrv_co_write_req_finish(child, offset, bytes, req, ret); 1954 1955 return ret; 1956 } 1957 1958 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child, 1959 int64_t offset, 1960 unsigned int bytes, 1961 BdrvRequestFlags flags, 1962 BdrvTrackedRequest *req) 1963 { 1964 BlockDriverState *bs = child->bs; 1965 QEMUIOVector local_qiov; 1966 uint64_t align = bs->bl.request_alignment; 1967 int ret = 0; 1968 bool padding; 1969 BdrvRequestPadding pad; 1970 1971 padding = bdrv_init_padding(bs, offset, bytes, &pad); 1972 if (padding) { 1973 bdrv_mark_request_serialising(req, align); 1974 1975 bdrv_padding_rmw_read(child, req, &pad, true); 1976 1977 if (pad.head || pad.merge_reads) { 1978 int64_t aligned_offset = offset & ~(align - 1); 1979 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align; 1980 1981 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes); 1982 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes, 1983 align, &local_qiov, 0, 1984 flags & ~BDRV_REQ_ZERO_WRITE); 1985 if (ret < 0 || pad.merge_reads) { 1986 /* Error or all work is done */ 1987 goto out; 1988 } 1989 offset += write_bytes - pad.head; 1990 bytes -= write_bytes - pad.head; 1991 } 1992 } 1993 1994 assert(!bytes || (offset & (align - 1)) == 0); 1995 if (bytes >= align) { 1996 /* Write the aligned part in the middle. */ 1997 uint64_t aligned_bytes = bytes & ~(align - 1); 1998 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align, 1999 NULL, 0, flags); 2000 if (ret < 0) { 2001 goto out; 2002 } 2003 bytes -= aligned_bytes; 2004 offset += aligned_bytes; 2005 } 2006 2007 assert(!bytes || (offset & (align - 1)) == 0); 2008 if (bytes) { 2009 assert(align == pad.tail + bytes); 2010 2011 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align); 2012 ret = bdrv_aligned_pwritev(child, req, offset, align, align, 2013 &local_qiov, 0, 2014 flags & ~BDRV_REQ_ZERO_WRITE); 2015 } 2016 2017 out: 2018 bdrv_padding_destroy(&pad); 2019 2020 return ret; 2021 } 2022 2023 /* 2024 * Handle a write request in coroutine context 2025 */ 2026 int coroutine_fn bdrv_co_pwritev(BdrvChild *child, 2027 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, 2028 BdrvRequestFlags flags) 2029 { 2030 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags); 2031 } 2032 2033 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child, 2034 int64_t offset, unsigned int bytes, QEMUIOVector *qiov, size_t qiov_offset, 2035 BdrvRequestFlags flags) 2036 { 2037 BlockDriverState *bs = child->bs; 2038 BdrvTrackedRequest req; 2039 uint64_t align = bs->bl.request_alignment; 2040 BdrvRequestPadding pad; 2041 int ret; 2042 2043 trace_bdrv_co_pwritev(child->bs, offset, bytes, flags); 2044 2045 if (!bs->drv) { 2046 return -ENOMEDIUM; 2047 } 2048 2049 ret = bdrv_check_byte_request(bs, offset, bytes); 2050 if (ret < 0) { 2051 return ret; 2052 } 2053 2054 /* If the request is misaligned then we can't make it efficient */ 2055 if ((flags & BDRV_REQ_NO_FALLBACK) && 2056 !QEMU_IS_ALIGNED(offset | bytes, align)) 2057 { 2058 return -ENOTSUP; 2059 } 2060 2061 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) { 2062 /* 2063 * Aligning zero request is nonsense. Even if driver has special meaning 2064 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass 2065 * it to driver due to request_alignment. 2066 * 2067 * Still, no reason to return an error if someone do unaligned 2068 * zero-length write occasionally. 2069 */ 2070 return 0; 2071 } 2072 2073 bdrv_inc_in_flight(bs); 2074 /* 2075 * Align write if necessary by performing a read-modify-write cycle. 2076 * Pad qiov with the read parts and be sure to have a tracked request not 2077 * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle. 2078 */ 2079 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE); 2080 2081 if (flags & BDRV_REQ_ZERO_WRITE) { 2082 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req); 2083 goto out; 2084 } 2085 2086 if (bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad)) { 2087 bdrv_mark_request_serialising(&req, align); 2088 bdrv_padding_rmw_read(child, &req, &pad, false); 2089 } 2090 2091 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align, 2092 qiov, qiov_offset, flags); 2093 2094 bdrv_padding_destroy(&pad); 2095 2096 out: 2097 tracked_request_end(&req); 2098 bdrv_dec_in_flight(bs); 2099 2100 return ret; 2101 } 2102 2103 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset, 2104 int bytes, BdrvRequestFlags flags) 2105 { 2106 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags); 2107 2108 if (!(child->bs->open_flags & BDRV_O_UNMAP)) { 2109 flags &= ~BDRV_REQ_MAY_UNMAP; 2110 } 2111 2112 return bdrv_co_pwritev(child, offset, bytes, NULL, 2113 BDRV_REQ_ZERO_WRITE | flags); 2114 } 2115 2116 /* 2117 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not. 2118 */ 2119 int bdrv_flush_all(void) 2120 { 2121 BdrvNextIterator it; 2122 BlockDriverState *bs = NULL; 2123 int result = 0; 2124 2125 /* 2126 * bdrv queue is managed by record/replay, 2127 * creating new flush request for stopping 2128 * the VM may break the determinism 2129 */ 2130 if (replay_events_enabled()) { 2131 return result; 2132 } 2133 2134 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { 2135 AioContext *aio_context = bdrv_get_aio_context(bs); 2136 int ret; 2137 2138 aio_context_acquire(aio_context); 2139 ret = bdrv_flush(bs); 2140 if (ret < 0 && !result) { 2141 result = ret; 2142 } 2143 aio_context_release(aio_context); 2144 } 2145 2146 return result; 2147 } 2148 2149 /* 2150 * Returns the allocation status of the specified sectors. 2151 * Drivers not implementing the functionality are assumed to not support 2152 * backing files, hence all their sectors are reported as allocated. 2153 * 2154 * If 'want_zero' is true, the caller is querying for mapping 2155 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and 2156 * _ZERO where possible; otherwise, the result favors larger 'pnum', 2157 * with a focus on accurate BDRV_BLOCK_ALLOCATED. 2158 * 2159 * If 'offset' is beyond the end of the disk image the return value is 2160 * BDRV_BLOCK_EOF and 'pnum' is set to 0. 2161 * 2162 * 'bytes' is the max value 'pnum' should be set to. If bytes goes 2163 * beyond the end of the disk image it will be clamped; if 'pnum' is set to 2164 * the end of the image, then the returned value will include BDRV_BLOCK_EOF. 2165 * 2166 * 'pnum' is set to the number of bytes (including and immediately 2167 * following the specified offset) that are easily known to be in the 2168 * same allocated/unallocated state. Note that a second call starting 2169 * at the original offset plus returned pnum may have the same status. 2170 * The returned value is non-zero on success except at end-of-file. 2171 * 2172 * Returns negative errno on failure. Otherwise, if the 2173 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are 2174 * set to the host mapping and BDS corresponding to the guest offset. 2175 */ 2176 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs, 2177 bool want_zero, 2178 int64_t offset, int64_t bytes, 2179 int64_t *pnum, int64_t *map, 2180 BlockDriverState **file) 2181 { 2182 int64_t total_size; 2183 int64_t n; /* bytes */ 2184 int ret; 2185 int64_t local_map = 0; 2186 BlockDriverState *local_file = NULL; 2187 int64_t aligned_offset, aligned_bytes; 2188 uint32_t align; 2189 bool has_filtered_child; 2190 2191 assert(pnum); 2192 *pnum = 0; 2193 total_size = bdrv_getlength(bs); 2194 if (total_size < 0) { 2195 ret = total_size; 2196 goto early_out; 2197 } 2198 2199 if (offset >= total_size) { 2200 ret = BDRV_BLOCK_EOF; 2201 goto early_out; 2202 } 2203 if (!bytes) { 2204 ret = 0; 2205 goto early_out; 2206 } 2207 2208 n = total_size - offset; 2209 if (n < bytes) { 2210 bytes = n; 2211 } 2212 2213 /* Must be non-NULL or bdrv_getlength() would have failed */ 2214 assert(bs->drv); 2215 has_filtered_child = bdrv_filter_child(bs); 2216 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) { 2217 *pnum = bytes; 2218 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED; 2219 if (offset + bytes == total_size) { 2220 ret |= BDRV_BLOCK_EOF; 2221 } 2222 if (bs->drv->protocol_name) { 2223 ret |= BDRV_BLOCK_OFFSET_VALID; 2224 local_map = offset; 2225 local_file = bs; 2226 } 2227 goto early_out; 2228 } 2229 2230 bdrv_inc_in_flight(bs); 2231 2232 /* Round out to request_alignment boundaries */ 2233 align = bs->bl.request_alignment; 2234 aligned_offset = QEMU_ALIGN_DOWN(offset, align); 2235 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset; 2236 2237 if (bs->drv->bdrv_co_block_status) { 2238 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset, 2239 aligned_bytes, pnum, &local_map, 2240 &local_file); 2241 } else { 2242 /* Default code for filters */ 2243 2244 local_file = bdrv_filter_bs(bs); 2245 assert(local_file); 2246 2247 *pnum = aligned_bytes; 2248 local_map = aligned_offset; 2249 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID; 2250 } 2251 if (ret < 0) { 2252 *pnum = 0; 2253 goto out; 2254 } 2255 2256 /* 2257 * The driver's result must be a non-zero multiple of request_alignment. 2258 * Clamp pnum and adjust map to original request. 2259 */ 2260 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) && 2261 align > offset - aligned_offset); 2262 if (ret & BDRV_BLOCK_RECURSE) { 2263 assert(ret & BDRV_BLOCK_DATA); 2264 assert(ret & BDRV_BLOCK_OFFSET_VALID); 2265 assert(!(ret & BDRV_BLOCK_ZERO)); 2266 } 2267 2268 *pnum -= offset - aligned_offset; 2269 if (*pnum > bytes) { 2270 *pnum = bytes; 2271 } 2272 if (ret & BDRV_BLOCK_OFFSET_VALID) { 2273 local_map += offset - aligned_offset; 2274 } 2275 2276 if (ret & BDRV_BLOCK_RAW) { 2277 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file); 2278 ret = bdrv_co_block_status(local_file, want_zero, local_map, 2279 *pnum, pnum, &local_map, &local_file); 2280 goto out; 2281 } 2282 2283 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) { 2284 ret |= BDRV_BLOCK_ALLOCATED; 2285 } else if (want_zero && bs->drv->supports_backing) { 2286 BlockDriverState *cow_bs = bdrv_cow_bs(bs); 2287 2288 if (cow_bs) { 2289 int64_t size2 = bdrv_getlength(cow_bs); 2290 2291 if (size2 >= 0 && offset >= size2) { 2292 ret |= BDRV_BLOCK_ZERO; 2293 } 2294 } else { 2295 ret |= BDRV_BLOCK_ZERO; 2296 } 2297 } 2298 2299 if (want_zero && ret & BDRV_BLOCK_RECURSE && 2300 local_file && local_file != bs && 2301 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) && 2302 (ret & BDRV_BLOCK_OFFSET_VALID)) { 2303 int64_t file_pnum; 2304 int ret2; 2305 2306 ret2 = bdrv_co_block_status(local_file, want_zero, local_map, 2307 *pnum, &file_pnum, NULL, NULL); 2308 if (ret2 >= 0) { 2309 /* Ignore errors. This is just providing extra information, it 2310 * is useful but not necessary. 2311 */ 2312 if (ret2 & BDRV_BLOCK_EOF && 2313 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) { 2314 /* 2315 * It is valid for the format block driver to read 2316 * beyond the end of the underlying file's current 2317 * size; such areas read as zero. 2318 */ 2319 ret |= BDRV_BLOCK_ZERO; 2320 } else { 2321 /* Limit request to the range reported by the protocol driver */ 2322 *pnum = file_pnum; 2323 ret |= (ret2 & BDRV_BLOCK_ZERO); 2324 } 2325 } 2326 } 2327 2328 out: 2329 bdrv_dec_in_flight(bs); 2330 if (ret >= 0 && offset + *pnum == total_size) { 2331 ret |= BDRV_BLOCK_EOF; 2332 } 2333 early_out: 2334 if (file) { 2335 *file = local_file; 2336 } 2337 if (map) { 2338 *map = local_map; 2339 } 2340 return ret; 2341 } 2342 2343 int coroutine_fn 2344 bdrv_co_common_block_status_above(BlockDriverState *bs, 2345 BlockDriverState *base, 2346 bool include_base, 2347 bool want_zero, 2348 int64_t offset, 2349 int64_t bytes, 2350 int64_t *pnum, 2351 int64_t *map, 2352 BlockDriverState **file) 2353 { 2354 int ret; 2355 BlockDriverState *p; 2356 int64_t eof = 0; 2357 2358 assert(!include_base || base); /* Can't include NULL base */ 2359 2360 if (!include_base && bs == base) { 2361 *pnum = bytes; 2362 return 0; 2363 } 2364 2365 ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file); 2366 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) { 2367 return ret; 2368 } 2369 2370 if (ret & BDRV_BLOCK_EOF) { 2371 eof = offset + *pnum; 2372 } 2373 2374 assert(*pnum <= bytes); 2375 bytes = *pnum; 2376 2377 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base; 2378 p = bdrv_filter_or_cow_bs(p)) 2379 { 2380 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map, 2381 file); 2382 if (ret < 0) { 2383 return ret; 2384 } 2385 if (*pnum == 0) { 2386 /* 2387 * The top layer deferred to this layer, and because this layer is 2388 * short, any zeroes that we synthesize beyond EOF behave as if they 2389 * were allocated at this layer. 2390 * 2391 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be 2392 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2393 * below. 2394 */ 2395 assert(ret & BDRV_BLOCK_EOF); 2396 *pnum = bytes; 2397 if (file) { 2398 *file = p; 2399 } 2400 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED; 2401 break; 2402 } 2403 if (ret & BDRV_BLOCK_ALLOCATED) { 2404 /* 2405 * We've found the node and the status, we must break. 2406 * 2407 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be 2408 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see 2409 * below. 2410 */ 2411 ret &= ~BDRV_BLOCK_EOF; 2412 break; 2413 } 2414 2415 if (p == base) { 2416 assert(include_base); 2417 break; 2418 } 2419 2420 /* 2421 * OK, [offset, offset + *pnum) region is unallocated on this layer, 2422 * let's continue the diving. 2423 */ 2424 assert(*pnum <= bytes); 2425 bytes = *pnum; 2426 } 2427 2428 if (offset + *pnum == eof) { 2429 ret |= BDRV_BLOCK_EOF; 2430 } 2431 2432 return ret; 2433 } 2434 2435 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base, 2436 int64_t offset, int64_t bytes, int64_t *pnum, 2437 int64_t *map, BlockDriverState **file) 2438 { 2439 return bdrv_common_block_status_above(bs, base, false, true, offset, bytes, 2440 pnum, map, file); 2441 } 2442 2443 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes, 2444 int64_t *pnum, int64_t *map, BlockDriverState **file) 2445 { 2446 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs), 2447 offset, bytes, pnum, map, file); 2448 } 2449 2450 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset, 2451 int64_t bytes, int64_t *pnum) 2452 { 2453 int ret; 2454 int64_t dummy; 2455 2456 ret = bdrv_common_block_status_above(bs, bs, true, false, offset, 2457 bytes, pnum ? pnum : &dummy, NULL, 2458 NULL); 2459 if (ret < 0) { 2460 return ret; 2461 } 2462 return !!(ret & BDRV_BLOCK_ALLOCATED); 2463 } 2464 2465 /* 2466 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP] 2467 * 2468 * Return 1 if (a prefix of) the given range is allocated in any image 2469 * between BASE and TOP (BASE is only included if include_base is set). 2470 * BASE can be NULL to check if the given offset is allocated in any 2471 * image of the chain. Return 0 otherwise, or negative errno on 2472 * failure. 2473 * 2474 * 'pnum' is set to the number of bytes (including and immediately 2475 * following the specified offset) that are known to be in the same 2476 * allocated/unallocated state. Note that a subsequent call starting 2477 * at 'offset + *pnum' may return the same allocation status (in other 2478 * words, the result is not necessarily the maximum possible range); 2479 * but 'pnum' will only be 0 when end of file is reached. 2480 */ 2481 int bdrv_is_allocated_above(BlockDriverState *top, 2482 BlockDriverState *base, 2483 bool include_base, int64_t offset, 2484 int64_t bytes, int64_t *pnum) 2485 { 2486 int ret = bdrv_common_block_status_above(top, base, include_base, false, 2487 offset, bytes, pnum, NULL, NULL); 2488 if (ret < 0) { 2489 return ret; 2490 } 2491 2492 return !!(ret & BDRV_BLOCK_ALLOCATED); 2493 } 2494 2495 int coroutine_fn 2496 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2497 { 2498 BlockDriver *drv = bs->drv; 2499 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2500 int ret = -ENOTSUP; 2501 2502 if (!drv) { 2503 return -ENOMEDIUM; 2504 } 2505 2506 bdrv_inc_in_flight(bs); 2507 2508 if (drv->bdrv_load_vmstate) { 2509 ret = drv->bdrv_load_vmstate(bs, qiov, pos); 2510 } else if (child_bs) { 2511 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos); 2512 } 2513 2514 bdrv_dec_in_flight(bs); 2515 2516 return ret; 2517 } 2518 2519 int coroutine_fn 2520 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos) 2521 { 2522 BlockDriver *drv = bs->drv; 2523 BlockDriverState *child_bs = bdrv_primary_bs(bs); 2524 int ret = -ENOTSUP; 2525 2526 if (!drv) { 2527 return -ENOMEDIUM; 2528 } 2529 2530 bdrv_inc_in_flight(bs); 2531 2532 if (drv->bdrv_save_vmstate) { 2533 ret = drv->bdrv_save_vmstate(bs, qiov, pos); 2534 } else if (child_bs) { 2535 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos); 2536 } 2537 2538 bdrv_dec_in_flight(bs); 2539 2540 return ret; 2541 } 2542 2543 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf, 2544 int64_t pos, int size) 2545 { 2546 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2547 int ret = bdrv_writev_vmstate(bs, &qiov, pos); 2548 2549 return ret < 0 ? ret : size; 2550 } 2551 2552 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf, 2553 int64_t pos, int size) 2554 { 2555 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size); 2556 int ret = bdrv_readv_vmstate(bs, &qiov, pos); 2557 2558 return ret < 0 ? ret : size; 2559 } 2560 2561 /**************************************************************/ 2562 /* async I/Os */ 2563 2564 void bdrv_aio_cancel(BlockAIOCB *acb) 2565 { 2566 qemu_aio_ref(acb); 2567 bdrv_aio_cancel_async(acb); 2568 while (acb->refcnt > 1) { 2569 if (acb->aiocb_info->get_aio_context) { 2570 aio_poll(acb->aiocb_info->get_aio_context(acb), true); 2571 } else if (acb->bs) { 2572 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so 2573 * assert that we're not using an I/O thread. Thread-safe 2574 * code should use bdrv_aio_cancel_async exclusively. 2575 */ 2576 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context()); 2577 aio_poll(bdrv_get_aio_context(acb->bs), true); 2578 } else { 2579 abort(); 2580 } 2581 } 2582 qemu_aio_unref(acb); 2583 } 2584 2585 /* Async version of aio cancel. The caller is not blocked if the acb implements 2586 * cancel_async, otherwise we do nothing and let the request normally complete. 2587 * In either case the completion callback must be called. */ 2588 void bdrv_aio_cancel_async(BlockAIOCB *acb) 2589 { 2590 if (acb->aiocb_info->cancel_async) { 2591 acb->aiocb_info->cancel_async(acb); 2592 } 2593 } 2594 2595 /**************************************************************/ 2596 /* Coroutine block device emulation */ 2597 2598 int coroutine_fn bdrv_co_flush(BlockDriverState *bs) 2599 { 2600 BdrvChild *primary_child = bdrv_primary_child(bs); 2601 BdrvChild *child; 2602 int current_gen; 2603 int ret = 0; 2604 2605 bdrv_inc_in_flight(bs); 2606 2607 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) || 2608 bdrv_is_sg(bs)) { 2609 goto early_exit; 2610 } 2611 2612 qemu_co_mutex_lock(&bs->reqs_lock); 2613 current_gen = qatomic_read(&bs->write_gen); 2614 2615 /* Wait until any previous flushes are completed */ 2616 while (bs->active_flush_req) { 2617 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock); 2618 } 2619 2620 /* Flushes reach this point in nondecreasing current_gen order. */ 2621 bs->active_flush_req = true; 2622 qemu_co_mutex_unlock(&bs->reqs_lock); 2623 2624 /* Write back all layers by calling one driver function */ 2625 if (bs->drv->bdrv_co_flush) { 2626 ret = bs->drv->bdrv_co_flush(bs); 2627 goto out; 2628 } 2629 2630 /* Write back cached data to the OS even with cache=unsafe */ 2631 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS); 2632 if (bs->drv->bdrv_co_flush_to_os) { 2633 ret = bs->drv->bdrv_co_flush_to_os(bs); 2634 if (ret < 0) { 2635 goto out; 2636 } 2637 } 2638 2639 /* But don't actually force it to the disk with cache=unsafe */ 2640 if (bs->open_flags & BDRV_O_NO_FLUSH) { 2641 goto flush_children; 2642 } 2643 2644 /* Check if we really need to flush anything */ 2645 if (bs->flushed_gen == current_gen) { 2646 goto flush_children; 2647 } 2648 2649 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK); 2650 if (!bs->drv) { 2651 /* bs->drv->bdrv_co_flush() might have ejected the BDS 2652 * (even in case of apparent success) */ 2653 ret = -ENOMEDIUM; 2654 goto out; 2655 } 2656 if (bs->drv->bdrv_co_flush_to_disk) { 2657 ret = bs->drv->bdrv_co_flush_to_disk(bs); 2658 } else if (bs->drv->bdrv_aio_flush) { 2659 BlockAIOCB *acb; 2660 CoroutineIOCompletion co = { 2661 .coroutine = qemu_coroutine_self(), 2662 }; 2663 2664 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co); 2665 if (acb == NULL) { 2666 ret = -EIO; 2667 } else { 2668 qemu_coroutine_yield(); 2669 ret = co.ret; 2670 } 2671 } else { 2672 /* 2673 * Some block drivers always operate in either writethrough or unsafe 2674 * mode and don't support bdrv_flush therefore. Usually qemu doesn't 2675 * know how the server works (because the behaviour is hardcoded or 2676 * depends on server-side configuration), so we can't ensure that 2677 * everything is safe on disk. Returning an error doesn't work because 2678 * that would break guests even if the server operates in writethrough 2679 * mode. 2680 * 2681 * Let's hope the user knows what he's doing. 2682 */ 2683 ret = 0; 2684 } 2685 2686 if (ret < 0) { 2687 goto out; 2688 } 2689 2690 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH 2691 * in the case of cache=unsafe, so there are no useless flushes. 2692 */ 2693 flush_children: 2694 ret = 0; 2695 QLIST_FOREACH(child, &bs->children, next) { 2696 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) { 2697 int this_child_ret = bdrv_co_flush(child->bs); 2698 if (!ret) { 2699 ret = this_child_ret; 2700 } 2701 } 2702 } 2703 2704 out: 2705 /* Notify any pending flushes that we have completed */ 2706 if (ret == 0) { 2707 bs->flushed_gen = current_gen; 2708 } 2709 2710 qemu_co_mutex_lock(&bs->reqs_lock); 2711 bs->active_flush_req = false; 2712 /* Return value is ignored - it's ok if wait queue is empty */ 2713 qemu_co_queue_next(&bs->flush_queue); 2714 qemu_co_mutex_unlock(&bs->reqs_lock); 2715 2716 early_exit: 2717 bdrv_dec_in_flight(bs); 2718 return ret; 2719 } 2720 2721 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset, 2722 int64_t bytes) 2723 { 2724 BdrvTrackedRequest req; 2725 int max_pdiscard, ret; 2726 int head, tail, align; 2727 BlockDriverState *bs = child->bs; 2728 2729 if (!bs || !bs->drv || !bdrv_is_inserted(bs)) { 2730 return -ENOMEDIUM; 2731 } 2732 2733 if (bdrv_has_readonly_bitmaps(bs)) { 2734 return -EPERM; 2735 } 2736 2737 if (offset < 0 || bytes < 0 || bytes > INT64_MAX - offset) { 2738 return -EIO; 2739 } 2740 2741 /* Do nothing if disabled. */ 2742 if (!(bs->open_flags & BDRV_O_UNMAP)) { 2743 return 0; 2744 } 2745 2746 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) { 2747 return 0; 2748 } 2749 2750 /* Discard is advisory, but some devices track and coalesce 2751 * unaligned requests, so we must pass everything down rather than 2752 * round here. Still, most devices will just silently ignore 2753 * unaligned requests (by returning -ENOTSUP), so we must fragment 2754 * the request accordingly. */ 2755 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment); 2756 assert(align % bs->bl.request_alignment == 0); 2757 head = offset % align; 2758 tail = (offset + bytes) % align; 2759 2760 bdrv_inc_in_flight(bs); 2761 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD); 2762 2763 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0); 2764 if (ret < 0) { 2765 goto out; 2766 } 2767 2768 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX), 2769 align); 2770 assert(max_pdiscard >= bs->bl.request_alignment); 2771 2772 while (bytes > 0) { 2773 int64_t num = bytes; 2774 2775 if (head) { 2776 /* Make small requests to get to alignment boundaries. */ 2777 num = MIN(bytes, align - head); 2778 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) { 2779 num %= bs->bl.request_alignment; 2780 } 2781 head = (head + num) % align; 2782 assert(num < max_pdiscard); 2783 } else if (tail) { 2784 if (num > align) { 2785 /* Shorten the request to the last aligned cluster. */ 2786 num -= tail; 2787 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) && 2788 tail > bs->bl.request_alignment) { 2789 tail %= bs->bl.request_alignment; 2790 num -= tail; 2791 } 2792 } 2793 /* limit request size */ 2794 if (num > max_pdiscard) { 2795 num = max_pdiscard; 2796 } 2797 2798 if (!bs->drv) { 2799 ret = -ENOMEDIUM; 2800 goto out; 2801 } 2802 if (bs->drv->bdrv_co_pdiscard) { 2803 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num); 2804 } else { 2805 BlockAIOCB *acb; 2806 CoroutineIOCompletion co = { 2807 .coroutine = qemu_coroutine_self(), 2808 }; 2809 2810 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num, 2811 bdrv_co_io_em_complete, &co); 2812 if (acb == NULL) { 2813 ret = -EIO; 2814 goto out; 2815 } else { 2816 qemu_coroutine_yield(); 2817 ret = co.ret; 2818 } 2819 } 2820 if (ret && ret != -ENOTSUP) { 2821 goto out; 2822 } 2823 2824 offset += num; 2825 bytes -= num; 2826 } 2827 ret = 0; 2828 out: 2829 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret); 2830 tracked_request_end(&req); 2831 bdrv_dec_in_flight(bs); 2832 return ret; 2833 } 2834 2835 int bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf) 2836 { 2837 BlockDriver *drv = bs->drv; 2838 CoroutineIOCompletion co = { 2839 .coroutine = qemu_coroutine_self(), 2840 }; 2841 BlockAIOCB *acb; 2842 2843 bdrv_inc_in_flight(bs); 2844 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) { 2845 co.ret = -ENOTSUP; 2846 goto out; 2847 } 2848 2849 if (drv->bdrv_co_ioctl) { 2850 co.ret = drv->bdrv_co_ioctl(bs, req, buf); 2851 } else { 2852 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co); 2853 if (!acb) { 2854 co.ret = -ENOTSUP; 2855 goto out; 2856 } 2857 qemu_coroutine_yield(); 2858 } 2859 out: 2860 bdrv_dec_in_flight(bs); 2861 return co.ret; 2862 } 2863 2864 void *qemu_blockalign(BlockDriverState *bs, size_t size) 2865 { 2866 return qemu_memalign(bdrv_opt_mem_align(bs), size); 2867 } 2868 2869 void *qemu_blockalign0(BlockDriverState *bs, size_t size) 2870 { 2871 return memset(qemu_blockalign(bs, size), 0, size); 2872 } 2873 2874 void *qemu_try_blockalign(BlockDriverState *bs, size_t size) 2875 { 2876 size_t align = bdrv_opt_mem_align(bs); 2877 2878 /* Ensure that NULL is never returned on success */ 2879 assert(align > 0); 2880 if (size == 0) { 2881 size = align; 2882 } 2883 2884 return qemu_try_memalign(align, size); 2885 } 2886 2887 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size) 2888 { 2889 void *mem = qemu_try_blockalign(bs, size); 2890 2891 if (mem) { 2892 memset(mem, 0, size); 2893 } 2894 2895 return mem; 2896 } 2897 2898 /* 2899 * Check if all memory in this vector is sector aligned. 2900 */ 2901 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov) 2902 { 2903 int i; 2904 size_t alignment = bdrv_min_mem_align(bs); 2905 2906 for (i = 0; i < qiov->niov; i++) { 2907 if ((uintptr_t) qiov->iov[i].iov_base % alignment) { 2908 return false; 2909 } 2910 if (qiov->iov[i].iov_len % alignment) { 2911 return false; 2912 } 2913 } 2914 2915 return true; 2916 } 2917 2918 void bdrv_add_before_write_notifier(BlockDriverState *bs, 2919 NotifierWithReturn *notifier) 2920 { 2921 notifier_with_return_list_add(&bs->before_write_notifiers, notifier); 2922 } 2923 2924 void bdrv_io_plug(BlockDriverState *bs) 2925 { 2926 BdrvChild *child; 2927 2928 QLIST_FOREACH(child, &bs->children, next) { 2929 bdrv_io_plug(child->bs); 2930 } 2931 2932 if (qatomic_fetch_inc(&bs->io_plugged) == 0) { 2933 BlockDriver *drv = bs->drv; 2934 if (drv && drv->bdrv_io_plug) { 2935 drv->bdrv_io_plug(bs); 2936 } 2937 } 2938 } 2939 2940 void bdrv_io_unplug(BlockDriverState *bs) 2941 { 2942 BdrvChild *child; 2943 2944 assert(bs->io_plugged); 2945 if (qatomic_fetch_dec(&bs->io_plugged) == 1) { 2946 BlockDriver *drv = bs->drv; 2947 if (drv && drv->bdrv_io_unplug) { 2948 drv->bdrv_io_unplug(bs); 2949 } 2950 } 2951 2952 QLIST_FOREACH(child, &bs->children, next) { 2953 bdrv_io_unplug(child->bs); 2954 } 2955 } 2956 2957 void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size) 2958 { 2959 BdrvChild *child; 2960 2961 if (bs->drv && bs->drv->bdrv_register_buf) { 2962 bs->drv->bdrv_register_buf(bs, host, size); 2963 } 2964 QLIST_FOREACH(child, &bs->children, next) { 2965 bdrv_register_buf(child->bs, host, size); 2966 } 2967 } 2968 2969 void bdrv_unregister_buf(BlockDriverState *bs, void *host) 2970 { 2971 BdrvChild *child; 2972 2973 if (bs->drv && bs->drv->bdrv_unregister_buf) { 2974 bs->drv->bdrv_unregister_buf(bs, host); 2975 } 2976 QLIST_FOREACH(child, &bs->children, next) { 2977 bdrv_unregister_buf(child->bs, host); 2978 } 2979 } 2980 2981 static int coroutine_fn bdrv_co_copy_range_internal( 2982 BdrvChild *src, uint64_t src_offset, BdrvChild *dst, 2983 uint64_t dst_offset, uint64_t bytes, 2984 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags, 2985 bool recurse_src) 2986 { 2987 BdrvTrackedRequest req; 2988 int ret; 2989 2990 /* TODO We can support BDRV_REQ_NO_FALLBACK here */ 2991 assert(!(read_flags & BDRV_REQ_NO_FALLBACK)); 2992 assert(!(write_flags & BDRV_REQ_NO_FALLBACK)); 2993 2994 if (!dst || !dst->bs) { 2995 return -ENOMEDIUM; 2996 } 2997 ret = bdrv_check_byte_request(dst->bs, dst_offset, bytes); 2998 if (ret) { 2999 return ret; 3000 } 3001 if (write_flags & BDRV_REQ_ZERO_WRITE) { 3002 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags); 3003 } 3004 3005 if (!src || !src->bs) { 3006 return -ENOMEDIUM; 3007 } 3008 ret = bdrv_check_byte_request(src->bs, src_offset, bytes); 3009 if (ret) { 3010 return ret; 3011 } 3012 3013 if (!src->bs->drv->bdrv_co_copy_range_from 3014 || !dst->bs->drv->bdrv_co_copy_range_to 3015 || src->bs->encrypted || dst->bs->encrypted) { 3016 return -ENOTSUP; 3017 } 3018 3019 if (recurse_src) { 3020 bdrv_inc_in_flight(src->bs); 3021 tracked_request_begin(&req, src->bs, src_offset, bytes, 3022 BDRV_TRACKED_READ); 3023 3024 /* BDRV_REQ_SERIALISING is only for write operation */ 3025 assert(!(read_flags & BDRV_REQ_SERIALISING)); 3026 bdrv_wait_serialising_requests(&req); 3027 3028 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs, 3029 src, src_offset, 3030 dst, dst_offset, 3031 bytes, 3032 read_flags, write_flags); 3033 3034 tracked_request_end(&req); 3035 bdrv_dec_in_flight(src->bs); 3036 } else { 3037 bdrv_inc_in_flight(dst->bs); 3038 tracked_request_begin(&req, dst->bs, dst_offset, bytes, 3039 BDRV_TRACKED_WRITE); 3040 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req, 3041 write_flags); 3042 if (!ret) { 3043 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs, 3044 src, src_offset, 3045 dst, dst_offset, 3046 bytes, 3047 read_flags, write_flags); 3048 } 3049 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret); 3050 tracked_request_end(&req); 3051 bdrv_dec_in_flight(dst->bs); 3052 } 3053 3054 return ret; 3055 } 3056 3057 /* Copy range from @src to @dst. 3058 * 3059 * See the comment of bdrv_co_copy_range for the parameter and return value 3060 * semantics. */ 3061 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, uint64_t src_offset, 3062 BdrvChild *dst, uint64_t dst_offset, 3063 uint64_t bytes, 3064 BdrvRequestFlags read_flags, 3065 BdrvRequestFlags write_flags) 3066 { 3067 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes, 3068 read_flags, write_flags); 3069 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3070 bytes, read_flags, write_flags, true); 3071 } 3072 3073 /* Copy range from @src to @dst. 3074 * 3075 * See the comment of bdrv_co_copy_range for the parameter and return value 3076 * semantics. */ 3077 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, uint64_t src_offset, 3078 BdrvChild *dst, uint64_t dst_offset, 3079 uint64_t bytes, 3080 BdrvRequestFlags read_flags, 3081 BdrvRequestFlags write_flags) 3082 { 3083 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes, 3084 read_flags, write_flags); 3085 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset, 3086 bytes, read_flags, write_flags, false); 3087 } 3088 3089 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset, 3090 BdrvChild *dst, uint64_t dst_offset, 3091 uint64_t bytes, BdrvRequestFlags read_flags, 3092 BdrvRequestFlags write_flags) 3093 { 3094 return bdrv_co_copy_range_from(src, src_offset, 3095 dst, dst_offset, 3096 bytes, read_flags, write_flags); 3097 } 3098 3099 static void bdrv_parent_cb_resize(BlockDriverState *bs) 3100 { 3101 BdrvChild *c; 3102 QLIST_FOREACH(c, &bs->parents, next_parent) { 3103 if (c->klass->resize) { 3104 c->klass->resize(c); 3105 } 3106 } 3107 } 3108 3109 /** 3110 * Truncate file to 'offset' bytes (needed only for file protocols) 3111 * 3112 * If 'exact' is true, the file must be resized to exactly the given 3113 * 'offset'. Otherwise, it is sufficient for the node to be at least 3114 * 'offset' bytes in length. 3115 */ 3116 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact, 3117 PreallocMode prealloc, BdrvRequestFlags flags, 3118 Error **errp) 3119 { 3120 BlockDriverState *bs = child->bs; 3121 BdrvChild *filtered, *backing; 3122 BlockDriver *drv = bs->drv; 3123 BdrvTrackedRequest req; 3124 int64_t old_size, new_bytes; 3125 int ret; 3126 3127 3128 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */ 3129 if (!drv) { 3130 error_setg(errp, "No medium inserted"); 3131 return -ENOMEDIUM; 3132 } 3133 if (offset < 0) { 3134 error_setg(errp, "Image size cannot be negative"); 3135 return -EINVAL; 3136 } 3137 3138 old_size = bdrv_getlength(bs); 3139 if (old_size < 0) { 3140 error_setg_errno(errp, -old_size, "Failed to get old image size"); 3141 return old_size; 3142 } 3143 3144 if (offset > old_size) { 3145 new_bytes = offset - old_size; 3146 } else { 3147 new_bytes = 0; 3148 } 3149 3150 bdrv_inc_in_flight(bs); 3151 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes, 3152 BDRV_TRACKED_TRUNCATE); 3153 3154 /* If we are growing the image and potentially using preallocation for the 3155 * new area, we need to make sure that no write requests are made to it 3156 * concurrently or they might be overwritten by preallocation. */ 3157 if (new_bytes) { 3158 bdrv_mark_request_serialising(&req, 1); 3159 } 3160 if (bs->read_only) { 3161 error_setg(errp, "Image is read-only"); 3162 ret = -EACCES; 3163 goto out; 3164 } 3165 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req, 3166 0); 3167 if (ret < 0) { 3168 error_setg_errno(errp, -ret, 3169 "Failed to prepare request for truncation"); 3170 goto out; 3171 } 3172 3173 filtered = bdrv_filter_child(bs); 3174 backing = bdrv_cow_child(bs); 3175 3176 /* 3177 * If the image has a backing file that is large enough that it would 3178 * provide data for the new area, we cannot leave it unallocated because 3179 * then the backing file content would become visible. Instead, zero-fill 3180 * the new area. 3181 * 3182 * Note that if the image has a backing file, but was opened without the 3183 * backing file, taking care of keeping things consistent with that backing 3184 * file is the user's responsibility. 3185 */ 3186 if (new_bytes && backing) { 3187 int64_t backing_len; 3188 3189 backing_len = bdrv_getlength(backing->bs); 3190 if (backing_len < 0) { 3191 ret = backing_len; 3192 error_setg_errno(errp, -ret, "Could not get backing file size"); 3193 goto out; 3194 } 3195 3196 if (backing_len > old_size) { 3197 flags |= BDRV_REQ_ZERO_WRITE; 3198 } 3199 } 3200 3201 if (drv->bdrv_co_truncate) { 3202 if (flags & ~bs->supported_truncate_flags) { 3203 error_setg(errp, "Block driver does not support requested flags"); 3204 ret = -ENOTSUP; 3205 goto out; 3206 } 3207 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp); 3208 } else if (filtered) { 3209 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp); 3210 } else { 3211 error_setg(errp, "Image format driver does not support resize"); 3212 ret = -ENOTSUP; 3213 goto out; 3214 } 3215 if (ret < 0) { 3216 goto out; 3217 } 3218 3219 ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS); 3220 if (ret < 0) { 3221 error_setg_errno(errp, -ret, "Could not refresh total sector count"); 3222 } else { 3223 offset = bs->total_sectors * BDRV_SECTOR_SIZE; 3224 } 3225 /* It's possible that truncation succeeded but refresh_total_sectors 3226 * failed, but the latter doesn't affect how we should finish the request. 3227 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */ 3228 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0); 3229 3230 out: 3231 tracked_request_end(&req); 3232 bdrv_dec_in_flight(bs); 3233 3234 return ret; 3235 } 3236