io.c (c7bc05f78ab31fb02fc9635f60b9bd22efc8d121) io.c (299403aedaeb7f08d8e98aa8614b29d4e5546066)
1/*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights

--- 222 unchanged lines hidden (view full) ---

231 assert(old >= 1);
232}
233
234typedef struct {
235 Coroutine *co;
236 BlockDriverState *bs;
237 bool done;
238 bool begin;
1/*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights

--- 222 unchanged lines hidden (view full) ---

231 assert(old >= 1);
232}
233
234typedef struct {
235 Coroutine *co;
236 BlockDriverState *bs;
237 bool done;
238 bool begin;
239 bool recursive;
240 bool poll;
241 BdrvChild *parent;
242 bool ignore_bds_parents;
243} BdrvCoDrainData;
244
245/* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
239 bool poll;
240 BdrvChild *parent;
241 bool ignore_bds_parents;
242} BdrvCoDrainData;
243
244/* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
246bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
247 BdrvChild *ignore_parent, bool ignore_bds_parents)
245bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent,
246 bool ignore_bds_parents)
248{
247{
249 BdrvChild *child, *next;
250 IO_OR_GS_CODE();
251
252 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
253 return true;
254 }
255
256 if (qatomic_read(&bs->in_flight)) {
257 return true;
258 }
259
248 IO_OR_GS_CODE();
249
250 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
251 return true;
252 }
253
254 if (qatomic_read(&bs->in_flight)) {
255 return true;
256 }
257
260 if (recursive) {
261 assert(!ignore_bds_parents);
262 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
263 if (bdrv_drain_poll(child->bs, recursive, child, false)) {
264 return true;
265 }
266 }
267 }
268
269 return false;
270}
271
258 return false;
259}
260
272static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
261static bool bdrv_drain_poll_top_level(BlockDriverState *bs,
273 BdrvChild *ignore_parent)
274{
262 BdrvChild *ignore_parent)
263{
275 return bdrv_drain_poll(bs, recursive, ignore_parent, false);
264 return bdrv_drain_poll(bs, ignore_parent, false);
276}
277
265}
266
278static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
279 BdrvChild *parent, bool ignore_bds_parents,
280 bool poll);
281static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
282 BdrvChild *parent, bool ignore_bds_parents);
267static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
268 bool ignore_bds_parents, bool poll);
269static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent,
270 bool ignore_bds_parents);
283
284static void bdrv_co_drain_bh_cb(void *opaque)
285{
286 BdrvCoDrainData *data = opaque;
287 Coroutine *co = data->co;
288 BlockDriverState *bs = data->bs;
289
290 if (bs) {
291 AioContext *ctx = bdrv_get_aio_context(bs);
292 aio_context_acquire(ctx);
293 bdrv_dec_in_flight(bs);
294 if (data->begin) {
271
272static void bdrv_co_drain_bh_cb(void *opaque)
273{
274 BdrvCoDrainData *data = opaque;
275 Coroutine *co = data->co;
276 BlockDriverState *bs = data->bs;
277
278 if (bs) {
279 AioContext *ctx = bdrv_get_aio_context(bs);
280 aio_context_acquire(ctx);
281 bdrv_dec_in_flight(bs);
282 if (data->begin) {
295 bdrv_do_drained_begin(bs, data->recursive, data->parent,
296 data->ignore_bds_parents, data->poll);
283 bdrv_do_drained_begin(bs, data->parent, data->ignore_bds_parents,
284 data->poll);
297 } else {
298 assert(!data->poll);
285 } else {
286 assert(!data->poll);
299 bdrv_do_drained_end(bs, data->recursive, data->parent,
300 data->ignore_bds_parents);
287 bdrv_do_drained_end(bs, data->parent, data->ignore_bds_parents);
301 }
302 aio_context_release(ctx);
303 } else {
304 assert(data->begin);
305 bdrv_drain_all_begin();
306 }
307
308 data->done = true;
309 aio_co_wake(co);
310}
311
312static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
288 }
289 aio_context_release(ctx);
290 } else {
291 assert(data->begin);
292 bdrv_drain_all_begin();
293 }
294
295 data->done = true;
296 aio_co_wake(co);
297}
298
299static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
313 bool begin, bool recursive,
300 bool begin,
314 BdrvChild *parent,
315 bool ignore_bds_parents,
316 bool poll)
317{
318 BdrvCoDrainData data;
319 Coroutine *self = qemu_coroutine_self();
320 AioContext *ctx = bdrv_get_aio_context(bs);
321 AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
322
323 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
324 * other coroutines run if they were queued by aio_co_enter(). */
325
326 assert(qemu_in_coroutine());
327 data = (BdrvCoDrainData) {
328 .co = self,
329 .bs = bs,
330 .done = false,
331 .begin = begin,
301 BdrvChild *parent,
302 bool ignore_bds_parents,
303 bool poll)
304{
305 BdrvCoDrainData data;
306 Coroutine *self = qemu_coroutine_self();
307 AioContext *ctx = bdrv_get_aio_context(bs);
308 AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
309
310 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
311 * other coroutines run if they were queued by aio_co_enter(). */
312
313 assert(qemu_in_coroutine());
314 data = (BdrvCoDrainData) {
315 .co = self,
316 .bs = bs,
317 .done = false,
318 .begin = begin,
332 .recursive = recursive,
333 .parent = parent,
334 .ignore_bds_parents = ignore_bds_parents,
335 .poll = poll,
336 };
337
338 if (bs) {
339 bdrv_inc_in_flight(bs);
340 }

--- 34 unchanged lines hidden (view full) ---

375 }
376
377 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
378 if (bs->drv && bs->drv->bdrv_drain_begin) {
379 bs->drv->bdrv_drain_begin(bs);
380 }
381}
382
319 .parent = parent,
320 .ignore_bds_parents = ignore_bds_parents,
321 .poll = poll,
322 };
323
324 if (bs) {
325 bdrv_inc_in_flight(bs);
326 }

--- 34 unchanged lines hidden (view full) ---

361 }
362
363 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
364 if (bs->drv && bs->drv->bdrv_drain_begin) {
365 bs->drv->bdrv_drain_begin(bs);
366 }
367}
368
383static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
384 BdrvChild *parent, bool ignore_bds_parents,
385 bool poll)
369static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
370 bool ignore_bds_parents, bool poll)
386{
371{
387 BdrvChild *child, *next;
388
389 if (qemu_in_coroutine()) {
372 if (qemu_in_coroutine()) {
390 bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
391 poll);
373 bdrv_co_yield_to_drain(bs, true, parent, ignore_bds_parents, poll);
392 return;
393 }
394
395 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
396
374 return;
375 }
376
377 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
378
397 if (recursive) {
398 assert(!ignore_bds_parents);
399 bs->recursive_quiesce_counter++;
400 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
401 bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents,
402 false);
403 }
404 }
405
406 /*
407 * Wait for drained requests to finish.
408 *
409 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
410 * call is needed so things in this AioContext can make progress even
411 * though we don't return to the main AioContext loop - this automatically
412 * includes other nodes in the same AioContext and therefore all child
413 * nodes.
414 */
415 if (poll) {
416 assert(!ignore_bds_parents);
379 /*
380 * Wait for drained requests to finish.
381 *
382 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
383 * call is needed so things in this AioContext can make progress even
384 * though we don't return to the main AioContext loop - this automatically
385 * includes other nodes in the same AioContext and therefore all child
386 * nodes.
387 */
388 if (poll) {
389 assert(!ignore_bds_parents);
417 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent));
390 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent));
418 }
419}
420
421void bdrv_drained_begin(BlockDriverState *bs)
422{
423 IO_OR_GS_CODE();
391 }
392}
393
394void bdrv_drained_begin(BlockDriverState *bs)
395{
396 IO_OR_GS_CODE();
424 bdrv_do_drained_begin(bs, false, NULL, false, true);
397 bdrv_do_drained_begin(bs, NULL, false, true);
425}
426
398}
399
427void bdrv_subtree_drained_begin(BlockDriverState *bs)
428{
429 IO_OR_GS_CODE();
430 bdrv_do_drained_begin(bs, true, NULL, false, true);
431}
432
433/**
434 * This function does not poll, nor must any of its recursively called
435 * functions.
436 */
400/**
401 * This function does not poll, nor must any of its recursively called
402 * functions.
403 */
437static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
438 BdrvChild *parent, bool ignore_bds_parents)
404static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent,
405 bool ignore_bds_parents)
439{
406{
440 BdrvChild *child;
441 int old_quiesce_counter;
442
443 if (qemu_in_coroutine()) {
407 int old_quiesce_counter;
408
409 if (qemu_in_coroutine()) {
444 bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
445 false);
410 bdrv_co_yield_to_drain(bs, false, parent, ignore_bds_parents, false);
446 return;
447 }
448 assert(bs->quiesce_counter > 0);
449
450 /* Re-enable things in child-to-parent order */
451 if (bs->drv && bs->drv->bdrv_drain_end) {
452 bs->drv->bdrv_drain_end(bs);
453 }
454 bdrv_parent_drained_end(bs, parent, ignore_bds_parents);
455
456 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
457 if (old_quiesce_counter == 1) {
458 aio_enable_external(bdrv_get_aio_context(bs));
459 }
411 return;
412 }
413 assert(bs->quiesce_counter > 0);
414
415 /* Re-enable things in child-to-parent order */
416 if (bs->drv && bs->drv->bdrv_drain_end) {
417 bs->drv->bdrv_drain_end(bs);
418 }
419 bdrv_parent_drained_end(bs, parent, ignore_bds_parents);
420
421 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
422 if (old_quiesce_counter == 1) {
423 aio_enable_external(bdrv_get_aio_context(bs));
424 }
460
461 if (recursive) {
462 assert(!ignore_bds_parents);
463 bs->recursive_quiesce_counter--;
464 QLIST_FOREACH(child, &bs->children, next) {
465 bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents);
466 }
467 }
468}
469
470void bdrv_drained_end(BlockDriverState *bs)
471{
472 IO_OR_GS_CODE();
425}
426
427void bdrv_drained_end(BlockDriverState *bs)
428{
429 IO_OR_GS_CODE();
473 bdrv_do_drained_end(bs, false, NULL, false);
430 bdrv_do_drained_end(bs, NULL, false);
474}
475
431}
432
476void bdrv_subtree_drained_end(BlockDriverState *bs)
477{
478 IO_OR_GS_CODE();
479 bdrv_do_drained_end(bs, true, NULL, false);
480}
481
482void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
483{
484 int i;
485 IO_OR_GS_CODE();
486
487 for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
488 bdrv_do_drained_begin(child->bs, true, child, false, true);
489 }
490}
491
492void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
493{
494 int i;
495 IO_OR_GS_CODE();
496
497 for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
498 bdrv_do_drained_end(child->bs, true, child, false);
499 }
500}
501
502void bdrv_drain(BlockDriverState *bs)
503{
504 IO_OR_GS_CODE();
505 bdrv_drained_begin(bs);
506 bdrv_drained_end(bs);
507}
508
509static void bdrv_drain_assert_idle(BlockDriverState *bs)

--- 14 unchanged lines hidden (view full) ---

524 bool result = false;
525 GLOBAL_STATE_CODE();
526
527 /* bdrv_drain_poll() can't make changes to the graph and we are holding the
528 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
529 while ((bs = bdrv_next_all_states(bs))) {
530 AioContext *aio_context = bdrv_get_aio_context(bs);
531 aio_context_acquire(aio_context);
433void bdrv_drain(BlockDriverState *bs)
434{
435 IO_OR_GS_CODE();
436 bdrv_drained_begin(bs);
437 bdrv_drained_end(bs);
438}
439
440static void bdrv_drain_assert_idle(BlockDriverState *bs)

--- 14 unchanged lines hidden (view full) ---

455 bool result = false;
456 GLOBAL_STATE_CODE();
457
458 /* bdrv_drain_poll() can't make changes to the graph and we are holding the
459 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
460 while ((bs = bdrv_next_all_states(bs))) {
461 AioContext *aio_context = bdrv_get_aio_context(bs);
462 aio_context_acquire(aio_context);
532 result |= bdrv_drain_poll(bs, false, NULL, true);
463 result |= bdrv_drain_poll(bs, NULL, true);
533 aio_context_release(aio_context);
534 }
535
536 return result;
537}
538
539/*
540 * Wait for pending requests to complete across all BlockDriverStates

--- 8 unchanged lines hidden (view full) ---

549 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
550 */
551void bdrv_drain_all_begin(void)
552{
553 BlockDriverState *bs = NULL;
554 GLOBAL_STATE_CODE();
555
556 if (qemu_in_coroutine()) {
464 aio_context_release(aio_context);
465 }
466
467 return result;
468}
469
470/*
471 * Wait for pending requests to complete across all BlockDriverStates

--- 8 unchanged lines hidden (view full) ---

480 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
481 */
482void bdrv_drain_all_begin(void)
483{
484 BlockDriverState *bs = NULL;
485 GLOBAL_STATE_CODE();
486
487 if (qemu_in_coroutine()) {
557 bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true);
488 bdrv_co_yield_to_drain(NULL, true, NULL, true, true);
558 return;
559 }
560
561 /*
562 * bdrv queue is managed by record/replay,
563 * waiting for finishing the I/O requests may
564 * be infinite
565 */

--- 8 unchanged lines hidden (view full) ---

574 bdrv_drain_all_count++;
575
576 /* Quiesce all nodes, without polling in-flight requests yet. The graph
577 * cannot change during this loop. */
578 while ((bs = bdrv_next_all_states(bs))) {
579 AioContext *aio_context = bdrv_get_aio_context(bs);
580
581 aio_context_acquire(aio_context);
489 return;
490 }
491
492 /*
493 * bdrv queue is managed by record/replay,
494 * waiting for finishing the I/O requests may
495 * be infinite
496 */

--- 8 unchanged lines hidden (view full) ---

505 bdrv_drain_all_count++;
506
507 /* Quiesce all nodes, without polling in-flight requests yet. The graph
508 * cannot change during this loop. */
509 while ((bs = bdrv_next_all_states(bs))) {
510 AioContext *aio_context = bdrv_get_aio_context(bs);
511
512 aio_context_acquire(aio_context);
582 bdrv_do_drained_begin(bs, false, NULL, true, false);
513 bdrv_do_drained_begin(bs, NULL, true, false);
583 aio_context_release(aio_context);
584 }
585
586 /* Now poll the in-flight requests */
587 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
588
589 while ((bs = bdrv_next_all_states(bs))) {
590 bdrv_drain_assert_idle(bs);
591 }
592}
593
594void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
595{
596 GLOBAL_STATE_CODE();
597
598 g_assert(bs->quiesce_counter > 0);
599 g_assert(!bs->refcnt);
600
601 while (bs->quiesce_counter) {
514 aio_context_release(aio_context);
515 }
516
517 /* Now poll the in-flight requests */
518 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
519
520 while ((bs = bdrv_next_all_states(bs))) {
521 bdrv_drain_assert_idle(bs);
522 }
523}
524
525void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
526{
527 GLOBAL_STATE_CODE();
528
529 g_assert(bs->quiesce_counter > 0);
530 g_assert(!bs->refcnt);
531
532 while (bs->quiesce_counter) {
602 bdrv_do_drained_end(bs, false, NULL, true);
533 bdrv_do_drained_end(bs, NULL, true);
603 }
604}
605
606void bdrv_drain_all_end(void)
607{
608 BlockDriverState *bs = NULL;
609 GLOBAL_STATE_CODE();
610

--- 5 unchanged lines hidden (view full) ---

616 if (replay_events_enabled()) {
617 return;
618 }
619
620 while ((bs = bdrv_next_all_states(bs))) {
621 AioContext *aio_context = bdrv_get_aio_context(bs);
622
623 aio_context_acquire(aio_context);
534 }
535}
536
537void bdrv_drain_all_end(void)
538{
539 BlockDriverState *bs = NULL;
540 GLOBAL_STATE_CODE();
541

--- 5 unchanged lines hidden (view full) ---

547 if (replay_events_enabled()) {
548 return;
549 }
550
551 while ((bs = bdrv_next_all_states(bs))) {
552 AioContext *aio_context = bdrv_get_aio_context(bs);
553
554 aio_context_acquire(aio_context);
624 bdrv_do_drained_end(bs, false, NULL, true);
555 bdrv_do_drained_end(bs, NULL, true);
625 aio_context_release(aio_context);
626 }
627
628 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
629 assert(bdrv_drain_all_count > 0);
630 bdrv_drain_all_count--;
631}
632

--- 2956 unchanged lines hidden --- 		556 aio_context_release(aio_context);
557 }
558
559 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
560 assert(bdrv_drain_all_count > 0);
561 bdrv_drain_all_count--;
562}
563

--- 2956 unchanged lines hidden ---