1 /* 2 * Block dirty bitmap postcopy migration 3 * 4 * Copyright IBM, Corp. 2009 5 * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved. 6 * 7 * Authors: 8 * Liran Schour <lirans@il.ibm.com> 9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2. See 12 * the COPYING file in the top-level directory. 13 * This file is derived from migration/block.c, so it's author and IBM copyright 14 * are here, although content is quite different. 15 * 16 * Contributions after 2012-01-13 are licensed under the terms of the 17 * GNU GPL, version 2 or (at your option) any later version. 18 * 19 * *** 20 * 21 * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable 22 * bitmaps are migrated. 23 * 24 * Bitmap migration implies creating bitmap with the same name and granularity 25 * in destination QEMU. If the bitmap with the same name (for the same node) 26 * already exists on destination an error will be generated. 27 * 28 * format of migration: 29 * 30 * # Header (shared for different chunk types) 31 * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags) 32 * [ 1 byte: node alias size ] \ flags & DEVICE_NAME 33 * [ n bytes: node alias ] / 34 * [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME 35 * [ n bytes: bitmap alias ] / 36 * 37 * # Start of bitmap migration (flags & START) 38 * header 39 * be64: granularity 40 * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap) 41 * bit 0 - bitmap is enabled 42 * bit 1 - bitmap is persistent 43 * bit 2 - bitmap is autoloading 44 * bits 3-7 - reserved, must be zero 45 * 46 * # Complete of bitmap migration (flags & COMPLETE) 47 * header 48 * 49 * # Data chunk of bitmap migration 50 * header 51 * be64: start sector 52 * be32: number of sectors 53 * [ be64: buffer size ] \ ! (flags & ZEROES) 54 * [ n bytes: buffer ] / 55 * 56 * The last chunk in stream should contain flags & EOS. The chunk may skip 57 * device and/or bitmap names, assuming them to be the same with the previous 58 * chunk. 59 */ 60 61 #include "qemu/osdep.h" 62 #include "block/block.h" 63 #include "block/block_int.h" 64 #include "sysemu/block-backend.h" 65 #include "sysemu/runstate.h" 66 #include "qemu/main-loop.h" 67 #include "qemu/error-report.h" 68 #include "migration/misc.h" 69 #include "migration/migration.h" 70 #include "qemu-file.h" 71 #include "migration/vmstate.h" 72 #include "migration/register.h" 73 #include "qemu/hbitmap.h" 74 #include "qemu/cutils.h" 75 #include "qemu/id.h" 76 #include "qapi/error.h" 77 #include "qapi/qapi-commands-migration.h" 78 #include "qapi/qapi-visit-migration.h" 79 #include "qapi/clone-visitor.h" 80 #include "trace.h" 81 82 #define CHUNK_SIZE (1 << 10) 83 84 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as 85 * follows: 86 * in first (most significant) byte bit 8 is clear --> one byte 87 * in first byte bit 8 is set --> two or four bytes, depending on second 88 * byte: 89 * | in second byte bit 8 is clear --> two bytes 90 * | in second byte bit 8 is set --> four bytes 91 */ 92 #define DIRTY_BITMAP_MIG_FLAG_EOS 0x01 93 #define DIRTY_BITMAP_MIG_FLAG_ZEROES 0x02 94 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME 0x04 95 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME 0x08 96 #define DIRTY_BITMAP_MIG_FLAG_START 0x10 97 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE 0x20 98 #define DIRTY_BITMAP_MIG_FLAG_BITS 0x40 99 100 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS 0x80 101 102 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED 0x01 103 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT 0x02 104 /* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */ 105 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK 0xf8 106 107 /* State of one bitmap during save process */ 108 typedef struct SaveBitmapState { 109 /* Written during setup phase. */ 110 BlockDriverState *bs; 111 char *node_alias; 112 char *bitmap_alias; 113 BdrvDirtyBitmap *bitmap; 114 uint64_t total_sectors; 115 uint64_t sectors_per_chunk; 116 QSIMPLEQ_ENTRY(SaveBitmapState) entry; 117 uint8_t flags; 118 119 /* For bulk phase. */ 120 bool bulk_completed; 121 uint64_t cur_sector; 122 } SaveBitmapState; 123 124 /* State of the dirty bitmap migration (DBM) during save process */ 125 typedef struct DBMSaveState { 126 QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list; 127 128 bool bulk_completed; 129 bool no_bitmaps; 130 131 /* for send_bitmap_bits() */ 132 BlockDriverState *prev_bs; 133 BdrvDirtyBitmap *prev_bitmap; 134 } DBMSaveState; 135 136 typedef struct LoadBitmapState { 137 BlockDriverState *bs; 138 BdrvDirtyBitmap *bitmap; 139 bool migrated; 140 bool enabled; 141 } LoadBitmapState; 142 143 /* State of the dirty bitmap migration (DBM) during load process */ 144 typedef struct DBMLoadState { 145 uint32_t flags; 146 char node_alias[256]; 147 char bitmap_alias[256]; 148 char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1]; 149 BlockDriverState *bs; 150 BdrvDirtyBitmap *bitmap; 151 152 bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */ 153 BitmapMigrationBitmapAlias *bmap_inner; 154 155 /* 156 * cancelled 157 * Incoming migration is cancelled for some reason. That means that we 158 * still should read our chunks from migration stream, to not affect other 159 * migration objects (like RAM), but just ignore them and do not touch any 160 * bitmaps or nodes. 161 */ 162 bool cancelled; 163 164 GSList *bitmaps; 165 QemuMutex lock; /* protect bitmaps */ 166 } DBMLoadState; 167 168 typedef struct DBMState { 169 DBMSaveState save; 170 DBMLoadState load; 171 } DBMState; 172 173 static DBMState dbm_state; 174 175 /* For hash tables that map node/bitmap names to aliases */ 176 typedef struct AliasMapInnerNode { 177 char *string; 178 GHashTable *subtree; 179 } AliasMapInnerNode; 180 181 static void free_alias_map_inner_node(void *amin_ptr) 182 { 183 AliasMapInnerNode *amin = amin_ptr; 184 185 g_free(amin->string); 186 g_hash_table_unref(amin->subtree); 187 g_free(amin); 188 } 189 190 /** 191 * Construct an alias map based on the given QMP structure. 192 * 193 * (Note that we cannot store such maps in the MigrationParameters 194 * object, because that struct is defined by the QAPI schema, which 195 * makes it basically impossible to have dicts with arbitrary keys. 196 * Therefore, we instead have to construct these maps when migration 197 * starts.) 198 * 199 * @bbm is the block_bitmap_mapping from the migration parameters. 200 * 201 * If @name_to_alias is true, the returned hash table will map node 202 * and bitmap names to their respective aliases (for outgoing 203 * migration). 204 * 205 * If @name_to_alias is false, the returned hash table will map node 206 * and bitmap aliases to their respective names (for incoming 207 * migration). 208 * 209 * The hash table maps node names/aliases to AliasMapInnerNode 210 * objects, whose .string is the respective node alias/name, and whose 211 * .subtree table maps bitmap names/aliases to the respective bitmap 212 * alias/name. 213 */ 214 static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm, 215 bool name_to_alias, 216 Error **errp) 217 { 218 GHashTable *alias_map; 219 size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1; 220 221 alias_map = g_hash_table_new_full(g_str_hash, g_str_equal, 222 g_free, free_alias_map_inner_node); 223 224 for (; bbm; bbm = bbm->next) { 225 const BitmapMigrationNodeAlias *bmna = bbm->value; 226 const BitmapMigrationBitmapAliasList *bmbal; 227 AliasMapInnerNode *amin; 228 GHashTable *bitmaps_map; 229 const char *node_map_from, *node_map_to; 230 GDestroyNotify gdn; 231 232 if (!id_wellformed(bmna->alias)) { 233 error_setg(errp, "The node alias '%s' is not well-formed", 234 bmna->alias); 235 goto fail; 236 } 237 238 if (strlen(bmna->alias) > UINT8_MAX) { 239 error_setg(errp, "The node alias '%s' is longer than %u bytes", 240 bmna->alias, UINT8_MAX); 241 goto fail; 242 } 243 244 if (strlen(bmna->node_name) > max_node_name_len) { 245 error_setg(errp, "The node name '%s' is longer than %zu bytes", 246 bmna->node_name, max_node_name_len); 247 goto fail; 248 } 249 250 if (name_to_alias) { 251 if (g_hash_table_contains(alias_map, bmna->node_name)) { 252 error_setg(errp, "The node name '%s' is mapped twice", 253 bmna->node_name); 254 goto fail; 255 } 256 257 node_map_from = bmna->node_name; 258 node_map_to = bmna->alias; 259 } else { 260 if (g_hash_table_contains(alias_map, bmna->alias)) { 261 error_setg(errp, "The node alias '%s' is used twice", 262 bmna->alias); 263 goto fail; 264 } 265 266 node_map_from = bmna->alias; 267 node_map_to = bmna->node_name; 268 } 269 270 gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias; 271 bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, 272 gdn); 273 274 amin = g_new(AliasMapInnerNode, 1); 275 *amin = (AliasMapInnerNode){ 276 .string = g_strdup(node_map_to), 277 .subtree = bitmaps_map, 278 }; 279 280 g_hash_table_insert(alias_map, g_strdup(node_map_from), amin); 281 282 for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) { 283 const BitmapMigrationBitmapAlias *bmba = bmbal->value; 284 const char *bmap_map_from; 285 286 if (strlen(bmba->alias) > UINT8_MAX) { 287 error_setg(errp, 288 "The bitmap alias '%s' is longer than %u bytes", 289 bmba->alias, UINT8_MAX); 290 goto fail; 291 } 292 293 if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) { 294 error_setg(errp, "The bitmap name '%s' is longer than %d bytes", 295 bmba->name, BDRV_BITMAP_MAX_NAME_SIZE); 296 goto fail; 297 } 298 299 if (name_to_alias) { 300 bmap_map_from = bmba->name; 301 302 if (g_hash_table_contains(bitmaps_map, bmba->name)) { 303 error_setg(errp, "The bitmap '%s'/'%s' is mapped twice", 304 bmna->node_name, bmba->name); 305 goto fail; 306 } 307 } else { 308 bmap_map_from = bmba->alias; 309 310 if (g_hash_table_contains(bitmaps_map, bmba->alias)) { 311 error_setg(errp, "The bitmap alias '%s'/'%s' is used twice", 312 bmna->alias, bmba->alias); 313 goto fail; 314 } 315 } 316 317 g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from), 318 QAPI_CLONE(BitmapMigrationBitmapAlias, bmba)); 319 } 320 } 321 322 return alias_map; 323 324 fail: 325 g_hash_table_destroy(alias_map); 326 return NULL; 327 } 328 329 /** 330 * Run construct_alias_map() in both directions to check whether @bbm 331 * is valid. 332 * (This function is to be used by migration/migration.c to validate 333 * the user-specified block-bitmap-mapping migration parameter.) 334 * 335 * Returns true if and only if the mapping is valid. 336 */ 337 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm, 338 Error **errp) 339 { 340 GHashTable *alias_map; 341 342 alias_map = construct_alias_map(bbm, true, errp); 343 if (!alias_map) { 344 return false; 345 } 346 g_hash_table_destroy(alias_map); 347 348 alias_map = construct_alias_map(bbm, false, errp); 349 if (!alias_map) { 350 return false; 351 } 352 g_hash_table_destroy(alias_map); 353 354 return true; 355 } 356 357 static uint32_t qemu_get_bitmap_flags(QEMUFile *f) 358 { 359 uint8_t flags = qemu_get_byte(f); 360 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) { 361 flags = flags << 8 | qemu_get_byte(f); 362 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) { 363 flags = flags << 16 | qemu_get_be16(f); 364 } 365 } 366 367 return flags; 368 } 369 370 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags) 371 { 372 /* The code currently does not send flags as more than one byte */ 373 assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS))); 374 375 qemu_put_byte(f, flags); 376 } 377 378 static void send_bitmap_header(QEMUFile *f, DBMSaveState *s, 379 SaveBitmapState *dbms, uint32_t additional_flags) 380 { 381 BlockDriverState *bs = dbms->bs; 382 BdrvDirtyBitmap *bitmap = dbms->bitmap; 383 uint32_t flags = additional_flags; 384 trace_send_bitmap_header_enter(); 385 386 if (bs != s->prev_bs) { 387 s->prev_bs = bs; 388 flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME; 389 } 390 391 if (bitmap != s->prev_bitmap) { 392 s->prev_bitmap = bitmap; 393 flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME; 394 } 395 396 qemu_put_bitmap_flags(f, flags); 397 398 if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { 399 qemu_put_counted_string(f, dbms->node_alias); 400 } 401 402 if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { 403 qemu_put_counted_string(f, dbms->bitmap_alias); 404 } 405 } 406 407 static void send_bitmap_start(QEMUFile *f, DBMSaveState *s, 408 SaveBitmapState *dbms) 409 { 410 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START); 411 qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap)); 412 qemu_put_byte(f, dbms->flags); 413 } 414 415 static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s, 416 SaveBitmapState *dbms) 417 { 418 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE); 419 } 420 421 static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s, 422 SaveBitmapState *dbms, 423 uint64_t start_sector, uint32_t nr_sectors) 424 { 425 /* align for buffer_is_zero() */ 426 uint64_t align = 4 * sizeof(long); 427 uint64_t unaligned_size = 428 bdrv_dirty_bitmap_serialization_size( 429 dbms->bitmap, start_sector << BDRV_SECTOR_BITS, 430 (uint64_t)nr_sectors << BDRV_SECTOR_BITS); 431 uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align); 432 uint8_t *buf = g_malloc0(buf_size); 433 uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS; 434 435 bdrv_dirty_bitmap_serialize_part( 436 dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS, 437 (uint64_t)nr_sectors << BDRV_SECTOR_BITS); 438 439 if (buffer_is_zero(buf, buf_size)) { 440 g_free(buf); 441 buf = NULL; 442 flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES; 443 } 444 445 trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size); 446 447 send_bitmap_header(f, s, dbms, flags); 448 449 qemu_put_be64(f, start_sector); 450 qemu_put_be32(f, nr_sectors); 451 452 /* if a block is zero we need to flush here since the network 453 * bandwidth is now a lot higher than the storage device bandwidth. 454 * thus if we queue zero blocks we slow down the migration. */ 455 if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) { 456 qemu_fflush(f); 457 } else { 458 qemu_put_be64(f, buf_size); 459 qemu_put_buffer(f, buf, buf_size); 460 } 461 462 g_free(buf); 463 } 464 465 /* Called with iothread lock taken. */ 466 static void dirty_bitmap_do_save_cleanup(DBMSaveState *s) 467 { 468 SaveBitmapState *dbms; 469 470 while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) { 471 QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry); 472 bdrv_dirty_bitmap_set_busy(dbms->bitmap, false); 473 bdrv_unref(dbms->bs); 474 g_free(dbms->node_alias); 475 g_free(dbms->bitmap_alias); 476 g_free(dbms); 477 } 478 } 479 480 /* Called with iothread lock taken. */ 481 static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs, 482 const char *bs_name, GHashTable *alias_map) 483 { 484 BdrvDirtyBitmap *bitmap; 485 SaveBitmapState *dbms; 486 GHashTable *bitmap_aliases; 487 const char *node_alias, *bitmap_name, *bitmap_alias; 488 Error *local_err = NULL; 489 490 /* When an alias map is given, @bs_name must be @bs's node name */ 491 assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs))); 492 493 FOR_EACH_DIRTY_BITMAP(bs, bitmap) { 494 if (bdrv_dirty_bitmap_name(bitmap)) { 495 break; 496 } 497 } 498 if (!bitmap) { 499 return 0; 500 } 501 502 bitmap_name = bdrv_dirty_bitmap_name(bitmap); 503 504 if (!bs_name || strcmp(bs_name, "") == 0) { 505 error_report("Bitmap '%s' in unnamed node can't be migrated", 506 bitmap_name); 507 return -1; 508 } 509 510 if (alias_map) { 511 const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name); 512 513 if (!amin) { 514 /* Skip bitmaps on nodes with no alias */ 515 return 0; 516 } 517 518 node_alias = amin->string; 519 bitmap_aliases = amin->subtree; 520 } else { 521 node_alias = bs_name; 522 bitmap_aliases = NULL; 523 } 524 525 if (node_alias[0] == '#') { 526 error_report("Bitmap '%s' in a node with auto-generated " 527 "name '%s' can't be migrated", 528 bitmap_name, node_alias); 529 return -1; 530 } 531 532 FOR_EACH_DIRTY_BITMAP(bs, bitmap) { 533 BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL; 534 bitmap_name = bdrv_dirty_bitmap_name(bitmap); 535 if (!bitmap_name) { 536 continue; 537 } 538 539 if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) { 540 error_report_err(local_err); 541 return -1; 542 } 543 544 if (bitmap_aliases) { 545 BitmapMigrationBitmapAlias *bmap_inner; 546 547 bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name); 548 if (!bmap_inner) { 549 /* Skip bitmaps with no alias */ 550 continue; 551 } 552 553 bitmap_alias = bmap_inner->alias; 554 if (bmap_inner->has_transform) { 555 bitmap_transform = bmap_inner->transform; 556 } 557 } else { 558 if (strlen(bitmap_name) > UINT8_MAX) { 559 error_report("Cannot migrate bitmap '%s' on node '%s': " 560 "Name is longer than %u bytes", 561 bitmap_name, bs_name, UINT8_MAX); 562 return -1; 563 } 564 bitmap_alias = bitmap_name; 565 } 566 567 bdrv_ref(bs); 568 bdrv_dirty_bitmap_set_busy(bitmap, true); 569 570 dbms = g_new0(SaveBitmapState, 1); 571 dbms->bs = bs; 572 dbms->node_alias = g_strdup(node_alias); 573 dbms->bitmap_alias = g_strdup(bitmap_alias); 574 dbms->bitmap = bitmap; 575 dbms->total_sectors = bdrv_nb_sectors(bs); 576 dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU * 577 (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS); 578 assert(dbms->sectors_per_chunk != 0); 579 if (bdrv_dirty_bitmap_enabled(bitmap)) { 580 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED; 581 } 582 if (bitmap_transform && 583 bitmap_transform->has_persistent) { 584 if (bitmap_transform->persistent) { 585 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT; 586 } 587 } else { 588 if (bdrv_dirty_bitmap_get_persistence(bitmap)) { 589 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT; 590 } 591 } 592 593 QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry); 594 } 595 596 return 0; 597 } 598 599 /* Called with iothread lock taken. */ 600 static int init_dirty_bitmap_migration(DBMSaveState *s) 601 { 602 BlockDriverState *bs; 603 SaveBitmapState *dbms; 604 GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL); 605 BlockBackend *blk; 606 const MigrationParameters *mig_params = &migrate_get_current()->parameters; 607 GHashTable *alias_map = NULL; 608 609 if (mig_params->has_block_bitmap_mapping) { 610 alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true, 611 &error_abort); 612 } 613 614 s->bulk_completed = false; 615 s->prev_bs = NULL; 616 s->prev_bitmap = NULL; 617 s->no_bitmaps = false; 618 619 if (!alias_map) { 620 /* 621 * Use blockdevice name for direct (or filtered) children of named block 622 * backends. 623 */ 624 for (blk = blk_next(NULL); blk; blk = blk_next(blk)) { 625 const char *name = blk_name(blk); 626 627 if (!name || strcmp(name, "") == 0) { 628 continue; 629 } 630 631 bs = blk_bs(blk); 632 633 /* Skip filters without bitmaps */ 634 while (bs && bs->drv && bs->drv->is_filter && 635 !bdrv_has_named_bitmaps(bs)) 636 { 637 bs = bdrv_filter_bs(bs); 638 } 639 640 if (bs && bs->drv && !bs->drv->is_filter) { 641 if (add_bitmaps_to_list(s, bs, name, NULL)) { 642 goto fail; 643 } 644 g_hash_table_add(handled_by_blk, bs); 645 } 646 } 647 } 648 649 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) { 650 if (g_hash_table_contains(handled_by_blk, bs)) { 651 continue; 652 } 653 654 if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) { 655 goto fail; 656 } 657 } 658 659 /* unset migration flags here, to not roll back it */ 660 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 661 bdrv_dirty_bitmap_skip_store(dbms->bitmap, true); 662 } 663 664 if (QSIMPLEQ_EMPTY(&s->dbms_list)) { 665 s->no_bitmaps = true; 666 } 667 668 g_hash_table_destroy(handled_by_blk); 669 if (alias_map) { 670 g_hash_table_destroy(alias_map); 671 } 672 673 return 0; 674 675 fail: 676 g_hash_table_destroy(handled_by_blk); 677 if (alias_map) { 678 g_hash_table_destroy(alias_map); 679 } 680 dirty_bitmap_do_save_cleanup(s); 681 682 return -1; 683 } 684 685 /* Called with no lock taken. */ 686 static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s, 687 SaveBitmapState *dbms) 688 { 689 uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector, 690 dbms->sectors_per_chunk); 691 692 send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors); 693 694 dbms->cur_sector += nr_sectors; 695 if (dbms->cur_sector >= dbms->total_sectors) { 696 dbms->bulk_completed = true; 697 } 698 } 699 700 /* Called with no lock taken. */ 701 static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit) 702 { 703 SaveBitmapState *dbms; 704 705 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 706 while (!dbms->bulk_completed) { 707 bulk_phase_send_chunk(f, s, dbms); 708 if (limit && qemu_file_rate_limit(f)) { 709 return; 710 } 711 } 712 } 713 714 s->bulk_completed = true; 715 } 716 717 /* for SaveVMHandlers */ 718 static void dirty_bitmap_save_cleanup(void *opaque) 719 { 720 DBMSaveState *s = &((DBMState *)opaque)->save; 721 722 dirty_bitmap_do_save_cleanup(s); 723 } 724 725 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque) 726 { 727 DBMSaveState *s = &((DBMState *)opaque)->save; 728 729 trace_dirty_bitmap_save_iterate(migration_in_postcopy()); 730 731 if (migration_in_postcopy() && !s->bulk_completed) { 732 bulk_phase(f, s, true); 733 } 734 735 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); 736 737 return s->bulk_completed; 738 } 739 740 /* Called with iothread lock taken. */ 741 742 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque) 743 { 744 DBMSaveState *s = &((DBMState *)opaque)->save; 745 SaveBitmapState *dbms; 746 trace_dirty_bitmap_save_complete_enter(); 747 748 if (!s->bulk_completed) { 749 bulk_phase(f, s, false); 750 } 751 752 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 753 send_bitmap_complete(f, s, dbms); 754 } 755 756 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); 757 758 trace_dirty_bitmap_save_complete_finish(); 759 760 dirty_bitmap_save_cleanup(opaque); 761 return 0; 762 } 763 764 static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque, 765 uint64_t max_size, 766 uint64_t *res_precopy_only, 767 uint64_t *res_compatible, 768 uint64_t *res_postcopy_only) 769 { 770 DBMSaveState *s = &((DBMState *)opaque)->save; 771 SaveBitmapState *dbms; 772 uint64_t pending = 0; 773 774 qemu_mutex_lock_iothread(); 775 776 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 777 uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap); 778 uint64_t sectors = dbms->bulk_completed ? 0 : 779 dbms->total_sectors - dbms->cur_sector; 780 781 pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran); 782 } 783 784 qemu_mutex_unlock_iothread(); 785 786 trace_dirty_bitmap_save_pending(pending, max_size); 787 788 *res_postcopy_only += pending; 789 } 790 791 /* First occurrence of this bitmap. It should be created if doesn't exist */ 792 static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s) 793 { 794 Error *local_err = NULL; 795 uint32_t granularity = qemu_get_be32(f); 796 uint8_t flags = qemu_get_byte(f); 797 LoadBitmapState *b; 798 bool persistent; 799 800 if (s->cancelled) { 801 return 0; 802 } 803 804 if (s->bitmap) { 805 error_report("Bitmap with the same name ('%s') already exists on " 806 "destination", bdrv_dirty_bitmap_name(s->bitmap)); 807 return -EINVAL; 808 } else { 809 s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity, 810 s->bitmap_name, &local_err); 811 if (!s->bitmap) { 812 error_report_err(local_err); 813 return -EINVAL; 814 } 815 } 816 817 if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) { 818 error_report("Unknown flags in migrated dirty bitmap header: %x", 819 flags); 820 return -EINVAL; 821 } 822 823 if (s->bmap_inner && 824 s->bmap_inner->has_transform && 825 s->bmap_inner->transform->has_persistent) { 826 persistent = s->bmap_inner->transform->persistent; 827 } else { 828 persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT; 829 } 830 831 if (persistent) { 832 bdrv_dirty_bitmap_set_persistence(s->bitmap, true); 833 } 834 835 bdrv_disable_dirty_bitmap(s->bitmap); 836 if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) { 837 bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err); 838 if (local_err) { 839 error_report_err(local_err); 840 return -EINVAL; 841 } 842 } 843 844 b = g_new(LoadBitmapState, 1); 845 b->bs = s->bs; 846 b->bitmap = s->bitmap; 847 b->migrated = false; 848 b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED; 849 850 s->bitmaps = g_slist_prepend(s->bitmaps, b); 851 852 return 0; 853 } 854 855 /* 856 * before_vm_start_handle_item 857 * 858 * g_slist_foreach helper 859 * 860 * item is LoadBitmapState* 861 * opaque is DBMLoadState* 862 */ 863 static void before_vm_start_handle_item(void *item, void *opaque) 864 { 865 DBMLoadState *s = opaque; 866 LoadBitmapState *b = item; 867 868 if (b->enabled) { 869 if (b->migrated) { 870 bdrv_enable_dirty_bitmap(b->bitmap); 871 } else { 872 bdrv_dirty_bitmap_enable_successor(b->bitmap); 873 } 874 } 875 876 if (b->migrated) { 877 s->bitmaps = g_slist_remove(s->bitmaps, b); 878 g_free(b); 879 } 880 } 881 882 void dirty_bitmap_mig_before_vm_start(void) 883 { 884 DBMLoadState *s = &dbm_state.load; 885 qemu_mutex_lock(&s->lock); 886 887 assert(!s->before_vm_start_handled); 888 g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s); 889 s->before_vm_start_handled = true; 890 891 qemu_mutex_unlock(&s->lock); 892 } 893 894 static void cancel_incoming_locked(DBMLoadState *s) 895 { 896 GSList *item; 897 898 if (s->cancelled) { 899 return; 900 } 901 902 s->cancelled = true; 903 s->bs = NULL; 904 s->bitmap = NULL; 905 906 /* Drop all unfinished bitmaps */ 907 for (item = s->bitmaps; item; item = g_slist_next(item)) { 908 LoadBitmapState *b = item->data; 909 910 /* 911 * Bitmap must be unfinished, as finished bitmaps should already be 912 * removed from the list. 913 */ 914 assert(!s->before_vm_start_handled || !b->migrated); 915 if (bdrv_dirty_bitmap_has_successor(b->bitmap)) { 916 bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort); 917 } 918 bdrv_release_dirty_bitmap(b->bitmap); 919 } 920 921 g_slist_free_full(s->bitmaps, g_free); 922 s->bitmaps = NULL; 923 } 924 925 void dirty_bitmap_mig_cancel_outgoing(void) 926 { 927 dirty_bitmap_do_save_cleanup(&dbm_state.save); 928 } 929 930 void dirty_bitmap_mig_cancel_incoming(void) 931 { 932 DBMLoadState *s = &dbm_state.load; 933 934 qemu_mutex_lock(&s->lock); 935 936 cancel_incoming_locked(s); 937 938 qemu_mutex_unlock(&s->lock); 939 } 940 941 static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s) 942 { 943 GSList *item; 944 trace_dirty_bitmap_load_complete(); 945 946 if (s->cancelled) { 947 return; 948 } 949 950 bdrv_dirty_bitmap_deserialize_finish(s->bitmap); 951 952 if (bdrv_dirty_bitmap_has_successor(s->bitmap)) { 953 bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort); 954 } 955 956 for (item = s->bitmaps; item; item = g_slist_next(item)) { 957 LoadBitmapState *b = item->data; 958 959 if (b->bitmap == s->bitmap) { 960 b->migrated = true; 961 if (s->before_vm_start_handled) { 962 s->bitmaps = g_slist_remove(s->bitmaps, b); 963 g_free(b); 964 } 965 break; 966 } 967 } 968 } 969 970 static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s) 971 { 972 uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS; 973 uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS; 974 trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS, 975 nr_bytes >> BDRV_SECTOR_BITS); 976 977 if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) { 978 trace_dirty_bitmap_load_bits_zeroes(); 979 if (!s->cancelled) { 980 bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte, 981 nr_bytes, false); 982 } 983 } else { 984 size_t ret; 985 g_autofree uint8_t *buf = NULL; 986 uint64_t buf_size = qemu_get_be64(f); 987 uint64_t needed_size; 988 989 /* 990 * The actual check for buf_size is done a bit later. We can't do it in 991 * cancelled mode as we don't have the bitmap to check the constraints 992 * (so, we allocate a buffer and read prior to the check). On the other 993 * hand, we shouldn't blindly g_malloc the number from the stream. 994 * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow 995 * a bit larger (which means that bitmap migration will fail anyway and 996 * the whole migration will most probably fail soon due to broken 997 * stream). 998 */ 999 if (buf_size > 10 * CHUNK_SIZE) { 1000 error_report("Bitmap migration stream buffer allocation request " 1001 "is too large"); 1002 return -EIO; 1003 } 1004 1005 buf = g_malloc(buf_size); 1006 ret = qemu_get_buffer(f, buf, buf_size); 1007 if (ret != buf_size) { 1008 error_report("Failed to read bitmap bits"); 1009 return -EIO; 1010 } 1011 1012 if (s->cancelled) { 1013 return 0; 1014 } 1015 1016 needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap, 1017 first_byte, 1018 nr_bytes); 1019 1020 if (needed_size > buf_size || 1021 buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long)) 1022 /* Here used same alignment as in send_bitmap_bits */ 1023 ) { 1024 error_report("Migrated bitmap granularity doesn't " 1025 "match the destination bitmap '%s' granularity", 1026 bdrv_dirty_bitmap_name(s->bitmap)); 1027 cancel_incoming_locked(s); 1028 return 0; 1029 } 1030 1031 bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes, 1032 false); 1033 } 1034 1035 return 0; 1036 } 1037 1038 static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s, 1039 GHashTable *alias_map) 1040 { 1041 GHashTable *bitmap_alias_map = NULL; 1042 Error *local_err = NULL; 1043 bool nothing; 1044 s->flags = qemu_get_bitmap_flags(f); 1045 trace_dirty_bitmap_load_header(s->flags); 1046 1047 nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS); 1048 1049 if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { 1050 if (!qemu_get_counted_string(f, s->node_alias)) { 1051 error_report("Unable to read node alias string"); 1052 return -EINVAL; 1053 } 1054 1055 if (!s->cancelled) { 1056 if (alias_map) { 1057 const AliasMapInnerNode *amin; 1058 1059 amin = g_hash_table_lookup(alias_map, s->node_alias); 1060 if (!amin) { 1061 error_setg(&local_err, "Error: Unknown node alias '%s'", 1062 s->node_alias); 1063 s->bs = NULL; 1064 } else { 1065 bitmap_alias_map = amin->subtree; 1066 s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err); 1067 } 1068 } else { 1069 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias, 1070 &local_err); 1071 } 1072 if (!s->bs) { 1073 error_report_err(local_err); 1074 cancel_incoming_locked(s); 1075 } 1076 } 1077 } else if (s->bs) { 1078 if (alias_map) { 1079 const AliasMapInnerNode *amin; 1080 1081 /* Must be present in the map, or s->bs would not be set */ 1082 amin = g_hash_table_lookup(alias_map, s->node_alias); 1083 assert(amin != NULL); 1084 1085 bitmap_alias_map = amin->subtree; 1086 } 1087 } else if (!nothing && !s->cancelled) { 1088 error_report("Error: block device name is not set"); 1089 cancel_incoming_locked(s); 1090 } 1091 1092 assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map); 1093 1094 if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { 1095 const char *bitmap_name; 1096 1097 if (!qemu_get_counted_string(f, s->bitmap_alias)) { 1098 error_report("Unable to read bitmap alias string"); 1099 return -EINVAL; 1100 } 1101 1102 bitmap_name = s->bitmap_alias; 1103 if (!s->cancelled && bitmap_alias_map) { 1104 BitmapMigrationBitmapAlias *bmap_inner; 1105 1106 bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias); 1107 if (!bmap_inner) { 1108 error_report("Error: Unknown bitmap alias '%s' on node " 1109 "'%s' (alias '%s')", s->bitmap_alias, 1110 s->bs->node_name, s->node_alias); 1111 cancel_incoming_locked(s); 1112 } else { 1113 bitmap_name = bmap_inner->name; 1114 } 1115 1116 s->bmap_inner = bmap_inner; 1117 } 1118 1119 if (!s->cancelled) { 1120 g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name)); 1121 s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name); 1122 1123 /* 1124 * bitmap may be NULL here, it wouldn't be an error if it is the 1125 * first occurrence of the bitmap 1126 */ 1127 if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) { 1128 error_report("Error: unknown dirty bitmap " 1129 "'%s' for block device '%s'", 1130 s->bitmap_name, s->bs->node_name); 1131 cancel_incoming_locked(s); 1132 } 1133 } 1134 } else if (!s->bitmap && !nothing && !s->cancelled) { 1135 error_report("Error: block device name is not set"); 1136 cancel_incoming_locked(s); 1137 } 1138 1139 return 0; 1140 } 1141 1142 /* 1143 * dirty_bitmap_load 1144 * 1145 * Load sequence of dirty bitmap chunks. Return error only on fatal io stream 1146 * violations. On other errors just cancel bitmaps incoming migration and return 1147 * 0. 1148 * 1149 * Note, than when incoming bitmap migration is canceled, we still must read all 1150 * our chunks (and just ignore them), to not affect other migration objects. 1151 */ 1152 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id) 1153 { 1154 GHashTable *alias_map = NULL; 1155 const MigrationParameters *mig_params = &migrate_get_current()->parameters; 1156 DBMLoadState *s = &((DBMState *)opaque)->load; 1157 int ret = 0; 1158 1159 trace_dirty_bitmap_load_enter(); 1160 1161 if (version_id != 1) { 1162 QEMU_LOCK_GUARD(&s->lock); 1163 cancel_incoming_locked(s); 1164 return -EINVAL; 1165 } 1166 1167 if (mig_params->has_block_bitmap_mapping) { 1168 alias_map = construct_alias_map(mig_params->block_bitmap_mapping, 1169 false, &error_abort); 1170 } 1171 1172 do { 1173 QEMU_LOCK_GUARD(&s->lock); 1174 1175 ret = dirty_bitmap_load_header(f, s, alias_map); 1176 if (ret < 0) { 1177 cancel_incoming_locked(s); 1178 goto fail; 1179 } 1180 1181 if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) { 1182 ret = dirty_bitmap_load_start(f, s); 1183 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) { 1184 dirty_bitmap_load_complete(f, s); 1185 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) { 1186 ret = dirty_bitmap_load_bits(f, s); 1187 } 1188 1189 if (!ret) { 1190 ret = qemu_file_get_error(f); 1191 } 1192 1193 if (ret) { 1194 cancel_incoming_locked(s); 1195 goto fail; 1196 } 1197 } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS)); 1198 1199 trace_dirty_bitmap_load_success(); 1200 ret = 0; 1201 fail: 1202 if (alias_map) { 1203 g_hash_table_destroy(alias_map); 1204 } 1205 return ret; 1206 } 1207 1208 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque) 1209 { 1210 DBMSaveState *s = &((DBMState *)opaque)->save; 1211 SaveBitmapState *dbms = NULL; 1212 if (init_dirty_bitmap_migration(s) < 0) { 1213 return -1; 1214 } 1215 1216 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 1217 send_bitmap_start(f, s, dbms); 1218 } 1219 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); 1220 1221 return 0; 1222 } 1223 1224 static bool dirty_bitmap_is_active(void *opaque) 1225 { 1226 DBMSaveState *s = &((DBMState *)opaque)->save; 1227 1228 return migrate_dirty_bitmaps() && !s->no_bitmaps; 1229 } 1230 1231 static bool dirty_bitmap_is_active_iterate(void *opaque) 1232 { 1233 return dirty_bitmap_is_active(opaque) && !runstate_is_running(); 1234 } 1235 1236 static bool dirty_bitmap_has_postcopy(void *opaque) 1237 { 1238 return true; 1239 } 1240 1241 static SaveVMHandlers savevm_dirty_bitmap_handlers = { 1242 .save_setup = dirty_bitmap_save_setup, 1243 .save_live_complete_postcopy = dirty_bitmap_save_complete, 1244 .save_live_complete_precopy = dirty_bitmap_save_complete, 1245 .has_postcopy = dirty_bitmap_has_postcopy, 1246 .save_live_pending = dirty_bitmap_save_pending, 1247 .save_live_iterate = dirty_bitmap_save_iterate, 1248 .is_active_iterate = dirty_bitmap_is_active_iterate, 1249 .load_state = dirty_bitmap_load, 1250 .save_cleanup = dirty_bitmap_save_cleanup, 1251 .is_active = dirty_bitmap_is_active, 1252 }; 1253 1254 void dirty_bitmap_mig_init(void) 1255 { 1256 QSIMPLEQ_INIT(&dbm_state.save.dbms_list); 1257 qemu_mutex_init(&dbm_state.load.lock); 1258 1259 register_savevm_live("dirty-bitmap", 0, 1, 1260 &savevm_dirty_bitmap_handlers, 1261 &dbm_state); 1262 } 1263