1 /* 2 * Block dirty bitmap postcopy migration 3 * 4 * Copyright IBM, Corp. 2009 5 * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved. 6 * 7 * Authors: 8 * Liran Schour <lirans@il.ibm.com> 9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> 10 * 11 * This work is licensed under the terms of the GNU GPL, version 2. See 12 * the COPYING file in the top-level directory. 13 * This file is derived from migration/block.c, so it's author and IBM copyright 14 * are here, although content is quite different. 15 * 16 * Contributions after 2012-01-13 are licensed under the terms of the 17 * GNU GPL, version 2 or (at your option) any later version. 18 * 19 * *** 20 * 21 * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable 22 * bitmaps are migrated. 23 * 24 * Bitmap migration implies creating bitmap with the same name and granularity 25 * in destination QEMU. If the bitmap with the same name (for the same node) 26 * already exists on destination an error will be generated. 27 * 28 * format of migration: 29 * 30 * # Header (shared for different chunk types) 31 * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags) 32 * [ 1 byte: node alias size ] \ flags & DEVICE_NAME 33 * [ n bytes: node alias ] / 34 * [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME 35 * [ n bytes: bitmap alias ] / 36 * 37 * # Start of bitmap migration (flags & START) 38 * header 39 * be64: granularity 40 * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap) 41 * bit 0 - bitmap is enabled 42 * bit 1 - bitmap is persistent 43 * bit 2 - bitmap is autoloading 44 * bits 3-7 - reserved, must be zero 45 * 46 * # Complete of bitmap migration (flags & COMPLETE) 47 * header 48 * 49 * # Data chunk of bitmap migration 50 * header 51 * be64: start sector 52 * be32: number of sectors 53 * [ be64: buffer size ] \ ! (flags & ZEROES) 54 * [ n bytes: buffer ] / 55 * 56 * The last chunk in stream should contain flags & EOS. The chunk may skip 57 * device and/or bitmap names, assuming them to be the same with the previous 58 * chunk. 59 */ 60 61 #include "qemu/osdep.h" 62 #include "block/block.h" 63 #include "block/block_int.h" 64 #include "sysemu/block-backend.h" 65 #include "sysemu/runstate.h" 66 #include "qemu/main-loop.h" 67 #include "qemu/error-report.h" 68 #include "migration/misc.h" 69 #include "migration/migration.h" 70 #include "qemu-file.h" 71 #include "migration/vmstate.h" 72 #include "migration/register.h" 73 #include "qemu/hbitmap.h" 74 #include "qemu/cutils.h" 75 #include "qemu/id.h" 76 #include "qapi/error.h" 77 #include "qapi/qapi-commands-migration.h" 78 #include "trace.h" 79 80 #define CHUNK_SIZE (1 << 10) 81 82 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as 83 * follows: 84 * in first (most significant) byte bit 8 is clear --> one byte 85 * in first byte bit 8 is set --> two or four bytes, depending on second 86 * byte: 87 * | in second byte bit 8 is clear --> two bytes 88 * | in second byte bit 8 is set --> four bytes 89 */ 90 #define DIRTY_BITMAP_MIG_FLAG_EOS 0x01 91 #define DIRTY_BITMAP_MIG_FLAG_ZEROES 0x02 92 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME 0x04 93 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME 0x08 94 #define DIRTY_BITMAP_MIG_FLAG_START 0x10 95 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE 0x20 96 #define DIRTY_BITMAP_MIG_FLAG_BITS 0x40 97 98 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS 0x80 99 100 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED 0x01 101 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT 0x02 102 /* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */ 103 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK 0xf8 104 105 /* State of one bitmap during save process */ 106 typedef struct SaveBitmapState { 107 /* Written during setup phase. */ 108 BlockDriverState *bs; 109 char *node_alias; 110 char *bitmap_alias; 111 BdrvDirtyBitmap *bitmap; 112 uint64_t total_sectors; 113 uint64_t sectors_per_chunk; 114 QSIMPLEQ_ENTRY(SaveBitmapState) entry; 115 uint8_t flags; 116 117 /* For bulk phase. */ 118 bool bulk_completed; 119 uint64_t cur_sector; 120 } SaveBitmapState; 121 122 /* State of the dirty bitmap migration (DBM) during save process */ 123 typedef struct DBMSaveState { 124 QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list; 125 126 bool bulk_completed; 127 bool no_bitmaps; 128 129 /* for send_bitmap_bits() */ 130 BlockDriverState *prev_bs; 131 BdrvDirtyBitmap *prev_bitmap; 132 } DBMSaveState; 133 134 typedef struct LoadBitmapState { 135 BlockDriverState *bs; 136 BdrvDirtyBitmap *bitmap; 137 bool migrated; 138 bool enabled; 139 } LoadBitmapState; 140 141 /* State of the dirty bitmap migration (DBM) during load process */ 142 typedef struct DBMLoadState { 143 uint32_t flags; 144 char node_alias[256]; 145 char bitmap_alias[256]; 146 char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1]; 147 BlockDriverState *bs; 148 BdrvDirtyBitmap *bitmap; 149 150 bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */ 151 152 /* 153 * cancelled 154 * Incoming migration is cancelled for some reason. That means that we 155 * still should read our chunks from migration stream, to not affect other 156 * migration objects (like RAM), but just ignore them and do not touch any 157 * bitmaps or nodes. 158 */ 159 bool cancelled; 160 161 GSList *bitmaps; 162 QemuMutex lock; /* protect bitmaps */ 163 } DBMLoadState; 164 165 typedef struct DBMState { 166 DBMSaveState save; 167 DBMLoadState load; 168 } DBMState; 169 170 static DBMState dbm_state; 171 172 /* For hash tables that map node/bitmap names to aliases */ 173 typedef struct AliasMapInnerNode { 174 char *string; 175 GHashTable *subtree; 176 } AliasMapInnerNode; 177 178 static void free_alias_map_inner_node(void *amin_ptr) 179 { 180 AliasMapInnerNode *amin = amin_ptr; 181 182 g_free(amin->string); 183 g_hash_table_unref(amin->subtree); 184 g_free(amin); 185 } 186 187 /** 188 * Construct an alias map based on the given QMP structure. 189 * 190 * (Note that we cannot store such maps in the MigrationParameters 191 * object, because that struct is defined by the QAPI schema, which 192 * makes it basically impossible to have dicts with arbitrary keys. 193 * Therefore, we instead have to construct these maps when migration 194 * starts.) 195 * 196 * @bbm is the block_bitmap_mapping from the migration parameters. 197 * 198 * If @name_to_alias is true, the returned hash table will map node 199 * and bitmap names to their respective aliases (for outgoing 200 * migration). 201 * 202 * If @name_to_alias is false, the returned hash table will map node 203 * and bitmap aliases to their respective names (for incoming 204 * migration). 205 * 206 * The hash table maps node names/aliases to AliasMapInnerNode 207 * objects, whose .string is the respective node alias/name, and whose 208 * .subtree table maps bitmap names/aliases to the respective bitmap 209 * alias/name. 210 */ 211 static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm, 212 bool name_to_alias, 213 Error **errp) 214 { 215 GHashTable *alias_map; 216 size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1; 217 218 alias_map = g_hash_table_new_full(g_str_hash, g_str_equal, 219 g_free, free_alias_map_inner_node); 220 221 for (; bbm; bbm = bbm->next) { 222 const BitmapMigrationNodeAlias *bmna = bbm->value; 223 const BitmapMigrationBitmapAliasList *bmbal; 224 AliasMapInnerNode *amin; 225 GHashTable *bitmaps_map; 226 const char *node_map_from, *node_map_to; 227 228 if (!id_wellformed(bmna->alias)) { 229 error_setg(errp, "The node alias '%s' is not well-formed", 230 bmna->alias); 231 goto fail; 232 } 233 234 if (strlen(bmna->alias) > UINT8_MAX) { 235 error_setg(errp, "The node alias '%s' is longer than %u bytes", 236 bmna->alias, UINT8_MAX); 237 goto fail; 238 } 239 240 if (strlen(bmna->node_name) > max_node_name_len) { 241 error_setg(errp, "The node name '%s' is longer than %zu bytes", 242 bmna->node_name, max_node_name_len); 243 goto fail; 244 } 245 246 if (name_to_alias) { 247 if (g_hash_table_contains(alias_map, bmna->node_name)) { 248 error_setg(errp, "The node name '%s' is mapped twice", 249 bmna->node_name); 250 goto fail; 251 } 252 253 node_map_from = bmna->node_name; 254 node_map_to = bmna->alias; 255 } else { 256 if (g_hash_table_contains(alias_map, bmna->alias)) { 257 error_setg(errp, "The node alias '%s' is used twice", 258 bmna->alias); 259 goto fail; 260 } 261 262 node_map_from = bmna->alias; 263 node_map_to = bmna->node_name; 264 } 265 266 bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, 267 g_free, g_free); 268 269 amin = g_new(AliasMapInnerNode, 1); 270 *amin = (AliasMapInnerNode){ 271 .string = g_strdup(node_map_to), 272 .subtree = bitmaps_map, 273 }; 274 275 g_hash_table_insert(alias_map, g_strdup(node_map_from), amin); 276 277 for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) { 278 const BitmapMigrationBitmapAlias *bmba = bmbal->value; 279 const char *bmap_map_from, *bmap_map_to; 280 281 if (strlen(bmba->alias) > UINT8_MAX) { 282 error_setg(errp, 283 "The bitmap alias '%s' is longer than %u bytes", 284 bmba->alias, UINT8_MAX); 285 goto fail; 286 } 287 288 if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) { 289 error_setg(errp, "The bitmap name '%s' is longer than %d bytes", 290 bmba->name, BDRV_BITMAP_MAX_NAME_SIZE); 291 goto fail; 292 } 293 294 if (name_to_alias) { 295 bmap_map_from = bmba->name; 296 bmap_map_to = bmba->alias; 297 298 if (g_hash_table_contains(bitmaps_map, bmba->name)) { 299 error_setg(errp, "The bitmap '%s'/'%s' is mapped twice", 300 bmna->node_name, bmba->name); 301 goto fail; 302 } 303 } else { 304 bmap_map_from = bmba->alias; 305 bmap_map_to = bmba->name; 306 307 if (g_hash_table_contains(bitmaps_map, bmba->alias)) { 308 error_setg(errp, "The bitmap alias '%s'/'%s' is used twice", 309 bmna->alias, bmba->alias); 310 goto fail; 311 } 312 } 313 314 g_hash_table_insert(bitmaps_map, 315 g_strdup(bmap_map_from), g_strdup(bmap_map_to)); 316 } 317 } 318 319 return alias_map; 320 321 fail: 322 g_hash_table_destroy(alias_map); 323 return NULL; 324 } 325 326 /** 327 * Run construct_alias_map() in both directions to check whether @bbm 328 * is valid. 329 * (This function is to be used by migration/migration.c to validate 330 * the user-specified block-bitmap-mapping migration parameter.) 331 * 332 * Returns true if and only if the mapping is valid. 333 */ 334 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm, 335 Error **errp) 336 { 337 GHashTable *alias_map; 338 339 alias_map = construct_alias_map(bbm, true, errp); 340 if (!alias_map) { 341 return false; 342 } 343 g_hash_table_destroy(alias_map); 344 345 alias_map = construct_alias_map(bbm, false, errp); 346 if (!alias_map) { 347 return false; 348 } 349 g_hash_table_destroy(alias_map); 350 351 return true; 352 } 353 354 static uint32_t qemu_get_bitmap_flags(QEMUFile *f) 355 { 356 uint8_t flags = qemu_get_byte(f); 357 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) { 358 flags = flags << 8 | qemu_get_byte(f); 359 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) { 360 flags = flags << 16 | qemu_get_be16(f); 361 } 362 } 363 364 return flags; 365 } 366 367 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags) 368 { 369 /* The code currently does not send flags as more than one byte */ 370 assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS))); 371 372 qemu_put_byte(f, flags); 373 } 374 375 static void send_bitmap_header(QEMUFile *f, DBMSaveState *s, 376 SaveBitmapState *dbms, uint32_t additional_flags) 377 { 378 BlockDriverState *bs = dbms->bs; 379 BdrvDirtyBitmap *bitmap = dbms->bitmap; 380 uint32_t flags = additional_flags; 381 trace_send_bitmap_header_enter(); 382 383 if (bs != s->prev_bs) { 384 s->prev_bs = bs; 385 flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME; 386 } 387 388 if (bitmap != s->prev_bitmap) { 389 s->prev_bitmap = bitmap; 390 flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME; 391 } 392 393 qemu_put_bitmap_flags(f, flags); 394 395 if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { 396 qemu_put_counted_string(f, dbms->node_alias); 397 } 398 399 if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { 400 qemu_put_counted_string(f, dbms->bitmap_alias); 401 } 402 } 403 404 static void send_bitmap_start(QEMUFile *f, DBMSaveState *s, 405 SaveBitmapState *dbms) 406 { 407 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START); 408 qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap)); 409 qemu_put_byte(f, dbms->flags); 410 } 411 412 static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s, 413 SaveBitmapState *dbms) 414 { 415 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE); 416 } 417 418 static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s, 419 SaveBitmapState *dbms, 420 uint64_t start_sector, uint32_t nr_sectors) 421 { 422 /* align for buffer_is_zero() */ 423 uint64_t align = 4 * sizeof(long); 424 uint64_t unaligned_size = 425 bdrv_dirty_bitmap_serialization_size( 426 dbms->bitmap, start_sector << BDRV_SECTOR_BITS, 427 (uint64_t)nr_sectors << BDRV_SECTOR_BITS); 428 uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align); 429 uint8_t *buf = g_malloc0(buf_size); 430 uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS; 431 432 bdrv_dirty_bitmap_serialize_part( 433 dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS, 434 (uint64_t)nr_sectors << BDRV_SECTOR_BITS); 435 436 if (buffer_is_zero(buf, buf_size)) { 437 g_free(buf); 438 buf = NULL; 439 flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES; 440 } 441 442 trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size); 443 444 send_bitmap_header(f, s, dbms, flags); 445 446 qemu_put_be64(f, start_sector); 447 qemu_put_be32(f, nr_sectors); 448 449 /* if a block is zero we need to flush here since the network 450 * bandwidth is now a lot higher than the storage device bandwidth. 451 * thus if we queue zero blocks we slow down the migration. */ 452 if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) { 453 qemu_fflush(f); 454 } else { 455 qemu_put_be64(f, buf_size); 456 qemu_put_buffer(f, buf, buf_size); 457 } 458 459 g_free(buf); 460 } 461 462 /* Called with iothread lock taken. */ 463 static void dirty_bitmap_do_save_cleanup(DBMSaveState *s) 464 { 465 SaveBitmapState *dbms; 466 467 while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) { 468 QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry); 469 bdrv_dirty_bitmap_set_busy(dbms->bitmap, false); 470 bdrv_unref(dbms->bs); 471 g_free(dbms->node_alias); 472 g_free(dbms->bitmap_alias); 473 g_free(dbms); 474 } 475 } 476 477 /* Called with iothread lock taken. */ 478 static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs, 479 const char *bs_name, GHashTable *alias_map) 480 { 481 BdrvDirtyBitmap *bitmap; 482 SaveBitmapState *dbms; 483 GHashTable *bitmap_aliases; 484 const char *node_alias, *bitmap_name, *bitmap_alias; 485 Error *local_err = NULL; 486 487 /* When an alias map is given, @bs_name must be @bs's node name */ 488 assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs))); 489 490 FOR_EACH_DIRTY_BITMAP(bs, bitmap) { 491 if (bdrv_dirty_bitmap_name(bitmap)) { 492 break; 493 } 494 } 495 if (!bitmap) { 496 return 0; 497 } 498 499 bitmap_name = bdrv_dirty_bitmap_name(bitmap); 500 501 if (!bs_name || strcmp(bs_name, "") == 0) { 502 error_report("Bitmap '%s' in unnamed node can't be migrated", 503 bitmap_name); 504 return -1; 505 } 506 507 if (alias_map) { 508 const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name); 509 510 if (!amin) { 511 /* Skip bitmaps on nodes with no alias */ 512 return 0; 513 } 514 515 node_alias = amin->string; 516 bitmap_aliases = amin->subtree; 517 } else { 518 node_alias = bs_name; 519 bitmap_aliases = NULL; 520 } 521 522 if (node_alias[0] == '#') { 523 error_report("Bitmap '%s' in a node with auto-generated " 524 "name '%s' can't be migrated", 525 bitmap_name, node_alias); 526 return -1; 527 } 528 529 FOR_EACH_DIRTY_BITMAP(bs, bitmap) { 530 bitmap_name = bdrv_dirty_bitmap_name(bitmap); 531 if (!bitmap_name) { 532 continue; 533 } 534 535 if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) { 536 error_report_err(local_err); 537 return -1; 538 } 539 540 if (bitmap_aliases) { 541 bitmap_alias = g_hash_table_lookup(bitmap_aliases, bitmap_name); 542 if (!bitmap_alias) { 543 /* Skip bitmaps with no alias */ 544 continue; 545 } 546 } else { 547 if (strlen(bitmap_name) > UINT8_MAX) { 548 error_report("Cannot migrate bitmap '%s' on node '%s': " 549 "Name is longer than %u bytes", 550 bitmap_name, bs_name, UINT8_MAX); 551 return -1; 552 } 553 bitmap_alias = bitmap_name; 554 } 555 556 bdrv_ref(bs); 557 bdrv_dirty_bitmap_set_busy(bitmap, true); 558 559 dbms = g_new0(SaveBitmapState, 1); 560 dbms->bs = bs; 561 dbms->node_alias = g_strdup(node_alias); 562 dbms->bitmap_alias = g_strdup(bitmap_alias); 563 dbms->bitmap = bitmap; 564 dbms->total_sectors = bdrv_nb_sectors(bs); 565 dbms->sectors_per_chunk = CHUNK_SIZE * 8 * 566 bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS; 567 if (bdrv_dirty_bitmap_enabled(bitmap)) { 568 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED; 569 } 570 if (bdrv_dirty_bitmap_get_persistence(bitmap)) { 571 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT; 572 } 573 574 QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry); 575 } 576 577 return 0; 578 } 579 580 /* Called with iothread lock taken. */ 581 static int init_dirty_bitmap_migration(DBMSaveState *s) 582 { 583 BlockDriverState *bs; 584 SaveBitmapState *dbms; 585 GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL); 586 BlockBackend *blk; 587 const MigrationParameters *mig_params = &migrate_get_current()->parameters; 588 GHashTable *alias_map = NULL; 589 590 if (mig_params->has_block_bitmap_mapping) { 591 alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true, 592 &error_abort); 593 } 594 595 s->bulk_completed = false; 596 s->prev_bs = NULL; 597 s->prev_bitmap = NULL; 598 s->no_bitmaps = false; 599 600 if (!alias_map) { 601 /* 602 * Use blockdevice name for direct (or filtered) children of named block 603 * backends. 604 */ 605 for (blk = blk_next(NULL); blk; blk = blk_next(blk)) { 606 const char *name = blk_name(blk); 607 608 if (!name || strcmp(name, "") == 0) { 609 continue; 610 } 611 612 bs = blk_bs(blk); 613 614 /* Skip filters without bitmaps */ 615 while (bs && bs->drv && bs->drv->is_filter && 616 !bdrv_has_named_bitmaps(bs)) 617 { 618 if (bs->backing) { 619 bs = bs->backing->bs; 620 } else if (bs->file) { 621 bs = bs->file->bs; 622 } else { 623 bs = NULL; 624 } 625 } 626 627 if (bs && bs->drv && !bs->drv->is_filter) { 628 if (add_bitmaps_to_list(s, bs, name, NULL)) { 629 goto fail; 630 } 631 g_hash_table_add(handled_by_blk, bs); 632 } 633 } 634 } 635 636 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) { 637 if (g_hash_table_contains(handled_by_blk, bs)) { 638 continue; 639 } 640 641 if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) { 642 goto fail; 643 } 644 } 645 646 /* unset migration flags here, to not roll back it */ 647 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 648 bdrv_dirty_bitmap_skip_store(dbms->bitmap, true); 649 } 650 651 if (QSIMPLEQ_EMPTY(&s->dbms_list)) { 652 s->no_bitmaps = true; 653 } 654 655 g_hash_table_destroy(handled_by_blk); 656 if (alias_map) { 657 g_hash_table_destroy(alias_map); 658 } 659 660 return 0; 661 662 fail: 663 g_hash_table_destroy(handled_by_blk); 664 if (alias_map) { 665 g_hash_table_destroy(alias_map); 666 } 667 dirty_bitmap_do_save_cleanup(s); 668 669 return -1; 670 } 671 672 /* Called with no lock taken. */ 673 static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s, 674 SaveBitmapState *dbms) 675 { 676 uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector, 677 dbms->sectors_per_chunk); 678 679 send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors); 680 681 dbms->cur_sector += nr_sectors; 682 if (dbms->cur_sector >= dbms->total_sectors) { 683 dbms->bulk_completed = true; 684 } 685 } 686 687 /* Called with no lock taken. */ 688 static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit) 689 { 690 SaveBitmapState *dbms; 691 692 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 693 while (!dbms->bulk_completed) { 694 bulk_phase_send_chunk(f, s, dbms); 695 if (limit && qemu_file_rate_limit(f)) { 696 return; 697 } 698 } 699 } 700 701 s->bulk_completed = true; 702 } 703 704 /* for SaveVMHandlers */ 705 static void dirty_bitmap_save_cleanup(void *opaque) 706 { 707 DBMSaveState *s = &((DBMState *)opaque)->save; 708 709 dirty_bitmap_do_save_cleanup(s); 710 } 711 712 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque) 713 { 714 DBMSaveState *s = &((DBMState *)opaque)->save; 715 716 trace_dirty_bitmap_save_iterate(migration_in_postcopy()); 717 718 if (migration_in_postcopy() && !s->bulk_completed) { 719 bulk_phase(f, s, true); 720 } 721 722 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); 723 724 return s->bulk_completed; 725 } 726 727 /* Called with iothread lock taken. */ 728 729 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque) 730 { 731 DBMSaveState *s = &((DBMState *)opaque)->save; 732 SaveBitmapState *dbms; 733 trace_dirty_bitmap_save_complete_enter(); 734 735 if (!s->bulk_completed) { 736 bulk_phase(f, s, false); 737 } 738 739 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 740 send_bitmap_complete(f, s, dbms); 741 } 742 743 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); 744 745 trace_dirty_bitmap_save_complete_finish(); 746 747 dirty_bitmap_save_cleanup(opaque); 748 return 0; 749 } 750 751 static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque, 752 uint64_t max_size, 753 uint64_t *res_precopy_only, 754 uint64_t *res_compatible, 755 uint64_t *res_postcopy_only) 756 { 757 DBMSaveState *s = &((DBMState *)opaque)->save; 758 SaveBitmapState *dbms; 759 uint64_t pending = 0; 760 761 qemu_mutex_lock_iothread(); 762 763 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 764 uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap); 765 uint64_t sectors = dbms->bulk_completed ? 0 : 766 dbms->total_sectors - dbms->cur_sector; 767 768 pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran); 769 } 770 771 qemu_mutex_unlock_iothread(); 772 773 trace_dirty_bitmap_save_pending(pending, max_size); 774 775 *res_postcopy_only += pending; 776 } 777 778 /* First occurrence of this bitmap. It should be created if doesn't exist */ 779 static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s) 780 { 781 Error *local_err = NULL; 782 uint32_t granularity = qemu_get_be32(f); 783 uint8_t flags = qemu_get_byte(f); 784 LoadBitmapState *b; 785 786 if (s->cancelled) { 787 return 0; 788 } 789 790 if (s->bitmap) { 791 error_report("Bitmap with the same name ('%s') already exists on " 792 "destination", bdrv_dirty_bitmap_name(s->bitmap)); 793 return -EINVAL; 794 } else { 795 s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity, 796 s->bitmap_name, &local_err); 797 if (!s->bitmap) { 798 error_report_err(local_err); 799 return -EINVAL; 800 } 801 } 802 803 if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) { 804 error_report("Unknown flags in migrated dirty bitmap header: %x", 805 flags); 806 return -EINVAL; 807 } 808 809 if (flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT) { 810 bdrv_dirty_bitmap_set_persistence(s->bitmap, true); 811 } 812 813 bdrv_disable_dirty_bitmap(s->bitmap); 814 if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) { 815 bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err); 816 if (local_err) { 817 error_report_err(local_err); 818 return -EINVAL; 819 } 820 } 821 822 b = g_new(LoadBitmapState, 1); 823 b->bs = s->bs; 824 b->bitmap = s->bitmap; 825 b->migrated = false; 826 b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED; 827 828 s->bitmaps = g_slist_prepend(s->bitmaps, b); 829 830 return 0; 831 } 832 833 /* 834 * before_vm_start_handle_item 835 * 836 * g_slist_foreach helper 837 * 838 * item is LoadBitmapState* 839 * opaque is DBMLoadState* 840 */ 841 static void before_vm_start_handle_item(void *item, void *opaque) 842 { 843 DBMLoadState *s = opaque; 844 LoadBitmapState *b = item; 845 846 if (b->enabled) { 847 if (b->migrated) { 848 bdrv_enable_dirty_bitmap(b->bitmap); 849 } else { 850 bdrv_dirty_bitmap_enable_successor(b->bitmap); 851 } 852 } 853 854 if (b->migrated) { 855 s->bitmaps = g_slist_remove(s->bitmaps, b); 856 g_free(b); 857 } 858 } 859 860 void dirty_bitmap_mig_before_vm_start(void) 861 { 862 DBMLoadState *s = &dbm_state.load; 863 qemu_mutex_lock(&s->lock); 864 865 assert(!s->before_vm_start_handled); 866 g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s); 867 s->before_vm_start_handled = true; 868 869 qemu_mutex_unlock(&s->lock); 870 } 871 872 static void cancel_incoming_locked(DBMLoadState *s) 873 { 874 GSList *item; 875 876 if (s->cancelled) { 877 return; 878 } 879 880 s->cancelled = true; 881 s->bs = NULL; 882 s->bitmap = NULL; 883 884 /* Drop all unfinished bitmaps */ 885 for (item = s->bitmaps; item; item = g_slist_next(item)) { 886 LoadBitmapState *b = item->data; 887 888 /* 889 * Bitmap must be unfinished, as finished bitmaps should already be 890 * removed from the list. 891 */ 892 assert(!s->before_vm_start_handled || !b->migrated); 893 if (bdrv_dirty_bitmap_has_successor(b->bitmap)) { 894 bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort); 895 } 896 bdrv_release_dirty_bitmap(b->bitmap); 897 } 898 899 g_slist_free_full(s->bitmaps, g_free); 900 s->bitmaps = NULL; 901 } 902 903 void dirty_bitmap_mig_cancel_outgoing(void) 904 { 905 dirty_bitmap_do_save_cleanup(&dbm_state.save); 906 } 907 908 void dirty_bitmap_mig_cancel_incoming(void) 909 { 910 DBMLoadState *s = &dbm_state.load; 911 912 qemu_mutex_lock(&s->lock); 913 914 cancel_incoming_locked(s); 915 916 qemu_mutex_unlock(&s->lock); 917 } 918 919 static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s) 920 { 921 GSList *item; 922 trace_dirty_bitmap_load_complete(); 923 924 if (s->cancelled) { 925 return; 926 } 927 928 bdrv_dirty_bitmap_deserialize_finish(s->bitmap); 929 930 if (bdrv_dirty_bitmap_has_successor(s->bitmap)) { 931 bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort); 932 } 933 934 for (item = s->bitmaps; item; item = g_slist_next(item)) { 935 LoadBitmapState *b = item->data; 936 937 if (b->bitmap == s->bitmap) { 938 b->migrated = true; 939 if (s->before_vm_start_handled) { 940 s->bitmaps = g_slist_remove(s->bitmaps, b); 941 g_free(b); 942 } 943 break; 944 } 945 } 946 } 947 948 static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s) 949 { 950 uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS; 951 uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS; 952 trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS, 953 nr_bytes >> BDRV_SECTOR_BITS); 954 955 if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) { 956 trace_dirty_bitmap_load_bits_zeroes(); 957 if (!s->cancelled) { 958 bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte, 959 nr_bytes, false); 960 } 961 } else { 962 size_t ret; 963 g_autofree uint8_t *buf = NULL; 964 uint64_t buf_size = qemu_get_be64(f); 965 uint64_t needed_size; 966 967 /* 968 * The actual check for buf_size is done a bit later. We can't do it in 969 * cancelled mode as we don't have the bitmap to check the constraints 970 * (so, we allocate a buffer and read prior to the check). On the other 971 * hand, we shouldn't blindly g_malloc the number from the stream. 972 * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow 973 * a bit larger (which means that bitmap migration will fail anyway and 974 * the whole migration will most probably fail soon due to broken 975 * stream). 976 */ 977 if (buf_size > 10 * CHUNK_SIZE) { 978 error_report("Bitmap migration stream buffer allocation request " 979 "is too large"); 980 return -EIO; 981 } 982 983 buf = g_malloc(buf_size); 984 ret = qemu_get_buffer(f, buf, buf_size); 985 if (ret != buf_size) { 986 error_report("Failed to read bitmap bits"); 987 return -EIO; 988 } 989 990 if (s->cancelled) { 991 return 0; 992 } 993 994 needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap, 995 first_byte, 996 nr_bytes); 997 998 if (needed_size > buf_size || 999 buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long)) 1000 /* Here used same alignment as in send_bitmap_bits */ 1001 ) { 1002 error_report("Migrated bitmap granularity doesn't " 1003 "match the destination bitmap '%s' granularity", 1004 bdrv_dirty_bitmap_name(s->bitmap)); 1005 cancel_incoming_locked(s); 1006 return 0; 1007 } 1008 1009 bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes, 1010 false); 1011 } 1012 1013 return 0; 1014 } 1015 1016 static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s, 1017 GHashTable *alias_map) 1018 { 1019 GHashTable *bitmap_alias_map = NULL; 1020 Error *local_err = NULL; 1021 bool nothing; 1022 s->flags = qemu_get_bitmap_flags(f); 1023 trace_dirty_bitmap_load_header(s->flags); 1024 1025 nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS); 1026 1027 if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { 1028 if (!qemu_get_counted_string(f, s->node_alias)) { 1029 error_report("Unable to read node alias string"); 1030 return -EINVAL; 1031 } 1032 1033 if (!s->cancelled) { 1034 if (alias_map) { 1035 const AliasMapInnerNode *amin; 1036 1037 amin = g_hash_table_lookup(alias_map, s->node_alias); 1038 if (!amin) { 1039 error_setg(&local_err, "Error: Unknown node alias '%s'", 1040 s->node_alias); 1041 s->bs = NULL; 1042 } else { 1043 bitmap_alias_map = amin->subtree; 1044 s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err); 1045 } 1046 } else { 1047 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias, 1048 &local_err); 1049 } 1050 if (!s->bs) { 1051 error_report_err(local_err); 1052 cancel_incoming_locked(s); 1053 } 1054 } 1055 } else if (s->bs) { 1056 if (alias_map) { 1057 const AliasMapInnerNode *amin; 1058 1059 /* Must be present in the map, or s->bs would not be set */ 1060 amin = g_hash_table_lookup(alias_map, s->node_alias); 1061 assert(amin != NULL); 1062 1063 bitmap_alias_map = amin->subtree; 1064 } 1065 } else if (!nothing && !s->cancelled) { 1066 error_report("Error: block device name is not set"); 1067 cancel_incoming_locked(s); 1068 } 1069 1070 assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map); 1071 1072 if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { 1073 const char *bitmap_name; 1074 1075 if (!qemu_get_counted_string(f, s->bitmap_alias)) { 1076 error_report("Unable to read bitmap alias string"); 1077 return -EINVAL; 1078 } 1079 1080 if (!s->cancelled) { 1081 if (bitmap_alias_map) { 1082 bitmap_name = g_hash_table_lookup(bitmap_alias_map, 1083 s->bitmap_alias); 1084 if (!bitmap_name) { 1085 error_report("Error: Unknown bitmap alias '%s' on node " 1086 "'%s' (alias '%s')", s->bitmap_alias, 1087 s->bs->node_name, s->node_alias); 1088 cancel_incoming_locked(s); 1089 } 1090 } else { 1091 bitmap_name = s->bitmap_alias; 1092 } 1093 } 1094 1095 if (!s->cancelled) { 1096 g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name)); 1097 s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name); 1098 1099 /* 1100 * bitmap may be NULL here, it wouldn't be an error if it is the 1101 * first occurrence of the bitmap 1102 */ 1103 if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) { 1104 error_report("Error: unknown dirty bitmap " 1105 "'%s' for block device '%s'", 1106 s->bitmap_name, s->bs->node_name); 1107 cancel_incoming_locked(s); 1108 } 1109 } 1110 } else if (!s->bitmap && !nothing && !s->cancelled) { 1111 error_report("Error: block device name is not set"); 1112 cancel_incoming_locked(s); 1113 } 1114 1115 return 0; 1116 } 1117 1118 /* 1119 * dirty_bitmap_load 1120 * 1121 * Load sequence of dirty bitmap chunks. Return error only on fatal io stream 1122 * violations. On other errors just cancel bitmaps incoming migration and return 1123 * 0. 1124 * 1125 * Note, than when incoming bitmap migration is canceled, we still must read all 1126 * our chunks (and just ignore them), to not affect other migration objects. 1127 */ 1128 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id) 1129 { 1130 GHashTable *alias_map = NULL; 1131 const MigrationParameters *mig_params = &migrate_get_current()->parameters; 1132 DBMLoadState *s = &((DBMState *)opaque)->load; 1133 int ret = 0; 1134 1135 trace_dirty_bitmap_load_enter(); 1136 1137 if (version_id != 1) { 1138 QEMU_LOCK_GUARD(&s->lock); 1139 cancel_incoming_locked(s); 1140 return -EINVAL; 1141 } 1142 1143 if (mig_params->has_block_bitmap_mapping) { 1144 alias_map = construct_alias_map(mig_params->block_bitmap_mapping, 1145 false, &error_abort); 1146 } 1147 1148 do { 1149 QEMU_LOCK_GUARD(&s->lock); 1150 1151 ret = dirty_bitmap_load_header(f, s, alias_map); 1152 if (ret < 0) { 1153 cancel_incoming_locked(s); 1154 goto fail; 1155 } 1156 1157 if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) { 1158 ret = dirty_bitmap_load_start(f, s); 1159 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) { 1160 dirty_bitmap_load_complete(f, s); 1161 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) { 1162 ret = dirty_bitmap_load_bits(f, s); 1163 } 1164 1165 if (!ret) { 1166 ret = qemu_file_get_error(f); 1167 } 1168 1169 if (ret) { 1170 cancel_incoming_locked(s); 1171 goto fail; 1172 } 1173 } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS)); 1174 1175 trace_dirty_bitmap_load_success(); 1176 ret = 0; 1177 fail: 1178 if (alias_map) { 1179 g_hash_table_destroy(alias_map); 1180 } 1181 return ret; 1182 } 1183 1184 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque) 1185 { 1186 DBMSaveState *s = &((DBMState *)opaque)->save; 1187 SaveBitmapState *dbms = NULL; 1188 if (init_dirty_bitmap_migration(s) < 0) { 1189 return -1; 1190 } 1191 1192 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { 1193 send_bitmap_start(f, s, dbms); 1194 } 1195 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); 1196 1197 return 0; 1198 } 1199 1200 static bool dirty_bitmap_is_active(void *opaque) 1201 { 1202 DBMSaveState *s = &((DBMState *)opaque)->save; 1203 1204 return migrate_dirty_bitmaps() && !s->no_bitmaps; 1205 } 1206 1207 static bool dirty_bitmap_is_active_iterate(void *opaque) 1208 { 1209 return dirty_bitmap_is_active(opaque) && !runstate_is_running(); 1210 } 1211 1212 static bool dirty_bitmap_has_postcopy(void *opaque) 1213 { 1214 return true; 1215 } 1216 1217 static SaveVMHandlers savevm_dirty_bitmap_handlers = { 1218 .save_setup = dirty_bitmap_save_setup, 1219 .save_live_complete_postcopy = dirty_bitmap_save_complete, 1220 .save_live_complete_precopy = dirty_bitmap_save_complete, 1221 .has_postcopy = dirty_bitmap_has_postcopy, 1222 .save_live_pending = dirty_bitmap_save_pending, 1223 .save_live_iterate = dirty_bitmap_save_iterate, 1224 .is_active_iterate = dirty_bitmap_is_active_iterate, 1225 .load_state = dirty_bitmap_load, 1226 .save_cleanup = dirty_bitmap_save_cleanup, 1227 .is_active = dirty_bitmap_is_active, 1228 }; 1229 1230 void dirty_bitmap_mig_init(void) 1231 { 1232 QSIMPLEQ_INIT(&dbm_state.save.dbms_list); 1233 qemu_mutex_init(&dbm_state.load.lock); 1234 1235 register_savevm_live("dirty-bitmap", 0, 1, 1236 &savevm_dirty_bitmap_handlers, 1237 &dbm_state); 1238 } 1239