1 /*
2  * Block dirty bitmap postcopy migration
3  *
4  * Copyright IBM, Corp. 2009
5  * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
6  *
7  * Authors:
8  *  Liran Schour   <lirans@il.ibm.com>
9  *  Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
10  *
11  * This work is licensed under the terms of the GNU GPL, version 2.  See
12  * the COPYING file in the top-level directory.
13  * This file is derived from migration/block.c, so it's author and IBM copyright
14  * are here, although content is quite different.
15  *
16  * Contributions after 2012-01-13 are licensed under the terms of the
17  * GNU GPL, version 2 or (at your option) any later version.
18  *
19  *                                ***
20  *
21  * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable
22  * bitmaps are migrated.
23  *
24  * Bitmap migration implies creating bitmap with the same name and granularity
25  * in destination QEMU. If the bitmap with the same name (for the same node)
26  * already exists on destination an error will be generated.
27  *
28  * format of migration:
29  *
30  * # Header (shared for different chunk types)
31  * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
32  * [ 1 byte: node alias size ] \  flags & DEVICE_NAME
33  * [ n bytes: node alias     ] /
34  * [ 1 byte: bitmap alias size ] \  flags & BITMAP_NAME
35  * [ n bytes: bitmap alias     ] /
36  *
37  * # Start of bitmap migration (flags & START)
38  * header
39  * be64: granularity
40  * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap)
41  *   bit 0    -  bitmap is enabled
42  *   bit 1    -  bitmap is persistent
43  *   bit 2    -  bitmap is autoloading
44  *   bits 3-7 - reserved, must be zero
45  *
46  * # Complete of bitmap migration (flags & COMPLETE)
47  * header
48  *
49  * # Data chunk of bitmap migration
50  * header
51  * be64: start sector
52  * be32: number of sectors
53  * [ be64: buffer size  ] \ ! (flags & ZEROES)
54  * [ n bytes: buffer    ] /
55  *
56  * The last chunk in stream should contain flags & EOS. The chunk may skip
57  * device and/or bitmap names, assuming them to be the same with the previous
58  * chunk.
59  */
60 
61 #include "qemu/osdep.h"
62 #include "block/block.h"
63 #include "block/block_int.h"
64 #include "block/dirty-bitmap.h"
65 #include "sysemu/block-backend.h"
66 #include "sysemu/runstate.h"
67 #include "qemu/main-loop.h"
68 #include "qemu/error-report.h"
69 #include "migration/misc.h"
70 #include "migration/migration.h"
71 #include "qemu-file.h"
72 #include "migration/vmstate.h"
73 #include "migration/register.h"
74 #include "qemu/hbitmap.h"
75 #include "qemu/cutils.h"
76 #include "qemu/id.h"
77 #include "qapi/error.h"
78 #include "qapi/qapi-commands-migration.h"
79 #include "qapi/qapi-visit-migration.h"
80 #include "qapi/clone-visitor.h"
81 #include "trace.h"
82 #include "options.h"
83 
84 #define CHUNK_SIZE     (1 << 10)
85 
86 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as
87  * follows:
88  * in first (most significant) byte bit 8 is clear  -->  one byte
89  * in first byte bit 8 is set    -->  two or four bytes, depending on second
90  *                                    byte:
91  *    | in second byte bit 8 is clear  -->  two bytes
92  *    | in second byte bit 8 is set    -->  four bytes
93  */
94 #define DIRTY_BITMAP_MIG_FLAG_EOS           0x01
95 #define DIRTY_BITMAP_MIG_FLAG_ZEROES        0x02
96 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME   0x04
97 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME   0x08
98 #define DIRTY_BITMAP_MIG_FLAG_START         0x10
99 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE      0x20
100 #define DIRTY_BITMAP_MIG_FLAG_BITS          0x40
101 
102 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS        0x80
103 
104 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED          0x01
105 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT       0x02
106 /* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */
107 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK    0xf8
108 
109 /* State of one bitmap during save process */
110 typedef struct SaveBitmapState {
111     /* Written during setup phase. */
112     BlockDriverState *bs;
113     char *node_alias;
114     char *bitmap_alias;
115     BdrvDirtyBitmap *bitmap;
116     uint64_t total_sectors;
117     uint64_t sectors_per_chunk;
118     QSIMPLEQ_ENTRY(SaveBitmapState) entry;
119     uint8_t flags;
120 
121     /* For bulk phase. */
122     bool bulk_completed;
123     uint64_t cur_sector;
124 } SaveBitmapState;
125 
126 /* State of the dirty bitmap migration (DBM) during save process */
127 typedef struct DBMSaveState {
128     QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list;
129 
130     bool bulk_completed;
131     bool no_bitmaps;
132 
133     /* for send_bitmap_bits() */
134     BlockDriverState *prev_bs;
135     BdrvDirtyBitmap *prev_bitmap;
136 } DBMSaveState;
137 
138 typedef struct LoadBitmapState {
139     BlockDriverState *bs;
140     BdrvDirtyBitmap *bitmap;
141     bool migrated;
142     bool enabled;
143 } LoadBitmapState;
144 
145 /* State of the dirty bitmap migration (DBM) during load process */
146 typedef struct DBMLoadState {
147     uint32_t flags;
148     char node_alias[256];
149     char bitmap_alias[256];
150     char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
151     BlockDriverState *bs;
152     BdrvDirtyBitmap *bitmap;
153 
154     bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */
155     BitmapMigrationBitmapAlias *bmap_inner;
156 
157     /*
158      * cancelled
159      * Incoming migration is cancelled for some reason. That means that we
160      * still should read our chunks from migration stream, to not affect other
161      * migration objects (like RAM), but just ignore them and do not touch any
162      * bitmaps or nodes.
163      */
164     bool cancelled;
165 
166     GSList *bitmaps;
167     QemuMutex lock; /* protect bitmaps */
168 } DBMLoadState;
169 
170 typedef struct DBMState {
171     DBMSaveState save;
172     DBMLoadState load;
173 } DBMState;
174 
175 static DBMState dbm_state;
176 
177 /* For hash tables that map node/bitmap names to aliases */
178 typedef struct AliasMapInnerNode {
179     char *string;
180     GHashTable *subtree;
181 } AliasMapInnerNode;
182 
183 static void free_alias_map_inner_node(void *amin_ptr)
184 {
185     AliasMapInnerNode *amin = amin_ptr;
186 
187     g_free(amin->string);
188     g_hash_table_unref(amin->subtree);
189     g_free(amin);
190 }
191 
192 /**
193  * Construct an alias map based on the given QMP structure.
194  *
195  * (Note that we cannot store such maps in the MigrationParameters
196  * object, because that struct is defined by the QAPI schema, which
197  * makes it basically impossible to have dicts with arbitrary keys.
198  * Therefore, we instead have to construct these maps when migration
199  * starts.)
200  *
201  * @bbm is the block_bitmap_mapping from the migration parameters.
202  *
203  * If @name_to_alias is true, the returned hash table will map node
204  * and bitmap names to their respective aliases (for outgoing
205  * migration).
206  *
207  * If @name_to_alias is false, the returned hash table will map node
208  * and bitmap aliases to their respective names (for incoming
209  * migration).
210  *
211  * The hash table maps node names/aliases to AliasMapInnerNode
212  * objects, whose .string is the respective node alias/name, and whose
213  * .subtree table maps bitmap names/aliases to the respective bitmap
214  * alias/name.
215  */
216 static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
217                                        bool name_to_alias,
218                                        Error **errp)
219 {
220     GHashTable *alias_map;
221     size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
222 
223     alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
224                                       g_free, free_alias_map_inner_node);
225 
226     for (; bbm; bbm = bbm->next) {
227         const BitmapMigrationNodeAlias *bmna = bbm->value;
228         const BitmapMigrationBitmapAliasList *bmbal;
229         AliasMapInnerNode *amin;
230         GHashTable *bitmaps_map;
231         const char *node_map_from, *node_map_to;
232         GDestroyNotify gdn;
233 
234         if (!id_wellformed(bmna->alias)) {
235             error_setg(errp, "The node alias '%s' is not well-formed",
236                        bmna->alias);
237             goto fail;
238         }
239 
240         if (strlen(bmna->alias) > UINT8_MAX) {
241             error_setg(errp, "The node alias '%s' is longer than %u bytes",
242                        bmna->alias, UINT8_MAX);
243             goto fail;
244         }
245 
246         if (strlen(bmna->node_name) > max_node_name_len) {
247             error_setg(errp, "The node name '%s' is longer than %zu bytes",
248                        bmna->node_name, max_node_name_len);
249             goto fail;
250         }
251 
252         if (name_to_alias) {
253             if (g_hash_table_contains(alias_map, bmna->node_name)) {
254                 error_setg(errp, "The node name '%s' is mapped twice",
255                            bmna->node_name);
256                 goto fail;
257             }
258 
259             node_map_from = bmna->node_name;
260             node_map_to = bmna->alias;
261         } else {
262             if (g_hash_table_contains(alias_map, bmna->alias)) {
263                 error_setg(errp, "The node alias '%s' is used twice",
264                            bmna->alias);
265                 goto fail;
266             }
267 
268             node_map_from = bmna->alias;
269             node_map_to = bmna->node_name;
270         }
271 
272         gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias;
273         bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
274                                             gdn);
275 
276         amin = g_new(AliasMapInnerNode, 1);
277         *amin = (AliasMapInnerNode){
278             .string = g_strdup(node_map_to),
279             .subtree = bitmaps_map,
280         };
281 
282         g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
283 
284         for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
285             const BitmapMigrationBitmapAlias *bmba = bmbal->value;
286             const char *bmap_map_from;
287 
288             if (strlen(bmba->alias) > UINT8_MAX) {
289                 error_setg(errp,
290                            "The bitmap alias '%s' is longer than %u bytes",
291                            bmba->alias, UINT8_MAX);
292                 goto fail;
293             }
294 
295             if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
296                 error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
297                            bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
298                 goto fail;
299             }
300 
301             if (name_to_alias) {
302                 bmap_map_from = bmba->name;
303 
304                 if (g_hash_table_contains(bitmaps_map, bmba->name)) {
305                     error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
306                                bmna->node_name, bmba->name);
307                     goto fail;
308                 }
309             } else {
310                 bmap_map_from = bmba->alias;
311 
312                 if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
313                     error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
314                                bmna->alias, bmba->alias);
315                     goto fail;
316                 }
317             }
318 
319             g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from),
320                                 QAPI_CLONE(BitmapMigrationBitmapAlias, bmba));
321         }
322     }
323 
324     return alias_map;
325 
326 fail:
327     g_hash_table_destroy(alias_map);
328     return NULL;
329 }
330 
331 /**
332  * Run construct_alias_map() in both directions to check whether @bbm
333  * is valid.
334  * (This function is to be used by migration/migration.c to validate
335  * the user-specified block-bitmap-mapping migration parameter.)
336  *
337  * Returns true if and only if the mapping is valid.
338  */
339 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
340                                       Error **errp)
341 {
342     GHashTable *alias_map;
343 
344     alias_map = construct_alias_map(bbm, true, errp);
345     if (!alias_map) {
346         return false;
347     }
348     g_hash_table_destroy(alias_map);
349 
350     alias_map = construct_alias_map(bbm, false, errp);
351     if (!alias_map) {
352         return false;
353     }
354     g_hash_table_destroy(alias_map);
355 
356     return true;
357 }
358 
359 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
360 {
361     uint8_t flags = qemu_get_byte(f);
362     if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
363         flags = flags << 8 | qemu_get_byte(f);
364         if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
365             flags = flags << 16 | qemu_get_be16(f);
366         }
367     }
368 
369     return flags;
370 }
371 
372 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
373 {
374     /* The code currently does not send flags as more than one byte */
375     assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
376 
377     qemu_put_byte(f, flags);
378 }
379 
380 static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
381                                SaveBitmapState *dbms, uint32_t additional_flags)
382 {
383     BlockDriverState *bs = dbms->bs;
384     BdrvDirtyBitmap *bitmap = dbms->bitmap;
385     uint32_t flags = additional_flags;
386     trace_send_bitmap_header_enter();
387 
388     if (bs != s->prev_bs) {
389         s->prev_bs = bs;
390         flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
391     }
392 
393     if (bitmap != s->prev_bitmap) {
394         s->prev_bitmap = bitmap;
395         flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
396     }
397 
398     qemu_put_bitmap_flags(f, flags);
399 
400     if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
401         qemu_put_counted_string(f, dbms->node_alias);
402     }
403 
404     if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
405         qemu_put_counted_string(f, dbms->bitmap_alias);
406     }
407 }
408 
409 static void send_bitmap_start(QEMUFile *f, DBMSaveState *s,
410                               SaveBitmapState *dbms)
411 {
412     send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START);
413     qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
414     qemu_put_byte(f, dbms->flags);
415 }
416 
417 static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s,
418                                  SaveBitmapState *dbms)
419 {
420     send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
421 }
422 
423 static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s,
424                              SaveBitmapState *dbms,
425                              uint64_t start_sector, uint32_t nr_sectors)
426 {
427     /* align for buffer_is_zero() */
428     uint64_t align = 4 * sizeof(long);
429     uint64_t unaligned_size =
430         bdrv_dirty_bitmap_serialization_size(
431             dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
432             (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
433     uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
434     uint8_t *buf = g_malloc0(buf_size);
435     uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
436 
437     bdrv_dirty_bitmap_serialize_part(
438         dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
439         (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
440 
441     if (buffer_is_zero(buf, buf_size)) {
442         g_free(buf);
443         buf = NULL;
444         flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
445     }
446 
447     trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
448 
449     send_bitmap_header(f, s, dbms, flags);
450 
451     qemu_put_be64(f, start_sector);
452     qemu_put_be32(f, nr_sectors);
453 
454     /* if a block is zero we need to flush here since the network
455      * bandwidth is now a lot higher than the storage device bandwidth.
456      * thus if we queue zero blocks we slow down the migration. */
457     if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
458         qemu_fflush(f);
459     } else {
460         qemu_put_be64(f, buf_size);
461         qemu_put_buffer(f, buf, buf_size);
462     }
463 
464     g_free(buf);
465 }
466 
467 /* Called with the BQL taken.  */
468 static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
469 {
470     SaveBitmapState *dbms;
471 
472     while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) {
473         QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
474         bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
475         bdrv_unref(dbms->bs);
476         g_free(dbms->node_alias);
477         g_free(dbms->bitmap_alias);
478         g_free(dbms);
479     }
480 }
481 
482 /* Called with the BQL taken. */
483 static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
484                                const char *bs_name, GHashTable *alias_map,
485                                Error **errp)
486 {
487     BdrvDirtyBitmap *bitmap;
488     SaveBitmapState *dbms;
489     GHashTable *bitmap_aliases;
490     const char *node_alias, *bitmap_name, *bitmap_alias;
491 
492     /* When an alias map is given, @bs_name must be @bs's node name */
493     assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
494 
495     FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
496         if (bdrv_dirty_bitmap_name(bitmap)) {
497             break;
498         }
499     }
500     if (!bitmap) {
501         return 0;
502     }
503 
504     bitmap_name = bdrv_dirty_bitmap_name(bitmap);
505 
506     if (!bs_name || strcmp(bs_name, "") == 0) {
507         error_setg(errp, "Bitmap '%s' in unnamed node can't be migrated",
508                    bitmap_name);
509         return -1;
510     }
511 
512     if (alias_map) {
513         const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
514 
515         if (!amin) {
516             /* Skip bitmaps on nodes with no alias */
517             return 0;
518         }
519 
520         node_alias = amin->string;
521         bitmap_aliases = amin->subtree;
522     } else {
523         node_alias = bs_name;
524         bitmap_aliases = NULL;
525     }
526 
527     if (node_alias[0] == '#') {
528         error_setg(errp, "Bitmap '%s' in a node with auto-generated "
529                    "name '%s' can't be migrated",
530                    bitmap_name, node_alias);
531         return -1;
532     }
533 
534     FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
535         BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL;
536         bitmap_name = bdrv_dirty_bitmap_name(bitmap);
537         if (!bitmap_name) {
538             continue;
539         }
540 
541         if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, errp)) {
542             return -1;
543         }
544 
545         if (bitmap_aliases) {
546             BitmapMigrationBitmapAlias *bmap_inner;
547 
548             bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name);
549             if (!bmap_inner) {
550                 /* Skip bitmaps with no alias */
551                 continue;
552             }
553 
554             bitmap_alias = bmap_inner->alias;
555             if (bmap_inner->transform) {
556                 bitmap_transform = bmap_inner->transform;
557             }
558         } else {
559             if (strlen(bitmap_name) > UINT8_MAX) {
560                 error_setg(errp, "Cannot migrate bitmap '%s' on node '%s': "
561                            "Name is longer than %u bytes",
562                            bitmap_name, bs_name, UINT8_MAX);
563                 return -1;
564             }
565             bitmap_alias = bitmap_name;
566         }
567 
568         bdrv_ref(bs);
569         bdrv_dirty_bitmap_set_busy(bitmap, true);
570 
571         dbms = g_new0(SaveBitmapState, 1);
572         dbms->bs = bs;
573         dbms->node_alias = g_strdup(node_alias);
574         dbms->bitmap_alias = g_strdup(bitmap_alias);
575         dbms->bitmap = bitmap;
576         dbms->total_sectors = bdrv_nb_sectors(bs);
577         dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU *
578             (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS);
579         assert(dbms->sectors_per_chunk != 0);
580         if (bdrv_dirty_bitmap_enabled(bitmap)) {
581             dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
582         }
583         if (bitmap_transform &&
584             bitmap_transform->has_persistent) {
585             if (bitmap_transform->persistent) {
586                 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
587             }
588         } else {
589             if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
590                 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
591             }
592         }
593 
594         QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry);
595     }
596 
597     return 0;
598 }
599 
600 /* Called with the BQL taken. */
601 static int init_dirty_bitmap_migration(DBMSaveState *s, Error **errp)
602 {
603     BlockDriverState *bs;
604     SaveBitmapState *dbms;
605     GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
606     BlockBackend *blk;
607     GHashTable *alias_map = NULL;
608 
609     /* Runs in the migration thread, but holds the BQL */
610     GLOBAL_STATE_CODE();
611     GRAPH_RDLOCK_GUARD_MAINLOOP();
612 
613     if (migrate_has_block_bitmap_mapping()) {
614         alias_map = construct_alias_map(migrate_block_bitmap_mapping(), true,
615                                         &error_abort);
616     }
617 
618     s->bulk_completed = false;
619     s->prev_bs = NULL;
620     s->prev_bitmap = NULL;
621     s->no_bitmaps = false;
622 
623     if (!alias_map) {
624         /*
625          * Use blockdevice name for direct (or filtered) children of named block
626          * backends.
627          */
628         for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
629             const char *name = blk_name(blk);
630 
631             if (!name || strcmp(name, "") == 0) {
632                 continue;
633             }
634 
635             bs = blk_bs(blk);
636 
637             /* Skip filters without bitmaps */
638             while (bs && bs->drv && bs->drv->is_filter &&
639                    !bdrv_has_named_bitmaps(bs))
640             {
641                 bs = bdrv_filter_bs(bs);
642             }
643 
644             if (bs && bs->drv && !bs->drv->is_filter) {
645                 if (add_bitmaps_to_list(s, bs, name, NULL, errp)) {
646                     goto fail;
647                 }
648                 g_hash_table_add(handled_by_blk, bs);
649             }
650         }
651     }
652 
653     for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
654         if (g_hash_table_contains(handled_by_blk, bs)) {
655             continue;
656         }
657 
658         if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map,
659                                 errp)) {
660             goto fail;
661         }
662     }
663 
664     /* unset migration flags here, to not roll back it */
665     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
666         bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
667     }
668 
669     if (QSIMPLEQ_EMPTY(&s->dbms_list)) {
670         s->no_bitmaps = true;
671     }
672 
673     g_hash_table_destroy(handled_by_blk);
674     if (alias_map) {
675         g_hash_table_destroy(alias_map);
676     }
677 
678     return 0;
679 
680 fail:
681     g_hash_table_destroy(handled_by_blk);
682     if (alias_map) {
683         g_hash_table_destroy(alias_map);
684     }
685     dirty_bitmap_do_save_cleanup(s);
686 
687     return -1;
688 }
689 
690 /* Called with no lock taken.  */
691 static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s,
692                                   SaveBitmapState *dbms)
693 {
694     uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
695                              dbms->sectors_per_chunk);
696 
697     send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors);
698 
699     dbms->cur_sector += nr_sectors;
700     if (dbms->cur_sector >= dbms->total_sectors) {
701         dbms->bulk_completed = true;
702     }
703 }
704 
705 /* Called with no lock taken.  */
706 static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit)
707 {
708     SaveBitmapState *dbms;
709 
710     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
711         while (!dbms->bulk_completed) {
712             bulk_phase_send_chunk(f, s, dbms);
713             if (limit && migration_rate_exceeded(f)) {
714                 return;
715             }
716         }
717     }
718 
719     s->bulk_completed = true;
720 }
721 
722 /* for SaveVMHandlers */
723 static void dirty_bitmap_save_cleanup(void *opaque)
724 {
725     DBMSaveState *s = &((DBMState *)opaque)->save;
726 
727     dirty_bitmap_do_save_cleanup(s);
728 }
729 
730 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
731 {
732     DBMSaveState *s = &((DBMState *)opaque)->save;
733 
734     trace_dirty_bitmap_save_iterate(migration_in_postcopy());
735 
736     if (migration_in_postcopy() && !s->bulk_completed) {
737         bulk_phase(f, s, true);
738     }
739 
740     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
741 
742     return s->bulk_completed;
743 }
744 
745 /* Called with the BQL taken.  */
746 
747 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
748 {
749     DBMSaveState *s = &((DBMState *)opaque)->save;
750     SaveBitmapState *dbms;
751     trace_dirty_bitmap_save_complete_enter();
752 
753     if (!s->bulk_completed) {
754         bulk_phase(f, s, false);
755     }
756 
757     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
758         send_bitmap_complete(f, s, dbms);
759     }
760 
761     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
762 
763     trace_dirty_bitmap_save_complete_finish();
764 
765     dirty_bitmap_save_cleanup(opaque);
766     return 0;
767 }
768 
769 static void dirty_bitmap_state_pending(void *opaque,
770                                        uint64_t *must_precopy,
771                                        uint64_t *can_postcopy)
772 {
773     DBMSaveState *s = &((DBMState *)opaque)->save;
774     SaveBitmapState *dbms;
775     uint64_t pending = 0;
776 
777     bql_lock();
778 
779     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
780         uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
781         uint64_t sectors = dbms->bulk_completed ? 0 :
782                            dbms->total_sectors - dbms->cur_sector;
783 
784         pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
785     }
786 
787     bql_unlock();
788 
789     trace_dirty_bitmap_state_pending(pending);
790 
791     *can_postcopy += pending;
792 }
793 
794 /* First occurrence of this bitmap. It should be created if doesn't exist */
795 static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s)
796 {
797     Error *local_err = NULL;
798     uint32_t granularity = qemu_get_be32(f);
799     uint8_t flags = qemu_get_byte(f);
800     LoadBitmapState *b;
801     bool persistent;
802 
803     if (s->cancelled) {
804         return 0;
805     }
806 
807     if (s->bitmap) {
808         error_report("Bitmap with the same name ('%s') already exists on "
809                      "destination", bdrv_dirty_bitmap_name(s->bitmap));
810         return -EINVAL;
811     } else {
812         s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
813                                              s->bitmap_name, &local_err);
814         if (!s->bitmap) {
815             error_report_err(local_err);
816             return -EINVAL;
817         }
818     }
819 
820     if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
821         error_report("Unknown flags in migrated dirty bitmap header: %x",
822                      flags);
823         return -EINVAL;
824     }
825 
826     if (s->bmap_inner &&
827         s->bmap_inner->transform &&
828         s->bmap_inner->transform->has_persistent) {
829         persistent = s->bmap_inner->transform->persistent;
830     } else {
831         persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
832     }
833 
834     if (persistent) {
835         bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
836     }
837 
838     bdrv_disable_dirty_bitmap(s->bitmap);
839     if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
840         bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
841         if (local_err) {
842             error_report_err(local_err);
843             return -EINVAL;
844         }
845     } else {
846         bdrv_dirty_bitmap_set_busy(s->bitmap, true);
847     }
848 
849     b = g_new(LoadBitmapState, 1);
850     b->bs = s->bs;
851     b->bitmap = s->bitmap;
852     b->migrated = false;
853     b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
854 
855     s->bitmaps = g_slist_prepend(s->bitmaps, b);
856 
857     return 0;
858 }
859 
860 /*
861  * before_vm_start_handle_item
862  *
863  * g_slist_foreach helper
864  *
865  * item is LoadBitmapState*
866  * opaque is DBMLoadState*
867  */
868 static void before_vm_start_handle_item(void *item, void *opaque)
869 {
870     DBMLoadState *s = opaque;
871     LoadBitmapState *b = item;
872 
873     if (b->enabled) {
874         if (b->migrated) {
875             bdrv_enable_dirty_bitmap(b->bitmap);
876         } else {
877             bdrv_dirty_bitmap_enable_successor(b->bitmap);
878         }
879     }
880 
881     if (b->migrated) {
882         s->bitmaps = g_slist_remove(s->bitmaps, b);
883         g_free(b);
884     }
885 }
886 
887 void dirty_bitmap_mig_before_vm_start(void)
888 {
889     DBMLoadState *s = &dbm_state.load;
890     qemu_mutex_lock(&s->lock);
891 
892     assert(!s->before_vm_start_handled);
893     g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s);
894     s->before_vm_start_handled = true;
895 
896     qemu_mutex_unlock(&s->lock);
897 }
898 
899 static void cancel_incoming_locked(DBMLoadState *s)
900 {
901     GSList *item;
902 
903     if (s->cancelled) {
904         return;
905     }
906 
907     s->cancelled = true;
908     s->bs = NULL;
909     s->bitmap = NULL;
910 
911     /* Drop all unfinished bitmaps */
912     for (item = s->bitmaps; item; item = g_slist_next(item)) {
913         LoadBitmapState *b = item->data;
914 
915         /*
916          * Bitmap must be unfinished, as finished bitmaps should already be
917          * removed from the list.
918          */
919         assert(!s->before_vm_start_handled || !b->migrated);
920         if (bdrv_dirty_bitmap_has_successor(b->bitmap)) {
921             bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort);
922         } else {
923             bdrv_dirty_bitmap_set_busy(b->bitmap, false);
924         }
925         bdrv_release_dirty_bitmap(b->bitmap);
926     }
927 
928     g_slist_free_full(s->bitmaps, g_free);
929     s->bitmaps = NULL;
930 }
931 
932 void dirty_bitmap_mig_cancel_outgoing(void)
933 {
934     dirty_bitmap_do_save_cleanup(&dbm_state.save);
935 }
936 
937 void dirty_bitmap_mig_cancel_incoming(void)
938 {
939     DBMLoadState *s = &dbm_state.load;
940 
941     qemu_mutex_lock(&s->lock);
942 
943     cancel_incoming_locked(s);
944 
945     qemu_mutex_unlock(&s->lock);
946 }
947 
948 static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
949 {
950     GSList *item;
951     trace_dirty_bitmap_load_complete();
952 
953     if (s->cancelled) {
954         return;
955     }
956 
957     bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
958 
959     if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
960         bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
961     } else {
962         bdrv_dirty_bitmap_set_busy(s->bitmap, false);
963     }
964 
965     for (item = s->bitmaps; item; item = g_slist_next(item)) {
966         LoadBitmapState *b = item->data;
967 
968         if (b->bitmap == s->bitmap) {
969             b->migrated = true;
970             if (s->before_vm_start_handled) {
971                 s->bitmaps = g_slist_remove(s->bitmaps, b);
972                 g_free(b);
973             }
974             break;
975         }
976     }
977 }
978 
979 static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
980 {
981     uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
982     uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
983     trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
984                                        nr_bytes >> BDRV_SECTOR_BITS);
985 
986     if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
987         trace_dirty_bitmap_load_bits_zeroes();
988         if (!s->cancelled) {
989             bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte,
990                                                  nr_bytes, false);
991         }
992     } else {
993         size_t ret;
994         g_autofree uint8_t *buf = NULL;
995         uint64_t buf_size = qemu_get_be64(f);
996         uint64_t needed_size;
997 
998         /*
999          * The actual check for buf_size is done a bit later. We can't do it in
1000          * cancelled mode as we don't have the bitmap to check the constraints
1001          * (so, we allocate a buffer and read prior to the check). On the other
1002          * hand, we shouldn't blindly g_malloc the number from the stream.
1003          * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow
1004          * a bit larger (which means that bitmap migration will fail anyway and
1005          * the whole migration will most probably fail soon due to broken
1006          * stream).
1007          */
1008         if (buf_size > 10 * CHUNK_SIZE) {
1009             error_report("Bitmap migration stream buffer allocation request "
1010                          "is too large");
1011             return -EIO;
1012         }
1013 
1014         buf = g_malloc(buf_size);
1015         ret = qemu_get_buffer(f, buf, buf_size);
1016         if (ret != buf_size) {
1017             error_report("Failed to read bitmap bits");
1018             return -EIO;
1019         }
1020 
1021         if (s->cancelled) {
1022             return 0;
1023         }
1024 
1025         needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
1026                                                            first_byte,
1027                                                            nr_bytes);
1028 
1029         if (needed_size > buf_size ||
1030             buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
1031              /* Here used same alignment as in send_bitmap_bits */
1032         ) {
1033             error_report("Migrated bitmap granularity doesn't "
1034                          "match the destination bitmap '%s' granularity",
1035                          bdrv_dirty_bitmap_name(s->bitmap));
1036             cancel_incoming_locked(s);
1037             return 0;
1038         }
1039 
1040         bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
1041                                            false);
1042     }
1043 
1044     return 0;
1045 }
1046 
1047 static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
1048                                     GHashTable *alias_map)
1049 {
1050     GHashTable *bitmap_alias_map = NULL;
1051     Error *local_err = NULL;
1052     bool nothing;
1053     s->flags = qemu_get_bitmap_flags(f);
1054     trace_dirty_bitmap_load_header(s->flags);
1055 
1056     nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
1057 
1058     if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
1059         if (!qemu_get_counted_string(f, s->node_alias)) {
1060             error_report("Unable to read node alias string");
1061             return -EINVAL;
1062         }
1063 
1064         if (!s->cancelled) {
1065             if (alias_map) {
1066                 const AliasMapInnerNode *amin;
1067 
1068                 amin = g_hash_table_lookup(alias_map, s->node_alias);
1069                 if (!amin) {
1070                     error_setg(&local_err, "Error: Unknown node alias '%s'",
1071                                s->node_alias);
1072                     s->bs = NULL;
1073                 } else {
1074                     bitmap_alias_map = amin->subtree;
1075                     s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
1076                 }
1077             } else {
1078                 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
1079                                        &local_err);
1080             }
1081             if (!s->bs) {
1082                 error_report_err(local_err);
1083                 cancel_incoming_locked(s);
1084             }
1085         }
1086     } else if (s->bs) {
1087         if (alias_map) {
1088             const AliasMapInnerNode *amin;
1089 
1090             /* Must be present in the map, or s->bs would not be set */
1091             amin = g_hash_table_lookup(alias_map, s->node_alias);
1092             assert(amin != NULL);
1093 
1094             bitmap_alias_map = amin->subtree;
1095         }
1096     } else if (!nothing && !s->cancelled) {
1097         error_report("Error: block device name is not set");
1098         cancel_incoming_locked(s);
1099     }
1100 
1101     assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
1102 
1103     if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
1104         const char *bitmap_name;
1105 
1106         if (!qemu_get_counted_string(f, s->bitmap_alias)) {
1107             error_report("Unable to read bitmap alias string");
1108             return -EINVAL;
1109         }
1110 
1111         bitmap_name = s->bitmap_alias;
1112         if (!s->cancelled && bitmap_alias_map) {
1113             BitmapMigrationBitmapAlias *bmap_inner;
1114 
1115             bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias);
1116             if (!bmap_inner) {
1117                 error_report("Error: Unknown bitmap alias '%s' on node "
1118                              "'%s' (alias '%s')", s->bitmap_alias,
1119                              s->bs->node_name, s->node_alias);
1120                 cancel_incoming_locked(s);
1121             } else {
1122                 bitmap_name = bmap_inner->name;
1123             }
1124 
1125             s->bmap_inner = bmap_inner;
1126         }
1127 
1128         if (!s->cancelled) {
1129             g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
1130             s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
1131 
1132             /*
1133              * bitmap may be NULL here, it wouldn't be an error if it is the
1134              * first occurrence of the bitmap
1135              */
1136             if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
1137                 error_report("Error: unknown dirty bitmap "
1138                              "'%s' for block device '%s'",
1139                              s->bitmap_name, s->bs->node_name);
1140                 cancel_incoming_locked(s);
1141             }
1142         }
1143     } else if (!s->bitmap && !nothing && !s->cancelled) {
1144         error_report("Error: block device name is not set");
1145         cancel_incoming_locked(s);
1146     }
1147 
1148     return 0;
1149 }
1150 
1151 /*
1152  * dirty_bitmap_load
1153  *
1154  * Load sequence of dirty bitmap chunks. Return error only on fatal io stream
1155  * violations. On other errors just cancel bitmaps incoming migration and return
1156  * 0.
1157  *
1158  * Note, than when incoming bitmap migration is canceled, we still must read all
1159  * our chunks (and just ignore them), to not affect other migration objects.
1160  */
1161 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
1162 {
1163     GHashTable *alias_map = NULL;
1164     DBMLoadState *s = &((DBMState *)opaque)->load;
1165     int ret = 0;
1166 
1167     trace_dirty_bitmap_load_enter();
1168 
1169     if (version_id != 1) {
1170         QEMU_LOCK_GUARD(&s->lock);
1171         cancel_incoming_locked(s);
1172         return -EINVAL;
1173     }
1174 
1175     if (migrate_has_block_bitmap_mapping()) {
1176         alias_map = construct_alias_map(migrate_block_bitmap_mapping(), false,
1177                                         &error_abort);
1178     }
1179 
1180     do {
1181         QEMU_LOCK_GUARD(&s->lock);
1182 
1183         ret = dirty_bitmap_load_header(f, s, alias_map);
1184         if (ret < 0) {
1185             cancel_incoming_locked(s);
1186             goto fail;
1187         }
1188 
1189         if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
1190             ret = dirty_bitmap_load_start(f, s);
1191         } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
1192             dirty_bitmap_load_complete(f, s);
1193         } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
1194             ret = dirty_bitmap_load_bits(f, s);
1195         }
1196 
1197         if (!ret) {
1198             ret = qemu_file_get_error(f);
1199         }
1200 
1201         if (ret) {
1202             cancel_incoming_locked(s);
1203             goto fail;
1204         }
1205     } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
1206 
1207     trace_dirty_bitmap_load_success();
1208     ret = 0;
1209 fail:
1210     if (alias_map) {
1211         g_hash_table_destroy(alias_map);
1212     }
1213     return ret;
1214 }
1215 
1216 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque, Error **errp)
1217 {
1218     DBMSaveState *s = &((DBMState *)opaque)->save;
1219     SaveBitmapState *dbms = NULL;
1220 
1221     if (init_dirty_bitmap_migration(s, errp) < 0) {
1222         return -1;
1223     }
1224 
1225     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
1226         send_bitmap_start(f, s, dbms);
1227     }
1228     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
1229     return 0;
1230 }
1231 
1232 static bool dirty_bitmap_is_active(void *opaque)
1233 {
1234     DBMSaveState *s = &((DBMState *)opaque)->save;
1235 
1236     return migrate_dirty_bitmaps() && !s->no_bitmaps;
1237 }
1238 
1239 static bool dirty_bitmap_is_active_iterate(void *opaque)
1240 {
1241     return dirty_bitmap_is_active(opaque) && !runstate_is_running();
1242 }
1243 
1244 static bool dirty_bitmap_has_postcopy(void *opaque)
1245 {
1246     return true;
1247 }
1248 
1249 static SaveVMHandlers savevm_dirty_bitmap_handlers = {
1250     .save_setup = dirty_bitmap_save_setup,
1251     .save_live_complete_postcopy = dirty_bitmap_save_complete,
1252     .save_live_complete_precopy = dirty_bitmap_save_complete,
1253     .has_postcopy = dirty_bitmap_has_postcopy,
1254     .state_pending_exact = dirty_bitmap_state_pending,
1255     .state_pending_estimate = dirty_bitmap_state_pending,
1256     .save_live_iterate = dirty_bitmap_save_iterate,
1257     .is_active_iterate = dirty_bitmap_is_active_iterate,
1258     .load_state = dirty_bitmap_load,
1259     .save_cleanup = dirty_bitmap_save_cleanup,
1260     .is_active = dirty_bitmap_is_active,
1261 };
1262 
1263 void dirty_bitmap_mig_init(void)
1264 {
1265     QSIMPLEQ_INIT(&dbm_state.save.dbms_list);
1266     qemu_mutex_init(&dbm_state.load.lock);
1267 
1268     register_savevm_live("dirty-bitmap", 0, 1,
1269                          &savevm_dirty_bitmap_handlers,
1270                          &dbm_state);
1271 }
1272