1 /*
2  * Block dirty bitmap postcopy migration
3  *
4  * Copyright IBM, Corp. 2009
5  * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
6  *
7  * Authors:
8  *  Liran Schour   <lirans@il.ibm.com>
9  *  Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
10  *
11  * This work is licensed under the terms of the GNU GPL, version 2.  See
12  * the COPYING file in the top-level directory.
13  * This file is derived from migration/block.c, so it's author and IBM copyright
14  * are here, although content is quite different.
15  *
16  * Contributions after 2012-01-13 are licensed under the terms of the
17  * GNU GPL, version 2 or (at your option) any later version.
18  *
19  *                                ***
20  *
21  * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable
22  * bitmaps are migrated.
23  *
24  * Bitmap migration implies creating bitmap with the same name and granularity
25  * in destination QEMU. If the bitmap with the same name (for the same node)
26  * already exists on destination an error will be generated.
27  *
28  * format of migration:
29  *
30  * # Header (shared for different chunk types)
31  * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
32  * [ 1 byte: node alias size ] \  flags & DEVICE_NAME
33  * [ n bytes: node alias     ] /
34  * [ 1 byte: bitmap alias size ] \  flags & BITMAP_NAME
35  * [ n bytes: bitmap alias     ] /
36  *
37  * # Start of bitmap migration (flags & START)
38  * header
39  * be64: granularity
40  * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap)
41  *   bit 0    -  bitmap is enabled
42  *   bit 1    -  bitmap is persistent
43  *   bit 2    -  bitmap is autoloading
44  *   bits 3-7 - reserved, must be zero
45  *
46  * # Complete of bitmap migration (flags & COMPLETE)
47  * header
48  *
49  * # Data chunk of bitmap migration
50  * header
51  * be64: start sector
52  * be32: number of sectors
53  * [ be64: buffer size  ] \ ! (flags & ZEROES)
54  * [ n bytes: buffer    ] /
55  *
56  * The last chunk in stream should contain flags & EOS. The chunk may skip
57  * device and/or bitmap names, assuming them to be the same with the previous
58  * chunk.
59  */
60 
61 #include "qemu/osdep.h"
62 #include "block/block.h"
63 #include "block/block_int.h"
64 #include "sysemu/block-backend.h"
65 #include "sysemu/runstate.h"
66 #include "qemu/main-loop.h"
67 #include "qemu/error-report.h"
68 #include "migration/misc.h"
69 #include "migration/migration.h"
70 #include "qemu-file.h"
71 #include "migration/vmstate.h"
72 #include "migration/register.h"
73 #include "qemu/hbitmap.h"
74 #include "qemu/cutils.h"
75 #include "qemu/id.h"
76 #include "qapi/error.h"
77 #include "qapi/qapi-commands-migration.h"
78 #include "qapi/qapi-visit-migration.h"
79 #include "qapi/clone-visitor.h"
80 #include "trace.h"
81 
82 #define CHUNK_SIZE     (1 << 10)
83 
84 /* Flags occupy one, two or four bytes (Big Endian). The size is determined as
85  * follows:
86  * in first (most significant) byte bit 8 is clear  -->  one byte
87  * in first byte bit 8 is set    -->  two or four bytes, depending on second
88  *                                    byte:
89  *    | in second byte bit 8 is clear  -->  two bytes
90  *    | in second byte bit 8 is set    -->  four bytes
91  */
92 #define DIRTY_BITMAP_MIG_FLAG_EOS           0x01
93 #define DIRTY_BITMAP_MIG_FLAG_ZEROES        0x02
94 #define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME   0x04
95 #define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME   0x08
96 #define DIRTY_BITMAP_MIG_FLAG_START         0x10
97 #define DIRTY_BITMAP_MIG_FLAG_COMPLETE      0x20
98 #define DIRTY_BITMAP_MIG_FLAG_BITS          0x40
99 
100 #define DIRTY_BITMAP_MIG_EXTRA_FLAGS        0x80
101 
102 #define DIRTY_BITMAP_MIG_START_FLAG_ENABLED          0x01
103 #define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT       0x02
104 /* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */
105 #define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK    0xf8
106 
107 /* State of one bitmap during save process */
108 typedef struct SaveBitmapState {
109     /* Written during setup phase. */
110     BlockDriverState *bs;
111     char *node_alias;
112     char *bitmap_alias;
113     BdrvDirtyBitmap *bitmap;
114     uint64_t total_sectors;
115     uint64_t sectors_per_chunk;
116     QSIMPLEQ_ENTRY(SaveBitmapState) entry;
117     uint8_t flags;
118 
119     /* For bulk phase. */
120     bool bulk_completed;
121     uint64_t cur_sector;
122 } SaveBitmapState;
123 
124 /* State of the dirty bitmap migration (DBM) during save process */
125 typedef struct DBMSaveState {
126     QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list;
127 
128     bool bulk_completed;
129     bool no_bitmaps;
130 
131     /* for send_bitmap_bits() */
132     BlockDriverState *prev_bs;
133     BdrvDirtyBitmap *prev_bitmap;
134 } DBMSaveState;
135 
136 typedef struct LoadBitmapState {
137     BlockDriverState *bs;
138     BdrvDirtyBitmap *bitmap;
139     bool migrated;
140     bool enabled;
141 } LoadBitmapState;
142 
143 /* State of the dirty bitmap migration (DBM) during load process */
144 typedef struct DBMLoadState {
145     uint32_t flags;
146     char node_alias[256];
147     char bitmap_alias[256];
148     char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
149     BlockDriverState *bs;
150     BdrvDirtyBitmap *bitmap;
151 
152     bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */
153     BitmapMigrationBitmapAlias *bmap_inner;
154 
155     /*
156      * cancelled
157      * Incoming migration is cancelled for some reason. That means that we
158      * still should read our chunks from migration stream, to not affect other
159      * migration objects (like RAM), but just ignore them and do not touch any
160      * bitmaps or nodes.
161      */
162     bool cancelled;
163 
164     GSList *bitmaps;
165     QemuMutex lock; /* protect bitmaps */
166 } DBMLoadState;
167 
168 typedef struct DBMState {
169     DBMSaveState save;
170     DBMLoadState load;
171 } DBMState;
172 
173 static DBMState dbm_state;
174 
175 /* For hash tables that map node/bitmap names to aliases */
176 typedef struct AliasMapInnerNode {
177     char *string;
178     GHashTable *subtree;
179 } AliasMapInnerNode;
180 
181 static void free_alias_map_inner_node(void *amin_ptr)
182 {
183     AliasMapInnerNode *amin = amin_ptr;
184 
185     g_free(amin->string);
186     g_hash_table_unref(amin->subtree);
187     g_free(amin);
188 }
189 
190 /**
191  * Construct an alias map based on the given QMP structure.
192  *
193  * (Note that we cannot store such maps in the MigrationParameters
194  * object, because that struct is defined by the QAPI schema, which
195  * makes it basically impossible to have dicts with arbitrary keys.
196  * Therefore, we instead have to construct these maps when migration
197  * starts.)
198  *
199  * @bbm is the block_bitmap_mapping from the migration parameters.
200  *
201  * If @name_to_alias is true, the returned hash table will map node
202  * and bitmap names to their respective aliases (for outgoing
203  * migration).
204  *
205  * If @name_to_alias is false, the returned hash table will map node
206  * and bitmap aliases to their respective names (for incoming
207  * migration).
208  *
209  * The hash table maps node names/aliases to AliasMapInnerNode
210  * objects, whose .string is the respective node alias/name, and whose
211  * .subtree table maps bitmap names/aliases to the respective bitmap
212  * alias/name.
213  */
214 static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
215                                        bool name_to_alias,
216                                        Error **errp)
217 {
218     GHashTable *alias_map;
219     size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
220 
221     alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
222                                       g_free, free_alias_map_inner_node);
223 
224     for (; bbm; bbm = bbm->next) {
225         const BitmapMigrationNodeAlias *bmna = bbm->value;
226         const BitmapMigrationBitmapAliasList *bmbal;
227         AliasMapInnerNode *amin;
228         GHashTable *bitmaps_map;
229         const char *node_map_from, *node_map_to;
230         GDestroyNotify gdn;
231 
232         if (!id_wellformed(bmna->alias)) {
233             error_setg(errp, "The node alias '%s' is not well-formed",
234                        bmna->alias);
235             goto fail;
236         }
237 
238         if (strlen(bmna->alias) > UINT8_MAX) {
239             error_setg(errp, "The node alias '%s' is longer than %u bytes",
240                        bmna->alias, UINT8_MAX);
241             goto fail;
242         }
243 
244         if (strlen(bmna->node_name) > max_node_name_len) {
245             error_setg(errp, "The node name '%s' is longer than %zu bytes",
246                        bmna->node_name, max_node_name_len);
247             goto fail;
248         }
249 
250         if (name_to_alias) {
251             if (g_hash_table_contains(alias_map, bmna->node_name)) {
252                 error_setg(errp, "The node name '%s' is mapped twice",
253                            bmna->node_name);
254                 goto fail;
255             }
256 
257             node_map_from = bmna->node_name;
258             node_map_to = bmna->alias;
259         } else {
260             if (g_hash_table_contains(alias_map, bmna->alias)) {
261                 error_setg(errp, "The node alias '%s' is used twice",
262                            bmna->alias);
263                 goto fail;
264             }
265 
266             node_map_from = bmna->alias;
267             node_map_to = bmna->node_name;
268         }
269 
270         gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias;
271         bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
272                                             gdn);
273 
274         amin = g_new(AliasMapInnerNode, 1);
275         *amin = (AliasMapInnerNode){
276             .string = g_strdup(node_map_to),
277             .subtree = bitmaps_map,
278         };
279 
280         g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
281 
282         for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
283             const BitmapMigrationBitmapAlias *bmba = bmbal->value;
284             const char *bmap_map_from;
285 
286             if (strlen(bmba->alias) > UINT8_MAX) {
287                 error_setg(errp,
288                            "The bitmap alias '%s' is longer than %u bytes",
289                            bmba->alias, UINT8_MAX);
290                 goto fail;
291             }
292 
293             if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
294                 error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
295                            bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
296                 goto fail;
297             }
298 
299             if (name_to_alias) {
300                 bmap_map_from = bmba->name;
301 
302                 if (g_hash_table_contains(bitmaps_map, bmba->name)) {
303                     error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
304                                bmna->node_name, bmba->name);
305                     goto fail;
306                 }
307             } else {
308                 bmap_map_from = bmba->alias;
309 
310                 if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
311                     error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
312                                bmna->alias, bmba->alias);
313                     goto fail;
314                 }
315             }
316 
317             g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from),
318                                 QAPI_CLONE(BitmapMigrationBitmapAlias, bmba));
319         }
320     }
321 
322     return alias_map;
323 
324 fail:
325     g_hash_table_destroy(alias_map);
326     return NULL;
327 }
328 
329 /**
330  * Run construct_alias_map() in both directions to check whether @bbm
331  * is valid.
332  * (This function is to be used by migration/migration.c to validate
333  * the user-specified block-bitmap-mapping migration parameter.)
334  *
335  * Returns true if and only if the mapping is valid.
336  */
337 bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
338                                       Error **errp)
339 {
340     GHashTable *alias_map;
341 
342     alias_map = construct_alias_map(bbm, true, errp);
343     if (!alias_map) {
344         return false;
345     }
346     g_hash_table_destroy(alias_map);
347 
348     alias_map = construct_alias_map(bbm, false, errp);
349     if (!alias_map) {
350         return false;
351     }
352     g_hash_table_destroy(alias_map);
353 
354     return true;
355 }
356 
357 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
358 {
359     uint8_t flags = qemu_get_byte(f);
360     if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
361         flags = flags << 8 | qemu_get_byte(f);
362         if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
363             flags = flags << 16 | qemu_get_be16(f);
364         }
365     }
366 
367     return flags;
368 }
369 
370 static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
371 {
372     /* The code currently does not send flags as more than one byte */
373     assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
374 
375     qemu_put_byte(f, flags);
376 }
377 
378 static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
379                                SaveBitmapState *dbms, uint32_t additional_flags)
380 {
381     BlockDriverState *bs = dbms->bs;
382     BdrvDirtyBitmap *bitmap = dbms->bitmap;
383     uint32_t flags = additional_flags;
384     trace_send_bitmap_header_enter();
385 
386     if (bs != s->prev_bs) {
387         s->prev_bs = bs;
388         flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
389     }
390 
391     if (bitmap != s->prev_bitmap) {
392         s->prev_bitmap = bitmap;
393         flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
394     }
395 
396     qemu_put_bitmap_flags(f, flags);
397 
398     if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
399         qemu_put_counted_string(f, dbms->node_alias);
400     }
401 
402     if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
403         qemu_put_counted_string(f, dbms->bitmap_alias);
404     }
405 }
406 
407 static void send_bitmap_start(QEMUFile *f, DBMSaveState *s,
408                               SaveBitmapState *dbms)
409 {
410     send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START);
411     qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
412     qemu_put_byte(f, dbms->flags);
413 }
414 
415 static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s,
416                                  SaveBitmapState *dbms)
417 {
418     send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
419 }
420 
421 static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s,
422                              SaveBitmapState *dbms,
423                              uint64_t start_sector, uint32_t nr_sectors)
424 {
425     /* align for buffer_is_zero() */
426     uint64_t align = 4 * sizeof(long);
427     uint64_t unaligned_size =
428         bdrv_dirty_bitmap_serialization_size(
429             dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
430             (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
431     uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
432     uint8_t *buf = g_malloc0(buf_size);
433     uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
434 
435     bdrv_dirty_bitmap_serialize_part(
436         dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
437         (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
438 
439     if (buffer_is_zero(buf, buf_size)) {
440         g_free(buf);
441         buf = NULL;
442         flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
443     }
444 
445     trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
446 
447     send_bitmap_header(f, s, dbms, flags);
448 
449     qemu_put_be64(f, start_sector);
450     qemu_put_be32(f, nr_sectors);
451 
452     /* if a block is zero we need to flush here since the network
453      * bandwidth is now a lot higher than the storage device bandwidth.
454      * thus if we queue zero blocks we slow down the migration. */
455     if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
456         qemu_fflush(f);
457     } else {
458         qemu_put_be64(f, buf_size);
459         qemu_put_buffer(f, buf, buf_size);
460     }
461 
462     g_free(buf);
463 }
464 
465 /* Called with iothread lock taken.  */
466 static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
467 {
468     SaveBitmapState *dbms;
469 
470     while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) {
471         QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
472         bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
473         bdrv_unref(dbms->bs);
474         g_free(dbms->node_alias);
475         g_free(dbms->bitmap_alias);
476         g_free(dbms);
477     }
478 }
479 
480 /* Called with iothread lock taken. */
481 static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
482                                const char *bs_name, GHashTable *alias_map)
483 {
484     BdrvDirtyBitmap *bitmap;
485     SaveBitmapState *dbms;
486     GHashTable *bitmap_aliases;
487     const char *node_alias, *bitmap_name, *bitmap_alias;
488     Error *local_err = NULL;
489 
490     /* When an alias map is given, @bs_name must be @bs's node name */
491     assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
492 
493     FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
494         if (bdrv_dirty_bitmap_name(bitmap)) {
495             break;
496         }
497     }
498     if (!bitmap) {
499         return 0;
500     }
501 
502     bitmap_name = bdrv_dirty_bitmap_name(bitmap);
503 
504     if (!bs_name || strcmp(bs_name, "") == 0) {
505         error_report("Bitmap '%s' in unnamed node can't be migrated",
506                      bitmap_name);
507         return -1;
508     }
509 
510     if (alias_map) {
511         const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
512 
513         if (!amin) {
514             /* Skip bitmaps on nodes with no alias */
515             return 0;
516         }
517 
518         node_alias = amin->string;
519         bitmap_aliases = amin->subtree;
520     } else {
521         node_alias = bs_name;
522         bitmap_aliases = NULL;
523     }
524 
525     if (node_alias[0] == '#') {
526         error_report("Bitmap '%s' in a node with auto-generated "
527                      "name '%s' can't be migrated",
528                      bitmap_name, node_alias);
529         return -1;
530     }
531 
532     FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
533         BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL;
534         bitmap_name = bdrv_dirty_bitmap_name(bitmap);
535         if (!bitmap_name) {
536             continue;
537         }
538 
539         if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
540             error_report_err(local_err);
541             return -1;
542         }
543 
544         if (bitmap_aliases) {
545             BitmapMigrationBitmapAlias *bmap_inner;
546 
547             bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name);
548             if (!bmap_inner) {
549                 /* Skip bitmaps with no alias */
550                 continue;
551             }
552 
553             bitmap_alias = bmap_inner->alias;
554             if (bmap_inner->transform) {
555                 bitmap_transform = bmap_inner->transform;
556             }
557         } else {
558             if (strlen(bitmap_name) > UINT8_MAX) {
559                 error_report("Cannot migrate bitmap '%s' on node '%s': "
560                              "Name is longer than %u bytes",
561                              bitmap_name, bs_name, UINT8_MAX);
562                 return -1;
563             }
564             bitmap_alias = bitmap_name;
565         }
566 
567         bdrv_ref(bs);
568         bdrv_dirty_bitmap_set_busy(bitmap, true);
569 
570         dbms = g_new0(SaveBitmapState, 1);
571         dbms->bs = bs;
572         dbms->node_alias = g_strdup(node_alias);
573         dbms->bitmap_alias = g_strdup(bitmap_alias);
574         dbms->bitmap = bitmap;
575         dbms->total_sectors = bdrv_nb_sectors(bs);
576         dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU *
577             (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS);
578         assert(dbms->sectors_per_chunk != 0);
579         if (bdrv_dirty_bitmap_enabled(bitmap)) {
580             dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
581         }
582         if (bitmap_transform &&
583             bitmap_transform->has_persistent) {
584             if (bitmap_transform->persistent) {
585                 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
586             }
587         } else {
588             if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
589                 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
590             }
591         }
592 
593         QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry);
594     }
595 
596     return 0;
597 }
598 
599 /* Called with iothread lock taken. */
600 static int init_dirty_bitmap_migration(DBMSaveState *s)
601 {
602     BlockDriverState *bs;
603     SaveBitmapState *dbms;
604     GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
605     BlockBackend *blk;
606     const MigrationParameters *mig_params = &migrate_get_current()->parameters;
607     GHashTable *alias_map = NULL;
608 
609     if (mig_params->has_block_bitmap_mapping) {
610         alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true,
611                                         &error_abort);
612     }
613 
614     s->bulk_completed = false;
615     s->prev_bs = NULL;
616     s->prev_bitmap = NULL;
617     s->no_bitmaps = false;
618 
619     if (!alias_map) {
620         /*
621          * Use blockdevice name for direct (or filtered) children of named block
622          * backends.
623          */
624         for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
625             const char *name = blk_name(blk);
626 
627             if (!name || strcmp(name, "") == 0) {
628                 continue;
629             }
630 
631             bs = blk_bs(blk);
632 
633             /* Skip filters without bitmaps */
634             while (bs && bs->drv && bs->drv->is_filter &&
635                    !bdrv_has_named_bitmaps(bs))
636             {
637                 bs = bdrv_filter_bs(bs);
638             }
639 
640             if (bs && bs->drv && !bs->drv->is_filter) {
641                 if (add_bitmaps_to_list(s, bs, name, NULL)) {
642                     goto fail;
643                 }
644                 g_hash_table_add(handled_by_blk, bs);
645             }
646         }
647     }
648 
649     for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
650         if (g_hash_table_contains(handled_by_blk, bs)) {
651             continue;
652         }
653 
654         if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) {
655             goto fail;
656         }
657     }
658 
659     /* unset migration flags here, to not roll back it */
660     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
661         bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
662     }
663 
664     if (QSIMPLEQ_EMPTY(&s->dbms_list)) {
665         s->no_bitmaps = true;
666     }
667 
668     g_hash_table_destroy(handled_by_blk);
669     if (alias_map) {
670         g_hash_table_destroy(alias_map);
671     }
672 
673     return 0;
674 
675 fail:
676     g_hash_table_destroy(handled_by_blk);
677     if (alias_map) {
678         g_hash_table_destroy(alias_map);
679     }
680     dirty_bitmap_do_save_cleanup(s);
681 
682     return -1;
683 }
684 
685 /* Called with no lock taken.  */
686 static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s,
687                                   SaveBitmapState *dbms)
688 {
689     uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
690                              dbms->sectors_per_chunk);
691 
692     send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors);
693 
694     dbms->cur_sector += nr_sectors;
695     if (dbms->cur_sector >= dbms->total_sectors) {
696         dbms->bulk_completed = true;
697     }
698 }
699 
700 /* Called with no lock taken.  */
701 static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit)
702 {
703     SaveBitmapState *dbms;
704 
705     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
706         while (!dbms->bulk_completed) {
707             bulk_phase_send_chunk(f, s, dbms);
708             if (limit && qemu_file_rate_limit(f)) {
709                 return;
710             }
711         }
712     }
713 
714     s->bulk_completed = true;
715 }
716 
717 /* for SaveVMHandlers */
718 static void dirty_bitmap_save_cleanup(void *opaque)
719 {
720     DBMSaveState *s = &((DBMState *)opaque)->save;
721 
722     dirty_bitmap_do_save_cleanup(s);
723 }
724 
725 static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
726 {
727     DBMSaveState *s = &((DBMState *)opaque)->save;
728 
729     trace_dirty_bitmap_save_iterate(migration_in_postcopy());
730 
731     if (migration_in_postcopy() && !s->bulk_completed) {
732         bulk_phase(f, s, true);
733     }
734 
735     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
736 
737     return s->bulk_completed;
738 }
739 
740 /* Called with iothread lock taken.  */
741 
742 static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
743 {
744     DBMSaveState *s = &((DBMState *)opaque)->save;
745     SaveBitmapState *dbms;
746     trace_dirty_bitmap_save_complete_enter();
747 
748     if (!s->bulk_completed) {
749         bulk_phase(f, s, false);
750     }
751 
752     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
753         send_bitmap_complete(f, s, dbms);
754     }
755 
756     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
757 
758     trace_dirty_bitmap_save_complete_finish();
759 
760     dirty_bitmap_save_cleanup(opaque);
761     return 0;
762 }
763 
764 static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque,
765                                       uint64_t max_size,
766                                       uint64_t *res_precopy_only,
767                                       uint64_t *res_compatible,
768                                       uint64_t *res_postcopy_only)
769 {
770     DBMSaveState *s = &((DBMState *)opaque)->save;
771     SaveBitmapState *dbms;
772     uint64_t pending = 0;
773 
774     qemu_mutex_lock_iothread();
775 
776     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
777         uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
778         uint64_t sectors = dbms->bulk_completed ? 0 :
779                            dbms->total_sectors - dbms->cur_sector;
780 
781         pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
782     }
783 
784     qemu_mutex_unlock_iothread();
785 
786     trace_dirty_bitmap_save_pending(pending, max_size);
787 
788     *res_postcopy_only += pending;
789 }
790 
791 /* First occurrence of this bitmap. It should be created if doesn't exist */
792 static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s)
793 {
794     Error *local_err = NULL;
795     uint32_t granularity = qemu_get_be32(f);
796     uint8_t flags = qemu_get_byte(f);
797     LoadBitmapState *b;
798     bool persistent;
799 
800     if (s->cancelled) {
801         return 0;
802     }
803 
804     if (s->bitmap) {
805         error_report("Bitmap with the same name ('%s') already exists on "
806                      "destination", bdrv_dirty_bitmap_name(s->bitmap));
807         return -EINVAL;
808     } else {
809         s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
810                                              s->bitmap_name, &local_err);
811         if (!s->bitmap) {
812             error_report_err(local_err);
813             return -EINVAL;
814         }
815     }
816 
817     if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
818         error_report("Unknown flags in migrated dirty bitmap header: %x",
819                      flags);
820         return -EINVAL;
821     }
822 
823     if (s->bmap_inner &&
824         s->bmap_inner->transform &&
825         s->bmap_inner->transform->has_persistent) {
826         persistent = s->bmap_inner->transform->persistent;
827     } else {
828         persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
829     }
830 
831     if (persistent) {
832         bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
833     }
834 
835     bdrv_disable_dirty_bitmap(s->bitmap);
836     if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
837         bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
838         if (local_err) {
839             error_report_err(local_err);
840             return -EINVAL;
841         }
842     } else {
843         bdrv_dirty_bitmap_set_busy(s->bitmap, true);
844     }
845 
846     b = g_new(LoadBitmapState, 1);
847     b->bs = s->bs;
848     b->bitmap = s->bitmap;
849     b->migrated = false;
850     b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
851 
852     s->bitmaps = g_slist_prepend(s->bitmaps, b);
853 
854     return 0;
855 }
856 
857 /*
858  * before_vm_start_handle_item
859  *
860  * g_slist_foreach helper
861  *
862  * item is LoadBitmapState*
863  * opaque is DBMLoadState*
864  */
865 static void before_vm_start_handle_item(void *item, void *opaque)
866 {
867     DBMLoadState *s = opaque;
868     LoadBitmapState *b = item;
869 
870     if (b->enabled) {
871         if (b->migrated) {
872             bdrv_enable_dirty_bitmap(b->bitmap);
873         } else {
874             bdrv_dirty_bitmap_enable_successor(b->bitmap);
875         }
876     }
877 
878     if (b->migrated) {
879         s->bitmaps = g_slist_remove(s->bitmaps, b);
880         g_free(b);
881     }
882 }
883 
884 void dirty_bitmap_mig_before_vm_start(void)
885 {
886     DBMLoadState *s = &dbm_state.load;
887     qemu_mutex_lock(&s->lock);
888 
889     assert(!s->before_vm_start_handled);
890     g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s);
891     s->before_vm_start_handled = true;
892 
893     qemu_mutex_unlock(&s->lock);
894 }
895 
896 static void cancel_incoming_locked(DBMLoadState *s)
897 {
898     GSList *item;
899 
900     if (s->cancelled) {
901         return;
902     }
903 
904     s->cancelled = true;
905     s->bs = NULL;
906     s->bitmap = NULL;
907 
908     /* Drop all unfinished bitmaps */
909     for (item = s->bitmaps; item; item = g_slist_next(item)) {
910         LoadBitmapState *b = item->data;
911 
912         /*
913          * Bitmap must be unfinished, as finished bitmaps should already be
914          * removed from the list.
915          */
916         assert(!s->before_vm_start_handled || !b->migrated);
917         if (bdrv_dirty_bitmap_has_successor(b->bitmap)) {
918             bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort);
919         } else {
920             bdrv_dirty_bitmap_set_busy(b->bitmap, false);
921         }
922         bdrv_release_dirty_bitmap(b->bitmap);
923     }
924 
925     g_slist_free_full(s->bitmaps, g_free);
926     s->bitmaps = NULL;
927 }
928 
929 void dirty_bitmap_mig_cancel_outgoing(void)
930 {
931     dirty_bitmap_do_save_cleanup(&dbm_state.save);
932 }
933 
934 void dirty_bitmap_mig_cancel_incoming(void)
935 {
936     DBMLoadState *s = &dbm_state.load;
937 
938     qemu_mutex_lock(&s->lock);
939 
940     cancel_incoming_locked(s);
941 
942     qemu_mutex_unlock(&s->lock);
943 }
944 
945 static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
946 {
947     GSList *item;
948     trace_dirty_bitmap_load_complete();
949 
950     if (s->cancelled) {
951         return;
952     }
953 
954     bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
955 
956     if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
957         bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
958     } else {
959         bdrv_dirty_bitmap_set_busy(s->bitmap, false);
960     }
961 
962     for (item = s->bitmaps; item; item = g_slist_next(item)) {
963         LoadBitmapState *b = item->data;
964 
965         if (b->bitmap == s->bitmap) {
966             b->migrated = true;
967             if (s->before_vm_start_handled) {
968                 s->bitmaps = g_slist_remove(s->bitmaps, b);
969                 g_free(b);
970             }
971             break;
972         }
973     }
974 }
975 
976 static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
977 {
978     uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
979     uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
980     trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
981                                        nr_bytes >> BDRV_SECTOR_BITS);
982 
983     if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
984         trace_dirty_bitmap_load_bits_zeroes();
985         if (!s->cancelled) {
986             bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte,
987                                                  nr_bytes, false);
988         }
989     } else {
990         size_t ret;
991         g_autofree uint8_t *buf = NULL;
992         uint64_t buf_size = qemu_get_be64(f);
993         uint64_t needed_size;
994 
995         /*
996          * The actual check for buf_size is done a bit later. We can't do it in
997          * cancelled mode as we don't have the bitmap to check the constraints
998          * (so, we allocate a buffer and read prior to the check). On the other
999          * hand, we shouldn't blindly g_malloc the number from the stream.
1000          * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow
1001          * a bit larger (which means that bitmap migration will fail anyway and
1002          * the whole migration will most probably fail soon due to broken
1003          * stream).
1004          */
1005         if (buf_size > 10 * CHUNK_SIZE) {
1006             error_report("Bitmap migration stream buffer allocation request "
1007                          "is too large");
1008             return -EIO;
1009         }
1010 
1011         buf = g_malloc(buf_size);
1012         ret = qemu_get_buffer(f, buf, buf_size);
1013         if (ret != buf_size) {
1014             error_report("Failed to read bitmap bits");
1015             return -EIO;
1016         }
1017 
1018         if (s->cancelled) {
1019             return 0;
1020         }
1021 
1022         needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
1023                                                            first_byte,
1024                                                            nr_bytes);
1025 
1026         if (needed_size > buf_size ||
1027             buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
1028              /* Here used same alignment as in send_bitmap_bits */
1029         ) {
1030             error_report("Migrated bitmap granularity doesn't "
1031                          "match the destination bitmap '%s' granularity",
1032                          bdrv_dirty_bitmap_name(s->bitmap));
1033             cancel_incoming_locked(s);
1034             return 0;
1035         }
1036 
1037         bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
1038                                            false);
1039     }
1040 
1041     return 0;
1042 }
1043 
1044 static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
1045                                     GHashTable *alias_map)
1046 {
1047     GHashTable *bitmap_alias_map = NULL;
1048     Error *local_err = NULL;
1049     bool nothing;
1050     s->flags = qemu_get_bitmap_flags(f);
1051     trace_dirty_bitmap_load_header(s->flags);
1052 
1053     nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
1054 
1055     if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
1056         if (!qemu_get_counted_string(f, s->node_alias)) {
1057             error_report("Unable to read node alias string");
1058             return -EINVAL;
1059         }
1060 
1061         if (!s->cancelled) {
1062             if (alias_map) {
1063                 const AliasMapInnerNode *amin;
1064 
1065                 amin = g_hash_table_lookup(alias_map, s->node_alias);
1066                 if (!amin) {
1067                     error_setg(&local_err, "Error: Unknown node alias '%s'",
1068                                s->node_alias);
1069                     s->bs = NULL;
1070                 } else {
1071                     bitmap_alias_map = amin->subtree;
1072                     s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
1073                 }
1074             } else {
1075                 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
1076                                        &local_err);
1077             }
1078             if (!s->bs) {
1079                 error_report_err(local_err);
1080                 cancel_incoming_locked(s);
1081             }
1082         }
1083     } else if (s->bs) {
1084         if (alias_map) {
1085             const AliasMapInnerNode *amin;
1086 
1087             /* Must be present in the map, or s->bs would not be set */
1088             amin = g_hash_table_lookup(alias_map, s->node_alias);
1089             assert(amin != NULL);
1090 
1091             bitmap_alias_map = amin->subtree;
1092         }
1093     } else if (!nothing && !s->cancelled) {
1094         error_report("Error: block device name is not set");
1095         cancel_incoming_locked(s);
1096     }
1097 
1098     assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
1099 
1100     if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
1101         const char *bitmap_name;
1102 
1103         if (!qemu_get_counted_string(f, s->bitmap_alias)) {
1104             error_report("Unable to read bitmap alias string");
1105             return -EINVAL;
1106         }
1107 
1108         bitmap_name = s->bitmap_alias;
1109         if (!s->cancelled && bitmap_alias_map) {
1110             BitmapMigrationBitmapAlias *bmap_inner;
1111 
1112             bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias);
1113             if (!bmap_inner) {
1114                 error_report("Error: Unknown bitmap alias '%s' on node "
1115                              "'%s' (alias '%s')", s->bitmap_alias,
1116                              s->bs->node_name, s->node_alias);
1117                 cancel_incoming_locked(s);
1118             } else {
1119                 bitmap_name = bmap_inner->name;
1120             }
1121 
1122             s->bmap_inner = bmap_inner;
1123         }
1124 
1125         if (!s->cancelled) {
1126             g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
1127             s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
1128 
1129             /*
1130              * bitmap may be NULL here, it wouldn't be an error if it is the
1131              * first occurrence of the bitmap
1132              */
1133             if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
1134                 error_report("Error: unknown dirty bitmap "
1135                              "'%s' for block device '%s'",
1136                              s->bitmap_name, s->bs->node_name);
1137                 cancel_incoming_locked(s);
1138             }
1139         }
1140     } else if (!s->bitmap && !nothing && !s->cancelled) {
1141         error_report("Error: block device name is not set");
1142         cancel_incoming_locked(s);
1143     }
1144 
1145     return 0;
1146 }
1147 
1148 /*
1149  * dirty_bitmap_load
1150  *
1151  * Load sequence of dirty bitmap chunks. Return error only on fatal io stream
1152  * violations. On other errors just cancel bitmaps incoming migration and return
1153  * 0.
1154  *
1155  * Note, than when incoming bitmap migration is canceled, we still must read all
1156  * our chunks (and just ignore them), to not affect other migration objects.
1157  */
1158 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
1159 {
1160     GHashTable *alias_map = NULL;
1161     const MigrationParameters *mig_params = &migrate_get_current()->parameters;
1162     DBMLoadState *s = &((DBMState *)opaque)->load;
1163     int ret = 0;
1164 
1165     trace_dirty_bitmap_load_enter();
1166 
1167     if (version_id != 1) {
1168         QEMU_LOCK_GUARD(&s->lock);
1169         cancel_incoming_locked(s);
1170         return -EINVAL;
1171     }
1172 
1173     if (mig_params->has_block_bitmap_mapping) {
1174         alias_map = construct_alias_map(mig_params->block_bitmap_mapping,
1175                                         false, &error_abort);
1176     }
1177 
1178     do {
1179         QEMU_LOCK_GUARD(&s->lock);
1180 
1181         ret = dirty_bitmap_load_header(f, s, alias_map);
1182         if (ret < 0) {
1183             cancel_incoming_locked(s);
1184             goto fail;
1185         }
1186 
1187         if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
1188             ret = dirty_bitmap_load_start(f, s);
1189         } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
1190             dirty_bitmap_load_complete(f, s);
1191         } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
1192             ret = dirty_bitmap_load_bits(f, s);
1193         }
1194 
1195         if (!ret) {
1196             ret = qemu_file_get_error(f);
1197         }
1198 
1199         if (ret) {
1200             cancel_incoming_locked(s);
1201             goto fail;
1202         }
1203     } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
1204 
1205     trace_dirty_bitmap_load_success();
1206     ret = 0;
1207 fail:
1208     if (alias_map) {
1209         g_hash_table_destroy(alias_map);
1210     }
1211     return ret;
1212 }
1213 
1214 static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
1215 {
1216     DBMSaveState *s = &((DBMState *)opaque)->save;
1217     SaveBitmapState *dbms = NULL;
1218 
1219     qemu_mutex_lock_iothread();
1220     if (init_dirty_bitmap_migration(s) < 0) {
1221         qemu_mutex_unlock_iothread();
1222         return -1;
1223     }
1224 
1225     QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
1226         send_bitmap_start(f, s, dbms);
1227     }
1228     qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
1229     qemu_mutex_unlock_iothread();
1230     return 0;
1231 }
1232 
1233 static bool dirty_bitmap_is_active(void *opaque)
1234 {
1235     DBMSaveState *s = &((DBMState *)opaque)->save;
1236 
1237     return migrate_dirty_bitmaps() && !s->no_bitmaps;
1238 }
1239 
1240 static bool dirty_bitmap_is_active_iterate(void *opaque)
1241 {
1242     return dirty_bitmap_is_active(opaque) && !runstate_is_running();
1243 }
1244 
1245 static bool dirty_bitmap_has_postcopy(void *opaque)
1246 {
1247     return true;
1248 }
1249 
1250 static SaveVMHandlers savevm_dirty_bitmap_handlers = {
1251     .save_setup = dirty_bitmap_save_setup,
1252     .save_live_complete_postcopy = dirty_bitmap_save_complete,
1253     .save_live_complete_precopy = dirty_bitmap_save_complete,
1254     .has_postcopy = dirty_bitmap_has_postcopy,
1255     .save_live_pending = dirty_bitmap_save_pending,
1256     .save_live_iterate = dirty_bitmap_save_iterate,
1257     .is_active_iterate = dirty_bitmap_is_active_iterate,
1258     .load_state = dirty_bitmap_load,
1259     .save_cleanup = dirty_bitmap_save_cleanup,
1260     .is_active = dirty_bitmap_is_active,
1261 };
1262 
1263 void dirty_bitmap_mig_init(void)
1264 {
1265     QSIMPLEQ_INIT(&dbm_state.save.dbms_list);
1266     qemu_mutex_init(&dbm_state.load.lock);
1267 
1268     register_savevm_live("dirty-bitmap", 0, 1,
1269                          &savevm_dirty_bitmap_handlers,
1270                          &dbm_state);
1271 }
1272