35     HBitmap *bitmap;            /* Dirty bitmap implementation */  member
61     BdrvDirtyBitmap *bitmap;  member
74 void bdrv_dirty_bitmap_lock(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_lock()  argument
76     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_lock()
79 void bdrv_dirty_bitmap_unlock(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_unlock()  argument
81     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_unlock()
105     BdrvDirtyBitmap *bitmap;  in bdrv_create_dirty_bitmap()  local
125     bitmap = g_new0(BdrvDirtyBitmap, 1);  in bdrv_create_dirty_bitmap()
126     bitmap->bs = bs;  in bdrv_create_dirty_bitmap()
127     bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(granularity));  in bdrv_create_dirty_bitmap()
128     bitmap->size = bitmap_size;  in bdrv_create_dirty_bitmap()
129     bitmap->name = g_strdup(name);  in bdrv_create_dirty_bitmap()
130     bitmap->disabled = false;  in bdrv_create_dirty_bitmap()
132     QLIST_INSERT_HEAD(&bs->dirty_bitmaps, bitmap, list);  in bdrv_create_dirty_bitmap()
134     return bitmap;  in bdrv_create_dirty_bitmap()
137 int64_t bdrv_dirty_bitmap_size(const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_size()  argument
139     return bitmap->size;  in bdrv_dirty_bitmap_size()
142 const char *bdrv_dirty_bitmap_name(const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_name()  argument
144     return bitmap->name;  in bdrv_dirty_bitmap_name()
148 bool bdrv_dirty_bitmap_has_successor(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_has_successor()  argument
150     return bitmap->successor;  in bdrv_dirty_bitmap_has_successor()
153 static bool bdrv_dirty_bitmap_busy(const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_busy()  argument
155     return bitmap->busy;  in bdrv_dirty_bitmap_busy()
158 void bdrv_dirty_bitmap_set_busy(BdrvDirtyBitmap *bitmap, bool busy)  in bdrv_dirty_bitmap_set_busy()  argument
160     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_set_busy()
161     bitmap->busy = busy;  in bdrv_dirty_bitmap_set_busy()
162     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_set_busy()
166 bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_enabled()  argument
168     return !bitmap->disabled;  in bdrv_dirty_bitmap_enabled()
172 static bool bdrv_dirty_bitmap_recording(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_recording()  argument
174     return !bitmap->disabled || (bitmap->successor &&  in bdrv_dirty_bitmap_recording()
175                                  !bitmap->successor->disabled);  in bdrv_dirty_bitmap_recording()
178 int bdrv_dirty_bitmap_check(const BdrvDirtyBitmap *bitmap, uint32_t flags,  in bdrv_dirty_bitmap_check()  argument
181     if ((flags & BDRV_BITMAP_BUSY) && bdrv_dirty_bitmap_busy(bitmap)) {  in bdrv_dirty_bitmap_check()
183                    " operation and cannot be used", bitmap->name);  in bdrv_dirty_bitmap_check()
187     if ((flags & BDRV_BITMAP_RO) && bdrv_dirty_bitmap_readonly(bitmap)) {  in bdrv_dirty_bitmap_check()
189                    bitmap->name);  in bdrv_dirty_bitmap_check()
194         bdrv_dirty_bitmap_inconsistent(bitmap)) {  in bdrv_dirty_bitmap_check()
196                    bitmap->name);  in bdrv_dirty_bitmap_check()
211 int bdrv_dirty_bitmap_create_successor(BdrvDirtyBitmap *bitmap, Error **errp)  in bdrv_dirty_bitmap_create_successor()  argument
216     if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_BUSY, errp)) {  in bdrv_dirty_bitmap_create_successor()
219     if (bdrv_dirty_bitmap_has_successor(bitmap)) {  in bdrv_dirty_bitmap_create_successor()
226     granularity = bdrv_dirty_bitmap_granularity(bitmap);  in bdrv_dirty_bitmap_create_successor()
227     child = bdrv_create_dirty_bitmap(bitmap->bs, granularity, NULL, errp);  in bdrv_dirty_bitmap_create_successor()
233     child->disabled = bitmap->disabled;  in bdrv_dirty_bitmap_create_successor()
234     bitmap->disabled = true;  in bdrv_dirty_bitmap_create_successor()
237     bitmap->successor = child;  in bdrv_dirty_bitmap_create_successor()
238     bitmap->busy = true;  in bdrv_dirty_bitmap_create_successor()
242 void bdrv_enable_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)  in bdrv_enable_dirty_bitmap_locked()  argument
244     bitmap->disabled = false;  in bdrv_enable_dirty_bitmap_locked()
248 void bdrv_dirty_bitmap_enable_successor(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_enable_successor()  argument
250     assert(bitmap->bs == bitmap->successor->bs);  in bdrv_dirty_bitmap_enable_successor()
251     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_enable_successor()
252     bdrv_enable_dirty_bitmap_locked(bitmap->successor);  in bdrv_dirty_bitmap_enable_successor()
253     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_enable_successor()
257 static void bdrv_release_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap)  in bdrv_release_dirty_bitmap_locked()  argument
259     assert(!bitmap->active_iterators);  in bdrv_release_dirty_bitmap_locked()
260     assert(!bdrv_dirty_bitmap_busy(bitmap));  in bdrv_release_dirty_bitmap_locked()
261     assert(!bdrv_dirty_bitmap_has_successor(bitmap));  in bdrv_release_dirty_bitmap_locked()
262     QLIST_REMOVE(bitmap, list);  in bdrv_release_dirty_bitmap_locked()
263     hbitmap_free(bitmap->bitmap);  in bdrv_release_dirty_bitmap_locked()
264     g_free(bitmap->name);  in bdrv_release_dirty_bitmap_locked()
265     g_free(bitmap);  in bdrv_release_dirty_bitmap_locked()
273 BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BdrvDirtyBitmap *bitmap,  in bdrv_dirty_bitmap_abdicate()  argument
277     BdrvDirtyBitmap *successor = bitmap->successor;  in bdrv_dirty_bitmap_abdicate()
285     name = bitmap->name;  in bdrv_dirty_bitmap_abdicate()
286     bitmap->name = NULL;  in bdrv_dirty_bitmap_abdicate()
288     bitmap->successor = NULL;  in bdrv_dirty_bitmap_abdicate()
289     successor->persistent = bitmap->persistent;  in bdrv_dirty_bitmap_abdicate()
290     bitmap->persistent = false;  in bdrv_dirty_bitmap_abdicate()
291     bitmap->busy = false;  in bdrv_dirty_bitmap_abdicate()
292     bdrv_release_dirty_bitmap(bitmap);  in bdrv_dirty_bitmap_abdicate()
314     hbitmap_merge(parent->bitmap, successor->bitmap, parent->bitmap);  in bdrv_reclaim_dirty_bitmap_locked()
343     BdrvDirtyBitmap *bitmap;  in bdrv_dirty_bitmap_truncate()  local
346     QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {  in bdrv_dirty_bitmap_truncate()
347         assert(!bdrv_dirty_bitmap_busy(bitmap));  in bdrv_dirty_bitmap_truncate()
348         assert(!bdrv_dirty_bitmap_has_successor(bitmap));  in bdrv_dirty_bitmap_truncate()
349         assert(!bitmap->active_iterators);  in bdrv_dirty_bitmap_truncate()
350         hbitmap_truncate(bitmap->bitmap, bytes);  in bdrv_dirty_bitmap_truncate()
351         bitmap->size = bytes;  in bdrv_dirty_bitmap_truncate()
357 void bdrv_release_dirty_bitmap(BdrvDirtyBitmap *bitmap)  in bdrv_release_dirty_bitmap()  argument
359     BlockDriverState *bs = bitmap->bs;  in bdrv_release_dirty_bitmap()
362     bdrv_release_dirty_bitmap_locked(bitmap);  in bdrv_release_dirty_bitmap()
438 void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)  in bdrv_disable_dirty_bitmap()  argument
440     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_disable_dirty_bitmap()
441     bitmap->disabled = true;  in bdrv_disable_dirty_bitmap()
442     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_disable_dirty_bitmap()
445 void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)  in bdrv_enable_dirty_bitmap()  argument
447     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_enable_dirty_bitmap()
448     bdrv_enable_dirty_bitmap_locked(bitmap);  in bdrv_enable_dirty_bitmap()
449     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_enable_dirty_bitmap()
478 bool bdrv_dirty_bitmap_get_locked(BdrvDirtyBitmap *bitmap, int64_t offset)  in bdrv_dirty_bitmap_get_locked()  argument
480     return hbitmap_get(bitmap->bitmap, offset);  in bdrv_dirty_bitmap_get_locked()
483 bool bdrv_dirty_bitmap_get(BdrvDirtyBitmap *bitmap, int64_t offset)  in bdrv_dirty_bitmap_get()  argument
486     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_get()
487     ret = bdrv_dirty_bitmap_get_locked(bitmap, offset);  in bdrv_dirty_bitmap_get()
488     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_get()
513 uint32_t bdrv_dirty_bitmap_granularity(const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_granularity()  argument
515     return 1U << hbitmap_granularity(bitmap->bitmap);  in bdrv_dirty_bitmap_granularity()
518 BdrvDirtyBitmapIter *bdrv_dirty_iter_new(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_iter_new()  argument
521     hbitmap_iter_init(&iter->hbi, bitmap->bitmap, 0);  in bdrv_dirty_iter_new()
522     iter->bitmap = bitmap;  in bdrv_dirty_iter_new()
523     bitmap->active_iterators++;  in bdrv_dirty_iter_new()
532     assert(iter->bitmap->active_iterators > 0);  in bdrv_dirty_iter_free()
533     iter->bitmap->active_iterators--;  in bdrv_dirty_iter_free()
543 void bdrv_set_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,  in bdrv_set_dirty_bitmap_locked()  argument
546     assert(!bdrv_dirty_bitmap_readonly(bitmap));  in bdrv_set_dirty_bitmap_locked()
547     hbitmap_set(bitmap->bitmap, offset, bytes);  in bdrv_set_dirty_bitmap_locked()
550 void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,  in bdrv_set_dirty_bitmap()  argument
553     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_set_dirty_bitmap()
554     bdrv_set_dirty_bitmap_locked(bitmap, offset, bytes);  in bdrv_set_dirty_bitmap()
555     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_set_dirty_bitmap()
559 void bdrv_reset_dirty_bitmap_locked(BdrvDirtyBitmap *bitmap,  in bdrv_reset_dirty_bitmap_locked()  argument
562     assert(!bdrv_dirty_bitmap_readonly(bitmap));  in bdrv_reset_dirty_bitmap_locked()
563     hbitmap_reset(bitmap->bitmap, offset, bytes);  in bdrv_reset_dirty_bitmap_locked()
566 void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,  in bdrv_reset_dirty_bitmap()  argument
569     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_reset_dirty_bitmap()
570     bdrv_reset_dirty_bitmap_locked(bitmap, offset, bytes);  in bdrv_reset_dirty_bitmap()
571     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_reset_dirty_bitmap()
574 void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out)  in bdrv_clear_dirty_bitmap()  argument
577     assert(!bdrv_dirty_bitmap_readonly(bitmap));  in bdrv_clear_dirty_bitmap()
578     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_clear_dirty_bitmap()
580         hbitmap_reset_all(bitmap->bitmap);  in bdrv_clear_dirty_bitmap()
582         HBitmap *backup = bitmap->bitmap;  in bdrv_clear_dirty_bitmap()
583         bitmap->bitmap = hbitmap_alloc(bitmap->size,  in bdrv_clear_dirty_bitmap()
587     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_clear_dirty_bitmap()
590 void bdrv_restore_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *backup)  in bdrv_restore_dirty_bitmap()  argument
592     HBitmap *tmp = bitmap->bitmap;  in bdrv_restore_dirty_bitmap()
593     assert(!bdrv_dirty_bitmap_readonly(bitmap));  in bdrv_restore_dirty_bitmap()
595     bitmap->bitmap = backup;  in bdrv_restore_dirty_bitmap()
599 uint64_t bdrv_dirty_bitmap_serialization_size(const BdrvDirtyBitmap *bitmap,  in bdrv_dirty_bitmap_serialization_size()  argument
602     return hbitmap_serialization_size(bitmap->bitmap, offset, bytes);  in bdrv_dirty_bitmap_serialization_size()
605 uint64_t bdrv_dirty_bitmap_serialization_align(const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_serialization_align()  argument
607     return hbitmap_serialization_align(bitmap->bitmap);  in bdrv_dirty_bitmap_serialization_align()
612                                                   const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_serialization_coverage()  argument
614     uint64_t granularity = bdrv_dirty_bitmap_granularity(bitmap);  in bdrv_dirty_bitmap_serialization_coverage()
618                            bdrv_dirty_bitmap_serialization_align(bitmap)));  in bdrv_dirty_bitmap_serialization_coverage()
623 void bdrv_dirty_bitmap_serialize_part(const BdrvDirtyBitmap *bitmap,  in bdrv_dirty_bitmap_serialize_part()  argument
627     hbitmap_serialize_part(bitmap->bitmap, buf, offset, bytes);  in bdrv_dirty_bitmap_serialize_part()
630 void bdrv_dirty_bitmap_deserialize_part(BdrvDirtyBitmap *bitmap,  in bdrv_dirty_bitmap_deserialize_part()  argument
634     hbitmap_deserialize_part(bitmap->bitmap, buf, offset, bytes, finish);  in bdrv_dirty_bitmap_deserialize_part()
637 void bdrv_dirty_bitmap_deserialize_zeroes(BdrvDirtyBitmap *bitmap,  in bdrv_dirty_bitmap_deserialize_zeroes()  argument
641     hbitmap_deserialize_zeroes(bitmap->bitmap, offset, bytes, finish);  in bdrv_dirty_bitmap_deserialize_zeroes()
644 void bdrv_dirty_bitmap_deserialize_ones(BdrvDirtyBitmap *bitmap,  in bdrv_dirty_bitmap_deserialize_ones()  argument
648     hbitmap_deserialize_ones(bitmap->bitmap, offset, bytes, finish);  in bdrv_dirty_bitmap_deserialize_ones()
651 void bdrv_dirty_bitmap_deserialize_finish(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_deserialize_finish()  argument
653     hbitmap_deserialize_finish(bitmap->bitmap);  in bdrv_dirty_bitmap_deserialize_finish()
658     BdrvDirtyBitmap *bitmap;  in bdrv_set_dirty()  local
666     QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {  in bdrv_set_dirty()
667         if (!bdrv_dirty_bitmap_enabled(bitmap)) {  in bdrv_set_dirty()
670         assert(!bdrv_dirty_bitmap_readonly(bitmap));  in bdrv_set_dirty()
671         hbitmap_set(bitmap->bitmap, offset, bytes);  in bdrv_set_dirty()
684 int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)  in bdrv_get_dirty_count()  argument
686     return hbitmap_count(bitmap->bitmap);  in bdrv_get_dirty_count()
689 bool bdrv_dirty_bitmap_readonly(const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_readonly()  argument
691     return bitmap->readonly;  in bdrv_dirty_bitmap_readonly()
695 void bdrv_dirty_bitmap_set_readonly(BdrvDirtyBitmap *bitmap, bool value)  in bdrv_dirty_bitmap_set_readonly()  argument
697     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_set_readonly()
698     bitmap->readonly = value;  in bdrv_dirty_bitmap_set_readonly()
699     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_set_readonly()
728 void bdrv_dirty_bitmap_set_persistence(BdrvDirtyBitmap *bitmap, bool persistent)  in bdrv_dirty_bitmap_set_persistence()  argument
730     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_set_persistence()
731     bitmap->persistent = persistent;  in bdrv_dirty_bitmap_set_persistence()
732     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_set_persistence()
736 void bdrv_dirty_bitmap_set_inconsistent(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_set_inconsistent()  argument
738     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_set_inconsistent()
739     assert(bitmap->persistent == true);  in bdrv_dirty_bitmap_set_inconsistent()
740     bitmap->inconsistent = true;  in bdrv_dirty_bitmap_set_inconsistent()
741     bitmap->disabled = true;  in bdrv_dirty_bitmap_set_inconsistent()
742     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_set_inconsistent()
746 void bdrv_dirty_bitmap_skip_store(BdrvDirtyBitmap *bitmap, bool skip)  in bdrv_dirty_bitmap_skip_store()  argument
748     bdrv_dirty_bitmaps_lock(bitmap->bs);  in bdrv_dirty_bitmap_skip_store()
749     bitmap->skip_store = skip;  in bdrv_dirty_bitmap_skip_store()
750     bdrv_dirty_bitmaps_unlock(bitmap->bs);  in bdrv_dirty_bitmap_skip_store()
753 bool bdrv_dirty_bitmap_get_persistence(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_get_persistence()  argument
755     return bitmap->persistent && !bitmap->skip_store;  in bdrv_dirty_bitmap_get_persistence()
758 bool bdrv_dirty_bitmap_inconsistent(const BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_inconsistent()  argument
760     return bitmap->inconsistent;  in bdrv_dirty_bitmap_inconsistent()
768 BdrvDirtyBitmap *bdrv_dirty_bitmap_next(BdrvDirtyBitmap *bitmap)  in bdrv_dirty_bitmap_next()  argument
770     return QLIST_NEXT(bitmap, list);  in bdrv_dirty_bitmap_next()
773 char *bdrv_dirty_bitmap_sha256(const BdrvDirtyBitmap *bitmap, Error **errp)  in bdrv_dirty_bitmap_sha256()  argument
775     return hbitmap_sha256(bitmap->bitmap, errp);  in bdrv_dirty_bitmap_sha256()
778 int64_t bdrv_dirty_bitmap_next_dirty(BdrvDirtyBitmap *bitmap, int64_t offset,  in bdrv_dirty_bitmap_next_dirty()  argument
781     return hbitmap_next_dirty(bitmap->bitmap, offset, bytes);  in bdrv_dirty_bitmap_next_dirty()
784 int64_t bdrv_dirty_bitmap_next_zero(BdrvDirtyBitmap *bitmap, int64_t offset,  in bdrv_dirty_bitmap_next_zero()  argument
787     return hbitmap_next_zero(bitmap->bitmap, offset, bytes);  in bdrv_dirty_bitmap_next_zero()
790 bool bdrv_dirty_bitmap_next_dirty_area(BdrvDirtyBitmap *bitmap,  in bdrv_dirty_bitmap_next_dirty_area()  argument
794     return hbitmap_next_dirty_area(bitmap->bitmap, start, end, max_dirty_count,  in bdrv_dirty_bitmap_next_dirty_area()
798 bool bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap, int64_t offset,  in bdrv_dirty_bitmap_status()  argument
801     return hbitmap_status(bitmap->bitmap, offset, bytes, count);  in bdrv_dirty_bitmap_status()
877         *backup = dest->bitmap;  in bdrv_dirty_bitmap_merge_internal()
878         dest->bitmap = hbitmap_alloc(dest->size, hbitmap_granularity(*backup));  in bdrv_dirty_bitmap_merge_internal()
879         hbitmap_merge(*backup, src->bitmap, dest->bitmap);  in bdrv_dirty_bitmap_merge_internal()
881         hbitmap_merge(dest->bitmap, src->bitmap, dest->bitmap);  in bdrv_dirty_bitmap_merge_internal()