1903cb1bfSPhilippe Mathieu-Daudé#!/usr/bin/env python3 29dd003a9SVladimir Sementsov-Ogievskiy# group: rw 3dfdc48d5SJohn Snow# 4dfdc48d5SJohn Snow# Test bitmap-sync backups (incremental, differential, and partials) 5dfdc48d5SJohn Snow# 6dfdc48d5SJohn Snow# Copyright (c) 2019 John Snow for Red Hat, Inc. 7dfdc48d5SJohn Snow# 8dfdc48d5SJohn Snow# This program is free software; you can redistribute it and/or modify 9dfdc48d5SJohn Snow# it under the terms of the GNU General Public License as published by 10dfdc48d5SJohn Snow# the Free Software Foundation; either version 2 of the License, or 11dfdc48d5SJohn Snow# (at your option) any later version. 12dfdc48d5SJohn Snow# 13dfdc48d5SJohn Snow# This program is distributed in the hope that it will be useful, 14dfdc48d5SJohn Snow# but WITHOUT ANY WARRANTY; without even the implied warranty of 15dfdc48d5SJohn Snow# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16dfdc48d5SJohn Snow# GNU General Public License for more details. 17dfdc48d5SJohn Snow# 18dfdc48d5SJohn Snow# You should have received a copy of the GNU General Public License 19dfdc48d5SJohn Snow# along with this program. If not, see <http://www.gnu.org/licenses/>. 20dfdc48d5SJohn Snow# 21dfdc48d5SJohn Snow# owner=jsnow@redhat.com 22dfdc48d5SJohn Snow 23dfdc48d5SJohn Snowimport math 24dfdc48d5SJohn Snowimport os 25dfdc48d5SJohn Snow 26dfdc48d5SJohn Snowimport iotests 27dfdc48d5SJohn Snowfrom iotests import log, qemu_img 28dfdc48d5SJohn Snow 29dfdc48d5SJohn SnowSIZE = 64 * 1024 * 1024 30dfdc48d5SJohn SnowGRANULARITY = 64 * 1024 31dfdc48d5SJohn Snow 32b0a32befSJohn Snow 33b0a32befSJohn Snowclass Pattern: 34b0a32befSJohn Snow def __init__(self, byte, offset, size=GRANULARITY): 35b0a32befSJohn Snow self.byte = byte 36b0a32befSJohn Snow self.offset = offset 37b0a32befSJohn Snow self.size = size 38b0a32befSJohn Snow 39b0a32befSJohn Snow def bits(self, granularity): 40b0a32befSJohn Snow lower = self.offset // granularity 41b0a32befSJohn Snow upper = (self.offset + self.size - 1) // granularity 42b0a32befSJohn Snow return set(range(lower, upper + 1)) 43b0a32befSJohn Snow 44dfdc48d5SJohn Snow 45dfdc48d5SJohn Snowclass PatternGroup: 46dfdc48d5SJohn Snow """Grouping of Pattern objects. Initialize with an iterable of Patterns.""" 47dfdc48d5SJohn Snow def __init__(self, patterns): 48dfdc48d5SJohn Snow self.patterns = patterns 49dfdc48d5SJohn Snow 50dfdc48d5SJohn Snow def bits(self, granularity): 51dfdc48d5SJohn Snow """Calculate the unique bits dirtied by this pattern grouping""" 52dfdc48d5SJohn Snow res = set() 53dfdc48d5SJohn Snow for pattern in self.patterns: 54b0a32befSJohn Snow res |= pattern.bits(granularity) 55dfdc48d5SJohn Snow return res 56dfdc48d5SJohn Snow 57b0a32befSJohn Snow 58dfdc48d5SJohn SnowGROUPS = [ 59dfdc48d5SJohn Snow PatternGroup([ 60dfdc48d5SJohn Snow # Batch 0: 4 clusters 61b0a32befSJohn Snow Pattern('0x49', 0x0000000), 62b0a32befSJohn Snow Pattern('0x6c', 0x0100000), # 1M 63b0a32befSJohn Snow Pattern('0x6f', 0x2000000), # 32M 64b0a32befSJohn Snow Pattern('0x76', 0x3ff0000)]), # 64M - 64K 65dfdc48d5SJohn Snow PatternGroup([ 66dfdc48d5SJohn Snow # Batch 1: 6 clusters (3 new) 67b0a32befSJohn Snow Pattern('0x65', 0x0000000), # Full overwrite 68b0a32befSJohn Snow Pattern('0x77', 0x00f8000), # Partial-left (1M-32K) 69b0a32befSJohn Snow Pattern('0x72', 0x2008000), # Partial-right (32M+32K) 70b0a32befSJohn Snow Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K) 71dfdc48d5SJohn Snow PatternGroup([ 72dfdc48d5SJohn Snow # Batch 2: 7 clusters (3 new) 73b0a32befSJohn Snow Pattern('0x74', 0x0010000), # Adjacent-right 74b0a32befSJohn Snow Pattern('0x69', 0x00e8000), # Partial-left (1M-96K) 75b0a32befSJohn Snow Pattern('0x6e', 0x2018000), # Partial-right (32M+96K) 76b0a32befSJohn Snow Pattern('0x67', 0x3fe0000, 77dfdc48d5SJohn Snow 2*GRANULARITY)]), # Overwrite [(64M-128K)-64M) 78dfdc48d5SJohn Snow PatternGroup([ 79dfdc48d5SJohn Snow # Batch 3: 8 clusters (5 new) 80dfdc48d5SJohn Snow # Carefully chosen such that nothing re-dirties the one cluster 81dfdc48d5SJohn Snow # that copies out successfully before failure in Group #1. 82b0a32befSJohn Snow Pattern('0xaa', 0x0010000, 83dfdc48d5SJohn Snow 3*GRANULARITY), # Overwrite and 2x Adjacent-right 84b0a32befSJohn Snow Pattern('0xbb', 0x00d8000), # Partial-left (1M-160K) 85b0a32befSJohn Snow Pattern('0xcc', 0x2028000), # Partial-right (32M+160K) 86b0a32befSJohn Snow Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right 87dfdc48d5SJohn Snow] 88dfdc48d5SJohn Snow 8932afa5a1SJohn Snow 9032afa5a1SJohn Snowclass EmulatedBitmap: 9132afa5a1SJohn Snow def __init__(self, granularity=GRANULARITY): 9232afa5a1SJohn Snow self._bits = set() 9332afa5a1SJohn Snow self.granularity = granularity 9432afa5a1SJohn Snow 9532afa5a1SJohn Snow def dirty_bits(self, bits): 9632afa5a1SJohn Snow self._bits |= set(bits) 9732afa5a1SJohn Snow 9832afa5a1SJohn Snow def dirty_group(self, n): 9932afa5a1SJohn Snow self.dirty_bits(GROUPS[n].bits(self.granularity)) 10032afa5a1SJohn Snow 10132afa5a1SJohn Snow def clear(self): 10232afa5a1SJohn Snow self._bits = set() 10332afa5a1SJohn Snow 10432afa5a1SJohn Snow def clear_bits(self, bits): 10532afa5a1SJohn Snow self._bits -= set(bits) 10632afa5a1SJohn Snow 10732afa5a1SJohn Snow def clear_bit(self, bit): 10832afa5a1SJohn Snow self.clear_bits({bit}) 10932afa5a1SJohn Snow 11032afa5a1SJohn Snow def clear_group(self, n): 11132afa5a1SJohn Snow self.clear_bits(GROUPS[n].bits(self.granularity)) 11232afa5a1SJohn Snow 11332afa5a1SJohn Snow @property 11432afa5a1SJohn Snow def first_bit(self): 11532afa5a1SJohn Snow return sorted(self.bits)[0] 11632afa5a1SJohn Snow 11732afa5a1SJohn Snow @property 11832afa5a1SJohn Snow def bits(self): 11932afa5a1SJohn Snow return self._bits 12032afa5a1SJohn Snow 12132afa5a1SJohn Snow @property 12232afa5a1SJohn Snow def count(self): 12332afa5a1SJohn Snow return len(self.bits) 12432afa5a1SJohn Snow 12532afa5a1SJohn Snow def compare(self, qmp_bitmap): 12632afa5a1SJohn Snow """ 12732afa5a1SJohn Snow Print a nice human-readable message checking that a bitmap as reported 12832afa5a1SJohn Snow by the QMP interface has as many bits set as we expect it to. 12932afa5a1SJohn Snow """ 13032afa5a1SJohn Snow 13132afa5a1SJohn Snow name = qmp_bitmap.get('name', '(anonymous)') 13232afa5a1SJohn Snow log("= Checking Bitmap {:s} =".format(name)) 13332afa5a1SJohn Snow 13432afa5a1SJohn Snow want = self.count 13532afa5a1SJohn Snow have = qmp_bitmap['count'] // qmp_bitmap['granularity'] 13632afa5a1SJohn Snow 13732afa5a1SJohn Snow log("expecting {:d} dirty sectors; have {:d}. {:s}".format( 13832afa5a1SJohn Snow want, have, "OK!" if want == have else "ERROR!")) 13932afa5a1SJohn Snow log('') 14032afa5a1SJohn Snow 14132afa5a1SJohn Snow 142dfdc48d5SJohn Snowclass Drive: 143dfdc48d5SJohn Snow """Represents, vaguely, a drive attached to a VM. 144dfdc48d5SJohn Snow Includes format, graph, and device information.""" 145dfdc48d5SJohn Snow 146dfdc48d5SJohn Snow def __init__(self, path, vm=None): 147dfdc48d5SJohn Snow self.path = path 148dfdc48d5SJohn Snow self.vm = vm 149dfdc48d5SJohn Snow self.fmt = None 150dfdc48d5SJohn Snow self.size = None 151dfdc48d5SJohn Snow self.node = None 152dfdc48d5SJohn Snow 153dfdc48d5SJohn Snow def img_create(self, fmt, size): 154dfdc48d5SJohn Snow self.fmt = fmt 155dfdc48d5SJohn Snow self.size = size 156dfdc48d5SJohn Snow iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size)) 157dfdc48d5SJohn Snow 158dfdc48d5SJohn Snow def create_target(self, name, fmt, size): 159dfdc48d5SJohn Snow basename = os.path.basename(self.path) 160dfdc48d5SJohn Snow file_node_name = "file_{}".format(basename) 161dfdc48d5SJohn Snow vm = self.vm 162dfdc48d5SJohn Snow 163*684750abSVladimir Sementsov-Ogievskiy log(vm.cmd('blockdev-create', job_id='bdc-file-job', 164dfdc48d5SJohn Snow options={ 165dfdc48d5SJohn Snow 'driver': 'file', 166dfdc48d5SJohn Snow 'filename': self.path, 167dfdc48d5SJohn Snow 'size': 0, 168dfdc48d5SJohn Snow })) 169dfdc48d5SJohn Snow vm.run_job('bdc-file-job') 170*684750abSVladimir Sementsov-Ogievskiy log(vm.cmd('blockdev-add', driver='file', 171dfdc48d5SJohn Snow node_name=file_node_name, filename=self.path)) 172dfdc48d5SJohn Snow 173*684750abSVladimir Sementsov-Ogievskiy log(vm.cmd('blockdev-create', job_id='bdc-fmt-job', 174dfdc48d5SJohn Snow options={ 175dfdc48d5SJohn Snow 'driver': fmt, 176dfdc48d5SJohn Snow 'file': file_node_name, 177dfdc48d5SJohn Snow 'size': size, 178dfdc48d5SJohn Snow })) 179dfdc48d5SJohn Snow vm.run_job('bdc-fmt-job') 180*684750abSVladimir Sementsov-Ogievskiy log(vm.cmd('blockdev-add', driver=fmt, 181dfdc48d5SJohn Snow node_name=name, 182dfdc48d5SJohn Snow file=file_node_name)) 183dfdc48d5SJohn Snow self.fmt = fmt 184dfdc48d5SJohn Snow self.size = size 185dfdc48d5SJohn Snow self.node = name 186dfdc48d5SJohn Snow 1870af2a09cSJohn Snowdef blockdev_backup(vm, device, target, sync, **kwargs): 1880af2a09cSJohn Snow # Strip any arguments explicitly nulled by the caller: 1890af2a09cSJohn Snow kwargs = {key: val for key, val in kwargs.items() if val is not None} 1900af2a09cSJohn Snow result = vm.qmp_log('blockdev-backup', 1910af2a09cSJohn Snow device=device, 1920af2a09cSJohn Snow target=target, 1930af2a09cSJohn Snow sync=sync, 19400e30f05SVladimir Sementsov-Ogievskiy filter_node_name='backup-top', 1952d0f32e3SVladimir Sementsov-Ogievskiy x_perf={'max-workers': 1}, 1960af2a09cSJohn Snow **kwargs) 1970af2a09cSJohn Snow return result 1980af2a09cSJohn Snow 1990af2a09cSJohn Snowdef blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs): 2000af2a09cSJohn Snow target_drive = Drive(filepath, vm=drive.vm) 2010af2a09cSJohn Snow target_drive.create_target(target_id, drive.fmt, drive.size) 202f1648454SVladimir Sementsov-Ogievskiy blockdev_backup(drive.vm, drive.node, target_id, sync, **kwargs) 2030af2a09cSJohn Snow 204dfdc48d5SJohn Snowdef reference_backup(drive, n, filepath): 205dfdc48d5SJohn Snow log("--- Reference Backup #{:d} ---\n".format(n)) 206dfdc48d5SJohn Snow target_id = "ref_target_{:d}".format(n) 207dfdc48d5SJohn Snow job_id = "ref_backup_{:d}".format(n) 2080af2a09cSJohn Snow blockdev_backup_mktarget(drive, target_id, filepath, "full", 2090af2a09cSJohn Snow job_id=job_id) 210dfdc48d5SJohn Snow drive.vm.run_job(job_id, auto_dismiss=True) 211dfdc48d5SJohn Snow log('') 212dfdc48d5SJohn Snow 2130af2a09cSJohn Snowdef backup(drive, n, filepath, sync, **kwargs): 2140af2a09cSJohn Snow log("--- Test Backup #{:d} ---\n".format(n)) 2150af2a09cSJohn Snow target_id = "backup_target_{:d}".format(n) 2160af2a09cSJohn Snow job_id = "backup_{:d}".format(n) 2170af2a09cSJohn Snow kwargs.setdefault('auto-finalize', False) 2180af2a09cSJohn Snow blockdev_backup_mktarget(drive, target_id, filepath, sync, 2190af2a09cSJohn Snow job_id=job_id, **kwargs) 220dfdc48d5SJohn Snow return job_id 221dfdc48d5SJohn Snow 22200e30f05SVladimir Sementsov-Ogievskiydef perform_writes(drive, n, filter_node_name=None): 223dfdc48d5SJohn Snow log("--- Write #{:d} ---\n".format(n)) 224dfdc48d5SJohn Snow for pattern in GROUPS[n].patterns: 225dfdc48d5SJohn Snow cmd = "write -P{:s} 0x{:07x} 0x{:x}".format( 226dfdc48d5SJohn Snow pattern.byte, 227dfdc48d5SJohn Snow pattern.offset, 228dfdc48d5SJohn Snow pattern.size) 229dfdc48d5SJohn Snow log(cmd) 23000e30f05SVladimir Sementsov-Ogievskiy log(drive.vm.hmp_qemu_io(filter_node_name or drive.node, cmd)) 2315c4343b8SVladimir Sementsov-Ogievskiy bitmaps = drive.vm.query_bitmaps() 2325c4343b8SVladimir Sementsov-Ogievskiy log({'bitmaps': bitmaps}, indent=2) 233dfdc48d5SJohn Snow log('') 234dfdc48d5SJohn Snow return bitmaps 235dfdc48d5SJohn Snow 236dfdc48d5SJohn Snow 237dfdc48d5SJohn Snowdef compare_images(image, reference, baseimg=None, expected_match=True): 238dfdc48d5SJohn Snow """ 239dfdc48d5SJohn Snow Print a nice human-readable message comparing these images. 240dfdc48d5SJohn Snow """ 241dfdc48d5SJohn Snow expected_ret = 0 if expected_match else 1 242dfdc48d5SJohn Snow if baseimg: 243fc272d3cSJohn Snow qemu_img("rebase", "-u", "-b", baseimg, '-F', iotests.imgfmt, image) 2442882ccf8SJohn Snow 2452882ccf8SJohn Snow sub = qemu_img("compare", image, reference, check=False) 2462882ccf8SJohn Snow 247dfdc48d5SJohn Snow log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format( 248dfdc48d5SJohn Snow image, reference, 2492882ccf8SJohn Snow "Identical" if sub.returncode == 0 else "Mismatch", 2502882ccf8SJohn Snow "OK!" if sub.returncode == expected_ret else "ERROR!"), 251dfdc48d5SJohn Snow filters=[iotests.filter_testfiles]) 252dfdc48d5SJohn Snow 2530af2a09cSJohn Snowdef test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None): 254dfdc48d5SJohn Snow """ 255dfdc48d5SJohn Snow Test bitmap backup routines. 256dfdc48d5SJohn Snow 257dfdc48d5SJohn Snow :param bsync_mode: Is the Bitmap Sync mode, and can be any of: 258dfdc48d5SJohn Snow - on-success: This is the "incremental" style mode. Bitmaps are 259dfdc48d5SJohn Snow synchronized to what was copied out only on success. 260dfdc48d5SJohn Snow (Partial images must be discarded.) 261dfdc48d5SJohn Snow - never: This is the "differential" style mode. 262dfdc48d5SJohn Snow Bitmaps are never synchronized. 263dfdc48d5SJohn Snow - always: This is a "best effort" style mode. 264dfdc48d5SJohn Snow Bitmaps are always synchronized, regardless of failure. 265dfdc48d5SJohn Snow (Partial images must be kept.) 266dfdc48d5SJohn Snow 267bd5ceebfSJohn Snow :param msync_mode: The mirror sync mode to use for the first backup. 268bd5ceebfSJohn Snow Can be any one of: 269bd5ceebfSJohn Snow - bitmap: Backups based on bitmap manifest. 270bd5ceebfSJohn Snow - full: Full backups. 271bd5ceebfSJohn Snow - top: Full backups of the top layer only. 272bd5ceebfSJohn Snow 273dfdc48d5SJohn Snow :param failure: Is the (optional) failure mode, and can be any of: 274dfdc48d5SJohn Snow - None: No failure. Test the normative path. Default. 275dfdc48d5SJohn Snow - simulated: Cancel the job right before it completes. 276dfdc48d5SJohn Snow This also tests writes "during" the job. 277dfdc48d5SJohn Snow - intermediate: This tests a job that fails mid-process and produces 278dfdc48d5SJohn Snow an incomplete backup. Testing limitations prevent 279dfdc48d5SJohn Snow testing competing writes. 280dfdc48d5SJohn Snow """ 2813192fad7SNir Soffer with iotests.FilePath( 282a242b19eSNir Soffer 'img', 'bsync1', 'bsync2', 'fbackup0', 'fbackup1', 'fbackup2') as \ 283a242b19eSNir Soffer (img_path, bsync1, bsync2, fbackup0, fbackup1, fbackup2), \ 284dfdc48d5SJohn Snow iotests.VM() as vm: 285dfdc48d5SJohn Snow 2860af2a09cSJohn Snow mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode) 287dfdc48d5SJohn Snow preposition = "with" if failure else "without" 288dfdc48d5SJohn Snow cond = "{:s} {:s}".format(preposition, 289dfdc48d5SJohn Snow "{:s} failure".format(failure) if failure 290dfdc48d5SJohn Snow else "failure") 291dfdc48d5SJohn Snow log("\n=== {:s} {:s} ===\n".format(mode, cond)) 292dfdc48d5SJohn Snow 293dfdc48d5SJohn Snow log('--- Preparing image & VM ---\n') 294dfdc48d5SJohn Snow drive0 = Drive(img_path, vm=vm) 295dfdc48d5SJohn Snow drive0.img_create(iotests.imgfmt, SIZE) 29622329f0dSLaurent Vivier vm.add_device("{},id=scsi0".format('virtio-scsi')) 297dfdc48d5SJohn Snow vm.launch() 298dfdc48d5SJohn Snow 299dfdc48d5SJohn Snow file_config = { 300dfdc48d5SJohn Snow 'driver': 'file', 301dfdc48d5SJohn Snow 'filename': drive0.path 302dfdc48d5SJohn Snow } 303dfdc48d5SJohn Snow 304dfdc48d5SJohn Snow if failure == 'intermediate': 305dfdc48d5SJohn Snow file_config = { 306dfdc48d5SJohn Snow 'driver': 'blkdebug', 307dfdc48d5SJohn Snow 'image': file_config, 308dfdc48d5SJohn Snow 'set-state': [{ 309dfdc48d5SJohn Snow 'event': 'flush_to_disk', 310dfdc48d5SJohn Snow 'state': 1, 311dfdc48d5SJohn Snow 'new_state': 2 312dfdc48d5SJohn Snow }, { 313dfdc48d5SJohn Snow 'event': 'read_aio', 314dfdc48d5SJohn Snow 'state': 2, 315dfdc48d5SJohn Snow 'new_state': 3 316dfdc48d5SJohn Snow }], 317dfdc48d5SJohn Snow 'inject-error': [{ 318dfdc48d5SJohn Snow 'event': 'read_aio', 319dfdc48d5SJohn Snow 'errno': 5, 320dfdc48d5SJohn Snow 'state': 3, 321dfdc48d5SJohn Snow 'immediately': False, 322dfdc48d5SJohn Snow 'once': True 323dfdc48d5SJohn Snow }] 324dfdc48d5SJohn Snow } 325dfdc48d5SJohn Snow 326f1648454SVladimir Sementsov-Ogievskiy drive0.node = 'drive0' 327dfdc48d5SJohn Snow vm.qmp_log('blockdev-add', 328dfdc48d5SJohn Snow filters=[iotests.filter_qmp_testfiles], 329f1648454SVladimir Sementsov-Ogievskiy node_name=drive0.node, 330dfdc48d5SJohn Snow driver=drive0.fmt, 331dfdc48d5SJohn Snow file=file_config) 332dfdc48d5SJohn Snow log('') 333dfdc48d5SJohn Snow 334dfdc48d5SJohn Snow # 0 - Writes and Reference Backup 335dfdc48d5SJohn Snow perform_writes(drive0, 0) 336dfdc48d5SJohn Snow reference_backup(drive0, 0, fbackup0) 337dfdc48d5SJohn Snow log('--- Add Bitmap ---\n') 338f1648454SVladimir Sementsov-Ogievskiy vm.qmp_log("block-dirty-bitmap-add", node=drive0.node, 339dfdc48d5SJohn Snow name="bitmap0", granularity=GRANULARITY) 340dfdc48d5SJohn Snow log('') 34132afa5a1SJohn Snow ebitmap = EmulatedBitmap() 342dfdc48d5SJohn Snow 343dfdc48d5SJohn Snow # 1 - Writes and Reference Backup 344dfdc48d5SJohn Snow bitmaps = perform_writes(drive0, 1) 34532afa5a1SJohn Snow ebitmap.dirty_group(1) 3465c4343b8SVladimir Sementsov-Ogievskiy bitmap = vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps) 34732afa5a1SJohn Snow ebitmap.compare(bitmap) 348dfdc48d5SJohn Snow reference_backup(drive0, 1, fbackup1) 349dfdc48d5SJohn Snow 3500af2a09cSJohn Snow # 1 - Test Backup (w/ Optional induced failure) 351dfdc48d5SJohn Snow if failure == 'intermediate': 352dfdc48d5SJohn Snow # Activate blkdebug induced failure for second-to-next read 353f1648454SVladimir Sementsov-Ogievskiy log(vm.hmp_qemu_io(drive0.node, 'flush')) 354dfdc48d5SJohn Snow log('') 3550af2a09cSJohn Snow job = backup(drive0, 1, bsync1, msync_mode, 3560af2a09cSJohn Snow bitmap="bitmap0", bitmap_mode=bsync_mode) 357dfdc48d5SJohn Snow 358dfdc48d5SJohn Snow def _callback(): 359dfdc48d5SJohn Snow """Issue writes while the job is open to test bitmap divergence.""" 360dfdc48d5SJohn Snow # Note: when `failure` is 'intermediate', this isn't called. 361dfdc48d5SJohn Snow log('') 36200e30f05SVladimir Sementsov-Ogievskiy bitmaps = perform_writes(drive0, 2, filter_node_name='backup-top') 363dfdc48d5SJohn Snow # Named bitmap (static, should be unchanged) 3645c4343b8SVladimir Sementsov-Ogievskiy ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', 3655c4343b8SVladimir Sementsov-Ogievskiy bitmaps=bitmaps)) 366dfdc48d5SJohn Snow # Anonymous bitmap (dynamic, shows new writes) 36732afa5a1SJohn Snow anonymous = EmulatedBitmap() 36832afa5a1SJohn Snow anonymous.dirty_group(2) 3695c4343b8SVladimir Sementsov-Ogievskiy anonymous.compare(vm.get_bitmap(drive0.node, '', recording=True, 3705c4343b8SVladimir Sementsov-Ogievskiy bitmaps=bitmaps)) 37132afa5a1SJohn Snow 37232afa5a1SJohn Snow # Simulate the order in which this will happen: 37332afa5a1SJohn Snow # group 1 gets cleared first, then group two gets written. 37432afa5a1SJohn Snow if ((bsync_mode == 'on-success' and not failure) or 37532afa5a1SJohn Snow (bsync_mode == 'always')): 376bd5ceebfSJohn Snow ebitmap.clear() 37732afa5a1SJohn Snow ebitmap.dirty_group(2) 378dfdc48d5SJohn Snow 379dfdc48d5SJohn Snow vm.run_job(job, auto_dismiss=True, auto_finalize=False, 380dfdc48d5SJohn Snow pre_finalize=_callback, 381dfdc48d5SJohn Snow cancel=(failure == 'simulated')) 3825c4343b8SVladimir Sementsov-Ogievskiy bitmaps = vm.query_bitmaps() 3835c4343b8SVladimir Sementsov-Ogievskiy log({'bitmaps': bitmaps}, indent=2) 384dfdc48d5SJohn Snow log('') 385dfdc48d5SJohn Snow 386dfdc48d5SJohn Snow if bsync_mode == 'always' and failure == 'intermediate': 387bd5ceebfSJohn Snow # TOP treats anything allocated as dirty, expect to see: 388bd5ceebfSJohn Snow if msync_mode == 'top': 389bd5ceebfSJohn Snow ebitmap.dirty_group(0) 390bd5ceebfSJohn Snow 391dfdc48d5SJohn Snow # We manage to copy one sector (one bit) before the error. 39232afa5a1SJohn Snow ebitmap.clear_bit(ebitmap.first_bit) 393bd5ceebfSJohn Snow 394bd5ceebfSJohn Snow # Full returns all bits set except what was copied/skipped 395bd5ceebfSJohn Snow if msync_mode == 'full': 396bd5ceebfSJohn Snow fail_bit = ebitmap.first_bit 397bd5ceebfSJohn Snow ebitmap.clear() 398bd5ceebfSJohn Snow ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY)) 399bd5ceebfSJohn Snow 4005c4343b8SVladimir Sementsov-Ogievskiy ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)) 401dfdc48d5SJohn Snow 402dfdc48d5SJohn Snow # 2 - Writes and Reference Backup 403dfdc48d5SJohn Snow bitmaps = perform_writes(drive0, 3) 40432afa5a1SJohn Snow ebitmap.dirty_group(3) 4055c4343b8SVladimir Sementsov-Ogievskiy ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)) 406dfdc48d5SJohn Snow reference_backup(drive0, 2, fbackup2) 407dfdc48d5SJohn Snow 408dfdc48d5SJohn Snow # 2 - Bitmap Backup (In failure modes, this is a recovery.) 4090af2a09cSJohn Snow job = backup(drive0, 2, bsync2, "bitmap", 4100af2a09cSJohn Snow bitmap="bitmap0", bitmap_mode=bsync_mode) 411dfdc48d5SJohn Snow vm.run_job(job, auto_dismiss=True, auto_finalize=False) 4125c4343b8SVladimir Sementsov-Ogievskiy bitmaps = vm.query_bitmaps() 4135c4343b8SVladimir Sementsov-Ogievskiy log({'bitmaps': bitmaps}, indent=2) 414dfdc48d5SJohn Snow log('') 41532afa5a1SJohn Snow if bsync_mode != 'never': 41632afa5a1SJohn Snow ebitmap.clear() 4175c4343b8SVladimir Sementsov-Ogievskiy ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)) 418dfdc48d5SJohn Snow 419dfdc48d5SJohn Snow log('--- Cleanup ---\n') 420dfdc48d5SJohn Snow vm.qmp_log("block-dirty-bitmap-remove", 421f1648454SVladimir Sementsov-Ogievskiy node=drive0.node, name="bitmap0") 4225c4343b8SVladimir Sementsov-Ogievskiy bitmaps = vm.query_bitmaps() 4235c4343b8SVladimir Sementsov-Ogievskiy log({'bitmaps': bitmaps}, indent=2) 424dfdc48d5SJohn Snow vm.shutdown() 425dfdc48d5SJohn Snow log('') 426dfdc48d5SJohn Snow 427dfdc48d5SJohn Snow log('--- Verification ---\n') 428dfdc48d5SJohn Snow # 'simulated' failures will actually all pass here because we canceled 429dfdc48d5SJohn Snow # while "pending". This is actually undefined behavior, 430dfdc48d5SJohn Snow # don't rely on this to be true! 431dfdc48d5SJohn Snow compare_images(bsync1, fbackup1, baseimg=fbackup0, 432dfdc48d5SJohn Snow expected_match=failure != 'intermediate') 433dfdc48d5SJohn Snow if not failure or bsync_mode == 'always': 434dfdc48d5SJohn Snow # Always keep the last backup on success or when using 'always' 435dfdc48d5SJohn Snow base = bsync1 436dfdc48d5SJohn Snow else: 437dfdc48d5SJohn Snow base = fbackup0 438dfdc48d5SJohn Snow compare_images(bsync2, fbackup2, baseimg=base) 439dfdc48d5SJohn Snow compare_images(img_path, fbackup2) 440dfdc48d5SJohn Snow log('') 441dfdc48d5SJohn Snow 442352092d3SJohn Snowdef test_backup_api(): 443352092d3SJohn Snow """ 444352092d3SJohn Snow Test malformed and prohibited invocations of the backup API. 445352092d3SJohn Snow """ 4463192fad7SNir Soffer with iotests.FilePath('img', 'bsync1') as (img_path, backup_path), \ 447352092d3SJohn Snow iotests.VM() as vm: 448352092d3SJohn Snow 449352092d3SJohn Snow log("\n=== API failure tests ===\n") 450352092d3SJohn Snow log('--- Preparing image & VM ---\n') 451352092d3SJohn Snow drive0 = Drive(img_path, vm=vm) 452352092d3SJohn Snow drive0.img_create(iotests.imgfmt, SIZE) 45322329f0dSLaurent Vivier vm.add_device("{},id=scsi0".format('virtio-scsi')) 454352092d3SJohn Snow vm.launch() 455352092d3SJohn Snow 456352092d3SJohn Snow file_config = { 457352092d3SJohn Snow 'driver': 'file', 458352092d3SJohn Snow 'filename': drive0.path 459352092d3SJohn Snow } 460352092d3SJohn Snow 461f1648454SVladimir Sementsov-Ogievskiy drive0.node = 'drive0' 462352092d3SJohn Snow vm.qmp_log('blockdev-add', 463352092d3SJohn Snow filters=[iotests.filter_qmp_testfiles], 464f1648454SVladimir Sementsov-Ogievskiy node_name=drive0.node, 465352092d3SJohn Snow driver=drive0.fmt, 466352092d3SJohn Snow file=file_config) 467352092d3SJohn Snow log('') 468352092d3SJohn Snow 469352092d3SJohn Snow target0 = Drive(backup_path, vm=vm) 470352092d3SJohn Snow target0.create_target("backup_target", drive0.fmt, drive0.size) 471352092d3SJohn Snow log('') 472352092d3SJohn Snow 473f1648454SVladimir Sementsov-Ogievskiy vm.qmp_log("block-dirty-bitmap-add", node=drive0.node, 474352092d3SJohn Snow name="bitmap0", granularity=GRANULARITY) 475352092d3SJohn Snow log('') 476352092d3SJohn Snow 477352092d3SJohn Snow log('-- Testing invalid QMP commands --\n') 478352092d3SJohn Snow 479352092d3SJohn Snow error_cases = { 480352092d3SJohn Snow 'incremental': { 481352092d3SJohn Snow None: ['on-success', 'always', 'never', None], 482352092d3SJohn Snow 'bitmap404': ['on-success', 'always', 'never', None], 483352092d3SJohn Snow 'bitmap0': ['always', 'never'] 484352092d3SJohn Snow }, 485352092d3SJohn Snow 'bitmap': { 486352092d3SJohn Snow None: ['on-success', 'always', 'never', None], 487352092d3SJohn Snow 'bitmap404': ['on-success', 'always', 'never', None], 488352092d3SJohn Snow 'bitmap0': [None], 489352092d3SJohn Snow }, 490bd5ceebfSJohn Snow 'full': { 491bd5ceebfSJohn Snow None: ['on-success', 'always', 'never'], 492bd5ceebfSJohn Snow 'bitmap404': ['on-success', 'always', 'never', None], 493bd5ceebfSJohn Snow 'bitmap0': ['never', None], 494bd5ceebfSJohn Snow }, 495bd5ceebfSJohn Snow 'top': { 496bd5ceebfSJohn Snow None: ['on-success', 'always', 'never'], 497bd5ceebfSJohn Snow 'bitmap404': ['on-success', 'always', 'never', None], 498bd5ceebfSJohn Snow 'bitmap0': ['never', None], 499bd5ceebfSJohn Snow }, 500bd5ceebfSJohn Snow 'none': { 501bd5ceebfSJohn Snow None: ['on-success', 'always', 'never'], 502bd5ceebfSJohn Snow 'bitmap404': ['on-success', 'always', 'never', None], 503bd5ceebfSJohn Snow 'bitmap0': ['on-success', 'always', 'never', None], 504bd5ceebfSJohn Snow } 505352092d3SJohn Snow } 506352092d3SJohn Snow 507352092d3SJohn Snow # Dicts, as always, are not stably-ordered prior to 3.7, so use tuples: 508bd5ceebfSJohn Snow for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'): 509352092d3SJohn Snow log("-- Sync mode {:s} tests --\n".format(sync_mode)) 510352092d3SJohn Snow for bitmap in (None, 'bitmap404', 'bitmap0'): 511352092d3SJohn Snow for policy in error_cases[sync_mode][bitmap]: 512f1648454SVladimir Sementsov-Ogievskiy blockdev_backup(drive0.vm, drive0.node, "backup_target", 513352092d3SJohn Snow sync_mode, job_id='api_job', 514352092d3SJohn Snow bitmap=bitmap, bitmap_mode=policy) 515352092d3SJohn Snow log('') 516352092d3SJohn Snow 517352092d3SJohn Snow 518dfdc48d5SJohn Snowdef main(): 519dfdc48d5SJohn Snow for bsync_mode in ("never", "on-success", "always"): 520dfdc48d5SJohn Snow for failure in ("simulated", "intermediate", None): 5210af2a09cSJohn Snow test_bitmap_sync(bsync_mode, "bitmap", failure) 522dfdc48d5SJohn Snow 523bd5ceebfSJohn Snow for sync_mode in ('full', 'top'): 524bd5ceebfSJohn Snow for bsync_mode in ('on-success', 'always'): 525bd5ceebfSJohn Snow for failure in ('simulated', 'intermediate', None): 526bd5ceebfSJohn Snow test_bitmap_sync(bsync_mode, sync_mode, failure) 527bd5ceebfSJohn Snow 528352092d3SJohn Snow test_backup_api() 529352092d3SJohn Snow 530dfdc48d5SJohn Snowif __name__ == '__main__': 531103cbc77SMax Reitz iotests.script_main(main, supported_fmts=['qcow2'], 532103cbc77SMax Reitz supported_protocols=['file']) 533