19f7264f5SJohn Snow#!/usr/bin/env python 29f7264f5SJohn Snow# 39f7264f5SJohn Snow# Tests for incremental drive-backup 49f7264f5SJohn Snow# 59f7264f5SJohn Snow# Copyright (C) 2015 John Snow for Red Hat, Inc. 69f7264f5SJohn Snow# 79f7264f5SJohn Snow# Based on 056. 89f7264f5SJohn Snow# 99f7264f5SJohn Snow# This program is free software; you can redistribute it and/or modify 109f7264f5SJohn Snow# it under the terms of the GNU General Public License as published by 119f7264f5SJohn Snow# the Free Software Foundation; either version 2 of the License, or 129f7264f5SJohn Snow# (at your option) any later version. 139f7264f5SJohn Snow# 149f7264f5SJohn Snow# This program is distributed in the hope that it will be useful, 159f7264f5SJohn Snow# but WITHOUT ANY WARRANTY; without even the implied warranty of 169f7264f5SJohn Snow# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 179f7264f5SJohn Snow# GNU General Public License for more details. 189f7264f5SJohn Snow# 199f7264f5SJohn Snow# You should have received a copy of the GNU General Public License 209f7264f5SJohn Snow# along with this program. If not, see <http://www.gnu.org/licenses/>. 219f7264f5SJohn Snow# 229f7264f5SJohn Snow 239f7264f5SJohn Snowimport os 249f7264f5SJohn Snowimport iotests 259f7264f5SJohn Snow 269f7264f5SJohn Snow 279f7264f5SJohn Snowdef io_write_patterns(img, patterns): 289f7264f5SJohn Snow for pattern in patterns: 299f7264f5SJohn Snow iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img) 309f7264f5SJohn Snow 319f7264f5SJohn Snow 32a3d71595SJohn Snowdef try_remove(img): 33a3d71595SJohn Snow try: 34a3d71595SJohn Snow os.remove(img) 35a3d71595SJohn Snow except OSError: 36a3d71595SJohn Snow pass 37a3d71595SJohn Snow 38a3d71595SJohn Snow 39a3d71595SJohn Snowclass Bitmap: 40a3d71595SJohn Snow def __init__(self, name, drive): 41a3d71595SJohn Snow self.name = name 42a3d71595SJohn Snow self.drive = drive 43a3d71595SJohn Snow self.num = 0 44a3d71595SJohn Snow self.backups = list() 45a3d71595SJohn Snow 46a3d71595SJohn Snow def base_target(self): 47a3d71595SJohn Snow return (self.drive['backup'], None) 48a3d71595SJohn Snow 49a3d71595SJohn Snow def new_target(self, num=None): 50a3d71595SJohn Snow if num is None: 51a3d71595SJohn Snow num = self.num 52a3d71595SJohn Snow self.num = num + 1 53a3d71595SJohn Snow base = os.path.join(iotests.test_dir, 54a3d71595SJohn Snow "%s.%s." % (self.drive['id'], self.name)) 55a3d71595SJohn Snow suff = "%i.%s" % (num, self.drive['fmt']) 56a3d71595SJohn Snow target = base + "inc" + suff 57a3d71595SJohn Snow reference = base + "ref" + suff 58a3d71595SJohn Snow self.backups.append((target, reference)) 59a3d71595SJohn Snow return (target, reference) 60a3d71595SJohn Snow 61a3d71595SJohn Snow def last_target(self): 62a3d71595SJohn Snow if self.backups: 63a3d71595SJohn Snow return self.backups[-1] 64a3d71595SJohn Snow return self.base_target() 65a3d71595SJohn Snow 66a3d71595SJohn Snow def del_target(self): 67a3d71595SJohn Snow for image in self.backups.pop(): 68a3d71595SJohn Snow try_remove(image) 69a3d71595SJohn Snow self.num -= 1 70a3d71595SJohn Snow 71a3d71595SJohn Snow def cleanup(self): 72a3d71595SJohn Snow for backup in self.backups: 73a3d71595SJohn Snow for image in backup: 74a3d71595SJohn Snow try_remove(image) 75a3d71595SJohn Snow 76a3d71595SJohn Snow 779f7264f5SJohn Snowclass TestIncrementalBackup(iotests.QMPTestCase): 789f7264f5SJohn Snow def setUp(self): 799f7264f5SJohn Snow self.bitmaps = list() 809f7264f5SJohn Snow self.files = list() 819f7264f5SJohn Snow self.drives = list() 829f7264f5SJohn Snow self.vm = iotests.VM() 839f7264f5SJohn Snow self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt) 849f7264f5SJohn Snow 859f7264f5SJohn Snow # Create a base image with a distinctive patterning 869f7264f5SJohn Snow drive0 = self.add_node('drive0') 879f7264f5SJohn Snow self.img_create(drive0['file'], drive0['fmt']) 889f7264f5SJohn Snow self.vm.add_drive(drive0['file']) 899f7264f5SJohn Snow io_write_patterns(drive0['file'], (('0x41', 0, 512), 909f7264f5SJohn Snow ('0xd5', '1M', '32k'), 919f7264f5SJohn Snow ('0xdc', '32M', '124k'))) 929f7264f5SJohn Snow self.vm.launch() 939f7264f5SJohn Snow 949f7264f5SJohn Snow 959f7264f5SJohn Snow def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None): 969f7264f5SJohn Snow if path is None: 979f7264f5SJohn Snow path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt)) 989f7264f5SJohn Snow if backup is None: 999f7264f5SJohn Snow backup = os.path.join(iotests.test_dir, 1009f7264f5SJohn Snow '%s.full.backup.%s' % (node_id, fmt)) 1019f7264f5SJohn Snow 1029f7264f5SJohn Snow self.drives.append({ 1039f7264f5SJohn Snow 'id': node_id, 1049f7264f5SJohn Snow 'file': path, 1059f7264f5SJohn Snow 'backup': backup, 1069f7264f5SJohn Snow 'fmt': fmt }) 1079f7264f5SJohn Snow return self.drives[-1] 1089f7264f5SJohn Snow 1099f7264f5SJohn Snow 1109f7264f5SJohn Snow def img_create(self, img, fmt=iotests.imgfmt, size='64M', 1119f7264f5SJohn Snow parent=None, parentFormat=None): 1129f7264f5SJohn Snow if parent: 1139f7264f5SJohn Snow if parentFormat is None: 1149f7264f5SJohn Snow parentFormat = fmt 1159f7264f5SJohn Snow iotests.qemu_img('create', '-f', fmt, img, size, 1169f7264f5SJohn Snow '-b', parent, '-F', parentFormat) 1179f7264f5SJohn Snow else: 1189f7264f5SJohn Snow iotests.qemu_img('create', '-f', fmt, img, size) 1199f7264f5SJohn Snow self.files.append(img) 1209f7264f5SJohn Snow 121a3d71595SJohn Snow 122a3d71595SJohn Snow def do_qmp_backup(self, error='Input/output error', **kwargs): 123a3d71595SJohn Snow res = self.vm.qmp('drive-backup', **kwargs) 124a3d71595SJohn Snow self.assert_qmp(res, 'return', {}) 125a3d71595SJohn Snow 126a3d71595SJohn Snow event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED", 127a3d71595SJohn Snow match={'data': {'device': kwargs['device']}}) 128*ff793890SJohn Snow self.assertNotEqual(event, None) 129a3d71595SJohn Snow 130a3d71595SJohn Snow try: 131a3d71595SJohn Snow failure = self.dictpath(event, 'data/error') 132a3d71595SJohn Snow except AssertionError: 133a3d71595SJohn Snow # Backup succeeded. 134a3d71595SJohn Snow self.assert_qmp(event, 'data/offset', event['data']['len']) 135a3d71595SJohn Snow return True 136a3d71595SJohn Snow else: 137a3d71595SJohn Snow # Backup failed. 138a3d71595SJohn Snow self.assert_qmp(event, 'data/error', error) 139a3d71595SJohn Snow return False 140a3d71595SJohn Snow 141a3d71595SJohn Snow 142a3d71595SJohn Snow def create_anchor_backup(self, drive=None): 143a3d71595SJohn Snow if drive is None: 144a3d71595SJohn Snow drive = self.drives[-1] 145a3d71595SJohn Snow res = self.do_qmp_backup(device=drive['id'], sync='full', 146a3d71595SJohn Snow format=drive['fmt'], target=drive['backup']) 147a3d71595SJohn Snow self.assertTrue(res) 148a3d71595SJohn Snow self.files.append(drive['backup']) 149a3d71595SJohn Snow return drive['backup'] 150a3d71595SJohn Snow 151a3d71595SJohn Snow 152a3d71595SJohn Snow def make_reference_backup(self, bitmap=None): 153a3d71595SJohn Snow if bitmap is None: 154a3d71595SJohn Snow bitmap = self.bitmaps[-1] 155a3d71595SJohn Snow _, reference = bitmap.last_target() 156a3d71595SJohn Snow res = self.do_qmp_backup(device=bitmap.drive['id'], sync='full', 157a3d71595SJohn Snow format=bitmap.drive['fmt'], target=reference) 158a3d71595SJohn Snow self.assertTrue(res) 159a3d71595SJohn Snow 160a3d71595SJohn Snow 16159fc5d84SJohn Snow def add_bitmap(self, name, drive, **kwargs): 162a3d71595SJohn Snow bitmap = Bitmap(name, drive) 163a3d71595SJohn Snow self.bitmaps.append(bitmap) 164a3d71595SJohn Snow result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'], 16559fc5d84SJohn Snow name=bitmap.name, **kwargs) 166a3d71595SJohn Snow self.assert_qmp(result, 'return', {}) 167a3d71595SJohn Snow return bitmap 168a3d71595SJohn Snow 169a3d71595SJohn Snow 170a3d71595SJohn Snow def prepare_backup(self, bitmap=None, parent=None): 171a3d71595SJohn Snow if bitmap is None: 172a3d71595SJohn Snow bitmap = self.bitmaps[-1] 173a3d71595SJohn Snow if parent is None: 174a3d71595SJohn Snow parent, _ = bitmap.last_target() 175a3d71595SJohn Snow 176a3d71595SJohn Snow target, _ = bitmap.new_target() 177a3d71595SJohn Snow self.img_create(target, bitmap.drive['fmt'], parent=parent) 178a3d71595SJohn Snow return target 179a3d71595SJohn Snow 180a3d71595SJohn Snow 181a3d71595SJohn Snow def create_incremental(self, bitmap=None, parent=None, 182a3d71595SJohn Snow parentFormat=None, validate=True): 183a3d71595SJohn Snow if bitmap is None: 184a3d71595SJohn Snow bitmap = self.bitmaps[-1] 185a3d71595SJohn Snow if parent is None: 186a3d71595SJohn Snow parent, _ = bitmap.last_target() 187a3d71595SJohn Snow 188a3d71595SJohn Snow target = self.prepare_backup(bitmap, parent) 189a3d71595SJohn Snow res = self.do_qmp_backup(device=bitmap.drive['id'], 190a3d71595SJohn Snow sync='dirty-bitmap', bitmap=bitmap.name, 191a3d71595SJohn Snow format=bitmap.drive['fmt'], target=target, 192a3d71595SJohn Snow mode='existing') 193a3d71595SJohn Snow if not res: 194a3d71595SJohn Snow bitmap.del_target(); 195a3d71595SJohn Snow self.assertFalse(validate) 196a3d71595SJohn Snow else: 197a3d71595SJohn Snow self.make_reference_backup(bitmap) 198a3d71595SJohn Snow return res 199a3d71595SJohn Snow 200a3d71595SJohn Snow 201a3d71595SJohn Snow def check_backups(self): 202a3d71595SJohn Snow for bitmap in self.bitmaps: 203a3d71595SJohn Snow for incremental, reference in bitmap.backups: 204a3d71595SJohn Snow self.assertTrue(iotests.compare_images(incremental, reference)) 205a3d71595SJohn Snow last = bitmap.last_target()[0] 206a3d71595SJohn Snow self.assertTrue(iotests.compare_images(last, bitmap.drive['file'])) 207a3d71595SJohn Snow 208a3d71595SJohn Snow 209a3d71595SJohn Snow def hmp_io_writes(self, drive, patterns): 210a3d71595SJohn Snow for pattern in patterns: 211a3d71595SJohn Snow self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern) 212a3d71595SJohn Snow self.vm.hmp_qemu_io(drive, 'flush') 213a3d71595SJohn Snow 214a3d71595SJohn Snow 21559fc5d84SJohn Snow def do_incremental_simple(self, **kwargs): 216a3d71595SJohn Snow self.create_anchor_backup() 21759fc5d84SJohn Snow self.add_bitmap('bitmap0', self.drives[0], **kwargs) 218a3d71595SJohn Snow 219a3d71595SJohn Snow # Sanity: Create a "hollow" incremental backup 220a3d71595SJohn Snow self.create_incremental() 221a3d71595SJohn Snow # Three writes: One complete overwrite, one new segment, 222a3d71595SJohn Snow # and one partial overlap. 223a3d71595SJohn Snow self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512), 224a3d71595SJohn Snow ('0xfe', '16M', '256k'), 225a3d71595SJohn Snow ('0x64', '32736k', '64k'))) 226a3d71595SJohn Snow self.create_incremental() 227a3d71595SJohn Snow # Three more writes, one of each kind, like above 228a3d71595SJohn Snow self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512), 229a3d71595SJohn Snow ('0x55', '8M', '352k'), 230a3d71595SJohn Snow ('0x78', '15872k', '1M'))) 231a3d71595SJohn Snow self.create_incremental() 232a3d71595SJohn Snow self.vm.shutdown() 233a3d71595SJohn Snow self.check_backups() 234a3d71595SJohn Snow 235a3d71595SJohn Snow 23659fc5d84SJohn Snow def test_incremental_simple(self): 23759fc5d84SJohn Snow ''' 23859fc5d84SJohn Snow Test: Create and verify three incremental backups. 23959fc5d84SJohn Snow 24059fc5d84SJohn Snow Create a bitmap and a full backup before VM execution begins, 24159fc5d84SJohn Snow then create a series of three incremental backups "during execution," 24259fc5d84SJohn Snow i.e.; after IO requests begin modifying the drive. 24359fc5d84SJohn Snow ''' 24459fc5d84SJohn Snow return self.do_incremental_simple() 24559fc5d84SJohn Snow 24659fc5d84SJohn Snow 24759fc5d84SJohn Snow def test_small_granularity(self): 24859fc5d84SJohn Snow ''' 24959fc5d84SJohn Snow Test: Create and verify backups made with a small granularity bitmap. 25059fc5d84SJohn Snow 25159fc5d84SJohn Snow Perform the same test as test_incremental_simple, but with a granularity 25259fc5d84SJohn Snow of only 32KiB instead of the present default of 64KiB. 25359fc5d84SJohn Snow ''' 25459fc5d84SJohn Snow return self.do_incremental_simple(granularity=32768) 25559fc5d84SJohn Snow 25659fc5d84SJohn Snow 25759fc5d84SJohn Snow def test_large_granularity(self): 25859fc5d84SJohn Snow ''' 25959fc5d84SJohn Snow Test: Create and verify backups made with a large granularity bitmap. 26059fc5d84SJohn Snow 26159fc5d84SJohn Snow Perform the same test as test_incremental_simple, but with a granularity 26259fc5d84SJohn Snow of 128KiB instead of the present default of 64KiB. 26359fc5d84SJohn Snow ''' 26459fc5d84SJohn Snow return self.do_incremental_simple(granularity=131072) 26559fc5d84SJohn Snow 26659fc5d84SJohn Snow 26724618f53SJohn Snow def test_incremental_failure(self): 26824618f53SJohn Snow '''Test: Verify backups made after a failure are correct. 26924618f53SJohn Snow 27024618f53SJohn Snow Simulate a failure during an incremental backup block job, 27124618f53SJohn Snow emulate additional writes, then create another incremental backup 27224618f53SJohn Snow afterwards and verify that the backup created is correct. 27324618f53SJohn Snow ''' 27424618f53SJohn Snow 27524618f53SJohn Snow # Create a blkdebug interface to this img as 'drive1', 27624618f53SJohn Snow # but don't actually create a new image. 27724618f53SJohn Snow drive1 = self.add_node('drive1', self.drives[0]['fmt'], 27824618f53SJohn Snow path=self.drives[0]['file'], 27924618f53SJohn Snow backup=self.drives[0]['backup']) 28024618f53SJohn Snow result = self.vm.qmp('blockdev-add', options={ 28124618f53SJohn Snow 'id': drive1['id'], 28224618f53SJohn Snow 'driver': drive1['fmt'], 28324618f53SJohn Snow 'file': { 28424618f53SJohn Snow 'driver': 'blkdebug', 28524618f53SJohn Snow 'image': { 28624618f53SJohn Snow 'driver': 'file', 28724618f53SJohn Snow 'filename': drive1['file'] 28824618f53SJohn Snow }, 28924618f53SJohn Snow 'set-state': [{ 29024618f53SJohn Snow 'event': 'flush_to_disk', 29124618f53SJohn Snow 'state': 1, 29224618f53SJohn Snow 'new_state': 2 29324618f53SJohn Snow }], 29424618f53SJohn Snow 'inject-error': [{ 29524618f53SJohn Snow 'event': 'read_aio', 29624618f53SJohn Snow 'errno': 5, 29724618f53SJohn Snow 'state': 2, 29824618f53SJohn Snow 'immediately': False, 29924618f53SJohn Snow 'once': True 30024618f53SJohn Snow }], 30124618f53SJohn Snow } 30224618f53SJohn Snow }) 30324618f53SJohn Snow self.assert_qmp(result, 'return', {}) 30424618f53SJohn Snow 30524618f53SJohn Snow self.create_anchor_backup(self.drives[0]) 30624618f53SJohn Snow self.add_bitmap('bitmap0', drive1) 30724618f53SJohn Snow # Note: at this point, during a normal execution, 30824618f53SJohn Snow # Assume that the VM resumes and begins issuing IO requests here. 30924618f53SJohn Snow 31024618f53SJohn Snow self.hmp_io_writes(drive1['id'], (('0xab', 0, 512), 31124618f53SJohn Snow ('0xfe', '16M', '256k'), 31224618f53SJohn Snow ('0x64', '32736k', '64k'))) 31324618f53SJohn Snow 31424618f53SJohn Snow result = self.create_incremental(validate=False) 31524618f53SJohn Snow self.assertFalse(result) 31624618f53SJohn Snow self.hmp_io_writes(drive1['id'], (('0x9a', 0, 512), 31724618f53SJohn Snow ('0x55', '8M', '352k'), 31824618f53SJohn Snow ('0x78', '15872k', '1M'))) 31924618f53SJohn Snow self.create_incremental() 32024618f53SJohn Snow self.vm.shutdown() 32124618f53SJohn Snow self.check_backups() 32224618f53SJohn Snow 32324618f53SJohn Snow 3249f7264f5SJohn Snow def test_sync_dirty_bitmap_missing(self): 3259f7264f5SJohn Snow self.assert_no_active_block_jobs() 3269f7264f5SJohn Snow self.files.append(self.err_img) 3279f7264f5SJohn Snow result = self.vm.qmp('drive-backup', device=self.drives[0]['id'], 3289f7264f5SJohn Snow sync='dirty-bitmap', format=self.drives[0]['fmt'], 3299f7264f5SJohn Snow target=self.err_img) 3309f7264f5SJohn Snow self.assert_qmp(result, 'error/class', 'GenericError') 3319f7264f5SJohn Snow 3329f7264f5SJohn Snow 3339f7264f5SJohn Snow def test_sync_dirty_bitmap_not_found(self): 3349f7264f5SJohn Snow self.assert_no_active_block_jobs() 3359f7264f5SJohn Snow self.files.append(self.err_img) 3369f7264f5SJohn Snow result = self.vm.qmp('drive-backup', device=self.drives[0]['id'], 3379f7264f5SJohn Snow sync='dirty-bitmap', bitmap='unknown', 3389f7264f5SJohn Snow format=self.drives[0]['fmt'], target=self.err_img) 3399f7264f5SJohn Snow self.assert_qmp(result, 'error/class', 'GenericError') 3409f7264f5SJohn Snow 3419f7264f5SJohn Snow 34259fc5d84SJohn Snow def test_sync_dirty_bitmap_bad_granularity(self): 34359fc5d84SJohn Snow ''' 34459fc5d84SJohn Snow Test: Test what happens if we provide an improper granularity. 34559fc5d84SJohn Snow 34659fc5d84SJohn Snow The granularity must always be a power of 2. 34759fc5d84SJohn Snow ''' 34859fc5d84SJohn Snow self.assert_no_active_block_jobs() 34959fc5d84SJohn Snow self.assertRaises(AssertionError, self.add_bitmap, 35059fc5d84SJohn Snow 'bitmap0', self.drives[0], 35159fc5d84SJohn Snow granularity=64000) 35259fc5d84SJohn Snow 35359fc5d84SJohn Snow 3549f7264f5SJohn Snow def tearDown(self): 3559f7264f5SJohn Snow self.vm.shutdown() 356a3d71595SJohn Snow for bitmap in self.bitmaps: 357a3d71595SJohn Snow bitmap.cleanup() 3589f7264f5SJohn Snow for filename in self.files: 359a3d71595SJohn Snow try_remove(filename) 3609f7264f5SJohn Snow 3619f7264f5SJohn Snow 3629f7264f5SJohn Snowif __name__ == '__main__': 3639f7264f5SJohn Snow iotests.main(supported_fmts=['qcow2']) 364