1#!/usr/bin/env python 2# 3# Tests for incremental drive-backup 4# 5# Copyright (C) 2015 John Snow for Red Hat, Inc. 6# 7# Based on 056. 8# 9# This program is free software; you can redistribute it and/or modify 10# it under the terms of the GNU General Public License as published by 11# the Free Software Foundation; either version 2 of the License, or 12# (at your option) any later version. 13# 14# This program is distributed in the hope that it will be useful, 15# but WITHOUT ANY WARRANTY; without even the implied warranty of 16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17# GNU General Public License for more details. 18# 19# You should have received a copy of the GNU General Public License 20# along with this program. If not, see <http://www.gnu.org/licenses/>. 21# 22 23import os 24import iotests 25 26 27def io_write_patterns(img, patterns): 28 for pattern in patterns: 29 iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img) 30 31 32def try_remove(img): 33 try: 34 os.remove(img) 35 except OSError: 36 pass 37 38 39def transaction_action(action, **kwargs): 40 return { 41 'type': action, 42 'data': dict((k.replace('_', '-'), v) for k, v in kwargs.items()) 43 } 44 45 46def transaction_bitmap_clear(node, name, **kwargs): 47 return transaction_action('block-dirty-bitmap-clear', 48 node=node, name=name, **kwargs) 49 50 51def transaction_drive_backup(device, target, **kwargs): 52 return transaction_action('drive-backup', job_id=device, device=device, 53 target=target, **kwargs) 54 55 56class Bitmap: 57 def __init__(self, name, drive): 58 self.name = name 59 self.drive = drive 60 self.num = 0 61 self.backups = list() 62 63 def base_target(self): 64 return (self.drive['backup'], None) 65 66 def new_target(self, num=None): 67 if num is None: 68 num = self.num 69 self.num = num + 1 70 base = os.path.join(iotests.test_dir, 71 "%s.%s." % (self.drive['id'], self.name)) 72 suff = "%i.%s" % (num, self.drive['fmt']) 73 target = base + "inc" + suff 74 reference = base + "ref" + suff 75 self.backups.append((target, reference)) 76 return (target, reference) 77 78 def last_target(self): 79 if self.backups: 80 return self.backups[-1] 81 return self.base_target() 82 83 def del_target(self): 84 for image in self.backups.pop(): 85 try_remove(image) 86 self.num -= 1 87 88 def cleanup(self): 89 for backup in self.backups: 90 for image in backup: 91 try_remove(image) 92 93 94class TestIncrementalBackupBase(iotests.QMPTestCase): 95 def __init__(self, *args): 96 super(TestIncrementalBackupBase, self).__init__(*args) 97 self.bitmaps = list() 98 self.files = list() 99 self.drives = list() 100 self.vm = iotests.VM() 101 self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt) 102 103 104 def setUp(self): 105 # Create a base image with a distinctive patterning 106 drive0 = self.add_node('drive0') 107 self.img_create(drive0['file'], drive0['fmt']) 108 self.vm.add_drive(drive0['file']) 109 self.write_default_pattern(drive0['file']) 110 self.vm.launch() 111 112 113 def write_default_pattern(self, target): 114 io_write_patterns(target, (('0x41', 0, 512), 115 ('0xd5', '1M', '32k'), 116 ('0xdc', '32M', '124k'))) 117 118 119 def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None): 120 if path is None: 121 path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt)) 122 if backup is None: 123 backup = os.path.join(iotests.test_dir, 124 '%s.full.backup.%s' % (node_id, fmt)) 125 126 self.drives.append({ 127 'id': node_id, 128 'file': path, 129 'backup': backup, 130 'fmt': fmt }) 131 return self.drives[-1] 132 133 134 def img_create(self, img, fmt=iotests.imgfmt, size='64M', 135 parent=None, parentFormat=None, **kwargs): 136 optargs = [] 137 for k,v in kwargs.items(): 138 optargs = optargs + ['-o', '%s=%s' % (k,v)] 139 args = ['create', '-f', fmt] + optargs + [img, size] 140 if parent: 141 if parentFormat is None: 142 parentFormat = fmt 143 args = args + ['-b', parent, '-F', parentFormat] 144 iotests.qemu_img(*args) 145 self.files.append(img) 146 147 148 def do_qmp_backup(self, error='Input/output error', **kwargs): 149 res = self.vm.qmp('drive-backup', **kwargs) 150 self.assert_qmp(res, 'return', {}) 151 return self.wait_qmp_backup(kwargs['device'], error) 152 153 154 def ignore_job_status_change_events(self): 155 while True: 156 e = self.vm.event_wait(name="JOB_STATUS_CHANGE") 157 if e['data']['status'] == 'null': 158 break 159 160 def wait_qmp_backup(self, device, error='Input/output error'): 161 event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED", 162 match={'data': {'device': device}}) 163 self.assertNotEqual(event, None) 164 self.ignore_job_status_change_events() 165 166 try: 167 failure = self.dictpath(event, 'data/error') 168 except AssertionError: 169 # Backup succeeded. 170 self.assert_qmp(event, 'data/offset', event['data']['len']) 171 return True 172 else: 173 # Backup failed. 174 self.assert_qmp(event, 'data/error', error) 175 return False 176 177 178 def wait_qmp_backup_cancelled(self, device): 179 event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED', 180 match={'data': {'device': device}}) 181 self.assertNotEqual(event, None) 182 self.ignore_job_status_change_events() 183 184 185 def create_anchor_backup(self, drive=None): 186 if drive is None: 187 drive = self.drives[-1] 188 res = self.do_qmp_backup(job_id=drive['id'], 189 device=drive['id'], sync='full', 190 format=drive['fmt'], target=drive['backup']) 191 self.assertTrue(res) 192 self.files.append(drive['backup']) 193 return drive['backup'] 194 195 196 def make_reference_backup(self, bitmap=None): 197 if bitmap is None: 198 bitmap = self.bitmaps[-1] 199 _, reference = bitmap.last_target() 200 res = self.do_qmp_backup(job_id=bitmap.drive['id'], 201 device=bitmap.drive['id'], sync='full', 202 format=bitmap.drive['fmt'], target=reference) 203 self.assertTrue(res) 204 205 206 def add_bitmap(self, name, drive, **kwargs): 207 bitmap = Bitmap(name, drive) 208 self.bitmaps.append(bitmap) 209 result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'], 210 name=bitmap.name, **kwargs) 211 self.assert_qmp(result, 'return', {}) 212 return bitmap 213 214 215 def prepare_backup(self, bitmap=None, parent=None): 216 if bitmap is None: 217 bitmap = self.bitmaps[-1] 218 if parent is None: 219 parent, _ = bitmap.last_target() 220 221 target, _ = bitmap.new_target() 222 self.img_create(target, bitmap.drive['fmt'], parent=parent) 223 return target 224 225 226 def create_incremental(self, bitmap=None, parent=None, 227 parentFormat=None, validate=True): 228 if bitmap is None: 229 bitmap = self.bitmaps[-1] 230 if parent is None: 231 parent, _ = bitmap.last_target() 232 233 target = self.prepare_backup(bitmap, parent) 234 res = self.do_qmp_backup(job_id=bitmap.drive['id'], 235 device=bitmap.drive['id'], 236 sync='incremental', bitmap=bitmap.name, 237 format=bitmap.drive['fmt'], target=target, 238 mode='existing') 239 if not res: 240 bitmap.del_target(); 241 self.assertFalse(validate) 242 else: 243 self.make_reference_backup(bitmap) 244 return res 245 246 247 def check_backups(self): 248 for bitmap in self.bitmaps: 249 for incremental, reference in bitmap.backups: 250 self.assertTrue(iotests.compare_images(incremental, reference)) 251 last = bitmap.last_target()[0] 252 self.assertTrue(iotests.compare_images(last, bitmap.drive['file'])) 253 254 255 def hmp_io_writes(self, drive, patterns): 256 for pattern in patterns: 257 self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern) 258 self.vm.hmp_qemu_io(drive, 'flush') 259 260 261 def do_incremental_simple(self, **kwargs): 262 self.create_anchor_backup() 263 self.add_bitmap('bitmap0', self.drives[0], **kwargs) 264 265 # Sanity: Create a "hollow" incremental backup 266 self.create_incremental() 267 # Three writes: One complete overwrite, one new segment, 268 # and one partial overlap. 269 self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512), 270 ('0xfe', '16M', '256k'), 271 ('0x64', '32736k', '64k'))) 272 self.create_incremental() 273 # Three more writes, one of each kind, like above 274 self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512), 275 ('0x55', '8M', '352k'), 276 ('0x78', '15872k', '1M'))) 277 self.create_incremental() 278 self.vm.shutdown() 279 self.check_backups() 280 281 282 def tearDown(self): 283 self.vm.shutdown() 284 for bitmap in self.bitmaps: 285 bitmap.cleanup() 286 for filename in self.files: 287 try_remove(filename) 288 289 290 291class TestIncrementalBackup(TestIncrementalBackupBase): 292 def test_incremental_simple(self): 293 ''' 294 Test: Create and verify three incremental backups. 295 296 Create a bitmap and a full backup before VM execution begins, 297 then create a series of three incremental backups "during execution," 298 i.e.; after IO requests begin modifying the drive. 299 ''' 300 return self.do_incremental_simple() 301 302 303 def test_small_granularity(self): 304 ''' 305 Test: Create and verify backups made with a small granularity bitmap. 306 307 Perform the same test as test_incremental_simple, but with a granularity 308 of only 32KiB instead of the present default of 64KiB. 309 ''' 310 return self.do_incremental_simple(granularity=32768) 311 312 313 def test_large_granularity(self): 314 ''' 315 Test: Create and verify backups made with a large granularity bitmap. 316 317 Perform the same test as test_incremental_simple, but with a granularity 318 of 128KiB instead of the present default of 64KiB. 319 ''' 320 return self.do_incremental_simple(granularity=131072) 321 322 323 def test_larger_cluster_target(self): 324 ''' 325 Test: Create and verify backups made to a larger cluster size target. 326 327 With a default granularity of 64KiB, verify that backups made to a 328 larger cluster size target of 128KiB without a backing file works. 329 ''' 330 drive0 = self.drives[0] 331 332 # Create a cluster_size=128k full backup / "anchor" backup 333 self.img_create(drive0['backup'], cluster_size='128k') 334 self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full', 335 format=drive0['fmt'], 336 target=drive0['backup'], 337 mode='existing')) 338 339 # Create bitmap and dirty it with some new writes. 340 # overwrite [32736, 32799] which will dirty bitmap clusters at 341 # 32M-64K and 32M. 32M+64K will be left undirtied. 342 bitmap0 = self.add_bitmap('bitmap0', drive0) 343 self.hmp_io_writes(drive0['id'], 344 (('0xab', 0, 512), 345 ('0xfe', '16M', '256k'), 346 ('0x64', '32736k', '64k'))) 347 # Check the dirty bitmap stats 348 result = self.vm.qmp('query-block') 349 self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/name', 'bitmap0') 350 self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/count', 458752) 351 self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/granularity', 65536) 352 self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/status', 'active') 353 self.assert_qmp(result, 'return[0]/dirty-bitmaps[0]/persistent', False) 354 355 # Prepare a cluster_size=128k backup target without a backing file. 356 (target, _) = bitmap0.new_target() 357 self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k') 358 359 # Perform Incremental Backup 360 self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'], 361 sync='incremental', 362 bitmap=bitmap0.name, 363 format=bitmap0.drive['fmt'], 364 target=target, 365 mode='existing')) 366 self.make_reference_backup(bitmap0) 367 368 # Add the backing file, then compare and exit. 369 iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b', 370 drive0['backup'], '-F', drive0['fmt'], target) 371 self.vm.shutdown() 372 self.check_backups() 373 374 375 def test_incremental_transaction(self): 376 '''Test: Verify backups made from transactionally created bitmaps. 377 378 Create a bitmap "before" VM execution begins, then create a second 379 bitmap AFTER writes have already occurred. Use transactions to create 380 a full backup and synchronize both bitmaps to this backup. 381 Create an incremental backup through both bitmaps and verify that 382 both backups match the current drive0 image. 383 ''' 384 385 drive0 = self.drives[0] 386 bitmap0 = self.add_bitmap('bitmap0', drive0) 387 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512), 388 ('0xfe', '16M', '256k'), 389 ('0x64', '32736k', '64k'))) 390 bitmap1 = self.add_bitmap('bitmap1', drive0) 391 392 result = self.vm.qmp('transaction', actions=[ 393 transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name), 394 transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name), 395 transaction_drive_backup(drive0['id'], drive0['backup'], 396 sync='full', format=drive0['fmt']) 397 ]) 398 self.assert_qmp(result, 'return', {}) 399 self.wait_until_completed(drive0['id']) 400 self.files.append(drive0['backup']) 401 402 self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512), 403 ('0x55', '8M', '352k'), 404 ('0x78', '15872k', '1M'))) 405 # Both bitmaps should be correctly in sync. 406 self.create_incremental(bitmap0) 407 self.create_incremental(bitmap1) 408 self.vm.shutdown() 409 self.check_backups() 410 411 412 def do_transaction_failure_test(self, race=False): 413 # Create a second drive, with pattern: 414 drive1 = self.add_node('drive1') 415 self.img_create(drive1['file'], drive1['fmt']) 416 io_write_patterns(drive1['file'], (('0x14', 0, 512), 417 ('0x5d', '1M', '32k'), 418 ('0xcd', '32M', '124k'))) 419 420 # Create a blkdebug interface to this img as 'drive1' 421 result = self.vm.qmp('blockdev-add', 422 node_name=drive1['id'], 423 driver=drive1['fmt'], 424 file={ 425 'driver': 'blkdebug', 426 'image': { 427 'driver': 'file', 428 'filename': drive1['file'] 429 }, 430 'set-state': [{ 431 'event': 'flush_to_disk', 432 'state': 1, 433 'new_state': 2 434 }], 435 'inject-error': [{ 436 'event': 'read_aio', 437 'errno': 5, 438 'state': 2, 439 'immediately': False, 440 'once': True 441 }], 442 } 443 ) 444 self.assert_qmp(result, 'return', {}) 445 446 # Create bitmaps and full backups for both drives 447 drive0 = self.drives[0] 448 dr0bm0 = self.add_bitmap('bitmap0', drive0) 449 dr1bm0 = self.add_bitmap('bitmap0', drive1) 450 self.create_anchor_backup(drive0) 451 self.create_anchor_backup(drive1) 452 self.assert_no_active_block_jobs() 453 self.assertFalse(self.vm.get_qmp_events(wait=False)) 454 455 # Emulate some writes 456 if not race: 457 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512), 458 ('0xfe', '16M', '256k'), 459 ('0x64', '32736k', '64k'))) 460 self.hmp_io_writes(drive1['id'], (('0xba', 0, 512), 461 ('0xef', '16M', '256k'), 462 ('0x46', '32736k', '64k'))) 463 464 # Create incremental backup targets 465 target0 = self.prepare_backup(dr0bm0) 466 target1 = self.prepare_backup(dr1bm0) 467 468 # Ask for a new incremental backup per-each drive, 469 # expecting drive1's backup to fail. In the 'race' test, 470 # we expect drive1 to attempt to cancel the empty drive0 job. 471 transaction = [ 472 transaction_drive_backup(drive0['id'], target0, sync='incremental', 473 format=drive0['fmt'], mode='existing', 474 bitmap=dr0bm0.name), 475 transaction_drive_backup(drive1['id'], target1, sync='incremental', 476 format=drive1['fmt'], mode='existing', 477 bitmap=dr1bm0.name) 478 ] 479 result = self.vm.qmp('transaction', actions=transaction, 480 properties={'completion-mode': 'grouped'} ) 481 self.assert_qmp(result, 'return', {}) 482 483 # Observe that drive0's backup is cancelled and drive1 completes with 484 # an error. 485 self.wait_qmp_backup_cancelled(drive0['id']) 486 self.assertFalse(self.wait_qmp_backup(drive1['id'])) 487 error = self.vm.event_wait('BLOCK_JOB_ERROR') 488 self.assert_qmp(error, 'data', {'device': drive1['id'], 489 'action': 'report', 490 'operation': 'read'}) 491 self.assertFalse(self.vm.get_qmp_events(wait=False)) 492 self.assert_no_active_block_jobs() 493 494 # Delete drive0's successful target and eliminate our record of the 495 # unsuccessful drive1 target. 496 dr0bm0.del_target() 497 dr1bm0.del_target() 498 if race: 499 # Don't re-run the transaction, we only wanted to test the race. 500 self.vm.shutdown() 501 return 502 503 # Re-run the same transaction: 504 target0 = self.prepare_backup(dr0bm0) 505 target1 = self.prepare_backup(dr1bm0) 506 507 # Re-run the exact same transaction. 508 result = self.vm.qmp('transaction', actions=transaction, 509 properties={'completion-mode':'grouped'}) 510 self.assert_qmp(result, 'return', {}) 511 512 # Both should complete successfully this time. 513 self.assertTrue(self.wait_qmp_backup(drive0['id'])) 514 self.assertTrue(self.wait_qmp_backup(drive1['id'])) 515 self.make_reference_backup(dr0bm0) 516 self.make_reference_backup(dr1bm0) 517 self.assertFalse(self.vm.get_qmp_events(wait=False)) 518 self.assert_no_active_block_jobs() 519 520 # And the images should of course validate. 521 self.vm.shutdown() 522 self.check_backups() 523 524 def test_transaction_failure(self): 525 '''Test: Verify backups made from a transaction that partially fails. 526 527 Add a second drive with its own unique pattern, and add a bitmap to each 528 drive. Use blkdebug to interfere with the backup on just one drive and 529 attempt to create a coherent incremental backup across both drives. 530 531 verify a failure in one but not both, then delete the failed stubs and 532 re-run the same transaction. 533 534 verify that both incrementals are created successfully. 535 ''' 536 self.do_transaction_failure_test() 537 538 def test_transaction_failure_race(self): 539 '''Test: Verify that transactions with jobs that have no data to 540 transfer do not cause race conditions in the cancellation of the entire 541 transaction job group. 542 ''' 543 self.do_transaction_failure_test(race=True) 544 545 546 def test_sync_dirty_bitmap_missing(self): 547 self.assert_no_active_block_jobs() 548 self.files.append(self.err_img) 549 result = self.vm.qmp('drive-backup', device=self.drives[0]['id'], 550 sync='incremental', format=self.drives[0]['fmt'], 551 target=self.err_img) 552 self.assert_qmp(result, 'error/class', 'GenericError') 553 554 555 def test_sync_dirty_bitmap_not_found(self): 556 self.assert_no_active_block_jobs() 557 self.files.append(self.err_img) 558 result = self.vm.qmp('drive-backup', device=self.drives[0]['id'], 559 sync='incremental', bitmap='unknown', 560 format=self.drives[0]['fmt'], target=self.err_img) 561 self.assert_qmp(result, 'error/class', 'GenericError') 562 563 564 def test_sync_dirty_bitmap_bad_granularity(self): 565 ''' 566 Test: Test what happens if we provide an improper granularity. 567 568 The granularity must always be a power of 2. 569 ''' 570 self.assert_no_active_block_jobs() 571 self.assertRaises(AssertionError, self.add_bitmap, 572 'bitmap0', self.drives[0], 573 granularity=64000) 574 575 576class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase): 577 '''Incremental backup tests that utilize a BlkDebug filter on drive0.''' 578 579 def setUp(self): 580 drive0 = self.add_node('drive0') 581 self.img_create(drive0['file'], drive0['fmt']) 582 self.write_default_pattern(drive0['file']) 583 self.vm.launch() 584 585 def test_incremental_failure(self): 586 '''Test: Verify backups made after a failure are correct. 587 588 Simulate a failure during an incremental backup block job, 589 emulate additional writes, then create another incremental backup 590 afterwards and verify that the backup created is correct. 591 ''' 592 593 drive0 = self.drives[0] 594 result = self.vm.qmp('blockdev-add', 595 node_name=drive0['id'], 596 driver=drive0['fmt'], 597 file={ 598 'driver': 'blkdebug', 599 'image': { 600 'driver': 'file', 601 'filename': drive0['file'] 602 }, 603 'set-state': [{ 604 'event': 'flush_to_disk', 605 'state': 1, 606 'new_state': 2 607 }], 608 'inject-error': [{ 609 'event': 'read_aio', 610 'errno': 5, 611 'state': 2, 612 'immediately': False, 613 'once': True 614 }], 615 } 616 ) 617 self.assert_qmp(result, 'return', {}) 618 619 self.create_anchor_backup(drive0) 620 self.add_bitmap('bitmap0', drive0) 621 # Note: at this point, during a normal execution, 622 # Assume that the VM resumes and begins issuing IO requests here. 623 624 self.hmp_io_writes(drive0['id'], (('0xab', 0, 512), 625 ('0xfe', '16M', '256k'), 626 ('0x64', '32736k', '64k'))) 627 628 result = self.create_incremental(validate=False) 629 self.assertFalse(result) 630 self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512), 631 ('0x55', '8M', '352k'), 632 ('0x78', '15872k', '1M'))) 633 self.create_incremental() 634 self.vm.shutdown() 635 self.check_backups() 636 637 638if __name__ == '__main__': 639 iotests.main(supported_fmts=['qcow2']) 640