1#!/usr/bin/env python3 2# group: rw 3# 4# Test bitmap-sync backups (incremental, differential, and partials) 5# 6# Copyright (c) 2019 John Snow for Red Hat, Inc. 7# 8# This program is free software; you can redistribute it and/or modify 9# it under the terms of the GNU General Public License as published by 10# the Free Software Foundation; either version 2 of the License, or 11# (at your option) any later version. 12# 13# This program is distributed in the hope that it will be useful, 14# but WITHOUT ANY WARRANTY; without even the implied warranty of 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16# GNU General Public License for more details. 17# 18# You should have received a copy of the GNU General Public License 19# along with this program. If not, see <http://www.gnu.org/licenses/>. 20# 21# owner=jsnow@redhat.com 22 23import math 24import os 25 26import iotests 27from iotests import log, qemu_img 28 29SIZE = 64 * 1024 * 1024 30GRANULARITY = 64 * 1024 31 32 33class Pattern: 34 def __init__(self, byte, offset, size=GRANULARITY): 35 self.byte = byte 36 self.offset = offset 37 self.size = size 38 39 def bits(self, granularity): 40 lower = self.offset // granularity 41 upper = (self.offset + self.size - 1) // granularity 42 return set(range(lower, upper + 1)) 43 44 45class PatternGroup: 46 """Grouping of Pattern objects. Initialize with an iterable of Patterns.""" 47 def __init__(self, patterns): 48 self.patterns = patterns 49 50 def bits(self, granularity): 51 """Calculate the unique bits dirtied by this pattern grouping""" 52 res = set() 53 for pattern in self.patterns: 54 res |= pattern.bits(granularity) 55 return res 56 57 58GROUPS = [ 59 PatternGroup([ 60 # Batch 0: 4 clusters 61 Pattern('0x49', 0x0000000), 62 Pattern('0x6c', 0x0100000), # 1M 63 Pattern('0x6f', 0x2000000), # 32M 64 Pattern('0x76', 0x3ff0000)]), # 64M - 64K 65 PatternGroup([ 66 # Batch 1: 6 clusters (3 new) 67 Pattern('0x65', 0x0000000), # Full overwrite 68 Pattern('0x77', 0x00f8000), # Partial-left (1M-32K) 69 Pattern('0x72', 0x2008000), # Partial-right (32M+32K) 70 Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K) 71 PatternGroup([ 72 # Batch 2: 7 clusters (3 new) 73 Pattern('0x74', 0x0010000), # Adjacent-right 74 Pattern('0x69', 0x00e8000), # Partial-left (1M-96K) 75 Pattern('0x6e', 0x2018000), # Partial-right (32M+96K) 76 Pattern('0x67', 0x3fe0000, 77 2*GRANULARITY)]), # Overwrite [(64M-128K)-64M) 78 PatternGroup([ 79 # Batch 3: 8 clusters (5 new) 80 # Carefully chosen such that nothing re-dirties the one cluster 81 # that copies out successfully before failure in Group #1. 82 Pattern('0xaa', 0x0010000, 83 3*GRANULARITY), # Overwrite and 2x Adjacent-right 84 Pattern('0xbb', 0x00d8000), # Partial-left (1M-160K) 85 Pattern('0xcc', 0x2028000), # Partial-right (32M+160K) 86 Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right 87] 88 89 90class EmulatedBitmap: 91 def __init__(self, granularity=GRANULARITY): 92 self._bits = set() 93 self.granularity = granularity 94 95 def dirty_bits(self, bits): 96 self._bits |= set(bits) 97 98 def dirty_group(self, n): 99 self.dirty_bits(GROUPS[n].bits(self.granularity)) 100 101 def clear(self): 102 self._bits = set() 103 104 def clear_bits(self, bits): 105 self._bits -= set(bits) 106 107 def clear_bit(self, bit): 108 self.clear_bits({bit}) 109 110 def clear_group(self, n): 111 self.clear_bits(GROUPS[n].bits(self.granularity)) 112 113 @property 114 def first_bit(self): 115 return sorted(self.bits)[0] 116 117 @property 118 def bits(self): 119 return self._bits 120 121 @property 122 def count(self): 123 return len(self.bits) 124 125 def compare(self, qmp_bitmap): 126 """ 127 Print a nice human-readable message checking that a bitmap as reported 128 by the QMP interface has as many bits set as we expect it to. 129 """ 130 131 name = qmp_bitmap.get('name', '(anonymous)') 132 log("= Checking Bitmap {:s} =".format(name)) 133 134 want = self.count 135 have = qmp_bitmap['count'] // qmp_bitmap['granularity'] 136 137 log("expecting {:d} dirty sectors; have {:d}. {:s}".format( 138 want, have, "OK!" if want == have else "ERROR!")) 139 log('') 140 141 142class Drive: 143 """Represents, vaguely, a drive attached to a VM. 144 Includes format, graph, and device information.""" 145 146 def __init__(self, path, vm=None): 147 self.path = path 148 self.vm = vm 149 self.fmt = None 150 self.size = None 151 self.node = None 152 153 def img_create(self, fmt, size): 154 self.fmt = fmt 155 self.size = size 156 iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size)) 157 158 def create_target(self, name, fmt, size): 159 basename = os.path.basename(self.path) 160 file_node_name = "file_{}".format(basename) 161 vm = self.vm 162 163 log(vm.cmd('blockdev-create', job_id='bdc-file-job', 164 options={ 165 'driver': 'file', 166 'filename': self.path, 167 'size': 0, 168 })) 169 vm.run_job('bdc-file-job') 170 log(vm.cmd('blockdev-add', driver='file', 171 node_name=file_node_name, filename=self.path)) 172 173 log(vm.cmd('blockdev-create', job_id='bdc-fmt-job', 174 options={ 175 'driver': fmt, 176 'file': file_node_name, 177 'size': size, 178 })) 179 vm.run_job('bdc-fmt-job') 180 log(vm.cmd('blockdev-add', driver=fmt, 181 node_name=name, 182 file=file_node_name)) 183 self.fmt = fmt 184 self.size = size 185 self.node = name 186 187def blockdev_backup(vm, device, target, sync, **kwargs): 188 # Strip any arguments explicitly nulled by the caller: 189 kwargs = {key: val for key, val in kwargs.items() if val is not None} 190 result = vm.qmp_log('blockdev-backup', 191 device=device, 192 target=target, 193 sync=sync, 194 filter_node_name='backup-top', 195 x_perf={'max-workers': 1}, 196 **kwargs) 197 return result 198 199def blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs): 200 target_drive = Drive(filepath, vm=drive.vm) 201 target_drive.create_target(target_id, drive.fmt, drive.size) 202 blockdev_backup(drive.vm, drive.node, target_id, sync, **kwargs) 203 204def reference_backup(drive, n, filepath): 205 log("--- Reference Backup #{:d} ---\n".format(n)) 206 target_id = "ref_target_{:d}".format(n) 207 job_id = "ref_backup_{:d}".format(n) 208 blockdev_backup_mktarget(drive, target_id, filepath, "full", 209 job_id=job_id) 210 drive.vm.run_job(job_id, auto_dismiss=True) 211 log('') 212 213def backup(drive, n, filepath, sync, **kwargs): 214 log("--- Test Backup #{:d} ---\n".format(n)) 215 target_id = "backup_target_{:d}".format(n) 216 job_id = "backup_{:d}".format(n) 217 kwargs.setdefault('auto-finalize', False) 218 blockdev_backup_mktarget(drive, target_id, filepath, sync, 219 job_id=job_id, **kwargs) 220 return job_id 221 222def perform_writes(drive, n, filter_node_name=None): 223 log("--- Write #{:d} ---\n".format(n)) 224 for pattern in GROUPS[n].patterns: 225 cmd = "write -P{:s} 0x{:07x} 0x{:x}".format( 226 pattern.byte, 227 pattern.offset, 228 pattern.size) 229 log(cmd) 230 log(drive.vm.hmp_qemu_io(filter_node_name or drive.node, cmd)) 231 bitmaps = drive.vm.query_bitmaps() 232 log({'bitmaps': bitmaps}, indent=2) 233 log('') 234 return bitmaps 235 236 237def compare_images(image, reference, baseimg=None, expected_match=True): 238 """ 239 Print a nice human-readable message comparing these images. 240 """ 241 expected_ret = 0 if expected_match else 1 242 if baseimg: 243 qemu_img("rebase", "-u", "-b", baseimg, '-F', iotests.imgfmt, image) 244 245 sub = qemu_img("compare", image, reference, check=False) 246 247 log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format( 248 image, reference, 249 "Identical" if sub.returncode == 0 else "Mismatch", 250 "OK!" if sub.returncode == expected_ret else "ERROR!"), 251 filters=[iotests.filter_testfiles]) 252 253def test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None): 254 """ 255 Test bitmap backup routines. 256 257 :param bsync_mode: Is the Bitmap Sync mode, and can be any of: 258 - on-success: This is the "incremental" style mode. Bitmaps are 259 synchronized to what was copied out only on success. 260 (Partial images must be discarded.) 261 - never: This is the "differential" style mode. 262 Bitmaps are never synchronized. 263 - always: This is a "best effort" style mode. 264 Bitmaps are always synchronized, regardless of failure. 265 (Partial images must be kept.) 266 267 :param msync_mode: The mirror sync mode to use for the first backup. 268 Can be any one of: 269 - bitmap: Backups based on bitmap manifest. 270 - full: Full backups. 271 - top: Full backups of the top layer only. 272 273 :param failure: Is the (optional) failure mode, and can be any of: 274 - None: No failure. Test the normative path. Default. 275 - simulated: Cancel the job right before it completes. 276 This also tests writes "during" the job. 277 - intermediate: This tests a job that fails mid-process and produces 278 an incomplete backup. Testing limitations prevent 279 testing competing writes. 280 """ 281 with iotests.FilePath( 282 'img', 'bsync1', 'bsync2', 'fbackup0', 'fbackup1', 'fbackup2') as \ 283 (img_path, bsync1, bsync2, fbackup0, fbackup1, fbackup2), \ 284 iotests.VM() as vm: 285 286 mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode) 287 preposition = "with" if failure else "without" 288 cond = "{:s} {:s}".format(preposition, 289 "{:s} failure".format(failure) if failure 290 else "failure") 291 log("\n=== {:s} {:s} ===\n".format(mode, cond)) 292 293 log('--- Preparing image & VM ---\n') 294 drive0 = Drive(img_path, vm=vm) 295 drive0.img_create(iotests.imgfmt, SIZE) 296 vm.add_device("{},id=scsi0".format('virtio-scsi')) 297 vm.launch() 298 299 file_config = { 300 'driver': 'file', 301 'filename': drive0.path 302 } 303 304 if failure == 'intermediate': 305 file_config = { 306 'driver': 'blkdebug', 307 'image': file_config, 308 'set-state': [{ 309 'event': 'flush_to_disk', 310 'state': 1, 311 'new_state': 2 312 }, { 313 'event': 'flush_to_disk', 314 'state': 2, 315 'new_state': 3 316 }, { 317 'event': "read_aio", 318 'state': 3, 319 'new_state': 4 320 }], 321 'inject-error': [{ 322 'event': 'read_aio', 323 'errno': 5, 324 'state': 4, 325 'immediately': False, 326 'once': True 327 }] 328 } 329 330 drive0.node = 'drive0' 331 vm.qmp_log('blockdev-add', 332 filters=[iotests.filter_qmp_testfiles], 333 node_name=drive0.node, 334 driver=drive0.fmt, 335 file=file_config) 336 log('') 337 338 # 0 - Writes and Reference Backup 339 perform_writes(drive0, 0) 340 reference_backup(drive0, 0, fbackup0) 341 log('--- Add Bitmap ---\n') 342 vm.qmp_log("block-dirty-bitmap-add", node=drive0.node, 343 name="bitmap0", granularity=GRANULARITY) 344 log('') 345 ebitmap = EmulatedBitmap() 346 347 # 1 - Writes and Reference Backup 348 bitmaps = perform_writes(drive0, 1) 349 ebitmap.dirty_group(1) 350 bitmap = vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps) 351 ebitmap.compare(bitmap) 352 reference_backup(drive0, 1, fbackup1) 353 354 # 1 - Test Backup (w/ Optional induced failure) 355 if failure == 'intermediate': 356 # Activate blkdebug induced failure for second-to-next read 357 log(vm.hmp_qemu_io(drive0.node, 'flush')) 358 log('') 359 job = backup(drive0, 1, bsync1, msync_mode, 360 bitmap="bitmap0", bitmap_mode=bsync_mode) 361 362 def _callback(): 363 """Issue writes while the job is open to test bitmap divergence.""" 364 # Note: when `failure` is 'intermediate', this isn't called. 365 log('') 366 bitmaps = perform_writes(drive0, 2, filter_node_name='backup-top') 367 # Named bitmap (static, should be unchanged) 368 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', 369 bitmaps=bitmaps)) 370 # Anonymous bitmap (dynamic, shows new writes) 371 anonymous = EmulatedBitmap() 372 anonymous.dirty_group(2) 373 anonymous.compare(vm.get_bitmap(drive0.node, '', recording=True, 374 bitmaps=bitmaps)) 375 376 # Simulate the order in which this will happen: 377 # group 1 gets cleared first, then group two gets written. 378 if ((bsync_mode == 'on-success' and not failure) or 379 (bsync_mode == 'always')): 380 ebitmap.clear() 381 ebitmap.dirty_group(2) 382 383 vm.run_job(job, auto_dismiss=True, auto_finalize=False, 384 pre_finalize=_callback, 385 cancel=(failure == 'simulated')) 386 bitmaps = vm.query_bitmaps() 387 log({'bitmaps': bitmaps}, indent=2) 388 log('') 389 390 if bsync_mode == 'always' and failure == 'intermediate': 391 # TOP treats anything allocated as dirty, expect to see: 392 if msync_mode == 'top': 393 ebitmap.dirty_group(0) 394 395 # We manage to copy one sector (one bit) before the error. 396 ebitmap.clear_bit(ebitmap.first_bit) 397 398 # Full returns all bits set except what was copied/skipped 399 if msync_mode == 'full': 400 fail_bit = ebitmap.first_bit 401 ebitmap.clear() 402 ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY)) 403 404 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)) 405 406 # 2 - Writes and Reference Backup 407 bitmaps = perform_writes(drive0, 3) 408 ebitmap.dirty_group(3) 409 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)) 410 reference_backup(drive0, 2, fbackup2) 411 412 # 2 - Bitmap Backup (In failure modes, this is a recovery.) 413 job = backup(drive0, 2, bsync2, "bitmap", 414 bitmap="bitmap0", bitmap_mode=bsync_mode) 415 vm.run_job(job, auto_dismiss=True, auto_finalize=False) 416 bitmaps = vm.query_bitmaps() 417 log({'bitmaps': bitmaps}, indent=2) 418 log('') 419 if bsync_mode != 'never': 420 ebitmap.clear() 421 ebitmap.compare(vm.get_bitmap(drive0.node, 'bitmap0', bitmaps=bitmaps)) 422 423 log('--- Cleanup ---\n') 424 vm.qmp_log("block-dirty-bitmap-remove", 425 node=drive0.node, name="bitmap0") 426 bitmaps = vm.query_bitmaps() 427 log({'bitmaps': bitmaps}, indent=2) 428 vm.shutdown() 429 log('') 430 431 log('--- Verification ---\n') 432 # 'simulated' failures will actually all pass here because we canceled 433 # while "pending". This is actually undefined behavior, 434 # don't rely on this to be true! 435 compare_images(bsync1, fbackup1, baseimg=fbackup0, 436 expected_match=failure != 'intermediate') 437 if not failure or bsync_mode == 'always': 438 # Always keep the last backup on success or when using 'always' 439 base = bsync1 440 else: 441 base = fbackup0 442 compare_images(bsync2, fbackup2, baseimg=base) 443 compare_images(img_path, fbackup2) 444 log('') 445 446def test_backup_api(): 447 """ 448 Test malformed and prohibited invocations of the backup API. 449 """ 450 with iotests.FilePath('img', 'bsync1') as (img_path, backup_path), \ 451 iotests.VM() as vm: 452 453 log("\n=== API failure tests ===\n") 454 log('--- Preparing image & VM ---\n') 455 drive0 = Drive(img_path, vm=vm) 456 drive0.img_create(iotests.imgfmt, SIZE) 457 vm.add_device("{},id=scsi0".format('virtio-scsi')) 458 vm.launch() 459 460 file_config = { 461 'driver': 'file', 462 'filename': drive0.path 463 } 464 465 drive0.node = 'drive0' 466 vm.qmp_log('blockdev-add', 467 filters=[iotests.filter_qmp_testfiles], 468 node_name=drive0.node, 469 driver=drive0.fmt, 470 file=file_config) 471 log('') 472 473 target0 = Drive(backup_path, vm=vm) 474 target0.create_target("backup_target", drive0.fmt, drive0.size) 475 log('') 476 477 vm.qmp_log("block-dirty-bitmap-add", node=drive0.node, 478 name="bitmap0", granularity=GRANULARITY) 479 log('') 480 481 log('-- Testing invalid QMP commands --\n') 482 483 error_cases = { 484 'incremental': { 485 None: ['on-success', 'always', 'never', None], 486 'bitmap404': ['on-success', 'always', 'never', None], 487 'bitmap0': ['always', 'never'] 488 }, 489 'bitmap': { 490 None: ['on-success', 'always', 'never', None], 491 'bitmap404': ['on-success', 'always', 'never', None], 492 'bitmap0': [None], 493 }, 494 'full': { 495 None: ['on-success', 'always', 'never'], 496 'bitmap404': ['on-success', 'always', 'never', None], 497 'bitmap0': ['never', None], 498 }, 499 'top': { 500 None: ['on-success', 'always', 'never'], 501 'bitmap404': ['on-success', 'always', 'never', None], 502 'bitmap0': ['never', None], 503 }, 504 'none': { 505 None: ['on-success', 'always', 'never'], 506 'bitmap404': ['on-success', 'always', 'never', None], 507 'bitmap0': ['on-success', 'always', 'never', None], 508 } 509 } 510 511 # Dicts, as always, are not stably-ordered prior to 3.7, so use tuples: 512 for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'): 513 log("-- Sync mode {:s} tests --\n".format(sync_mode)) 514 for bitmap in (None, 'bitmap404', 'bitmap0'): 515 for policy in error_cases[sync_mode][bitmap]: 516 blockdev_backup(drive0.vm, drive0.node, "backup_target", 517 sync_mode, job_id='api_job', 518 bitmap=bitmap, bitmap_mode=policy) 519 log('') 520 521 522def main(): 523 for bsync_mode in ("never", "on-success", "always"): 524 for failure in ("simulated", "intermediate", None): 525 test_bitmap_sync(bsync_mode, "bitmap", failure) 526 527 for sync_mode in ('full', 'top'): 528 for bsync_mode in ('on-success', 'always'): 529 for failure in ('simulated', 'intermediate', None): 530 test_bitmap_sync(bsync_mode, sync_mode, failure) 531 532 test_backup_api() 533 534if __name__ == '__main__': 535 iotests.script_main(main, supported_fmts=['qcow2'], 536 supported_protocols=['file']) 537