1#!/usr/bin/env python3 2# group: rw 3# 4# Tests for active mirroring 5# 6# Copyright (C) 2018 Red Hat, Inc. 7# 8# This program is free software; you can redistribute it and/or modify 9# it under the terms of the GNU General Public License as published by 10# the Free Software Foundation; either version 2 of the License, or 11# (at your option) any later version. 12# 13# This program is distributed in the hope that it will be useful, 14# but WITHOUT ANY WARRANTY; without even the implied warranty of 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16# GNU General Public License for more details. 17# 18# You should have received a copy of the GNU General Public License 19# along with this program. If not, see <http://www.gnu.org/licenses/>. 20# 21 22import math 23import os 24import subprocess 25import time 26from typing import List, Optional 27import iotests 28from iotests import qemu_img 29 30source_img = os.path.join(iotests.test_dir, 'source.' + iotests.imgfmt) 31target_img = os.path.join(iotests.test_dir, 'target.' + iotests.imgfmt) 32 33class TestActiveMirror(iotests.QMPTestCase): 34 image_len = 128 * 1024 * 1024 # MB 35 potential_writes_in_flight = True 36 37 def setUp(self): 38 qemu_img('create', '-f', iotests.imgfmt, source_img, '128M') 39 qemu_img('create', '-f', iotests.imgfmt, target_img, '128M') 40 41 blk_source = {'id': 'source', 42 'if': 'none', 43 'node-name': 'source-node', 44 'driver': iotests.imgfmt, 45 'file': {'driver': 'blkdebug', 46 'image': {'driver': 'file', 47 'filename': source_img}}} 48 49 blk_target = {'node-name': 'target-node', 50 'driver': iotests.imgfmt, 51 'file': {'driver': 'file', 52 'filename': target_img}} 53 54 self.vm = iotests.VM() 55 self.vm.add_drive_raw(self.vm.qmp_to_opts(blk_source)) 56 self.vm.add_blockdev(self.vm.qmp_to_opts(blk_target)) 57 self.vm.add_device('virtio-blk,id=vblk,drive=source') 58 self.vm.launch() 59 60 def tearDown(self): 61 self.vm.shutdown() 62 63 if not self.potential_writes_in_flight: 64 self.assertTrue(iotests.compare_images(source_img, target_img), 65 'mirror target does not match source') 66 67 os.remove(source_img) 68 os.remove(target_img) 69 70 def doActiveIO(self, sync_source_and_target): 71 # Fill the source image 72 self.vm.hmp_qemu_io('source', 73 'write -P 1 0 %i' % self.image_len); 74 75 # Start some background requests 76 for offset in range(1 * self.image_len // 8, 3 * self.image_len // 8, 1024 * 1024): 77 self.vm.hmp_qemu_io('source', 'aio_write -P 2 %i 1M' % offset) 78 for offset in range(2 * self.image_len // 8, 3 * self.image_len // 8, 1024 * 1024): 79 self.vm.hmp_qemu_io('source', 'aio_write -z %i 1M' % offset) 80 81 # Start the block job 82 result = self.vm.qmp('blockdev-mirror', 83 job_id='mirror', 84 filter_node_name='mirror-node', 85 device='source-node', 86 target='target-node', 87 sync='full', 88 copy_mode='write-blocking') 89 self.assert_qmp(result, 'return', {}) 90 91 # Start some more requests 92 for offset in range(3 * self.image_len // 8, 5 * self.image_len // 8, 1024 * 1024): 93 self.vm.hmp_qemu_io('source', 'aio_write -P 3 %i 1M' % offset) 94 for offset in range(4 * self.image_len // 8, 5 * self.image_len // 8, 1024 * 1024): 95 self.vm.hmp_qemu_io('source', 'aio_write -z %i 1M' % offset) 96 97 # Wait for the READY event 98 self.wait_ready(drive='mirror') 99 100 # Now start some final requests; all of these (which land on 101 # the source) should be settled using the active mechanism. 102 # The mirror code itself asserts that the source BDS's dirty 103 # bitmap will stay clean between READY and COMPLETED. 104 for offset in range(5 * self.image_len // 8, 7 * self.image_len // 8, 1024 * 1024): 105 self.vm.hmp_qemu_io('source', 'aio_write -P 3 %i 1M' % offset) 106 for offset in range(6 * self.image_len // 8, 7 * self.image_len // 8, 1024 * 1024): 107 self.vm.hmp_qemu_io('source', 'aio_write -z %i 1M' % offset) 108 109 if sync_source_and_target: 110 # If source and target should be in sync after the mirror, 111 # we have to flush before completion 112 self.vm.hmp_qemu_io('source', 'aio_flush') 113 self.potential_writes_in_flight = False 114 115 self.complete_and_wait(drive='mirror', wait_ready=False) 116 117 def testActiveIO(self): 118 self.doActiveIO(False) 119 120 def testActiveIOFlushed(self): 121 self.doActiveIO(True) 122 123 def testUnalignedActiveIO(self): 124 # Fill the source image 125 result = self.vm.hmp_qemu_io('source', 'write -P 1 0 2M') 126 127 # Start the block job (very slowly) 128 result = self.vm.qmp('blockdev-mirror', 129 job_id='mirror', 130 filter_node_name='mirror-node', 131 device='source-node', 132 target='target-node', 133 sync='full', 134 copy_mode='write-blocking', 135 buf_size=(1048576 // 4), 136 speed=1) 137 self.assert_qmp(result, 'return', {}) 138 139 # Start an unaligned request to a dirty area 140 result = self.vm.hmp_qemu_io('source', 'write -P 2 %i 1' % (1048576 + 42)) 141 142 # Let the job finish 143 result = self.vm.qmp('block-job-set-speed', device='mirror', speed=0) 144 self.assert_qmp(result, 'return', {}) 145 self.complete_and_wait(drive='mirror') 146 147 self.potential_writes_in_flight = False 148 149 def testIntersectingActiveIO(self): 150 # Fill the source image 151 result = self.vm.hmp_qemu_io('source', 'write -P 1 0 2M') 152 153 # Start the block job (very slowly) 154 result = self.vm.qmp('blockdev-mirror', 155 job_id='mirror', 156 filter_node_name='mirror-node', 157 device='source-node', 158 target='target-node', 159 sync='full', 160 copy_mode='write-blocking', 161 speed=1) 162 self.assert_qmp(result, 'return', {}) 163 164 self.vm.hmp_qemu_io('source', 'break write_aio A') 165 self.vm.hmp_qemu_io('source', 'aio_write 0 1M') # 1 166 self.vm.hmp_qemu_io('source', 'wait_break A') 167 self.vm.hmp_qemu_io('source', 'aio_write 0 2M') # 2 168 self.vm.hmp_qemu_io('source', 'aio_write 0 2M') # 3 169 170 # Now 2 and 3 are in mirror_wait_on_conflicts, waiting for 1 171 172 self.vm.hmp_qemu_io('source', 'break write_aio B') 173 self.vm.hmp_qemu_io('source', 'aio_write 1M 2M') # 4 174 self.vm.hmp_qemu_io('source', 'wait_break B') 175 176 # 4 doesn't wait for 2 and 3, because they didn't yet set 177 # in_flight_bitmap. So, nothing prevents 4 to go except for our 178 # break-point B. 179 180 self.vm.hmp_qemu_io('source', 'resume A') 181 182 # Now we resumed 1, so 2 and 3 goes to the next iteration of while loop 183 # in mirror_wait_on_conflicts(). They don't exit, as bitmap is dirty 184 # due to request 4. 185 # In the past at that point 2 and 3 would wait for each other producing 186 # a dead-lock. Now this is fixed and they will wait for request 4. 187 188 self.vm.hmp_qemu_io('source', 'resume B') 189 190 # After resuming 4, one of 2 and 3 goes first and set in_flight_bitmap, 191 # so the other will wait for it. 192 193 result = self.vm.qmp('block-job-set-speed', device='mirror', speed=0) 194 self.assert_qmp(result, 'return', {}) 195 self.complete_and_wait(drive='mirror') 196 197 self.potential_writes_in_flight = False 198 199 200class TestThrottledWithNbdExportBase(iotests.QMPTestCase): 201 image_len = 128 * 1024 * 1024 # MB 202 iops: Optional[int] = None 203 background_processes: List['subprocess.Popen[str]'] = [] 204 205 def setUp(self): 206 # Must be set by subclasses 207 self.assertIsNotNone(self.iops) 208 209 qemu_img('create', '-f', iotests.imgfmt, source_img, '128M') 210 qemu_img('create', '-f', iotests.imgfmt, target_img, '128M') 211 212 self.vm = iotests.VM() 213 self.vm.launch() 214 215 result = self.vm.qmp('object-add', **{ 216 'qom-type': 'throttle-group', 217 'id': 'thrgr', 218 'limits': { 219 'iops-total': self.iops, 220 'iops-total-max': self.iops 221 } 222 }) 223 self.assert_qmp(result, 'return', {}) 224 225 result = self.vm.qmp('blockdev-add', **{ 226 'node-name': 'source-node', 227 'driver': 'throttle', 228 'throttle-group': 'thrgr', 229 'file': { 230 'driver': iotests.imgfmt, 231 'file': { 232 'driver': 'file', 233 'filename': source_img 234 } 235 } 236 }) 237 self.assert_qmp(result, 'return', {}) 238 239 result = self.vm.qmp('blockdev-add', **{ 240 'node-name': 'target-node', 241 'driver': iotests.imgfmt, 242 'file': { 243 'driver': 'file', 244 'filename': target_img 245 } 246 }) 247 self.assert_qmp(result, 'return', {}) 248 249 self.nbd_sock = iotests.file_path('nbd.sock', 250 base_dir=iotests.sock_dir) 251 self.nbd_url = f'nbd+unix:///source-node?socket={self.nbd_sock}' 252 253 result = self.vm.qmp('nbd-server-start', addr={ 254 'type': 'unix', 255 'data': { 256 'path': self.nbd_sock 257 } 258 }) 259 self.assert_qmp(result, 'return', {}) 260 261 result = self.vm.qmp('block-export-add', id='exp0', type='nbd', 262 node_name='source-node', writable=True) 263 self.assert_qmp(result, 'return', {}) 264 265 def tearDown(self): 266 # Wait for background requests to settle 267 try: 268 while True: 269 p = self.background_processes.pop() 270 while True: 271 try: 272 p.wait(timeout=0.0) 273 break 274 except subprocess.TimeoutExpired: 275 self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}') 276 except IndexError: 277 pass 278 279 # Cancel ongoing block jobs 280 for job in self.vm.qmp('query-jobs')['return']: 281 self.vm.qmp('block-job-cancel', device=job['id'], force=True) 282 283 while True: 284 self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}') 285 if len(self.vm.qmp('query-jobs')['return']) == 0: 286 break 287 288 self.vm.shutdown() 289 os.remove(source_img) 290 os.remove(target_img) 291 292 293class TestLowThrottledWithNbdExport(TestThrottledWithNbdExportBase): 294 iops = 16 295 296 def testUnderLoad(self): 297 ''' 298 Throttle the source node, then issue a whole bunch of external requests 299 while the mirror job (in write-blocking mode) is running. We want to 300 see background requests being issued even while the source is under 301 full load by active writes, so that progress can be made towards READY. 302 ''' 303 304 # Fill the first half of the source image; do not fill the second half, 305 # that is where we will have active requests occur. This ensures that 306 # active mirroring itself will not directly contribute to the job's 307 # progress (because when the job was started, those areas were not 308 # intended to be copied, so active mirroring will only lead to not 309 # losing progress, but also not making any). 310 self.vm.hmp_qemu_io('source-node', 311 f'aio_write -P 1 0 {self.image_len // 2}') 312 self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}') 313 314 # Launch the mirror job 315 mirror_buf_size = 65536 316 result = self.vm.qmp('blockdev-mirror', 317 job_id='mirror', 318 filter_node_name='mirror-node', 319 device='source-node', 320 target='target-node', 321 sync='full', 322 copy_mode='write-blocking', 323 buf_size=mirror_buf_size) 324 self.assert_qmp(result, 'return', {}) 325 326 # We create the external requests via qemu-io processes on the NBD 327 # server. Have their offset start in the middle of the image so they 328 # do not overlap with the background requests (which start from the 329 # beginning). 330 active_request_offset = self.image_len // 2 331 active_request_len = 4096 332 333 # Create enough requests to saturate the node for 5 seconds 334 for _ in range(0, 5 * self.iops): 335 req = f'write -P 42 {active_request_offset} {active_request_len}' 336 active_request_offset += active_request_len 337 p = iotests.qemu_io_popen('-f', 'nbd', self.nbd_url, '-c', req) 338 self.background_processes += [p] 339 340 # Now advance the clock one I/O operation at a time by the 4 seconds 341 # (i.e. one less than 5). We expect the mirror job to issue background 342 # operations here, even though active requests are still in flight. 343 # The active requests will take precedence, however, because they have 344 # been issued earlier than mirror's background requests. 345 # Once the active requests we have started above are done (i.e. after 5 346 # virtual seconds), we expect those background requests to be worked 347 # on. We only advance 4 seconds here to avoid race conditions. 348 for _ in range(0, 4 * self.iops): 349 step = math.ceil(1 * 1000 * 1000 * 1000 / self.iops) 350 self.vm.qtest(f'clock_step {step}') 351 352 # Note how much remains to be done until the mirror job is finished 353 job_status = self.vm.qmp('query-jobs')['return'][0] 354 start_remaining = job_status['total-progress'] - \ 355 job_status['current-progress'] 356 357 # Create a whole bunch of more active requests 358 for _ in range(0, 10 * self.iops): 359 req = f'write -P 42 {active_request_offset} {active_request_len}' 360 active_request_offset += active_request_len 361 p = iotests.qemu_io_popen('-f', 'nbd', self.nbd_url, '-c', req) 362 self.background_processes += [p] 363 364 # Let the clock advance more. After 1 second, as noted above, we 365 # expect the background requests to be worked on. Give them a couple 366 # of seconds (specifically 4) to see their impact. 367 for _ in range(0, 5 * self.iops): 368 step = math.ceil(1 * 1000 * 1000 * 1000 / self.iops) 369 self.vm.qtest(f'clock_step {step}') 370 371 # Note how much remains to be done now. We expect this number to be 372 # reduced thanks to those background requests. 373 job_status = self.vm.qmp('query-jobs')['return'][0] 374 end_remaining = job_status['total-progress'] - \ 375 job_status['current-progress'] 376 377 # See that indeed progress was being made on the job, even while the 378 # node was saturated with active requests 379 self.assertGreater(start_remaining - end_remaining, 0) 380 381 382class TestHighThrottledWithNbdExport(TestThrottledWithNbdExportBase): 383 iops = 1024 384 385 def testActiveOnCreation(self): 386 ''' 387 Issue requests on the mirror source node right as the mirror is 388 instated. It's possible that requests occur before the actual job is 389 created, but after the node has been put into the graph. Write 390 requests across the node must in that case be forwarded to the source 391 node without attempting to mirror them (there is no job object yet, so 392 attempting to access it would cause a segfault). 393 We do this with a lightly throttled node (i.e. quite high IOPS limit). 394 Using throttling seems to increase reproductivity, but if the limit is 395 too low, all requests allowed per second will be submitted before 396 mirror_start_job() gets to the problematic point. 397 ''' 398 399 # Let qemu-img bench create write requests (enough for two seconds on 400 # the virtual clock) 401 bench_args = ['bench', '-w', '-d', '1024', '-f', 'nbd', 402 '-c', str(self.iops * 2), self.nbd_url] 403 p = iotests.qemu_tool_popen(iotests.qemu_img_args + bench_args) 404 self.background_processes += [p] 405 406 # Give qemu-img bench time to start up and issue requests 407 time.sleep(1.0) 408 # Flush the request queue, so new requests can come in right as we 409 # start blockdev-mirror 410 self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}') 411 412 result = self.vm.qmp('blockdev-mirror', 413 job_id='mirror', 414 device='source-node', 415 target='target-node', 416 sync='full', 417 copy_mode='write-blocking') 418 self.assert_qmp(result, 'return', {}) 419 420 421if __name__ == '__main__': 422 iotests.main(supported_fmts=['qcow2', 'raw'], 423 supported_protocols=['file']) 424