1#!/usr/bin/env python3
2# group: rw migration
3#
4# Tests for dirty bitmaps migration.
5#
6# Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation; either version 2 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License
19# along with this program.  If not, see <http://www.gnu.org/licenses/>.
20#
21
22import itertools
23import operator
24import os
25import re
26
27import iotests
28from iotests import qemu_img, qemu_img_create, Timeout
29
30
31disk_a = os.path.join(iotests.test_dir, 'disk_a')
32disk_b = os.path.join(iotests.test_dir, 'disk_b')
33base_a = os.path.join(iotests.test_dir, 'base_a')
34size = '1M'
35mig_file = os.path.join(iotests.test_dir, 'mig_file')
36mig_cmd = 'exec: cat > ' + mig_file
37incoming_cmd = 'exec: cat ' + mig_file
38
39
40def get_bitmap_hash(vm):
41    result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
42                    node='drive0', name='bitmap0')
43    return result['return']['sha256']
44
45
46class TestDirtyBitmapMigration(iotests.QMPTestCase):
47    def tearDown(self):
48        self.vm_a.shutdown()
49        self.vm_b.shutdown()
50        os.remove(disk_a)
51        os.remove(disk_b)
52        os.remove(mig_file)
53
54    def setUp(self):
55        qemu_img('create', '-f', iotests.imgfmt, disk_a, size)
56        qemu_img('create', '-f', iotests.imgfmt, disk_b, size)
57
58        self.vm_a = iotests.VM(path_suffix='a').add_drive(disk_a)
59        self.vm_a.launch()
60
61        self.vm_b = iotests.VM(path_suffix='b')
62
63    def add_bitmap(self, vm, granularity, persistent):
64        params = {'node': 'drive0',
65                  'name': 'bitmap0',
66                  'granularity': granularity}
67        if persistent:
68            params['persistent'] = True
69
70        result = vm.qmp('block-dirty-bitmap-add', **params)
71        self.assert_qmp(result, 'return', {})
72
73    def check_bitmap(self, vm, sha256):
74        result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
75                        node='drive0', name='bitmap0')
76        if sha256:
77            self.assert_qmp(result, 'return/sha256', sha256)
78        else:
79            self.assert_qmp(result, 'error/desc',
80                            "Dirty bitmap 'bitmap0' not found")
81
82    def do_test_migration_resume_source(self, persistent, migrate_bitmaps):
83        granularity = 512
84
85        # regions = ((start, count), ...)
86        regions = ((0, 0x10000),
87                   (0xf0000, 0x10000),
88                   (0xa0201, 0x1000))
89
90        mig_caps = [{'capability': 'events', 'state': True}]
91        if migrate_bitmaps:
92            mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})
93
94        result = self.vm_a.qmp('migrate-set-capabilities',
95                               capabilities=mig_caps)
96        self.assert_qmp(result, 'return', {})
97
98        self.add_bitmap(self.vm_a, granularity, persistent)
99        for r in regions:
100            self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
101        sha256 = get_bitmap_hash(self.vm_a)
102
103        result = self.vm_a.qmp('migrate', uri=mig_cmd)
104        while True:
105            event = self.vm_a.event_wait('MIGRATION')
106            if event['data']['status'] == 'completed':
107                break
108        while True:
109            result = self.vm_a.qmp('query-status')
110            if result['return']['status'] == 'postmigrate':
111                break
112
113        # test that bitmap is still here
114        removed = (not migrate_bitmaps) and persistent
115        self.check_bitmap(self.vm_a, False if removed else sha256)
116
117        result = self.vm_a.qmp('cont')
118        self.assert_qmp(result, 'return', {})
119
120        # test that bitmap is still here after invalidation
121        self.check_bitmap(self.vm_a, sha256)
122
123        # shutdown and check that invalidation didn't fail
124        self.vm_a.shutdown()
125
126        # catch 'Could not reopen qcow2 layer: Bitmap already exists'
127        # possible error
128        log = self.vm_a.get_log()
129        log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
130        log = re.sub(r'^(wrote .* bytes at offset .*\n.*KiB.*ops.*sec.*\n){3}',
131                     '', log)
132        log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
133        self.assertEqual(log, '')
134
135        # test that bitmap is still persistent
136        self.vm_a.launch()
137        self.check_bitmap(self.vm_a, sha256 if persistent else False)
138
139    def do_test_migration(self, persistent, migrate_bitmaps, online,
140                          shared_storage, pre_shutdown):
141        granularity = 512
142
143        # regions = ((start, count), ...)
144        regions = ((0, 0x10000),
145                   (0xf0000, 0x10000),
146                   (0xa0201, 0x1000))
147
148        should_migrate = \
149            (migrate_bitmaps and (persistent or not pre_shutdown)) or \
150            (persistent and shared_storage)
151        mig_caps = [{'capability': 'events', 'state': True}]
152        if migrate_bitmaps:
153            mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})
154
155        self.vm_b.add_incoming(incoming_cmd if online else "defer")
156        self.vm_b.add_drive(disk_a if shared_storage else disk_b)
157
158        if online:
159            os.mkfifo(mig_file)
160            self.vm_b.launch()
161            result = self.vm_b.qmp('migrate-set-capabilities',
162                                   capabilities=mig_caps)
163            self.assert_qmp(result, 'return', {})
164
165        self.add_bitmap(self.vm_a, granularity, persistent)
166        for r in regions:
167            self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
168        sha256 = get_bitmap_hash(self.vm_a)
169
170        if pre_shutdown:
171            self.vm_a.shutdown()
172            self.vm_a.launch()
173
174        result = self.vm_a.qmp('migrate-set-capabilities',
175                               capabilities=mig_caps)
176        self.assert_qmp(result, 'return', {})
177
178        result = self.vm_a.qmp('migrate', uri=mig_cmd)
179        while True:
180            event = self.vm_a.event_wait('MIGRATION')
181            if event['data']['status'] == 'completed':
182                break
183
184        if not online:
185            self.vm_a.shutdown()
186            self.vm_b.launch()
187            result = self.vm_b.qmp('migrate-set-capabilities',
188                                   capabilities=mig_caps)
189            self.assert_qmp(result, 'return', {})
190            result = self.vm_b.qmp('migrate-incoming', uri=incoming_cmd)
191            self.assert_qmp(result, 'return', {})
192
193        while True:
194            event = self.vm_b.event_wait('MIGRATION')
195            if event['data']['status'] == 'completed':
196                break
197
198        self.check_bitmap(self.vm_b, sha256 if should_migrate else False)
199
200        if should_migrate:
201            self.vm_b.shutdown()
202
203            # catch 'Could not reopen qcow2 layer: Bitmap already exists'
204            # possible error
205            log = self.vm_b.get_log()
206            log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
207            log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
208            self.assertEqual(log, '')
209
210            # recreate vm_b, as we don't want -incoming option (this will lead
211            # to "cat" process left alive after test finish)
212            self.vm_b = iotests.VM(path_suffix='b')
213            self.vm_b.add_drive(disk_a if shared_storage else disk_b)
214            self.vm_b.launch()
215            self.check_bitmap(self.vm_b, sha256 if persistent else False)
216
217
218def inject_test_case(klass, suffix, method, *args, **kwargs):
219    mc = operator.methodcaller(method, *args, **kwargs)
220    # We want to add a function attribute to `klass`, so that it is
221    # correctly converted to a method on instantiation.  The
222    # methodcaller object `mc` is a callable, not a function, so we
223    # need the lambda to turn it into a function.
224    # pylint: disable=unnecessary-lambda
225    setattr(klass, 'test_' + method + suffix, lambda self: mc(self))
226
227
228class TestDirtyBitmapBackingMigration(iotests.QMPTestCase):
229    def setUp(self):
230        qemu_img_create('-f', iotests.imgfmt, base_a, size)
231        qemu_img_create('-f', iotests.imgfmt, '-F', iotests.imgfmt,
232                        '-b', base_a, disk_a, size)
233
234        for f in (disk_a, base_a):
235            qemu_img('bitmap', '--add', f, 'bmap0')
236
237        blockdev = {
238            'node-name': 'node0',
239            'driver': iotests.imgfmt,
240            'file': {
241                'driver': 'file',
242                'filename': disk_a
243            },
244            'backing': {
245                'node-name': 'node0-base',
246                'driver': iotests.imgfmt,
247                'file': {
248                    'driver': 'file',
249                    'filename': base_a
250                }
251            }
252        }
253
254        self.vm = iotests.VM()
255        self.vm.launch()
256
257        result = self.vm.qmp('blockdev-add', **blockdev)
258        self.assert_qmp(result, 'return', {})
259
260        # Check that the bitmaps are there
261        nodes = self.vm.qmp('query-named-block-nodes', flat=True)['return']
262        for node in nodes:
263            if 'node0' in node['node-name']:
264                self.assert_qmp(node, 'dirty-bitmaps[0]/name', 'bmap0')
265
266        caps = [{'capability': 'events', 'state': True}]
267        result = self.vm.qmp('migrate-set-capabilities', capabilities=caps)
268        self.assert_qmp(result, 'return', {})
269
270    def tearDown(self):
271        self.vm.shutdown()
272        for f in (disk_a, base_a):
273            os.remove(f)
274
275    def test_cont_on_source(self):
276        """
277        Continue the source after migration.
278        """
279        result = self.vm.qmp('migrate', uri='exec: cat > /dev/null')
280        self.assert_qmp(result, 'return', {})
281
282        with Timeout(10, 'Migration timeout'):
283            self.vm.wait_migration('postmigrate')
284
285        result = self.vm.qmp('cont')
286        self.assert_qmp(result, 'return', {})
287
288
289def main() -> None:
290    for cmb in list(itertools.product((True, False), repeat=5)):
291        name = ('_' if cmb[0] else '_not_') + 'persistent_'
292        name += ('_' if cmb[1] else '_not_') + 'migbitmap_'
293        name += '_online' if cmb[2] else '_offline'
294        name += '_shared' if cmb[3] else '_nonshared'
295        if cmb[4]:
296            name += '__pre_shutdown'
297
298        inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration',
299                         *list(cmb))
300
301    for cmb in list(itertools.product((True, False), repeat=2)):
302        name = ('_' if cmb[0] else '_not_') + 'persistent_'
303        name += ('_' if cmb[1] else '_not_') + 'migbitmap'
304
305        inject_test_case(TestDirtyBitmapMigration, name,
306                         'do_test_migration_resume_source', *list(cmb))
307
308    iotests.main(
309        supported_fmts=['qcow2'],
310        supported_protocols=['file'],
311        unsupported_imgopts=['compat']
312    )
313
314
315if __name__ == '__main__':
316    main()
317