1#!/usr/bin/env python3
2# group: rw migration
3#
4# Tests for dirty bitmaps migration.
5#
6# Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation; either version 2 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License
19# along with this program.  If not, see <http://www.gnu.org/licenses/>.
20#
21
22import os
23import itertools
24import operator
25import re
26import iotests
27from iotests import qemu_img, qemu_img_create, Timeout
28
29
30disk_a = os.path.join(iotests.test_dir, 'disk_a')
31disk_b = os.path.join(iotests.test_dir, 'disk_b')
32base_a = os.path.join(iotests.test_dir, 'base_a')
33size = '1M'
34mig_file = os.path.join(iotests.test_dir, 'mig_file')
35mig_cmd = 'exec: cat > ' + mig_file
36incoming_cmd = 'exec: cat ' + mig_file
37
38
39def get_bitmap_hash(vm):
40    result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
41                    node='drive0', name='bitmap0')
42    return result['return']['sha256']
43
44
45class TestDirtyBitmapMigration(iotests.QMPTestCase):
46    def tearDown(self):
47        self.vm_a.shutdown()
48        self.vm_b.shutdown()
49        os.remove(disk_a)
50        os.remove(disk_b)
51        os.remove(mig_file)
52
53    def setUp(self):
54        qemu_img('create', '-f', iotests.imgfmt, disk_a, size)
55        qemu_img('create', '-f', iotests.imgfmt, disk_b, size)
56
57        self.vm_a = iotests.VM(path_suffix='a').add_drive(disk_a)
58        self.vm_a.launch()
59
60        self.vm_b = iotests.VM(path_suffix='b')
61
62    def add_bitmap(self, vm, granularity, persistent):
63        params = {'node': 'drive0',
64                  'name': 'bitmap0',
65                  'granularity': granularity}
66        if persistent:
67            params['persistent'] = True
68
69        result = vm.qmp('block-dirty-bitmap-add', **params)
70        self.assert_qmp(result, 'return', {})
71
72    def check_bitmap(self, vm, sha256):
73        result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
74                        node='drive0', name='bitmap0')
75        if sha256:
76            self.assert_qmp(result, 'return/sha256', sha256)
77        else:
78            self.assert_qmp(result, 'error/desc',
79                            "Dirty bitmap 'bitmap0' not found")
80
81    def do_test_migration_resume_source(self, persistent, migrate_bitmaps):
82        granularity = 512
83
84        # regions = ((start, count), ...)
85        regions = ((0, 0x10000),
86                   (0xf0000, 0x10000),
87                   (0xa0201, 0x1000))
88
89        mig_caps = [{'capability': 'events', 'state': True}]
90        if migrate_bitmaps:
91            mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})
92
93        result = self.vm_a.qmp('migrate-set-capabilities',
94                               capabilities=mig_caps)
95        self.assert_qmp(result, 'return', {})
96
97        self.add_bitmap(self.vm_a, granularity, persistent)
98        for r in regions:
99            self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
100        sha256 = get_bitmap_hash(self.vm_a)
101
102        result = self.vm_a.qmp('migrate', uri=mig_cmd)
103        while True:
104            event = self.vm_a.event_wait('MIGRATION')
105            if event['data']['status'] == 'completed':
106                break
107        while True:
108            result = self.vm_a.qmp('query-status')
109            if result['return']['status'] == 'postmigrate':
110                break
111
112        # test that bitmap is still here
113        removed = (not migrate_bitmaps) and persistent
114        self.check_bitmap(self.vm_a, False if removed else sha256)
115
116        result = self.vm_a.qmp('cont')
117        self.assert_qmp(result, 'return', {})
118
119        # test that bitmap is still here after invalidation
120        self.check_bitmap(self.vm_a, sha256)
121
122        # shutdown and check that invalidation didn't fail
123        self.vm_a.shutdown()
124
125        # catch 'Could not reopen qcow2 layer: Bitmap already exists'
126        # possible error
127        log = self.vm_a.get_log()
128        log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
129        log = re.sub(r'^(wrote .* bytes at offset .*\n.*KiB.*ops.*sec.*\n){3}',
130                     '', log)
131        log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
132        self.assertEqual(log, '')
133
134        # test that bitmap is still persistent
135        self.vm_a.launch()
136        self.check_bitmap(self.vm_a, sha256 if persistent else False)
137
138    def do_test_migration(self, persistent, migrate_bitmaps, online,
139                          shared_storage, pre_shutdown):
140        granularity = 512
141
142        # regions = ((start, count), ...)
143        regions = ((0, 0x10000),
144                   (0xf0000, 0x10000),
145                   (0xa0201, 0x1000))
146
147        should_migrate = \
148            (migrate_bitmaps and (persistent or not pre_shutdown)) or \
149            (persistent and shared_storage)
150        mig_caps = [{'capability': 'events', 'state': True}]
151        if migrate_bitmaps:
152            mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})
153
154        self.vm_b.add_incoming(incoming_cmd if online else "defer")
155        self.vm_b.add_drive(disk_a if shared_storage else disk_b)
156
157        if online:
158            os.mkfifo(mig_file)
159            self.vm_b.launch()
160            result = self.vm_b.qmp('migrate-set-capabilities',
161                                   capabilities=mig_caps)
162            self.assert_qmp(result, 'return', {})
163
164        self.add_bitmap(self.vm_a, granularity, persistent)
165        for r in regions:
166            self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
167        sha256 = get_bitmap_hash(self.vm_a)
168
169        if pre_shutdown:
170            self.vm_a.shutdown()
171            self.vm_a.launch()
172
173        result = self.vm_a.qmp('migrate-set-capabilities',
174                               capabilities=mig_caps)
175        self.assert_qmp(result, 'return', {})
176
177        result = self.vm_a.qmp('migrate', uri=mig_cmd)
178        while True:
179            event = self.vm_a.event_wait('MIGRATION')
180            if event['data']['status'] == 'completed':
181                break
182
183        if not online:
184            self.vm_a.shutdown()
185            self.vm_b.launch()
186            result = self.vm_b.qmp('migrate-set-capabilities',
187                                   capabilities=mig_caps)
188            self.assert_qmp(result, 'return', {})
189            result = self.vm_b.qmp('migrate-incoming', uri=incoming_cmd)
190            self.assert_qmp(result, 'return', {})
191
192        while True:
193            event = self.vm_b.event_wait('MIGRATION')
194            if event['data']['status'] == 'completed':
195                break
196
197        self.check_bitmap(self.vm_b, sha256 if should_migrate else False)
198
199        if should_migrate:
200            self.vm_b.shutdown()
201
202            # catch 'Could not reopen qcow2 layer: Bitmap already exists'
203            # possible error
204            log = self.vm_b.get_log()
205            log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
206            log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
207            self.assertEqual(log, '')
208
209            # recreate vm_b, as we don't want -incoming option (this will lead
210            # to "cat" process left alive after test finish)
211            self.vm_b = iotests.VM(path_suffix='b')
212            self.vm_b.add_drive(disk_a if shared_storage else disk_b)
213            self.vm_b.launch()
214            self.check_bitmap(self.vm_b, sha256 if persistent else False)
215
216
217def inject_test_case(klass, suffix, method, *args, **kwargs):
218    mc = operator.methodcaller(method, *args, **kwargs)
219    # We want to add a function attribute to `klass`, so that it is
220    # correctly converted to a method on instantiation.  The
221    # methodcaller object `mc` is a callable, not a function, so we
222    # need the lambda to turn it into a function.
223    # pylint: disable=unnecessary-lambda
224    setattr(klass, 'test_' + method + suffix, lambda self: mc(self))
225
226
227for cmb in list(itertools.product((True, False), repeat=5)):
228    name = ('_' if cmb[0] else '_not_') + 'persistent_'
229    name += ('_' if cmb[1] else '_not_') + 'migbitmap_'
230    name += '_online' if cmb[2] else '_offline'
231    name += '_shared' if cmb[3] else '_nonshared'
232    if cmb[4]:
233        name += '__pre_shutdown'
234
235    inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration',
236                     *list(cmb))
237
238for cmb in list(itertools.product((True, False), repeat=2)):
239    name = ('_' if cmb[0] else '_not_') + 'persistent_'
240    name += ('_' if cmb[1] else '_not_') + 'migbitmap'
241
242    inject_test_case(TestDirtyBitmapMigration, name,
243                     'do_test_migration_resume_source', *list(cmb))
244
245
246class TestDirtyBitmapBackingMigration(iotests.QMPTestCase):
247    def setUp(self):
248        qemu_img_create('-f', iotests.imgfmt, base_a, size)
249        qemu_img_create('-f', iotests.imgfmt, '-F', iotests.imgfmt,
250                        '-b', base_a, disk_a, size)
251
252        for f in (disk_a, base_a):
253            qemu_img('bitmap', '--add', f, 'bmap0')
254
255        blockdev = {
256            'node-name': 'node0',
257            'driver': iotests.imgfmt,
258            'file': {
259                'driver': 'file',
260                'filename': disk_a
261            },
262            'backing': {
263                'node-name': 'node0-base',
264                'driver': iotests.imgfmt,
265                'file': {
266                    'driver': 'file',
267                    'filename': base_a
268                }
269            }
270        }
271
272        self.vm = iotests.VM()
273        self.vm.launch()
274
275        result = self.vm.qmp('blockdev-add', **blockdev)
276        self.assert_qmp(result, 'return', {})
277
278        # Check that the bitmaps are there
279        nodes = self.vm.qmp('query-named-block-nodes', flat=True)['return']
280        for node in nodes:
281            if 'node0' in node['node-name']:
282                self.assert_qmp(node, 'dirty-bitmaps[0]/name', 'bmap0')
283
284        caps = [{'capability': 'events', 'state': True}]
285        result = self.vm.qmp('migrate-set-capabilities', capabilities=caps)
286        self.assert_qmp(result, 'return', {})
287
288    def tearDown(self):
289        self.vm.shutdown()
290        for f in (disk_a, base_a):
291            os.remove(f)
292
293    def test_cont_on_source(self):
294        """
295        Continue the source after migration.
296        """
297        result = self.vm.qmp('migrate', uri='exec: cat > /dev/null')
298        self.assert_qmp(result, 'return', {})
299
300        with Timeout(10, 'Migration timeout'):
301            self.vm.wait_migration('postmigrate')
302
303        result = self.vm.qmp('cont')
304        self.assert_qmp(result, 'return', {})
305
306
307if __name__ == '__main__':
308    iotests.main(supported_fmts=['qcow2'],
309                 supported_protocols=['file'])
310