xref: /openbmc/qemu/tests/qemu-iotests/testenv.py (revision 5b73b248) 
1# TestEnv class to manage test environment variables.
2#
3# Copyright (c) 2020-2021 Virtuozzo International GmbH
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17#
18
19import os
20import sys
21import tempfile
22from pathlib import Path
23import shutil
24import collections
25import random
26import subprocess
27import glob
28from typing import List, Dict, Any, Optional
29
30if sys.version_info >= (3, 9):
31    from contextlib import AbstractContextManager as ContextManager
32else:
33    from typing import ContextManager
34
35DEF_GDB_OPTIONS = 'localhost:12345'
36
37def isxfile(path: str) -> bool:
38    return os.path.isfile(path) and os.access(path, os.X_OK)
39
40
41def get_default_machine(qemu_prog: str) -> str:
42    outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True,
43                          universal_newlines=True,
44                          stdout=subprocess.PIPE).stdout
45
46    machines = outp.split('\n')
47    try:
48        default_machine = next(m for m in machines if ' (default)' in m)
49    except StopIteration:
50        return ''
51    default_machine = default_machine.split(' ', 1)[0]
52
53    alias_suf = ' (alias of {})'.format(default_machine)
54    alias = next((m for m in machines if m.endswith(alias_suf)), None)
55    if alias is not None:
56        default_machine = alias.split(' ', 1)[0]
57
58    return default_machine
59
60
61class TestEnv(ContextManager['TestEnv']):
62    """
63    Manage system environment for running tests
64
65    The following variables are supported/provided. They are represented by
66    lower-cased TestEnv attributes.
67    """
68
69    # We store environment variables as instance attributes, and there are a
70    # lot of them. Silence pylint:
71    # pylint: disable=too-many-instance-attributes
72
73    env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR',
74                     'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG',
75                     'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG',
76                     'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS',
77                     'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT',
78                     'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO',
79                     'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU',
80                     'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX',
81                     'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_',
82                     'GDB_OPTIONS', 'PRINT_QEMU']
83
84    def prepare_subprocess(self, args: List[str]) -> Dict[str, str]:
85        if self.debug:
86            args.append('-d')
87
88        with open(args[0], encoding="utf-8") as f:
89            try:
90                if f.readline().rstrip() == '#!/usr/bin/env python3':
91                    args.insert(0, self.python)
92            except UnicodeDecodeError:  # binary test? for future.
93                pass
94
95        os_env = os.environ.copy()
96        os_env.update(self.get_env())
97        return os_env
98
99    def get_env(self) -> Dict[str, str]:
100        env = {}
101        for v in self.env_variables:
102            val = getattr(self, v.lower(), None)
103            if val is not None:
104                env[v] = val
105
106        return env
107
108    def init_directories(self) -> None:
109        """Init directory variables:
110             PYTHONPATH
111             TEST_DIR
112             SOCK_DIR
113             SAMPLE_IMG_DIR
114        """
115
116        # Path where qemu goodies live in this source tree.
117        qemu_srctree_path = Path(__file__, '../../../python').resolve()
118
119        self.pythonpath = os.pathsep.join(filter(None, (
120            self.source_iotests,
121            str(qemu_srctree_path),
122            os.getenv('PYTHONPATH'),
123        )))
124
125        self.test_dir = os.getenv('TEST_DIR',
126                                  os.path.join(os.getcwd(), 'scratch'))
127        Path(self.test_dir).mkdir(parents=True, exist_ok=True)
128
129        try:
130            self.sock_dir = os.environ['SOCK_DIR']
131            self.tmp_sock_dir = False
132            Path(self.sock_dir).mkdir(parents=True, exist_ok=True)
133        except KeyError:
134            self.sock_dir = tempfile.mkdtemp(prefix="qemu-iotests-")
135            self.tmp_sock_dir = True
136
137        self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR',
138                                        os.path.join(self.source_iotests,
139                                                     'sample_images'))
140
141    def init_binaries(self) -> None:
142        """Init binary path variables:
143             PYTHON (for bash tests)
144             QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
145        """
146        self.python = sys.executable
147
148        def root(*names: str) -> str:
149            return os.path.join(self.build_root, *names)
150
151        arch = os.uname().machine
152        if 'ppc64' in arch:
153            arch = 'ppc64'
154
155        self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}'))
156        if not os.path.exists(self.qemu_prog):
157            pattern = root('qemu-system-*')
158            try:
159                progs = sorted(glob.iglob(pattern))
160                self.qemu_prog = next(p for p in progs if isxfile(p))
161            except StopIteration:
162                sys.exit("Not found any Qemu executable binary by pattern "
163                         f"'{pattern}'")
164
165        self.qemu_img_prog = os.getenv('QEMU_IMG_PROG', root('qemu-img'))
166        self.qemu_io_prog = os.getenv('QEMU_IO_PROG', root('qemu-io'))
167        self.qemu_nbd_prog = os.getenv('QEMU_NBD_PROG', root('qemu-nbd'))
168        self.qsd_prog = os.getenv('QSD_PROG', root('storage-daemon',
169                                                   'qemu-storage-daemon'))
170
171        for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog,
172                  self.qemu_prog, self.qsd_prog]:
173            if not os.path.exists(b):
174                sys.exit('No such file: ' + b)
175            if not isxfile(b):
176                sys.exit('Not executable: ' + b)
177
178    def __init__(self, source_dir: str, build_dir: str,
179                 imgfmt: str, imgproto: str, aiomode: str,
180                 cachemode: Optional[str] = None,
181                 imgopts: Optional[str] = None,
182                 misalign: bool = False,
183                 debug: bool = False,
184                 valgrind: bool = False,
185                 gdb: bool = False,
186                 qprint: bool = False,
187                 dry_run: bool = False) -> None:
188        self.imgfmt = imgfmt
189        self.imgproto = imgproto
190        self.aiomode = aiomode
191        self.imgopts = imgopts
192        self.misalign = misalign
193        self.debug = debug
194
195        if qprint:
196            self.print_qemu = 'y'
197
198        if gdb:
199            self.gdb_options = os.getenv('GDB_OPTIONS', DEF_GDB_OPTIONS)
200            if not self.gdb_options:
201                # cover the case 'export GDB_OPTIONS='
202                self.gdb_options = DEF_GDB_OPTIONS
203        elif 'GDB_OPTIONS' in os.environ:
204            # to not propagate it in prepare_subprocess()
205            del os.environ['GDB_OPTIONS']
206
207        if valgrind:
208            self.valgrind_qemu = 'y'
209
210        if cachemode is None:
211            self.cachemode_is_default = 'true'
212            self.cachemode = 'writeback'
213        else:
214            self.cachemode_is_default = 'false'
215            self.cachemode = cachemode
216
217        # Initialize generic paths: build_root, build_iotests, source_iotests,
218        # which are needed to initialize some environment variables. They are
219        # used by init_*() functions as well.
220
221        self.source_iotests = source_dir
222        self.build_iotests = build_dir
223
224        self.build_root = Path(self.build_iotests).parent.parent
225
226        self.init_directories()
227
228        if dry_run:
229            return
230
231        self.init_binaries()
232
233        self.malloc_perturb_ = os.getenv('MALLOC_PERTURB_',
234                                         str(random.randrange(1, 255)))
235
236        # QEMU_OPTIONS
237        self.qemu_options = '-nodefaults -display none -accel qtest'
238        machine_map = (
239            ('arm', 'virt'),
240            ('aarch64', 'virt'),
241            ('avr', 'mega2560'),
242            ('m68k', 'virt'),
243            ('riscv32', 'virt'),
244            ('riscv64', 'virt'),
245            ('rx', 'gdbsim-r5f562n8'),
246            ('tricore', 'tricore_testboard')
247        )
248        for suffix, machine in machine_map:
249            if self.qemu_prog.endswith(f'qemu-system-{suffix}'):
250                self.qemu_options += f' -machine {machine}'
251
252        # QEMU_DEFAULT_MACHINE
253        self.qemu_default_machine = get_default_machine(self.qemu_prog)
254
255        self.qemu_img_options = os.getenv('QEMU_IMG_OPTIONS')
256        self.qemu_nbd_options = os.getenv('QEMU_NBD_OPTIONS')
257
258        is_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg', 'vvfat']
259        self.imgfmt_generic = 'true' if is_generic else 'false'
260
261        self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}'
262        if self.misalign:
263            self.qemu_io_options += ' --misalign'
264
265        self.qemu_io_options_no_fmt = self.qemu_io_options
266
267        if self.imgfmt == 'luks':
268            self.imgoptssyntax = 'true'
269            self.imgkeysecret = '123456'
270            if not self.imgopts:
271                self.imgopts = 'iter-time=10'
272            elif 'iter-time=' not in self.imgopts:
273                self.imgopts += ',iter-time=10'
274        else:
275            self.imgoptssyntax = 'false'
276            self.qemu_io_options += ' -f ' + self.imgfmt
277
278        if self.imgfmt == 'vmdk':
279            if not self.imgopts:
280                self.imgopts = 'zeroed_grain=on'
281            elif 'zeroed_grain=' not in self.imgopts:
282                self.imgopts += ',zeroed_grain=on'
283
284    def close(self) -> None:
285        if self.tmp_sock_dir:
286            shutil.rmtree(self.sock_dir)
287
288    def __enter__(self) -> 'TestEnv':
289        return self
290
291    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
292        self.close()
293
294    def print_env(self, prefix: str = '') -> None:
295        template = """\
296{prefix}QEMU          -- "{QEMU_PROG}" {QEMU_OPTIONS}
297{prefix}QEMU_IMG      -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS}
298{prefix}QEMU_IO       -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS}
299{prefix}QEMU_NBD      -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS}
300{prefix}IMGFMT        -- {IMGFMT}{imgopts}
301{prefix}IMGPROTO      -- {IMGPROTO}
302{prefix}PLATFORM      -- {platform}
303{prefix}TEST_DIR      -- {TEST_DIR}
304{prefix}SOCK_DIR      -- {SOCK_DIR}
305{prefix}GDB_OPTIONS   -- {GDB_OPTIONS}
306{prefix}VALGRIND_QEMU -- {VALGRIND_QEMU}
307{prefix}PRINT_QEMU_OUTPUT -- {PRINT_QEMU}
308{prefix}"""
309
310        args = collections.defaultdict(str, self.get_env())
311
312        if 'IMGOPTS' in args:
313            args['imgopts'] = f" ({args['IMGOPTS']})"
314
315        u = os.uname()
316        args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}'
317        args['prefix'] = prefix
318        print(template.format_map(args))
319