1# TestEnv class to manage test environment variables. 2# 3# Copyright (c) 2020-2021 Virtuozzo International GmbH 4# 5# This program is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 2 of the License, or 8# (at your option) any later version. 9# 10# This program is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with this program. If not, see <http://www.gnu.org/licenses/>. 17# 18 19import os 20import sys 21import tempfile 22from pathlib import Path 23import shutil 24import collections 25import contextlib 26import random 27import subprocess 28import glob 29from typing import List, Dict, Any, Optional 30 31 32DEF_GDB_OPTIONS = 'localhost:12345' 33 34def isxfile(path: str) -> bool: 35 return os.path.isfile(path) and os.access(path, os.X_OK) 36 37 38def get_default_machine(qemu_prog: str) -> str: 39 outp = subprocess.run([qemu_prog, '-machine', 'help'], check=True, 40 universal_newlines=True, 41 stdout=subprocess.PIPE).stdout 42 43 machines = outp.split('\n') 44 try: 45 default_machine = next(m for m in machines if ' (default)' in m) 46 except StopIteration: 47 return '' 48 default_machine = default_machine.split(' ', 1)[0] 49 50 alias_suf = ' (alias of {})'.format(default_machine) 51 alias = next((m for m in machines if m.endswith(alias_suf)), None) 52 if alias is not None: 53 default_machine = alias.split(' ', 1)[0] 54 55 return default_machine 56 57 58class TestEnv(contextlib.AbstractContextManager['TestEnv']): 59 """ 60 Manage system environment for running tests 61 62 The following variables are supported/provided. They are represented by 63 lower-cased TestEnv attributes. 64 """ 65 66 # We store environment variables as instance attributes, and there are a 67 # lot of them. Silence pylint: 68 # pylint: disable=too-many-instance-attributes 69 70 env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR', 71 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG', 72 'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG', 73 'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS', 74 'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT', 75 'QEMU_NBD_OPTIONS', 'IMGOPTS', 'IMGFMT', 'IMGPROTO', 76 'AIOMODE', 'CACHEMODE', 'VALGRIND_QEMU', 77 'CACHEMODE_IS_DEFAULT', 'IMGFMT_GENERIC', 'IMGOPTSSYNTAX', 78 'IMGKEYSECRET', 'QEMU_DEFAULT_MACHINE', 'MALLOC_PERTURB_', 79 'GDB_OPTIONS', 'PRINT_QEMU'] 80 81 def prepare_subprocess(self, args: List[str]) -> Dict[str, str]: 82 if self.debug: 83 args.append('-d') 84 85 with open(args[0], encoding="utf-8") as f: 86 try: 87 if f.readline().rstrip() == '#!/usr/bin/env python3': 88 args.insert(0, self.python) 89 except UnicodeDecodeError: # binary test? for future. 90 pass 91 92 os_env = os.environ.copy() 93 os_env.update(self.get_env()) 94 return os_env 95 96 def get_env(self) -> Dict[str, str]: 97 env = {} 98 for v in self.env_variables: 99 val = getattr(self, v.lower(), None) 100 if val is not None: 101 env[v] = val 102 103 return env 104 105 def init_directories(self) -> None: 106 """Init directory variables: 107 PYTHONPATH 108 TEST_DIR 109 SOCK_DIR 110 SAMPLE_IMG_DIR 111 """ 112 113 # Path where qemu goodies live in this source tree. 114 qemu_srctree_path = Path(__file__, '../../../python').resolve() 115 116 self.pythonpath = os.pathsep.join(filter(None, ( 117 self.source_iotests, 118 str(qemu_srctree_path), 119 os.getenv('PYTHONPATH'), 120 ))) 121 122 self.test_dir = os.getenv('TEST_DIR', 123 os.path.join(os.getcwd(), 'scratch')) 124 Path(self.test_dir).mkdir(parents=True, exist_ok=True) 125 126 try: 127 self.sock_dir = os.environ['SOCK_DIR'] 128 self.tmp_sock_dir = False 129 Path(self.sock_dir).mkdir(parents=True, exist_ok=True) 130 except KeyError: 131 self.sock_dir = tempfile.mkdtemp(prefix="qemu-iotests-") 132 self.tmp_sock_dir = True 133 134 self.sample_img_dir = os.getenv('SAMPLE_IMG_DIR', 135 os.path.join(self.source_iotests, 136 'sample_images')) 137 138 def init_binaries(self) -> None: 139 """Init binary path variables: 140 PYTHON (for bash tests) 141 QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG 142 """ 143 self.python = sys.executable 144 145 def root(*names: str) -> str: 146 return os.path.join(self.build_root, *names) 147 148 arch = os.uname().machine 149 if 'ppc64' in arch: 150 arch = 'ppc64' 151 152 self.qemu_prog = os.getenv('QEMU_PROG', root(f'qemu-system-{arch}')) 153 if not os.path.exists(self.qemu_prog): 154 pattern = root('qemu-system-*') 155 try: 156 progs = sorted(glob.iglob(pattern)) 157 self.qemu_prog = next(p for p in progs if isxfile(p)) 158 except StopIteration: 159 sys.exit("Not found any Qemu executable binary by pattern " 160 f"'{pattern}'") 161 162 self.qemu_img_prog = os.getenv('QEMU_IMG_PROG', root('qemu-img')) 163 self.qemu_io_prog = os.getenv('QEMU_IO_PROG', root('qemu-io')) 164 self.qemu_nbd_prog = os.getenv('QEMU_NBD_PROG', root('qemu-nbd')) 165 self.qsd_prog = os.getenv('QSD_PROG', root('storage-daemon', 166 'qemu-storage-daemon')) 167 168 for b in [self.qemu_img_prog, self.qemu_io_prog, self.qemu_nbd_prog, 169 self.qemu_prog, self.qsd_prog]: 170 if not os.path.exists(b): 171 sys.exit('No such file: ' + b) 172 if not isxfile(b): 173 sys.exit('Not executable: ' + b) 174 175 def __init__(self, source_dir: str, build_dir: str, 176 imgfmt: str, imgproto: str, aiomode: str, 177 cachemode: Optional[str] = None, 178 imgopts: Optional[str] = None, 179 misalign: bool = False, 180 debug: bool = False, 181 valgrind: bool = False, 182 gdb: bool = False, 183 qprint: bool = False, 184 dry_run: bool = False) -> None: 185 self.imgfmt = imgfmt 186 self.imgproto = imgproto 187 self.aiomode = aiomode 188 self.imgopts = imgopts 189 self.misalign = misalign 190 self.debug = debug 191 192 if qprint: 193 self.print_qemu = 'y' 194 195 if gdb: 196 self.gdb_options = os.getenv('GDB_OPTIONS', DEF_GDB_OPTIONS) 197 if not self.gdb_options: 198 # cover the case 'export GDB_OPTIONS=' 199 self.gdb_options = DEF_GDB_OPTIONS 200 elif 'GDB_OPTIONS' in os.environ: 201 # to not propagate it in prepare_subprocess() 202 del os.environ['GDB_OPTIONS'] 203 204 if valgrind: 205 self.valgrind_qemu = 'y' 206 207 if cachemode is None: 208 self.cachemode_is_default = 'true' 209 self.cachemode = 'writeback' 210 else: 211 self.cachemode_is_default = 'false' 212 self.cachemode = cachemode 213 214 # Initialize generic paths: build_root, build_iotests, source_iotests, 215 # which are needed to initialize some environment variables. They are 216 # used by init_*() functions as well. 217 218 self.source_iotests = source_dir 219 self.build_iotests = build_dir 220 221 self.build_root = Path(self.build_iotests).parent.parent 222 223 self.init_directories() 224 225 if dry_run: 226 return 227 228 self.init_binaries() 229 230 self.malloc_perturb_ = os.getenv('MALLOC_PERTURB_', 231 str(random.randrange(1, 255))) 232 233 # QEMU_OPTIONS 234 self.qemu_options = '-nodefaults -display none -accel qtest' 235 machine_map = ( 236 ('arm', 'virt'), 237 ('aarch64', 'virt'), 238 ('avr', 'mega2560'), 239 ('m68k', 'virt'), 240 ('or1k', 'virt'), 241 ('riscv32', 'virt'), 242 ('riscv64', 'virt'), 243 ('rx', 'gdbsim-r5f562n8'), 244 ('sh4', 'r2d'), 245 ('sh4eb', 'r2d'), 246 ('tricore', 'tricore_testboard') 247 ) 248 for suffix, machine in machine_map: 249 if self.qemu_prog.endswith(f'qemu-system-{suffix}'): 250 self.qemu_options += f' -machine {machine}' 251 252 # QEMU_DEFAULT_MACHINE 253 self.qemu_default_machine = get_default_machine(self.qemu_prog) 254 255 self.qemu_img_options = os.getenv('QEMU_IMG_OPTIONS') 256 self.qemu_nbd_options = os.getenv('QEMU_NBD_OPTIONS') 257 258 is_generic = self.imgfmt not in ['bochs', 'cloop', 'dmg', 'vvfat'] 259 self.imgfmt_generic = 'true' if is_generic else 'false' 260 261 self.qemu_io_options = f'--cache {self.cachemode} --aio {self.aiomode}' 262 if self.misalign: 263 self.qemu_io_options += ' --misalign' 264 265 self.qemu_io_options_no_fmt = self.qemu_io_options 266 267 if self.imgfmt == 'luks': 268 self.imgoptssyntax = 'true' 269 self.imgkeysecret = '123456' 270 if not self.imgopts: 271 self.imgopts = 'iter-time=10' 272 elif 'iter-time=' not in self.imgopts: 273 self.imgopts += ',iter-time=10' 274 else: 275 self.imgoptssyntax = 'false' 276 self.qemu_io_options += ' -f ' + self.imgfmt 277 278 if self.imgfmt == 'vmdk': 279 if not self.imgopts: 280 self.imgopts = 'zeroed_grain=on' 281 elif 'zeroed_grain=' not in self.imgopts: 282 self.imgopts += ',zeroed_grain=on' 283 284 def close(self) -> None: 285 if self.tmp_sock_dir: 286 shutil.rmtree(self.sock_dir) 287 288 def __enter__(self) -> 'TestEnv': 289 return self 290 291 def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: 292 self.close() 293 294 def print_env(self, prefix: str = '') -> None: 295 template = """\ 296{prefix}QEMU -- "{QEMU_PROG}" {QEMU_OPTIONS} 297{prefix}QEMU_IMG -- "{QEMU_IMG_PROG}" {QEMU_IMG_OPTIONS} 298{prefix}QEMU_IO -- "{QEMU_IO_PROG}" {QEMU_IO_OPTIONS} 299{prefix}QEMU_NBD -- "{QEMU_NBD_PROG}" {QEMU_NBD_OPTIONS} 300{prefix}IMGFMT -- {IMGFMT}{imgopts} 301{prefix}IMGPROTO -- {IMGPROTO} 302{prefix}PLATFORM -- {platform} 303{prefix}TEST_DIR -- {TEST_DIR} 304{prefix}SOCK_DIR -- {SOCK_DIR} 305{prefix}GDB_OPTIONS -- {GDB_OPTIONS} 306{prefix}VALGRIND_QEMU -- {VALGRIND_QEMU} 307{prefix}PRINT_QEMU_OUTPUT -- {PRINT_QEMU} 308{prefix}""" 309 310 args = collections.defaultdict(str, self.get_env()) 311 312 if 'IMGOPTS' in args: 313 args['imgopts'] = f" ({args['IMGOPTS']})" 314 315 u = os.uname() 316 args['platform'] = f'{u.sysname}/{u.machine} {u.nodename} {u.release}' 317 args['prefix'] = prefix 318 print(template.format_map(args)) 319