1f345cfd0SStefan Hajnoczi# Common utilities and Python wrappers for qemu-iotests 2f345cfd0SStefan Hajnoczi# 3f345cfd0SStefan Hajnoczi# Copyright (C) 2012 IBM Corp. 4f345cfd0SStefan Hajnoczi# 5f345cfd0SStefan Hajnoczi# This program is free software; you can redistribute it and/or modify 6f345cfd0SStefan Hajnoczi# it under the terms of the GNU General Public License as published by 7f345cfd0SStefan Hajnoczi# the Free Software Foundation; either version 2 of the License, or 8f345cfd0SStefan Hajnoczi# (at your option) any later version. 9f345cfd0SStefan Hajnoczi# 10f345cfd0SStefan Hajnoczi# This program is distributed in the hope that it will be useful, 11f345cfd0SStefan Hajnoczi# but WITHOUT ANY WARRANTY; without even the implied warranty of 12f345cfd0SStefan Hajnoczi# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13f345cfd0SStefan Hajnoczi# GNU General Public License for more details. 14f345cfd0SStefan Hajnoczi# 15f345cfd0SStefan Hajnoczi# You should have received a copy of the GNU General Public License 16f345cfd0SStefan Hajnoczi# along with this program. If not, see <http://www.gnu.org/licenses/>. 17f345cfd0SStefan Hajnoczi# 18f345cfd0SStefan Hajnoczi 1922e29bceSVladimir Sementsov-Ogievskiyimport argparse 20ef6e9228SVladimir Sementsov-Ogievskiyimport atexit 2155b11630SVladimir Sementsov-Ogievskiyimport bz2 220706e87dSJohn Snowfrom collections import OrderedDict 23aa1cbeb8SKevin Wolfimport faulthandler 24b404b13bSJohn Snowimport json 25b404b13bSJohn Snowimport logging 26b404b13bSJohn Snowimport os 27b404b13bSJohn Snowimport re 2855b11630SVladimir Sementsov-Ogievskiyimport shutil 29b404b13bSJohn Snowimport signal 30b404b13bSJohn Snowimport struct 31b404b13bSJohn Snowimport subprocess 32b404b13bSJohn Snowimport sys 33b7719bcaSNir Sofferimport time 34206dc475SJohn Snowfrom typing import (Any, Callable, Dict, Iterable, Iterator, 35f29f4c25SPaolo Bonzini List, Optional, Sequence, TextIO, Tuple, Type, TypeVar) 36b404b13bSJohn Snowimport unittest 37f345cfd0SStefan Hajnoczi 38b7719bcaSNir Sofferfrom contextlib import contextmanager 39b7719bcaSNir Soffer 40beb6b57bSJohn Snowfrom qemu.machine import qtest 413a8736cfSVladimir Sementsov-Ogievskiyfrom qemu.qmp.legacy import QMPMessage, QMPReturnValue, QEMUMonitorProtocol 422882ccf8SJohn Snowfrom qemu.utils import VerboseProcessError 4302f3a911SVladimir Sementsov-Ogievskiy 4452ea799eSJohn Snow# Use this logger for logging messages directly from the iotests module 4552ea799eSJohn Snowlogger = logging.getLogger('qemu.iotests') 4652ea799eSJohn Snowlogger.addHandler(logging.NullHandler()) 4752ea799eSJohn Snow 4852ea799eSJohn Snow# Use this logger for messages that ought to be used for diff output. 4952ea799eSJohn Snowtest_logger = logging.getLogger('qemu.iotests.diff_io') 5052ea799eSJohn Snow 5152ea799eSJohn Snow 52aa1cbeb8SKevin Wolffaulthandler.enable() 53aa1cbeb8SKevin Wolf 54934659c4SMax Reitz# This will not work if arguments contain spaces but is necessary if we 55f345cfd0SStefan Hajnoczi# want to support the override options that ./check supports. 56934659c4SMax Reitzqemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')] 57934659c4SMax Reitzif os.environ.get('QEMU_IMG_OPTIONS'): 58934659c4SMax Reitz qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ') 59934659c4SMax Reitz 60934659c4SMax Reitzqemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 61934659c4SMax Reitzif os.environ.get('QEMU_IO_OPTIONS'): 62934659c4SMax Reitz qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ') 63934659c4SMax Reitz 64a4d925f8SAndrey Shinkevichqemu_io_args_no_fmt = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 65a4d925f8SAndrey Shinkevichif os.environ.get('QEMU_IO_OPTIONS_NO_FMT'): 66a4d925f8SAndrey Shinkevich qemu_io_args_no_fmt += \ 67a4d925f8SAndrey Shinkevich os.environ['QEMU_IO_OPTIONS_NO_FMT'].strip().split(' ') 68a4d925f8SAndrey Shinkevich 6981b6b2bcSKevin Wolfqemu_nbd_prog = os.environ.get('QEMU_NBD_PROG', 'qemu-nbd') 7081b6b2bcSKevin Wolfqemu_nbd_args = [qemu_nbd_prog] 71bec87774SMax Reitzif os.environ.get('QEMU_NBD_OPTIONS'): 72bec87774SMax Reitz qemu_nbd_args += os.environ['QEMU_NBD_OPTIONS'].strip().split(' ') 73bec87774SMax Reitz 744c44b4a4SDaniel P. Berrangeqemu_prog = os.environ.get('QEMU_PROG', 'qemu') 7566613974SDaniel P. Berrangeqemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ') 76f345cfd0SStefan Hajnoczi 77091dc7b2SHanna Reitzqsd_prog = os.environ.get('QSD_PROG', 'qemu-storage-daemon') 78091dc7b2SHanna Reitz 79cfb9b0b7SEmanuele Giuseppe Espositogdb_qemu_env = os.environ.get('GDB_OPTIONS') 80cfb9b0b7SEmanuele Giuseppe Espositoqemu_gdb = [] 81cfb9b0b7SEmanuele Giuseppe Espositoif gdb_qemu_env: 82cfb9b0b7SEmanuele Giuseppe Esposito qemu_gdb = ['gdbserver'] + gdb_qemu_env.strip().split(' ') 83cfb9b0b7SEmanuele Giuseppe Esposito 84eb7a91d0SEmanuele Giuseppe Espositoqemu_print = os.environ.get('PRINT_QEMU', False) 85eb7a91d0SEmanuele Giuseppe Esposito 86f345cfd0SStefan Hajnocziimgfmt = os.environ.get('IMGFMT', 'raw') 87f345cfd0SStefan Hajnocziimgproto = os.environ.get('IMGPROTO', 'file') 883e0105e0SMax Reitz 893e0105e0SMax Reitztry: 903e0105e0SMax Reitz test_dir = os.environ['TEST_DIR'] 913e0105e0SMax Reitz sock_dir = os.environ['SOCK_DIR'] 923e0105e0SMax Reitz cachemode = os.environ['CACHEMODE'] 933e0105e0SMax Reitz aiomode = os.environ['AIOMODE'] 943e0105e0SMax Reitz qemu_default_machine = os.environ['QEMU_DEFAULT_MACHINE'] 953e0105e0SMax Reitzexcept KeyError: 963e0105e0SMax Reitz # We are using these variables as proxies to indicate that we're 973e0105e0SMax Reitz # not being run via "check". There may be other things set up by 983e0105e0SMax Reitz # "check" that individual test cases rely on. 993e0105e0SMax Reitz sys.stderr.write('Please run this test via the "check" script\n') 1003e0105e0SMax Reitz sys.exit(os.EX_USAGE) 101f345cfd0SStefan Hajnoczi 102a9b4c6bbSEmanuele Giuseppe Espositoqemu_valgrind = [] 103a9b4c6bbSEmanuele Giuseppe Espositoif os.environ.get('VALGRIND_QEMU') == "y" and \ 104a9b4c6bbSEmanuele Giuseppe Esposito os.environ.get('NO_VALGRIND') != "y": 105a9b4c6bbSEmanuele Giuseppe Esposito valgrind_logfile = "--log-file=" + test_dir 106a9b4c6bbSEmanuele Giuseppe Esposito # %p allows to put the valgrind process PID, since 107a9b4c6bbSEmanuele Giuseppe Esposito # we don't know it a priori (subprocess.Popen is 108a9b4c6bbSEmanuele Giuseppe Esposito # not yet invoked) 109a9b4c6bbSEmanuele Giuseppe Esposito valgrind_logfile += "/%p.valgrind" 110a9b4c6bbSEmanuele Giuseppe Esposito 111a9b4c6bbSEmanuele Giuseppe Esposito qemu_valgrind = ['valgrind', valgrind_logfile, '--error-exitcode=99'] 112a9b4c6bbSEmanuele Giuseppe Esposito 11385a353a0SVladimir Sementsov-Ogievskiyluks_default_secret_object = 'secret,id=keysec0,data=' + \ 11458ebcb65SJohn Snow os.environ.get('IMGKEYSECRET', '') 11585a353a0SVladimir Sementsov-Ogievskiyluks_default_key_secret_opt = 'key-secret=keysec0' 11685a353a0SVladimir Sementsov-Ogievskiy 11755b11630SVladimir Sementsov-Ogievskiysample_img_dir = os.environ['SAMPLE_IMG_DIR'] 11855b11630SVladimir Sementsov-Ogievskiy 11955b11630SVladimir Sementsov-Ogievskiy 120206dc475SJohn Snow@contextmanager 121206dc475SJohn Snowdef change_log_level( 122206dc475SJohn Snow logger_name: str, level: int = logging.CRITICAL) -> Iterator[None]: 123206dc475SJohn Snow """ 124206dc475SJohn Snow Utility function for temporarily changing the log level of a logger. 125206dc475SJohn Snow 126206dc475SJohn Snow This can be used to silence errors that are expected or uninteresting. 127206dc475SJohn Snow """ 128206dc475SJohn Snow _logger = logging.getLogger(logger_name) 129206dc475SJohn Snow current_level = _logger.level 130206dc475SJohn Snow _logger.setLevel(level) 131206dc475SJohn Snow 132206dc475SJohn Snow try: 133206dc475SJohn Snow yield 134206dc475SJohn Snow finally: 135206dc475SJohn Snow _logger.setLevel(current_level) 136206dc475SJohn Snow 137206dc475SJohn Snow 13855b11630SVladimir Sementsov-Ogievskiydef unarchive_sample_image(sample, fname): 13955b11630SVladimir Sementsov-Ogievskiy sample_fname = os.path.join(sample_img_dir, sample + '.bz2') 14055b11630SVladimir Sementsov-Ogievskiy with bz2.open(sample_fname) as f_in, open(fname, 'wb') as f_out: 14155b11630SVladimir Sementsov-Ogievskiy shutil.copyfileobj(f_in, f_out) 14255b11630SVladimir Sementsov-Ogievskiy 14385a353a0SVladimir Sementsov-Ogievskiy 144c34ec513SVladimir Sementsov-Ogievskiydef qemu_tool_popen(args: Sequence[str], 145c34ec513SVladimir Sementsov-Ogievskiy connect_stderr: bool = True) -> 'subprocess.Popen[str]': 146c34ec513SVladimir Sementsov-Ogievskiy stderr = subprocess.STDOUT if connect_stderr else None 147c34ec513SVladimir Sementsov-Ogievskiy # pylint: disable=consider-using-with 148c34ec513SVladimir Sementsov-Ogievskiy return subprocess.Popen(args, 149c34ec513SVladimir Sementsov-Ogievskiy stdout=subprocess.PIPE, 150c34ec513SVladimir Sementsov-Ogievskiy stderr=stderr, 151c34ec513SVladimir Sementsov-Ogievskiy universal_newlines=True) 152c34ec513SVladimir Sementsov-Ogievskiy 153c34ec513SVladimir Sementsov-Ogievskiy 15491efbae9SKevin Wolfdef qemu_tool_pipe_and_status(tool: str, args: Sequence[str], 155a70eeb3dSVladimir Sementsov-Ogievskiy connect_stderr: bool = True, 156a70eeb3dSVladimir Sementsov-Ogievskiy drop_successful_output: bool = False) \ 157a70eeb3dSVladimir Sementsov-Ogievskiy -> Tuple[str, int]: 15891efbae9SKevin Wolf """ 15991efbae9SKevin Wolf Run a tool and return both its output and its exit code 16091efbae9SKevin Wolf """ 161c34ec513SVladimir Sementsov-Ogievskiy with qemu_tool_popen(args, connect_stderr) as subp: 16291efbae9SKevin Wolf output = subp.communicate()[0] 16391efbae9SKevin Wolf if subp.returncode < 0: 1645bd04f61SVladimir Sementsov-Ogievskiy cmd = ' '.join(args) 165ac4e14f5SEmanuele Giuseppe Esposito sys.stderr.write(f'{tool} received signal \ 166ac4e14f5SEmanuele Giuseppe Esposito {-subp.returncode}: {cmd}\n') 167a70eeb3dSVladimir Sementsov-Ogievskiy if drop_successful_output and subp.returncode == 0: 168a70eeb3dSVladimir Sementsov-Ogievskiy output = '' 16991efbae9SKevin Wolf return (output, subp.returncode) 17091efbae9SKevin Wolf 17122e29bceSVladimir Sementsov-Ogievskiydef qemu_img_create_prepare_args(args: List[str]) -> List[str]: 17222e29bceSVladimir Sementsov-Ogievskiy if not args or args[0] != 'create': 17322e29bceSVladimir Sementsov-Ogievskiy return list(args) 17422e29bceSVladimir Sementsov-Ogievskiy args = args[1:] 17522e29bceSVladimir Sementsov-Ogievskiy 17622e29bceSVladimir Sementsov-Ogievskiy p = argparse.ArgumentParser(allow_abbrev=False) 17728a5ad93SVladimir Sementsov-Ogievskiy # -o option may be specified several times 17828a5ad93SVladimir Sementsov-Ogievskiy p.add_argument('-o', action='append', default=[]) 17922e29bceSVladimir Sementsov-Ogievskiy p.add_argument('-f') 18022e29bceSVladimir Sementsov-Ogievskiy parsed, remaining = p.parse_known_args(args) 18122e29bceSVladimir Sementsov-Ogievskiy 18228a5ad93SVladimir Sementsov-Ogievskiy opts_list = parsed.o 18328a5ad93SVladimir Sementsov-Ogievskiy 18422e29bceSVladimir Sementsov-Ogievskiy result = ['create'] 18522e29bceSVladimir Sementsov-Ogievskiy if parsed.f is not None: 18622e29bceSVladimir Sementsov-Ogievskiy result += ['-f', parsed.f] 18722e29bceSVladimir Sementsov-Ogievskiy 18822e29bceSVladimir Sementsov-Ogievskiy # IMGOPTS most probably contain options specific for the selected format, 18922e29bceSVladimir Sementsov-Ogievskiy # like extended_l2 or compression_type for qcow2. Test may want to create 19022e29bceSVladimir Sementsov-Ogievskiy # additional images in other formats that doesn't support these options. 19122e29bceSVladimir Sementsov-Ogievskiy # So, use IMGOPTS only for images created in imgfmt format. 19228a5ad93SVladimir Sementsov-Ogievskiy imgopts = os.environ.get('IMGOPTS') 19328a5ad93SVladimir Sementsov-Ogievskiy if imgopts and parsed.f == imgfmt: 19428a5ad93SVladimir Sementsov-Ogievskiy opts_list.insert(0, imgopts) 19528a5ad93SVladimir Sementsov-Ogievskiy 19628a5ad93SVladimir Sementsov-Ogievskiy # default luks support 19728a5ad93SVladimir Sementsov-Ogievskiy if parsed.f == 'luks' and \ 19828a5ad93SVladimir Sementsov-Ogievskiy all('key-secret' not in opts for opts in opts_list): 19928a5ad93SVladimir Sementsov-Ogievskiy result += ['--object', luks_default_secret_object] 20028a5ad93SVladimir Sementsov-Ogievskiy opts_list.append(luks_default_key_secret_opt) 20128a5ad93SVladimir Sementsov-Ogievskiy 20228a5ad93SVladimir Sementsov-Ogievskiy for opts in opts_list: 20328a5ad93SVladimir Sementsov-Ogievskiy result += ['-o', opts] 20422e29bceSVladimir Sementsov-Ogievskiy 20522e29bceSVladimir Sementsov-Ogievskiy result += remaining 20622e29bceSVladimir Sementsov-Ogievskiy 20722e29bceSVladimir Sementsov-Ogievskiy return result 20822e29bceSVladimir Sementsov-Ogievskiy 209b2d68a8eSJohn Snow 210b2d68a8eSJohn Snowdef qemu_tool(*args: str, check: bool = True, combine_stdio: bool = True 2112882ccf8SJohn Snow ) -> 'subprocess.CompletedProcess[str]': 2122882ccf8SJohn Snow """ 213b2d68a8eSJohn Snow Run a qemu tool and return its status code and console output. 2142882ccf8SJohn Snow 215b2d68a8eSJohn Snow :param args: full command line to run. 2162882ccf8SJohn Snow :param check: Enforce a return code of zero. 2172882ccf8SJohn Snow :param combine_stdio: set to False to keep stdout/stderr separated. 2182882ccf8SJohn Snow 2192882ccf8SJohn Snow :raise VerboseProcessError: 2202882ccf8SJohn Snow When the return code is negative, or on any non-zero exit code 2212882ccf8SJohn Snow when 'check=True' was provided (the default). This exception has 2222882ccf8SJohn Snow 'stdout', 'stderr', and 'returncode' properties that may be 2232882ccf8SJohn Snow inspected to show greater detail. If this exception is not 2242882ccf8SJohn Snow handled, the command-line, return code, and all console output 2252882ccf8SJohn Snow will be included at the bottom of the stack trace. 2262882ccf8SJohn Snow 2272882ccf8SJohn Snow :return: 2282882ccf8SJohn Snow a CompletedProcess. This object has args, returncode, and stdout 2292882ccf8SJohn Snow properties. If streams are not combined, it will also have a 2302882ccf8SJohn Snow stderr property. 2312882ccf8SJohn Snow """ 2322882ccf8SJohn Snow subp = subprocess.run( 233b2d68a8eSJohn Snow args, 2342882ccf8SJohn Snow stdout=subprocess.PIPE, 2352882ccf8SJohn Snow stderr=subprocess.STDOUT if combine_stdio else subprocess.PIPE, 2362882ccf8SJohn Snow universal_newlines=True, 2372882ccf8SJohn Snow check=False 2382882ccf8SJohn Snow ) 2392882ccf8SJohn Snow 2402882ccf8SJohn Snow if check and subp.returncode or (subp.returncode < 0): 2412882ccf8SJohn Snow raise VerboseProcessError( 242b2d68a8eSJohn Snow subp.returncode, args, 2432882ccf8SJohn Snow output=subp.stdout, 2442882ccf8SJohn Snow stderr=subp.stderr, 2452882ccf8SJohn Snow ) 2462882ccf8SJohn Snow 2472882ccf8SJohn Snow return subp 2482882ccf8SJohn Snow 249f345cfd0SStefan Hajnoczi 250b2d68a8eSJohn Snowdef qemu_img(*args: str, check: bool = True, combine_stdio: bool = True 251b2d68a8eSJohn Snow ) -> 'subprocess.CompletedProcess[str]': 252b2d68a8eSJohn Snow """ 253b2d68a8eSJohn Snow Run QEMU_IMG_PROG and return its status code and console output. 254b2d68a8eSJohn Snow 255b2d68a8eSJohn Snow This function always prepends QEMU_IMG_OPTIONS and may further alter 256b2d68a8eSJohn Snow the args for 'create' commands. 257b2d68a8eSJohn Snow 258b2d68a8eSJohn Snow See `qemu_tool()` for greater detail. 259b2d68a8eSJohn Snow """ 260b2d68a8eSJohn Snow full_args = qemu_img_args + qemu_img_create_prepare_args(list(args)) 261b2d68a8eSJohn Snow return qemu_tool(*full_args, check=check, combine_stdio=combine_stdio) 262b2d68a8eSJohn Snow 263b2d68a8eSJohn Snow 2648a57a4beSMax Reitzdef ordered_qmp(qmsg, conv_keys=True): 265039be85cSJohn Snow # Dictionaries are not ordered prior to 3.6, therefore: 266039be85cSJohn Snow if isinstance(qmsg, list): 267039be85cSJohn Snow return [ordered_qmp(atom) for atom in qmsg] 268039be85cSJohn Snow if isinstance(qmsg, dict): 2690706e87dSJohn Snow od = OrderedDict() 270039be85cSJohn Snow for k, v in sorted(qmsg.items()): 2718a57a4beSMax Reitz if conv_keys: 2728a57a4beSMax Reitz k = k.replace('_', '-') 2738a57a4beSMax Reitz od[k] = ordered_qmp(v, conv_keys=False) 2740706e87dSJohn Snow return od 275039be85cSJohn Snow return qmsg 2760706e87dSJohn Snow 2772882ccf8SJohn Snowdef qemu_img_create(*args: str) -> 'subprocess.CompletedProcess[str]': 27828a5ad93SVladimir Sementsov-Ogievskiy return qemu_img('create', *args) 27985a353a0SVladimir Sementsov-Ogievskiy 28029768d04SJohn Snowdef qemu_img_json(*args: str) -> Any: 28129768d04SJohn Snow """ 28229768d04SJohn Snow Run qemu-img and return its output as deserialized JSON. 28329768d04SJohn Snow 28429768d04SJohn Snow :raise CalledProcessError: 28529768d04SJohn Snow When qemu-img crashes, or returns a non-zero exit code without 28629768d04SJohn Snow producing a valid JSON document to stdout. 28729768d04SJohn Snow :raise JSONDecoderError: 28829768d04SJohn Snow When qemu-img returns 0, but failed to produce a valid JSON document. 28929768d04SJohn Snow 29029768d04SJohn Snow :return: A deserialized JSON object; probably a dict[str, Any]. 29129768d04SJohn Snow """ 29229768d04SJohn Snow try: 29329768d04SJohn Snow res = qemu_img(*args, combine_stdio=False) 29429768d04SJohn Snow except subprocess.CalledProcessError as exc: 29529768d04SJohn Snow # Terminated due to signal. Don't bother. 29629768d04SJohn Snow if exc.returncode < 0: 29729768d04SJohn Snow raise 29829768d04SJohn Snow 29929768d04SJohn Snow # Commands like 'check' can return failure (exit codes 2 and 3) 30029768d04SJohn Snow # to indicate command completion, but with errors found. For 30129768d04SJohn Snow # multi-command flexibility, ignore the exact error codes and 30229768d04SJohn Snow # *try* to load JSON. 30329768d04SJohn Snow try: 30429768d04SJohn Snow return json.loads(exc.stdout) 30529768d04SJohn Snow except json.JSONDecodeError: 30629768d04SJohn Snow # Nope. This thing is toast. Raise the /process/ error. 30729768d04SJohn Snow pass 30829768d04SJohn Snow raise 30929768d04SJohn Snow 31029768d04SJohn Snow return json.loads(res.stdout) 31129768d04SJohn Snow 3120f7d7d72SJohn Snowdef qemu_img_measure(*args: str) -> Any: 3130f7d7d72SJohn Snow return qemu_img_json("measure", "--output", "json", *args) 3144b914b01SNir Soffer 3150f7d7d72SJohn Snowdef qemu_img_check(*args: str) -> Any: 3160f7d7d72SJohn Snow return qemu_img_json("check", "--output", "json", *args) 3174b914b01SNir Soffer 3189ebb2b76SJohn Snowdef qemu_img_info(*args: str) -> Any: 3199ebb2b76SJohn Snow return qemu_img_json('info', "--output", "json", *args) 3209ebb2b76SJohn Snow 3211670ae7aSJohn Snowdef qemu_img_map(*args: str) -> Any: 3221670ae7aSJohn Snow return qemu_img_json('map', "--output", "json", *args) 3231670ae7aSJohn Snow 3248f685ac3SJohn Snowdef qemu_img_log(*args: str, check: bool = True 3258f685ac3SJohn Snow ) -> 'subprocess.CompletedProcess[str]': 3268f685ac3SJohn Snow result = qemu_img(*args, check=check) 327f400e14dSJohn Snow log(result.stdout, filters=[filter_testfiles]) 328ac6fb43eSKevin Wolf return result 329ac6fb43eSKevin Wolf 330f400e14dSJohn Snowdef img_info_log(filename: str, filter_path: Optional[str] = None, 331f400e14dSJohn Snow use_image_opts: bool = False, extra_args: Sequence[str] = (), 332bcc6777aSHanna Reitz check: bool = True, drop_child_info: bool = True, 333f400e14dSJohn Snow ) -> None: 3345ba141dcSKevin Wolf args = ['info'] 3353bd2b942SVladimir Sementsov-Ogievskiy if use_image_opts: 3365ba141dcSKevin Wolf args.append('--image-opts') 3375ba141dcSKevin Wolf else: 3385ba141dcSKevin Wolf args += ['-f', imgfmt] 3395ba141dcSKevin Wolf args += extra_args 3405ba141dcSKevin Wolf args.append(filename) 3415ba141dcSKevin Wolf 3428f685ac3SJohn Snow output = qemu_img(*args, check=check).stdout 3436b605adeSKevin Wolf if not filter_path: 3446b605adeSKevin Wolf filter_path = filename 345bcc6777aSHanna Reitz log(filter_img_info(output, filter_path, drop_child_info)) 3466b605adeSKevin Wolf 34794a781f2SVladimir Sementsov-Ogievskiydef qemu_io_wrap_args(args: Sequence[str]) -> List[str]: 34894a781f2SVladimir Sementsov-Ogievskiy if '-f' in args or '--image-opts' in args: 34994a781f2SVladimir Sementsov-Ogievskiy return qemu_io_args_no_fmt + list(args) 35094a781f2SVladimir Sementsov-Ogievskiy else: 35194a781f2SVladimir Sementsov-Ogievskiy return qemu_io_args + list(args) 35294a781f2SVladimir Sementsov-Ogievskiy 35375c90eeeSVladimir Sementsov-Ogievskiydef qemu_io_popen(*args): 35475c90eeeSVladimir Sementsov-Ogievskiy return qemu_tool_popen(qemu_io_wrap_args(args)) 35575c90eeeSVladimir Sementsov-Ogievskiy 3566dede6a4SJohn Snowdef qemu_io(*args: str, check: bool = True, combine_stdio: bool = True 3576dede6a4SJohn Snow ) -> 'subprocess.CompletedProcess[str]': 3586dede6a4SJohn Snow """ 3596dede6a4SJohn Snow Run QEMU_IO_PROG and return the status code and console output. 3606dede6a4SJohn Snow 3616dede6a4SJohn Snow This function always prepends either QEMU_IO_OPTIONS or 3626dede6a4SJohn Snow QEMU_IO_OPTIONS_NO_FMT. 3636dede6a4SJohn Snow """ 3646dede6a4SJohn Snow return qemu_tool(*qemu_io_wrap_args(args), 3656dede6a4SJohn Snow check=check, combine_stdio=combine_stdio) 366f345cfd0SStefan Hajnoczi 36740bfeae1SJohn Snowdef qemu_io_log(*args: str, check: bool = True 36840bfeae1SJohn Snow ) -> 'subprocess.CompletedProcess[str]': 36940bfeae1SJohn Snow result = qemu_io(*args, check=check) 3706dede6a4SJohn Snow log(result.stdout, filters=[filter_testfiles, filter_qemu_io]) 371a96f0350SKevin Wolf return result 372a96f0350SKevin Wolf 3739fa90eecSVladimir Sementsov-Ogievskiyclass QemuIoInteractive: 3749fa90eecSVladimir Sementsov-Ogievskiy def __init__(self, *args): 37594a781f2SVladimir Sementsov-Ogievskiy self.args = qemu_io_wrap_args(args) 376ac4e14f5SEmanuele Giuseppe Esposito # We need to keep the Popen objext around, and not 377ac4e14f5SEmanuele Giuseppe Esposito # close it immediately. Therefore, disable the pylint check: 378ac4e14f5SEmanuele Giuseppe Esposito # pylint: disable=consider-using-with 3799fa90eecSVladimir Sementsov-Ogievskiy self._p = subprocess.Popen(self.args, stdin=subprocess.PIPE, 3809fa90eecSVladimir Sementsov-Ogievskiy stdout=subprocess.PIPE, 3818eb5e674SMax Reitz stderr=subprocess.STDOUT, 3828eb5e674SMax Reitz universal_newlines=True) 3831f4b774aSVladimir Sementsov-Ogievskiy out = self._p.stdout.read(9) 3841f4b774aSVladimir Sementsov-Ogievskiy if out != 'qemu-io> ': 3851f4b774aSVladimir Sementsov-Ogievskiy # Most probably qemu-io just failed to start. 3861f4b774aSVladimir Sementsov-Ogievskiy # Let's collect the whole output and exit. 3871f4b774aSVladimir Sementsov-Ogievskiy out += self._p.stdout.read() 3881f4b774aSVladimir Sementsov-Ogievskiy self._p.wait(timeout=1) 3891f4b774aSVladimir Sementsov-Ogievskiy raise ValueError(out) 3909fa90eecSVladimir Sementsov-Ogievskiy 3919fa90eecSVladimir Sementsov-Ogievskiy def close(self): 3929fa90eecSVladimir Sementsov-Ogievskiy self._p.communicate('q\n') 3939fa90eecSVladimir Sementsov-Ogievskiy 3949fa90eecSVladimir Sementsov-Ogievskiy def _read_output(self): 3959fa90eecSVladimir Sementsov-Ogievskiy pattern = 'qemu-io> ' 3969fa90eecSVladimir Sementsov-Ogievskiy n = len(pattern) 3979fa90eecSVladimir Sementsov-Ogievskiy pos = 0 3989fa90eecSVladimir Sementsov-Ogievskiy s = [] 3999fa90eecSVladimir Sementsov-Ogievskiy while pos != n: 4009fa90eecSVladimir Sementsov-Ogievskiy c = self._p.stdout.read(1) 4019fa90eecSVladimir Sementsov-Ogievskiy # check unexpected EOF 4029fa90eecSVladimir Sementsov-Ogievskiy assert c != '' 4039fa90eecSVladimir Sementsov-Ogievskiy s.append(c) 4049fa90eecSVladimir Sementsov-Ogievskiy if c == pattern[pos]: 4059fa90eecSVladimir Sementsov-Ogievskiy pos += 1 4069fa90eecSVladimir Sementsov-Ogievskiy else: 4079fa90eecSVladimir Sementsov-Ogievskiy pos = 0 4089fa90eecSVladimir Sementsov-Ogievskiy 4099fa90eecSVladimir Sementsov-Ogievskiy return ''.join(s[:-n]) 4109fa90eecSVladimir Sementsov-Ogievskiy 4119fa90eecSVladimir Sementsov-Ogievskiy def cmd(self, cmd): 4129fa90eecSVladimir Sementsov-Ogievskiy # quit command is in close(), '\n' is added automatically 4139fa90eecSVladimir Sementsov-Ogievskiy assert '\n' not in cmd 4149fa90eecSVladimir Sementsov-Ogievskiy cmd = cmd.strip() 4156a96d87cSJohn Snow assert cmd not in ('q', 'quit') 4169fa90eecSVladimir Sementsov-Ogievskiy self._p.stdin.write(cmd + '\n') 417f544adf7SMax Reitz self._p.stdin.flush() 4189fa90eecSVladimir Sementsov-Ogievskiy return self._read_output() 4199fa90eecSVladimir Sementsov-Ogievskiy 4209fa90eecSVladimir Sementsov-Ogievskiy 421091dc7b2SHanna Reitzclass QemuStorageDaemon: 422ec88eed8SHanna Reitz _qmp: Optional[QEMUMonitorProtocol] = None 423ec88eed8SHanna Reitz _qmpsock: Optional[str] = None 424ec88eed8SHanna Reitz # Python < 3.8 would complain if this type were not a string literal 425ec88eed8SHanna Reitz # (importing `annotations` from `__future__` would work; but not on <= 3.6) 426ec88eed8SHanna Reitz _p: 'Optional[subprocess.Popen[bytes]]' = None 427ec88eed8SHanna Reitz 428ec88eed8SHanna Reitz def __init__(self, *args: str, instance_id: str = 'a', qmp: bool = False): 429091dc7b2SHanna Reitz assert '--pidfile' not in args 430091dc7b2SHanna Reitz self.pidfile = os.path.join(test_dir, f'qsd-{instance_id}-pid') 431091dc7b2SHanna Reitz all_args = [qsd_prog] + list(args) + ['--pidfile', self.pidfile] 432091dc7b2SHanna Reitz 433ec88eed8SHanna Reitz if qmp: 434ec88eed8SHanna Reitz self._qmpsock = os.path.join(sock_dir, f'qsd-{instance_id}.sock') 435ec88eed8SHanna Reitz all_args += ['--chardev', 436ec88eed8SHanna Reitz f'socket,id=qmp-sock,path={self._qmpsock}', 437ec88eed8SHanna Reitz '--monitor', 'qmp-sock'] 438ec88eed8SHanna Reitz 439ec88eed8SHanna Reitz self._qmp = QEMUMonitorProtocol(self._qmpsock, server=True) 440ec88eed8SHanna Reitz 441091dc7b2SHanna Reitz # Cannot use with here, we want the subprocess to stay around 442091dc7b2SHanna Reitz # pylint: disable=consider-using-with 443091dc7b2SHanna Reitz self._p = subprocess.Popen(all_args) 444ec88eed8SHanna Reitz if self._qmp is not None: 445ec88eed8SHanna Reitz self._qmp.accept() 446091dc7b2SHanna Reitz while not os.path.exists(self.pidfile): 447091dc7b2SHanna Reitz if self._p.poll() is not None: 448091dc7b2SHanna Reitz cmd = ' '.join(all_args) 449091dc7b2SHanna Reitz raise RuntimeError( 450091dc7b2SHanna Reitz 'qemu-storage-daemon terminated with exit code ' + 451091dc7b2SHanna Reitz f'{self._p.returncode}: {cmd}') 452091dc7b2SHanna Reitz 453091dc7b2SHanna Reitz time.sleep(0.01) 454091dc7b2SHanna Reitz 455091dc7b2SHanna Reitz with open(self.pidfile, encoding='utf-8') as f: 456091dc7b2SHanna Reitz self._pid = int(f.read().strip()) 457091dc7b2SHanna Reitz 458091dc7b2SHanna Reitz assert self._pid == self._p.pid 459091dc7b2SHanna Reitz 460ec88eed8SHanna Reitz def qmp(self, cmd: str, args: Optional[Dict[str, object]] = None) \ 461ec88eed8SHanna Reitz -> QMPMessage: 462ec88eed8SHanna Reitz assert self._qmp is not None 46337274707SVladimir Sementsov-Ogievskiy return self._qmp.cmd_raw(cmd, args) 464ec88eed8SHanna Reitz 46595fdd8dbSKevin Wolf def get_qmp(self) -> QEMUMonitorProtocol: 46695fdd8dbSKevin Wolf assert self._qmp is not None 46795fdd8dbSKevin Wolf return self._qmp 46895fdd8dbSKevin Wolf 4693a8736cfSVladimir Sementsov-Ogievskiy def cmd(self, cmd: str, args: Optional[Dict[str, object]] = None) \ 4703a8736cfSVladimir Sementsov-Ogievskiy -> QMPReturnValue: 4713a8736cfSVladimir Sementsov-Ogievskiy assert self._qmp is not None 4723a8736cfSVladimir Sementsov-Ogievskiy return self._qmp.cmd(cmd, **(args or {})) 4733a8736cfSVladimir Sementsov-Ogievskiy 474091dc7b2SHanna Reitz def stop(self, kill_signal=15): 475091dc7b2SHanna Reitz self._p.send_signal(kill_signal) 476091dc7b2SHanna Reitz self._p.wait() 477091dc7b2SHanna Reitz self._p = None 478091dc7b2SHanna Reitz 479ec88eed8SHanna Reitz if self._qmp: 480ec88eed8SHanna Reitz self._qmp.close() 481ec88eed8SHanna Reitz 482ec88eed8SHanna Reitz if self._qmpsock is not None: 483ec88eed8SHanna Reitz try: 484ec88eed8SHanna Reitz os.remove(self._qmpsock) 485ec88eed8SHanna Reitz except OSError: 486ec88eed8SHanna Reitz pass 487091dc7b2SHanna Reitz try: 488091dc7b2SHanna Reitz os.remove(self.pidfile) 489091dc7b2SHanna Reitz except OSError: 490091dc7b2SHanna Reitz pass 491091dc7b2SHanna Reitz 492091dc7b2SHanna Reitz def __del__(self): 493091dc7b2SHanna Reitz if self._p is not None: 494091dc7b2SHanna Reitz self.stop(kill_signal=9) 495091dc7b2SHanna Reitz 496091dc7b2SHanna Reitz 497bec87774SMax Reitzdef qemu_nbd(*args): 498bec87774SMax Reitz '''Run qemu-nbd in daemon mode and return the parent's exit code''' 499bec87774SMax Reitz return subprocess.call(qemu_nbd_args + ['--fork'] + list(args)) 500bec87774SMax Reitz 50191efbae9SKevin Wolfdef qemu_nbd_early_pipe(*args: str) -> Tuple[int, str]: 502e1e6eccdSMax Reitz '''Run qemu-nbd in daemon mode and return both the parent's exit code 5036177b584SMax Reitz and its output in case of an error''' 50491efbae9SKevin Wolf full_args = qemu_nbd_args + ['--fork'] + list(args) 50591efbae9SKevin Wolf output, returncode = qemu_tool_pipe_and_status('qemu-nbd', full_args, 50691efbae9SKevin Wolf connect_stderr=False) 50791efbae9SKevin Wolf return returncode, output if returncode else '' 508e1e6eccdSMax Reitz 50981b6b2bcSKevin Wolfdef qemu_nbd_list_log(*args: str) -> str: 51081b6b2bcSKevin Wolf '''Run qemu-nbd to list remote exports''' 51181b6b2bcSKevin Wolf full_args = [qemu_nbd_prog, '-L'] + list(args) 51281b6b2bcSKevin Wolf output, _ = qemu_tool_pipe_and_status('qemu-nbd', full_args) 51381b6b2bcSKevin Wolf log(output, filters=[filter_testfiles, filter_nbd_exports]) 51481b6b2bcSKevin Wolf return output 51581b6b2bcSKevin Wolf 516b7719bcaSNir Soffer@contextmanager 51723ee0ec2SVladimir Sementsov-Ogievskiydef qemu_nbd_popen(*args): 518b7719bcaSNir Soffer '''Context manager running qemu-nbd within the context''' 5193f7db418SVladimir Sementsov-Ogievskiy pid_file = file_path("qemu_nbd_popen-nbd-pid-file") 5203f7db418SVladimir Sementsov-Ogievskiy 5213f7db418SVladimir Sementsov-Ogievskiy assert not os.path.exists(pid_file) 522b7719bcaSNir Soffer 523b7719bcaSNir Soffer cmd = list(qemu_nbd_args) 524b7719bcaSNir Soffer cmd.extend(('--persistent', '--pid-file', pid_file)) 525b7719bcaSNir Soffer cmd.extend(args) 526b7719bcaSNir Soffer 527b7719bcaSNir Soffer log('Start NBD server') 528ac4e14f5SEmanuele Giuseppe Esposito with subprocess.Popen(cmd) as p: 529b7719bcaSNir Soffer try: 530b7719bcaSNir Soffer while not os.path.exists(pid_file): 531b7719bcaSNir Soffer if p.poll() is not None: 532b7719bcaSNir Soffer raise RuntimeError( 533b7719bcaSNir Soffer "qemu-nbd terminated with exit code {}: {}" 534b7719bcaSNir Soffer .format(p.returncode, ' '.join(cmd))) 535b7719bcaSNir Soffer 536b7719bcaSNir Soffer time.sleep(0.01) 537b7719bcaSNir Soffer yield 538b7719bcaSNir Soffer finally: 5393f7db418SVladimir Sementsov-Ogievskiy if os.path.exists(pid_file): 5403f7db418SVladimir Sementsov-Ogievskiy os.remove(pid_file) 541b7719bcaSNir Soffer log('Kill NBD server') 542b7719bcaSNir Soffer p.kill() 543b7719bcaSNir Soffer p.wait() 54423ee0ec2SVladimir Sementsov-Ogievskiy 545569131d5SJohn Snowdef compare_images(img1: str, img2: str, 546569131d5SJohn Snow fmt1: str = imgfmt, fmt2: str = imgfmt) -> bool: 547569131d5SJohn Snow """ 548569131d5SJohn Snow Compare two images with QEMU_IMG; return True if they are identical. 549569131d5SJohn Snow 550569131d5SJohn Snow :raise CalledProcessError: 551569131d5SJohn Snow when qemu-img crashes or returns a status code of anything other 552569131d5SJohn Snow than 0 (identical) or 1 (different). 553569131d5SJohn Snow """ 554569131d5SJohn Snow try: 555569131d5SJohn Snow qemu_img('compare', '-f', fmt1, '-F', fmt2, img1, img2) 556569131d5SJohn Snow return True 557569131d5SJohn Snow except subprocess.CalledProcessError as exc: 558569131d5SJohn Snow if exc.returncode == 1: 559569131d5SJohn Snow return False 560569131d5SJohn Snow raise 5613a3918c3SStefan Hajnoczi 5622499a096SStefan Hajnoczidef create_image(name, size): 5632499a096SStefan Hajnoczi '''Create a fully-allocated raw image with sector markers''' 564ac4e14f5SEmanuele Giuseppe Esposito with open(name, 'wb') as file: 5652499a096SStefan Hajnoczi i = 0 5662499a096SStefan Hajnoczi while i < size: 5679a3a9a63SMax Reitz sector = struct.pack('>l504xl', i // 512, i // 512) 5682499a096SStefan Hajnoczi file.write(sector) 5692499a096SStefan Hajnoczi i = i + 512 5702499a096SStefan Hajnoczi 5719ebb2b76SJohn Snowdef image_size(img: str) -> int: 5729ebb2b76SJohn Snow """Return image's virtual size""" 5739ebb2b76SJohn Snow value = qemu_img_info('-f', imgfmt, img)['virtual-size'] 5749ebb2b76SJohn Snow if not isinstance(value, int): 5759ebb2b76SJohn Snow type_name = type(value).__name__ 5769ebb2b76SJohn Snow raise TypeError("Expected 'int' for 'virtual-size', " 5779ebb2b76SJohn Snow f"got '{value}' of type '{type_name}'") 5789ebb2b76SJohn Snow return value 57974f69050SFam Zheng 580011a5761SMax Reitzdef is_str(val): 581011a5761SMax Reitz return isinstance(val, str) 582011a5761SMax Reitz 583a2d1c8fdSDaniel P. Berrangetest_dir_re = re.compile(r"%s" % test_dir) 584a2d1c8fdSDaniel P. Berrangedef filter_test_dir(msg): 585a2d1c8fdSDaniel P. Berrange return test_dir_re.sub("TEST_DIR", msg) 586a2d1c8fdSDaniel P. Berrange 587a2d1c8fdSDaniel P. Berrangewin32_re = re.compile(r"\r") 588a2d1c8fdSDaniel P. Berrangedef filter_win32(msg): 589a2d1c8fdSDaniel P. Berrange return win32_re.sub("", msg) 590a2d1c8fdSDaniel P. Berrange 591b031e9a5SJohn Snowqemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* " 592b031e9a5SJohn Snow r"\([0-9\/.inf]* [EPTGMKiBbytes]*\/sec " 593b031e9a5SJohn Snow r"and [0-9\/.inf]* ops\/sec\)") 594a2d1c8fdSDaniel P. Berrangedef filter_qemu_io(msg): 595a2d1c8fdSDaniel P. Berrange msg = filter_win32(msg) 596b031e9a5SJohn Snow return qemu_io_re.sub("X ops; XX:XX:XX.X " 597b031e9a5SJohn Snow "(XXX YYY/sec and XXX ops/sec)", msg) 598a2d1c8fdSDaniel P. Berrange 599a2d1c8fdSDaniel P. Berrangechown_re = re.compile(r"chown [0-9]+:[0-9]+") 600a2d1c8fdSDaniel P. Berrangedef filter_chown(msg): 601a2d1c8fdSDaniel P. Berrange return chown_re.sub("chown UID:GID", msg) 602a2d1c8fdSDaniel P. Berrange 60312314f2dSStefan Hajnoczidef filter_qmp_event(event): 60412314f2dSStefan Hajnoczi '''Filter a QMP event dict''' 60512314f2dSStefan Hajnoczi event = dict(event) 60612314f2dSStefan Hajnoczi if 'timestamp' in event: 60712314f2dSStefan Hajnoczi event['timestamp']['seconds'] = 'SECS' 60812314f2dSStefan Hajnoczi event['timestamp']['microseconds'] = 'USECS' 60912314f2dSStefan Hajnoczi return event 61012314f2dSStefan Hajnoczi 61108fcd611SJohn Snowdef filter_qmp(qmsg, filter_fn): 61208fcd611SJohn Snow '''Given a string filter, filter a QMP object's values. 61308fcd611SJohn Snow filter_fn takes a (key, value) pair.''' 61408fcd611SJohn Snow # Iterate through either lists or dicts; 61508fcd611SJohn Snow if isinstance(qmsg, list): 61608fcd611SJohn Snow items = enumerate(qmsg) 617da9d88d8SHanna Reitz elif isinstance(qmsg, dict): 61808fcd611SJohn Snow items = qmsg.items() 619da9d88d8SHanna Reitz else: 620da9d88d8SHanna Reitz return filter_fn(None, qmsg) 62108fcd611SJohn Snow 62208fcd611SJohn Snow for k, v in items: 6236a96d87cSJohn Snow if isinstance(v, (dict, list)): 62408fcd611SJohn Snow qmsg[k] = filter_qmp(v, filter_fn) 62508fcd611SJohn Snow else: 62608fcd611SJohn Snow qmsg[k] = filter_fn(k, v) 62708fcd611SJohn Snow return qmsg 62808fcd611SJohn Snow 629e234398aSKevin Wolfdef filter_testfiles(msg): 630df0e032bSVladimir Sementsov-Ogievskiy pref1 = os.path.join(test_dir, "%s-" % (os.getpid())) 631df0e032bSVladimir Sementsov-Ogievskiy pref2 = os.path.join(sock_dir, "%s-" % (os.getpid())) 632df0e032bSVladimir Sementsov-Ogievskiy return msg.replace(pref1, 'TEST_DIR/PID-').replace(pref2, 'SOCK_DIR/PID-') 633e234398aSKevin Wolf 63408fcd611SJohn Snowdef filter_qmp_testfiles(qmsg): 6356a96d87cSJohn Snow def _filter(_key, value): 63656a6e5d0SMax Reitz if is_str(value): 63708fcd611SJohn Snow return filter_testfiles(value) 63808fcd611SJohn Snow return value 63908fcd611SJohn Snow return filter_qmp(qmsg, _filter) 64008fcd611SJohn Snow 6412daba442SMaxim Levitskydef filter_virtio_scsi(output: str) -> str: 6422daba442SMaxim Levitsky return re.sub(r'(virtio-scsi)-(ccw|pci)', r'\1', output) 6432daba442SMaxim Levitsky 6442daba442SMaxim Levitskydef filter_qmp_virtio_scsi(qmsg): 6452daba442SMaxim Levitsky def _filter(_key, value): 6462daba442SMaxim Levitsky if is_str(value): 6472daba442SMaxim Levitsky return filter_virtio_scsi(value) 6482daba442SMaxim Levitsky return value 6492daba442SMaxim Levitsky return filter_qmp(qmsg, _filter) 6502daba442SMaxim Levitsky 651fa1151f8SJohn Snowdef filter_generated_node_ids(msg): 652fa1151f8SJohn Snow return re.sub("#block[0-9]+", "NODE_NAME", msg) 653fa1151f8SJohn Snow 654*d7a64c45SStefan Hajnoczidef filter_qmp_generated_node_ids(qmsg): 655*d7a64c45SStefan Hajnoczi def _filter(_key, value): 656*d7a64c45SStefan Hajnoczi if is_str(value): 657*d7a64c45SStefan Hajnoczi return filter_generated_node_ids(value) 658*d7a64c45SStefan Hajnoczi return value 659*d7a64c45SStefan Hajnoczi return filter_qmp(qmsg, _filter) 660*d7a64c45SStefan Hajnoczi 661bcc6777aSHanna Reitzdef filter_img_info(output: str, filename: str, 662bcc6777aSHanna Reitz drop_child_info: bool = True) -> str: 6636b605adeSKevin Wolf lines = [] 664bcc6777aSHanna Reitz drop_indented = False 6656b605adeSKevin Wolf for line in output.split('\n'): 6666b605adeSKevin Wolf if 'disk size' in line or 'actual-size' in line: 6676b605adeSKevin Wolf continue 668bcc6777aSHanna Reitz 669bcc6777aSHanna Reitz # Drop child node info 670bcc6777aSHanna Reitz if drop_indented: 671bcc6777aSHanna Reitz if line.startswith(' '): 672bcc6777aSHanna Reitz continue 673bcc6777aSHanna Reitz drop_indented = False 674bcc6777aSHanna Reitz if drop_child_info and "Child node '/" in line: 675bcc6777aSHanna Reitz drop_indented = True 676bcc6777aSHanna Reitz continue 677bcc6777aSHanna Reitz 678fd586ce8SKevin Wolf line = line.replace(filename, 'TEST_IMG') 679fd586ce8SKevin Wolf line = filter_testfiles(line) 680fd586ce8SKevin Wolf line = line.replace(imgfmt, 'IMGFMT') 6816b605adeSKevin Wolf line = re.sub('iters: [0-9]+', 'iters: XXX', line) 682b031e9a5SJohn Snow line = re.sub('uuid: [-a-f0-9]+', 683b031e9a5SJohn Snow 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', 684b031e9a5SJohn Snow line) 685bab4feb2SFam Zheng line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line) 686e877bba3SVladimir Sementsov-Ogievskiy line = re.sub('(compression type: )(zlib|zstd)', r'\1COMPRESSION_TYPE', 687e877bba3SVladimir Sementsov-Ogievskiy line) 6886b605adeSKevin Wolf lines.append(line) 6896b605adeSKevin Wolf return '\n'.join(lines) 6906b605adeSKevin Wolf 691f2ea0b20SMax Reitzdef filter_imgfmt(msg): 692f2ea0b20SMax Reitz return msg.replace(imgfmt, 'IMGFMT') 693f2ea0b20SMax Reitz 694f2ea0b20SMax Reitzdef filter_qmp_imgfmt(qmsg): 6956a96d87cSJohn Snow def _filter(_key, value): 696f2ea0b20SMax Reitz if is_str(value): 697f2ea0b20SMax Reitz return filter_imgfmt(value) 698f2ea0b20SMax Reitz return value 699f2ea0b20SMax Reitz return filter_qmp(qmsg, _filter) 700f2ea0b20SMax Reitz 70181b6b2bcSKevin Wolfdef filter_nbd_exports(output: str) -> str: 70281b6b2bcSKevin Wolf return re.sub(r'((min|opt|max) block): [0-9]+', r'\1: XXX', output) 70381b6b2bcSKevin Wolf 7041cd0dbfcSJohn Snow 7051cd0dbfcSJohn SnowMsg = TypeVar('Msg', Dict[str, Any], List[Any], str) 7061cd0dbfcSJohn Snow 7071cd0dbfcSJohn Snowdef log(msg: Msg, 7081cd0dbfcSJohn Snow filters: Iterable[Callable[[Msg], Msg]] = (), 7091cd0dbfcSJohn Snow indent: Optional[int] = None) -> None: 7101cd0dbfcSJohn Snow """ 7111cd0dbfcSJohn Snow Logs either a string message or a JSON serializable message (like QMP). 7121cd0dbfcSJohn Snow If indent is provided, JSON serializable messages are pretty-printed. 7131cd0dbfcSJohn Snow """ 714a2d1c8fdSDaniel P. Berrange for flt in filters: 715a2d1c8fdSDaniel P. Berrange msg = flt(msg) 7166a96d87cSJohn Snow if isinstance(msg, (dict, list)): 7170706e87dSJohn Snow # Don't sort if it's already sorted 7180706e87dSJohn Snow do_sort = not isinstance(msg, OrderedDict) 71952ea799eSJohn Snow test_logger.info(json.dumps(msg, sort_keys=do_sort, indent=indent)) 720e21b5f34SMax Reitz else: 72152ea799eSJohn Snow test_logger.info(msg) 722a2d1c8fdSDaniel P. Berrange 7232c93c5cbSKevin Wolfclass Timeout: 7242c93c5cbSKevin Wolf def __init__(self, seconds, errmsg="Timeout"): 7252c93c5cbSKevin Wolf self.seconds = seconds 7262c93c5cbSKevin Wolf self.errmsg = errmsg 7272c93c5cbSKevin Wolf def __enter__(self): 728d0c34326SEmanuele Giuseppe Esposito if qemu_gdb or qemu_valgrind: 729d3ec2022SEmanuele Giuseppe Esposito return self 7302c93c5cbSKevin Wolf signal.signal(signal.SIGALRM, self.timeout) 7312c93c5cbSKevin Wolf signal.setitimer(signal.ITIMER_REAL, self.seconds) 7322c93c5cbSKevin Wolf return self 7336a96d87cSJohn Snow def __exit__(self, exc_type, value, traceback): 734d0c34326SEmanuele Giuseppe Esposito if qemu_gdb or qemu_valgrind: 735d3ec2022SEmanuele Giuseppe Esposito return False 7362c93c5cbSKevin Wolf signal.setitimer(signal.ITIMER_REAL, 0) 7372c93c5cbSKevin Wolf return False 7382c93c5cbSKevin Wolf def timeout(self, signum, frame): 739aef633e7SJohn Snow raise TimeoutError(self.errmsg) 7402c93c5cbSKevin Wolf 741de263986SJohn Snowdef file_pattern(name): 742de263986SJohn Snow return "{0}-{1}".format(os.getpid(), name) 743f4844ac0SStefan Hajnoczi 7443192fad7SNir Sofferclass FilePath: 745de263986SJohn Snow """ 746f765af87SNir Soffer Context manager generating multiple file names. The generated files are 747f765af87SNir Soffer removed when exiting the context. 748f4844ac0SStefan Hajnoczi 749f765af87SNir Soffer Example usage: 750f4844ac0SStefan Hajnoczi 7513192fad7SNir Soffer with FilePath('a.img', 'b.img') as (img_a, img_b): 752f765af87SNir Soffer # Use img_a and img_b here... 753f765af87SNir Soffer 754f765af87SNir Soffer # a.img and b.img are automatically removed here. 755f765af87SNir Soffer 756f765af87SNir Soffer By default images are created in iotests.test_dir. To create sockets use 757f765af87SNir Soffer iotests.sock_dir: 758f765af87SNir Soffer 7593192fad7SNir Soffer with FilePath('a.sock', base_dir=iotests.sock_dir) as sock: 7603192fad7SNir Soffer 7613192fad7SNir Soffer For convenience, calling with one argument yields a single file instead of 7623192fad7SNir Soffer a tuple with one item. 763f765af87SNir Soffer 764de263986SJohn Snow """ 765a242b19eSNir Soffer def __init__(self, *names, base_dir=test_dir): 7667cc002a0SNir Soffer self.paths = [os.path.join(base_dir, file_pattern(name)) 7677cc002a0SNir Soffer for name in names] 768f4844ac0SStefan Hajnoczi 769f4844ac0SStefan Hajnoczi def __enter__(self): 7703192fad7SNir Soffer if len(self.paths) == 1: 7713192fad7SNir Soffer return self.paths[0] 7723192fad7SNir Soffer else: 773de263986SJohn Snow return self.paths 774f4844ac0SStefan Hajnoczi 775f4844ac0SStefan Hajnoczi def __exit__(self, exc_type, exc_val, exc_tb): 776de263986SJohn Snow for path in self.paths: 777a7971702SNir Soffer try: 778de263986SJohn Snow os.remove(path) 779f4844ac0SStefan Hajnoczi except OSError: 780f4844ac0SStefan Hajnoczi pass 781f4844ac0SStefan Hajnoczi return False 782f4844ac0SStefan Hajnoczi 783f4844ac0SStefan Hajnoczi 784c5ff5a3cSMax Reitzdef try_remove(img): 785ef6e9228SVladimir Sementsov-Ogievskiy try: 786c5ff5a3cSMax Reitz os.remove(img) 787ef6e9228SVladimir Sementsov-Ogievskiy except OSError: 788ef6e9228SVladimir Sementsov-Ogievskiy pass 789ef6e9228SVladimir Sementsov-Ogievskiy 790c5ff5a3cSMax Reitzdef file_path_remover(): 791c5ff5a3cSMax Reitz for path in reversed(file_path_remover.paths): 792c5ff5a3cSMax Reitz try_remove(path) 793c5ff5a3cSMax Reitz 794ef6e9228SVladimir Sementsov-Ogievskiy 79593b78ea5SMax Reitzdef file_path(*names, base_dir=test_dir): 796ef6e9228SVladimir Sementsov-Ogievskiy ''' Another way to get auto-generated filename that cleans itself up. 797ef6e9228SVladimir Sementsov-Ogievskiy 798ef6e9228SVladimir Sementsov-Ogievskiy Use is as simple as: 799ef6e9228SVladimir Sementsov-Ogievskiy 800ef6e9228SVladimir Sementsov-Ogievskiy img_a, img_b = file_path('a.img', 'b.img') 801ef6e9228SVladimir Sementsov-Ogievskiy sock = file_path('socket') 802ef6e9228SVladimir Sementsov-Ogievskiy ''' 803ef6e9228SVladimir Sementsov-Ogievskiy 804ef6e9228SVladimir Sementsov-Ogievskiy if not hasattr(file_path_remover, 'paths'): 805ef6e9228SVladimir Sementsov-Ogievskiy file_path_remover.paths = [] 806ef6e9228SVladimir Sementsov-Ogievskiy atexit.register(file_path_remover) 807ef6e9228SVladimir Sementsov-Ogievskiy 808ef6e9228SVladimir Sementsov-Ogievskiy paths = [] 809ef6e9228SVladimir Sementsov-Ogievskiy for name in names: 810de263986SJohn Snow filename = file_pattern(name) 81193b78ea5SMax Reitz path = os.path.join(base_dir, filename) 812ef6e9228SVladimir Sementsov-Ogievskiy file_path_remover.paths.append(path) 813ef6e9228SVladimir Sementsov-Ogievskiy paths.append(path) 814ef6e9228SVladimir Sementsov-Ogievskiy 815ef6e9228SVladimir Sementsov-Ogievskiy return paths[0] if len(paths) == 1 else paths 816ef6e9228SVladimir Sementsov-Ogievskiy 8175a259e86SKevin Wolfdef remote_filename(path): 8185a259e86SKevin Wolf if imgproto == 'file': 8195a259e86SKevin Wolf return path 8205a259e86SKevin Wolf elif imgproto == 'ssh': 821b8c1f901SMax Reitz return "ssh://%s@127.0.0.1:22%s" % (os.environ.get('USER'), path) 8225a259e86SKevin Wolf else: 823aef633e7SJohn Snow raise ValueError("Protocol %s not supported" % (imgproto)) 824ef6e9228SVladimir Sementsov-Ogievskiy 8254c44b4a4SDaniel P. Berrangeclass VM(qtest.QEMUQtestMachine): 826f345cfd0SStefan Hajnoczi '''A QEMU VM''' 827f345cfd0SStefan Hajnoczi 8285fcbdf50SMax Reitz def __init__(self, path_suffix=''): 8295fcbdf50SMax Reitz name = "qemu%s-%d" % (path_suffix, os.getpid()) 830d0c34326SEmanuele Giuseppe Esposito timer = 15.0 if not (qemu_gdb or qemu_valgrind) else None 8314032d1f6SEmanuele Giuseppe Esposito if qemu_gdb and qemu_valgrind: 8324032d1f6SEmanuele Giuseppe Esposito sys.stderr.write('gdb and valgrind are mutually exclusive\n') 8334032d1f6SEmanuele Giuseppe Esposito sys.exit(1) 8344032d1f6SEmanuele Giuseppe Esposito wrapper = qemu_gdb if qemu_gdb else qemu_valgrind 8354032d1f6SEmanuele Giuseppe Esposito super().__init__(qemu_prog, qemu_opts, wrapper=wrapper, 836776b9974SEmanuele Giuseppe Esposito name=name, 8372ca6e26cSCleber Rosa base_temp_dir=test_dir, 83846d4747aSJohn Snow qmp_timer=timer) 839f345cfd0SStefan Hajnoczi self._num_drives = 0 84030b005d9SWenchao Xia 841d792c863SEmanuele Giuseppe Esposito def _post_shutdown(self) -> None: 842d792c863SEmanuele Giuseppe Esposito super()._post_shutdown() 843d792c863SEmanuele Giuseppe Esposito if not qemu_valgrind or not self._popen: 844d792c863SEmanuele Giuseppe Esposito return 845d792c863SEmanuele Giuseppe Esposito valgrind_filename = f"{test_dir}/{self._popen.pid}.valgrind" 846d792c863SEmanuele Giuseppe Esposito if self.exitcode() == 99: 84781dcb9caSHanna Reitz with open(valgrind_filename, encoding='utf-8') as f: 848d792c863SEmanuele Giuseppe Esposito print(f.read()) 849d792c863SEmanuele Giuseppe Esposito else: 850d792c863SEmanuele Giuseppe Esposito os.remove(valgrind_filename) 851d792c863SEmanuele Giuseppe Esposito 852eb7a91d0SEmanuele Giuseppe Esposito def _pre_launch(self) -> None: 853eb7a91d0SEmanuele Giuseppe Esposito super()._pre_launch() 854eb7a91d0SEmanuele Giuseppe Esposito if qemu_print: 855eb7a91d0SEmanuele Giuseppe Esposito # set QEMU binary output to stdout 856eb7a91d0SEmanuele Giuseppe Esposito self._close_qemu_log_file() 857eb7a91d0SEmanuele Giuseppe Esposito 858ccc15f7dSStefan Hajnoczi def add_object(self, opts): 859ccc15f7dSStefan Hajnoczi self._args.append('-object') 860ccc15f7dSStefan Hajnoczi self._args.append(opts) 861ccc15f7dSStefan Hajnoczi return self 862ccc15f7dSStefan Hajnoczi 863486b88bdSKevin Wolf def add_device(self, opts): 864486b88bdSKevin Wolf self._args.append('-device') 865486b88bdSKevin Wolf self._args.append(opts) 866486b88bdSKevin Wolf return self 867486b88bdSKevin Wolf 86878b666f4SFam Zheng def add_drive_raw(self, opts): 86978b666f4SFam Zheng self._args.append('-drive') 87078b666f4SFam Zheng self._args.append(opts) 87178b666f4SFam Zheng return self 87278b666f4SFam Zheng 8731d3d4b63SJohn Snow def add_drive(self, path, opts='', interface='virtio', img_format=imgfmt): 874f345cfd0SStefan Hajnoczi '''Add a virtio-blk drive to the VM''' 8758e492253SMax Reitz options = ['if=%s' % interface, 876f345cfd0SStefan Hajnoczi 'id=drive%d' % self._num_drives] 8778e492253SMax Reitz 8788e492253SMax Reitz if path is not None: 8798e492253SMax Reitz options.append('file=%s' % path) 8801d3d4b63SJohn Snow options.append('format=%s' % img_format) 881fc17c259SKevin Wolf options.append('cache=%s' % cachemode) 8827156ca48SAarushi Mehta options.append('aio=%s' % aiomode) 8838e492253SMax Reitz 884f345cfd0SStefan Hajnoczi if opts: 885f345cfd0SStefan Hajnoczi options.append(opts) 886f345cfd0SStefan Hajnoczi 8871d3d4b63SJohn Snow if img_format == 'luks' and 'key-secret' not in opts: 88885a353a0SVladimir Sementsov-Ogievskiy # default luks support 88985a353a0SVladimir Sementsov-Ogievskiy if luks_default_secret_object not in self._args: 89085a353a0SVladimir Sementsov-Ogievskiy self.add_object(luks_default_secret_object) 89185a353a0SVladimir Sementsov-Ogievskiy 89285a353a0SVladimir Sementsov-Ogievskiy options.append(luks_default_key_secret_opt) 89385a353a0SVladimir Sementsov-Ogievskiy 894f345cfd0SStefan Hajnoczi self._args.append('-drive') 895f345cfd0SStefan Hajnoczi self._args.append(','.join(options)) 896f345cfd0SStefan Hajnoczi self._num_drives += 1 897f345cfd0SStefan Hajnoczi return self 898f345cfd0SStefan Hajnoczi 8995694923aSMax Reitz def add_blockdev(self, opts): 9005694923aSMax Reitz self._args.append('-blockdev') 9015694923aSMax Reitz if isinstance(opts, str): 9025694923aSMax Reitz self._args.append(opts) 9035694923aSMax Reitz else: 9045694923aSMax Reitz self._args.append(','.join(opts)) 9055694923aSMax Reitz return self 9065694923aSMax Reitz 90712314f2dSStefan Hajnoczi def add_incoming(self, addr): 90812314f2dSStefan Hajnoczi self._args.append('-incoming') 90912314f2dSStefan Hajnoczi self._args.append(addr) 91012314f2dSStefan Hajnoczi return self 91112314f2dSStefan Hajnoczi 9122012453dSJohn Snow def hmp(self, command_line: str, use_log: bool = False) -> QMPMessage: 913239bbcc0SJohn Snow cmd = 'human-monitor-command' 914090744d5SJohn Snow kwargs: Dict[str, Any] = {'command-line': command_line} 915239bbcc0SJohn Snow if use_log: 916239bbcc0SJohn Snow return self.qmp_log(cmd, **kwargs) 917239bbcc0SJohn Snow else: 918239bbcc0SJohn Snow return self.qmp(cmd, **kwargs) 919239bbcc0SJohn Snow 920239bbcc0SJohn Snow def pause_drive(self, drive: str, event: Optional[str] = None) -> None: 921239bbcc0SJohn Snow """Pause drive r/w operations""" 9223cf53c77SFam Zheng if not event: 9233cf53c77SFam Zheng self.pause_drive(drive, "read_aio") 9243cf53c77SFam Zheng self.pause_drive(drive, "write_aio") 9253cf53c77SFam Zheng return 926239bbcc0SJohn Snow self.hmp(f'qemu-io {drive} "break {event} bp_{drive}"') 9273cf53c77SFam Zheng 928239bbcc0SJohn Snow def resume_drive(self, drive: str) -> None: 929239bbcc0SJohn Snow """Resume drive r/w operations""" 930239bbcc0SJohn Snow self.hmp(f'qemu-io {drive} "remove_break bp_{drive}"') 9313cf53c77SFam Zheng 932239bbcc0SJohn Snow def hmp_qemu_io(self, drive: str, cmd: str, 933e89c0c8dSVladimir Sementsov-Ogievskiy use_log: bool = False, qdev: bool = False) -> QMPMessage: 934239bbcc0SJohn Snow """Write to a given drive using an HMP command""" 935e89c0c8dSVladimir Sementsov-Ogievskiy d = '-d ' if qdev else '' 936e89c0c8dSVladimir Sementsov-Ogievskiy return self.hmp(f'qemu-io {d}{drive} "{cmd}"', use_log=use_log) 937e3409362SIan Main 93862a94288SKevin Wolf def flatten_qmp_object(self, obj, output=None, basestr=''): 93962a94288SKevin Wolf if output is None: 940cc16153fSHanna Reitz output = {} 94162a94288SKevin Wolf if isinstance(obj, list): 9426a96d87cSJohn Snow for i, item in enumerate(obj): 9436a96d87cSJohn Snow self.flatten_qmp_object(item, output, basestr + str(i) + '.') 94462a94288SKevin Wolf elif isinstance(obj, dict): 94562a94288SKevin Wolf for key in obj: 94662a94288SKevin Wolf self.flatten_qmp_object(obj[key], output, basestr + key + '.') 94762a94288SKevin Wolf else: 94862a94288SKevin Wolf output[basestr[:-1]] = obj # Strip trailing '.' 94962a94288SKevin Wolf return output 95062a94288SKevin Wolf 95162a94288SKevin Wolf def qmp_to_opts(self, obj): 95262a94288SKevin Wolf obj = self.flatten_qmp_object(obj) 953cc16153fSHanna Reitz output_list = [] 95462a94288SKevin Wolf for key in obj: 95562a94288SKevin Wolf output_list += [key + '=' + obj[key]] 95662a94288SKevin Wolf return ','.join(output_list) 95762a94288SKevin Wolf 9588b6f5f8bSJohn Snow def get_qmp_events_filtered(self, wait=60.0): 9595ad1dbf7SKevin Wolf result = [] 9605ad1dbf7SKevin Wolf for ev in self.get_qmp_events(wait=wait): 9615ad1dbf7SKevin Wolf result.append(filter_qmp_event(ev)) 9625ad1dbf7SKevin Wolf return result 96362a94288SKevin Wolf 9644eabe051SJohn Snow def qmp_log(self, cmd, filters=(), indent=None, **kwargs): 9650706e87dSJohn Snow full_cmd = OrderedDict(( 9660706e87dSJohn Snow ("execute", cmd), 967039be85cSJohn Snow ("arguments", ordered_qmp(kwargs)) 9680706e87dSJohn Snow )) 96955cd64eaSJohn Snow log(full_cmd, filters, indent=indent) 970e234398aSKevin Wolf result = self.qmp(cmd, **kwargs) 97155cd64eaSJohn Snow log(result, filters, indent=indent) 972e234398aSKevin Wolf return result 973e234398aSKevin Wolf 9746a4e88e1SMax Reitz # Returns None on success, and an error string on failure 975da9d88d8SHanna Reitz def run_job(self, job: str, auto_finalize: bool = True, 976da9d88d8SHanna Reitz auto_dismiss: bool = False, 977da9d88d8SHanna Reitz pre_finalize: Optional[Callable[[], None]] = None, 978da9d88d8SHanna Reitz cancel: bool = False, wait: float = 60.0, 979da9d88d8SHanna Reitz filters: Iterable[Callable[[Any], Any]] = (), 980da9d88d8SHanna Reitz ) -> Optional[str]: 981d443b74bSJohn Snow """ 982d443b74bSJohn Snow run_job moves a job from creation through to dismissal. 983d443b74bSJohn Snow 984d443b74bSJohn Snow :param job: String. ID of recently-launched job 985d443b74bSJohn Snow :param auto_finalize: Bool. True if the job was launched with 986d443b74bSJohn Snow auto_finalize. Defaults to True. 987d443b74bSJohn Snow :param auto_dismiss: Bool. True if the job was launched with 988d443b74bSJohn Snow auto_dismiss=True. Defaults to False. 989d443b74bSJohn Snow :param pre_finalize: Callback. A callable that takes no arguments to be 990d443b74bSJohn Snow invoked prior to issuing job-finalize, if any. 991d443b74bSJohn Snow :param cancel: Bool. When true, cancels the job after the pre_finalize 992d443b74bSJohn Snow callback. 993d443b74bSJohn Snow :param wait: Float. Timeout value specifying how long to wait for any 994d443b74bSJohn Snow event, in seconds. Defaults to 60.0. 995d443b74bSJohn Snow """ 996d6a79af0SJohn Snow match_device = {'data': {'device': job}} 997d6a79af0SJohn Snow match_id = {'data': {'id': job}} 998d6a79af0SJohn Snow events = [ 999d6a79af0SJohn Snow ('BLOCK_JOB_COMPLETED', match_device), 1000d6a79af0SJohn Snow ('BLOCK_JOB_CANCELLED', match_device), 1001d6a79af0SJohn Snow ('BLOCK_JOB_ERROR', match_device), 1002d6a79af0SJohn Snow ('BLOCK_JOB_READY', match_device), 1003d6a79af0SJohn Snow ('BLOCK_JOB_PENDING', match_id), 1004d6a79af0SJohn Snow ('JOB_STATUS_CHANGE', match_id) 1005d6a79af0SJohn Snow ] 10066a4e88e1SMax Reitz error = None 1007fc47d851SKevin Wolf while True: 100855824e09SKevin Wolf ev = filter_qmp_event(self.events_wait(events, timeout=wait)) 1009d6a79af0SJohn Snow if ev['event'] != 'JOB_STATUS_CHANGE': 1010da9d88d8SHanna Reitz log(ev, filters=filters) 1011d6a79af0SJohn Snow continue 1012fc47d851SKevin Wolf status = ev['data']['status'] 1013fc47d851SKevin Wolf if status == 'aborting': 1014fc47d851SKevin Wolf result = self.qmp('query-jobs') 1015fc47d851SKevin Wolf for j in result['return']: 1016fc47d851SKevin Wolf if j['id'] == job: 10176a4e88e1SMax Reitz error = j['error'] 1018da9d88d8SHanna Reitz log('Job failed: %s' % (j['error']), filters=filters) 10194688c4e3SKevin Wolf elif status == 'ready': 1020da9d88d8SHanna Reitz self.qmp_log('job-complete', id=job, filters=filters) 1021fc47d851SKevin Wolf elif status == 'pending' and not auto_finalize: 1022ac6fb43eSKevin Wolf if pre_finalize: 1023ac6fb43eSKevin Wolf pre_finalize() 102452ea799eSJohn Snow if cancel: 1025da9d88d8SHanna Reitz self.qmp_log('job-cancel', id=job, filters=filters) 102652ea799eSJohn Snow else: 1027da9d88d8SHanna Reitz self.qmp_log('job-finalize', id=job, filters=filters) 1028fc47d851SKevin Wolf elif status == 'concluded' and not auto_dismiss: 1029da9d88d8SHanna Reitz self.qmp_log('job-dismiss', id=job, filters=filters) 1030fc47d851SKevin Wolf elif status == 'null': 10316a4e88e1SMax Reitz return error 1032fc47d851SKevin Wolf 1033e9dbd1caSKevin Wolf # Returns None on success, and an error string on failure 1034e9dbd1caSKevin Wolf def blockdev_create(self, options, job_id='job0', filters=None): 1035e9dbd1caSKevin Wolf if filters is None: 1036e9dbd1caSKevin Wolf filters = [filter_qmp_testfiles] 1037e9dbd1caSKevin Wolf result = self.qmp_log('blockdev-create', filters=filters, 1038e9dbd1caSKevin Wolf job_id=job_id, options=options) 1039e9dbd1caSKevin Wolf 1040e9dbd1caSKevin Wolf if 'return' in result: 1041e9dbd1caSKevin Wolf assert result['return'] == {} 1042da9d88d8SHanna Reitz job_result = self.run_job(job_id, filters=filters) 1043e9dbd1caSKevin Wolf else: 1044e9dbd1caSKevin Wolf job_result = result['error'] 1045e9dbd1caSKevin Wolf 1046e9dbd1caSKevin Wolf log("") 1047e9dbd1caSKevin Wolf return job_result 1048e9dbd1caSKevin Wolf 1049980448f1SKevin Wolf def enable_migration_events(self, name): 1050980448f1SKevin Wolf log('Enabling migration QMP events on %s...' % name) 1051980448f1SKevin Wolf log(self.qmp('migrate-set-capabilities', capabilities=[ 1052980448f1SKevin Wolf { 1053980448f1SKevin Wolf 'capability': 'events', 1054980448f1SKevin Wolf 'state': True 1055980448f1SKevin Wolf } 1056980448f1SKevin Wolf ])) 1057980448f1SKevin Wolf 10584bf63c80SMax Reitz def wait_migration(self, expect_runstate: Optional[str]) -> bool: 1059980448f1SKevin Wolf while True: 1060980448f1SKevin Wolf event = self.event_wait('MIGRATION') 1061503c2b31SKevin Wolf # We use the default timeout, and with a timeout, event_wait() 1062503c2b31SKevin Wolf # never returns None 1063503c2b31SKevin Wolf assert event 1064503c2b31SKevin Wolf 1065980448f1SKevin Wolf log(event, filters=[filter_qmp_event]) 10664bf63c80SMax Reitz if event['data']['status'] in ('completed', 'failed'): 1067980448f1SKevin Wolf break 10684bf63c80SMax Reitz 10694bf63c80SMax Reitz if event['data']['status'] == 'completed': 10708da7969bSMax Reitz # The event may occur in finish-migrate, so wait for the expected 10718da7969bSMax Reitz # post-migration runstate 10724bf63c80SMax Reitz runstate = None 10734bf63c80SMax Reitz while runstate != expect_runstate: 10744bf63c80SMax Reitz runstate = self.qmp('query-status')['return']['status'] 10754bf63c80SMax Reitz return True 10764bf63c80SMax Reitz else: 10774bf63c80SMax Reitz return False 1078980448f1SKevin Wolf 1079ef7afd63SMax Reitz def node_info(self, node_name): 1080ef7afd63SMax Reitz nodes = self.qmp('query-named-block-nodes') 1081ef7afd63SMax Reitz for x in nodes['return']: 1082ef7afd63SMax Reitz if x['node-name'] == node_name: 1083ef7afd63SMax Reitz return x 1084ef7afd63SMax Reitz return None 1085ef7afd63SMax Reitz 10865c4343b8SVladimir Sementsov-Ogievskiy def query_bitmaps(self): 10875c4343b8SVladimir Sementsov-Ogievskiy res = self.qmp("query-named-block-nodes") 10885c4343b8SVladimir Sementsov-Ogievskiy return {device['node-name']: device['dirty-bitmaps'] 10895c4343b8SVladimir Sementsov-Ogievskiy for device in res['return'] if 'dirty-bitmaps' in device} 10905c4343b8SVladimir Sementsov-Ogievskiy 10915c4343b8SVladimir Sementsov-Ogievskiy def get_bitmap(self, node_name, bitmap_name, recording=None, bitmaps=None): 10925c4343b8SVladimir Sementsov-Ogievskiy """ 10935c4343b8SVladimir Sementsov-Ogievskiy get a specific bitmap from the object returned by query_bitmaps. 10945c4343b8SVladimir Sementsov-Ogievskiy :param recording: If specified, filter results by the specified value. 10955c4343b8SVladimir Sementsov-Ogievskiy :param bitmaps: If specified, use it instead of call query_bitmaps() 10965c4343b8SVladimir Sementsov-Ogievskiy """ 10975c4343b8SVladimir Sementsov-Ogievskiy if bitmaps is None: 10985c4343b8SVladimir Sementsov-Ogievskiy bitmaps = self.query_bitmaps() 10995c4343b8SVladimir Sementsov-Ogievskiy 11005c4343b8SVladimir Sementsov-Ogievskiy for bitmap in bitmaps[node_name]: 11015c4343b8SVladimir Sementsov-Ogievskiy if bitmap.get('name', '') == bitmap_name: 11026a96d87cSJohn Snow if recording is None or bitmap.get('recording') == recording: 11035c4343b8SVladimir Sementsov-Ogievskiy return bitmap 11045c4343b8SVladimir Sementsov-Ogievskiy return None 11055c4343b8SVladimir Sementsov-Ogievskiy 11065c4343b8SVladimir Sementsov-Ogievskiy def check_bitmap_status(self, node_name, bitmap_name, fields): 11075c4343b8SVladimir Sementsov-Ogievskiy ret = self.get_bitmap(node_name, bitmap_name) 11085c4343b8SVladimir Sementsov-Ogievskiy 11095c4343b8SVladimir Sementsov-Ogievskiy return fields.items() <= ret.items() 11105c4343b8SVladimir Sementsov-Ogievskiy 11116a3d0f1eSMax Reitz def assert_block_path(self, root, path, expected_node, graph=None): 11126a3d0f1eSMax Reitz """ 11136a3d0f1eSMax Reitz Check whether the node under the given path in the block graph 11146a3d0f1eSMax Reitz is @expected_node. 11156a3d0f1eSMax Reitz 11166a3d0f1eSMax Reitz @root is the node name of the node where the @path is rooted. 11176a3d0f1eSMax Reitz 11186a3d0f1eSMax Reitz @path is a string that consists of child names separated by 11196a3d0f1eSMax Reitz slashes. It must begin with a slash. 11206a3d0f1eSMax Reitz 11216a3d0f1eSMax Reitz Examples for @root + @path: 11226a3d0f1eSMax Reitz - root="qcow2-node", path="/backing/file" 11236a3d0f1eSMax Reitz - root="quorum-node", path="/children.2/file" 11246a3d0f1eSMax Reitz 11256a3d0f1eSMax Reitz Hypothetically, @path could be empty, in which case it would 11266a3d0f1eSMax Reitz point to @root. However, in practice this case is not useful 11276a3d0f1eSMax Reitz and hence not allowed. 11286a3d0f1eSMax Reitz 11296a3d0f1eSMax Reitz @expected_node may be None. (All elements of the path but the 11306a3d0f1eSMax Reitz leaf must still exist.) 11316a3d0f1eSMax Reitz 11326a3d0f1eSMax Reitz @graph may be None or the result of an x-debug-query-block-graph 11336a3d0f1eSMax Reitz call that has already been performed. 11346a3d0f1eSMax Reitz """ 11356a3d0f1eSMax Reitz if graph is None: 11366a3d0f1eSMax Reitz graph = self.qmp('x-debug-query-block-graph')['return'] 11376a3d0f1eSMax Reitz 11386a3d0f1eSMax Reitz iter_path = iter(path.split('/')) 11396a3d0f1eSMax Reitz 11406a3d0f1eSMax Reitz # Must start with a / 11416a3d0f1eSMax Reitz assert next(iter_path) == '' 11426a3d0f1eSMax Reitz 11436a3d0f1eSMax Reitz node = next((node for node in graph['nodes'] if node['name'] == root), 11446a3d0f1eSMax Reitz None) 11456a3d0f1eSMax Reitz 11466a3d0f1eSMax Reitz # An empty @path is not allowed, so the root node must be present 11476a3d0f1eSMax Reitz assert node is not None, 'Root node %s not found' % root 11486a3d0f1eSMax Reitz 11496a3d0f1eSMax Reitz for child_name in iter_path: 11506a3d0f1eSMax Reitz assert node is not None, 'Cannot follow path %s%s' % (root, path) 11516a3d0f1eSMax Reitz 11526a3d0f1eSMax Reitz try: 11536a96d87cSJohn Snow node_id = next(edge['child'] for edge in graph['edges'] 11546a96d87cSJohn Snow if (edge['parent'] == node['id'] and 11556a96d87cSJohn Snow edge['name'] == child_name)) 11566a3d0f1eSMax Reitz 11576a96d87cSJohn Snow node = next(node for node in graph['nodes'] 11586a3d0f1eSMax Reitz if node['id'] == node_id) 11596a96d87cSJohn Snow 11606a3d0f1eSMax Reitz except StopIteration: 11616a3d0f1eSMax Reitz node = None 11626a3d0f1eSMax Reitz 11636a3d0f1eSMax Reitz if node is None: 11646a3d0f1eSMax Reitz assert expected_node is None, \ 11656a3d0f1eSMax Reitz 'No node found under %s (but expected %s)' % \ 11666a3d0f1eSMax Reitz (path, expected_node) 11676a3d0f1eSMax Reitz else: 11686a3d0f1eSMax Reitz assert node['name'] == expected_node, \ 11696a3d0f1eSMax Reitz 'Found node %s under %s (but expected %s)' % \ 11706a3d0f1eSMax Reitz (node['name'], path, expected_node) 11717898f74eSJohn Snow 1172f345cfd0SStefan Hajnocziindex_re = re.compile(r'([^\[]+)\[([^\]]+)\]') 1173f345cfd0SStefan Hajnoczi 1174f345cfd0SStefan Hajnocziclass QMPTestCase(unittest.TestCase): 1175f345cfd0SStefan Hajnoczi '''Abstract base class for QMP test cases''' 1176f345cfd0SStefan Hajnoczi 11776a96d87cSJohn Snow def __init__(self, *args, **kwargs): 11786a96d87cSJohn Snow super().__init__(*args, **kwargs) 11796a96d87cSJohn Snow # Many users of this class set a VM property we rely on heavily 11806a96d87cSJohn Snow # in the methods below. 11816a96d87cSJohn Snow self.vm = None 11826a96d87cSJohn Snow 1183f345cfd0SStefan Hajnoczi def dictpath(self, d, path): 1184f345cfd0SStefan Hajnoczi '''Traverse a path in a nested dict''' 1185f345cfd0SStefan Hajnoczi for component in path.split('/'): 1186f345cfd0SStefan Hajnoczi m = index_re.match(component) 1187f345cfd0SStefan Hajnoczi if m: 1188f345cfd0SStefan Hajnoczi component, idx = m.groups() 1189f345cfd0SStefan Hajnoczi idx = int(idx) 1190f345cfd0SStefan Hajnoczi 1191f345cfd0SStefan Hajnoczi if not isinstance(d, dict) or component not in d: 1192b031e9a5SJohn Snow self.fail(f'failed path traversal for "{path}" in "{d}"') 1193f345cfd0SStefan Hajnoczi d = d[component] 1194f345cfd0SStefan Hajnoczi 1195f345cfd0SStefan Hajnoczi if m: 1196f345cfd0SStefan Hajnoczi if not isinstance(d, list): 1197b031e9a5SJohn Snow self.fail(f'path component "{component}" in "{path}" ' 1198b031e9a5SJohn Snow f'is not a list in "{d}"') 1199f345cfd0SStefan Hajnoczi try: 1200f345cfd0SStefan Hajnoczi d = d[idx] 1201f345cfd0SStefan Hajnoczi except IndexError: 1202b031e9a5SJohn Snow self.fail(f'invalid index "{idx}" in path "{path}" ' 1203b031e9a5SJohn Snow f'in "{d}"') 1204f345cfd0SStefan Hajnoczi return d 1205f345cfd0SStefan Hajnoczi 120690f0b711SPaolo Bonzini def assert_qmp_absent(self, d, path): 120790f0b711SPaolo Bonzini try: 120890f0b711SPaolo Bonzini result = self.dictpath(d, path) 120990f0b711SPaolo Bonzini except AssertionError: 121090f0b711SPaolo Bonzini return 121190f0b711SPaolo Bonzini self.fail('path "%s" has value "%s"' % (path, str(result))) 121290f0b711SPaolo Bonzini 1213f345cfd0SStefan Hajnoczi def assert_qmp(self, d, path, value): 1214a93a42bdSMax Reitz '''Assert that the value for a specific path in a QMP dict 1215a93a42bdSMax Reitz matches. When given a list of values, assert that any of 1216a93a42bdSMax Reitz them matches.''' 1217a93a42bdSMax Reitz 1218f345cfd0SStefan Hajnoczi result = self.dictpath(d, path) 1219a93a42bdSMax Reitz 1220a93a42bdSMax Reitz # [] makes no sense as a list of valid values, so treat it as 1221a93a42bdSMax Reitz # an actual single value. 1222a93a42bdSMax Reitz if isinstance(value, list) and value != []: 1223a93a42bdSMax Reitz for v in value: 1224a93a42bdSMax Reitz if result == v: 1225a93a42bdSMax Reitz return 1226a93a42bdSMax Reitz self.fail('no match for "%s" in %s' % (str(result), str(value))) 1227a93a42bdSMax Reitz else: 1228a93a42bdSMax Reitz self.assertEqual(result, value, 1229dbf231d7SVladimir Sementsov-Ogievskiy '"%s" is "%s", expected "%s"' 1230dbf231d7SVladimir Sementsov-Ogievskiy % (path, str(result), str(value))) 1231f345cfd0SStefan Hajnoczi 1232ecc1c88eSStefan Hajnoczi def assert_no_active_block_jobs(self): 1233ecc1c88eSStefan Hajnoczi result = self.vm.qmp('query-block-jobs') 1234ecc1c88eSStefan Hajnoczi self.assert_qmp(result, 'return', []) 1235ecc1c88eSStefan Hajnoczi 1236e71fc0baSFam Zheng def assert_has_block_node(self, node_name=None, file_name=None): 1237e71fc0baSFam Zheng """Issue a query-named-block-nodes and assert node_name and/or 1238e71fc0baSFam Zheng file_name is present in the result""" 1239e71fc0baSFam Zheng def check_equal_or_none(a, b): 12406a96d87cSJohn Snow return a is None or b is None or a == b 1241e71fc0baSFam Zheng assert node_name or file_name 1242e71fc0baSFam Zheng result = self.vm.qmp('query-named-block-nodes') 1243e71fc0baSFam Zheng for x in result["return"]: 1244e71fc0baSFam Zheng if check_equal_or_none(x.get("node-name"), node_name) and \ 1245e71fc0baSFam Zheng check_equal_or_none(x.get("file"), file_name): 1246e71fc0baSFam Zheng return 12476a96d87cSJohn Snow self.fail("Cannot find %s %s in result:\n%s" % 1248e71fc0baSFam Zheng (node_name, file_name, result)) 1249e71fc0baSFam Zheng 1250e07375f5SMax Reitz def assert_json_filename_equal(self, json_filename, reference): 1251e07375f5SMax Reitz '''Asserts that the given filename is a json: filename and that its 1252e07375f5SMax Reitz content is equal to the given reference object''' 1253e07375f5SMax Reitz self.assertEqual(json_filename[:5], 'json:') 1254b031e9a5SJohn Snow self.assertEqual( 1255b031e9a5SJohn Snow self.vm.flatten_qmp_object(json.loads(json_filename[5:])), 1256b031e9a5SJohn Snow self.vm.flatten_qmp_object(reference) 1257b031e9a5SJohn Snow ) 1258e07375f5SMax Reitz 1259b031e9a5SJohn Snow def cancel_and_wait(self, drive='drive0', force=False, 1260b031e9a5SJohn Snow resume=False, wait=60.0): 12612575fe16SStefan Hajnoczi '''Cancel a block job and wait for it to finish, returning the event''' 1262b6aed193SVladimir Sementsov-Ogievskiy self.vm.cmd('block-job-cancel', device=drive, force=force) 12632575fe16SStefan Hajnoczi 12643cf53c77SFam Zheng if resume: 12653cf53c77SFam Zheng self.vm.resume_drive(drive) 12663cf53c77SFam Zheng 12672575fe16SStefan Hajnoczi cancelled = False 12682575fe16SStefan Hajnoczi result = None 12692575fe16SStefan Hajnoczi while not cancelled: 12708b6f5f8bSJohn Snow for event in self.vm.get_qmp_events(wait=wait): 12712575fe16SStefan Hajnoczi if event['event'] == 'BLOCK_JOB_COMPLETED' or \ 12722575fe16SStefan Hajnoczi event['event'] == 'BLOCK_JOB_CANCELLED': 12732575fe16SStefan Hajnoczi self.assert_qmp(event, 'data/device', drive) 12742575fe16SStefan Hajnoczi result = event 12752575fe16SStefan Hajnoczi cancelled = True 12761dac83f1SKevin Wolf elif event['event'] == 'JOB_STATUS_CHANGE': 12771dac83f1SKevin Wolf self.assert_qmp(event, 'data/id', drive) 12781dac83f1SKevin Wolf 12792575fe16SStefan Hajnoczi 12802575fe16SStefan Hajnoczi self.assert_no_active_block_jobs() 12812575fe16SStefan Hajnoczi return result 12822575fe16SStefan Hajnoczi 1283b031e9a5SJohn Snow def wait_until_completed(self, drive='drive0', check_offset=True, 1284b031e9a5SJohn Snow wait=60.0, error=None): 12850dbe8a1bSStefan Hajnoczi '''Wait for a block job to finish, returning the event''' 1286c3988519SPeter Xu while True: 12878b6f5f8bSJohn Snow for event in self.vm.get_qmp_events(wait=wait): 12880dbe8a1bSStefan Hajnoczi if event['event'] == 'BLOCK_JOB_COMPLETED': 12890dbe8a1bSStefan Hajnoczi self.assert_qmp(event, 'data/device', drive) 1290216656f5SMax Reitz if error is None: 12910dbe8a1bSStefan Hajnoczi self.assert_qmp_absent(event, 'data/error') 12929974ad40SFam Zheng if check_offset: 1293216656f5SMax Reitz self.assert_qmp(event, 'data/offset', 1294216656f5SMax Reitz event['data']['len']) 1295216656f5SMax Reitz else: 1296216656f5SMax Reitz self.assert_qmp(event, 'data/error', error) 12970dbe8a1bSStefan Hajnoczi self.assert_no_active_block_jobs() 12980dbe8a1bSStefan Hajnoczi return event 12996a96d87cSJohn Snow if event['event'] == 'JOB_STATUS_CHANGE': 13001dac83f1SKevin Wolf self.assert_qmp(event, 'data/id', drive) 13010dbe8a1bSStefan Hajnoczi 1302866323f3SFam Zheng def wait_ready(self, drive='drive0'): 13036a96d87cSJohn Snow """Wait until a BLOCK_JOB_READY event, and return the event.""" 1304c682bf18SMax Reitz return self.vm.events_wait([ 1305c682bf18SMax Reitz ('BLOCK_JOB_READY', 1306c682bf18SMax Reitz {'data': {'type': 'mirror', 'device': drive}}), 1307c682bf18SMax Reitz ('BLOCK_JOB_READY', 1308c682bf18SMax Reitz {'data': {'type': 'commit', 'device': drive}}) 1309c682bf18SMax Reitz ]) 1310866323f3SFam Zheng 1311866323f3SFam Zheng def wait_ready_and_cancel(self, drive='drive0'): 1312866323f3SFam Zheng self.wait_ready(drive=drive) 1313866323f3SFam Zheng event = self.cancel_and_wait(drive=drive) 1314fa1cfb40SKevin Wolf self.assertEqual(event['event'], 'BLOCK_JOB_COMPLETED') 1315866323f3SFam Zheng self.assert_qmp(event, 'data/type', 'mirror') 1316866323f3SFam Zheng self.assert_qmp(event, 'data/offset', event['data']['len']) 1317866323f3SFam Zheng 1318216656f5SMax Reitz def complete_and_wait(self, drive='drive0', wait_ready=True, 1319216656f5SMax Reitz completion_error=None): 1320866323f3SFam Zheng '''Complete a block job and wait for it to finish''' 1321866323f3SFam Zheng if wait_ready: 1322866323f3SFam Zheng self.wait_ready(drive=drive) 1323866323f3SFam Zheng 1324b6aed193SVladimir Sementsov-Ogievskiy self.vm.cmd('block-job-complete', device=drive) 1325866323f3SFam Zheng 1326216656f5SMax Reitz event = self.wait_until_completed(drive=drive, error=completion_error) 1327c682bf18SMax Reitz self.assertTrue(event['data']['type'] in ['mirror', 'commit']) 1328866323f3SFam Zheng 1329f03d9d24SJohn Snow def pause_wait(self, job_id='job0'): 1330e1df89bbSKevin Wolf with Timeout(3, "Timeout waiting for job to pause"): 13312c93c5cbSKevin Wolf while True: 13322c93c5cbSKevin Wolf result = self.vm.qmp('query-block-jobs') 1333c1bac161SVladimir Sementsov-Ogievskiy found = False 13342c93c5cbSKevin Wolf for job in result['return']: 1335c1bac161SVladimir Sementsov-Ogievskiy if job['device'] == job_id: 1336c1bac161SVladimir Sementsov-Ogievskiy found = True 13376a96d87cSJohn Snow if job['paused'] and not job['busy']: 13382c93c5cbSKevin Wolf return job 1339c1bac161SVladimir Sementsov-Ogievskiy break 1340c1bac161SVladimir Sementsov-Ogievskiy assert found 13412c93c5cbSKevin Wolf 1342f03d9d24SJohn Snow def pause_job(self, job_id='job0', wait=True): 1343b6aed193SVladimir Sementsov-Ogievskiy self.vm.cmd('block-job-pause', device=job_id) 1344f03d9d24SJohn Snow if wait: 134539995e21SVladimir Sementsov-Ogievskiy self.pause_wait(job_id) 1346f03d9d24SJohn Snow 13476be01225SMax Reitz def case_skip(self, reason): 13486be01225SMax Reitz '''Skip this test case''' 13496be01225SMax Reitz case_notrun(reason) 13506be01225SMax Reitz self.skipTest(reason) 13516be01225SMax Reitz 13522c93c5cbSKevin Wolf 1353f345cfd0SStefan Hajnoczidef notrun(reason): 1354f345cfd0SStefan Hajnoczi '''Skip this test suite''' 1355f345cfd0SStefan Hajnoczi # Each test in qemu-iotests has a number ("seq") 1356f345cfd0SStefan Hajnoczi seq = os.path.basename(sys.argv[0]) 1357f345cfd0SStefan Hajnoczi 13581a8fcca0SHanna Reitz with open('%s/%s.notrun' % (test_dir, seq), 'w', encoding='utf-8') \ 135981dcb9caSHanna Reitz as outfile: 136006aad78bSJohn Snow outfile.write(reason + '\n') 136152ea799eSJohn Snow logger.warning("%s not run: %s", seq, reason) 1362f345cfd0SStefan Hajnoczi sys.exit(0) 1363f345cfd0SStefan Hajnoczi 136457ed557fSAndrey Shinkevichdef case_notrun(reason): 13656be01225SMax Reitz '''Mark this test case as not having been run (without actually 13666be01225SMax Reitz skipping it, that is left to the caller). See 13676be01225SMax Reitz QMPTestCase.case_skip() for a variant that actually skips the 13686be01225SMax Reitz current test case.''' 13696be01225SMax Reitz 137057ed557fSAndrey Shinkevich # Each test in qemu-iotests has a number ("seq") 137157ed557fSAndrey Shinkevich seq = os.path.basename(sys.argv[0]) 137257ed557fSAndrey Shinkevich 13731a8fcca0SHanna Reitz with open('%s/%s.casenotrun' % (test_dir, seq), 'a', encoding='utf-8') \ 137481dcb9caSHanna Reitz as outfile: 137506aad78bSJohn Snow outfile.write(' [case not run] ' + reason + '\n') 137657ed557fSAndrey Shinkevich 137759c29869SJohn Snowdef _verify_image_format(supported_fmts: Sequence[str] = (), 137859c29869SJohn Snow unsupported_fmts: Sequence[str] = ()) -> None: 1379f48351d2SVladimir Sementsov-Ogievskiy if 'generic' in supported_fmts and \ 1380f48351d2SVladimir Sementsov-Ogievskiy os.environ.get('IMGFMT_GENERIC', 'true') == 'true': 1381f48351d2SVladimir Sementsov-Ogievskiy # similar to 1382f48351d2SVladimir Sementsov-Ogievskiy # _supported_fmt generic 1383f48351d2SVladimir Sementsov-Ogievskiy # for bash tests 1384f7125522SKevin Wolf supported_fmts = () 1385f48351d2SVladimir Sementsov-Ogievskiy 1386f48351d2SVladimir Sementsov-Ogievskiy not_sup = supported_fmts and (imgfmt not in supported_fmts) 1387f48351d2SVladimir Sementsov-Ogievskiy if not_sup or (imgfmt in unsupported_fmts): 13883f5c4076SDaniel P. Berrange notrun('not suitable for this image format: %s' % imgfmt) 1389c6a92369SDaniel P. Berrange 1390d2a839edSMax Reitz if imgfmt == 'luks': 1391d2a839edSMax Reitz verify_working_luks() 1392d2a839edSMax Reitz 139359c29869SJohn Snowdef _verify_protocol(supported: Sequence[str] = (), 139459c29869SJohn Snow unsupported: Sequence[str] = ()) -> None: 13955a259e86SKevin Wolf assert not (supported and unsupported) 13965a259e86SKevin Wolf 13975a259e86SKevin Wolf if 'generic' in supported: 13985a259e86SKevin Wolf return 13995a259e86SKevin Wolf 14005a259e86SKevin Wolf not_sup = supported and (imgproto not in supported) 14015a259e86SKevin Wolf if not_sup or (imgproto in unsupported): 14025a259e86SKevin Wolf notrun('not suitable for this protocol: %s' % imgproto) 14035a259e86SKevin Wolf 140459c29869SJohn Snowdef _verify_platform(supported: Sequence[str] = (), 140559c29869SJohn Snow unsupported: Sequence[str] = ()) -> None: 140672b29030SJohn Snow if any((sys.platform.startswith(x) for x in unsupported)): 140772b29030SJohn Snow notrun('not suitable for this OS: %s' % sys.platform) 140872b29030SJohn Snow 14097d814059SJohn Snow if supported: 141072b29030SJohn Snow if not any((sys.platform.startswith(x) for x in supported)): 1411c6a92369SDaniel P. Berrange notrun('not suitable for this OS: %s' % sys.platform) 1412c6a92369SDaniel P. Berrange 141359c29869SJohn Snowdef _verify_cache_mode(supported_cache_modes: Sequence[str] = ()) -> None: 1414ac8bd439SVladimir Sementsov-Ogievskiy if supported_cache_modes and (cachemode not in supported_cache_modes): 1415ac8bd439SVladimir Sementsov-Ogievskiy notrun('not suitable for this cache mode: %s' % cachemode) 1416ac8bd439SVladimir Sementsov-Ogievskiy 1417cd8f5b75SKevin Wolfdef _verify_aio_mode(supported_aio_modes: Sequence[str] = ()) -> None: 14187156ca48SAarushi Mehta if supported_aio_modes and (aiomode not in supported_aio_modes): 14197156ca48SAarushi Mehta notrun('not suitable for this aio mode: %s' % aiomode) 14207156ca48SAarushi Mehta 142118654716SVladimir Sementsov-Ogievskiydef _verify_formats(required_formats: Sequence[str] = ()) -> None: 142218654716SVladimir Sementsov-Ogievskiy usf_list = list(set(required_formats) - set(supported_formats())) 142318654716SVladimir Sementsov-Ogievskiy if usf_list: 142418654716SVladimir Sementsov-Ogievskiy notrun(f'formats {usf_list} are not whitelisted') 142518654716SVladimir Sementsov-Ogievskiy 1426f203080bSVladimir Sementsov-Ogievskiy 1427f203080bSVladimir Sementsov-Ogievskiydef _verify_virtio_blk() -> None: 1428f203080bSVladimir Sementsov-Ogievskiy out = qemu_pipe('-M', 'none', '-device', 'help') 1429f203080bSVladimir Sementsov-Ogievskiy if 'virtio-blk' not in out: 1430f203080bSVladimir Sementsov-Ogievskiy notrun('Missing virtio-blk in QEMU binary') 1431f203080bSVladimir Sementsov-Ogievskiy 1432bc05c638SKevin Wolfdef verify_virtio_scsi_pci_or_ccw() -> None: 1433359a8562SLaurent Vivier out = qemu_pipe('-M', 'none', '-device', 'help') 1434359a8562SLaurent Vivier if 'virtio-scsi-pci' not in out and 'virtio-scsi-ccw' not in out: 1435359a8562SLaurent Vivier notrun('Missing virtio-scsi-pci or virtio-scsi-ccw in QEMU binary') 1436359a8562SLaurent Vivier 1437f203080bSVladimir Sementsov-Ogievskiy 14387c15400cSVladimir Sementsov-Ogievskiydef _verify_imgopts(unsupported: Sequence[str] = ()) -> None: 14397c15400cSVladimir Sementsov-Ogievskiy imgopts = os.environ.get('IMGOPTS') 14407c15400cSVladimir Sementsov-Ogievskiy # One of usage examples for IMGOPTS is "data_file=$TEST_IMG.ext_data_file" 14417c15400cSVladimir Sementsov-Ogievskiy # but it supported only for bash tests. We don't have a concept of global 14427c15400cSVladimir Sementsov-Ogievskiy # TEST_IMG in iotests.py, not saying about somehow parsing $variables. 14437c15400cSVladimir Sementsov-Ogievskiy # So, for simplicity let's just not support any IMGOPTS with '$' inside. 14447c15400cSVladimir Sementsov-Ogievskiy unsup = list(unsupported) + ['$'] 14457c15400cSVladimir Sementsov-Ogievskiy if imgopts and any(x in imgopts for x in unsup): 14467c15400cSVladimir Sementsov-Ogievskiy notrun(f'not suitable for this imgopts: {imgopts}') 14477c15400cSVladimir Sementsov-Ogievskiy 14487c15400cSVladimir Sementsov-Ogievskiy 144978d04761SJohn Snowdef supports_quorum() -> bool: 145078d04761SJohn Snow return 'quorum' in qemu_img('--help').stdout 1451b0f90495SAlberto Garcia 14523f647b51SSascha Silbedef verify_quorum(): 14533f647b51SSascha Silbe '''Skip test suite if quorum support is not available''' 1454b0f90495SAlberto Garcia if not supports_quorum(): 14553f647b51SSascha Silbe notrun('quorum support missing') 14563f647b51SSascha Silbe 14576649f4bdSMax Reitzdef has_working_luks() -> Tuple[bool, str]: 14586649f4bdSMax Reitz """ 14596649f4bdSMax Reitz Check whether our LUKS driver can actually create images 14606649f4bdSMax Reitz (this extends to LUKS encryption for qcow2). 14616649f4bdSMax Reitz 14626649f4bdSMax Reitz If not, return the reason why. 14636649f4bdSMax Reitz """ 14646649f4bdSMax Reitz 14656649f4bdSMax Reitz img_file = f'{test_dir}/luks-test.luks' 146697576f8cSJohn Snow res = qemu_img('create', '-f', 'luks', 14676649f4bdSMax Reitz '--object', luks_default_secret_object, 14686649f4bdSMax Reitz '-o', luks_default_key_secret_opt, 14696649f4bdSMax Reitz '-o', 'iter-time=10', 147097576f8cSJohn Snow img_file, '1G', 147197576f8cSJohn Snow check=False) 14726649f4bdSMax Reitz try: 14736649f4bdSMax Reitz os.remove(img_file) 14746649f4bdSMax Reitz except OSError: 14756649f4bdSMax Reitz pass 14766649f4bdSMax Reitz 147797576f8cSJohn Snow if res.returncode: 147897576f8cSJohn Snow reason = res.stdout 147997576f8cSJohn Snow for line in res.stdout.splitlines(): 14806649f4bdSMax Reitz if img_file + ':' in line: 14816649f4bdSMax Reitz reason = line.split(img_file + ':', 1)[1].strip() 14826649f4bdSMax Reitz break 14836649f4bdSMax Reitz 14846649f4bdSMax Reitz return (False, reason) 14856649f4bdSMax Reitz else: 14866649f4bdSMax Reitz return (True, '') 14876649f4bdSMax Reitz 14886649f4bdSMax Reitzdef verify_working_luks(): 14896649f4bdSMax Reitz """ 14906649f4bdSMax Reitz Skip test suite if LUKS does not work 14916649f4bdSMax Reitz """ 14926649f4bdSMax Reitz (working, reason) = has_working_luks() 14936649f4bdSMax Reitz if not working: 14946649f4bdSMax Reitz notrun(reason) 14956649f4bdSMax Reitz 14969ba271f0SHanna Reitzdef supports_qcow2_zstd_compression() -> bool: 14979ba271f0SHanna Reitz img_file = f'{test_dir}/qcow2-zstd-test.qcow2' 14989ba271f0SHanna Reitz res = qemu_img('create', '-f', 'qcow2', '-o', 'compression_type=zstd', 14999ba271f0SHanna Reitz img_file, '0', 15009ba271f0SHanna Reitz check=False) 15019ba271f0SHanna Reitz try: 15029ba271f0SHanna Reitz os.remove(img_file) 15039ba271f0SHanna Reitz except OSError: 15049ba271f0SHanna Reitz pass 15059ba271f0SHanna Reitz 15069ba271f0SHanna Reitz if res.returncode == 1 and \ 15079ba271f0SHanna Reitz "'compression-type' does not accept value 'zstd'" in res.stdout: 15089ba271f0SHanna Reitz return False 15099ba271f0SHanna Reitz else: 15109ba271f0SHanna Reitz return True 15119ba271f0SHanna Reitz 15129ba271f0SHanna Reitzdef verify_qcow2_zstd_compression(): 15139ba271f0SHanna Reitz if not supports_qcow2_zstd_compression(): 15149ba271f0SHanna Reitz notrun('zstd compression not supported') 15159ba271f0SHanna Reitz 151691efbae9SKevin Wolfdef qemu_pipe(*args: str) -> str: 1517b031e9a5SJohn Snow """ 1518b031e9a5SJohn Snow Run qemu with an option to print something and exit (e.g. a help option). 1519b031e9a5SJohn Snow 1520b031e9a5SJohn Snow :return: QEMU's stdout output. 1521b031e9a5SJohn Snow """ 152291efbae9SKevin Wolf full_args = [qemu_prog] + qemu_opts + list(args) 152391efbae9SKevin Wolf output, _ = qemu_tool_pipe_and_status('qemu', full_args) 152449438972SMax Reitz return output 152557ed557fSAndrey Shinkevich 152657ed557fSAndrey Shinkevichdef supported_formats(read_only=False): 152757ed557fSAndrey Shinkevich '''Set 'read_only' to True to check ro-whitelist 152857ed557fSAndrey Shinkevich Otherwise, rw-whitelist is checked''' 1529767de537SMax Reitz 1530767de537SMax Reitz if not hasattr(supported_formats, "formats"): 1531767de537SMax Reitz supported_formats.formats = {} 1532767de537SMax Reitz 1533767de537SMax Reitz if read_only not in supported_formats.formats: 153457ed557fSAndrey Shinkevich format_message = qemu_pipe("-drive", "format=help") 153557ed557fSAndrey Shinkevich line = 1 if read_only else 0 1536767de537SMax Reitz supported_formats.formats[read_only] = \ 1537767de537SMax Reitz format_message.splitlines()[line].split(":")[1].split() 1538767de537SMax Reitz 1539767de537SMax Reitz return supported_formats.formats[read_only] 154057ed557fSAndrey Shinkevich 15414eabe051SJohn Snowdef skip_if_unsupported(required_formats=(), read_only=False): 154257ed557fSAndrey Shinkevich '''Skip Test Decorator 154357ed557fSAndrey Shinkevich Runs the test if all the required formats are whitelisted''' 154457ed557fSAndrey Shinkevich def skip_test_decorator(func): 1545cd8f5b75SKevin Wolf def func_wrapper(test_case: QMPTestCase, *args: List[Any], 1546cd8f5b75SKevin Wolf **kwargs: Dict[str, Any]) -> None: 15477448be83SMax Reitz if callable(required_formats): 15487448be83SMax Reitz fmts = required_formats(test_case) 15497448be83SMax Reitz else: 15507448be83SMax Reitz fmts = required_formats 15517448be83SMax Reitz 15527448be83SMax Reitz usf_list = list(set(fmts) - set(supported_formats(read_only))) 155357ed557fSAndrey Shinkevich if usf_list: 1554b031e9a5SJohn Snow msg = f'{test_case}: formats {usf_list} are not whitelisted' 1555b031e9a5SJohn Snow test_case.case_skip(msg) 155657ed557fSAndrey Shinkevich else: 1557cd8f5b75SKevin Wolf func(test_case, *args, **kwargs) 155857ed557fSAndrey Shinkevich return func_wrapper 155957ed557fSAndrey Shinkevich return skip_test_decorator 156057ed557fSAndrey Shinkevich 1561ff3caf5aSMax Reitzdef skip_for_formats(formats: Sequence[str] = ()) \ 1562ff3caf5aSMax Reitz -> Callable[[Callable[[QMPTestCase, List[Any], Dict[str, Any]], None]], 1563ff3caf5aSMax Reitz Callable[[QMPTestCase, List[Any], Dict[str, Any]], None]]: 1564ff3caf5aSMax Reitz '''Skip Test Decorator 1565ff3caf5aSMax Reitz Skips the test for the given formats''' 1566ff3caf5aSMax Reitz def skip_test_decorator(func): 1567ff3caf5aSMax Reitz def func_wrapper(test_case: QMPTestCase, *args: List[Any], 1568ff3caf5aSMax Reitz **kwargs: Dict[str, Any]) -> None: 1569ff3caf5aSMax Reitz if imgfmt in formats: 1570ff3caf5aSMax Reitz msg = f'{test_case}: Skipped for format {imgfmt}' 1571ff3caf5aSMax Reitz test_case.case_skip(msg) 1572ff3caf5aSMax Reitz else: 1573ff3caf5aSMax Reitz func(test_case, *args, **kwargs) 1574ff3caf5aSMax Reitz return func_wrapper 1575ff3caf5aSMax Reitz return skip_test_decorator 1576ff3caf5aSMax Reitz 1577d926f4ddSKevin Wolfdef skip_if_user_is_root(func): 1578d926f4ddSKevin Wolf '''Skip Test Decorator 1579d926f4ddSKevin Wolf Runs the test only without root permissions''' 1580d926f4ddSKevin Wolf def func_wrapper(*args, **kwargs): 1581d926f4ddSKevin Wolf if os.getuid() == 0: 1582d926f4ddSKevin Wolf case_notrun('{}: cannot be run as root'.format(args[0])) 15836a96d87cSJohn Snow return None 1584d926f4ddSKevin Wolf else: 1585d926f4ddSKevin Wolf return func(*args, **kwargs) 1586d926f4ddSKevin Wolf return func_wrapper 1587d926f4ddSKevin Wolf 1588f29f4c25SPaolo Bonzini# We need to filter out the time taken from the output so that 1589f29f4c25SPaolo Bonzini# qemu-iotest can reliably diff the results against master output, 1590f29f4c25SPaolo Bonzini# and hide skipped tests from the reference output. 1591f29f4c25SPaolo Bonzini 1592f29f4c25SPaolo Bonziniclass ReproducibleTestResult(unittest.TextTestResult): 1593f29f4c25SPaolo Bonzini def addSkip(self, test, reason): 1594f29f4c25SPaolo Bonzini # Same as TextTestResult, but print dot instead of "s" 1595f29f4c25SPaolo Bonzini unittest.TestResult.addSkip(self, test, reason) 1596f29f4c25SPaolo Bonzini if self.showAll: 1597f29f4c25SPaolo Bonzini self.stream.writeln("skipped {0!r}".format(reason)) 1598f29f4c25SPaolo Bonzini elif self.dots: 1599f29f4c25SPaolo Bonzini self.stream.write(".") 1600f29f4c25SPaolo Bonzini self.stream.flush() 1601f29f4c25SPaolo Bonzini 1602f29f4c25SPaolo Bonziniclass ReproducibleStreamWrapper: 1603f29f4c25SPaolo Bonzini def __init__(self, stream: TextIO): 1604f29f4c25SPaolo Bonzini self.stream = stream 1605f29f4c25SPaolo Bonzini 1606f29f4c25SPaolo Bonzini def __getattr__(self, attr): 1607f29f4c25SPaolo Bonzini if attr in ('stream', '__getstate__'): 1608f29f4c25SPaolo Bonzini raise AttributeError(attr) 1609f29f4c25SPaolo Bonzini return getattr(self.stream, attr) 1610f29f4c25SPaolo Bonzini 1611f29f4c25SPaolo Bonzini def write(self, arg=None): 1612f29f4c25SPaolo Bonzini arg = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', arg) 1613f29f4c25SPaolo Bonzini arg = re.sub(r' \(skipped=\d+\)', r'', arg) 1614f29f4c25SPaolo Bonzini self.stream.write(arg) 1615f29f4c25SPaolo Bonzini 1616f29f4c25SPaolo Bonziniclass ReproducibleTestRunner(unittest.TextTestRunner): 1617f29f4c25SPaolo Bonzini def __init__(self, stream: Optional[TextIO] = None, 161887e4d4a2SEmanuele Giuseppe Esposito resultclass: Type[unittest.TestResult] = 161987e4d4a2SEmanuele Giuseppe Esposito ReproducibleTestResult, 1620f29f4c25SPaolo Bonzini **kwargs: Any) -> None: 1621f29f4c25SPaolo Bonzini rstream = ReproducibleStreamWrapper(stream or sys.stdout) 1622f29f4c25SPaolo Bonzini super().__init__(stream=rstream, # type: ignore 1623f29f4c25SPaolo Bonzini descriptions=True, 1624f29f4c25SPaolo Bonzini resultclass=resultclass, 1625f29f4c25SPaolo Bonzini **kwargs) 1626f29f4c25SPaolo Bonzini 162700dbc85eSPaolo Bonzinidef execute_unittest(argv: List[str], debug: bool = False) -> None: 16287d814059SJohn Snow """Executes unittests within the calling module.""" 16297d814059SJohn Snow 163000dbc85eSPaolo Bonzini # Some tests have warnings, especially ResourceWarnings for unclosed 163100dbc85eSPaolo Bonzini # files and sockets. Ignore them for now to ensure reproducibility of 163200dbc85eSPaolo Bonzini # the test output. 163300dbc85eSPaolo Bonzini unittest.main(argv=argv, 163400dbc85eSPaolo Bonzini testRunner=ReproducibleTestRunner, 163500dbc85eSPaolo Bonzini verbosity=2 if debug else 1, 163600dbc85eSPaolo Bonzini warnings=None if sys.warnoptions else 'ignore') 1637f345cfd0SStefan Hajnoczi 16387d814059SJohn Snowdef execute_setup_common(supported_fmts: Sequence[str] = (), 16397d814059SJohn Snow supported_platforms: Sequence[str] = (), 16407d814059SJohn Snow supported_cache_modes: Sequence[str] = (), 16417d814059SJohn Snow supported_aio_modes: Sequence[str] = (), 16427d814059SJohn Snow unsupported_fmts: Sequence[str] = (), 16437d814059SJohn Snow supported_protocols: Sequence[str] = (), 164418654716SVladimir Sementsov-Ogievskiy unsupported_protocols: Sequence[str] = (), 16457c15400cSVladimir Sementsov-Ogievskiy required_fmts: Sequence[str] = (), 16467c15400cSVladimir Sementsov-Ogievskiy unsupported_imgopts: Sequence[str] = ()) -> bool: 16477d814059SJohn Snow """ 16487d814059SJohn Snow Perform necessary setup for either script-style or unittest-style tests. 16497d814059SJohn Snow 16507d814059SJohn Snow :return: Bool; Whether or not debug mode has been requested via the CLI. 16517d814059SJohn Snow """ 16527d814059SJohn Snow # Note: Python 3.6 and pylint do not like 'Collection' so use 'Sequence'. 1653c0088d79SKevin Wolf 165444a46a9cSJohn Snow debug = '-d' in sys.argv 165544a46a9cSJohn Snow if debug: 165644a46a9cSJohn Snow sys.argv.remove('-d') 165744a46a9cSJohn Snow logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) 165844a46a9cSJohn Snow 165959c29869SJohn Snow _verify_image_format(supported_fmts, unsupported_fmts) 166059c29869SJohn Snow _verify_protocol(supported_protocols, unsupported_protocols) 166159c29869SJohn Snow _verify_platform(supported=supported_platforms) 166259c29869SJohn Snow _verify_cache_mode(supported_cache_modes) 166359c29869SJohn Snow _verify_aio_mode(supported_aio_modes) 166418654716SVladimir Sementsov-Ogievskiy _verify_formats(required_fmts) 1665f203080bSVladimir Sementsov-Ogievskiy _verify_virtio_blk() 16667c15400cSVladimir Sementsov-Ogievskiy _verify_imgopts(unsupported_imgopts) 1667bc521696SFam Zheng 16687d814059SJohn Snow return debug 16697d814059SJohn Snow 16707d814059SJohn Snowdef execute_test(*args, test_function=None, **kwargs): 16717d814059SJohn Snow """Run either unittest or script-style tests.""" 16727d814059SJohn Snow 16737d814059SJohn Snow debug = execute_setup_common(*args, **kwargs) 1674456a2d5aSJohn Snow if not test_function: 167500dbc85eSPaolo Bonzini execute_unittest(sys.argv, debug) 1676456a2d5aSJohn Snow else: 1677456a2d5aSJohn Snow test_function() 1678f345cfd0SStefan Hajnoczi 167952ea799eSJohn Snowdef activate_logging(): 168052ea799eSJohn Snow """Activate iotests.log() output to stdout for script-style tests.""" 168152ea799eSJohn Snow handler = logging.StreamHandler(stream=sys.stdout) 168252ea799eSJohn Snow formatter = logging.Formatter('%(message)s') 168352ea799eSJohn Snow handler.setFormatter(formatter) 168452ea799eSJohn Snow test_logger.addHandler(handler) 168552ea799eSJohn Snow test_logger.setLevel(logging.INFO) 168652ea799eSJohn Snow test_logger.propagate = False 168752ea799eSJohn Snow 16887d814059SJohn Snow# This is called from script-style iotests without a single point of entry 16897d814059SJohn Snowdef script_initialize(*args, **kwargs): 16907d814059SJohn Snow """Initialize script-style tests without running any tests.""" 169152ea799eSJohn Snow activate_logging() 16927d814059SJohn Snow execute_setup_common(*args, **kwargs) 16937d814059SJohn Snow 16947d814059SJohn Snow# This is called from script-style iotests with a single point of entry 1695456a2d5aSJohn Snowdef script_main(test_function, *args, **kwargs): 1696456a2d5aSJohn Snow """Run script-style tests outside of the unittest framework""" 169752ea799eSJohn Snow activate_logging() 16987d814059SJohn Snow execute_test(*args, test_function=test_function, **kwargs) 1699456a2d5aSJohn Snow 17007d814059SJohn Snow# This is called from unittest style iotests 1701456a2d5aSJohn Snowdef main(*args, **kwargs): 1702456a2d5aSJohn Snow """Run tests using the unittest framework""" 17037d814059SJohn Snow execute_test(*args, **kwargs) 1704