1from __future__ import print_function 2# Common utilities and Python wrappers for qemu-iotests 3# 4# Copyright (C) 2012 IBM Corp. 5# 6# This program is free software; you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation; either version 2 of the License, or 9# (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program. If not, see <http://www.gnu.org/licenses/>. 18# 19 20import errno 21import os 22import re 23import subprocess 24import string 25import unittest 26import sys 27import struct 28import json 29import signal 30import logging 31import atexit 32import io 33from collections import OrderedDict 34 35sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 36from qemu import qtest 37 38assert sys.version_info >= (3,6) 39 40# This will not work if arguments contain spaces but is necessary if we 41# want to support the override options that ./check supports. 42qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')] 43if os.environ.get('QEMU_IMG_OPTIONS'): 44 qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ') 45 46qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 47if os.environ.get('QEMU_IO_OPTIONS'): 48 qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ') 49 50qemu_io_args_no_fmt = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 51if os.environ.get('QEMU_IO_OPTIONS_NO_FMT'): 52 qemu_io_args_no_fmt += \ 53 os.environ['QEMU_IO_OPTIONS_NO_FMT'].strip().split(' ') 54 55qemu_nbd_args = [os.environ.get('QEMU_NBD_PROG', 'qemu-nbd')] 56if os.environ.get('QEMU_NBD_OPTIONS'): 57 qemu_nbd_args += os.environ['QEMU_NBD_OPTIONS'].strip().split(' ') 58 59qemu_prog = os.environ.get('QEMU_PROG', 'qemu') 60qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ') 61 62imgfmt = os.environ.get('IMGFMT', 'raw') 63imgproto = os.environ.get('IMGPROTO', 'file') 64test_dir = os.environ.get('TEST_DIR') 65sock_dir = os.environ.get('SOCK_DIR') 66output_dir = os.environ.get('OUTPUT_DIR', '.') 67cachemode = os.environ.get('CACHEMODE') 68qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') 69 70socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') 71 72luks_default_secret_object = 'secret,id=keysec0,data=' + \ 73 os.environ.get('IMGKEYSECRET', '') 74luks_default_key_secret_opt = 'key-secret=keysec0' 75 76 77def qemu_img(*args): 78 '''Run qemu-img and return the exit code''' 79 devnull = open('/dev/null', 'r+') 80 exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) 81 if exitcode < 0: 82 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 83 return exitcode 84 85def ordered_qmp(qmsg, conv_keys=True): 86 # Dictionaries are not ordered prior to 3.6, therefore: 87 if isinstance(qmsg, list): 88 return [ordered_qmp(atom) for atom in qmsg] 89 if isinstance(qmsg, dict): 90 od = OrderedDict() 91 for k, v in sorted(qmsg.items()): 92 if conv_keys: 93 k = k.replace('_', '-') 94 od[k] = ordered_qmp(v, conv_keys=False) 95 return od 96 return qmsg 97 98def qemu_img_create(*args): 99 args = list(args) 100 101 # default luks support 102 if '-f' in args and args[args.index('-f') + 1] == 'luks': 103 if '-o' in args: 104 i = args.index('-o') 105 if 'key-secret' not in args[i + 1]: 106 args[i + 1].append(luks_default_key_secret_opt) 107 args.insert(i + 2, '--object') 108 args.insert(i + 3, luks_default_secret_object) 109 else: 110 args = ['-o', luks_default_key_secret_opt, 111 '--object', luks_default_secret_object] + args 112 113 args.insert(0, 'create') 114 115 return qemu_img(*args) 116 117def qemu_img_verbose(*args): 118 '''Run qemu-img without suppressing its output and return the exit code''' 119 exitcode = subprocess.call(qemu_img_args + list(args)) 120 if exitcode < 0: 121 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 122 return exitcode 123 124def qemu_img_pipe(*args): 125 '''Run qemu-img and return its output''' 126 subp = subprocess.Popen(qemu_img_args + list(args), 127 stdout=subprocess.PIPE, 128 stderr=subprocess.STDOUT, 129 universal_newlines=True) 130 exitcode = subp.wait() 131 if exitcode < 0: 132 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 133 return subp.communicate()[0] 134 135def qemu_img_log(*args): 136 result = qemu_img_pipe(*args) 137 log(result, filters=[filter_testfiles]) 138 return result 139 140def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]): 141 args = [ 'info' ] 142 if imgopts: 143 args.append('--image-opts') 144 else: 145 args += [ '-f', imgfmt ] 146 args += extra_args 147 args.append(filename) 148 149 output = qemu_img_pipe(*args) 150 if not filter_path: 151 filter_path = filename 152 log(filter_img_info(output, filter_path)) 153 154def qemu_io(*args): 155 '''Run qemu-io and return the stdout data''' 156 args = qemu_io_args + list(args) 157 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 158 stderr=subprocess.STDOUT, 159 universal_newlines=True) 160 exitcode = subp.wait() 161 if exitcode < 0: 162 sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args))) 163 return subp.communicate()[0] 164 165def qemu_io_log(*args): 166 result = qemu_io(*args) 167 log(result, filters=[filter_testfiles, filter_qemu_io]) 168 return result 169 170def qemu_io_silent(*args): 171 '''Run qemu-io and return the exit code, suppressing stdout''' 172 args = qemu_io_args + list(args) 173 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w')) 174 if exitcode < 0: 175 sys.stderr.write('qemu-io received signal %i: %s\n' % 176 (-exitcode, ' '.join(args))) 177 return exitcode 178 179def qemu_io_silent_check(*args): 180 '''Run qemu-io and return the true if subprocess returned 0''' 181 args = qemu_io_args + list(args) 182 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w'), 183 stderr=subprocess.STDOUT) 184 return exitcode == 0 185 186def get_virtio_scsi_device(): 187 if qemu_default_machine == 's390-ccw-virtio': 188 return 'virtio-scsi-ccw' 189 return 'virtio-scsi-pci' 190 191class QemuIoInteractive: 192 def __init__(self, *args): 193 self.args = qemu_io_args + list(args) 194 self._p = subprocess.Popen(self.args, stdin=subprocess.PIPE, 195 stdout=subprocess.PIPE, 196 stderr=subprocess.STDOUT, 197 universal_newlines=True) 198 assert self._p.stdout.read(9) == 'qemu-io> ' 199 200 def close(self): 201 self._p.communicate('q\n') 202 203 def _read_output(self): 204 pattern = 'qemu-io> ' 205 n = len(pattern) 206 pos = 0 207 s = [] 208 while pos != n: 209 c = self._p.stdout.read(1) 210 # check unexpected EOF 211 assert c != '' 212 s.append(c) 213 if c == pattern[pos]: 214 pos += 1 215 else: 216 pos = 0 217 218 return ''.join(s[:-n]) 219 220 def cmd(self, cmd): 221 # quit command is in close(), '\n' is added automatically 222 assert '\n' not in cmd 223 cmd = cmd.strip() 224 assert cmd != 'q' and cmd != 'quit' 225 self._p.stdin.write(cmd + '\n') 226 self._p.stdin.flush() 227 return self._read_output() 228 229 230def qemu_nbd(*args): 231 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 232 return subprocess.call(qemu_nbd_args + ['--fork'] + list(args)) 233 234def qemu_nbd_early_pipe(*args): 235 '''Run qemu-nbd in daemon mode and return both the parent's exit code 236 and its output in case of an error''' 237 subp = subprocess.Popen(qemu_nbd_args + ['--fork'] + list(args), 238 stdout=subprocess.PIPE, 239 stderr=subprocess.STDOUT, 240 universal_newlines=True) 241 exitcode = subp.wait() 242 if exitcode < 0: 243 sys.stderr.write('qemu-nbd received signal %i: %s\n' % 244 (-exitcode, 245 ' '.join(qemu_nbd_args + ['--fork'] + list(args)))) 246 if exitcode == 0: 247 return exitcode, '' 248 else: 249 return exitcode, subp.communicate()[0] 250 251def qemu_nbd_popen(*args): 252 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 253 return subprocess.Popen(qemu_nbd_args + ['--persistent'] + list(args)) 254 255def compare_images(img1, img2, fmt1=imgfmt, fmt2=imgfmt): 256 '''Return True if two image files are identical''' 257 return qemu_img('compare', '-f', fmt1, 258 '-F', fmt2, img1, img2) == 0 259 260def create_image(name, size): 261 '''Create a fully-allocated raw image with sector markers''' 262 file = open(name, 'wb') 263 i = 0 264 while i < size: 265 sector = struct.pack('>l504xl', i // 512, i // 512) 266 file.write(sector) 267 i = i + 512 268 file.close() 269 270def image_size(img): 271 '''Return image's virtual size''' 272 r = qemu_img_pipe('info', '--output=json', '-f', imgfmt, img) 273 return json.loads(r)['virtual-size'] 274 275def is_str(val): 276 return isinstance(val, str) 277 278test_dir_re = re.compile(r"%s" % test_dir) 279def filter_test_dir(msg): 280 return test_dir_re.sub("TEST_DIR", msg) 281 282win32_re = re.compile(r"\r") 283def filter_win32(msg): 284 return win32_re.sub("", msg) 285 286qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)") 287def filter_qemu_io(msg): 288 msg = filter_win32(msg) 289 return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg) 290 291chown_re = re.compile(r"chown [0-9]+:[0-9]+") 292def filter_chown(msg): 293 return chown_re.sub("chown UID:GID", msg) 294 295def filter_qmp_event(event): 296 '''Filter a QMP event dict''' 297 event = dict(event) 298 if 'timestamp' in event: 299 event['timestamp']['seconds'] = 'SECS' 300 event['timestamp']['microseconds'] = 'USECS' 301 return event 302 303def filter_qmp(qmsg, filter_fn): 304 '''Given a string filter, filter a QMP object's values. 305 filter_fn takes a (key, value) pair.''' 306 # Iterate through either lists or dicts; 307 if isinstance(qmsg, list): 308 items = enumerate(qmsg) 309 else: 310 items = qmsg.items() 311 312 for k, v in items: 313 if isinstance(v, list) or isinstance(v, dict): 314 qmsg[k] = filter_qmp(v, filter_fn) 315 else: 316 qmsg[k] = filter_fn(k, v) 317 return qmsg 318 319def filter_testfiles(msg): 320 prefix = os.path.join(test_dir, "%s-" % (os.getpid())) 321 return msg.replace(prefix, 'TEST_DIR/PID-') 322 323def filter_qmp_testfiles(qmsg): 324 def _filter(key, value): 325 if is_str(value): 326 return filter_testfiles(value) 327 return value 328 return filter_qmp(qmsg, _filter) 329 330def filter_generated_node_ids(msg): 331 return re.sub("#block[0-9]+", "NODE_NAME", msg) 332 333def filter_img_info(output, filename): 334 lines = [] 335 for line in output.split('\n'): 336 if 'disk size' in line or 'actual-size' in line: 337 continue 338 line = line.replace(filename, 'TEST_IMG') \ 339 .replace(imgfmt, 'IMGFMT') 340 line = re.sub('iters: [0-9]+', 'iters: XXX', line) 341 line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line) 342 line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line) 343 lines.append(line) 344 return '\n'.join(lines) 345 346def filter_imgfmt(msg): 347 return msg.replace(imgfmt, 'IMGFMT') 348 349def filter_qmp_imgfmt(qmsg): 350 def _filter(key, value): 351 if is_str(value): 352 return filter_imgfmt(value) 353 return value 354 return filter_qmp(qmsg, _filter) 355 356def log(msg, filters=[], indent=None): 357 '''Logs either a string message or a JSON serializable message (like QMP). 358 If indent is provided, JSON serializable messages are pretty-printed.''' 359 for flt in filters: 360 msg = flt(msg) 361 if isinstance(msg, dict) or isinstance(msg, list): 362 # Python < 3.4 needs to know not to add whitespace when pretty-printing: 363 separators = (', ', ': ') if indent is None else (',', ': ') 364 # Don't sort if it's already sorted 365 do_sort = not isinstance(msg, OrderedDict) 366 print(json.dumps(msg, sort_keys=do_sort, 367 indent=indent, separators=separators)) 368 else: 369 print(msg) 370 371class Timeout: 372 def __init__(self, seconds, errmsg = "Timeout"): 373 self.seconds = seconds 374 self.errmsg = errmsg 375 def __enter__(self): 376 signal.signal(signal.SIGALRM, self.timeout) 377 signal.setitimer(signal.ITIMER_REAL, self.seconds) 378 return self 379 def __exit__(self, type, value, traceback): 380 signal.setitimer(signal.ITIMER_REAL, 0) 381 return False 382 def timeout(self, signum, frame): 383 raise Exception(self.errmsg) 384 385def file_pattern(name): 386 return "{0}-{1}".format(os.getpid(), name) 387 388class FilePaths(object): 389 """ 390 FilePaths is an auto-generated filename that cleans itself up. 391 392 Use this context manager to generate filenames and ensure that the file 393 gets deleted:: 394 395 with FilePaths(['test.img']) as img_path: 396 qemu_img('create', img_path, '1G') 397 # migration_sock_path is automatically deleted 398 """ 399 def __init__(self, names, base_dir=test_dir): 400 self.paths = [] 401 for name in names: 402 self.paths.append(os.path.join(base_dir, file_pattern(name))) 403 404 def __enter__(self): 405 return self.paths 406 407 def __exit__(self, exc_type, exc_val, exc_tb): 408 try: 409 for path in self.paths: 410 os.remove(path) 411 except OSError: 412 pass 413 return False 414 415class FilePath(FilePaths): 416 """ 417 FilePath is a specialization of FilePaths that takes a single filename. 418 """ 419 def __init__(self, name, base_dir=test_dir): 420 super(FilePath, self).__init__([name], base_dir) 421 422 def __enter__(self): 423 return self.paths[0] 424 425def file_path_remover(): 426 for path in reversed(file_path_remover.paths): 427 try: 428 os.remove(path) 429 except OSError: 430 pass 431 432 433def file_path(*names, base_dir=test_dir): 434 ''' Another way to get auto-generated filename that cleans itself up. 435 436 Use is as simple as: 437 438 img_a, img_b = file_path('a.img', 'b.img') 439 sock = file_path('socket') 440 ''' 441 442 if not hasattr(file_path_remover, 'paths'): 443 file_path_remover.paths = [] 444 atexit.register(file_path_remover) 445 446 paths = [] 447 for name in names: 448 filename = file_pattern(name) 449 path = os.path.join(base_dir, filename) 450 file_path_remover.paths.append(path) 451 paths.append(path) 452 453 return paths[0] if len(paths) == 1 else paths 454 455def remote_filename(path): 456 if imgproto == 'file': 457 return path 458 elif imgproto == 'ssh': 459 return "ssh://%s@127.0.0.1:22%s" % (os.environ.get('USER'), path) 460 else: 461 raise Exception("Protocol %s not supported" % (imgproto)) 462 463class VM(qtest.QEMUQtestMachine): 464 '''A QEMU VM''' 465 466 def __init__(self, path_suffix=''): 467 name = "qemu%s-%d" % (path_suffix, os.getpid()) 468 super(VM, self).__init__(qemu_prog, qemu_opts, name=name, 469 test_dir=test_dir, 470 socket_scm_helper=socket_scm_helper, 471 sock_dir=sock_dir) 472 self._num_drives = 0 473 474 def add_object(self, opts): 475 self._args.append('-object') 476 self._args.append(opts) 477 return self 478 479 def add_device(self, opts): 480 self._args.append('-device') 481 self._args.append(opts) 482 return self 483 484 def add_drive_raw(self, opts): 485 self._args.append('-drive') 486 self._args.append(opts) 487 return self 488 489 def add_drive(self, path, opts='', interface='virtio', format=imgfmt): 490 '''Add a virtio-blk drive to the VM''' 491 options = ['if=%s' % interface, 492 'id=drive%d' % self._num_drives] 493 494 if path is not None: 495 options.append('file=%s' % path) 496 options.append('format=%s' % format) 497 options.append('cache=%s' % cachemode) 498 499 if opts: 500 options.append(opts) 501 502 if format == 'luks' and 'key-secret' not in opts: 503 # default luks support 504 if luks_default_secret_object not in self._args: 505 self.add_object(luks_default_secret_object) 506 507 options.append(luks_default_key_secret_opt) 508 509 self._args.append('-drive') 510 self._args.append(','.join(options)) 511 self._num_drives += 1 512 return self 513 514 def add_blockdev(self, opts): 515 self._args.append('-blockdev') 516 if isinstance(opts, str): 517 self._args.append(opts) 518 else: 519 self._args.append(','.join(opts)) 520 return self 521 522 def add_incoming(self, addr): 523 self._args.append('-incoming') 524 self._args.append(addr) 525 return self 526 527 def pause_drive(self, drive, event=None): 528 '''Pause drive r/w operations''' 529 if not event: 530 self.pause_drive(drive, "read_aio") 531 self.pause_drive(drive, "write_aio") 532 return 533 self.qmp('human-monitor-command', 534 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) 535 536 def resume_drive(self, drive): 537 self.qmp('human-monitor-command', 538 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) 539 540 def hmp_qemu_io(self, drive, cmd): 541 '''Write to a given drive using an HMP command''' 542 return self.qmp('human-monitor-command', 543 command_line='qemu-io %s "%s"' % (drive, cmd)) 544 545 def flatten_qmp_object(self, obj, output=None, basestr=''): 546 if output is None: 547 output = dict() 548 if isinstance(obj, list): 549 for i in range(len(obj)): 550 self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.') 551 elif isinstance(obj, dict): 552 for key in obj: 553 self.flatten_qmp_object(obj[key], output, basestr + key + '.') 554 else: 555 output[basestr[:-1]] = obj # Strip trailing '.' 556 return output 557 558 def qmp_to_opts(self, obj): 559 obj = self.flatten_qmp_object(obj) 560 output_list = list() 561 for key in obj: 562 output_list += [key + '=' + obj[key]] 563 return ','.join(output_list) 564 565 def get_qmp_events_filtered(self, wait=60.0): 566 result = [] 567 for ev in self.get_qmp_events(wait=wait): 568 result.append(filter_qmp_event(ev)) 569 return result 570 571 def qmp_log(self, cmd, filters=[], indent=None, **kwargs): 572 full_cmd = OrderedDict(( 573 ("execute", cmd), 574 ("arguments", ordered_qmp(kwargs)) 575 )) 576 log(full_cmd, filters, indent=indent) 577 result = self.qmp(cmd, **kwargs) 578 log(result, filters, indent=indent) 579 return result 580 581 # Returns None on success, and an error string on failure 582 def run_job(self, job, auto_finalize=True, auto_dismiss=False, 583 pre_finalize=None, cancel=False, use_log=True, wait=60.0): 584 """ 585 run_job moves a job from creation through to dismissal. 586 587 :param job: String. ID of recently-launched job 588 :param auto_finalize: Bool. True if the job was launched with 589 auto_finalize. Defaults to True. 590 :param auto_dismiss: Bool. True if the job was launched with 591 auto_dismiss=True. Defaults to False. 592 :param pre_finalize: Callback. A callable that takes no arguments to be 593 invoked prior to issuing job-finalize, if any. 594 :param cancel: Bool. When true, cancels the job after the pre_finalize 595 callback. 596 :param use_log: Bool. When false, does not log QMP messages. 597 :param wait: Float. Timeout value specifying how long to wait for any 598 event, in seconds. Defaults to 60.0. 599 """ 600 match_device = {'data': {'device': job}} 601 match_id = {'data': {'id': job}} 602 events = [ 603 ('BLOCK_JOB_COMPLETED', match_device), 604 ('BLOCK_JOB_CANCELLED', match_device), 605 ('BLOCK_JOB_ERROR', match_device), 606 ('BLOCK_JOB_READY', match_device), 607 ('BLOCK_JOB_PENDING', match_id), 608 ('JOB_STATUS_CHANGE', match_id) 609 ] 610 error = None 611 while True: 612 ev = filter_qmp_event(self.events_wait(events, timeout=wait)) 613 if ev['event'] != 'JOB_STATUS_CHANGE': 614 if use_log: 615 log(ev) 616 continue 617 status = ev['data']['status'] 618 if status == 'aborting': 619 result = self.qmp('query-jobs') 620 for j in result['return']: 621 if j['id'] == job: 622 error = j['error'] 623 if use_log: 624 log('Job failed: %s' % (j['error'])) 625 elif status == 'ready': 626 self.qmp_log('job-complete', id=job) 627 elif status == 'pending' and not auto_finalize: 628 if pre_finalize: 629 pre_finalize() 630 if cancel and use_log: 631 self.qmp_log('job-cancel', id=job) 632 elif cancel: 633 self.qmp('job-cancel', id=job) 634 elif use_log: 635 self.qmp_log('job-finalize', id=job) 636 else: 637 self.qmp('job-finalize', id=job) 638 elif status == 'concluded' and not auto_dismiss: 639 if use_log: 640 self.qmp_log('job-dismiss', id=job) 641 else: 642 self.qmp('job-dismiss', id=job) 643 elif status == 'null': 644 return error 645 646 # Returns None on success, and an error string on failure 647 def blockdev_create(self, options, job_id='job0', filters=None): 648 if filters is None: 649 filters = [filter_qmp_testfiles] 650 result = self.qmp_log('blockdev-create', filters=filters, 651 job_id=job_id, options=options) 652 653 if 'return' in result: 654 assert result['return'] == {} 655 job_result = self.run_job(job_id) 656 else: 657 job_result = result['error'] 658 659 log("") 660 return job_result 661 662 def enable_migration_events(self, name): 663 log('Enabling migration QMP events on %s...' % name) 664 log(self.qmp('migrate-set-capabilities', capabilities=[ 665 { 666 'capability': 'events', 667 'state': True 668 } 669 ])) 670 671 def wait_migration(self, expect_runstate): 672 while True: 673 event = self.event_wait('MIGRATION') 674 log(event, filters=[filter_qmp_event]) 675 if event['data']['status'] == 'completed': 676 break 677 # The event may occur in finish-migrate, so wait for the expected 678 # post-migration runstate 679 while self.qmp('query-status')['return']['status'] != expect_runstate: 680 pass 681 682 def node_info(self, node_name): 683 nodes = self.qmp('query-named-block-nodes') 684 for x in nodes['return']: 685 if x['node-name'] == node_name: 686 return x 687 return None 688 689 def query_bitmaps(self): 690 res = self.qmp("query-named-block-nodes") 691 return {device['node-name']: device['dirty-bitmaps'] 692 for device in res['return'] if 'dirty-bitmaps' in device} 693 694 def get_bitmap(self, node_name, bitmap_name, recording=None, bitmaps=None): 695 """ 696 get a specific bitmap from the object returned by query_bitmaps. 697 :param recording: If specified, filter results by the specified value. 698 :param bitmaps: If specified, use it instead of call query_bitmaps() 699 """ 700 if bitmaps is None: 701 bitmaps = self.query_bitmaps() 702 703 for bitmap in bitmaps[node_name]: 704 if bitmap.get('name', '') == bitmap_name: 705 if recording is None: 706 return bitmap 707 elif bitmap.get('recording') == recording: 708 return bitmap 709 return None 710 711 def check_bitmap_status(self, node_name, bitmap_name, fields): 712 ret = self.get_bitmap(node_name, bitmap_name) 713 714 return fields.items() <= ret.items() 715 716 717index_re = re.compile(r'([^\[]+)\[([^\]]+)\]') 718 719class QMPTestCase(unittest.TestCase): 720 '''Abstract base class for QMP test cases''' 721 722 def dictpath(self, d, path): 723 '''Traverse a path in a nested dict''' 724 for component in path.split('/'): 725 m = index_re.match(component) 726 if m: 727 component, idx = m.groups() 728 idx = int(idx) 729 730 if not isinstance(d, dict) or component not in d: 731 self.fail('failed path traversal for "%s" in "%s"' % (path, str(d))) 732 d = d[component] 733 734 if m: 735 if not isinstance(d, list): 736 self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d))) 737 try: 738 d = d[idx] 739 except IndexError: 740 self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d))) 741 return d 742 743 def assert_qmp_absent(self, d, path): 744 try: 745 result = self.dictpath(d, path) 746 except AssertionError: 747 return 748 self.fail('path "%s" has value "%s"' % (path, str(result))) 749 750 def assert_qmp(self, d, path, value): 751 '''Assert that the value for a specific path in a QMP dict 752 matches. When given a list of values, assert that any of 753 them matches.''' 754 755 result = self.dictpath(d, path) 756 757 # [] makes no sense as a list of valid values, so treat it as 758 # an actual single value. 759 if isinstance(value, list) and value != []: 760 for v in value: 761 if result == v: 762 return 763 self.fail('no match for "%s" in %s' % (str(result), str(value))) 764 else: 765 self.assertEqual(result, value, 766 '"%s" is "%s", expected "%s"' 767 % (path, str(result), str(value))) 768 769 def assert_no_active_block_jobs(self): 770 result = self.vm.qmp('query-block-jobs') 771 self.assert_qmp(result, 'return', []) 772 773 def assert_has_block_node(self, node_name=None, file_name=None): 774 """Issue a query-named-block-nodes and assert node_name and/or 775 file_name is present in the result""" 776 def check_equal_or_none(a, b): 777 return a == None or b == None or a == b 778 assert node_name or file_name 779 result = self.vm.qmp('query-named-block-nodes') 780 for x in result["return"]: 781 if check_equal_or_none(x.get("node-name"), node_name) and \ 782 check_equal_or_none(x.get("file"), file_name): 783 return 784 self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \ 785 (node_name, file_name, result)) 786 787 def assert_json_filename_equal(self, json_filename, reference): 788 '''Asserts that the given filename is a json: filename and that its 789 content is equal to the given reference object''' 790 self.assertEqual(json_filename[:5], 'json:') 791 self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])), 792 self.vm.flatten_qmp_object(reference)) 793 794 def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0): 795 '''Cancel a block job and wait for it to finish, returning the event''' 796 result = self.vm.qmp('block-job-cancel', device=drive, force=force) 797 self.assert_qmp(result, 'return', {}) 798 799 if resume: 800 self.vm.resume_drive(drive) 801 802 cancelled = False 803 result = None 804 while not cancelled: 805 for event in self.vm.get_qmp_events(wait=wait): 806 if event['event'] == 'BLOCK_JOB_COMPLETED' or \ 807 event['event'] == 'BLOCK_JOB_CANCELLED': 808 self.assert_qmp(event, 'data/device', drive) 809 result = event 810 cancelled = True 811 elif event['event'] == 'JOB_STATUS_CHANGE': 812 self.assert_qmp(event, 'data/id', drive) 813 814 815 self.assert_no_active_block_jobs() 816 return result 817 818 def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0, 819 error=None): 820 '''Wait for a block job to finish, returning the event''' 821 while True: 822 for event in self.vm.get_qmp_events(wait=wait): 823 if event['event'] == 'BLOCK_JOB_COMPLETED': 824 self.assert_qmp(event, 'data/device', drive) 825 if error is None: 826 self.assert_qmp_absent(event, 'data/error') 827 if check_offset: 828 self.assert_qmp(event, 'data/offset', 829 event['data']['len']) 830 else: 831 self.assert_qmp(event, 'data/error', error) 832 self.assert_no_active_block_jobs() 833 return event 834 elif event['event'] == 'JOB_STATUS_CHANGE': 835 self.assert_qmp(event, 'data/id', drive) 836 837 def wait_ready(self, drive='drive0'): 838 '''Wait until a block job BLOCK_JOB_READY event''' 839 f = {'data': {'type': 'mirror', 'device': drive } } 840 event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f) 841 842 def wait_ready_and_cancel(self, drive='drive0'): 843 self.wait_ready(drive=drive) 844 event = self.cancel_and_wait(drive=drive) 845 self.assertEqual(event['event'], 'BLOCK_JOB_COMPLETED') 846 self.assert_qmp(event, 'data/type', 'mirror') 847 self.assert_qmp(event, 'data/offset', event['data']['len']) 848 849 def complete_and_wait(self, drive='drive0', wait_ready=True, 850 completion_error=None): 851 '''Complete a block job and wait for it to finish''' 852 if wait_ready: 853 self.wait_ready(drive=drive) 854 855 result = self.vm.qmp('block-job-complete', device=drive) 856 self.assert_qmp(result, 'return', {}) 857 858 event = self.wait_until_completed(drive=drive, error=completion_error) 859 self.assert_qmp(event, 'data/type', 'mirror') 860 861 def pause_wait(self, job_id='job0'): 862 with Timeout(1, "Timeout waiting for job to pause"): 863 while True: 864 result = self.vm.qmp('query-block-jobs') 865 found = False 866 for job in result['return']: 867 if job['device'] == job_id: 868 found = True 869 if job['paused'] == True and job['busy'] == False: 870 return job 871 break 872 assert found 873 874 def pause_job(self, job_id='job0', wait=True): 875 result = self.vm.qmp('block-job-pause', device=job_id) 876 self.assert_qmp(result, 'return', {}) 877 if wait: 878 return self.pause_wait(job_id) 879 return result 880 881 def case_skip(self, reason): 882 '''Skip this test case''' 883 case_notrun(reason) 884 self.skipTest(reason) 885 886 887def notrun(reason): 888 '''Skip this test suite''' 889 # Each test in qemu-iotests has a number ("seq") 890 seq = os.path.basename(sys.argv[0]) 891 892 open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n') 893 print('%s not run: %s' % (seq, reason)) 894 sys.exit(0) 895 896def case_notrun(reason): 897 '''Mark this test case as not having been run (without actually 898 skipping it, that is left to the caller). See 899 QMPTestCase.case_skip() for a variant that actually skips the 900 current test case.''' 901 902 # Each test in qemu-iotests has a number ("seq") 903 seq = os.path.basename(sys.argv[0]) 904 905 open('%s/%s.casenotrun' % (output_dir, seq), 'a').write( 906 ' [case not run] ' + reason + '\n') 907 908def verify_image_format(supported_fmts=[], unsupported_fmts=[]): 909 assert not (supported_fmts and unsupported_fmts) 910 911 if 'generic' in supported_fmts and \ 912 os.environ.get('IMGFMT_GENERIC', 'true') == 'true': 913 # similar to 914 # _supported_fmt generic 915 # for bash tests 916 return 917 918 not_sup = supported_fmts and (imgfmt not in supported_fmts) 919 if not_sup or (imgfmt in unsupported_fmts): 920 notrun('not suitable for this image format: %s' % imgfmt) 921 922def verify_protocol(supported=[], unsupported=[]): 923 assert not (supported and unsupported) 924 925 if 'generic' in supported: 926 return 927 928 not_sup = supported and (imgproto not in supported) 929 if not_sup or (imgproto in unsupported): 930 notrun('not suitable for this protocol: %s' % imgproto) 931 932def verify_platform(supported_oses=['linux']): 933 if True not in [sys.platform.startswith(x) for x in supported_oses]: 934 notrun('not suitable for this OS: %s' % sys.platform) 935 936def verify_cache_mode(supported_cache_modes=[]): 937 if supported_cache_modes and (cachemode not in supported_cache_modes): 938 notrun('not suitable for this cache mode: %s' % cachemode) 939 940def supports_quorum(): 941 return 'quorum' in qemu_img_pipe('--help') 942 943def verify_quorum(): 944 '''Skip test suite if quorum support is not available''' 945 if not supports_quorum(): 946 notrun('quorum support missing') 947 948def qemu_pipe(*args): 949 '''Run qemu with an option to print something and exit (e.g. a help option), 950 and return its output''' 951 args = [qemu_prog] + qemu_opts + list(args) 952 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 953 stderr=subprocess.STDOUT, 954 universal_newlines=True) 955 exitcode = subp.wait() 956 if exitcode < 0: 957 sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, 958 ' '.join(args))) 959 return subp.communicate()[0] 960 961def supported_formats(read_only=False): 962 '''Set 'read_only' to True to check ro-whitelist 963 Otherwise, rw-whitelist is checked''' 964 965 if not hasattr(supported_formats, "formats"): 966 supported_formats.formats = {} 967 968 if read_only not in supported_formats.formats: 969 format_message = qemu_pipe("-drive", "format=help") 970 line = 1 if read_only else 0 971 supported_formats.formats[read_only] = \ 972 format_message.splitlines()[line].split(":")[1].split() 973 974 return supported_formats.formats[read_only] 975 976def skip_if_unsupported(required_formats=[], read_only=False): 977 '''Skip Test Decorator 978 Runs the test if all the required formats are whitelisted''' 979 def skip_test_decorator(func): 980 def func_wrapper(test_case: QMPTestCase, *args, **kwargs): 981 if callable(required_formats): 982 fmts = required_formats(test_case) 983 else: 984 fmts = required_formats 985 986 usf_list = list(set(fmts) - set(supported_formats(read_only))) 987 if usf_list: 988 test_case.case_skip('{}: formats {} are not whitelisted'.format( 989 test_case, usf_list)) 990 else: 991 return func(test_case, *args, **kwargs) 992 return func_wrapper 993 return skip_test_decorator 994 995def skip_if_user_is_root(func): 996 '''Skip Test Decorator 997 Runs the test only without root permissions''' 998 def func_wrapper(*args, **kwargs): 999 if os.getuid() == 0: 1000 case_notrun('{}: cannot be run as root'.format(args[0])) 1001 else: 1002 return func(*args, **kwargs) 1003 return func_wrapper 1004 1005def execute_unittest(output, verbosity, debug): 1006 runner = unittest.TextTestRunner(stream=output, descriptions=True, 1007 verbosity=verbosity) 1008 try: 1009 # unittest.main() will use sys.exit(); so expect a SystemExit 1010 # exception 1011 unittest.main(testRunner=runner) 1012 finally: 1013 if not debug: 1014 out = output.getvalue() 1015 out = re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', out) 1016 1017 # Hide skipped tests from the reference output 1018 out = re.sub(r'OK \(skipped=\d+\)', 'OK', out) 1019 out_first_line, out_rest = out.split('\n', 1) 1020 out = out_first_line.replace('s', '.') + '\n' + out_rest 1021 1022 sys.stderr.write(out) 1023 1024def execute_test(test_function=None, 1025 supported_fmts=[], supported_oses=['linux'], 1026 supported_cache_modes=[], unsupported_fmts=[], 1027 supported_protocols=[], unsupported_protocols=[]): 1028 """Run either unittest or script-style tests.""" 1029 1030 # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to 1031 # indicate that we're not being run via "check". There may be 1032 # other things set up by "check" that individual test cases rely 1033 # on. 1034 if test_dir is None or qemu_default_machine is None: 1035 sys.stderr.write('Please run this test via the "check" script\n') 1036 sys.exit(os.EX_USAGE) 1037 1038 debug = '-d' in sys.argv 1039 verbosity = 1 1040 verify_image_format(supported_fmts, unsupported_fmts) 1041 verify_protocol(supported_protocols, unsupported_protocols) 1042 verify_platform(supported_oses) 1043 verify_cache_mode(supported_cache_modes) 1044 1045 if debug: 1046 output = sys.stdout 1047 verbosity = 2 1048 sys.argv.remove('-d') 1049 else: 1050 # We need to filter out the time taken from the output so that 1051 # qemu-iotest can reliably diff the results against master output. 1052 output = io.StringIO() 1053 1054 logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) 1055 1056 if not test_function: 1057 execute_unittest(output, verbosity, debug) 1058 else: 1059 test_function() 1060 1061def script_main(test_function, *args, **kwargs): 1062 """Run script-style tests outside of the unittest framework""" 1063 execute_test(test_function, *args, **kwargs) 1064 1065def main(*args, **kwargs): 1066 """Run tests using the unittest framework""" 1067 execute_test(None, *args, **kwargs) 1068