1from __future__ import print_function 2# Common utilities and Python wrappers for qemu-iotests 3# 4# Copyright (C) 2012 IBM Corp. 5# 6# This program is free software; you can redistribute it and/or modify 7# it under the terms of the GNU General Public License as published by 8# the Free Software Foundation; either version 2 of the License, or 9# (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program. If not, see <http://www.gnu.org/licenses/>. 18# 19 20import errno 21import os 22import re 23import subprocess 24import string 25import unittest 26import sys 27import struct 28import json 29import signal 30import logging 31import atexit 32import io 33from collections import OrderedDict 34 35sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) 36from qemu import qtest 37 38assert sys.version_info >= (3,6) 39 40# This will not work if arguments contain spaces but is necessary if we 41# want to support the override options that ./check supports. 42qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')] 43if os.environ.get('QEMU_IMG_OPTIONS'): 44 qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ') 45 46qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] 47if os.environ.get('QEMU_IO_OPTIONS'): 48 qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ') 49 50qemu_nbd_args = [os.environ.get('QEMU_NBD_PROG', 'qemu-nbd')] 51if os.environ.get('QEMU_NBD_OPTIONS'): 52 qemu_nbd_args += os.environ['QEMU_NBD_OPTIONS'].strip().split(' ') 53 54qemu_prog = os.environ.get('QEMU_PROG', 'qemu') 55qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ') 56 57imgfmt = os.environ.get('IMGFMT', 'raw') 58imgproto = os.environ.get('IMGPROTO', 'file') 59test_dir = os.environ.get('TEST_DIR') 60output_dir = os.environ.get('OUTPUT_DIR', '.') 61cachemode = os.environ.get('CACHEMODE') 62qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') 63 64socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') 65 66luks_default_secret_object = 'secret,id=keysec0,data=' + \ 67 os.environ.get('IMGKEYSECRET', '') 68luks_default_key_secret_opt = 'key-secret=keysec0' 69 70 71def qemu_img(*args): 72 '''Run qemu-img and return the exit code''' 73 devnull = open('/dev/null', 'r+') 74 exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) 75 if exitcode < 0: 76 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 77 return exitcode 78 79def ordered_qmp(qmsg, conv_keys=True): 80 # Dictionaries are not ordered prior to 3.6, therefore: 81 if isinstance(qmsg, list): 82 return [ordered_qmp(atom) for atom in qmsg] 83 if isinstance(qmsg, dict): 84 od = OrderedDict() 85 for k, v in sorted(qmsg.items()): 86 if conv_keys: 87 k = k.replace('_', '-') 88 od[k] = ordered_qmp(v, conv_keys=False) 89 return od 90 return qmsg 91 92def qemu_img_create(*args): 93 args = list(args) 94 95 # default luks support 96 if '-f' in args and args[args.index('-f') + 1] == 'luks': 97 if '-o' in args: 98 i = args.index('-o') 99 if 'key-secret' not in args[i + 1]: 100 args[i + 1].append(luks_default_key_secret_opt) 101 args.insert(i + 2, '--object') 102 args.insert(i + 3, luks_default_secret_object) 103 else: 104 args = ['-o', luks_default_key_secret_opt, 105 '--object', luks_default_secret_object] + args 106 107 args.insert(0, 'create') 108 109 return qemu_img(*args) 110 111def qemu_img_verbose(*args): 112 '''Run qemu-img without suppressing its output and return the exit code''' 113 exitcode = subprocess.call(qemu_img_args + list(args)) 114 if exitcode < 0: 115 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 116 return exitcode 117 118def qemu_img_pipe(*args): 119 '''Run qemu-img and return its output''' 120 subp = subprocess.Popen(qemu_img_args + list(args), 121 stdout=subprocess.PIPE, 122 stderr=subprocess.STDOUT, 123 universal_newlines=True) 124 exitcode = subp.wait() 125 if exitcode < 0: 126 sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) 127 return subp.communicate()[0] 128 129def qemu_img_log(*args): 130 result = qemu_img_pipe(*args) 131 log(result, filters=[filter_testfiles]) 132 return result 133 134def img_info_log(filename, filter_path=None, imgopts=False, extra_args=[]): 135 args = [ 'info' ] 136 if imgopts: 137 args.append('--image-opts') 138 else: 139 args += [ '-f', imgfmt ] 140 args += extra_args 141 args.append(filename) 142 143 output = qemu_img_pipe(*args) 144 if not filter_path: 145 filter_path = filename 146 log(filter_img_info(output, filter_path)) 147 148def qemu_io(*args): 149 '''Run qemu-io and return the stdout data''' 150 args = qemu_io_args + list(args) 151 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 152 stderr=subprocess.STDOUT, 153 universal_newlines=True) 154 exitcode = subp.wait() 155 if exitcode < 0: 156 sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args))) 157 return subp.communicate()[0] 158 159def qemu_io_silent(*args): 160 '''Run qemu-io and return the exit code, suppressing stdout''' 161 args = qemu_io_args + list(args) 162 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w')) 163 if exitcode < 0: 164 sys.stderr.write('qemu-io received signal %i: %s\n' % 165 (-exitcode, ' '.join(args))) 166 return exitcode 167 168def qemu_io_silent_check(*args): 169 '''Run qemu-io and return the true if subprocess returned 0''' 170 args = qemu_io_args + list(args) 171 exitcode = subprocess.call(args, stdout=open('/dev/null', 'w'), 172 stderr=subprocess.STDOUT) 173 return exitcode == 0 174 175def get_virtio_scsi_device(): 176 if qemu_default_machine == 's390-ccw-virtio': 177 return 'virtio-scsi-ccw' 178 return 'virtio-scsi-pci' 179 180class QemuIoInteractive: 181 def __init__(self, *args): 182 self.args = qemu_io_args + list(args) 183 self._p = subprocess.Popen(self.args, stdin=subprocess.PIPE, 184 stdout=subprocess.PIPE, 185 stderr=subprocess.STDOUT, 186 universal_newlines=True) 187 assert self._p.stdout.read(9) == 'qemu-io> ' 188 189 def close(self): 190 self._p.communicate('q\n') 191 192 def _read_output(self): 193 pattern = 'qemu-io> ' 194 n = len(pattern) 195 pos = 0 196 s = [] 197 while pos != n: 198 c = self._p.stdout.read(1) 199 # check unexpected EOF 200 assert c != '' 201 s.append(c) 202 if c == pattern[pos]: 203 pos += 1 204 else: 205 pos = 0 206 207 return ''.join(s[:-n]) 208 209 def cmd(self, cmd): 210 # quit command is in close(), '\n' is added automatically 211 assert '\n' not in cmd 212 cmd = cmd.strip() 213 assert cmd != 'q' and cmd != 'quit' 214 self._p.stdin.write(cmd + '\n') 215 self._p.stdin.flush() 216 return self._read_output() 217 218 219def qemu_nbd(*args): 220 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 221 return subprocess.call(qemu_nbd_args + ['--fork'] + list(args)) 222 223def qemu_nbd_early_pipe(*args): 224 '''Run qemu-nbd in daemon mode and return both the parent's exit code 225 and its output in case of an error''' 226 subp = subprocess.Popen(qemu_nbd_args + ['--fork'] + list(args), 227 stdout=subprocess.PIPE, 228 stderr=subprocess.STDOUT, 229 universal_newlines=True) 230 exitcode = subp.wait() 231 if exitcode < 0: 232 sys.stderr.write('qemu-nbd received signal %i: %s\n' % 233 (-exitcode, 234 ' '.join(qemu_nbd_args + ['--fork'] + list(args)))) 235 if exitcode == 0: 236 return exitcode, '' 237 else: 238 return exitcode, subp.communicate()[0] 239 240def qemu_nbd_popen(*args): 241 '''Run qemu-nbd in daemon mode and return the parent's exit code''' 242 return subprocess.Popen(qemu_nbd_args + ['--persistent'] + list(args)) 243 244def compare_images(img1, img2, fmt1=imgfmt, fmt2=imgfmt): 245 '''Return True if two image files are identical''' 246 return qemu_img('compare', '-f', fmt1, 247 '-F', fmt2, img1, img2) == 0 248 249def create_image(name, size): 250 '''Create a fully-allocated raw image with sector markers''' 251 file = open(name, 'wb') 252 i = 0 253 while i < size: 254 sector = struct.pack('>l504xl', i // 512, i // 512) 255 file.write(sector) 256 i = i + 512 257 file.close() 258 259def image_size(img): 260 '''Return image's virtual size''' 261 r = qemu_img_pipe('info', '--output=json', '-f', imgfmt, img) 262 return json.loads(r)['virtual-size'] 263 264def is_str(val): 265 return isinstance(val, str) 266 267test_dir_re = re.compile(r"%s" % test_dir) 268def filter_test_dir(msg): 269 return test_dir_re.sub("TEST_DIR", msg) 270 271win32_re = re.compile(r"\r") 272def filter_win32(msg): 273 return win32_re.sub("", msg) 274 275qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)") 276def filter_qemu_io(msg): 277 msg = filter_win32(msg) 278 return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg) 279 280chown_re = re.compile(r"chown [0-9]+:[0-9]+") 281def filter_chown(msg): 282 return chown_re.sub("chown UID:GID", msg) 283 284def filter_qmp_event(event): 285 '''Filter a QMP event dict''' 286 event = dict(event) 287 if 'timestamp' in event: 288 event['timestamp']['seconds'] = 'SECS' 289 event['timestamp']['microseconds'] = 'USECS' 290 return event 291 292def filter_qmp(qmsg, filter_fn): 293 '''Given a string filter, filter a QMP object's values. 294 filter_fn takes a (key, value) pair.''' 295 # Iterate through either lists or dicts; 296 if isinstance(qmsg, list): 297 items = enumerate(qmsg) 298 else: 299 items = qmsg.items() 300 301 for k, v in items: 302 if isinstance(v, list) or isinstance(v, dict): 303 qmsg[k] = filter_qmp(v, filter_fn) 304 else: 305 qmsg[k] = filter_fn(k, v) 306 return qmsg 307 308def filter_testfiles(msg): 309 prefix = os.path.join(test_dir, "%s-" % (os.getpid())) 310 return msg.replace(prefix, 'TEST_DIR/PID-') 311 312def filter_qmp_testfiles(qmsg): 313 def _filter(key, value): 314 if is_str(value): 315 return filter_testfiles(value) 316 return value 317 return filter_qmp(qmsg, _filter) 318 319def filter_generated_node_ids(msg): 320 return re.sub("#block[0-9]+", "NODE_NAME", msg) 321 322def filter_img_info(output, filename): 323 lines = [] 324 for line in output.split('\n'): 325 if 'disk size' in line or 'actual-size' in line: 326 continue 327 line = line.replace(filename, 'TEST_IMG') \ 328 .replace(imgfmt, 'IMGFMT') 329 line = re.sub('iters: [0-9]+', 'iters: XXX', line) 330 line = re.sub('uuid: [-a-f0-9]+', 'uuid: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', line) 331 line = re.sub('cid: [0-9]+', 'cid: XXXXXXXXXX', line) 332 lines.append(line) 333 return '\n'.join(lines) 334 335def filter_imgfmt(msg): 336 return msg.replace(imgfmt, 'IMGFMT') 337 338def filter_qmp_imgfmt(qmsg): 339 def _filter(key, value): 340 if is_str(value): 341 return filter_imgfmt(value) 342 return value 343 return filter_qmp(qmsg, _filter) 344 345def log(msg, filters=[], indent=None): 346 '''Logs either a string message or a JSON serializable message (like QMP). 347 If indent is provided, JSON serializable messages are pretty-printed.''' 348 for flt in filters: 349 msg = flt(msg) 350 if isinstance(msg, dict) or isinstance(msg, list): 351 # Python < 3.4 needs to know not to add whitespace when pretty-printing: 352 separators = (', ', ': ') if indent is None else (',', ': ') 353 # Don't sort if it's already sorted 354 do_sort = not isinstance(msg, OrderedDict) 355 print(json.dumps(msg, sort_keys=do_sort, 356 indent=indent, separators=separators)) 357 else: 358 print(msg) 359 360class Timeout: 361 def __init__(self, seconds, errmsg = "Timeout"): 362 self.seconds = seconds 363 self.errmsg = errmsg 364 def __enter__(self): 365 signal.signal(signal.SIGALRM, self.timeout) 366 signal.setitimer(signal.ITIMER_REAL, self.seconds) 367 return self 368 def __exit__(self, type, value, traceback): 369 signal.setitimer(signal.ITIMER_REAL, 0) 370 return False 371 def timeout(self, signum, frame): 372 raise Exception(self.errmsg) 373 374def file_pattern(name): 375 return "{0}-{1}".format(os.getpid(), name) 376 377class FilePaths(object): 378 """ 379 FilePaths is an auto-generated filename that cleans itself up. 380 381 Use this context manager to generate filenames and ensure that the file 382 gets deleted:: 383 384 with FilePaths(['test.img']) as img_path: 385 qemu_img('create', img_path, '1G') 386 # migration_sock_path is automatically deleted 387 """ 388 def __init__(self, names): 389 self.paths = [] 390 for name in names: 391 self.paths.append(os.path.join(test_dir, file_pattern(name))) 392 393 def __enter__(self): 394 return self.paths 395 396 def __exit__(self, exc_type, exc_val, exc_tb): 397 try: 398 for path in self.paths: 399 os.remove(path) 400 except OSError: 401 pass 402 return False 403 404class FilePath(FilePaths): 405 """ 406 FilePath is a specialization of FilePaths that takes a single filename. 407 """ 408 def __init__(self, name): 409 super(FilePath, self).__init__([name]) 410 411 def __enter__(self): 412 return self.paths[0] 413 414def file_path_remover(): 415 for path in reversed(file_path_remover.paths): 416 try: 417 os.remove(path) 418 except OSError: 419 pass 420 421 422def file_path(*names): 423 ''' Another way to get auto-generated filename that cleans itself up. 424 425 Use is as simple as: 426 427 img_a, img_b = file_path('a.img', 'b.img') 428 sock = file_path('socket') 429 ''' 430 431 if not hasattr(file_path_remover, 'paths'): 432 file_path_remover.paths = [] 433 atexit.register(file_path_remover) 434 435 paths = [] 436 for name in names: 437 filename = file_pattern(name) 438 path = os.path.join(test_dir, filename) 439 file_path_remover.paths.append(path) 440 paths.append(path) 441 442 return paths[0] if len(paths) == 1 else paths 443 444def remote_filename(path): 445 if imgproto == 'file': 446 return path 447 elif imgproto == 'ssh': 448 return "ssh://%s@127.0.0.1:22%s" % (os.environ.get('USER'), path) 449 else: 450 raise Exception("Protocol %s not supported" % (imgproto)) 451 452class VM(qtest.QEMUQtestMachine): 453 '''A QEMU VM''' 454 455 def __init__(self, path_suffix=''): 456 name = "qemu%s-%d" % (path_suffix, os.getpid()) 457 super(VM, self).__init__(qemu_prog, qemu_opts, name=name, 458 test_dir=test_dir, 459 socket_scm_helper=socket_scm_helper) 460 self._num_drives = 0 461 462 def add_object(self, opts): 463 self._args.append('-object') 464 self._args.append(opts) 465 return self 466 467 def add_device(self, opts): 468 self._args.append('-device') 469 self._args.append(opts) 470 return self 471 472 def add_drive_raw(self, opts): 473 self._args.append('-drive') 474 self._args.append(opts) 475 return self 476 477 def add_drive(self, path, opts='', interface='virtio', format=imgfmt): 478 '''Add a virtio-blk drive to the VM''' 479 options = ['if=%s' % interface, 480 'id=drive%d' % self._num_drives] 481 482 if path is not None: 483 options.append('file=%s' % path) 484 options.append('format=%s' % format) 485 options.append('cache=%s' % cachemode) 486 487 if opts: 488 options.append(opts) 489 490 if format == 'luks' and 'key-secret' not in opts: 491 # default luks support 492 if luks_default_secret_object not in self._args: 493 self.add_object(luks_default_secret_object) 494 495 options.append(luks_default_key_secret_opt) 496 497 self._args.append('-drive') 498 self._args.append(','.join(options)) 499 self._num_drives += 1 500 return self 501 502 def add_blockdev(self, opts): 503 self._args.append('-blockdev') 504 if isinstance(opts, str): 505 self._args.append(opts) 506 else: 507 self._args.append(','.join(opts)) 508 return self 509 510 def add_incoming(self, addr): 511 self._args.append('-incoming') 512 self._args.append(addr) 513 return self 514 515 def pause_drive(self, drive, event=None): 516 '''Pause drive r/w operations''' 517 if not event: 518 self.pause_drive(drive, "read_aio") 519 self.pause_drive(drive, "write_aio") 520 return 521 self.qmp('human-monitor-command', 522 command_line='qemu-io %s "break %s bp_%s"' % (drive, event, drive)) 523 524 def resume_drive(self, drive): 525 self.qmp('human-monitor-command', 526 command_line='qemu-io %s "remove_break bp_%s"' % (drive, drive)) 527 528 def hmp_qemu_io(self, drive, cmd): 529 '''Write to a given drive using an HMP command''' 530 return self.qmp('human-monitor-command', 531 command_line='qemu-io %s "%s"' % (drive, cmd)) 532 533 def flatten_qmp_object(self, obj, output=None, basestr=''): 534 if output is None: 535 output = dict() 536 if isinstance(obj, list): 537 for i in range(len(obj)): 538 self.flatten_qmp_object(obj[i], output, basestr + str(i) + '.') 539 elif isinstance(obj, dict): 540 for key in obj: 541 self.flatten_qmp_object(obj[key], output, basestr + key + '.') 542 else: 543 output[basestr[:-1]] = obj # Strip trailing '.' 544 return output 545 546 def qmp_to_opts(self, obj): 547 obj = self.flatten_qmp_object(obj) 548 output_list = list() 549 for key in obj: 550 output_list += [key + '=' + obj[key]] 551 return ','.join(output_list) 552 553 def get_qmp_events_filtered(self, wait=60.0): 554 result = [] 555 for ev in self.get_qmp_events(wait=wait): 556 result.append(filter_qmp_event(ev)) 557 return result 558 559 def qmp_log(self, cmd, filters=[], indent=None, **kwargs): 560 full_cmd = OrderedDict(( 561 ("execute", cmd), 562 ("arguments", ordered_qmp(kwargs)) 563 )) 564 log(full_cmd, filters, indent=indent) 565 result = self.qmp(cmd, **kwargs) 566 log(result, filters, indent=indent) 567 return result 568 569 # Returns None on success, and an error string on failure 570 def run_job(self, job, auto_finalize=True, auto_dismiss=False, 571 pre_finalize=None, cancel=False, use_log=True, wait=60.0): 572 """ 573 run_job moves a job from creation through to dismissal. 574 575 :param job: String. ID of recently-launched job 576 :param auto_finalize: Bool. True if the job was launched with 577 auto_finalize. Defaults to True. 578 :param auto_dismiss: Bool. True if the job was launched with 579 auto_dismiss=True. Defaults to False. 580 :param pre_finalize: Callback. A callable that takes no arguments to be 581 invoked prior to issuing job-finalize, if any. 582 :param cancel: Bool. When true, cancels the job after the pre_finalize 583 callback. 584 :param use_log: Bool. When false, does not log QMP messages. 585 :param wait: Float. Timeout value specifying how long to wait for any 586 event, in seconds. Defaults to 60.0. 587 """ 588 match_device = {'data': {'device': job}} 589 match_id = {'data': {'id': job}} 590 events = [ 591 ('BLOCK_JOB_COMPLETED', match_device), 592 ('BLOCK_JOB_CANCELLED', match_device), 593 ('BLOCK_JOB_ERROR', match_device), 594 ('BLOCK_JOB_READY', match_device), 595 ('BLOCK_JOB_PENDING', match_id), 596 ('JOB_STATUS_CHANGE', match_id) 597 ] 598 error = None 599 while True: 600 ev = filter_qmp_event(self.events_wait(events)) 601 if ev['event'] != 'JOB_STATUS_CHANGE': 602 if use_log: 603 log(ev) 604 continue 605 status = ev['data']['status'] 606 if status == 'aborting': 607 result = self.qmp('query-jobs') 608 for j in result['return']: 609 if j['id'] == job: 610 error = j['error'] 611 if use_log: 612 log('Job failed: %s' % (j['error'])) 613 elif status == 'pending' and not auto_finalize: 614 if pre_finalize: 615 pre_finalize() 616 if cancel and use_log: 617 self.qmp_log('job-cancel', id=job) 618 elif cancel: 619 self.qmp('job-cancel', id=job) 620 elif use_log: 621 self.qmp_log('job-finalize', id=job) 622 else: 623 self.qmp('job-finalize', id=job) 624 elif status == 'concluded' and not auto_dismiss: 625 if use_log: 626 self.qmp_log('job-dismiss', id=job) 627 else: 628 self.qmp('job-dismiss', id=job) 629 elif status == 'null': 630 return error 631 632 def enable_migration_events(self, name): 633 log('Enabling migration QMP events on %s...' % name) 634 log(self.qmp('migrate-set-capabilities', capabilities=[ 635 { 636 'capability': 'events', 637 'state': True 638 } 639 ])) 640 641 def wait_migration(self): 642 while True: 643 event = self.event_wait('MIGRATION') 644 log(event, filters=[filter_qmp_event]) 645 if event['data']['status'] == 'completed': 646 break 647 648 def node_info(self, node_name): 649 nodes = self.qmp('query-named-block-nodes') 650 for x in nodes['return']: 651 if x['node-name'] == node_name: 652 return x 653 return None 654 655 def query_bitmaps(self): 656 res = self.qmp("query-named-block-nodes") 657 return {device['node-name']: device['dirty-bitmaps'] 658 for device in res['return'] if 'dirty-bitmaps' in device} 659 660 def get_bitmap(self, node_name, bitmap_name, recording=None, bitmaps=None): 661 """ 662 get a specific bitmap from the object returned by query_bitmaps. 663 :param recording: If specified, filter results by the specified value. 664 :param bitmaps: If specified, use it instead of call query_bitmaps() 665 """ 666 if bitmaps is None: 667 bitmaps = self.query_bitmaps() 668 669 for bitmap in bitmaps[node_name]: 670 if bitmap.get('name', '') == bitmap_name: 671 if recording is None: 672 return bitmap 673 elif bitmap.get('recording') == recording: 674 return bitmap 675 return None 676 677 def check_bitmap_status(self, node_name, bitmap_name, fields): 678 ret = self.get_bitmap(node_name, bitmap_name) 679 680 return fields.items() <= ret.items() 681 682 683index_re = re.compile(r'([^\[]+)\[([^\]]+)\]') 684 685class QMPTestCase(unittest.TestCase): 686 '''Abstract base class for QMP test cases''' 687 688 def dictpath(self, d, path): 689 '''Traverse a path in a nested dict''' 690 for component in path.split('/'): 691 m = index_re.match(component) 692 if m: 693 component, idx = m.groups() 694 idx = int(idx) 695 696 if not isinstance(d, dict) or component not in d: 697 self.fail('failed path traversal for "%s" in "%s"' % (path, str(d))) 698 d = d[component] 699 700 if m: 701 if not isinstance(d, list): 702 self.fail('path component "%s" in "%s" is not a list in "%s"' % (component, path, str(d))) 703 try: 704 d = d[idx] 705 except IndexError: 706 self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d))) 707 return d 708 709 def assert_qmp_absent(self, d, path): 710 try: 711 result = self.dictpath(d, path) 712 except AssertionError: 713 return 714 self.fail('path "%s" has value "%s"' % (path, str(result))) 715 716 def assert_qmp(self, d, path, value): 717 '''Assert that the value for a specific path in a QMP dict 718 matches. When given a list of values, assert that any of 719 them matches.''' 720 721 result = self.dictpath(d, path) 722 723 # [] makes no sense as a list of valid values, so treat it as 724 # an actual single value. 725 if isinstance(value, list) and value != []: 726 for v in value: 727 if result == v: 728 return 729 self.fail('no match for "%s" in %s' % (str(result), str(value))) 730 else: 731 self.assertEqual(result, value, 732 'values not equal "%s" and "%s"' 733 % (str(result), str(value))) 734 735 def assert_no_active_block_jobs(self): 736 result = self.vm.qmp('query-block-jobs') 737 self.assert_qmp(result, 'return', []) 738 739 def assert_has_block_node(self, node_name=None, file_name=None): 740 """Issue a query-named-block-nodes and assert node_name and/or 741 file_name is present in the result""" 742 def check_equal_or_none(a, b): 743 return a == None or b == None or a == b 744 assert node_name or file_name 745 result = self.vm.qmp('query-named-block-nodes') 746 for x in result["return"]: 747 if check_equal_or_none(x.get("node-name"), node_name) and \ 748 check_equal_or_none(x.get("file"), file_name): 749 return 750 self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \ 751 (node_name, file_name, result)) 752 753 def assert_json_filename_equal(self, json_filename, reference): 754 '''Asserts that the given filename is a json: filename and that its 755 content is equal to the given reference object''' 756 self.assertEqual(json_filename[:5], 'json:') 757 self.assertEqual(self.vm.flatten_qmp_object(json.loads(json_filename[5:])), 758 self.vm.flatten_qmp_object(reference)) 759 760 def cancel_and_wait(self, drive='drive0', force=False, resume=False, wait=60.0): 761 '''Cancel a block job and wait for it to finish, returning the event''' 762 result = self.vm.qmp('block-job-cancel', device=drive, force=force) 763 self.assert_qmp(result, 'return', {}) 764 765 if resume: 766 self.vm.resume_drive(drive) 767 768 cancelled = False 769 result = None 770 while not cancelled: 771 for event in self.vm.get_qmp_events(wait=wait): 772 if event['event'] == 'BLOCK_JOB_COMPLETED' or \ 773 event['event'] == 'BLOCK_JOB_CANCELLED': 774 self.assert_qmp(event, 'data/device', drive) 775 result = event 776 cancelled = True 777 elif event['event'] == 'JOB_STATUS_CHANGE': 778 self.assert_qmp(event, 'data/id', drive) 779 780 781 self.assert_no_active_block_jobs() 782 return result 783 784 def wait_until_completed(self, drive='drive0', check_offset=True, wait=60.0): 785 '''Wait for a block job to finish, returning the event''' 786 while True: 787 for event in self.vm.get_qmp_events(wait=wait): 788 if event['event'] == 'BLOCK_JOB_COMPLETED': 789 self.assert_qmp(event, 'data/device', drive) 790 self.assert_qmp_absent(event, 'data/error') 791 if check_offset: 792 self.assert_qmp(event, 'data/offset', event['data']['len']) 793 self.assert_no_active_block_jobs() 794 return event 795 elif event['event'] == 'JOB_STATUS_CHANGE': 796 self.assert_qmp(event, 'data/id', drive) 797 798 def wait_ready(self, drive='drive0'): 799 '''Wait until a block job BLOCK_JOB_READY event''' 800 f = {'data': {'type': 'mirror', 'device': drive } } 801 event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f) 802 803 def wait_ready_and_cancel(self, drive='drive0'): 804 self.wait_ready(drive=drive) 805 event = self.cancel_and_wait(drive=drive) 806 self.assertEqual(event['event'], 'BLOCK_JOB_COMPLETED') 807 self.assert_qmp(event, 'data/type', 'mirror') 808 self.assert_qmp(event, 'data/offset', event['data']['len']) 809 810 def complete_and_wait(self, drive='drive0', wait_ready=True): 811 '''Complete a block job and wait for it to finish''' 812 if wait_ready: 813 self.wait_ready(drive=drive) 814 815 result = self.vm.qmp('block-job-complete', device=drive) 816 self.assert_qmp(result, 'return', {}) 817 818 event = self.wait_until_completed(drive=drive) 819 self.assert_qmp(event, 'data/type', 'mirror') 820 821 def pause_wait(self, job_id='job0'): 822 with Timeout(1, "Timeout waiting for job to pause"): 823 while True: 824 result = self.vm.qmp('query-block-jobs') 825 found = False 826 for job in result['return']: 827 if job['device'] == job_id: 828 found = True 829 if job['paused'] == True and job['busy'] == False: 830 return job 831 break 832 assert found 833 834 def pause_job(self, job_id='job0', wait=True): 835 result = self.vm.qmp('block-job-pause', device=job_id) 836 self.assert_qmp(result, 'return', {}) 837 if wait: 838 return self.pause_wait(job_id) 839 return result 840 841 842def notrun(reason): 843 '''Skip this test suite''' 844 # Each test in qemu-iotests has a number ("seq") 845 seq = os.path.basename(sys.argv[0]) 846 847 open('%s/%s.notrun' % (output_dir, seq), 'w').write(reason + '\n') 848 print('%s not run: %s' % (seq, reason)) 849 sys.exit(0) 850 851def case_notrun(reason): 852 '''Skip this test case''' 853 # Each test in qemu-iotests has a number ("seq") 854 seq = os.path.basename(sys.argv[0]) 855 856 open('%s/%s.casenotrun' % (output_dir, seq), 'a').write( 857 ' [case not run] ' + reason + '\n') 858 859def verify_image_format(supported_fmts=[], unsupported_fmts=[]): 860 assert not (supported_fmts and unsupported_fmts) 861 862 if 'generic' in supported_fmts and \ 863 os.environ.get('IMGFMT_GENERIC', 'true') == 'true': 864 # similar to 865 # _supported_fmt generic 866 # for bash tests 867 return 868 869 not_sup = supported_fmts and (imgfmt not in supported_fmts) 870 if not_sup or (imgfmt in unsupported_fmts): 871 notrun('not suitable for this image format: %s' % imgfmt) 872 873def verify_protocol(supported=[], unsupported=[]): 874 assert not (supported and unsupported) 875 876 if 'generic' in supported: 877 return 878 879 not_sup = supported and (imgproto not in supported) 880 if not_sup or (imgproto in unsupported): 881 notrun('not suitable for this protocol: %s' % imgproto) 882 883def verify_platform(supported_oses=['linux']): 884 if True not in [sys.platform.startswith(x) for x in supported_oses]: 885 notrun('not suitable for this OS: %s' % sys.platform) 886 887def verify_cache_mode(supported_cache_modes=[]): 888 if supported_cache_modes and (cachemode not in supported_cache_modes): 889 notrun('not suitable for this cache mode: %s' % cachemode) 890 891def supports_quorum(): 892 return 'quorum' in qemu_img_pipe('--help') 893 894def verify_quorum(): 895 '''Skip test suite if quorum support is not available''' 896 if not supports_quorum(): 897 notrun('quorum support missing') 898 899def qemu_pipe(*args): 900 '''Run qemu with an option to print something and exit (e.g. a help option), 901 and return its output''' 902 args = [qemu_prog] + qemu_opts + list(args) 903 subp = subprocess.Popen(args, stdout=subprocess.PIPE, 904 stderr=subprocess.STDOUT, 905 universal_newlines=True) 906 exitcode = subp.wait() 907 if exitcode < 0: 908 sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, 909 ' '.join(args))) 910 return subp.communicate()[0] 911 912def supported_formats(read_only=False): 913 '''Set 'read_only' to True to check ro-whitelist 914 Otherwise, rw-whitelist is checked''' 915 format_message = qemu_pipe("-drive", "format=help") 916 line = 1 if read_only else 0 917 return format_message.splitlines()[line].split(":")[1].split() 918 919def skip_if_unsupported(required_formats=[], read_only=False): 920 '''Skip Test Decorator 921 Runs the test if all the required formats are whitelisted''' 922 def skip_test_decorator(func): 923 def func_wrapper(*args, **kwargs): 924 usf_list = list(set(required_formats) - 925 set(supported_formats(read_only))) 926 if usf_list: 927 case_notrun('{}: formats {} are not whitelisted'.format( 928 args[0], usf_list)) 929 else: 930 return func(*args, **kwargs) 931 return func_wrapper 932 return skip_test_decorator 933 934def skip_if_user_is_root(func): 935 '''Skip Test Decorator 936 Runs the test only without root permissions''' 937 def func_wrapper(*args, **kwargs): 938 if os.getuid() == 0: 939 case_notrun('{}: cannot be run as root'.format(args[0])) 940 else: 941 return func(*args, **kwargs) 942 return func_wrapper 943 944def execute_unittest(output, verbosity, debug): 945 runner = unittest.TextTestRunner(stream=output, descriptions=True, 946 verbosity=verbosity) 947 try: 948 # unittest.main() will use sys.exit(); so expect a SystemExit 949 # exception 950 unittest.main(testRunner=runner) 951 finally: 952 if not debug: 953 sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', 954 r'Ran \1 tests', output.getvalue())) 955 956def execute_test(test_function=None, 957 supported_fmts=[], supported_oses=['linux'], 958 supported_cache_modes=[], unsupported_fmts=[], 959 supported_protocols=[], unsupported_protocols=[]): 960 """Run either unittest or script-style tests.""" 961 962 # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to 963 # indicate that we're not being run via "check". There may be 964 # other things set up by "check" that individual test cases rely 965 # on. 966 if test_dir is None or qemu_default_machine is None: 967 sys.stderr.write('Please run this test via the "check" script\n') 968 sys.exit(os.EX_USAGE) 969 970 debug = '-d' in sys.argv 971 verbosity = 1 972 verify_image_format(supported_fmts, unsupported_fmts) 973 verify_protocol(supported_protocols, unsupported_protocols) 974 verify_platform(supported_oses) 975 verify_cache_mode(supported_cache_modes) 976 977 if debug: 978 output = sys.stdout 979 verbosity = 2 980 sys.argv.remove('-d') 981 else: 982 # We need to filter out the time taken from the output so that 983 # qemu-iotest can reliably diff the results against master output. 984 output = io.StringIO() 985 986 logging.basicConfig(level=(logging.DEBUG if debug else logging.WARN)) 987 988 if not test_function: 989 execute_unittest(output, verbosity, debug) 990 else: 991 test_function() 992 993def script_main(test_function, *args, **kwargs): 994 """Run script-style tests outside of the unittest framework""" 995 execute_test(test_function, *args, **kwargs) 996 997def main(*args, **kwargs): 998 """Run tests using the unittest framework""" 999 execute_test(None, *args, **kwargs) 1000