1#!/usr/bin/env python 2# 3# Copyright (c) 2017 Red Hat Inc 4# 5# Author: 6# Eduardo Habkost <ehabkost@redhat.com> 7# 8# This program is free software; you can redistribute it and/or modify 9# it under the terms of the GNU General Public License as published by 10# the Free Software Foundation; either version 2 of the License, or 11# (at your option) any later version. 12# 13# This program is distributed in the hope that it will be useful, 14# but WITHOUT ANY WARRANTY; without even the implied warranty of 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16# GNU General Public License for more details. 17# 18# You should have received a copy of the GNU General Public License along 19# with this program; if not, write to the Free Software Foundation, Inc., 20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 21 22""" 23Run QEMU with all combinations of -machine and -device types, 24check for crashes and unexpected errors. 25""" 26from __future__ import print_function 27 28import sys 29import os 30import glob 31import logging 32import traceback 33import re 34import random 35import argparse 36from itertools import chain 37 38from qemu import QEMUMachine 39 40logger = logging.getLogger('device-crash-test') 41dbg = logger.debug 42 43 44# Purposes of the following whitelist: 45# * Avoiding verbose log messages when we find known non-fatal 46# (exitcode=1) errors 47# * Avoiding fatal errors when we find known crashes 48# * Skipping machines/devices that are known not to work out of 49# the box, when running in --quick mode 50# 51# Keeping the whitelist updated is desirable, but not required, 52# because unexpected cases where QEMU exits with exitcode=1 will 53# just trigger a INFO message. 54 55# Valid whitelist entry keys: 56# * accel: regexp, full match only 57# * machine: regexp, full match only 58# * device: regexp, full match only 59# * log: regexp, partial match allowed 60# * exitcode: if not present, defaults to 1. If None, matches any exitcode 61# * warn: if True, matching failures will be logged as warnings 62# * expected: if True, QEMU is expected to always fail every time 63# when testing the corresponding test case 64# * loglevel: log level of log output when there's a match. 65ERROR_WHITELIST = [ 66 # Machines that won't work out of the box: 67 # MACHINE | ERROR MESSAGE 68 {'machine':'niagara', 'expected':True}, # Unable to load a firmware for -M niagara 69 {'machine':'boston', 'expected':True}, # Please provide either a -kernel or -bios argument 70 {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null) 71 72 # devices that don't work out of the box because they require extra options to "-device DEV": 73 # DEVICE | ERROR MESSAGE 74 {'device':'.*-(i386|x86_64)-cpu', 'expected':True}, # CPU socket-id is not set 75 {'device':'icp', 'expected':True}, # icp_realize: required link 'xics' not found: Property '.xics' not found 76 {'device':'ics', 'expected':True}, # ics_base_realize: required link 'xics' not found: Property '.xics' not found 77 # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True 78 {'device':'ide-cd'}, # No drive specified 79 {'device':'ide-drive', 'expected':True}, # No drive specified 80 {'device':'ide-hd', 'expected':True}, # No drive specified 81 {'device':'ipmi-bmc-extern', 'expected':True}, # IPMI external bmc requires chardev attribute 82 {'device':'isa-debugcon', 'expected':True}, # Can't create serial device, empty char device 83 {'device':'isa-ipmi-bt', 'expected':True}, # IPMI device requires a bmc attribute to be set 84 {'device':'isa-ipmi-kcs', 'expected':True}, # IPMI device requires a bmc attribute to be set 85 {'device':'isa-parallel', 'expected':True}, # Can't create serial device, empty char device 86 {'device':'ivshmem', 'expected':True}, # You must specify either 'shm' or 'chardev' 87 {'device':'ivshmem-doorbell', 'expected':True}, # You must specify a 'chardev' 88 {'device':'ivshmem-plain', 'expected':True}, # You must specify a 'memdev' 89 {'device':'loader', 'expected':True}, # please include valid arguments 90 {'device':'nand', 'expected':True}, # Unsupported NAND block size 0x1 91 {'device':'nvdimm', 'expected':True}, # 'memdev' property is not set 92 {'device':'nvme', 'expected':True}, # Device initialization failed 93 {'device':'pc-dimm', 'expected':True}, # 'memdev' property is not set 94 {'device':'pci-bridge', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. 95 {'device':'pci-bridge-seat', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. 96 {'device':'pxb', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. 97 {'device':'scsi-block', 'expected':True}, # drive property not set 98 {'device':'scsi-disk', 'expected':True}, # drive property not set 99 {'device':'scsi-generic', 'expected':True}, # drive property not set 100 {'device':'scsi-hd', 'expected':True}, # drive property not set 101 {'device':'spapr-pci-host-bridge', 'expected':True}, # BUID not specified for PHB 102 {'device':'spapr-rng', 'expected':True}, # spapr-rng needs an RNG backend! 103 {'device':'spapr-vty', 'expected':True}, # chardev property not set 104 {'device':'tpm-tis', 'expected':True}, # tpm_tis: backend driver with id (null) could not be found 105 {'device':'unimplemented-device', 'expected':True}, # property 'size' not specified or zero 106 {'device':'usb-braille', 'expected':True}, # Property chardev is required 107 {'device':'usb-mtp', 'expected':True}, # rootdir property must be configured 108 {'device':'usb-redir', 'expected':True}, # Parameter 'chardev' is missing 109 {'device':'usb-serial', 'expected':True}, # Property chardev is required 110 {'device':'usb-storage', 'expected':True}, # drive property not set 111 {'device':'vfio-amd-xgbe', 'expected':True}, # -device vfio-amd-xgbe: vfio error: wrong host device name 112 {'device':'vfio-calxeda-xgmac', 'expected':True}, # -device vfio-calxeda-xgmac: vfio error: wrong host device name 113 {'device':'vfio-pci', 'expected':True}, # No provided host device 114 {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0 115 {'device':'vhost-scsi.*', 'expected':True}, # vhost-scsi: missing wwpn 116 {'device':'vhost-vsock-device', 'expected':True}, # guest-cid property must be greater than 2 117 {'device':'vhost-vsock-pci', 'expected':True}, # guest-cid property must be greater than 2 118 {'device':'virtio-9p-ccw', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL 119 {'device':'virtio-9p-device', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL 120 {'device':'virtio-9p-pci', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL 121 {'device':'virtio-blk-ccw', 'expected':True}, # drive property not set 122 {'device':'virtio-blk-device', 'expected':True}, # drive property not set 123 {'device':'virtio-blk-device', 'expected':True}, # drive property not set 124 {'device':'virtio-blk-pci', 'expected':True}, # drive property not set 125 {'device':'virtio-crypto-ccw', 'expected':True}, # 'cryptodev' parameter expects a valid object 126 {'device':'virtio-crypto-device', 'expected':True}, # 'cryptodev' parameter expects a valid object 127 {'device':'virtio-crypto-pci', 'expected':True}, # 'cryptodev' parameter expects a valid object 128 {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required 129 {'device':'virtio-input-host-pci', 'expected':True}, # evdev property is required 130 {'device':'xen-pvdevice', 'expected':True}, # Device ID invalid, it must always be supplied 131 {'device':'vhost-vsock-ccw', 'expected':True}, # guest-cid property must be greater than 2 132 {'device':'zpci', 'expected':True}, # target must be defined 133 {'device':'pnv-(occ|icp|lpc)', 'expected':True}, # required link 'xics' not found: Property '.xics' not found 134 {'device':'powernv-cpu-.*', 'expected':True}, # pnv_core_realize: required link 'xics' not found: Property '.xics' not found 135 136 # ioapic devices are already created by pc and will fail: 137 {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed 138 {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True}, # Only 1 ioapics allowed 139 140 # "spapr-cpu-core needs a pseries machine" 141 {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True}, 142 143 # KVM-specific devices shouldn't be tried without accel=kvm: 144 {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True}, 145 146 # xen-specific machines and devices: 147 {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True}, 148 {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True}, 149 150 # this fails on some machine-types, but not all, so they don't have expected=True: 151 {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide 152 153 # Silence INFO messages for errors that are common on multiple 154 # devices/machines: 155 {'log':r"No '[\w-]+' bus found for device '[\w-]+'"}, 156 {'log':r"images* must be given with the 'pflash' parameter"}, 157 {'log':r"(Guest|ROM|Flash|Kernel) image must be specified"}, 158 {'log':r"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"}, 159 {'log':r"Couldn't find rom image '[\w-]+\.bin'"}, 160 {'log':r"speed mismatch trying to attach usb device"}, 161 {'log':r"Can't create a second ISA bus"}, 162 {'log':r"duplicate fw_cfg file name"}, 163 # sysbus-related error messages: most machines reject most dynamic sysbus devices: 164 {'log':r"Option '-device [\w.,-]+' cannot be handled by this machine"}, 165 {'log':r"Device [\w.,-]+ is not supported by this machine yet"}, 166 {'log':r"Device [\w.,-]+ can not be dynamically instantiated"}, 167 {'log':r"Platform Bus: Can not fit MMIO region of size "}, 168 # other more specific errors we will ignore: 169 {'device':'.*-spapr-cpu-core', 'log':r"CPU core type should be"}, 170 {'log':r"MSI(-X)? is not supported by interrupt controller"}, 171 {'log':r"pxb-pcie? devices cannot reside on a PCIe? bus"}, 172 {'log':r"Ignoring smp_cpus value"}, 173 {'log':r"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"}, 174 {'log':r"This CPU requires a smaller page size than the system is using"}, 175 {'log':r"MSI-X support is mandatory in the S390 architecture"}, 176 {'log':r"rom check and register reset failed"}, 177 {'log':r"Unable to initialize GIC, CPUState for CPU#0 not valid"}, 178 {'log':r"Multiple VT220 operator consoles are not supported"}, 179 {'log':r"core 0 already populated"}, 180 {'log':r"could not find stage1 bootloader"}, 181 182 # other exitcode=1 failures not listed above will just generate INFO messages: 183 {'exitcode':1, 'loglevel':logging.INFO}, 184 185 # KNOWN CRASHES: 186 # Known crashes will generate error messages, but won't be fatal. 187 # Those entries must be removed once we fix the crashes. 188 {'exitcode':-6, 'log':r"Device 'serial0' is in use", 'loglevel':logging.ERROR}, 189 {'exitcode':-6, 'log':r"qemu_net_client_setup: Assertion `!peer->peer' failed", 'loglevel':logging.ERROR}, 190 {'exitcode':-6, 'log':r'RAMBlock "[\w.-]+" already registered', 'loglevel':logging.ERROR}, 191 {'exitcode':-6, 'log':r"find_ram_offset: Assertion `size != 0' failed.", 'loglevel':logging.ERROR}, 192 {'exitcode':-6, 'log':r"add_cpreg_to_hashtable: code should not be reached", 'loglevel':logging.ERROR}, 193 {'exitcode':-6, 'log':r"qemu_alloc_display: Assertion `surface->image != NULL' failed", 'loglevel':logging.ERROR}, 194 {'exitcode':-6, 'log':r"Unexpected error in error_set_from_qdev_prop_error", 'loglevel':logging.ERROR}, 195 {'exitcode':-6, 'log':r"Object .* is not an instance of type spapr-machine", 'loglevel':logging.ERROR}, 196 {'exitcode':-6, 'log':r"Object .* is not an instance of type generic-pc-machine", 'loglevel':logging.ERROR}, 197 {'exitcode':-6, 'log':r"Object .* is not an instance of type e500-ccsr", 'loglevel':logging.ERROR}, 198 {'exitcode':-6, 'log':r"vmstate_register_with_alias_id: Assertion `!se->compat \|\| se->instance_id == 0' failed", 'loglevel':logging.ERROR}, 199 200 # everything else (including SIGABRT and SIGSEGV) will be a fatal error: 201 {'exitcode':None, 'fatal':True, 'loglevel':logging.FATAL}, 202] 203 204 205def whitelistTestCaseMatch(wl, t): 206 """Check if a test case specification can match a whitelist entry 207 208 This only checks if a whitelist entry is a candidate match 209 for a given test case, it won't check if the test case 210 results/output match the entry. See whitelistResultMatch(). 211 """ 212 return (('machine' not in wl or 213 'machine' not in t or 214 re.match(wl['machine'] + '$', t['machine'])) and 215 ('accel' not in wl or 216 'accel' not in t or 217 re.match(wl['accel'] + '$', t['accel'])) and 218 ('device' not in wl or 219 'device' not in t or 220 re.match(wl['device'] + '$', t['device']))) 221 222 223def whitelistCandidates(t): 224 """Generate the list of candidates that can match a test case""" 225 for i, wl in enumerate(ERROR_WHITELIST): 226 if whitelistTestCaseMatch(wl, t): 227 yield (i, wl) 228 229 230def findExpectedResult(t): 231 """Check if there's an expected=True whitelist entry for a test case 232 233 Returns (i, wl) tuple, where i is the index in 234 ERROR_WHITELIST and wl is the whitelist entry itself. 235 """ 236 for i, wl in whitelistCandidates(t): 237 if wl.get('expected'): 238 return (i, wl) 239 240 241def whitelistResultMatch(wl, r): 242 """Check if test case results/output match a whitelist entry 243 244 It is valid to call this function only if 245 whitelistTestCaseMatch() is True for the entry (e.g. on 246 entries returned by whitelistCandidates()) 247 """ 248 assert whitelistTestCaseMatch(wl, r['testcase']) 249 return ((wl.get('exitcode', 1) is None or 250 r['exitcode'] == wl.get('exitcode', 1)) and 251 ('log' not in wl or 252 re.search(wl['log'], r['log'], re.MULTILINE))) 253 254 255def checkResultWhitelist(r): 256 """Look up whitelist entry for a given test case result 257 258 Returns (i, wl) tuple, where i is the index in 259 ERROR_WHITELIST and wl is the whitelist entry itself. 260 """ 261 for i, wl in whitelistCandidates(r['testcase']): 262 if whitelistResultMatch(wl, r): 263 return i, wl 264 265 raise Exception("this should never happen") 266 267 268def qemuOptsEscape(s): 269 """Escape option value QemuOpts""" 270 return s.replace(",", ",,") 271 272 273def formatTestCase(t): 274 """Format test case info as "key=value key=value" for prettier logging output""" 275 return ' '.join('%s=%s' % (k, v) for k, v in t.items()) 276 277 278def qomListTypeNames(vm, **kwargs): 279 """Run qom-list-types QMP command, return type names""" 280 types = vm.command('qom-list-types', **kwargs) 281 return [t['name'] for t in types] 282 283 284def infoQDM(vm): 285 """Parse 'info qdm' output""" 286 args = {'command-line': 'info qdm'} 287 devhelp = vm.command('human-monitor-command', **args) 288 for l in devhelp.split('\n'): 289 l = l.strip() 290 if l == '' or l.endswith(':'): 291 continue 292 d = {'name': re.search(r'name "([^"]+)"', l).group(1), 293 'no-user': (re.search(', no-user', l) is not None)} 294 yield d 295 296 297class QemuBinaryInfo(object): 298 def __init__(self, binary, devtype): 299 if devtype is None: 300 devtype = 'device' 301 302 self.binary = binary 303 self._machine_info = {} 304 305 dbg("devtype: %r", devtype) 306 args = ['-S', '-machine', 'none,accel=kvm:tcg'] 307 dbg("querying info for QEMU binary: %s", binary) 308 vm = QEMUMachine(binary=binary, args=args) 309 vm.launch() 310 try: 311 self.alldevs = set(qomListTypeNames(vm, implements=devtype, abstract=False)) 312 # there's no way to query DeviceClass::user_creatable using QMP, 313 # so use 'info qdm': 314 self.no_user_devs = set([d['name'] for d in infoQDM(vm, ) if d['no-user']]) 315 self.machines = list(m['name'] for m in vm.command('query-machines')) 316 self.user_devs = self.alldevs.difference(self.no_user_devs) 317 self.kvm_available = vm.command('query-kvm')['enabled'] 318 finally: 319 vm.shutdown() 320 321 def machineInfo(self, machine): 322 """Query for information on a specific machine-type 323 324 Results are cached internally, in case the same machine- 325 type is queried multiple times. 326 """ 327 if machine in self._machine_info: 328 return self._machine_info[machine] 329 330 mi = {} 331 args = ['-S', '-machine', '%s' % (machine)] 332 dbg("querying machine info for binary=%s machine=%s", self.binary, machine) 333 vm = QEMUMachine(binary=self.binary, args=args) 334 try: 335 vm.launch() 336 mi['runnable'] = True 337 except KeyboardInterrupt: 338 raise 339 except: 340 dbg("exception trying to run binary=%s machine=%s", self.binary, machine, exc_info=sys.exc_info()) 341 dbg("log: %r", vm.get_log()) 342 mi['runnable'] = False 343 344 vm.shutdown() 345 self._machine_info[machine] = mi 346 return mi 347 348 349BINARY_INFO = {} 350 351 352def getBinaryInfo(args, binary): 353 if binary not in BINARY_INFO: 354 BINARY_INFO[binary] = QemuBinaryInfo(binary, args.devtype) 355 return BINARY_INFO[binary] 356 357 358def checkOneCase(args, testcase): 359 """Check one specific case 360 361 Returns a dictionary containing failure information on error, 362 or None on success 363 """ 364 binary = testcase['binary'] 365 accel = testcase['accel'] 366 machine = testcase['machine'] 367 device = testcase['device'] 368 369 dbg("will test: %r", testcase) 370 371 args = ['-S', '-machine', '%s,accel=%s' % (machine, accel), 372 '-device', qemuOptsEscape(device)] 373 cmdline = ' '.join([binary] + args) 374 dbg("will launch QEMU: %s", cmdline) 375 vm = QEMUMachine(binary=binary, args=args) 376 377 exc_traceback = None 378 try: 379 vm.launch() 380 except KeyboardInterrupt: 381 raise 382 except: 383 exc_traceback = traceback.format_exc() 384 dbg("Exception while running test case") 385 finally: 386 vm.shutdown() 387 ec = vm.exitcode() 388 log = vm.get_log() 389 390 if exc_traceback is not None or ec != 0: 391 return {'exc_traceback':exc_traceback, 392 'exitcode':ec, 393 'log':log, 394 'testcase':testcase, 395 'cmdline':cmdline} 396 397 398def binariesToTest(args, testcase): 399 if args.qemu: 400 r = args.qemu 401 else: 402 r = glob.glob('./*-softmmu/qemu-system-*') 403 return r 404 405 406def accelsToTest(args, testcase): 407 if getBinaryInfo(args, testcase['binary']).kvm_available: 408 yield 'kvm' 409 yield 'tcg' 410 411 412def machinesToTest(args, testcase): 413 return getBinaryInfo(args, testcase['binary']).machines 414 415 416def devicesToTest(args, testcase): 417 return getBinaryInfo(args, testcase['binary']).user_devs 418 419 420TESTCASE_VARIABLES = [ 421 ('binary', binariesToTest), 422 ('accel', accelsToTest), 423 ('machine', machinesToTest), 424 ('device', devicesToTest), 425] 426 427 428def genCases1(args, testcases, var, fn): 429 """Generate new testcases for one variable 430 431 If an existing item already has a variable set, don't 432 generate new items and just return it directly. This 433 allows the "-t" command-line option to be used to choose 434 a specific test case. 435 """ 436 for testcase in testcases: 437 if var in testcase: 438 yield testcase.copy() 439 else: 440 for i in fn(args, testcase): 441 t = testcase.copy() 442 t[var] = i 443 yield t 444 445 446def genCases(args, testcase): 447 """Generate test cases for all variables 448 """ 449 cases = [testcase.copy()] 450 for var, fn in TESTCASE_VARIABLES: 451 dbg("var: %r, fn: %r", var, fn) 452 cases = genCases1(args, cases, var, fn) 453 return cases 454 455 456def casesToTest(args, testcase): 457 cases = genCases(args, testcase) 458 if args.random: 459 cases = list(cases) 460 cases = random.sample(cases, min(args.random, len(cases))) 461 if args.debug: 462 cases = list(cases) 463 dbg("%d test cases to test", len(cases)) 464 if args.shuffle: 465 cases = list(cases) 466 random.shuffle(cases) 467 return cases 468 469 470def logFailure(f, level): 471 t = f['testcase'] 472 logger.log(level, "failed: %s", formatTestCase(t)) 473 logger.log(level, "cmdline: %s", f['cmdline']) 474 for l in f['log'].strip().split('\n'): 475 logger.log(level, "log: %s", l) 476 logger.log(level, "exit code: %r", f['exitcode']) 477 if f['exc_traceback']: 478 logger.log(level, "exception:") 479 for l in f['exc_traceback'].split('\n'): 480 logger.log(level, " %s", l.rstrip('\n')) 481 482 483def main(): 484 parser = argparse.ArgumentParser(description="QEMU -device crash test") 485 parser.add_argument('-t', metavar='KEY=VALUE', nargs='*', 486 help="Limit test cases to KEY=VALUE", 487 action='append', dest='testcases', default=[]) 488 parser.add_argument('-d', '--debug', action='store_true', 489 help='debug output') 490 parser.add_argument('-v', '--verbose', action='store_true', default=True, 491 help='verbose output') 492 parser.add_argument('-q', '--quiet', dest='verbose', action='store_false', 493 help='non-verbose output') 494 parser.add_argument('-r', '--random', type=int, metavar='COUNT', 495 help='run a random sample of COUNT test cases', 496 default=0) 497 parser.add_argument('--shuffle', action='store_true', 498 help='Run test cases in random order') 499 parser.add_argument('--dry-run', action='store_true', 500 help="Don't run any tests, just generate list") 501 parser.add_argument('-D', '--devtype', metavar='TYPE', 502 help="Test only device types that implement TYPE") 503 parser.add_argument('-Q', '--quick', action='store_true', default=True, 504 help="Quick mode: skip test cases that are expected to fail") 505 parser.add_argument('-F', '--full', action='store_false', dest='quick', 506 help="Full mode: test cases that are expected to fail") 507 parser.add_argument('--strict', action='store_true', dest='strict', 508 help="Treat all warnings as fatal") 509 parser.add_argument('qemu', nargs='*', metavar='QEMU', 510 help='QEMU binary to run') 511 args = parser.parse_args() 512 513 if args.debug: 514 lvl = logging.DEBUG 515 elif args.verbose: 516 lvl = logging.INFO 517 else: 518 lvl = logging.WARN 519 logging.basicConfig(stream=sys.stdout, level=lvl, format='%(levelname)s: %(message)s') 520 521 fatal_failures = [] 522 wl_stats = {} 523 skipped = 0 524 total = 0 525 526 tc = {} 527 dbg("testcases: %r", args.testcases) 528 if args.testcases: 529 for t in chain(*args.testcases): 530 for kv in t.split(): 531 k, v = kv.split('=', 1) 532 tc[k] = v 533 534 if len(binariesToTest(args, tc)) == 0: 535 print("No QEMU binary found", file=sys.stderr) 536 parser.print_usage(sys.stderr) 537 return 1 538 539 for t in casesToTest(args, tc): 540 logger.info("running test case: %s", formatTestCase(t)) 541 total += 1 542 543 expected_match = findExpectedResult(t) 544 if (args.quick and 545 (expected_match or 546 not getBinaryInfo(args, t['binary']).machineInfo(t['machine'])['runnable'])): 547 dbg("skipped: %s", formatTestCase(t)) 548 skipped += 1 549 continue 550 551 if args.dry_run: 552 continue 553 554 try: 555 f = checkOneCase(args, t) 556 except KeyboardInterrupt: 557 break 558 559 if f: 560 i, wl = checkResultWhitelist(f) 561 dbg("testcase: %r, whitelist match: %r", t, wl) 562 wl_stats.setdefault(i, []).append(f) 563 level = wl.get('loglevel', logging.DEBUG) 564 logFailure(f, level) 565 if wl.get('fatal') or (args.strict and level >= logging.WARN): 566 fatal_failures.append(f) 567 else: 568 dbg("success: %s", formatTestCase(t)) 569 if expected_match: 570 logger.warn("Didn't fail as expected: %s", formatTestCase(t)) 571 572 logger.info("Total: %d test cases", total) 573 if skipped: 574 logger.info("Skipped %d test cases", skipped) 575 576 if args.debug: 577 stats = sorted([(len(wl_stats.get(i, [])), wl) for i, wl in enumerate(ERROR_WHITELIST)]) 578 for count, wl in stats: 579 dbg("whitelist entry stats: %d: %r", count, wl) 580 581 if fatal_failures: 582 for f in fatal_failures: 583 t = f['testcase'] 584 logger.error("Fatal failure: %s", formatTestCase(t)) 585 logger.error("Fatal failures on some machine/device combinations") 586 return 1 587 588if __name__ == '__main__': 589 sys.exit(main()) 590