183d290c5STom Rini# SPDX-License-Identifier: GPL-2.0 2d201506cSStephen Warren# Copyright (c) 2015 Stephen Warren 3d201506cSStephen Warren# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. 4d201506cSStephen Warren 5d201506cSStephen Warren# Implementation of pytest run-time hook functions. These are invoked by 6d201506cSStephen Warren# pytest at certain points during operation, e.g. startup, for each executed 7d201506cSStephen Warren# test, at shutdown etc. These hooks perform functions such as: 8d201506cSStephen Warren# - Parsing custom command-line options. 9d201506cSStephen Warren# - Pullilng in user-specified board configuration. 10d201506cSStephen Warren# - Creating the U-Boot console test fixture. 11d201506cSStephen Warren# - Creating the HTML log file. 12d201506cSStephen Warren# - Monitoring each test's results. 13d201506cSStephen Warren# - Implementing custom pytest markers. 14d201506cSStephen Warren 15d201506cSStephen Warrenimport atexit 16d201506cSStephen Warrenimport errno 17d201506cSStephen Warrenimport os 18d201506cSStephen Warrenimport os.path 19d201506cSStephen Warrenimport pytest 20d201506cSStephen Warrenfrom _pytest.runner import runtestprotocol 21d201506cSStephen Warrenimport ConfigParser 221cd85f57SStephen Warrenimport re 23d201506cSStephen Warrenimport StringIO 24d201506cSStephen Warrenimport sys 25d201506cSStephen Warren 26d201506cSStephen Warren# Globals: The HTML log file, and the connection to the U-Boot console. 27d201506cSStephen Warrenlog = None 28d201506cSStephen Warrenconsole = None 29d201506cSStephen Warren 30d201506cSStephen Warrendef mkdir_p(path): 31e8debf39SStephen Warren """Create a directory path. 32d201506cSStephen Warren 33d201506cSStephen Warren This includes creating any intermediate/parent directories. Any errors 34d201506cSStephen Warren caused due to already extant directories are ignored. 35d201506cSStephen Warren 36d201506cSStephen Warren Args: 37d201506cSStephen Warren path: The directory path to create. 38d201506cSStephen Warren 39d201506cSStephen Warren Returns: 40d201506cSStephen Warren Nothing. 41e8debf39SStephen Warren """ 42d201506cSStephen Warren 43d201506cSStephen Warren try: 44d201506cSStephen Warren os.makedirs(path) 45d201506cSStephen Warren except OSError as exc: 46d201506cSStephen Warren if exc.errno == errno.EEXIST and os.path.isdir(path): 47d201506cSStephen Warren pass 48d201506cSStephen Warren else: 49d201506cSStephen Warren raise 50d201506cSStephen Warren 51d201506cSStephen Warrendef pytest_addoption(parser): 52e8debf39SStephen Warren """pytest hook: Add custom command-line options to the cmdline parser. 53d201506cSStephen Warren 54d201506cSStephen Warren Args: 55d201506cSStephen Warren parser: The pytest command-line parser. 56d201506cSStephen Warren 57d201506cSStephen Warren Returns: 58d201506cSStephen Warren Nothing. 59e8debf39SStephen Warren """ 60d201506cSStephen Warren 61d201506cSStephen Warren parser.addoption('--build-dir', default=None, 62d201506cSStephen Warren help='U-Boot build directory (O=)') 63d201506cSStephen Warren parser.addoption('--result-dir', default=None, 64d201506cSStephen Warren help='U-Boot test result/tmp directory') 65d201506cSStephen Warren parser.addoption('--persistent-data-dir', default=None, 66d201506cSStephen Warren help='U-Boot test persistent generated data directory') 67d201506cSStephen Warren parser.addoption('--board-type', '--bd', '-B', default='sandbox', 68d201506cSStephen Warren help='U-Boot board type') 69d201506cSStephen Warren parser.addoption('--board-identity', '--id', default='na', 70d201506cSStephen Warren help='U-Boot board identity/instance') 71d201506cSStephen Warren parser.addoption('--build', default=False, action='store_true', 72d201506cSStephen Warren help='Compile U-Boot before running tests') 7389ab8410SStephen Warren parser.addoption('--gdbserver', default=None, 7489ab8410SStephen Warren help='Run sandbox under gdbserver. The argument is the channel '+ 7589ab8410SStephen Warren 'over which gdbserver should communicate, e.g. localhost:1234') 76d201506cSStephen Warren 77d201506cSStephen Warrendef pytest_configure(config): 78e8debf39SStephen Warren """pytest hook: Perform custom initialization at startup time. 79d201506cSStephen Warren 80d201506cSStephen Warren Args: 81d201506cSStephen Warren config: The pytest configuration. 82d201506cSStephen Warren 83d201506cSStephen Warren Returns: 84d201506cSStephen Warren Nothing. 85e8debf39SStephen Warren """ 86d201506cSStephen Warren 87d201506cSStephen Warren global log 88d201506cSStephen Warren global console 89d201506cSStephen Warren global ubconfig 90d201506cSStephen Warren 91d201506cSStephen Warren test_py_dir = os.path.dirname(os.path.abspath(__file__)) 92d201506cSStephen Warren source_dir = os.path.dirname(os.path.dirname(test_py_dir)) 93d201506cSStephen Warren 94d201506cSStephen Warren board_type = config.getoption('board_type') 95d201506cSStephen Warren board_type_filename = board_type.replace('-', '_') 96d201506cSStephen Warren 97d201506cSStephen Warren board_identity = config.getoption('board_identity') 98d201506cSStephen Warren board_identity_filename = board_identity.replace('-', '_') 99d201506cSStephen Warren 100d201506cSStephen Warren build_dir = config.getoption('build_dir') 101d201506cSStephen Warren if not build_dir: 102d201506cSStephen Warren build_dir = source_dir + '/build-' + board_type 103d201506cSStephen Warren mkdir_p(build_dir) 104d201506cSStephen Warren 105d201506cSStephen Warren result_dir = config.getoption('result_dir') 106d201506cSStephen Warren if not result_dir: 107d201506cSStephen Warren result_dir = build_dir 108d201506cSStephen Warren mkdir_p(result_dir) 109d201506cSStephen Warren 110d201506cSStephen Warren persistent_data_dir = config.getoption('persistent_data_dir') 111d201506cSStephen Warren if not persistent_data_dir: 112d201506cSStephen Warren persistent_data_dir = build_dir + '/persistent-data' 113d201506cSStephen Warren mkdir_p(persistent_data_dir) 114d201506cSStephen Warren 11589ab8410SStephen Warren gdbserver = config.getoption('gdbserver') 11689ab8410SStephen Warren if gdbserver and board_type != 'sandbox': 11789ab8410SStephen Warren raise Exception('--gdbserver only supported with sandbox') 11889ab8410SStephen Warren 119d201506cSStephen Warren import multiplexed_log 120d201506cSStephen Warren log = multiplexed_log.Logfile(result_dir + '/test-log.html') 121d201506cSStephen Warren 122d201506cSStephen Warren if config.getoption('build'): 123d201506cSStephen Warren if build_dir != source_dir: 124d201506cSStephen Warren o_opt = 'O=%s' % build_dir 125d201506cSStephen Warren else: 126d201506cSStephen Warren o_opt = '' 127d201506cSStephen Warren cmds = ( 128d201506cSStephen Warren ['make', o_opt, '-s', board_type + '_defconfig'], 129d201506cSStephen Warren ['make', o_opt, '-s', '-j8'], 130d201506cSStephen Warren ) 13183357fd5SStephen Warren with log.section('make'): 132d201506cSStephen Warren runner = log.get_runner('make', sys.stdout) 133d201506cSStephen Warren for cmd in cmds: 134d201506cSStephen Warren runner.run(cmd, cwd=source_dir) 135d201506cSStephen Warren runner.close() 13683357fd5SStephen Warren log.status_pass('OK') 137d201506cSStephen Warren 138d201506cSStephen Warren class ArbitraryAttributeContainer(object): 139d201506cSStephen Warren pass 140d201506cSStephen Warren 141d201506cSStephen Warren ubconfig = ArbitraryAttributeContainer() 142d201506cSStephen Warren ubconfig.brd = dict() 143d201506cSStephen Warren ubconfig.env = dict() 144d201506cSStephen Warren 145d201506cSStephen Warren modules = [ 146d201506cSStephen Warren (ubconfig.brd, 'u_boot_board_' + board_type_filename), 147d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename), 148d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' + 149d201506cSStephen Warren board_identity_filename), 150d201506cSStephen Warren ] 151d201506cSStephen Warren for (dict_to_fill, module_name) in modules: 152d201506cSStephen Warren try: 153d201506cSStephen Warren module = __import__(module_name) 154d201506cSStephen Warren except ImportError: 155d201506cSStephen Warren continue 156d201506cSStephen Warren dict_to_fill.update(module.__dict__) 157d201506cSStephen Warren 158d201506cSStephen Warren ubconfig.buildconfig = dict() 159d201506cSStephen Warren 160d201506cSStephen Warren for conf_file in ('.config', 'include/autoconf.mk'): 161d201506cSStephen Warren dot_config = build_dir + '/' + conf_file 162d201506cSStephen Warren if not os.path.exists(dot_config): 163d201506cSStephen Warren raise Exception(conf_file + ' does not exist; ' + 164d201506cSStephen Warren 'try passing --build option?') 165d201506cSStephen Warren 166d201506cSStephen Warren with open(dot_config, 'rt') as f: 167d201506cSStephen Warren ini_str = '[root]\n' + f.read() 168d201506cSStephen Warren ini_sio = StringIO.StringIO(ini_str) 169d201506cSStephen Warren parser = ConfigParser.RawConfigParser() 170d201506cSStephen Warren parser.readfp(ini_sio) 171d201506cSStephen Warren ubconfig.buildconfig.update(parser.items('root')) 172d201506cSStephen Warren 173d201506cSStephen Warren ubconfig.test_py_dir = test_py_dir 174d201506cSStephen Warren ubconfig.source_dir = source_dir 175d201506cSStephen Warren ubconfig.build_dir = build_dir 176d201506cSStephen Warren ubconfig.result_dir = result_dir 177d201506cSStephen Warren ubconfig.persistent_data_dir = persistent_data_dir 178d201506cSStephen Warren ubconfig.board_type = board_type 179d201506cSStephen Warren ubconfig.board_identity = board_identity 18089ab8410SStephen Warren ubconfig.gdbserver = gdbserver 1810671960bSSimon Glass ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb' 182d201506cSStephen Warren 183d201506cSStephen Warren env_vars = ( 184d201506cSStephen Warren 'board_type', 185d201506cSStephen Warren 'board_identity', 186d201506cSStephen Warren 'source_dir', 187d201506cSStephen Warren 'test_py_dir', 188d201506cSStephen Warren 'build_dir', 189d201506cSStephen Warren 'result_dir', 190d201506cSStephen Warren 'persistent_data_dir', 191d201506cSStephen Warren ) 192d201506cSStephen Warren for v in env_vars: 193d201506cSStephen Warren os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v) 194d201506cSStephen Warren 1952fedbaa4SSimon Glass if board_type.startswith('sandbox'): 196d201506cSStephen Warren import u_boot_console_sandbox 197d201506cSStephen Warren console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig) 198d201506cSStephen Warren else: 199d201506cSStephen Warren import u_boot_console_exec_attach 200d201506cSStephen Warren console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig) 201d201506cSStephen Warren 2021f0fe88dSSimon Glassre_ut_test_list = re.compile(r'_u_boot_list_2_(.*)_test_2_\1_test_(.*)\s*$') 2031cd85f57SStephen Warrendef generate_ut_subtest(metafunc, fixture_name): 2041cd85f57SStephen Warren """Provide parametrization for a ut_subtest fixture. 2051cd85f57SStephen Warren 2061cd85f57SStephen Warren Determines the set of unit tests built into a U-Boot binary by parsing the 2071cd85f57SStephen Warren list of symbols generated by the build process. Provides this information 2081cd85f57SStephen Warren to test functions by parameterizing their ut_subtest fixture parameter. 2091cd85f57SStephen Warren 2101cd85f57SStephen Warren Args: 2111cd85f57SStephen Warren metafunc: The pytest test function. 2121cd85f57SStephen Warren fixture_name: The fixture name to test. 2131cd85f57SStephen Warren 2141cd85f57SStephen Warren Returns: 2151cd85f57SStephen Warren Nothing. 2161cd85f57SStephen Warren """ 2171cd85f57SStephen Warren 2181cd85f57SStephen Warren fn = console.config.build_dir + '/u-boot.sym' 2191cd85f57SStephen Warren try: 2201cd85f57SStephen Warren with open(fn, 'rt') as f: 2211cd85f57SStephen Warren lines = f.readlines() 2221cd85f57SStephen Warren except: 2231cd85f57SStephen Warren lines = [] 2241cd85f57SStephen Warren lines.sort() 2251cd85f57SStephen Warren 2261cd85f57SStephen Warren vals = [] 2271cd85f57SStephen Warren for l in lines: 2281cd85f57SStephen Warren m = re_ut_test_list.search(l) 2291cd85f57SStephen Warren if not m: 2301cd85f57SStephen Warren continue 2311cd85f57SStephen Warren vals.append(m.group(1) + ' ' + m.group(2)) 2321cd85f57SStephen Warren 2331cd85f57SStephen Warren ids = ['ut_' + s.replace(' ', '_') for s in vals] 2341cd85f57SStephen Warren metafunc.parametrize(fixture_name, vals, ids=ids) 2351cd85f57SStephen Warren 2361cd85f57SStephen Warrendef generate_config(metafunc, fixture_name): 2371cd85f57SStephen Warren """Provide parametrization for {env,brd}__ fixtures. 238d201506cSStephen Warren 239d201506cSStephen Warren If a test function takes parameter(s) (fixture names) of the form brd__xxx 240d201506cSStephen Warren or env__xxx, the brd and env configuration dictionaries are consulted to 241d201506cSStephen Warren find the list of values to use for those parameters, and the test is 242d201506cSStephen Warren parametrized so that it runs once for each combination of values. 243d201506cSStephen Warren 244d201506cSStephen Warren Args: 245d201506cSStephen Warren metafunc: The pytest test function. 2461cd85f57SStephen Warren fixture_name: The fixture name to test. 247d201506cSStephen Warren 248d201506cSStephen Warren Returns: 249d201506cSStephen Warren Nothing. 250e8debf39SStephen Warren """ 251d201506cSStephen Warren 252d201506cSStephen Warren subconfigs = { 253d201506cSStephen Warren 'brd': console.config.brd, 254d201506cSStephen Warren 'env': console.config.env, 255d201506cSStephen Warren } 2561cd85f57SStephen Warren parts = fixture_name.split('__') 257d201506cSStephen Warren if len(parts) < 2: 2581cd85f57SStephen Warren return 259d201506cSStephen Warren if parts[0] not in subconfigs: 2601cd85f57SStephen Warren return 261d201506cSStephen Warren subconfig = subconfigs[parts[0]] 262d201506cSStephen Warren vals = [] 2631cd85f57SStephen Warren val = subconfig.get(fixture_name, []) 264d201506cSStephen Warren # If that exact name is a key in the data source: 265d201506cSStephen Warren if val: 266d201506cSStephen Warren # ... use the dict value as a single parameter value. 267d201506cSStephen Warren vals = (val, ) 268d201506cSStephen Warren else: 269d201506cSStephen Warren # ... otherwise, see if there's a key that contains a list of 270d201506cSStephen Warren # values to use instead. 2711cd85f57SStephen Warren vals = subconfig.get(fixture_name+ 's', []) 272d20e5e97SStephen Warren def fixture_id(index, val): 273d20e5e97SStephen Warren try: 2741cd85f57SStephen Warren return val['fixture_id'] 275d20e5e97SStephen Warren except: 2761cd85f57SStephen Warren return fixture_name + str(index) 277d20e5e97SStephen Warren ids = [fixture_id(index, val) for (index, val) in enumerate(vals)] 2781cd85f57SStephen Warren metafunc.parametrize(fixture_name, vals, ids=ids) 2791cd85f57SStephen Warren 2801cd85f57SStephen Warrendef pytest_generate_tests(metafunc): 2811cd85f57SStephen Warren """pytest hook: parameterize test functions based on custom rules. 2821cd85f57SStephen Warren 2831cd85f57SStephen Warren Check each test function parameter (fixture name) to see if it is one of 2841cd85f57SStephen Warren our custom names, and if so, provide the correct parametrization for that 2851cd85f57SStephen Warren parameter. 2861cd85f57SStephen Warren 2871cd85f57SStephen Warren Args: 2881cd85f57SStephen Warren metafunc: The pytest test function. 2891cd85f57SStephen Warren 2901cd85f57SStephen Warren Returns: 2911cd85f57SStephen Warren Nothing. 2921cd85f57SStephen Warren """ 2931cd85f57SStephen Warren 2941cd85f57SStephen Warren for fn in metafunc.fixturenames: 2951cd85f57SStephen Warren if fn == 'ut_subtest': 2961cd85f57SStephen Warren generate_ut_subtest(metafunc, fn) 2971cd85f57SStephen Warren continue 2981cd85f57SStephen Warren generate_config(metafunc, fn) 299d201506cSStephen Warren 300d8c1e033SStefan Brüns@pytest.fixture(scope='session') 301d8c1e033SStefan Brünsdef u_boot_log(request): 302d8c1e033SStefan Brüns """Generate the value of a test's log fixture. 303d8c1e033SStefan Brüns 304d8c1e033SStefan Brüns Args: 305d8c1e033SStefan Brüns request: The pytest request. 306d8c1e033SStefan Brüns 307d8c1e033SStefan Brüns Returns: 308d8c1e033SStefan Brüns The fixture value. 309d8c1e033SStefan Brüns """ 310d8c1e033SStefan Brüns 311d8c1e033SStefan Brüns return console.log 312d8c1e033SStefan Brüns 313d8c1e033SStefan Brüns@pytest.fixture(scope='session') 314d8c1e033SStefan Brünsdef u_boot_config(request): 315d8c1e033SStefan Brüns """Generate the value of a test's u_boot_config fixture. 316d8c1e033SStefan Brüns 317d8c1e033SStefan Brüns Args: 318d8c1e033SStefan Brüns request: The pytest request. 319d8c1e033SStefan Brüns 320d8c1e033SStefan Brüns Returns: 321d8c1e033SStefan Brüns The fixture value. 322d8c1e033SStefan Brüns """ 323d8c1e033SStefan Brüns 324d8c1e033SStefan Brüns return console.config 325d8c1e033SStefan Brüns 326636f38d8SStephen Warren@pytest.fixture(scope='function') 327d201506cSStephen Warrendef u_boot_console(request): 328e8debf39SStephen Warren """Generate the value of a test's u_boot_console fixture. 329d201506cSStephen Warren 330d201506cSStephen Warren Args: 331d201506cSStephen Warren request: The pytest request. 332d201506cSStephen Warren 333d201506cSStephen Warren Returns: 334d201506cSStephen Warren The fixture value. 335e8debf39SStephen Warren """ 336d201506cSStephen Warren 337636f38d8SStephen Warren console.ensure_spawned() 338d201506cSStephen Warren return console 339d201506cSStephen Warren 34083357fd5SStephen Warrenanchors = {} 3411326022cSStephen Warrentests_not_run = [] 3421326022cSStephen Warrentests_failed = [] 3431326022cSStephen Warrentests_xpassed = [] 3441326022cSStephen Warrentests_xfailed = [] 3451326022cSStephen Warrentests_skipped = [] 34632090e50SStephen Warrentests_warning = [] 3471326022cSStephen Warrentests_passed = [] 348d201506cSStephen Warren 349d201506cSStephen Warrendef pytest_itemcollected(item): 350e8debf39SStephen Warren """pytest hook: Called once for each test found during collection. 351d201506cSStephen Warren 352d201506cSStephen Warren This enables our custom result analysis code to see the list of all tests 353d201506cSStephen Warren that should eventually be run. 354d201506cSStephen Warren 355d201506cSStephen Warren Args: 356d201506cSStephen Warren item: The item that was collected. 357d201506cSStephen Warren 358d201506cSStephen Warren Returns: 359d201506cSStephen Warren Nothing. 360e8debf39SStephen Warren """ 361d201506cSStephen Warren 3621326022cSStephen Warren tests_not_run.append(item.name) 363d201506cSStephen Warren 364d201506cSStephen Warrendef cleanup(): 365e8debf39SStephen Warren """Clean up all global state. 366d201506cSStephen Warren 367d201506cSStephen Warren Executed (via atexit) once the entire test process is complete. This 368d201506cSStephen Warren includes logging the status of all tests, and the identity of any failed 369d201506cSStephen Warren or skipped tests. 370d201506cSStephen Warren 371d201506cSStephen Warren Args: 372d201506cSStephen Warren None. 373d201506cSStephen Warren 374d201506cSStephen Warren Returns: 375d201506cSStephen Warren Nothing. 376e8debf39SStephen Warren """ 377d201506cSStephen Warren 378d201506cSStephen Warren if console: 379d201506cSStephen Warren console.close() 380d201506cSStephen Warren if log: 38183357fd5SStephen Warren with log.section('Status Report', 'status_report'): 382d201506cSStephen Warren log.status_pass('%d passed' % len(tests_passed)) 38332090e50SStephen Warren if tests_warning: 38432090e50SStephen Warren log.status_warning('%d passed with warning' % len(tests_warning)) 38532090e50SStephen Warren for test in tests_warning: 38632090e50SStephen Warren anchor = anchors.get(test, None) 38732090e50SStephen Warren log.status_warning('... ' + test, anchor) 388d201506cSStephen Warren if tests_skipped: 389d201506cSStephen Warren log.status_skipped('%d skipped' % len(tests_skipped)) 390d201506cSStephen Warren for test in tests_skipped: 39183357fd5SStephen Warren anchor = anchors.get(test, None) 39283357fd5SStephen Warren log.status_skipped('... ' + test, anchor) 39378b39cc3SStephen Warren if tests_xpassed: 39478b39cc3SStephen Warren log.status_xpass('%d xpass' % len(tests_xpassed)) 39578b39cc3SStephen Warren for test in tests_xpassed: 39683357fd5SStephen Warren anchor = anchors.get(test, None) 39783357fd5SStephen Warren log.status_xpass('... ' + test, anchor) 39878b39cc3SStephen Warren if tests_xfailed: 39978b39cc3SStephen Warren log.status_xfail('%d xfail' % len(tests_xfailed)) 40078b39cc3SStephen Warren for test in tests_xfailed: 40183357fd5SStephen Warren anchor = anchors.get(test, None) 40283357fd5SStephen Warren log.status_xfail('... ' + test, anchor) 403d201506cSStephen Warren if tests_failed: 404d201506cSStephen Warren log.status_fail('%d failed' % len(tests_failed)) 405d201506cSStephen Warren for test in tests_failed: 40683357fd5SStephen Warren anchor = anchors.get(test, None) 40783357fd5SStephen Warren log.status_fail('... ' + test, anchor) 408d201506cSStephen Warren if tests_not_run: 409d201506cSStephen Warren log.status_fail('%d not run' % len(tests_not_run)) 410d201506cSStephen Warren for test in tests_not_run: 41183357fd5SStephen Warren anchor = anchors.get(test, None) 41283357fd5SStephen Warren log.status_fail('... ' + test, anchor) 413d201506cSStephen Warren log.close() 414d201506cSStephen Warrenatexit.register(cleanup) 415d201506cSStephen Warren 416d201506cSStephen Warrendef setup_boardspec(item): 417e8debf39SStephen Warren """Process any 'boardspec' marker for a test. 418d201506cSStephen Warren 419d201506cSStephen Warren Such a marker lists the set of board types that a test does/doesn't 420d201506cSStephen Warren support. If tests are being executed on an unsupported board, the test is 421d201506cSStephen Warren marked to be skipped. 422d201506cSStephen Warren 423d201506cSStephen Warren Args: 424d201506cSStephen Warren item: The pytest test item. 425d201506cSStephen Warren 426d201506cSStephen Warren Returns: 427d201506cSStephen Warren Nothing. 428e8debf39SStephen Warren """ 429d201506cSStephen Warren 430d201506cSStephen Warren mark = item.get_marker('boardspec') 431d201506cSStephen Warren if not mark: 432d201506cSStephen Warren return 433d201506cSStephen Warren required_boards = [] 434d201506cSStephen Warren for board in mark.args: 435d201506cSStephen Warren if board.startswith('!'): 436d201506cSStephen Warren if ubconfig.board_type == board[1:]: 437d5170448SStephen Warren pytest.skip('board "%s" not supported' % ubconfig.board_type) 438d201506cSStephen Warren return 439d201506cSStephen Warren else: 440d201506cSStephen Warren required_boards.append(board) 441d201506cSStephen Warren if required_boards and ubconfig.board_type not in required_boards: 442d5170448SStephen Warren pytest.skip('board "%s" not supported' % ubconfig.board_type) 443d201506cSStephen Warren 444d201506cSStephen Warrendef setup_buildconfigspec(item): 445e8debf39SStephen Warren """Process any 'buildconfigspec' marker for a test. 446d201506cSStephen Warren 447d201506cSStephen Warren Such a marker lists some U-Boot configuration feature that the test 448d201506cSStephen Warren requires. If tests are being executed on an U-Boot build that doesn't 449d201506cSStephen Warren have the required feature, the test is marked to be skipped. 450d201506cSStephen Warren 451d201506cSStephen Warren Args: 452d201506cSStephen Warren item: The pytest test item. 453d201506cSStephen Warren 454d201506cSStephen Warren Returns: 455d201506cSStephen Warren Nothing. 456e8debf39SStephen Warren """ 457d201506cSStephen Warren 458d201506cSStephen Warren mark = item.get_marker('buildconfigspec') 459d201506cSStephen Warren if not mark: 460d201506cSStephen Warren return 461d201506cSStephen Warren for option in mark.args: 462d201506cSStephen Warren if not ubconfig.buildconfig.get('config_' + option.lower(), None): 463d5170448SStephen Warren pytest.skip('.config feature "%s" not enabled' % option.lower()) 464d201506cSStephen Warren 4652d26bf6cSStephen Warrendef tool_is_in_path(tool): 4662d26bf6cSStephen Warren for path in os.environ["PATH"].split(os.pathsep): 4672d26bf6cSStephen Warren fn = os.path.join(path, tool) 4682d26bf6cSStephen Warren if os.path.isfile(fn) and os.access(fn, os.X_OK): 4692d26bf6cSStephen Warren return True 4702d26bf6cSStephen Warren return False 4712d26bf6cSStephen Warren 4722d26bf6cSStephen Warrendef setup_requiredtool(item): 4732d26bf6cSStephen Warren """Process any 'requiredtool' marker for a test. 4742d26bf6cSStephen Warren 4752d26bf6cSStephen Warren Such a marker lists some external tool (binary, executable, application) 4762d26bf6cSStephen Warren that the test requires. If tests are being executed on a system that 4772d26bf6cSStephen Warren doesn't have the required tool, the test is marked to be skipped. 4782d26bf6cSStephen Warren 4792d26bf6cSStephen Warren Args: 4802d26bf6cSStephen Warren item: The pytest test item. 4812d26bf6cSStephen Warren 4822d26bf6cSStephen Warren Returns: 4832d26bf6cSStephen Warren Nothing. 4842d26bf6cSStephen Warren """ 4852d26bf6cSStephen Warren 4862d26bf6cSStephen Warren mark = item.get_marker('requiredtool') 4872d26bf6cSStephen Warren if not mark: 4882d26bf6cSStephen Warren return 4892d26bf6cSStephen Warren for tool in mark.args: 4902d26bf6cSStephen Warren if not tool_is_in_path(tool): 4912d26bf6cSStephen Warren pytest.skip('tool "%s" not in $PATH' % tool) 4922d26bf6cSStephen Warren 493b0a928a1SStephen Warrendef start_test_section(item): 494b0a928a1SStephen Warren anchors[item.name] = log.start_section(item.name) 495b0a928a1SStephen Warren 496d201506cSStephen Warrendef pytest_runtest_setup(item): 497e8debf39SStephen Warren """pytest hook: Configure (set up) a test item. 498d201506cSStephen Warren 499d201506cSStephen Warren Called once for each test to perform any custom configuration. This hook 500d201506cSStephen Warren is used to skip the test if certain conditions apply. 501d201506cSStephen Warren 502d201506cSStephen Warren Args: 503d201506cSStephen Warren item: The pytest test item. 504d201506cSStephen Warren 505d201506cSStephen Warren Returns: 506d201506cSStephen Warren Nothing. 507e8debf39SStephen Warren """ 508d201506cSStephen Warren 509b0a928a1SStephen Warren start_test_section(item) 510d201506cSStephen Warren setup_boardspec(item) 511d201506cSStephen Warren setup_buildconfigspec(item) 5122d26bf6cSStephen Warren setup_requiredtool(item) 513d201506cSStephen Warren 514d201506cSStephen Warrendef pytest_runtest_protocol(item, nextitem): 515e8debf39SStephen Warren """pytest hook: Called to execute a test. 516d201506cSStephen Warren 517d201506cSStephen Warren This hook wraps the standard pytest runtestprotocol() function in order 518d201506cSStephen Warren to acquire visibility into, and record, each test function's result. 519d201506cSStephen Warren 520d201506cSStephen Warren Args: 521d201506cSStephen Warren item: The pytest test item to execute. 522d201506cSStephen Warren nextitem: The pytest test item that will be executed after this one. 523d201506cSStephen Warren 524d201506cSStephen Warren Returns: 525d201506cSStephen Warren A list of pytest reports (test result data). 526e8debf39SStephen Warren """ 527d201506cSStephen Warren 52832090e50SStephen Warren log.get_and_reset_warning() 529d201506cSStephen Warren reports = runtestprotocol(item, nextitem=nextitem) 53032090e50SStephen Warren was_warning = log.get_and_reset_warning() 53178b39cc3SStephen Warren 532b0a928a1SStephen Warren # In pytest 3, runtestprotocol() may not call pytest_runtest_setup() if 533b0a928a1SStephen Warren # the test is skipped. That call is required to create the test's section 534b0a928a1SStephen Warren # in the log file. The call to log.end_section() requires that the log 535b0a928a1SStephen Warren # contain a section for this test. Create a section for the test if it 536b0a928a1SStephen Warren # doesn't already exist. 537b0a928a1SStephen Warren if not item.name in anchors: 538b0a928a1SStephen Warren start_test_section(item) 539b0a928a1SStephen Warren 54078b39cc3SStephen Warren failure_cleanup = False 54132090e50SStephen Warren if not was_warning: 54278b39cc3SStephen Warren test_list = tests_passed 54378b39cc3SStephen Warren msg = 'OK' 54478b39cc3SStephen Warren msg_log = log.status_pass 54532090e50SStephen Warren else: 54632090e50SStephen Warren test_list = tests_warning 54732090e50SStephen Warren msg = 'OK (with warning)' 54832090e50SStephen Warren msg_log = log.status_warning 549d201506cSStephen Warren for report in reports: 550d201506cSStephen Warren if report.outcome == 'failed': 55178b39cc3SStephen Warren if hasattr(report, 'wasxfail'): 55278b39cc3SStephen Warren test_list = tests_xpassed 55378b39cc3SStephen Warren msg = 'XPASSED' 55478b39cc3SStephen Warren msg_log = log.status_xpass 55578b39cc3SStephen Warren else: 55678b39cc3SStephen Warren failure_cleanup = True 55778b39cc3SStephen Warren test_list = tests_failed 55878b39cc3SStephen Warren msg = 'FAILED:\n' + str(report.longrepr) 55978b39cc3SStephen Warren msg_log = log.status_fail 560d201506cSStephen Warren break 561d201506cSStephen Warren if report.outcome == 'skipped': 56278b39cc3SStephen Warren if hasattr(report, 'wasxfail'): 56378b39cc3SStephen Warren failure_cleanup = True 56478b39cc3SStephen Warren test_list = tests_xfailed 56578b39cc3SStephen Warren msg = 'XFAILED:\n' + str(report.longrepr) 56678b39cc3SStephen Warren msg_log = log.status_xfail 56778b39cc3SStephen Warren break 56878b39cc3SStephen Warren test_list = tests_skipped 56978b39cc3SStephen Warren msg = 'SKIPPED:\n' + str(report.longrepr) 57078b39cc3SStephen Warren msg_log = log.status_skipped 571d201506cSStephen Warren 57278b39cc3SStephen Warren if failure_cleanup: 573c10eb9d3SStephen Warren console.drain_console() 57478b39cc3SStephen Warren 5751326022cSStephen Warren test_list.append(item.name) 576d201506cSStephen Warren tests_not_run.remove(item.name) 577d201506cSStephen Warren 578d201506cSStephen Warren try: 57978b39cc3SStephen Warren msg_log(msg) 580d201506cSStephen Warren except: 581d201506cSStephen Warren # If something went wrong with logging, it's better to let the test 582d201506cSStephen Warren # process continue, which may report other exceptions that triggered 583d201506cSStephen Warren # the logging issue (e.g. console.log wasn't created). Hence, just 584d201506cSStephen Warren # squash the exception. If the test setup failed due to e.g. syntax 585d201506cSStephen Warren # error somewhere else, this won't be seen. However, once that issue 586d201506cSStephen Warren # is fixed, if this exception still exists, it will then be logged as 587d201506cSStephen Warren # part of the test's stdout. 588d201506cSStephen Warren import traceback 589*dffd56d1SPaul Burton print('Exception occurred while logging runtest status:') 590d201506cSStephen Warren traceback.print_exc() 591d201506cSStephen Warren # FIXME: Can we force a test failure here? 592d201506cSStephen Warren 593d201506cSStephen Warren log.end_section(item.name) 594d201506cSStephen Warren 59578b39cc3SStephen Warren if failure_cleanup: 596d201506cSStephen Warren console.cleanup_spawn() 597d201506cSStephen Warren 598d201506cSStephen Warren return reports 599