183d290c5STom Rini# SPDX-License-Identifier: GPL-2.0 2d201506cSStephen Warren# Copyright (c) 2015 Stephen Warren 3d201506cSStephen Warren# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. 4d201506cSStephen Warren 5d201506cSStephen Warren# Implementation of pytest run-time hook functions. These are invoked by 6d201506cSStephen Warren# pytest at certain points during operation, e.g. startup, for each executed 7d201506cSStephen Warren# test, at shutdown etc. These hooks perform functions such as: 8d201506cSStephen Warren# - Parsing custom command-line options. 9d201506cSStephen Warren# - Pullilng in user-specified board configuration. 10d201506cSStephen Warren# - Creating the U-Boot console test fixture. 11d201506cSStephen Warren# - Creating the HTML log file. 12d201506cSStephen Warren# - Monitoring each test's results. 13d201506cSStephen Warren# - Implementing custom pytest markers. 14d201506cSStephen Warren 15d201506cSStephen Warrenimport atexit 16d201506cSStephen Warrenimport errno 17d201506cSStephen Warrenimport os 18d201506cSStephen Warrenimport os.path 19d201506cSStephen Warrenimport pytest 20d201506cSStephen Warrenfrom _pytest.runner import runtestprotocol 211cd85f57SStephen Warrenimport re 22d201506cSStephen Warrenimport StringIO 23d201506cSStephen Warrenimport sys 24d201506cSStephen Warren 25*052ca37dSPaul Burtontry: 26*052ca37dSPaul Burton import configparser 27*052ca37dSPaul Burtonexcept: 28*052ca37dSPaul Burton import ConfigParser as configparser 29*052ca37dSPaul Burton 30d201506cSStephen Warren# Globals: The HTML log file, and the connection to the U-Boot console. 31d201506cSStephen Warrenlog = None 32d201506cSStephen Warrenconsole = None 33d201506cSStephen Warren 34d201506cSStephen Warrendef mkdir_p(path): 35e8debf39SStephen Warren """Create a directory path. 36d201506cSStephen Warren 37d201506cSStephen Warren This includes creating any intermediate/parent directories. Any errors 38d201506cSStephen Warren caused due to already extant directories are ignored. 39d201506cSStephen Warren 40d201506cSStephen Warren Args: 41d201506cSStephen Warren path: The directory path to create. 42d201506cSStephen Warren 43d201506cSStephen Warren Returns: 44d201506cSStephen Warren Nothing. 45e8debf39SStephen Warren """ 46d201506cSStephen Warren 47d201506cSStephen Warren try: 48d201506cSStephen Warren os.makedirs(path) 49d201506cSStephen Warren except OSError as exc: 50d201506cSStephen Warren if exc.errno == errno.EEXIST and os.path.isdir(path): 51d201506cSStephen Warren pass 52d201506cSStephen Warren else: 53d201506cSStephen Warren raise 54d201506cSStephen Warren 55d201506cSStephen Warrendef pytest_addoption(parser): 56e8debf39SStephen Warren """pytest hook: Add custom command-line options to the cmdline parser. 57d201506cSStephen Warren 58d201506cSStephen Warren Args: 59d201506cSStephen Warren parser: The pytest command-line parser. 60d201506cSStephen Warren 61d201506cSStephen Warren Returns: 62d201506cSStephen Warren Nothing. 63e8debf39SStephen Warren """ 64d201506cSStephen Warren 65d201506cSStephen Warren parser.addoption('--build-dir', default=None, 66d201506cSStephen Warren help='U-Boot build directory (O=)') 67d201506cSStephen Warren parser.addoption('--result-dir', default=None, 68d201506cSStephen Warren help='U-Boot test result/tmp directory') 69d201506cSStephen Warren parser.addoption('--persistent-data-dir', default=None, 70d201506cSStephen Warren help='U-Boot test persistent generated data directory') 71d201506cSStephen Warren parser.addoption('--board-type', '--bd', '-B', default='sandbox', 72d201506cSStephen Warren help='U-Boot board type') 73d201506cSStephen Warren parser.addoption('--board-identity', '--id', default='na', 74d201506cSStephen Warren help='U-Boot board identity/instance') 75d201506cSStephen Warren parser.addoption('--build', default=False, action='store_true', 76d201506cSStephen Warren help='Compile U-Boot before running tests') 7789ab8410SStephen Warren parser.addoption('--gdbserver', default=None, 7889ab8410SStephen Warren help='Run sandbox under gdbserver. The argument is the channel '+ 7989ab8410SStephen Warren 'over which gdbserver should communicate, e.g. localhost:1234') 80d201506cSStephen Warren 81d201506cSStephen Warrendef pytest_configure(config): 82e8debf39SStephen Warren """pytest hook: Perform custom initialization at startup time. 83d201506cSStephen Warren 84d201506cSStephen Warren Args: 85d201506cSStephen Warren config: The pytest configuration. 86d201506cSStephen Warren 87d201506cSStephen Warren Returns: 88d201506cSStephen Warren Nothing. 89e8debf39SStephen Warren """ 90d201506cSStephen Warren 91d201506cSStephen Warren global log 92d201506cSStephen Warren global console 93d201506cSStephen Warren global ubconfig 94d201506cSStephen Warren 95d201506cSStephen Warren test_py_dir = os.path.dirname(os.path.abspath(__file__)) 96d201506cSStephen Warren source_dir = os.path.dirname(os.path.dirname(test_py_dir)) 97d201506cSStephen Warren 98d201506cSStephen Warren board_type = config.getoption('board_type') 99d201506cSStephen Warren board_type_filename = board_type.replace('-', '_') 100d201506cSStephen Warren 101d201506cSStephen Warren board_identity = config.getoption('board_identity') 102d201506cSStephen Warren board_identity_filename = board_identity.replace('-', '_') 103d201506cSStephen Warren 104d201506cSStephen Warren build_dir = config.getoption('build_dir') 105d201506cSStephen Warren if not build_dir: 106d201506cSStephen Warren build_dir = source_dir + '/build-' + board_type 107d201506cSStephen Warren mkdir_p(build_dir) 108d201506cSStephen Warren 109d201506cSStephen Warren result_dir = config.getoption('result_dir') 110d201506cSStephen Warren if not result_dir: 111d201506cSStephen Warren result_dir = build_dir 112d201506cSStephen Warren mkdir_p(result_dir) 113d201506cSStephen Warren 114d201506cSStephen Warren persistent_data_dir = config.getoption('persistent_data_dir') 115d201506cSStephen Warren if not persistent_data_dir: 116d201506cSStephen Warren persistent_data_dir = build_dir + '/persistent-data' 117d201506cSStephen Warren mkdir_p(persistent_data_dir) 118d201506cSStephen Warren 11989ab8410SStephen Warren gdbserver = config.getoption('gdbserver') 12089ab8410SStephen Warren if gdbserver and board_type != 'sandbox': 12189ab8410SStephen Warren raise Exception('--gdbserver only supported with sandbox') 12289ab8410SStephen Warren 123d201506cSStephen Warren import multiplexed_log 124d201506cSStephen Warren log = multiplexed_log.Logfile(result_dir + '/test-log.html') 125d201506cSStephen Warren 126d201506cSStephen Warren if config.getoption('build'): 127d201506cSStephen Warren if build_dir != source_dir: 128d201506cSStephen Warren o_opt = 'O=%s' % build_dir 129d201506cSStephen Warren else: 130d201506cSStephen Warren o_opt = '' 131d201506cSStephen Warren cmds = ( 132d201506cSStephen Warren ['make', o_opt, '-s', board_type + '_defconfig'], 133d201506cSStephen Warren ['make', o_opt, '-s', '-j8'], 134d201506cSStephen Warren ) 13583357fd5SStephen Warren with log.section('make'): 136d201506cSStephen Warren runner = log.get_runner('make', sys.stdout) 137d201506cSStephen Warren for cmd in cmds: 138d201506cSStephen Warren runner.run(cmd, cwd=source_dir) 139d201506cSStephen Warren runner.close() 14083357fd5SStephen Warren log.status_pass('OK') 141d201506cSStephen Warren 142d201506cSStephen Warren class ArbitraryAttributeContainer(object): 143d201506cSStephen Warren pass 144d201506cSStephen Warren 145d201506cSStephen Warren ubconfig = ArbitraryAttributeContainer() 146d201506cSStephen Warren ubconfig.brd = dict() 147d201506cSStephen Warren ubconfig.env = dict() 148d201506cSStephen Warren 149d201506cSStephen Warren modules = [ 150d201506cSStephen Warren (ubconfig.brd, 'u_boot_board_' + board_type_filename), 151d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename), 152d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' + 153d201506cSStephen Warren board_identity_filename), 154d201506cSStephen Warren ] 155d201506cSStephen Warren for (dict_to_fill, module_name) in modules: 156d201506cSStephen Warren try: 157d201506cSStephen Warren module = __import__(module_name) 158d201506cSStephen Warren except ImportError: 159d201506cSStephen Warren continue 160d201506cSStephen Warren dict_to_fill.update(module.__dict__) 161d201506cSStephen Warren 162d201506cSStephen Warren ubconfig.buildconfig = dict() 163d201506cSStephen Warren 164d201506cSStephen Warren for conf_file in ('.config', 'include/autoconf.mk'): 165d201506cSStephen Warren dot_config = build_dir + '/' + conf_file 166d201506cSStephen Warren if not os.path.exists(dot_config): 167d201506cSStephen Warren raise Exception(conf_file + ' does not exist; ' + 168d201506cSStephen Warren 'try passing --build option?') 169d201506cSStephen Warren 170d201506cSStephen Warren with open(dot_config, 'rt') as f: 171d201506cSStephen Warren ini_str = '[root]\n' + f.read() 172d201506cSStephen Warren ini_sio = StringIO.StringIO(ini_str) 173*052ca37dSPaul Burton parser = configparser.RawConfigParser() 174d201506cSStephen Warren parser.readfp(ini_sio) 175d201506cSStephen Warren ubconfig.buildconfig.update(parser.items('root')) 176d201506cSStephen Warren 177d201506cSStephen Warren ubconfig.test_py_dir = test_py_dir 178d201506cSStephen Warren ubconfig.source_dir = source_dir 179d201506cSStephen Warren ubconfig.build_dir = build_dir 180d201506cSStephen Warren ubconfig.result_dir = result_dir 181d201506cSStephen Warren ubconfig.persistent_data_dir = persistent_data_dir 182d201506cSStephen Warren ubconfig.board_type = board_type 183d201506cSStephen Warren ubconfig.board_identity = board_identity 18489ab8410SStephen Warren ubconfig.gdbserver = gdbserver 1850671960bSSimon Glass ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb' 186d201506cSStephen Warren 187d201506cSStephen Warren env_vars = ( 188d201506cSStephen Warren 'board_type', 189d201506cSStephen Warren 'board_identity', 190d201506cSStephen Warren 'source_dir', 191d201506cSStephen Warren 'test_py_dir', 192d201506cSStephen Warren 'build_dir', 193d201506cSStephen Warren 'result_dir', 194d201506cSStephen Warren 'persistent_data_dir', 195d201506cSStephen Warren ) 196d201506cSStephen Warren for v in env_vars: 197d201506cSStephen Warren os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v) 198d201506cSStephen Warren 1992fedbaa4SSimon Glass if board_type.startswith('sandbox'): 200d201506cSStephen Warren import u_boot_console_sandbox 201d201506cSStephen Warren console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig) 202d201506cSStephen Warren else: 203d201506cSStephen Warren import u_boot_console_exec_attach 204d201506cSStephen Warren console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig) 205d201506cSStephen Warren 2061f0fe88dSSimon Glassre_ut_test_list = re.compile(r'_u_boot_list_2_(.*)_test_2_\1_test_(.*)\s*$') 2071cd85f57SStephen Warrendef generate_ut_subtest(metafunc, fixture_name): 2081cd85f57SStephen Warren """Provide parametrization for a ut_subtest fixture. 2091cd85f57SStephen Warren 2101cd85f57SStephen Warren Determines the set of unit tests built into a U-Boot binary by parsing the 2111cd85f57SStephen Warren list of symbols generated by the build process. Provides this information 2121cd85f57SStephen Warren to test functions by parameterizing their ut_subtest fixture parameter. 2131cd85f57SStephen Warren 2141cd85f57SStephen Warren Args: 2151cd85f57SStephen Warren metafunc: The pytest test function. 2161cd85f57SStephen Warren fixture_name: The fixture name to test. 2171cd85f57SStephen Warren 2181cd85f57SStephen Warren Returns: 2191cd85f57SStephen Warren Nothing. 2201cd85f57SStephen Warren """ 2211cd85f57SStephen Warren 2221cd85f57SStephen Warren fn = console.config.build_dir + '/u-boot.sym' 2231cd85f57SStephen Warren try: 2241cd85f57SStephen Warren with open(fn, 'rt') as f: 2251cd85f57SStephen Warren lines = f.readlines() 2261cd85f57SStephen Warren except: 2271cd85f57SStephen Warren lines = [] 2281cd85f57SStephen Warren lines.sort() 2291cd85f57SStephen Warren 2301cd85f57SStephen Warren vals = [] 2311cd85f57SStephen Warren for l in lines: 2321cd85f57SStephen Warren m = re_ut_test_list.search(l) 2331cd85f57SStephen Warren if not m: 2341cd85f57SStephen Warren continue 2351cd85f57SStephen Warren vals.append(m.group(1) + ' ' + m.group(2)) 2361cd85f57SStephen Warren 2371cd85f57SStephen Warren ids = ['ut_' + s.replace(' ', '_') for s in vals] 2381cd85f57SStephen Warren metafunc.parametrize(fixture_name, vals, ids=ids) 2391cd85f57SStephen Warren 2401cd85f57SStephen Warrendef generate_config(metafunc, fixture_name): 2411cd85f57SStephen Warren """Provide parametrization for {env,brd}__ fixtures. 242d201506cSStephen Warren 243d201506cSStephen Warren If a test function takes parameter(s) (fixture names) of the form brd__xxx 244d201506cSStephen Warren or env__xxx, the brd and env configuration dictionaries are consulted to 245d201506cSStephen Warren find the list of values to use for those parameters, and the test is 246d201506cSStephen Warren parametrized so that it runs once for each combination of values. 247d201506cSStephen Warren 248d201506cSStephen Warren Args: 249d201506cSStephen Warren metafunc: The pytest test function. 2501cd85f57SStephen Warren fixture_name: The fixture name to test. 251d201506cSStephen Warren 252d201506cSStephen Warren Returns: 253d201506cSStephen Warren Nothing. 254e8debf39SStephen Warren """ 255d201506cSStephen Warren 256d201506cSStephen Warren subconfigs = { 257d201506cSStephen Warren 'brd': console.config.brd, 258d201506cSStephen Warren 'env': console.config.env, 259d201506cSStephen Warren } 2601cd85f57SStephen Warren parts = fixture_name.split('__') 261d201506cSStephen Warren if len(parts) < 2: 2621cd85f57SStephen Warren return 263d201506cSStephen Warren if parts[0] not in subconfigs: 2641cd85f57SStephen Warren return 265d201506cSStephen Warren subconfig = subconfigs[parts[0]] 266d201506cSStephen Warren vals = [] 2671cd85f57SStephen Warren val = subconfig.get(fixture_name, []) 268d201506cSStephen Warren # If that exact name is a key in the data source: 269d201506cSStephen Warren if val: 270d201506cSStephen Warren # ... use the dict value as a single parameter value. 271d201506cSStephen Warren vals = (val, ) 272d201506cSStephen Warren else: 273d201506cSStephen Warren # ... otherwise, see if there's a key that contains a list of 274d201506cSStephen Warren # values to use instead. 2751cd85f57SStephen Warren vals = subconfig.get(fixture_name+ 's', []) 276d20e5e97SStephen Warren def fixture_id(index, val): 277d20e5e97SStephen Warren try: 2781cd85f57SStephen Warren return val['fixture_id'] 279d20e5e97SStephen Warren except: 2801cd85f57SStephen Warren return fixture_name + str(index) 281d20e5e97SStephen Warren ids = [fixture_id(index, val) for (index, val) in enumerate(vals)] 2821cd85f57SStephen Warren metafunc.parametrize(fixture_name, vals, ids=ids) 2831cd85f57SStephen Warren 2841cd85f57SStephen Warrendef pytest_generate_tests(metafunc): 2851cd85f57SStephen Warren """pytest hook: parameterize test functions based on custom rules. 2861cd85f57SStephen Warren 2871cd85f57SStephen Warren Check each test function parameter (fixture name) to see if it is one of 2881cd85f57SStephen Warren our custom names, and if so, provide the correct parametrization for that 2891cd85f57SStephen Warren parameter. 2901cd85f57SStephen Warren 2911cd85f57SStephen Warren Args: 2921cd85f57SStephen Warren metafunc: The pytest test function. 2931cd85f57SStephen Warren 2941cd85f57SStephen Warren Returns: 2951cd85f57SStephen Warren Nothing. 2961cd85f57SStephen Warren """ 2971cd85f57SStephen Warren 2981cd85f57SStephen Warren for fn in metafunc.fixturenames: 2991cd85f57SStephen Warren if fn == 'ut_subtest': 3001cd85f57SStephen Warren generate_ut_subtest(metafunc, fn) 3011cd85f57SStephen Warren continue 3021cd85f57SStephen Warren generate_config(metafunc, fn) 303d201506cSStephen Warren 304d8c1e033SStefan Brüns@pytest.fixture(scope='session') 305d8c1e033SStefan Brünsdef u_boot_log(request): 306d8c1e033SStefan Brüns """Generate the value of a test's log fixture. 307d8c1e033SStefan Brüns 308d8c1e033SStefan Brüns Args: 309d8c1e033SStefan Brüns request: The pytest request. 310d8c1e033SStefan Brüns 311d8c1e033SStefan Brüns Returns: 312d8c1e033SStefan Brüns The fixture value. 313d8c1e033SStefan Brüns """ 314d8c1e033SStefan Brüns 315d8c1e033SStefan Brüns return console.log 316d8c1e033SStefan Brüns 317d8c1e033SStefan Brüns@pytest.fixture(scope='session') 318d8c1e033SStefan Brünsdef u_boot_config(request): 319d8c1e033SStefan Brüns """Generate the value of a test's u_boot_config fixture. 320d8c1e033SStefan Brüns 321d8c1e033SStefan Brüns Args: 322d8c1e033SStefan Brüns request: The pytest request. 323d8c1e033SStefan Brüns 324d8c1e033SStefan Brüns Returns: 325d8c1e033SStefan Brüns The fixture value. 326d8c1e033SStefan Brüns """ 327d8c1e033SStefan Brüns 328d8c1e033SStefan Brüns return console.config 329d8c1e033SStefan Brüns 330636f38d8SStephen Warren@pytest.fixture(scope='function') 331d201506cSStephen Warrendef u_boot_console(request): 332e8debf39SStephen Warren """Generate the value of a test's u_boot_console fixture. 333d201506cSStephen Warren 334d201506cSStephen Warren Args: 335d201506cSStephen Warren request: The pytest request. 336d201506cSStephen Warren 337d201506cSStephen Warren Returns: 338d201506cSStephen Warren The fixture value. 339e8debf39SStephen Warren """ 340d201506cSStephen Warren 341636f38d8SStephen Warren console.ensure_spawned() 342d201506cSStephen Warren return console 343d201506cSStephen Warren 34483357fd5SStephen Warrenanchors = {} 3451326022cSStephen Warrentests_not_run = [] 3461326022cSStephen Warrentests_failed = [] 3471326022cSStephen Warrentests_xpassed = [] 3481326022cSStephen Warrentests_xfailed = [] 3491326022cSStephen Warrentests_skipped = [] 35032090e50SStephen Warrentests_warning = [] 3511326022cSStephen Warrentests_passed = [] 352d201506cSStephen Warren 353d201506cSStephen Warrendef pytest_itemcollected(item): 354e8debf39SStephen Warren """pytest hook: Called once for each test found during collection. 355d201506cSStephen Warren 356d201506cSStephen Warren This enables our custom result analysis code to see the list of all tests 357d201506cSStephen Warren that should eventually be run. 358d201506cSStephen Warren 359d201506cSStephen Warren Args: 360d201506cSStephen Warren item: The item that was collected. 361d201506cSStephen Warren 362d201506cSStephen Warren Returns: 363d201506cSStephen Warren Nothing. 364e8debf39SStephen Warren """ 365d201506cSStephen Warren 3661326022cSStephen Warren tests_not_run.append(item.name) 367d201506cSStephen Warren 368d201506cSStephen Warrendef cleanup(): 369e8debf39SStephen Warren """Clean up all global state. 370d201506cSStephen Warren 371d201506cSStephen Warren Executed (via atexit) once the entire test process is complete. This 372d201506cSStephen Warren includes logging the status of all tests, and the identity of any failed 373d201506cSStephen Warren or skipped tests. 374d201506cSStephen Warren 375d201506cSStephen Warren Args: 376d201506cSStephen Warren None. 377d201506cSStephen Warren 378d201506cSStephen Warren Returns: 379d201506cSStephen Warren Nothing. 380e8debf39SStephen Warren """ 381d201506cSStephen Warren 382d201506cSStephen Warren if console: 383d201506cSStephen Warren console.close() 384d201506cSStephen Warren if log: 38583357fd5SStephen Warren with log.section('Status Report', 'status_report'): 386d201506cSStephen Warren log.status_pass('%d passed' % len(tests_passed)) 38732090e50SStephen Warren if tests_warning: 38832090e50SStephen Warren log.status_warning('%d passed with warning' % len(tests_warning)) 38932090e50SStephen Warren for test in tests_warning: 39032090e50SStephen Warren anchor = anchors.get(test, None) 39132090e50SStephen Warren log.status_warning('... ' + test, anchor) 392d201506cSStephen Warren if tests_skipped: 393d201506cSStephen Warren log.status_skipped('%d skipped' % len(tests_skipped)) 394d201506cSStephen Warren for test in tests_skipped: 39583357fd5SStephen Warren anchor = anchors.get(test, None) 39683357fd5SStephen Warren log.status_skipped('... ' + test, anchor) 39778b39cc3SStephen Warren if tests_xpassed: 39878b39cc3SStephen Warren log.status_xpass('%d xpass' % len(tests_xpassed)) 39978b39cc3SStephen Warren for test in tests_xpassed: 40083357fd5SStephen Warren anchor = anchors.get(test, None) 40183357fd5SStephen Warren log.status_xpass('... ' + test, anchor) 40278b39cc3SStephen Warren if tests_xfailed: 40378b39cc3SStephen Warren log.status_xfail('%d xfail' % len(tests_xfailed)) 40478b39cc3SStephen Warren for test in tests_xfailed: 40583357fd5SStephen Warren anchor = anchors.get(test, None) 40683357fd5SStephen Warren log.status_xfail('... ' + test, anchor) 407d201506cSStephen Warren if tests_failed: 408d201506cSStephen Warren log.status_fail('%d failed' % len(tests_failed)) 409d201506cSStephen Warren for test in tests_failed: 41083357fd5SStephen Warren anchor = anchors.get(test, None) 41183357fd5SStephen Warren log.status_fail('... ' + test, anchor) 412d201506cSStephen Warren if tests_not_run: 413d201506cSStephen Warren log.status_fail('%d not run' % len(tests_not_run)) 414d201506cSStephen Warren for test in tests_not_run: 41583357fd5SStephen Warren anchor = anchors.get(test, None) 41683357fd5SStephen Warren log.status_fail('... ' + test, anchor) 417d201506cSStephen Warren log.close() 418d201506cSStephen Warrenatexit.register(cleanup) 419d201506cSStephen Warren 420d201506cSStephen Warrendef setup_boardspec(item): 421e8debf39SStephen Warren """Process any 'boardspec' marker for a test. 422d201506cSStephen Warren 423d201506cSStephen Warren Such a marker lists the set of board types that a test does/doesn't 424d201506cSStephen Warren support. If tests are being executed on an unsupported board, the test is 425d201506cSStephen Warren marked to be skipped. 426d201506cSStephen Warren 427d201506cSStephen Warren Args: 428d201506cSStephen Warren item: The pytest test item. 429d201506cSStephen Warren 430d201506cSStephen Warren Returns: 431d201506cSStephen Warren Nothing. 432e8debf39SStephen Warren """ 433d201506cSStephen Warren 434d201506cSStephen Warren mark = item.get_marker('boardspec') 435d201506cSStephen Warren if not mark: 436d201506cSStephen Warren return 437d201506cSStephen Warren required_boards = [] 438d201506cSStephen Warren for board in mark.args: 439d201506cSStephen Warren if board.startswith('!'): 440d201506cSStephen Warren if ubconfig.board_type == board[1:]: 441d5170448SStephen Warren pytest.skip('board "%s" not supported' % ubconfig.board_type) 442d201506cSStephen Warren return 443d201506cSStephen Warren else: 444d201506cSStephen Warren required_boards.append(board) 445d201506cSStephen Warren if required_boards and ubconfig.board_type not in required_boards: 446d5170448SStephen Warren pytest.skip('board "%s" not supported' % ubconfig.board_type) 447d201506cSStephen Warren 448d201506cSStephen Warrendef setup_buildconfigspec(item): 449e8debf39SStephen Warren """Process any 'buildconfigspec' marker for a test. 450d201506cSStephen Warren 451d201506cSStephen Warren Such a marker lists some U-Boot configuration feature that the test 452d201506cSStephen Warren requires. If tests are being executed on an U-Boot build that doesn't 453d201506cSStephen Warren have the required feature, the test is marked to be skipped. 454d201506cSStephen Warren 455d201506cSStephen Warren Args: 456d201506cSStephen Warren item: The pytest test item. 457d201506cSStephen Warren 458d201506cSStephen Warren Returns: 459d201506cSStephen Warren Nothing. 460e8debf39SStephen Warren """ 461d201506cSStephen Warren 462d201506cSStephen Warren mark = item.get_marker('buildconfigspec') 463d201506cSStephen Warren if not mark: 464d201506cSStephen Warren return 465d201506cSStephen Warren for option in mark.args: 466d201506cSStephen Warren if not ubconfig.buildconfig.get('config_' + option.lower(), None): 467d5170448SStephen Warren pytest.skip('.config feature "%s" not enabled' % option.lower()) 468d201506cSStephen Warren 4692d26bf6cSStephen Warrendef tool_is_in_path(tool): 4702d26bf6cSStephen Warren for path in os.environ["PATH"].split(os.pathsep): 4712d26bf6cSStephen Warren fn = os.path.join(path, tool) 4722d26bf6cSStephen Warren if os.path.isfile(fn) and os.access(fn, os.X_OK): 4732d26bf6cSStephen Warren return True 4742d26bf6cSStephen Warren return False 4752d26bf6cSStephen Warren 4762d26bf6cSStephen Warrendef setup_requiredtool(item): 4772d26bf6cSStephen Warren """Process any 'requiredtool' marker for a test. 4782d26bf6cSStephen Warren 4792d26bf6cSStephen Warren Such a marker lists some external tool (binary, executable, application) 4802d26bf6cSStephen Warren that the test requires. If tests are being executed on a system that 4812d26bf6cSStephen Warren doesn't have the required tool, the test is marked to be skipped. 4822d26bf6cSStephen Warren 4832d26bf6cSStephen Warren Args: 4842d26bf6cSStephen Warren item: The pytest test item. 4852d26bf6cSStephen Warren 4862d26bf6cSStephen Warren Returns: 4872d26bf6cSStephen Warren Nothing. 4882d26bf6cSStephen Warren """ 4892d26bf6cSStephen Warren 4902d26bf6cSStephen Warren mark = item.get_marker('requiredtool') 4912d26bf6cSStephen Warren if not mark: 4922d26bf6cSStephen Warren return 4932d26bf6cSStephen Warren for tool in mark.args: 4942d26bf6cSStephen Warren if not tool_is_in_path(tool): 4952d26bf6cSStephen Warren pytest.skip('tool "%s" not in $PATH' % tool) 4962d26bf6cSStephen Warren 497b0a928a1SStephen Warrendef start_test_section(item): 498b0a928a1SStephen Warren anchors[item.name] = log.start_section(item.name) 499b0a928a1SStephen Warren 500d201506cSStephen Warrendef pytest_runtest_setup(item): 501e8debf39SStephen Warren """pytest hook: Configure (set up) a test item. 502d201506cSStephen Warren 503d201506cSStephen Warren Called once for each test to perform any custom configuration. This hook 504d201506cSStephen Warren is used to skip the test if certain conditions apply. 505d201506cSStephen Warren 506d201506cSStephen Warren Args: 507d201506cSStephen Warren item: The pytest test item. 508d201506cSStephen Warren 509d201506cSStephen Warren Returns: 510d201506cSStephen Warren Nothing. 511e8debf39SStephen Warren """ 512d201506cSStephen Warren 513b0a928a1SStephen Warren start_test_section(item) 514d201506cSStephen Warren setup_boardspec(item) 515d201506cSStephen Warren setup_buildconfigspec(item) 5162d26bf6cSStephen Warren setup_requiredtool(item) 517d201506cSStephen Warren 518d201506cSStephen Warrendef pytest_runtest_protocol(item, nextitem): 519e8debf39SStephen Warren """pytest hook: Called to execute a test. 520d201506cSStephen Warren 521d201506cSStephen Warren This hook wraps the standard pytest runtestprotocol() function in order 522d201506cSStephen Warren to acquire visibility into, and record, each test function's result. 523d201506cSStephen Warren 524d201506cSStephen Warren Args: 525d201506cSStephen Warren item: The pytest test item to execute. 526d201506cSStephen Warren nextitem: The pytest test item that will be executed after this one. 527d201506cSStephen Warren 528d201506cSStephen Warren Returns: 529d201506cSStephen Warren A list of pytest reports (test result data). 530e8debf39SStephen Warren """ 531d201506cSStephen Warren 53232090e50SStephen Warren log.get_and_reset_warning() 533d201506cSStephen Warren reports = runtestprotocol(item, nextitem=nextitem) 53432090e50SStephen Warren was_warning = log.get_and_reset_warning() 53578b39cc3SStephen Warren 536b0a928a1SStephen Warren # In pytest 3, runtestprotocol() may not call pytest_runtest_setup() if 537b0a928a1SStephen Warren # the test is skipped. That call is required to create the test's section 538b0a928a1SStephen Warren # in the log file. The call to log.end_section() requires that the log 539b0a928a1SStephen Warren # contain a section for this test. Create a section for the test if it 540b0a928a1SStephen Warren # doesn't already exist. 541b0a928a1SStephen Warren if not item.name in anchors: 542b0a928a1SStephen Warren start_test_section(item) 543b0a928a1SStephen Warren 54478b39cc3SStephen Warren failure_cleanup = False 54532090e50SStephen Warren if not was_warning: 54678b39cc3SStephen Warren test_list = tests_passed 54778b39cc3SStephen Warren msg = 'OK' 54878b39cc3SStephen Warren msg_log = log.status_pass 54932090e50SStephen Warren else: 55032090e50SStephen Warren test_list = tests_warning 55132090e50SStephen Warren msg = 'OK (with warning)' 55232090e50SStephen Warren msg_log = log.status_warning 553d201506cSStephen Warren for report in reports: 554d201506cSStephen Warren if report.outcome == 'failed': 55578b39cc3SStephen Warren if hasattr(report, 'wasxfail'): 55678b39cc3SStephen Warren test_list = tests_xpassed 55778b39cc3SStephen Warren msg = 'XPASSED' 55878b39cc3SStephen Warren msg_log = log.status_xpass 55978b39cc3SStephen Warren else: 56078b39cc3SStephen Warren failure_cleanup = True 56178b39cc3SStephen Warren test_list = tests_failed 56278b39cc3SStephen Warren msg = 'FAILED:\n' + str(report.longrepr) 56378b39cc3SStephen Warren msg_log = log.status_fail 564d201506cSStephen Warren break 565d201506cSStephen Warren if report.outcome == 'skipped': 56678b39cc3SStephen Warren if hasattr(report, 'wasxfail'): 56778b39cc3SStephen Warren failure_cleanup = True 56878b39cc3SStephen Warren test_list = tests_xfailed 56978b39cc3SStephen Warren msg = 'XFAILED:\n' + str(report.longrepr) 57078b39cc3SStephen Warren msg_log = log.status_xfail 57178b39cc3SStephen Warren break 57278b39cc3SStephen Warren test_list = tests_skipped 57378b39cc3SStephen Warren msg = 'SKIPPED:\n' + str(report.longrepr) 57478b39cc3SStephen Warren msg_log = log.status_skipped 575d201506cSStephen Warren 57678b39cc3SStephen Warren if failure_cleanup: 577c10eb9d3SStephen Warren console.drain_console() 57878b39cc3SStephen Warren 5791326022cSStephen Warren test_list.append(item.name) 580d201506cSStephen Warren tests_not_run.remove(item.name) 581d201506cSStephen Warren 582d201506cSStephen Warren try: 58378b39cc3SStephen Warren msg_log(msg) 584d201506cSStephen Warren except: 585d201506cSStephen Warren # If something went wrong with logging, it's better to let the test 586d201506cSStephen Warren # process continue, which may report other exceptions that triggered 587d201506cSStephen Warren # the logging issue (e.g. console.log wasn't created). Hence, just 588d201506cSStephen Warren # squash the exception. If the test setup failed due to e.g. syntax 589d201506cSStephen Warren # error somewhere else, this won't be seen. However, once that issue 590d201506cSStephen Warren # is fixed, if this exception still exists, it will then be logged as 591d201506cSStephen Warren # part of the test's stdout. 592d201506cSStephen Warren import traceback 593dffd56d1SPaul Burton print('Exception occurred while logging runtest status:') 594d201506cSStephen Warren traceback.print_exc() 595d201506cSStephen Warren # FIXME: Can we force a test failure here? 596d201506cSStephen Warren 597d201506cSStephen Warren log.end_section(item.name) 598d201506cSStephen Warren 59978b39cc3SStephen Warren if failure_cleanup: 600d201506cSStephen Warren console.cleanup_spawn() 601d201506cSStephen Warren 602d201506cSStephen Warren return reports 603