1d201506cSStephen Warren# Copyright (c) 2015 Stephen Warren 2d201506cSStephen Warren# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. 3d201506cSStephen Warren# 4d201506cSStephen Warren# SPDX-License-Identifier: GPL-2.0 5d201506cSStephen Warren 6d201506cSStephen Warren# Implementation of pytest run-time hook functions. These are invoked by 7d201506cSStephen Warren# pytest at certain points during operation, e.g. startup, for each executed 8d201506cSStephen Warren# test, at shutdown etc. These hooks perform functions such as: 9d201506cSStephen Warren# - Parsing custom command-line options. 10d201506cSStephen Warren# - Pullilng in user-specified board configuration. 11d201506cSStephen Warren# - Creating the U-Boot console test fixture. 12d201506cSStephen Warren# - Creating the HTML log file. 13d201506cSStephen Warren# - Monitoring each test's results. 14d201506cSStephen Warren# - Implementing custom pytest markers. 15d201506cSStephen Warren 16d201506cSStephen Warrenimport atexit 17d201506cSStephen Warrenimport errno 18d201506cSStephen Warrenimport os 19d201506cSStephen Warrenimport os.path 20d201506cSStephen Warrenimport pexpect 21d201506cSStephen Warrenimport pytest 22d201506cSStephen Warrenfrom _pytest.runner import runtestprotocol 23d201506cSStephen Warrenimport ConfigParser 24d201506cSStephen Warrenimport StringIO 25d201506cSStephen Warrenimport sys 26d201506cSStephen Warren 27d201506cSStephen Warren# Globals: The HTML log file, and the connection to the U-Boot console. 28d201506cSStephen Warrenlog = None 29d201506cSStephen Warrenconsole = None 30d201506cSStephen Warren 31d201506cSStephen Warrendef mkdir_p(path): 32e8debf39SStephen Warren """Create a directory path. 33d201506cSStephen Warren 34d201506cSStephen Warren This includes creating any intermediate/parent directories. Any errors 35d201506cSStephen Warren caused due to already extant directories are ignored. 36d201506cSStephen Warren 37d201506cSStephen Warren Args: 38d201506cSStephen Warren path: The directory path to create. 39d201506cSStephen Warren 40d201506cSStephen Warren Returns: 41d201506cSStephen Warren Nothing. 42e8debf39SStephen Warren """ 43d201506cSStephen Warren 44d201506cSStephen Warren try: 45d201506cSStephen Warren os.makedirs(path) 46d201506cSStephen Warren except OSError as exc: 47d201506cSStephen Warren if exc.errno == errno.EEXIST and os.path.isdir(path): 48d201506cSStephen Warren pass 49d201506cSStephen Warren else: 50d201506cSStephen Warren raise 51d201506cSStephen Warren 52d201506cSStephen Warrendef pytest_addoption(parser): 53e8debf39SStephen Warren """pytest hook: Add custom command-line options to the cmdline parser. 54d201506cSStephen Warren 55d201506cSStephen Warren Args: 56d201506cSStephen Warren parser: The pytest command-line parser. 57d201506cSStephen Warren 58d201506cSStephen Warren Returns: 59d201506cSStephen Warren Nothing. 60e8debf39SStephen Warren """ 61d201506cSStephen Warren 62d201506cSStephen Warren parser.addoption('--build-dir', default=None, 63d201506cSStephen Warren help='U-Boot build directory (O=)') 64d201506cSStephen Warren parser.addoption('--result-dir', default=None, 65d201506cSStephen Warren help='U-Boot test result/tmp directory') 66d201506cSStephen Warren parser.addoption('--persistent-data-dir', default=None, 67d201506cSStephen Warren help='U-Boot test persistent generated data directory') 68d201506cSStephen Warren parser.addoption('--board-type', '--bd', '-B', default='sandbox', 69d201506cSStephen Warren help='U-Boot board type') 70d201506cSStephen Warren parser.addoption('--board-identity', '--id', default='na', 71d201506cSStephen Warren help='U-Boot board identity/instance') 72d201506cSStephen Warren parser.addoption('--build', default=False, action='store_true', 73d201506cSStephen Warren help='Compile U-Boot before running tests') 74d201506cSStephen Warren 75d201506cSStephen Warrendef pytest_configure(config): 76e8debf39SStephen Warren """pytest hook: Perform custom initialization at startup time. 77d201506cSStephen Warren 78d201506cSStephen Warren Args: 79d201506cSStephen Warren config: The pytest configuration. 80d201506cSStephen Warren 81d201506cSStephen Warren Returns: 82d201506cSStephen Warren Nothing. 83e8debf39SStephen Warren """ 84d201506cSStephen Warren 85d201506cSStephen Warren global log 86d201506cSStephen Warren global console 87d201506cSStephen Warren global ubconfig 88d201506cSStephen Warren 89d201506cSStephen Warren test_py_dir = os.path.dirname(os.path.abspath(__file__)) 90d201506cSStephen Warren source_dir = os.path.dirname(os.path.dirname(test_py_dir)) 91d201506cSStephen Warren 92d201506cSStephen Warren board_type = config.getoption('board_type') 93d201506cSStephen Warren board_type_filename = board_type.replace('-', '_') 94d201506cSStephen Warren 95d201506cSStephen Warren board_identity = config.getoption('board_identity') 96d201506cSStephen Warren board_identity_filename = board_identity.replace('-', '_') 97d201506cSStephen Warren 98d201506cSStephen Warren build_dir = config.getoption('build_dir') 99d201506cSStephen Warren if not build_dir: 100d201506cSStephen Warren build_dir = source_dir + '/build-' + board_type 101d201506cSStephen Warren mkdir_p(build_dir) 102d201506cSStephen Warren 103d201506cSStephen Warren result_dir = config.getoption('result_dir') 104d201506cSStephen Warren if not result_dir: 105d201506cSStephen Warren result_dir = build_dir 106d201506cSStephen Warren mkdir_p(result_dir) 107d201506cSStephen Warren 108d201506cSStephen Warren persistent_data_dir = config.getoption('persistent_data_dir') 109d201506cSStephen Warren if not persistent_data_dir: 110d201506cSStephen Warren persistent_data_dir = build_dir + '/persistent-data' 111d201506cSStephen Warren mkdir_p(persistent_data_dir) 112d201506cSStephen Warren 113d201506cSStephen Warren import multiplexed_log 114d201506cSStephen Warren log = multiplexed_log.Logfile(result_dir + '/test-log.html') 115d201506cSStephen Warren 116d201506cSStephen Warren if config.getoption('build'): 117d201506cSStephen Warren if build_dir != source_dir: 118d201506cSStephen Warren o_opt = 'O=%s' % build_dir 119d201506cSStephen Warren else: 120d201506cSStephen Warren o_opt = '' 121d201506cSStephen Warren cmds = ( 122d201506cSStephen Warren ['make', o_opt, '-s', board_type + '_defconfig'], 123d201506cSStephen Warren ['make', o_opt, '-s', '-j8'], 124d201506cSStephen Warren ) 125d201506cSStephen Warren runner = log.get_runner('make', sys.stdout) 126d201506cSStephen Warren for cmd in cmds: 127d201506cSStephen Warren runner.run(cmd, cwd=source_dir) 128d201506cSStephen Warren runner.close() 129d201506cSStephen Warren 130d201506cSStephen Warren class ArbitraryAttributeContainer(object): 131d201506cSStephen Warren pass 132d201506cSStephen Warren 133d201506cSStephen Warren ubconfig = ArbitraryAttributeContainer() 134d201506cSStephen Warren ubconfig.brd = dict() 135d201506cSStephen Warren ubconfig.env = dict() 136d201506cSStephen Warren 137d201506cSStephen Warren modules = [ 138d201506cSStephen Warren (ubconfig.brd, 'u_boot_board_' + board_type_filename), 139d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename), 140d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' + 141d201506cSStephen Warren board_identity_filename), 142d201506cSStephen Warren ] 143d201506cSStephen Warren for (dict_to_fill, module_name) in modules: 144d201506cSStephen Warren try: 145d201506cSStephen Warren module = __import__(module_name) 146d201506cSStephen Warren except ImportError: 147d201506cSStephen Warren continue 148d201506cSStephen Warren dict_to_fill.update(module.__dict__) 149d201506cSStephen Warren 150d201506cSStephen Warren ubconfig.buildconfig = dict() 151d201506cSStephen Warren 152d201506cSStephen Warren for conf_file in ('.config', 'include/autoconf.mk'): 153d201506cSStephen Warren dot_config = build_dir + '/' + conf_file 154d201506cSStephen Warren if not os.path.exists(dot_config): 155d201506cSStephen Warren raise Exception(conf_file + ' does not exist; ' + 156d201506cSStephen Warren 'try passing --build option?') 157d201506cSStephen Warren 158d201506cSStephen Warren with open(dot_config, 'rt') as f: 159d201506cSStephen Warren ini_str = '[root]\n' + f.read() 160d201506cSStephen Warren ini_sio = StringIO.StringIO(ini_str) 161d201506cSStephen Warren parser = ConfigParser.RawConfigParser() 162d201506cSStephen Warren parser.readfp(ini_sio) 163d201506cSStephen Warren ubconfig.buildconfig.update(parser.items('root')) 164d201506cSStephen Warren 165d201506cSStephen Warren ubconfig.test_py_dir = test_py_dir 166d201506cSStephen Warren ubconfig.source_dir = source_dir 167d201506cSStephen Warren ubconfig.build_dir = build_dir 168d201506cSStephen Warren ubconfig.result_dir = result_dir 169d201506cSStephen Warren ubconfig.persistent_data_dir = persistent_data_dir 170d201506cSStephen Warren ubconfig.board_type = board_type 171d201506cSStephen Warren ubconfig.board_identity = board_identity 172d201506cSStephen Warren 173d201506cSStephen Warren env_vars = ( 174d201506cSStephen Warren 'board_type', 175d201506cSStephen Warren 'board_identity', 176d201506cSStephen Warren 'source_dir', 177d201506cSStephen Warren 'test_py_dir', 178d201506cSStephen Warren 'build_dir', 179d201506cSStephen Warren 'result_dir', 180d201506cSStephen Warren 'persistent_data_dir', 181d201506cSStephen Warren ) 182d201506cSStephen Warren for v in env_vars: 183d201506cSStephen Warren os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v) 184d201506cSStephen Warren 185d201506cSStephen Warren if board_type == 'sandbox': 186d201506cSStephen Warren import u_boot_console_sandbox 187d201506cSStephen Warren console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig) 188d201506cSStephen Warren else: 189d201506cSStephen Warren import u_boot_console_exec_attach 190d201506cSStephen Warren console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig) 191d201506cSStephen Warren 192d201506cSStephen Warrendef pytest_generate_tests(metafunc): 193e8debf39SStephen Warren """pytest hook: parameterize test functions based on custom rules. 194d201506cSStephen Warren 195d201506cSStephen Warren If a test function takes parameter(s) (fixture names) of the form brd__xxx 196d201506cSStephen Warren or env__xxx, the brd and env configuration dictionaries are consulted to 197d201506cSStephen Warren find the list of values to use for those parameters, and the test is 198d201506cSStephen Warren parametrized so that it runs once for each combination of values. 199d201506cSStephen Warren 200d201506cSStephen Warren Args: 201d201506cSStephen Warren metafunc: The pytest test function. 202d201506cSStephen Warren 203d201506cSStephen Warren Returns: 204d201506cSStephen Warren Nothing. 205e8debf39SStephen Warren """ 206d201506cSStephen Warren 207d201506cSStephen Warren subconfigs = { 208d201506cSStephen Warren 'brd': console.config.brd, 209d201506cSStephen Warren 'env': console.config.env, 210d201506cSStephen Warren } 211d201506cSStephen Warren for fn in metafunc.fixturenames: 212d201506cSStephen Warren parts = fn.split('__') 213d201506cSStephen Warren if len(parts) < 2: 214d201506cSStephen Warren continue 215d201506cSStephen Warren if parts[0] not in subconfigs: 216d201506cSStephen Warren continue 217d201506cSStephen Warren subconfig = subconfigs[parts[0]] 218d201506cSStephen Warren vals = [] 219d201506cSStephen Warren val = subconfig.get(fn, []) 220d201506cSStephen Warren # If that exact name is a key in the data source: 221d201506cSStephen Warren if val: 222d201506cSStephen Warren # ... use the dict value as a single parameter value. 223d201506cSStephen Warren vals = (val, ) 224d201506cSStephen Warren else: 225d201506cSStephen Warren # ... otherwise, see if there's a key that contains a list of 226d201506cSStephen Warren # values to use instead. 227d201506cSStephen Warren vals = subconfig.get(fn + 's', []) 228*d20e5e97SStephen Warren def fixture_id(index, val): 229*d20e5e97SStephen Warren try: 230*d20e5e97SStephen Warren return val["fixture_id"] 231*d20e5e97SStephen Warren except: 232*d20e5e97SStephen Warren return fn + str(index) 233*d20e5e97SStephen Warren ids = [fixture_id(index, val) for (index, val) in enumerate(vals)] 234*d20e5e97SStephen Warren metafunc.parametrize(fn, vals, ids=ids) 235d201506cSStephen Warren 236636f38d8SStephen Warren@pytest.fixture(scope='function') 237d201506cSStephen Warrendef u_boot_console(request): 238e8debf39SStephen Warren """Generate the value of a test's u_boot_console fixture. 239d201506cSStephen Warren 240d201506cSStephen Warren Args: 241d201506cSStephen Warren request: The pytest request. 242d201506cSStephen Warren 243d201506cSStephen Warren Returns: 244d201506cSStephen Warren The fixture value. 245e8debf39SStephen Warren """ 246d201506cSStephen Warren 247636f38d8SStephen Warren console.ensure_spawned() 248d201506cSStephen Warren return console 249d201506cSStephen Warren 250d201506cSStephen Warrentests_not_run = set() 251d201506cSStephen Warrentests_failed = set() 252d201506cSStephen Warrentests_skipped = set() 253d201506cSStephen Warrentests_passed = set() 254d201506cSStephen Warren 255d201506cSStephen Warrendef pytest_itemcollected(item): 256e8debf39SStephen Warren """pytest hook: Called once for each test found during collection. 257d201506cSStephen Warren 258d201506cSStephen Warren This enables our custom result analysis code to see the list of all tests 259d201506cSStephen Warren that should eventually be run. 260d201506cSStephen Warren 261d201506cSStephen Warren Args: 262d201506cSStephen Warren item: The item that was collected. 263d201506cSStephen Warren 264d201506cSStephen Warren Returns: 265d201506cSStephen Warren Nothing. 266e8debf39SStephen Warren """ 267d201506cSStephen Warren 268d201506cSStephen Warren tests_not_run.add(item.name) 269d201506cSStephen Warren 270d201506cSStephen Warrendef cleanup(): 271e8debf39SStephen Warren """Clean up all global state. 272d201506cSStephen Warren 273d201506cSStephen Warren Executed (via atexit) once the entire test process is complete. This 274d201506cSStephen Warren includes logging the status of all tests, and the identity of any failed 275d201506cSStephen Warren or skipped tests. 276d201506cSStephen Warren 277d201506cSStephen Warren Args: 278d201506cSStephen Warren None. 279d201506cSStephen Warren 280d201506cSStephen Warren Returns: 281d201506cSStephen Warren Nothing. 282e8debf39SStephen Warren """ 283d201506cSStephen Warren 284d201506cSStephen Warren if console: 285d201506cSStephen Warren console.close() 286d201506cSStephen Warren if log: 287d201506cSStephen Warren log.status_pass('%d passed' % len(tests_passed)) 288d201506cSStephen Warren if tests_skipped: 289d201506cSStephen Warren log.status_skipped('%d skipped' % len(tests_skipped)) 290d201506cSStephen Warren for test in tests_skipped: 291d201506cSStephen Warren log.status_skipped('... ' + test) 292d201506cSStephen Warren if tests_failed: 293d201506cSStephen Warren log.status_fail('%d failed' % len(tests_failed)) 294d201506cSStephen Warren for test in tests_failed: 295d201506cSStephen Warren log.status_fail('... ' + test) 296d201506cSStephen Warren if tests_not_run: 297d201506cSStephen Warren log.status_fail('%d not run' % len(tests_not_run)) 298d201506cSStephen Warren for test in tests_not_run: 299d201506cSStephen Warren log.status_fail('... ' + test) 300d201506cSStephen Warren log.close() 301d201506cSStephen Warrenatexit.register(cleanup) 302d201506cSStephen Warren 303d201506cSStephen Warrendef setup_boardspec(item): 304e8debf39SStephen Warren """Process any 'boardspec' marker for a test. 305d201506cSStephen Warren 306d201506cSStephen Warren Such a marker lists the set of board types that a test does/doesn't 307d201506cSStephen Warren support. If tests are being executed on an unsupported board, the test is 308d201506cSStephen Warren marked to be skipped. 309d201506cSStephen Warren 310d201506cSStephen Warren Args: 311d201506cSStephen Warren item: The pytest test item. 312d201506cSStephen Warren 313d201506cSStephen Warren Returns: 314d201506cSStephen Warren Nothing. 315e8debf39SStephen Warren """ 316d201506cSStephen Warren 317d201506cSStephen Warren mark = item.get_marker('boardspec') 318d201506cSStephen Warren if not mark: 319d201506cSStephen Warren return 320d201506cSStephen Warren required_boards = [] 321d201506cSStephen Warren for board in mark.args: 322d201506cSStephen Warren if board.startswith('!'): 323d201506cSStephen Warren if ubconfig.board_type == board[1:]: 324d201506cSStephen Warren pytest.skip('board not supported') 325d201506cSStephen Warren return 326d201506cSStephen Warren else: 327d201506cSStephen Warren required_boards.append(board) 328d201506cSStephen Warren if required_boards and ubconfig.board_type not in required_boards: 329d201506cSStephen Warren pytest.skip('board not supported') 330d201506cSStephen Warren 331d201506cSStephen Warrendef setup_buildconfigspec(item): 332e8debf39SStephen Warren """Process any 'buildconfigspec' marker for a test. 333d201506cSStephen Warren 334d201506cSStephen Warren Such a marker lists some U-Boot configuration feature that the test 335d201506cSStephen Warren requires. If tests are being executed on an U-Boot build that doesn't 336d201506cSStephen Warren have the required feature, the test is marked to be skipped. 337d201506cSStephen Warren 338d201506cSStephen Warren Args: 339d201506cSStephen Warren item: The pytest test item. 340d201506cSStephen Warren 341d201506cSStephen Warren Returns: 342d201506cSStephen Warren Nothing. 343e8debf39SStephen Warren """ 344d201506cSStephen Warren 345d201506cSStephen Warren mark = item.get_marker('buildconfigspec') 346d201506cSStephen Warren if not mark: 347d201506cSStephen Warren return 348d201506cSStephen Warren for option in mark.args: 349d201506cSStephen Warren if not ubconfig.buildconfig.get('config_' + option.lower(), None): 350d201506cSStephen Warren pytest.skip('.config feature not enabled') 351d201506cSStephen Warren 352d201506cSStephen Warrendef pytest_runtest_setup(item): 353e8debf39SStephen Warren """pytest hook: Configure (set up) a test item. 354d201506cSStephen Warren 355d201506cSStephen Warren Called once for each test to perform any custom configuration. This hook 356d201506cSStephen Warren is used to skip the test if certain conditions apply. 357d201506cSStephen Warren 358d201506cSStephen Warren Args: 359d201506cSStephen Warren item: The pytest test item. 360d201506cSStephen Warren 361d201506cSStephen Warren Returns: 362d201506cSStephen Warren Nothing. 363e8debf39SStephen Warren """ 364d201506cSStephen Warren 365d201506cSStephen Warren log.start_section(item.name) 366d201506cSStephen Warren setup_boardspec(item) 367d201506cSStephen Warren setup_buildconfigspec(item) 368d201506cSStephen Warren 369d201506cSStephen Warrendef pytest_runtest_protocol(item, nextitem): 370e8debf39SStephen Warren """pytest hook: Called to execute a test. 371d201506cSStephen Warren 372d201506cSStephen Warren This hook wraps the standard pytest runtestprotocol() function in order 373d201506cSStephen Warren to acquire visibility into, and record, each test function's result. 374d201506cSStephen Warren 375d201506cSStephen Warren Args: 376d201506cSStephen Warren item: The pytest test item to execute. 377d201506cSStephen Warren nextitem: The pytest test item that will be executed after this one. 378d201506cSStephen Warren 379d201506cSStephen Warren Returns: 380d201506cSStephen Warren A list of pytest reports (test result data). 381e8debf39SStephen Warren """ 382d201506cSStephen Warren 383d201506cSStephen Warren reports = runtestprotocol(item, nextitem=nextitem) 384d201506cSStephen Warren failed = None 385d201506cSStephen Warren skipped = None 386d201506cSStephen Warren for report in reports: 387d201506cSStephen Warren if report.outcome == 'failed': 388d201506cSStephen Warren failed = report 389d201506cSStephen Warren break 390d201506cSStephen Warren if report.outcome == 'skipped': 391d201506cSStephen Warren if not skipped: 392d201506cSStephen Warren skipped = report 393d201506cSStephen Warren 394d201506cSStephen Warren if failed: 395c10eb9d3SStephen Warren console.drain_console() 396d201506cSStephen Warren tests_failed.add(item.name) 397d201506cSStephen Warren elif skipped: 398d201506cSStephen Warren tests_skipped.add(item.name) 399d201506cSStephen Warren else: 400d201506cSStephen Warren tests_passed.add(item.name) 401d201506cSStephen Warren tests_not_run.remove(item.name) 402d201506cSStephen Warren 403d201506cSStephen Warren try: 404d201506cSStephen Warren if failed: 405d201506cSStephen Warren msg = 'FAILED:\n' + str(failed.longrepr) 406d201506cSStephen Warren log.status_fail(msg) 407d201506cSStephen Warren elif skipped: 408d201506cSStephen Warren msg = 'SKIPPED:\n' + str(skipped.longrepr) 409d201506cSStephen Warren log.status_skipped(msg) 410d201506cSStephen Warren else: 411d201506cSStephen Warren log.status_pass('OK') 412d201506cSStephen Warren except: 413d201506cSStephen Warren # If something went wrong with logging, it's better to let the test 414d201506cSStephen Warren # process continue, which may report other exceptions that triggered 415d201506cSStephen Warren # the logging issue (e.g. console.log wasn't created). Hence, just 416d201506cSStephen Warren # squash the exception. If the test setup failed due to e.g. syntax 417d201506cSStephen Warren # error somewhere else, this won't be seen. However, once that issue 418d201506cSStephen Warren # is fixed, if this exception still exists, it will then be logged as 419d201506cSStephen Warren # part of the test's stdout. 420d201506cSStephen Warren import traceback 421d201506cSStephen Warren print 'Exception occurred while logging runtest status:' 422d201506cSStephen Warren traceback.print_exc() 423d201506cSStephen Warren # FIXME: Can we force a test failure here? 424d201506cSStephen Warren 425d201506cSStephen Warren log.end_section(item.name) 426d201506cSStephen Warren 427d201506cSStephen Warren if failed: 428d201506cSStephen Warren console.cleanup_spawn() 429d201506cSStephen Warren 430d201506cSStephen Warren return reports 431