1*d201506cSStephen Warren# Copyright (c) 2015 Stephen Warren 2*d201506cSStephen Warren# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved. 3*d201506cSStephen Warren# 4*d201506cSStephen Warren# SPDX-License-Identifier: GPL-2.0 5*d201506cSStephen Warren 6*d201506cSStephen Warren# Implementation of pytest run-time hook functions. These are invoked by 7*d201506cSStephen Warren# pytest at certain points during operation, e.g. startup, for each executed 8*d201506cSStephen Warren# test, at shutdown etc. These hooks perform functions such as: 9*d201506cSStephen Warren# - Parsing custom command-line options. 10*d201506cSStephen Warren# - Pullilng in user-specified board configuration. 11*d201506cSStephen Warren# - Creating the U-Boot console test fixture. 12*d201506cSStephen Warren# - Creating the HTML log file. 13*d201506cSStephen Warren# - Monitoring each test's results. 14*d201506cSStephen Warren# - Implementing custom pytest markers. 15*d201506cSStephen Warren 16*d201506cSStephen Warrenimport atexit 17*d201506cSStephen Warrenimport errno 18*d201506cSStephen Warrenimport os 19*d201506cSStephen Warrenimport os.path 20*d201506cSStephen Warrenimport pexpect 21*d201506cSStephen Warrenimport pytest 22*d201506cSStephen Warrenfrom _pytest.runner import runtestprotocol 23*d201506cSStephen Warrenimport ConfigParser 24*d201506cSStephen Warrenimport StringIO 25*d201506cSStephen Warrenimport sys 26*d201506cSStephen Warren 27*d201506cSStephen Warren# Globals: The HTML log file, and the connection to the U-Boot console. 28*d201506cSStephen Warrenlog = None 29*d201506cSStephen Warrenconsole = None 30*d201506cSStephen Warren 31*d201506cSStephen Warrendef mkdir_p(path): 32*d201506cSStephen Warren '''Create a directory path. 33*d201506cSStephen Warren 34*d201506cSStephen Warren This includes creating any intermediate/parent directories. Any errors 35*d201506cSStephen Warren caused due to already extant directories are ignored. 36*d201506cSStephen Warren 37*d201506cSStephen Warren Args: 38*d201506cSStephen Warren path: The directory path to create. 39*d201506cSStephen Warren 40*d201506cSStephen Warren Returns: 41*d201506cSStephen Warren Nothing. 42*d201506cSStephen Warren ''' 43*d201506cSStephen Warren 44*d201506cSStephen Warren try: 45*d201506cSStephen Warren os.makedirs(path) 46*d201506cSStephen Warren except OSError as exc: 47*d201506cSStephen Warren if exc.errno == errno.EEXIST and os.path.isdir(path): 48*d201506cSStephen Warren pass 49*d201506cSStephen Warren else: 50*d201506cSStephen Warren raise 51*d201506cSStephen Warren 52*d201506cSStephen Warrendef pytest_addoption(parser): 53*d201506cSStephen Warren '''pytest hook: Add custom command-line options to the cmdline parser. 54*d201506cSStephen Warren 55*d201506cSStephen Warren Args: 56*d201506cSStephen Warren parser: The pytest command-line parser. 57*d201506cSStephen Warren 58*d201506cSStephen Warren Returns: 59*d201506cSStephen Warren Nothing. 60*d201506cSStephen Warren ''' 61*d201506cSStephen Warren 62*d201506cSStephen Warren parser.addoption('--build-dir', default=None, 63*d201506cSStephen Warren help='U-Boot build directory (O=)') 64*d201506cSStephen Warren parser.addoption('--result-dir', default=None, 65*d201506cSStephen Warren help='U-Boot test result/tmp directory') 66*d201506cSStephen Warren parser.addoption('--persistent-data-dir', default=None, 67*d201506cSStephen Warren help='U-Boot test persistent generated data directory') 68*d201506cSStephen Warren parser.addoption('--board-type', '--bd', '-B', default='sandbox', 69*d201506cSStephen Warren help='U-Boot board type') 70*d201506cSStephen Warren parser.addoption('--board-identity', '--id', default='na', 71*d201506cSStephen Warren help='U-Boot board identity/instance') 72*d201506cSStephen Warren parser.addoption('--build', default=False, action='store_true', 73*d201506cSStephen Warren help='Compile U-Boot before running tests') 74*d201506cSStephen Warren 75*d201506cSStephen Warrendef pytest_configure(config): 76*d201506cSStephen Warren '''pytest hook: Perform custom initialization at startup time. 77*d201506cSStephen Warren 78*d201506cSStephen Warren Args: 79*d201506cSStephen Warren config: The pytest configuration. 80*d201506cSStephen Warren 81*d201506cSStephen Warren Returns: 82*d201506cSStephen Warren Nothing. 83*d201506cSStephen Warren ''' 84*d201506cSStephen Warren 85*d201506cSStephen Warren global log 86*d201506cSStephen Warren global console 87*d201506cSStephen Warren global ubconfig 88*d201506cSStephen Warren 89*d201506cSStephen Warren test_py_dir = os.path.dirname(os.path.abspath(__file__)) 90*d201506cSStephen Warren source_dir = os.path.dirname(os.path.dirname(test_py_dir)) 91*d201506cSStephen Warren 92*d201506cSStephen Warren board_type = config.getoption('board_type') 93*d201506cSStephen Warren board_type_filename = board_type.replace('-', '_') 94*d201506cSStephen Warren 95*d201506cSStephen Warren board_identity = config.getoption('board_identity') 96*d201506cSStephen Warren board_identity_filename = board_identity.replace('-', '_') 97*d201506cSStephen Warren 98*d201506cSStephen Warren build_dir = config.getoption('build_dir') 99*d201506cSStephen Warren if not build_dir: 100*d201506cSStephen Warren build_dir = source_dir + '/build-' + board_type 101*d201506cSStephen Warren mkdir_p(build_dir) 102*d201506cSStephen Warren 103*d201506cSStephen Warren result_dir = config.getoption('result_dir') 104*d201506cSStephen Warren if not result_dir: 105*d201506cSStephen Warren result_dir = build_dir 106*d201506cSStephen Warren mkdir_p(result_dir) 107*d201506cSStephen Warren 108*d201506cSStephen Warren persistent_data_dir = config.getoption('persistent_data_dir') 109*d201506cSStephen Warren if not persistent_data_dir: 110*d201506cSStephen Warren persistent_data_dir = build_dir + '/persistent-data' 111*d201506cSStephen Warren mkdir_p(persistent_data_dir) 112*d201506cSStephen Warren 113*d201506cSStephen Warren import multiplexed_log 114*d201506cSStephen Warren log = multiplexed_log.Logfile(result_dir + '/test-log.html') 115*d201506cSStephen Warren 116*d201506cSStephen Warren if config.getoption('build'): 117*d201506cSStephen Warren if build_dir != source_dir: 118*d201506cSStephen Warren o_opt = 'O=%s' % build_dir 119*d201506cSStephen Warren else: 120*d201506cSStephen Warren o_opt = '' 121*d201506cSStephen Warren cmds = ( 122*d201506cSStephen Warren ['make', o_opt, '-s', board_type + '_defconfig'], 123*d201506cSStephen Warren ['make', o_opt, '-s', '-j8'], 124*d201506cSStephen Warren ) 125*d201506cSStephen Warren runner = log.get_runner('make', sys.stdout) 126*d201506cSStephen Warren for cmd in cmds: 127*d201506cSStephen Warren runner.run(cmd, cwd=source_dir) 128*d201506cSStephen Warren runner.close() 129*d201506cSStephen Warren 130*d201506cSStephen Warren class ArbitraryAttributeContainer(object): 131*d201506cSStephen Warren pass 132*d201506cSStephen Warren 133*d201506cSStephen Warren ubconfig = ArbitraryAttributeContainer() 134*d201506cSStephen Warren ubconfig.brd = dict() 135*d201506cSStephen Warren ubconfig.env = dict() 136*d201506cSStephen Warren 137*d201506cSStephen Warren modules = [ 138*d201506cSStephen Warren (ubconfig.brd, 'u_boot_board_' + board_type_filename), 139*d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename), 140*d201506cSStephen Warren (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' + 141*d201506cSStephen Warren board_identity_filename), 142*d201506cSStephen Warren ] 143*d201506cSStephen Warren for (dict_to_fill, module_name) in modules: 144*d201506cSStephen Warren try: 145*d201506cSStephen Warren module = __import__(module_name) 146*d201506cSStephen Warren except ImportError: 147*d201506cSStephen Warren continue 148*d201506cSStephen Warren dict_to_fill.update(module.__dict__) 149*d201506cSStephen Warren 150*d201506cSStephen Warren ubconfig.buildconfig = dict() 151*d201506cSStephen Warren 152*d201506cSStephen Warren for conf_file in ('.config', 'include/autoconf.mk'): 153*d201506cSStephen Warren dot_config = build_dir + '/' + conf_file 154*d201506cSStephen Warren if not os.path.exists(dot_config): 155*d201506cSStephen Warren raise Exception(conf_file + ' does not exist; ' + 156*d201506cSStephen Warren 'try passing --build option?') 157*d201506cSStephen Warren 158*d201506cSStephen Warren with open(dot_config, 'rt') as f: 159*d201506cSStephen Warren ini_str = '[root]\n' + f.read() 160*d201506cSStephen Warren ini_sio = StringIO.StringIO(ini_str) 161*d201506cSStephen Warren parser = ConfigParser.RawConfigParser() 162*d201506cSStephen Warren parser.readfp(ini_sio) 163*d201506cSStephen Warren ubconfig.buildconfig.update(parser.items('root')) 164*d201506cSStephen Warren 165*d201506cSStephen Warren ubconfig.test_py_dir = test_py_dir 166*d201506cSStephen Warren ubconfig.source_dir = source_dir 167*d201506cSStephen Warren ubconfig.build_dir = build_dir 168*d201506cSStephen Warren ubconfig.result_dir = result_dir 169*d201506cSStephen Warren ubconfig.persistent_data_dir = persistent_data_dir 170*d201506cSStephen Warren ubconfig.board_type = board_type 171*d201506cSStephen Warren ubconfig.board_identity = board_identity 172*d201506cSStephen Warren 173*d201506cSStephen Warren env_vars = ( 174*d201506cSStephen Warren 'board_type', 175*d201506cSStephen Warren 'board_identity', 176*d201506cSStephen Warren 'source_dir', 177*d201506cSStephen Warren 'test_py_dir', 178*d201506cSStephen Warren 'build_dir', 179*d201506cSStephen Warren 'result_dir', 180*d201506cSStephen Warren 'persistent_data_dir', 181*d201506cSStephen Warren ) 182*d201506cSStephen Warren for v in env_vars: 183*d201506cSStephen Warren os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v) 184*d201506cSStephen Warren 185*d201506cSStephen Warren if board_type == 'sandbox': 186*d201506cSStephen Warren import u_boot_console_sandbox 187*d201506cSStephen Warren console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig) 188*d201506cSStephen Warren else: 189*d201506cSStephen Warren import u_boot_console_exec_attach 190*d201506cSStephen Warren console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig) 191*d201506cSStephen Warren 192*d201506cSStephen Warrendef pytest_generate_tests(metafunc): 193*d201506cSStephen Warren '''pytest hook: parameterize test functions based on custom rules. 194*d201506cSStephen Warren 195*d201506cSStephen Warren If a test function takes parameter(s) (fixture names) of the form brd__xxx 196*d201506cSStephen Warren or env__xxx, the brd and env configuration dictionaries are consulted to 197*d201506cSStephen Warren find the list of values to use for those parameters, and the test is 198*d201506cSStephen Warren parametrized so that it runs once for each combination of values. 199*d201506cSStephen Warren 200*d201506cSStephen Warren Args: 201*d201506cSStephen Warren metafunc: The pytest test function. 202*d201506cSStephen Warren 203*d201506cSStephen Warren Returns: 204*d201506cSStephen Warren Nothing. 205*d201506cSStephen Warren ''' 206*d201506cSStephen Warren 207*d201506cSStephen Warren subconfigs = { 208*d201506cSStephen Warren 'brd': console.config.brd, 209*d201506cSStephen Warren 'env': console.config.env, 210*d201506cSStephen Warren } 211*d201506cSStephen Warren for fn in metafunc.fixturenames: 212*d201506cSStephen Warren parts = fn.split('__') 213*d201506cSStephen Warren if len(parts) < 2: 214*d201506cSStephen Warren continue 215*d201506cSStephen Warren if parts[0] not in subconfigs: 216*d201506cSStephen Warren continue 217*d201506cSStephen Warren subconfig = subconfigs[parts[0]] 218*d201506cSStephen Warren vals = [] 219*d201506cSStephen Warren val = subconfig.get(fn, []) 220*d201506cSStephen Warren # If that exact name is a key in the data source: 221*d201506cSStephen Warren if val: 222*d201506cSStephen Warren # ... use the dict value as a single parameter value. 223*d201506cSStephen Warren vals = (val, ) 224*d201506cSStephen Warren else: 225*d201506cSStephen Warren # ... otherwise, see if there's a key that contains a list of 226*d201506cSStephen Warren # values to use instead. 227*d201506cSStephen Warren vals = subconfig.get(fn + 's', []) 228*d201506cSStephen Warren metafunc.parametrize(fn, vals) 229*d201506cSStephen Warren 230*d201506cSStephen Warren@pytest.fixture(scope='session') 231*d201506cSStephen Warrendef u_boot_console(request): 232*d201506cSStephen Warren '''Generate the value of a test's u_boot_console fixture. 233*d201506cSStephen Warren 234*d201506cSStephen Warren Args: 235*d201506cSStephen Warren request: The pytest request. 236*d201506cSStephen Warren 237*d201506cSStephen Warren Returns: 238*d201506cSStephen Warren The fixture value. 239*d201506cSStephen Warren ''' 240*d201506cSStephen Warren 241*d201506cSStephen Warren return console 242*d201506cSStephen Warren 243*d201506cSStephen Warrentests_not_run = set() 244*d201506cSStephen Warrentests_failed = set() 245*d201506cSStephen Warrentests_skipped = set() 246*d201506cSStephen Warrentests_passed = set() 247*d201506cSStephen Warren 248*d201506cSStephen Warrendef pytest_itemcollected(item): 249*d201506cSStephen Warren '''pytest hook: Called once for each test found during collection. 250*d201506cSStephen Warren 251*d201506cSStephen Warren This enables our custom result analysis code to see the list of all tests 252*d201506cSStephen Warren that should eventually be run. 253*d201506cSStephen Warren 254*d201506cSStephen Warren Args: 255*d201506cSStephen Warren item: The item that was collected. 256*d201506cSStephen Warren 257*d201506cSStephen Warren Returns: 258*d201506cSStephen Warren Nothing. 259*d201506cSStephen Warren ''' 260*d201506cSStephen Warren 261*d201506cSStephen Warren tests_not_run.add(item.name) 262*d201506cSStephen Warren 263*d201506cSStephen Warrendef cleanup(): 264*d201506cSStephen Warren '''Clean up all global state. 265*d201506cSStephen Warren 266*d201506cSStephen Warren Executed (via atexit) once the entire test process is complete. This 267*d201506cSStephen Warren includes logging the status of all tests, and the identity of any failed 268*d201506cSStephen Warren or skipped tests. 269*d201506cSStephen Warren 270*d201506cSStephen Warren Args: 271*d201506cSStephen Warren None. 272*d201506cSStephen Warren 273*d201506cSStephen Warren Returns: 274*d201506cSStephen Warren Nothing. 275*d201506cSStephen Warren ''' 276*d201506cSStephen Warren 277*d201506cSStephen Warren if console: 278*d201506cSStephen Warren console.close() 279*d201506cSStephen Warren if log: 280*d201506cSStephen Warren log.status_pass('%d passed' % len(tests_passed)) 281*d201506cSStephen Warren if tests_skipped: 282*d201506cSStephen Warren log.status_skipped('%d skipped' % len(tests_skipped)) 283*d201506cSStephen Warren for test in tests_skipped: 284*d201506cSStephen Warren log.status_skipped('... ' + test) 285*d201506cSStephen Warren if tests_failed: 286*d201506cSStephen Warren log.status_fail('%d failed' % len(tests_failed)) 287*d201506cSStephen Warren for test in tests_failed: 288*d201506cSStephen Warren log.status_fail('... ' + test) 289*d201506cSStephen Warren if tests_not_run: 290*d201506cSStephen Warren log.status_fail('%d not run' % len(tests_not_run)) 291*d201506cSStephen Warren for test in tests_not_run: 292*d201506cSStephen Warren log.status_fail('... ' + test) 293*d201506cSStephen Warren log.close() 294*d201506cSStephen Warrenatexit.register(cleanup) 295*d201506cSStephen Warren 296*d201506cSStephen Warrendef setup_boardspec(item): 297*d201506cSStephen Warren '''Process any 'boardspec' marker for a test. 298*d201506cSStephen Warren 299*d201506cSStephen Warren Such a marker lists the set of board types that a test does/doesn't 300*d201506cSStephen Warren support. If tests are being executed on an unsupported board, the test is 301*d201506cSStephen Warren marked to be skipped. 302*d201506cSStephen Warren 303*d201506cSStephen Warren Args: 304*d201506cSStephen Warren item: The pytest test item. 305*d201506cSStephen Warren 306*d201506cSStephen Warren Returns: 307*d201506cSStephen Warren Nothing. 308*d201506cSStephen Warren ''' 309*d201506cSStephen Warren 310*d201506cSStephen Warren mark = item.get_marker('boardspec') 311*d201506cSStephen Warren if not mark: 312*d201506cSStephen Warren return 313*d201506cSStephen Warren required_boards = [] 314*d201506cSStephen Warren for board in mark.args: 315*d201506cSStephen Warren if board.startswith('!'): 316*d201506cSStephen Warren if ubconfig.board_type == board[1:]: 317*d201506cSStephen Warren pytest.skip('board not supported') 318*d201506cSStephen Warren return 319*d201506cSStephen Warren else: 320*d201506cSStephen Warren required_boards.append(board) 321*d201506cSStephen Warren if required_boards and ubconfig.board_type not in required_boards: 322*d201506cSStephen Warren pytest.skip('board not supported') 323*d201506cSStephen Warren 324*d201506cSStephen Warrendef setup_buildconfigspec(item): 325*d201506cSStephen Warren '''Process any 'buildconfigspec' marker for a test. 326*d201506cSStephen Warren 327*d201506cSStephen Warren Such a marker lists some U-Boot configuration feature that the test 328*d201506cSStephen Warren requires. If tests are being executed on an U-Boot build that doesn't 329*d201506cSStephen Warren have the required feature, the test is marked to be skipped. 330*d201506cSStephen Warren 331*d201506cSStephen Warren Args: 332*d201506cSStephen Warren item: The pytest test item. 333*d201506cSStephen Warren 334*d201506cSStephen Warren Returns: 335*d201506cSStephen Warren Nothing. 336*d201506cSStephen Warren ''' 337*d201506cSStephen Warren 338*d201506cSStephen Warren mark = item.get_marker('buildconfigspec') 339*d201506cSStephen Warren if not mark: 340*d201506cSStephen Warren return 341*d201506cSStephen Warren for option in mark.args: 342*d201506cSStephen Warren if not ubconfig.buildconfig.get('config_' + option.lower(), None): 343*d201506cSStephen Warren pytest.skip('.config feature not enabled') 344*d201506cSStephen Warren 345*d201506cSStephen Warrendef pytest_runtest_setup(item): 346*d201506cSStephen Warren '''pytest hook: Configure (set up) a test item. 347*d201506cSStephen Warren 348*d201506cSStephen Warren Called once for each test to perform any custom configuration. This hook 349*d201506cSStephen Warren is used to skip the test if certain conditions apply. 350*d201506cSStephen Warren 351*d201506cSStephen Warren Args: 352*d201506cSStephen Warren item: The pytest test item. 353*d201506cSStephen Warren 354*d201506cSStephen Warren Returns: 355*d201506cSStephen Warren Nothing. 356*d201506cSStephen Warren ''' 357*d201506cSStephen Warren 358*d201506cSStephen Warren log.start_section(item.name) 359*d201506cSStephen Warren setup_boardspec(item) 360*d201506cSStephen Warren setup_buildconfigspec(item) 361*d201506cSStephen Warren 362*d201506cSStephen Warrendef pytest_runtest_protocol(item, nextitem): 363*d201506cSStephen Warren '''pytest hook: Called to execute a test. 364*d201506cSStephen Warren 365*d201506cSStephen Warren This hook wraps the standard pytest runtestprotocol() function in order 366*d201506cSStephen Warren to acquire visibility into, and record, each test function's result. 367*d201506cSStephen Warren 368*d201506cSStephen Warren Args: 369*d201506cSStephen Warren item: The pytest test item to execute. 370*d201506cSStephen Warren nextitem: The pytest test item that will be executed after this one. 371*d201506cSStephen Warren 372*d201506cSStephen Warren Returns: 373*d201506cSStephen Warren A list of pytest reports (test result data). 374*d201506cSStephen Warren ''' 375*d201506cSStephen Warren 376*d201506cSStephen Warren reports = runtestprotocol(item, nextitem=nextitem) 377*d201506cSStephen Warren failed = None 378*d201506cSStephen Warren skipped = None 379*d201506cSStephen Warren for report in reports: 380*d201506cSStephen Warren if report.outcome == 'failed': 381*d201506cSStephen Warren failed = report 382*d201506cSStephen Warren break 383*d201506cSStephen Warren if report.outcome == 'skipped': 384*d201506cSStephen Warren if not skipped: 385*d201506cSStephen Warren skipped = report 386*d201506cSStephen Warren 387*d201506cSStephen Warren if failed: 388*d201506cSStephen Warren tests_failed.add(item.name) 389*d201506cSStephen Warren elif skipped: 390*d201506cSStephen Warren tests_skipped.add(item.name) 391*d201506cSStephen Warren else: 392*d201506cSStephen Warren tests_passed.add(item.name) 393*d201506cSStephen Warren tests_not_run.remove(item.name) 394*d201506cSStephen Warren 395*d201506cSStephen Warren try: 396*d201506cSStephen Warren if failed: 397*d201506cSStephen Warren msg = 'FAILED:\n' + str(failed.longrepr) 398*d201506cSStephen Warren log.status_fail(msg) 399*d201506cSStephen Warren elif skipped: 400*d201506cSStephen Warren msg = 'SKIPPED:\n' + str(skipped.longrepr) 401*d201506cSStephen Warren log.status_skipped(msg) 402*d201506cSStephen Warren else: 403*d201506cSStephen Warren log.status_pass('OK') 404*d201506cSStephen Warren except: 405*d201506cSStephen Warren # If something went wrong with logging, it's better to let the test 406*d201506cSStephen Warren # process continue, which may report other exceptions that triggered 407*d201506cSStephen Warren # the logging issue (e.g. console.log wasn't created). Hence, just 408*d201506cSStephen Warren # squash the exception. If the test setup failed due to e.g. syntax 409*d201506cSStephen Warren # error somewhere else, this won't be seen. However, once that issue 410*d201506cSStephen Warren # is fixed, if this exception still exists, it will then be logged as 411*d201506cSStephen Warren # part of the test's stdout. 412*d201506cSStephen Warren import traceback 413*d201506cSStephen Warren print 'Exception occurred while logging runtest status:' 414*d201506cSStephen Warren traceback.print_exc() 415*d201506cSStephen Warren # FIXME: Can we force a test failure here? 416*d201506cSStephen Warren 417*d201506cSStephen Warren log.end_section(item.name) 418*d201506cSStephen Warren 419*d201506cSStephen Warren if failed: 420*d201506cSStephen Warren console.cleanup_spawn() 421*d201506cSStephen Warren 422*d201506cSStephen Warren return reports 423