xref: /openbmc/u-boot/test/py/conftest.py (revision e3f44f5c)
1# Copyright (c) 2015 Stephen Warren
2# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved.
3#
4# SPDX-License-Identifier: GPL-2.0
5
6# Implementation of pytest run-time hook functions. These are invoked by
7# pytest at certain points during operation, e.g. startup, for each executed
8# test, at shutdown etc. These hooks perform functions such as:
9# - Parsing custom command-line options.
10# - Pullilng in user-specified board configuration.
11# - Creating the U-Boot console test fixture.
12# - Creating the HTML log file.
13# - Monitoring each test's results.
14# - Implementing custom pytest markers.
15
16import atexit
17import errno
18import os
19import os.path
20import pytest
21from _pytest.runner import runtestprotocol
22import ConfigParser
23import re
24import StringIO
25import sys
26
27# Globals: The HTML log file, and the connection to the U-Boot console.
28log = None
29console = None
30
31def mkdir_p(path):
32    """Create a directory path.
33
34    This includes creating any intermediate/parent directories. Any errors
35    caused due to already extant directories are ignored.
36
37    Args:
38        path: The directory path to create.
39
40    Returns:
41        Nothing.
42    """
43
44    try:
45        os.makedirs(path)
46    except OSError as exc:
47        if exc.errno == errno.EEXIST and os.path.isdir(path):
48            pass
49        else:
50            raise
51
52def pytest_addoption(parser):
53    """pytest hook: Add custom command-line options to the cmdline parser.
54
55    Args:
56        parser: The pytest command-line parser.
57
58    Returns:
59        Nothing.
60    """
61
62    parser.addoption('--build-dir', default=None,
63        help='U-Boot build directory (O=)')
64    parser.addoption('--result-dir', default=None,
65        help='U-Boot test result/tmp directory')
66    parser.addoption('--persistent-data-dir', default=None,
67        help='U-Boot test persistent generated data directory')
68    parser.addoption('--board-type', '--bd', '-B', default='sandbox',
69        help='U-Boot board type')
70    parser.addoption('--board-identity', '--id', default='na',
71        help='U-Boot board identity/instance')
72    parser.addoption('--build', default=False, action='store_true',
73        help='Compile U-Boot before running tests')
74    parser.addoption('--gdbserver', default=None,
75        help='Run sandbox under gdbserver. The argument is the channel '+
76        'over which gdbserver should communicate, e.g. localhost:1234')
77
78def pytest_configure(config):
79    """pytest hook: Perform custom initialization at startup time.
80
81    Args:
82        config: The pytest configuration.
83
84    Returns:
85        Nothing.
86    """
87
88    global log
89    global console
90    global ubconfig
91
92    test_py_dir = os.path.dirname(os.path.abspath(__file__))
93    source_dir = os.path.dirname(os.path.dirname(test_py_dir))
94
95    board_type = config.getoption('board_type')
96    board_type_filename = board_type.replace('-', '_')
97
98    board_identity = config.getoption('board_identity')
99    board_identity_filename = board_identity.replace('-', '_')
100
101    build_dir = config.getoption('build_dir')
102    if not build_dir:
103        build_dir = source_dir + '/build-' + board_type
104    mkdir_p(build_dir)
105
106    result_dir = config.getoption('result_dir')
107    if not result_dir:
108        result_dir = build_dir
109    mkdir_p(result_dir)
110
111    persistent_data_dir = config.getoption('persistent_data_dir')
112    if not persistent_data_dir:
113        persistent_data_dir = build_dir + '/persistent-data'
114    mkdir_p(persistent_data_dir)
115
116    gdbserver = config.getoption('gdbserver')
117    if gdbserver and board_type != 'sandbox':
118        raise Exception('--gdbserver only supported with sandbox')
119
120    import multiplexed_log
121    log = multiplexed_log.Logfile(result_dir + '/test-log.html')
122
123    if config.getoption('build'):
124        if build_dir != source_dir:
125            o_opt = 'O=%s' % build_dir
126        else:
127            o_opt = ''
128        cmds = (
129            ['make', o_opt, '-s', board_type + '_defconfig'],
130            ['make', o_opt, '-s', '-j8'],
131        )
132        with log.section('make'):
133            runner = log.get_runner('make', sys.stdout)
134            for cmd in cmds:
135                runner.run(cmd, cwd=source_dir)
136            runner.close()
137            log.status_pass('OK')
138
139    class ArbitraryAttributeContainer(object):
140        pass
141
142    ubconfig = ArbitraryAttributeContainer()
143    ubconfig.brd = dict()
144    ubconfig.env = dict()
145
146    modules = [
147        (ubconfig.brd, 'u_boot_board_' + board_type_filename),
148        (ubconfig.env, 'u_boot_boardenv_' + board_type_filename),
149        (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' +
150            board_identity_filename),
151    ]
152    for (dict_to_fill, module_name) in modules:
153        try:
154            module = __import__(module_name)
155        except ImportError:
156            continue
157        dict_to_fill.update(module.__dict__)
158
159    ubconfig.buildconfig = dict()
160
161    for conf_file in ('.config', 'include/autoconf.mk'):
162        dot_config = build_dir + '/' + conf_file
163        if not os.path.exists(dot_config):
164            raise Exception(conf_file + ' does not exist; ' +
165                'try passing --build option?')
166
167        with open(dot_config, 'rt') as f:
168            ini_str = '[root]\n' + f.read()
169            ini_sio = StringIO.StringIO(ini_str)
170            parser = ConfigParser.RawConfigParser()
171            parser.readfp(ini_sio)
172            ubconfig.buildconfig.update(parser.items('root'))
173
174    ubconfig.test_py_dir = test_py_dir
175    ubconfig.source_dir = source_dir
176    ubconfig.build_dir = build_dir
177    ubconfig.result_dir = result_dir
178    ubconfig.persistent_data_dir = persistent_data_dir
179    ubconfig.board_type = board_type
180    ubconfig.board_identity = board_identity
181    ubconfig.gdbserver = gdbserver
182    ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb'
183
184    env_vars = (
185        'board_type',
186        'board_identity',
187        'source_dir',
188        'test_py_dir',
189        'build_dir',
190        'result_dir',
191        'persistent_data_dir',
192    )
193    for v in env_vars:
194        os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v)
195
196    if board_type.startswith('sandbox'):
197        import u_boot_console_sandbox
198        console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig)
199    else:
200        import u_boot_console_exec_attach
201        console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig)
202
203re_ut_test_list = re.compile(r'_u_boot_list_2_(dm|env)_test_2_\1_test_(.*)\s*$')
204def generate_ut_subtest(metafunc, fixture_name):
205    """Provide parametrization for a ut_subtest fixture.
206
207    Determines the set of unit tests built into a U-Boot binary by parsing the
208    list of symbols generated by the build process. Provides this information
209    to test functions by parameterizing their ut_subtest fixture parameter.
210
211    Args:
212        metafunc: The pytest test function.
213        fixture_name: The fixture name to test.
214
215    Returns:
216        Nothing.
217    """
218
219    fn = console.config.build_dir + '/u-boot.sym'
220    try:
221        with open(fn, 'rt') as f:
222            lines = f.readlines()
223    except:
224        lines = []
225    lines.sort()
226
227    vals = []
228    for l in lines:
229        m = re_ut_test_list.search(l)
230        if not m:
231            continue
232        vals.append(m.group(1) + ' ' + m.group(2))
233
234    ids = ['ut_' + s.replace(' ', '_') for s in vals]
235    metafunc.parametrize(fixture_name, vals, ids=ids)
236
237def generate_config(metafunc, fixture_name):
238    """Provide parametrization for {env,brd}__ fixtures.
239
240    If a test function takes parameter(s) (fixture names) of the form brd__xxx
241    or env__xxx, the brd and env configuration dictionaries are consulted to
242    find the list of values to use for those parameters, and the test is
243    parametrized so that it runs once for each combination of values.
244
245    Args:
246        metafunc: The pytest test function.
247        fixture_name: The fixture name to test.
248
249    Returns:
250        Nothing.
251    """
252
253    subconfigs = {
254        'brd': console.config.brd,
255        'env': console.config.env,
256    }
257    parts = fixture_name.split('__')
258    if len(parts) < 2:
259        return
260    if parts[0] not in subconfigs:
261        return
262    subconfig = subconfigs[parts[0]]
263    vals = []
264    val = subconfig.get(fixture_name, [])
265    # If that exact name is a key in the data source:
266    if val:
267        # ... use the dict value as a single parameter value.
268        vals = (val, )
269    else:
270        # ... otherwise, see if there's a key that contains a list of
271        # values to use instead.
272        vals = subconfig.get(fixture_name+ 's', [])
273    def fixture_id(index, val):
274        try:
275            return val['fixture_id']
276        except:
277            return fixture_name + str(index)
278    ids = [fixture_id(index, val) for (index, val) in enumerate(vals)]
279    metafunc.parametrize(fixture_name, vals, ids=ids)
280
281def pytest_generate_tests(metafunc):
282    """pytest hook: parameterize test functions based on custom rules.
283
284    Check each test function parameter (fixture name) to see if it is one of
285    our custom names, and if so, provide the correct parametrization for that
286    parameter.
287
288    Args:
289        metafunc: The pytest test function.
290
291    Returns:
292        Nothing.
293    """
294
295    for fn in metafunc.fixturenames:
296        if fn == 'ut_subtest':
297            generate_ut_subtest(metafunc, fn)
298            continue
299        generate_config(metafunc, fn)
300
301@pytest.fixture(scope='session')
302def u_boot_log(request):
303     """Generate the value of a test's log fixture.
304
305     Args:
306         request: The pytest request.
307
308     Returns:
309         The fixture value.
310     """
311
312     return console.log
313
314@pytest.fixture(scope='session')
315def u_boot_config(request):
316     """Generate the value of a test's u_boot_config fixture.
317
318     Args:
319         request: The pytest request.
320
321     Returns:
322         The fixture value.
323     """
324
325     return console.config
326
327@pytest.fixture(scope='function')
328def u_boot_console(request):
329    """Generate the value of a test's u_boot_console fixture.
330
331    Args:
332        request: The pytest request.
333
334    Returns:
335        The fixture value.
336    """
337
338    console.ensure_spawned()
339    return console
340
341anchors = {}
342tests_not_run = []
343tests_failed = []
344tests_xpassed = []
345tests_xfailed = []
346tests_skipped = []
347tests_passed = []
348
349def pytest_itemcollected(item):
350    """pytest hook: Called once for each test found during collection.
351
352    This enables our custom result analysis code to see the list of all tests
353    that should eventually be run.
354
355    Args:
356        item: The item that was collected.
357
358    Returns:
359        Nothing.
360    """
361
362    tests_not_run.append(item.name)
363
364def cleanup():
365    """Clean up all global state.
366
367    Executed (via atexit) once the entire test process is complete. This
368    includes logging the status of all tests, and the identity of any failed
369    or skipped tests.
370
371    Args:
372        None.
373
374    Returns:
375        Nothing.
376    """
377
378    if console:
379        console.close()
380    if log:
381        with log.section('Status Report', 'status_report'):
382            log.status_pass('%d passed' % len(tests_passed))
383            if tests_skipped:
384                log.status_skipped('%d skipped' % len(tests_skipped))
385                for test in tests_skipped:
386                    anchor = anchors.get(test, None)
387                    log.status_skipped('... ' + test, anchor)
388            if tests_xpassed:
389                log.status_xpass('%d xpass' % len(tests_xpassed))
390                for test in tests_xpassed:
391                    anchor = anchors.get(test, None)
392                    log.status_xpass('... ' + test, anchor)
393            if tests_xfailed:
394                log.status_xfail('%d xfail' % len(tests_xfailed))
395                for test in tests_xfailed:
396                    anchor = anchors.get(test, None)
397                    log.status_xfail('... ' + test, anchor)
398            if tests_failed:
399                log.status_fail('%d failed' % len(tests_failed))
400                for test in tests_failed:
401                    anchor = anchors.get(test, None)
402                    log.status_fail('... ' + test, anchor)
403            if tests_not_run:
404                log.status_fail('%d not run' % len(tests_not_run))
405                for test in tests_not_run:
406                    anchor = anchors.get(test, None)
407                    log.status_fail('... ' + test, anchor)
408        log.close()
409atexit.register(cleanup)
410
411def setup_boardspec(item):
412    """Process any 'boardspec' marker for a test.
413
414    Such a marker lists the set of board types that a test does/doesn't
415    support. If tests are being executed on an unsupported board, the test is
416    marked to be skipped.
417
418    Args:
419        item: The pytest test item.
420
421    Returns:
422        Nothing.
423    """
424
425    mark = item.get_marker('boardspec')
426    if not mark:
427        return
428    required_boards = []
429    for board in mark.args:
430        if board.startswith('!'):
431            if ubconfig.board_type == board[1:]:
432                pytest.skip('board not supported')
433                return
434        else:
435            required_boards.append(board)
436    if required_boards and ubconfig.board_type not in required_boards:
437        pytest.skip('board not supported')
438
439def setup_buildconfigspec(item):
440    """Process any 'buildconfigspec' marker for a test.
441
442    Such a marker lists some U-Boot configuration feature that the test
443    requires. If tests are being executed on an U-Boot build that doesn't
444    have the required feature, the test is marked to be skipped.
445
446    Args:
447        item: The pytest test item.
448
449    Returns:
450        Nothing.
451    """
452
453    mark = item.get_marker('buildconfigspec')
454    if not mark:
455        return
456    for option in mark.args:
457        if not ubconfig.buildconfig.get('config_' + option.lower(), None):
458            pytest.skip('.config feature not enabled')
459
460def start_test_section(item):
461    anchors[item.name] = log.start_section(item.name)
462
463def pytest_runtest_setup(item):
464    """pytest hook: Configure (set up) a test item.
465
466    Called once for each test to perform any custom configuration. This hook
467    is used to skip the test if certain conditions apply.
468
469    Args:
470        item: The pytest test item.
471
472    Returns:
473        Nothing.
474    """
475
476    start_test_section(item)
477    setup_boardspec(item)
478    setup_buildconfigspec(item)
479
480def pytest_runtest_protocol(item, nextitem):
481    """pytest hook: Called to execute a test.
482
483    This hook wraps the standard pytest runtestprotocol() function in order
484    to acquire visibility into, and record, each test function's result.
485
486    Args:
487        item: The pytest test item to execute.
488        nextitem: The pytest test item that will be executed after this one.
489
490    Returns:
491        A list of pytest reports (test result data).
492    """
493
494    reports = runtestprotocol(item, nextitem=nextitem)
495
496    # In pytest 3, runtestprotocol() may not call pytest_runtest_setup() if
497    # the test is skipped. That call is required to create the test's section
498    # in the log file. The call to log.end_section() requires that the log
499    # contain a section for this test. Create a section for the test if it
500    # doesn't already exist.
501    if not item.name in anchors:
502        start_test_section(item)
503
504    failure_cleanup = False
505    test_list = tests_passed
506    msg = 'OK'
507    msg_log = log.status_pass
508    for report in reports:
509        if report.outcome == 'failed':
510            if hasattr(report, 'wasxfail'):
511                test_list = tests_xpassed
512                msg = 'XPASSED'
513                msg_log = log.status_xpass
514            else:
515                failure_cleanup = True
516                test_list = tests_failed
517                msg = 'FAILED:\n' + str(report.longrepr)
518                msg_log = log.status_fail
519            break
520        if report.outcome == 'skipped':
521            if hasattr(report, 'wasxfail'):
522                failure_cleanup = True
523                test_list = tests_xfailed
524                msg = 'XFAILED:\n' + str(report.longrepr)
525                msg_log = log.status_xfail
526                break
527            test_list = tests_skipped
528            msg = 'SKIPPED:\n' + str(report.longrepr)
529            msg_log = log.status_skipped
530
531    if failure_cleanup:
532        console.drain_console()
533
534    test_list.append(item.name)
535    tests_not_run.remove(item.name)
536
537    try:
538        msg_log(msg)
539    except:
540        # If something went wrong with logging, it's better to let the test
541        # process continue, which may report other exceptions that triggered
542        # the logging issue (e.g. console.log wasn't created). Hence, just
543        # squash the exception. If the test setup failed due to e.g. syntax
544        # error somewhere else, this won't be seen. However, once that issue
545        # is fixed, if this exception still exists, it will then be logged as
546        # part of the test's stdout.
547        import traceback
548        print 'Exception occurred while logging runtest status:'
549        traceback.print_exc()
550        # FIXME: Can we force a test failure here?
551
552    log.end_section(item.name)
553
554    if failure_cleanup:
555        console.cleanup_spawn()
556
557    return reports
558