1#!/usr/bin/env python3
2
3# Yocto Project layer checking tool
4#
5# Copyright (C) 2017 Intel Corporation
6#
7# SPDX-License-Identifier: MIT
8#
9
10import os
11import sys
12import argparse
13import logging
14import time
15import signal
16import shutil
17import collections
18
19scripts_path = os.path.dirname(os.path.realpath(__file__))
20lib_path = scripts_path + '/lib'
21sys.path = sys.path + [lib_path]
22import scriptutils
23import scriptpath
24scriptpath.add_oe_lib_path()
25scriptpath.add_bitbake_lib_path()
26
27from checklayer import LayerType, detect_layers, add_layers, add_layer_dependencies, get_layer_dependencies, get_signatures, check_bblayers, sanity_check_layers
28from oeqa.utils.commands import get_bb_vars
29
30PROGNAME = 'yocto-check-layer'
31CASES_PATHS = [os.path.join(os.path.abspath(os.path.dirname(__file__)),
32                'lib', 'checklayer', 'cases')]
33logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout)
34
35def test_layer(td, layer, test_software_layer_signatures):
36    from checklayer.context import CheckLayerTestContext
37    logger.info("Starting to analyze: %s" % layer['name'])
38    logger.info("----------------------------------------------------------------------")
39
40    tc = CheckLayerTestContext(td=td, logger=logger, layer=layer, test_software_layer_signatures=test_software_layer_signatures)
41    tc.loadTests(CASES_PATHS)
42    return tc.runTests()
43
44def dump_layer_debug(layer):
45    logger.debug("Found layer %s (%s)" % (layer["name"], layer["path"]))
46    collections = layer.get("collections", {})
47    if collections:
48        logger.debug("%s collections: %s" % (layer["name"], ", ".join(collections)))
49
50def main():
51    parser = argparse.ArgumentParser(
52            description="Yocto Project layer checking tool",
53            add_help=False)
54    parser.add_argument('layers', metavar='LAYER_DIR', nargs='+',
55            help='Layer to check')
56    parser.add_argument('-o', '--output-log',
57            help='File to output log (optional)', action='store')
58    parser.add_argument('--dependency', nargs="+",
59            help='Layers to process for dependencies', action='store')
60    parser.add_argument('--no-auto-dependency', help='Disable automatic testing of dependencies',
61            action='store_true')
62    parser.add_argument('--machines', nargs="+",
63            help='List of MACHINEs to be used during testing', action='store')
64    parser.add_argument('--additional-layers', nargs="+",
65            help='List of additional layers to add during testing', action='store')
66    group = parser.add_mutually_exclusive_group()
67    group.add_argument('--with-software-layer-signature-check', action='store_true', dest='test_software_layer_signatures',
68                       default=True,
69                       help='check that software layers do not change signatures (on by default)')
70    group.add_argument('--without-software-layer-signature-check', action='store_false', dest='test_software_layer_signatures',
71                       help='disable signature checking for software layers')
72    parser.add_argument('-n', '--no-auto', help='Disable auto layer discovery',
73            action='store_true')
74    parser.add_argument('-d', '--debug', help='Enable debug output',
75            action='store_true')
76    parser.add_argument('-q', '--quiet', help='Print only errors',
77            action='store_true')
78
79    parser.add_argument('-h', '--help', action='help',
80            default=argparse.SUPPRESS,
81            help='show this help message and exit')
82
83    args = parser.parse_args()
84
85    if args.output_log:
86        fh = logging.FileHandler(args.output_log)
87        fh.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
88        logger.addHandler(fh)
89    if args.debug:
90        logger.setLevel(logging.DEBUG)
91    elif args.quiet:
92        logger.setLevel(logging.ERROR)
93
94    if not 'BUILDDIR' in os.environ:
95        logger.error("You must source the environment before running this script.")
96        logger.error("$ source oe-init-build-env")
97        return 1
98    builddir = os.environ['BUILDDIR']
99    bblayersconf = os.path.join(builddir, 'conf', 'bblayers.conf')
100
101    layers = detect_layers(args.layers, args.no_auto)
102    if not layers:
103        logger.error("Failed to detect layers")
104        return 1
105    if args.additional_layers:
106        additional_layers = detect_layers(args.additional_layers, args.no_auto)
107    else:
108        additional_layers = []
109    if args.dependency:
110        dep_layers = detect_layers(args.dependency, args.no_auto)
111        dep_layers = dep_layers + layers
112    else:
113        dep_layers = layers
114
115    logger.debug("Found additional layers:")
116    for l in additional_layers:
117        dump_layer_debug(l)
118    logger.debug("Found dependency layers:")
119    for l in dep_layers:
120        dump_layer_debug(l)
121
122    if not sanity_check_layers(additional_layers + dep_layers, logger):
123        logger.error("Failed layer validation")
124        return 1
125
126    logger.info("Detected layers:")
127    for layer in layers:
128        if layer['type'] == LayerType.ERROR_BSP_DISTRO:
129            logger.error("%s: Can't be DISTRO and BSP type at the same time."\
130                     " Both conf/distro and conf/machine folders were found."\
131                     % layer['name'])
132            layers.remove(layer)
133        elif layer['type'] == LayerType.ERROR_NO_LAYER_CONF:
134            logger.info("%s: Doesn't have conf/layer.conf file, so ignoring"\
135                     % layer['name'])
136            layers.remove(layer)
137        else:
138            logger.info("%s: %s, %s" % (layer['name'], layer['type'],
139                layer['path']))
140    if not layers:
141        return 1
142
143    # Find all dependencies, and get them checked too
144    if not args.no_auto_dependency:
145        depends = []
146        for layer in layers:
147            layer_depends = get_layer_dependencies(layer, dep_layers, logger)
148            if layer_depends:
149                for d in layer_depends:
150                    if d not in depends:
151                        depends.append(d)
152
153        for d in depends:
154            if d not in layers:
155                logger.info("Adding %s to the list of layers to test, as a dependency", d['name'])
156                layers.append(d)
157
158    shutil.copyfile(bblayersconf, bblayersconf + '.backup')
159    def cleanup_bblayers(signum, frame):
160        shutil.copyfile(bblayersconf + '.backup', bblayersconf)
161        os.unlink(bblayersconf + '.backup')
162    signal.signal(signal.SIGTERM, cleanup_bblayers)
163    signal.signal(signal.SIGINT, cleanup_bblayers)
164
165    td = {}
166    results = collections.OrderedDict()
167    results_status = collections.OrderedDict()
168
169    layers_tested = 0
170    for layer in layers:
171        if layer['type'] in (LayerType.ERROR_NO_LAYER_CONF, LayerType.ERROR_BSP_DISTRO):
172            continue
173
174        # Reset to a clean backup copy for each run
175        shutil.copyfile(bblayersconf + '.backup', bblayersconf)
176
177        if layer['type'] not in (LayerType.CORE, ) and check_bblayers(bblayersconf, layer['path'], logger):
178            logger.info("%s already in %s. To capture initial signatures, layer under test should not present "
179               "in BBLAYERS. Please remove %s from BBLAYERS." % (layer['name'], bblayersconf, layer['name']))
180            results[layer['name']] = None
181            results_status[layer['name']] = 'SKIPPED (Layer under test should not present in BBLAYERS)'
182            continue
183
184        logger.info('')
185        logger.info("Setting up for %s(%s), %s" % (layer['name'], layer['type'],
186            layer['path']))
187
188        missing_dependencies = not add_layer_dependencies(bblayersconf, layer, dep_layers, logger)
189        if not missing_dependencies:
190            for additional_layer in additional_layers:
191                if not add_layer_dependencies(bblayersconf, additional_layer, dep_layers, logger):
192                    missing_dependencies = True
193                    break
194        if missing_dependencies:
195            logger.info('Skipping %s due to missing dependencies.' % layer['name'])
196            results[layer['name']] = None
197            results_status[layer['name']] = 'SKIPPED (Missing dependencies)'
198            layers_tested = layers_tested + 1
199            continue
200
201        if any(map(lambda additional_layer: not add_layers(bblayersconf, [additional_layer], logger),
202                   additional_layers)):
203            logger.info('Skipping %s due to missing additional layers.' % layer['name'])
204            results[layer['name']] = None
205            results_status[layer['name']] = 'SKIPPED (Missing additional layers)'
206            layers_tested = layers_tested + 1
207            continue
208
209        logger.info('Getting initial bitbake variables ...')
210        td['bbvars'] = get_bb_vars()
211        logger.info('Getting initial signatures ...')
212        td['builddir'] = builddir
213        try:
214            td['sigs'], td['tunetasks'] = get_signatures(td['builddir'])
215        except RuntimeError as e:
216            logger.info(str(e))
217            results[layer['name']] = None
218            results_status[layer['name']] = 'FAIL (Generating world signatures)'
219            layers_tested = layers_tested + 1
220            continue
221        td['machines'] = args.machines
222
223        if not add_layers(bblayersconf, [layer], logger):
224            logger.info('Skipping %s ???.' % layer['name'])
225            results[layer['name']] = None
226            results_status[layer['name']] = 'SKIPPED (Unknown)'
227            layers_tested = layers_tested + 1
228            continue
229
230        result = test_layer(td, layer, args.test_software_layer_signatures)
231        results[layer['name']] = result
232        results_status[layer['name']] = 'PASS' if results[layer['name']].wasSuccessful() else 'FAIL'
233        layers_tested = layers_tested + 1
234
235    ret = 0
236    if layers_tested:
237        logger.info('')
238        logger.info('Summary of results:')
239        logger.info('')
240        for layer_name in results_status:
241            logger.info('%s ... %s' % (layer_name, results_status[layer_name]))
242            if not results[layer_name] or not results[layer_name].wasSuccessful():
243                ret = 2 # ret = 1 used for initialization errors
244
245    cleanup_bblayers(None, None)
246
247    return ret
248
249if __name__ == '__main__':
250    try:
251        ret =  main()
252    except Exception:
253        ret = 1
254        import traceback
255        traceback.print_exc()
256    sys.exit(ret)
257