1#!/usr/bin/env python3 2 3# Yocto Project layer checking tool 4# 5# Copyright (C) 2017 Intel Corporation 6# 7# SPDX-License-Identifier: MIT 8# 9 10import os 11import sys 12import argparse 13import logging 14import time 15import signal 16import shutil 17import collections 18 19scripts_path = os.path.dirname(os.path.realpath(__file__)) 20lib_path = scripts_path + '/lib' 21sys.path = sys.path + [lib_path] 22import scriptutils 23import scriptpath 24scriptpath.add_oe_lib_path() 25scriptpath.add_bitbake_lib_path() 26 27from checklayer import LayerType, detect_layers, add_layers, add_layer_dependencies, get_layer_dependencies, get_signatures, check_bblayers, sanity_check_layers 28from oeqa.utils.commands import get_bb_vars 29 30PROGNAME = 'yocto-check-layer' 31CASES_PATHS = [os.path.join(os.path.abspath(os.path.dirname(__file__)), 32 'lib', 'checklayer', 'cases')] 33logger = scriptutils.logger_create(PROGNAME, stream=sys.stdout) 34 35def test_layer(td, layer, test_software_layer_signatures): 36 from checklayer.context import CheckLayerTestContext 37 logger.info("Starting to analyze: %s" % layer['name']) 38 logger.info("----------------------------------------------------------------------") 39 40 tc = CheckLayerTestContext(td=td, logger=logger, layer=layer, test_software_layer_signatures=test_software_layer_signatures) 41 tc.loadTests(CASES_PATHS) 42 return tc.runTests() 43 44def dump_layer_debug(layer): 45 logger.debug("Found layer %s (%s)" % (layer["name"], layer["path"])) 46 collections = layer.get("collections", {}) 47 if collections: 48 logger.debug("%s collections: %s" % (layer["name"], ", ".join(collections))) 49 50def main(): 51 parser = argparse.ArgumentParser( 52 description="Yocto Project layer checking tool", 53 add_help=False) 54 parser.add_argument('layers', metavar='LAYER_DIR', nargs='+', 55 help='Layer to check') 56 parser.add_argument('-o', '--output-log', 57 help='File to output log (optional)', action='store') 58 parser.add_argument('--dependency', nargs="+", 59 help='Layers to process for dependencies', action='store') 60 parser.add_argument('--no-auto-dependency', help='Disable automatic testing of dependencies', 61 action='store_true') 62 parser.add_argument('--machines', nargs="+", 63 help='List of MACHINEs to be used during testing', action='store') 64 parser.add_argument('--additional-layers', nargs="+", 65 help='List of additional layers to add during testing', action='store') 66 group = parser.add_mutually_exclusive_group() 67 group.add_argument('--with-software-layer-signature-check', action='store_true', dest='test_software_layer_signatures', 68 default=True, 69 help='check that software layers do not change signatures (on by default)') 70 group.add_argument('--without-software-layer-signature-check', action='store_false', dest='test_software_layer_signatures', 71 help='disable signature checking for software layers') 72 parser.add_argument('-n', '--no-auto', help='Disable auto layer discovery', 73 action='store_true') 74 parser.add_argument('-d', '--debug', help='Enable debug output', 75 action='store_true') 76 parser.add_argument('-q', '--quiet', help='Print only errors', 77 action='store_true') 78 79 parser.add_argument('-h', '--help', action='help', 80 default=argparse.SUPPRESS, 81 help='show this help message and exit') 82 83 args = parser.parse_args() 84 85 if args.output_log: 86 fh = logging.FileHandler(args.output_log) 87 fh.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) 88 logger.addHandler(fh) 89 if args.debug: 90 logger.setLevel(logging.DEBUG) 91 elif args.quiet: 92 logger.setLevel(logging.ERROR) 93 94 if not 'BUILDDIR' in os.environ: 95 logger.error("You must source the environment before running this script.") 96 logger.error("$ source oe-init-build-env") 97 return 1 98 builddir = os.environ['BUILDDIR'] 99 bblayersconf = os.path.join(builddir, 'conf', 'bblayers.conf') 100 101 layers = detect_layers(args.layers, args.no_auto) 102 if not layers: 103 logger.error("Failed to detect layers") 104 return 1 105 if args.additional_layers: 106 additional_layers = detect_layers(args.additional_layers, args.no_auto) 107 else: 108 additional_layers = [] 109 if args.dependency: 110 dep_layers = detect_layers(args.dependency, args.no_auto) 111 dep_layers = dep_layers + layers 112 else: 113 dep_layers = layers 114 115 logger.debug("Found additional layers:") 116 for l in additional_layers: 117 dump_layer_debug(l) 118 logger.debug("Found dependency layers:") 119 for l in dep_layers: 120 dump_layer_debug(l) 121 122 if not sanity_check_layers(additional_layers + dep_layers, logger): 123 logger.error("Failed layer validation") 124 return 1 125 126 logger.info("Detected layers:") 127 for layer in layers: 128 if layer['type'] == LayerType.ERROR_BSP_DISTRO: 129 logger.error("%s: Can't be DISTRO and BSP type at the same time."\ 130 " Both conf/distro and conf/machine folders were found."\ 131 % layer['name']) 132 layers.remove(layer) 133 elif layer['type'] == LayerType.ERROR_NO_LAYER_CONF: 134 logger.info("%s: Doesn't have conf/layer.conf file, so ignoring"\ 135 % layer['name']) 136 layers.remove(layer) 137 else: 138 logger.info("%s: %s, %s" % (layer['name'], layer['type'], 139 layer['path'])) 140 if not layers: 141 return 1 142 143 # Find all dependencies, and get them checked too 144 if not args.no_auto_dependency: 145 depends = [] 146 for layer in layers: 147 layer_depends = get_layer_dependencies(layer, dep_layers, logger) 148 if layer_depends: 149 for d in layer_depends: 150 if d not in depends: 151 depends.append(d) 152 153 for d in depends: 154 if d not in layers: 155 logger.info("Adding %s to the list of layers to test, as a dependency", d['name']) 156 layers.append(d) 157 158 shutil.copyfile(bblayersconf, bblayersconf + '.backup') 159 def cleanup_bblayers(signum, frame): 160 shutil.copyfile(bblayersconf + '.backup', bblayersconf) 161 os.unlink(bblayersconf + '.backup') 162 signal.signal(signal.SIGTERM, cleanup_bblayers) 163 signal.signal(signal.SIGINT, cleanup_bblayers) 164 165 td = {} 166 results = collections.OrderedDict() 167 results_status = collections.OrderedDict() 168 169 layers_tested = 0 170 for layer in layers: 171 if layer['type'] in (LayerType.ERROR_NO_LAYER_CONF, LayerType.ERROR_BSP_DISTRO): 172 continue 173 174 # Reset to a clean backup copy for each run 175 shutil.copyfile(bblayersconf + '.backup', bblayersconf) 176 177 if layer['type'] not in (LayerType.CORE, ) and check_bblayers(bblayersconf, layer['path'], logger): 178 logger.info("%s already in %s. To capture initial signatures, layer under test should not present " 179 "in BBLAYERS. Please remove %s from BBLAYERS." % (layer['name'], bblayersconf, layer['name'])) 180 results[layer['name']] = None 181 results_status[layer['name']] = 'SKIPPED (Layer under test should not present in BBLAYERS)' 182 continue 183 184 logger.info('') 185 logger.info("Setting up for %s(%s), %s" % (layer['name'], layer['type'], 186 layer['path'])) 187 188 missing_dependencies = not add_layer_dependencies(bblayersconf, layer, dep_layers, logger) 189 if not missing_dependencies: 190 for additional_layer in additional_layers: 191 if not add_layer_dependencies(bblayersconf, additional_layer, dep_layers, logger): 192 missing_dependencies = True 193 break 194 if missing_dependencies: 195 logger.info('Skipping %s due to missing dependencies.' % layer['name']) 196 results[layer['name']] = None 197 results_status[layer['name']] = 'SKIPPED (Missing dependencies)' 198 layers_tested = layers_tested + 1 199 continue 200 201 if any(map(lambda additional_layer: not add_layers(bblayersconf, [additional_layer], logger), 202 additional_layers)): 203 logger.info('Skipping %s due to missing additional layers.' % layer['name']) 204 results[layer['name']] = None 205 results_status[layer['name']] = 'SKIPPED (Missing additional layers)' 206 layers_tested = layers_tested + 1 207 continue 208 209 logger.info('Getting initial bitbake variables ...') 210 td['bbvars'] = get_bb_vars() 211 logger.info('Getting initial signatures ...') 212 td['builddir'] = builddir 213 try: 214 td['sigs'], td['tunetasks'] = get_signatures(td['builddir']) 215 except RuntimeError as e: 216 logger.info(str(e)) 217 results[layer['name']] = None 218 results_status[layer['name']] = 'FAIL (Generating world signatures)' 219 layers_tested = layers_tested + 1 220 continue 221 td['machines'] = args.machines 222 223 if not add_layers(bblayersconf, [layer], logger): 224 logger.info('Skipping %s ???.' % layer['name']) 225 results[layer['name']] = None 226 results_status[layer['name']] = 'SKIPPED (Unknown)' 227 layers_tested = layers_tested + 1 228 continue 229 230 result = test_layer(td, layer, args.test_software_layer_signatures) 231 results[layer['name']] = result 232 results_status[layer['name']] = 'PASS' if results[layer['name']].wasSuccessful() else 'FAIL' 233 layers_tested = layers_tested + 1 234 235 ret = 0 236 if layers_tested: 237 logger.info('') 238 logger.info('Summary of results:') 239 logger.info('') 240 for layer_name in results_status: 241 logger.info('%s ... %s' % (layer_name, results_status[layer_name])) 242 if not results[layer_name] or not results[layer_name].wasSuccessful(): 243 ret = 2 # ret = 1 used for initialization errors 244 245 cleanup_bblayers(None, None) 246 247 return ret 248 249if __name__ == '__main__': 250 try: 251 ret = main() 252 except Exception: 253 ret = 1 254 import traceback 255 traceback.print_exc() 256 sys.exit(ret) 257