1# Yocto Project layer check tool 2# 3# Copyright (C) 2017 Intel Corporation 4# 5# SPDX-License-Identifier: MIT 6# 7 8import os 9import re 10import subprocess 11from enum import Enum 12 13import bb.tinfoil 14 15class LayerType(Enum): 16 BSP = 0 17 DISTRO = 1 18 SOFTWARE = 2 19 ERROR_NO_LAYER_CONF = 98 20 ERROR_BSP_DISTRO = 99 21 22def _get_configurations(path): 23 configs = [] 24 25 for f in os.listdir(path): 26 file_path = os.path.join(path, f) 27 if os.path.isfile(file_path) and f.endswith('.conf'): 28 configs.append(f[:-5]) # strip .conf 29 return configs 30 31def _get_layer_collections(layer_path, lconf=None, data=None): 32 import bb.parse 33 import bb.data 34 35 if lconf is None: 36 lconf = os.path.join(layer_path, 'conf', 'layer.conf') 37 38 if data is None: 39 ldata = bb.data.init() 40 bb.parse.init_parser(ldata) 41 else: 42 ldata = data.createCopy() 43 44 ldata.setVar('LAYERDIR', layer_path) 45 try: 46 ldata = bb.parse.handle(lconf, ldata, include=True) 47 except: 48 raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path) 49 ldata.expandVarref('LAYERDIR') 50 51 collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split() 52 if not collections: 53 name = os.path.basename(layer_path) 54 collections = [name] 55 56 collections = {c: {} for c in collections} 57 for name in collections: 58 priority = ldata.getVar('BBFILE_PRIORITY_%s' % name) 59 pattern = ldata.getVar('BBFILE_PATTERN_%s' % name) 60 depends = ldata.getVar('LAYERDEPENDS_%s' % name) 61 compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name) 62 collections[name]['priority'] = priority 63 collections[name]['pattern'] = pattern 64 collections[name]['depends'] = depends 65 collections[name]['compat'] = compat 66 67 return collections 68 69def _detect_layer(layer_path): 70 """ 71 Scans layer directory to detect what type of layer 72 is BSP, Distro or Software. 73 74 Returns a dictionary with layer name, type and path. 75 """ 76 77 layer = {} 78 layer_name = os.path.basename(layer_path) 79 80 layer['name'] = layer_name 81 layer['path'] = layer_path 82 layer['conf'] = {} 83 84 if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')): 85 layer['type'] = LayerType.ERROR_NO_LAYER_CONF 86 return layer 87 88 machine_conf = os.path.join(layer_path, 'conf', 'machine') 89 distro_conf = os.path.join(layer_path, 'conf', 'distro') 90 91 is_bsp = False 92 is_distro = False 93 94 if os.path.isdir(machine_conf): 95 machines = _get_configurations(machine_conf) 96 if machines: 97 is_bsp = True 98 99 if os.path.isdir(distro_conf): 100 distros = _get_configurations(distro_conf) 101 if distros: 102 is_distro = True 103 104 if is_bsp and is_distro: 105 layer['type'] = LayerType.ERROR_BSP_DISTRO 106 elif is_bsp: 107 layer['type'] = LayerType.BSP 108 layer['conf']['machines'] = machines 109 elif is_distro: 110 layer['type'] = LayerType.DISTRO 111 layer['conf']['distros'] = distros 112 else: 113 layer['type'] = LayerType.SOFTWARE 114 115 layer['collections'] = _get_layer_collections(layer['path']) 116 117 return layer 118 119def detect_layers(layer_directories, no_auto): 120 layers = [] 121 122 for directory in layer_directories: 123 directory = os.path.realpath(directory) 124 if directory[-1] == '/': 125 directory = directory[0:-1] 126 127 if no_auto: 128 conf_dir = os.path.join(directory, 'conf') 129 if os.path.isdir(conf_dir): 130 layer = _detect_layer(directory) 131 if layer: 132 layers.append(layer) 133 else: 134 for root, dirs, files in os.walk(directory): 135 dir_name = os.path.basename(root) 136 conf_dir = os.path.join(root, 'conf') 137 if os.path.isdir(conf_dir): 138 layer = _detect_layer(root) 139 if layer: 140 layers.append(layer) 141 142 return layers 143 144def _find_layer_depends(depend, layers): 145 for layer in layers: 146 for collection in layer['collections']: 147 if depend == collection: 148 return layer 149 return None 150 151def add_layer_dependencies(bblayersconf, layer, layers, logger): 152 def recurse_dependencies(depends, layer, layers, logger, ret = []): 153 logger.debug('Processing dependencies %s for layer %s.' % \ 154 (depends, layer['name'])) 155 156 for depend in depends.split(): 157 # core (oe-core) is suppose to be provided 158 if depend == 'core': 159 continue 160 161 layer_depend = _find_layer_depends(depend, layers) 162 if not layer_depend: 163 logger.error('Layer %s depends on %s and isn\'t found.' % \ 164 (layer['name'], depend)) 165 ret = None 166 continue 167 168 # We keep processing, even if ret is None, this allows us to report 169 # multiple errors at once 170 if ret is not None and layer_depend not in ret: 171 ret.append(layer_depend) 172 else: 173 # we might have processed this dependency already, in which case 174 # we should not do it again (avoid recursive loop) 175 continue 176 177 # Recursively process... 178 if 'collections' not in layer_depend: 179 continue 180 181 for collection in layer_depend['collections']: 182 collect_deps = layer_depend['collections'][collection]['depends'] 183 if not collect_deps: 184 continue 185 ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret) 186 187 return ret 188 189 layer_depends = [] 190 for collection in layer['collections']: 191 depends = layer['collections'][collection]['depends'] 192 if not depends: 193 continue 194 195 layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends) 196 197 # Note: [] (empty) is allowed, None is not! 198 if layer_depends is None: 199 return False 200 else: 201 add_layers(bblayersconf, layer_depends, logger) 202 203 return True 204 205def add_layers(bblayersconf, layers, logger): 206 # Don't add a layer that is already present. 207 added = set() 208 output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8') 209 for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE): 210 added.add(path) 211 212 with open(bblayersconf, 'a+') as f: 213 for layer in layers: 214 logger.info('Adding layer %s' % layer['name']) 215 name = layer['name'] 216 path = layer['path'] 217 if path in added: 218 logger.info('%s is already in %s' % (name, bblayersconf)) 219 else: 220 added.add(path) 221 f.write("\nBBLAYERS += \"%s\"\n" % path) 222 return True 223 224def check_command(error_msg, cmd, cwd=None): 225 ''' 226 Run a command under a shell, capture stdout and stderr in a single stream, 227 throw an error when command returns non-zero exit code. Returns the output. 228 ''' 229 230 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) 231 output, _ = p.communicate() 232 if p.returncode: 233 msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8')) 234 raise RuntimeError(msg) 235 return output 236 237def get_signatures(builddir, failsafe=False, machine=None): 238 import re 239 240 # some recipes needs to be excluded like meta-world-pkgdata 241 # because a layer can add recipes to a world build so signature 242 # will be change 243 exclude_recipes = ('meta-world-pkgdata',) 244 245 sigs = {} 246 tune2tasks = {} 247 248 cmd = 'BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" ' 249 if machine: 250 cmd += 'MACHINE=%s ' % machine 251 cmd += 'bitbake ' 252 if failsafe: 253 cmd += '-k ' 254 cmd += '-S none world' 255 sigs_file = os.path.join(builddir, 'locked-sigs.inc') 256 if os.path.exists(sigs_file): 257 os.unlink(sigs_file) 258 try: 259 check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.', 260 cmd, builddir) 261 except RuntimeError as ex: 262 if failsafe and os.path.exists(sigs_file): 263 # Ignore the error here. Most likely some recipes active 264 # in a world build lack some dependencies. There is a 265 # separate test_machine_world_build which exposes the 266 # failure. 267 pass 268 else: 269 raise 270 271 sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$") 272 tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") 273 current_tune = None 274 with open(sigs_file, 'r') as f: 275 for line in f.readlines(): 276 line = line.strip() 277 t = tune_regex.search(line) 278 if t: 279 current_tune = t.group('tune') 280 s = sig_regex.match(line) 281 if s: 282 exclude = False 283 for er in exclude_recipes: 284 (recipe, task) = s.group('task').split(':') 285 if er == recipe: 286 exclude = True 287 break 288 if exclude: 289 continue 290 291 sigs[s.group('task')] = s.group('hash') 292 tune2tasks.setdefault(current_tune, []).append(s.group('task')) 293 294 if not sigs: 295 raise RuntimeError('Can\'t load signatures from %s' % sigs_file) 296 297 return (sigs, tune2tasks) 298 299def get_depgraph(targets=['world'], failsafe=False): 300 ''' 301 Returns the dependency graph for the given target(s). 302 The dependency graph is taken directly from DepTreeEvent. 303 ''' 304 depgraph = None 305 with bb.tinfoil.Tinfoil() as tinfoil: 306 tinfoil.prepare(config_only=False) 307 tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted']) 308 if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'): 309 raise RuntimeError('starting generateDepTreeEvent failed') 310 while True: 311 event = tinfoil.wait_event(timeout=1000) 312 if event: 313 if isinstance(event, bb.command.CommandFailed): 314 raise RuntimeError('Generating dependency information failed: %s' % event.error) 315 elif isinstance(event, bb.command.CommandCompleted): 316 break 317 elif isinstance(event, bb.event.NoProvider): 318 if failsafe: 319 # The event is informational, we will get information about the 320 # remaining dependencies eventually and thus can ignore this 321 # here like we do in get_signatures(), if desired. 322 continue 323 if event._reasons: 324 raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons)) 325 else: 326 raise RuntimeError('Nothing provides %s.' % (event._item)) 327 elif isinstance(event, bb.event.DepTreeGenerated): 328 depgraph = event._depgraph 329 330 if depgraph is None: 331 raise RuntimeError('Could not retrieve the depgraph.') 332 return depgraph 333 334def compare_signatures(old_sigs, curr_sigs): 335 ''' 336 Compares the result of two get_signatures() calls. Returns None if no 337 problems found, otherwise a string that can be used as additional 338 explanation in self.fail(). 339 ''' 340 # task -> (old signature, new signature) 341 sig_diff = {} 342 for task in old_sigs: 343 if task in curr_sigs and \ 344 old_sigs[task] != curr_sigs[task]: 345 sig_diff[task] = (old_sigs[task], curr_sigs[task]) 346 347 if not sig_diff: 348 return None 349 350 # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures() 351 # uses <pn>:<taskname>. Need to convert sometimes. The output follows 352 # the convention from get_signatures() because that seems closer to 353 # normal bitbake output. 354 def sig2graph(task): 355 pn, taskname = task.rsplit(':', 1) 356 return pn + '.' + taskname 357 def graph2sig(task): 358 pn, taskname = task.rsplit('.', 1) 359 return pn + ':' + taskname 360 depgraph = get_depgraph(failsafe=True) 361 depends = depgraph['tdepends'] 362 363 # If a task A has a changed signature, but none of its 364 # dependencies, then we need to report it because it is 365 # the one which introduces a change. Any task depending on 366 # A (directly or indirectly) will also have a changed 367 # signature, but we don't need to report it. It might have 368 # its own changes, which will become apparent once the 369 # issues that we do report are fixed and the test gets run 370 # again. 371 sig_diff_filtered = [] 372 for task, (old_sig, new_sig) in sig_diff.items(): 373 deps_tainted = False 374 for dep in depends.get(sig2graph(task), ()): 375 if graph2sig(dep) in sig_diff: 376 deps_tainted = True 377 break 378 if not deps_tainted: 379 sig_diff_filtered.append((task, old_sig, new_sig)) 380 381 msg = [] 382 msg.append('%d signatures changed, initial differences (first hash before, second after):' % 383 len(sig_diff)) 384 for diff in sorted(sig_diff_filtered): 385 recipe, taskname = diff[0].rsplit(':', 1) 386 cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \ 387 (recipe, taskname, diff[1], diff[2]) 388 msg.append(' %s: %s -> %s' % diff) 389 msg.append(' %s' % cmd) 390 try: 391 output = check_command('Determining signature difference failed.', 392 cmd).decode('utf-8') 393 except RuntimeError as error: 394 output = str(error) 395 if output: 396 msg.extend([' ' + line for line in output.splitlines()]) 397 msg.append('') 398 return '\n'.join(msg) 399