1# Yocto Project layer check tool
2#
3# Copyright (C) 2017 Intel Corporation
4#
5# SPDX-License-Identifier: MIT
6#
7
8import os
9import re
10import subprocess
11from enum import Enum
12
13import bb.tinfoil
14
15class LayerType(Enum):
16    BSP = 0
17    DISTRO = 1
18    SOFTWARE = 2
19    ERROR_NO_LAYER_CONF = 98
20    ERROR_BSP_DISTRO = 99
21
22def _get_configurations(path):
23    configs = []
24
25    for f in os.listdir(path):
26        file_path = os.path.join(path, f)
27        if os.path.isfile(file_path) and f.endswith('.conf'):
28            configs.append(f[:-5]) # strip .conf
29    return configs
30
31def _get_layer_collections(layer_path, lconf=None, data=None):
32    import bb.parse
33    import bb.data
34
35    if lconf is None:
36        lconf = os.path.join(layer_path, 'conf', 'layer.conf')
37
38    if data is None:
39        ldata = bb.data.init()
40        bb.parse.init_parser(ldata)
41    else:
42        ldata = data.createCopy()
43
44    ldata.setVar('LAYERDIR', layer_path)
45    try:
46        ldata = bb.parse.handle(lconf, ldata, include=True)
47    except:
48        raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
49    ldata.expandVarref('LAYERDIR')
50
51    collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split()
52    if not collections:
53        name = os.path.basename(layer_path)
54        collections = [name]
55
56    collections = {c: {} for c in collections}
57    for name in collections:
58        priority = ldata.getVar('BBFILE_PRIORITY_%s' % name)
59        pattern = ldata.getVar('BBFILE_PATTERN_%s' % name)
60        depends = ldata.getVar('LAYERDEPENDS_%s' % name)
61        compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name)
62        collections[name]['priority'] = priority
63        collections[name]['pattern'] = pattern
64        collections[name]['depends'] = depends
65        collections[name]['compat'] = compat
66
67    return collections
68
69def _detect_layer(layer_path):
70    """
71        Scans layer directory to detect what type of layer
72        is BSP, Distro or Software.
73
74        Returns a dictionary with layer name, type and path.
75    """
76
77    layer = {}
78    layer_name = os.path.basename(layer_path)
79
80    layer['name'] = layer_name
81    layer['path'] = layer_path
82    layer['conf'] = {}
83
84    if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')):
85        layer['type'] = LayerType.ERROR_NO_LAYER_CONF
86        return layer
87
88    machine_conf = os.path.join(layer_path, 'conf', 'machine')
89    distro_conf = os.path.join(layer_path, 'conf', 'distro')
90
91    is_bsp = False
92    is_distro = False
93
94    if os.path.isdir(machine_conf):
95        machines = _get_configurations(machine_conf)
96        if machines:
97            is_bsp = True
98
99    if os.path.isdir(distro_conf):
100        distros = _get_configurations(distro_conf)
101        if distros:
102            is_distro = True
103
104    if is_bsp and is_distro:
105        layer['type'] = LayerType.ERROR_BSP_DISTRO
106    elif is_bsp:
107        layer['type'] = LayerType.BSP
108        layer['conf']['machines'] = machines
109    elif is_distro:
110        layer['type'] = LayerType.DISTRO
111        layer['conf']['distros'] = distros
112    else:
113        layer['type'] = LayerType.SOFTWARE
114
115    layer['collections'] = _get_layer_collections(layer['path'])
116
117    return layer
118
119def detect_layers(layer_directories, no_auto):
120    layers = []
121
122    for directory in layer_directories:
123        directory = os.path.realpath(directory)
124        if directory[-1] == '/':
125            directory = directory[0:-1]
126
127        if no_auto:
128            conf_dir = os.path.join(directory, 'conf')
129            if os.path.isdir(conf_dir):
130                layer = _detect_layer(directory)
131                if layer:
132                    layers.append(layer)
133        else:
134            for root, dirs, files in os.walk(directory):
135                dir_name = os.path.basename(root)
136                conf_dir = os.path.join(root, 'conf')
137                if os.path.isdir(conf_dir):
138                    layer = _detect_layer(root)
139                    if layer:
140                        layers.append(layer)
141
142    return layers
143
144def _find_layer_depends(depend, layers):
145    for layer in layers:
146        for collection in layer['collections']:
147            if depend == collection:
148                return layer
149    return None
150
151def add_layer_dependencies(bblayersconf, layer, layers, logger):
152    def recurse_dependencies(depends, layer, layers, logger, ret = []):
153        logger.debug('Processing dependencies %s for layer %s.' % \
154                    (depends, layer['name']))
155
156        for depend in depends.split():
157            # core (oe-core) is suppose to be provided
158            if depend == 'core':
159                continue
160
161            layer_depend = _find_layer_depends(depend, layers)
162            if not layer_depend:
163                logger.error('Layer %s depends on %s and isn\'t found.' % \
164                        (layer['name'], depend))
165                ret = None
166                continue
167
168            # We keep processing, even if ret is None, this allows us to report
169            # multiple errors at once
170            if ret is not None and layer_depend not in ret:
171                ret.append(layer_depend)
172            else:
173                # we might have processed this dependency already, in which case
174                # we should not do it again (avoid recursive loop)
175                continue
176
177            # Recursively process...
178            if 'collections' not in layer_depend:
179                continue
180
181            for collection in layer_depend['collections']:
182                collect_deps = layer_depend['collections'][collection]['depends']
183                if not collect_deps:
184                    continue
185                ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret)
186
187        return ret
188
189    layer_depends = []
190    for collection in layer['collections']:
191        depends = layer['collections'][collection]['depends']
192        if not depends:
193            continue
194
195        layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
196
197    # Note: [] (empty) is allowed, None is not!
198    if layer_depends is None:
199        return False
200    else:
201        add_layers(bblayersconf, layer_depends, logger)
202
203    return True
204
205def add_layers(bblayersconf, layers, logger):
206    # Don't add a layer that is already present.
207    added = set()
208    output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8')
209    for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE):
210        added.add(path)
211
212    with open(bblayersconf, 'a+') as f:
213        for layer in layers:
214            logger.info('Adding layer %s' % layer['name'])
215            name = layer['name']
216            path = layer['path']
217            if path in added:
218                logger.info('%s is already in %s' % (name, bblayersconf))
219            else:
220                added.add(path)
221                f.write("\nBBLAYERS += \"%s\"\n" % path)
222    return True
223
224def check_command(error_msg, cmd, cwd=None):
225    '''
226    Run a command under a shell, capture stdout and stderr in a single stream,
227    throw an error when command returns non-zero exit code. Returns the output.
228    '''
229
230    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
231    output, _ = p.communicate()
232    if p.returncode:
233        msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8'))
234        raise RuntimeError(msg)
235    return output
236
237def get_signatures(builddir, failsafe=False, machine=None):
238    import re
239
240    # some recipes needs to be excluded like meta-world-pkgdata
241    # because a layer can add recipes to a world build so signature
242    # will be change
243    exclude_recipes = ('meta-world-pkgdata',)
244
245    sigs = {}
246    tune2tasks = {}
247
248    cmd = 'BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
249    if machine:
250        cmd += 'MACHINE=%s ' % machine
251    cmd += 'bitbake '
252    if failsafe:
253        cmd += '-k '
254    cmd += '-S none world'
255    sigs_file = os.path.join(builddir, 'locked-sigs.inc')
256    if os.path.exists(sigs_file):
257        os.unlink(sigs_file)
258    try:
259        check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.',
260                      cmd, builddir)
261    except RuntimeError as ex:
262        if failsafe and os.path.exists(sigs_file):
263            # Ignore the error here. Most likely some recipes active
264            # in a world build lack some dependencies. There is a
265            # separate test_machine_world_build which exposes the
266            # failure.
267            pass
268        else:
269            raise
270
271    sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
272    tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
273    current_tune = None
274    with open(sigs_file, 'r') as f:
275        for line in f.readlines():
276            line = line.strip()
277            t = tune_regex.search(line)
278            if t:
279                current_tune = t.group('tune')
280            s = sig_regex.match(line)
281            if s:
282                exclude = False
283                for er in exclude_recipes:
284                    (recipe, task) = s.group('task').split(':')
285                    if er == recipe:
286                        exclude = True
287                        break
288                if exclude:
289                    continue
290
291                sigs[s.group('task')] = s.group('hash')
292                tune2tasks.setdefault(current_tune, []).append(s.group('task'))
293
294    if not sigs:
295        raise RuntimeError('Can\'t load signatures from %s' % sigs_file)
296
297    return (sigs, tune2tasks)
298
299def get_depgraph(targets=['world'], failsafe=False):
300    '''
301    Returns the dependency graph for the given target(s).
302    The dependency graph is taken directly from DepTreeEvent.
303    '''
304    depgraph = None
305    with bb.tinfoil.Tinfoil() as tinfoil:
306        tinfoil.prepare(config_only=False)
307        tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted'])
308        if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'):
309            raise RuntimeError('starting generateDepTreeEvent failed')
310        while True:
311            event = tinfoil.wait_event(timeout=1000)
312            if event:
313                if isinstance(event, bb.command.CommandFailed):
314                    raise RuntimeError('Generating dependency information failed: %s' % event.error)
315                elif isinstance(event, bb.command.CommandCompleted):
316                    break
317                elif isinstance(event, bb.event.NoProvider):
318                    if failsafe:
319                        # The event is informational, we will get information about the
320                        # remaining dependencies eventually and thus can ignore this
321                        # here like we do in get_signatures(), if desired.
322                        continue
323                    if event._reasons:
324                        raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons))
325                    else:
326                        raise RuntimeError('Nothing provides %s.' % (event._item))
327                elif isinstance(event, bb.event.DepTreeGenerated):
328                    depgraph = event._depgraph
329
330    if depgraph is None:
331        raise RuntimeError('Could not retrieve the depgraph.')
332    return depgraph
333
334def compare_signatures(old_sigs, curr_sigs):
335    '''
336    Compares the result of two get_signatures() calls. Returns None if no
337    problems found, otherwise a string that can be used as additional
338    explanation in self.fail().
339    '''
340    # task -> (old signature, new signature)
341    sig_diff = {}
342    for task in old_sigs:
343        if task in curr_sigs and \
344           old_sigs[task] != curr_sigs[task]:
345            sig_diff[task] = (old_sigs[task], curr_sigs[task])
346
347    if not sig_diff:
348        return None
349
350    # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures()
351    # uses <pn>:<taskname>. Need to convert sometimes. The output follows
352    # the convention from get_signatures() because that seems closer to
353    # normal bitbake output.
354    def sig2graph(task):
355        pn, taskname = task.rsplit(':', 1)
356        return pn + '.' + taskname
357    def graph2sig(task):
358        pn, taskname = task.rsplit('.', 1)
359        return pn + ':' + taskname
360    depgraph = get_depgraph(failsafe=True)
361    depends = depgraph['tdepends']
362
363    # If a task A has a changed signature, but none of its
364    # dependencies, then we need to report it because it is
365    # the one which introduces a change. Any task depending on
366    # A (directly or indirectly) will also have a changed
367    # signature, but we don't need to report it. It might have
368    # its own changes, which will become apparent once the
369    # issues that we do report are fixed and the test gets run
370    # again.
371    sig_diff_filtered = []
372    for task, (old_sig, new_sig) in sig_diff.items():
373        deps_tainted = False
374        for dep in depends.get(sig2graph(task), ()):
375            if graph2sig(dep) in sig_diff:
376                deps_tainted = True
377                break
378        if not deps_tainted:
379            sig_diff_filtered.append((task, old_sig, new_sig))
380
381    msg = []
382    msg.append('%d signatures changed, initial differences (first hash before, second after):' %
383               len(sig_diff))
384    for diff in sorted(sig_diff_filtered):
385        recipe, taskname = diff[0].rsplit(':', 1)
386        cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \
387              (recipe, taskname, diff[1], diff[2])
388        msg.append('   %s: %s -> %s' % diff)
389        msg.append('      %s' % cmd)
390        try:
391            output = check_command('Determining signature difference failed.',
392                                   cmd).decode('utf-8')
393        except RuntimeError as error:
394            output = str(error)
395        if output:
396            msg.extend(['      ' + line for line in output.splitlines()])
397            msg.append('')
398    return '\n'.join(msg)
399