1#
2# Copyright BitBake Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import collections
8import fnmatch
9import logging
10import sys
11import os
12import re
13
14import bb.utils
15
16from bblayers.common import LayerPlugin
17
18logger = logging.getLogger('bitbake-layers')
19
20
21def plugin_init(plugins):
22    return QueryPlugin()
23
24
25class QueryPlugin(LayerPlugin):
26    def __init__(self):
27        super(QueryPlugin, self).__init__()
28        self.collection_res = {}
29
30    def do_show_layers(self, args):
31        """show current configured layers."""
32        logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
33        logger.plain('=' * 74)
34        for layer, _, regex, pri in self.tinfoil.cooker.bbfile_config_priorities:
35            layerdir = self.bbfile_collections.get(layer, None)
36            layername = self.get_layer_name(layerdir)
37            logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), pri))
38
39    def version_str(self, pe, pv, pr = None):
40        verstr = "%s" % pv
41        if pr:
42            verstr = "%s-%s" % (verstr, pr)
43        if pe:
44            verstr = "%s:%s" % (pe, verstr)
45        return verstr
46
47    def do_show_overlayed(self, args):
48        """list overlayed recipes (where the same recipe exists in another layer)
49
50Lists the names of overlayed recipes and the available versions in each
51layer, with the preferred version first. Note that skipped recipes that
52are overlayed will also be listed, with a " (skipped)" suffix.
53"""
54
55        items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, False, True, None, False, None, args.mc)
56
57        # Check for overlayed .bbclass files
58        classes = collections.defaultdict(list)
59        for layerdir in self.bblayers:
60            for c in ["classes-global", "classes-recipe", "classes"]:
61                classdir = os.path.join(layerdir, c)
62                if os.path.exists(classdir):
63                    for classfile in os.listdir(classdir):
64                        if os.path.splitext(classfile)[1] == '.bbclass':
65                            classes[classfile].append(classdir)
66
67        # Locating classes and other files is a bit more complicated than recipes -
68        # layer priority is not a factor; instead BitBake uses the first matching
69        # file in BBPATH, which is manipulated directly by each layer's
70        # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a
71        # factor - however, each layer.conf is free to either prepend or append to
72        # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
73        # not be exactly the order present in bblayers.conf either.
74        bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
75        overlayed_class_found = False
76        for (classfile, classdirs) in classes.items():
77            if len(classdirs) > 1:
78                if not overlayed_class_found:
79                    logger.plain('=== Overlayed classes ===')
80                    overlayed_class_found = True
81
82                mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile))
83                if args.filenames:
84                    logger.plain('%s' % mainfile)
85                else:
86                    # We effectively have to guess the layer here
87                    logger.plain('%s:' % classfile)
88                    mainlayername = '?'
89                    for layerdir in self.bblayers:
90                        classdir = os.path.join(layerdir, 'classes')
91                        if mainfile.startswith(classdir):
92                            mainlayername = self.get_layer_name(layerdir)
93                    logger.plain('  %s' % mainlayername)
94                for classdir in classdirs:
95                    fullpath = os.path.join(classdir, classfile)
96                    if fullpath != mainfile:
97                        if args.filenames:
98                            print('  %s' % fullpath)
99                        else:
100                            print('  %s' % self.get_layer_name(os.path.dirname(classdir)))
101
102        if overlayed_class_found:
103            items_listed = True;
104
105        if not items_listed:
106            logger.plain('No overlayed files found.')
107
108    def do_show_recipes(self, args):
109        """list available recipes, showing the layer they are provided by
110
111Lists the names of recipes and the available versions in each
112layer, with the preferred version first. Optionally you may specify
113pnspec to match a specified recipe name (supports wildcards). Note that
114skipped recipes will also be listed, with a " (skipped)" suffix.
115"""
116
117        inheritlist = args.inherits.split(',') if args.inherits else []
118        if inheritlist or args.pnspec or args.multiple:
119            title = 'Matching recipes:'
120        else:
121            title = 'Available recipes:'
122        self.list_recipes(title, args.pnspec, False, False, args.filenames, args.recipes_only, args.multiple, args.layer, args.bare, inheritlist, args.mc)
123
124    def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_recipes_only, show_multi_provider_only, selected_layer, bare, inherits, mc):
125        if inherits:
126            bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
127            for classname in inherits:
128                found = False
129                for c in ["classes-global", "classes-recipe", "classes"]:
130                    cfile = c + '/%s.bbclass' % classname
131                    if bb.utils.which(bbpath, cfile, history=False):
132                        found = True
133                        break
134                if not found:
135                    logger.error('No class named %s found in BBPATH', classname)
136                    sys.exit(1)
137
138        pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn
139        (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc)
140        allproviders = self.tinfoil.get_all_providers(mc)
141
142        # Ensure we list skipped recipes
143        # We are largely guessing about PN, PV and the preferred version here,
144        # but we have no choice since skipped recipes are not fully parsed
145        skiplist = list(self.tinfoil.cooker.skiplist.keys())
146        mcspec = 'mc:%s:' % mc
147        if mc:
148            skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)]
149
150        for fn in skiplist:
151            recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
152            p = recipe_parts[0]
153            if len(recipe_parts) > 1:
154                ver = (None, recipe_parts[1], None)
155            else:
156                ver = (None, 'unknown', None)
157            allproviders[p].append((ver, fn))
158            if not p in pkg_pn:
159                pkg_pn[p] = 'dummy'
160                preferred_versions[p] = (ver, fn)
161
162        def print_item(f, pn, ver, layer, ispref):
163            if not selected_layer or layer == selected_layer:
164                if not bare and f in skiplist:
165                    skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist[f].skipreason
166                else:
167                    skipped = ''
168                if show_filenames:
169                    if ispref:
170                        logger.plain("%s%s", f, skipped)
171                    else:
172                        logger.plain("  %s%s", f, skipped)
173                elif show_recipes_only:
174                    if pn not in show_unique_pn:
175                        show_unique_pn.append(pn)
176                        logger.plain("%s%s", pn, skipped)
177                else:
178                    if ispref:
179                        logger.plain("%s:", pn)
180                    logger.plain("  %s %s%s", layer.ljust(20), ver, skipped)
181
182        global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split()
183        cls_re = re.compile('classes.*/')
184
185        preffiles = []
186        show_unique_pn = []
187        items_listed = False
188        for p in sorted(pkg_pn):
189            if pnspec:
190                found=False
191                for pnm in pnspec:
192                    if fnmatch.fnmatch(p, pnm):
193                        found=True
194                        break
195                if not found:
196                    continue
197
198            if len(allproviders[p]) > 1 or not show_multi_provider_only:
199                pref = preferred_versions[p]
200                realfn = bb.cache.virtualfn2realfn(pref[1])
201                preffile = realfn[0]
202
203                # We only display once per recipe, we should prefer non extended versions of the
204                # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl
205                # which would otherwise sort first).
206                if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[mc].pkg_fn:
207                    continue
208
209                if inherits:
210                    matchcount = 0
211                    recipe_inherits = self.tinfoil.cooker_data.inherits.get(preffile, [])
212                    for cls in recipe_inherits:
213                        if cls_re.match(cls):
214                            continue
215                        classname = os.path.splitext(os.path.basename(cls))[0]
216                        if classname in global_inherit:
217                            continue
218                        elif classname in inherits:
219                            matchcount += 1
220                    if matchcount != len(inherits):
221                        # No match - skip this recipe
222                        continue
223
224                if preffile not in preffiles:
225                    preflayer = self.get_file_layer(preffile)
226                    multilayer = False
227                    same_ver = True
228                    provs = []
229                    for prov in allproviders[p]:
230                        provfile = bb.cache.virtualfn2realfn(prov[1])[0]
231                        provlayer = self.get_file_layer(provfile)
232                        provs.append((provfile, provlayer, prov[0]))
233                        if provlayer != preflayer:
234                            multilayer = True
235                        if prov[0] != pref[0]:
236                            same_ver = False
237                    if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
238                        if not items_listed:
239                            logger.plain('=== %s ===' % title)
240                            items_listed = True
241                        print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
242                        for (provfile, provlayer, provver) in provs:
243                            if provfile != preffile:
244                                print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False)
245                        # Ensure we don't show two entries for BBCLASSEXTENDed recipes
246                        preffiles.append(preffile)
247
248        return items_listed
249
250    def get_file_layer(self, filename):
251        layerdir = self.get_file_layerdir(filename)
252        if layerdir:
253            return self.get_layer_name(layerdir)
254        else:
255            return '?'
256
257    def get_collection_res(self):
258        if not self.collection_res:
259            self.collection_res = bb.utils.get_collection_res(self.tinfoil.config_data)
260        return self.collection_res
261
262    def get_file_layerdir(self, filename):
263        layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data, self.get_collection_res())
264        return self.bbfile_collections.get(layer, None)
265
266    def remove_layer_prefix(self, f):
267        """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the
268           return value will be: layer_dir/foo/blah"""
269        f_layerdir = self.get_file_layerdir(f)
270        if not f_layerdir:
271            return f
272        prefix = os.path.join(os.path.dirname(f_layerdir), '')
273        return f[len(prefix):] if f.startswith(prefix) else f
274
275    def do_show_appends(self, args):
276        """list bbappend files and recipe files they apply to
277
278Lists recipes with the bbappends that apply to them as subitems.
279"""
280        if args.pnspec:
281            logger.plain('=== Matched appended recipes ===')
282        else:
283            logger.plain('=== Appended recipes ===')
284
285        pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys())
286        pnlist.sort()
287        appends = False
288        for pn in pnlist:
289            if args.pnspec:
290                found=False
291                for pnm in args.pnspec:
292                    if fnmatch.fnmatch(pn, pnm):
293                        found=True
294                        break
295                if not found:
296                    continue
297
298            if self.show_appends_for_pn(pn):
299                appends = True
300
301        if not args.pnspec and self.show_appends_for_skipped():
302            appends = True
303
304        if not appends:
305            logger.plain('No append files found')
306
307    def show_appends_for_pn(self, pn):
308        filenames = self.tinfoil.cooker_data.pkg_pn[pn]
309
310        best = self.tinfoil.find_best_provider(pn)
311        best_filename = os.path.basename(best[3])
312
313        return self.show_appends_output(filenames, best_filename)
314
315    def show_appends_for_skipped(self):
316        filenames = [os.path.basename(f)
317                    for f in self.tinfoil.cooker.skiplist.keys()]
318        return self.show_appends_output(filenames, None, " (skipped)")
319
320    def show_appends_output(self, filenames, best_filename, name_suffix = ''):
321        appended, missing = self.get_appends_for_files(filenames)
322        if appended:
323            for basename, appends in appended:
324                logger.plain('%s%s:', basename, name_suffix)
325                for append in appends:
326                    logger.plain('  %s', append)
327
328            if best_filename:
329                if best_filename in missing:
330                    logger.warning('%s: missing append for preferred version',
331                                   best_filename)
332            return True
333        else:
334            return False
335
336    def get_appends_for_files(self, filenames):
337        appended, notappended = [], []
338        for filename in filenames:
339            _, cls, mc = bb.cache.virtualfn2realfn(filename)
340            if cls:
341                continue
342
343            basename = os.path.basename(filename)
344            appends = self.tinfoil.cooker.collections[mc].get_file_appends(basename)
345            if appends:
346                appended.append((basename, list(appends)))
347            else:
348                notappended.append(basename)
349        return appended, notappended
350
351    def do_show_cross_depends(self, args):
352        """Show dependencies between recipes that cross layer boundaries.
353
354Figure out the dependencies between recipes that cross layer boundaries.
355
356NOTE: .bbappend files can impact the dependencies.
357"""
358        ignore_layers = (args.ignore or '').split(',')
359
360        pkg_fn = self.tinfoil.cooker_data.pkg_fn
361        bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
362        self.require_re = re.compile(r"require\s+(.+)")
363        self.include_re = re.compile(r"include\s+(.+)")
364        self.inherit_re = re.compile(r"inherit\s+(.+)")
365
366        global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split()
367
368        # The bb's DEPENDS and RDEPENDS
369        for f in pkg_fn:
370            f = bb.cache.virtualfn2realfn(f)[0]
371            # Get the layername that the file is in
372            layername = self.get_file_layer(f)
373
374            # The DEPENDS
375            deps = self.tinfoil.cooker_data.deps[f]
376            for pn in deps:
377                if pn in self.tinfoil.cooker_data.pkg_pn:
378                    best = self.tinfoil.find_best_provider(pn)
379                    self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
380
381            # The RDPENDS
382            all_rdeps = self.tinfoil.cooker_data.rundeps[f].values()
383            # Remove the duplicated or null one.
384            sorted_rdeps = {}
385            # The all_rdeps is the list in list, so we need two for loops
386            for k1 in all_rdeps:
387                for k2 in k1:
388                    sorted_rdeps[k2] = 1
389            all_rdeps = sorted_rdeps.keys()
390            for rdep in all_rdeps:
391                all_p, best = self.tinfoil.get_runtime_providers(rdep)
392                if all_p:
393                    if f in all_p:
394                        # The recipe provides this one itself, ignore
395                        continue
396                    self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
397
398            # The RRECOMMENDS
399            all_rrecs = self.tinfoil.cooker_data.runrecs[f].values()
400            # Remove the duplicated or null one.
401            sorted_rrecs = {}
402            # The all_rrecs is the list in list, so we need two for loops
403            for k1 in all_rrecs:
404                for k2 in k1:
405                    sorted_rrecs[k2] = 1
406            all_rrecs = sorted_rrecs.keys()
407            for rrec in all_rrecs:
408                all_p, best = self.tinfoil.get_runtime_providers(rrec)
409                if all_p:
410                    if f in all_p:
411                        # The recipe provides this one itself, ignore
412                        continue
413                    self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
414
415            # The inherit class
416            cls_re = re.compile('classes.*/')
417            if f in self.tinfoil.cooker_data.inherits:
418                inherits = self.tinfoil.cooker_data.inherits[f]
419                for cls in inherits:
420                    # The inherits' format is [classes/cls, /path/to/classes/cls]
421                    # ignore the classes/cls.
422                    if not cls_re.match(cls):
423                        classname = os.path.splitext(os.path.basename(cls))[0]
424                        if classname in global_inherit:
425                            continue
426                        inherit_layername = self.get_file_layer(cls)
427                        if inherit_layername != layername and not inherit_layername in ignore_layers:
428                            if not args.filenames:
429                                f_short = self.remove_layer_prefix(f)
430                                cls = self.remove_layer_prefix(cls)
431                            else:
432                                f_short = f
433                            logger.plain("%s inherits %s" % (f_short, cls))
434
435            # The 'require/include xxx' in the bb file
436            pv_re = re.compile(r"\${PV}")
437            with open(f, 'r') as fnfile:
438                line = fnfile.readline()
439                while line:
440                    m, keyword = self.match_require_include(line)
441                    # Found the 'require/include xxxx'
442                    if m:
443                        needed_file = m.group(1)
444                        # Replace the ${PV} with the real PV
445                        if pv_re.search(needed_file) and f in self.tinfoil.cooker_data.pkg_pepvpr:
446                            pv = self.tinfoil.cooker_data.pkg_pepvpr[f][1]
447                            needed_file = re.sub(r"\${PV}", pv, needed_file)
448                        self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
449                    line = fnfile.readline()
450
451        # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
452        conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$")
453        inc_re = re.compile(r".*\.inc$")
454        # The "inherit xxx" in .bbclass
455        bbclass_re = re.compile(r".*\.bbclass$")
456        for layerdir in self.bblayers:
457            layername = self.get_layer_name(layerdir)
458            for dirpath, dirnames, filenames in os.walk(layerdir):
459                for name in filenames:
460                    f = os.path.join(dirpath, name)
461                    s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f)
462                    if s:
463                        with open(f, 'r') as ffile:
464                            line = ffile.readline()
465                            while line:
466                                m, keyword = self.match_require_include(line)
467                                # Only bbclass has the "inherit xxx" here.
468                                bbclass=""
469                                if not m and f.endswith(".bbclass"):
470                                    m, keyword = self.match_inherit(line)
471                                    bbclass=".bbclass"
472                                # Find a 'require/include xxxx'
473                                if m:
474                                    self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
475                                line = ffile.readline()
476
477    def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers):
478        """Print the depends that crosses a layer boundary"""
479        needed_file = bb.utils.which(bbpath, needed_filename)
480        if needed_file:
481            # Which layer is this file from
482            needed_layername = self.get_file_layer(needed_file)
483            if needed_layername != layername and not needed_layername in ignore_layers:
484                if not show_filenames:
485                    f = self.remove_layer_prefix(f)
486                    needed_file = self.remove_layer_prefix(needed_file)
487                logger.plain("%s %s %s" %(f, keyword, needed_file))
488
489    def match_inherit(self, line):
490        """Match the inherit xxx line"""
491        return (self.inherit_re.match(line), "inherits")
492
493    def match_require_include(self, line):
494        """Match the require/include xxx line"""
495        m = self.require_re.match(line)
496        keyword = "requires"
497        if not m:
498            m = self.include_re.match(line)
499            keyword = "includes"
500        return (m, keyword)
501
502    def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers):
503        """Print the DEPENDS/RDEPENDS file that crosses a layer boundary"""
504        best_realfn = bb.cache.virtualfn2realfn(needed_file)[0]
505        needed_layername = self.get_file_layer(best_realfn)
506        if needed_layername != layername and not needed_layername in ignore_layers:
507            if not show_filenames:
508                f = self.remove_layer_prefix(f)
509                best_realfn = self.remove_layer_prefix(best_realfn)
510
511            logger.plain("%s %s %s" % (f, keyword, best_realfn))
512
513    def register_commands(self, sp):
514        self.add_command(sp, 'show-layers', self.do_show_layers, parserecipes=False)
515
516        parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed)
517        parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
518        parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true')
519        parser_show_overlayed.add_argument('--mc', help='use specified multiconfig', default='')
520
521        parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes)
522        parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
523        parser_show_recipes.add_argument('-r', '--recipes-only', help='instead of the default formatting, list recipes only', action='store_true')
524        parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true')
525        parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class(es) - separate multiple classes using , (without spaces)', metavar='CLASS', default='')
526        parser_show_recipes.add_argument('-l', '--layer', help='only list recipes from the selected layer', default='')
527        parser_show_recipes.add_argument('-b', '--bare', help='output just names without the "(skipped)" marker', action='store_true')
528        parser_show_recipes.add_argument('--mc', help='use specified multiconfig', default='')
529        parser_show_recipes.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
530
531        parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends)
532        parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
533
534        parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends)
535        parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')
536        parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME')
537