1#
2# Copyright BitBake Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import fnmatch
8import logging
9import os
10import shutil
11import sys
12import tempfile
13
14from bb.cookerdata import findTopdir
15import bb.utils
16
17from bblayers.common import LayerPlugin
18
19logger = logging.getLogger('bitbake-layers')
20
21
22def plugin_init(plugins):
23    return ActionPlugin()
24
25
26class ActionPlugin(LayerPlugin):
27    def do_add_layer(self, args):
28        """Add one or more layers to bblayers.conf."""
29        layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir]
30
31        for layerdir in layerdirs:
32            if not os.path.exists(layerdir):
33                sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir)
34                return 1
35
36            layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
37            if not os.path.exists(layer_conf):
38                sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir)
39                return 1
40
41        bblayers_conf = os.path.join(findTopdir(),'conf', 'bblayers.conf')
42        if not os.path.exists(bblayers_conf):
43            sys.stderr.write("Unable to find bblayers.conf\n")
44            return 1
45
46        # Back up bblayers.conf to tempdir before we add layers
47        tempdir = tempfile.mkdtemp()
48        backup = tempdir + "/bblayers.conf.bak"
49        shutil.copy2(bblayers_conf, backup)
50
51        try:
52            notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
53            self.tinfoil.modified_files()
54            if not (args.force or notadded):
55                try:
56                    self.tinfoil.run_command('parseConfiguration')
57                except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
58                    # Restore the back up copy of bblayers.conf
59                    shutil.copy2(backup, bblayers_conf)
60                    self.tinfoil.modified_files()
61                    bb.fatal("Parse failure with the specified layer added, exiting.")
62                else:
63                    for item in notadded:
64                        sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
65        finally:
66            # Remove the back up copy of bblayers.conf
67            shutil.rmtree(tempdir)
68
69    def do_remove_layer(self, args):
70        """Remove one or more layers from bblayers.conf."""
71        bblayers_conf = os.path.join(findTopdir() ,'conf', 'bblayers.conf')
72        if not os.path.exists(bblayers_conf):
73            sys.stderr.write("Unable to find bblayers.conf\n")
74            return 1
75
76        layerdirs = []
77        for item in args.layerdir:
78            if item.startswith('*'):
79                layerdir = item
80            elif not '/' in item:
81                layerdir = '*/%s' % item
82            else:
83                layerdir = os.path.abspath(item)
84            layerdirs.append(layerdir)
85        (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs)
86        self.tinfoil.modified_files()
87        if notremoved:
88            for item in notremoved:
89                sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
90            return 1
91
92    def do_flatten(self, args):
93        """flatten layer configuration into a separate output directory.
94
95Takes the specified layers (or all layers in the current layer
96configuration if none are specified) and builds a "flattened" directory
97containing the contents of all layers, with any overlayed recipes removed
98and bbappends appended to the corresponding recipes. Note that some manual
99cleanup may still be necessary afterwards, in particular:
100
101* where non-recipe files (such as patches) are overwritten (the flatten
102  command will show a warning for these)
103* where anything beyond the normal layer setup has been added to
104  layer.conf (only the lowest priority number layer's layer.conf is used)
105* overridden/appended items from bbappends will need to be tidied up
106* when the flattened layers do not have the same directory structure (the
107  flatten command should show a warning when this will cause a problem)
108
109Warning: if you flatten several layers where another layer is intended to
110be used "inbetween" them (in layer priority order) such that recipes /
111bbappends in the layers interact, and then attempt to use the new output
112layer together with that other layer, you may no longer get the same
113build results (as the layer priority order has effectively changed).
114"""
115        if len(args.layer) == 1:
116            logger.error('If you specify layers to flatten you must specify at least two')
117            return 1
118
119        outputdir = args.outputdir
120        if os.path.exists(outputdir) and os.listdir(outputdir):
121            logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
122            return 1
123
124        layers = self.bblayers
125        if len(args.layer) > 2:
126            layernames = args.layer
127            found_layernames = []
128            found_layerdirs = []
129            for layerdir in layers:
130                layername = self.get_layer_name(layerdir)
131                if layername in layernames:
132                    found_layerdirs.append(layerdir)
133                    found_layernames.append(layername)
134
135            for layername in layernames:
136                if not layername in found_layernames:
137                    logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
138                    return
139            layers = found_layerdirs
140        else:
141            layernames = []
142
143        # Ensure a specified path matches our list of layers
144        def layer_path_match(path):
145            for layerdir in layers:
146                if path.startswith(os.path.join(layerdir, '')):
147                    return layerdir
148            return None
149
150        applied_appends = []
151        for layer in layers:
152            overlayed = set()
153            for mc in self.tinfoil.cooker.multiconfigs:
154                for f in self.tinfoil.cooker.collections[mc].overlayed.keys():
155                    for of in self.tinfoil.cooker.collections[mc].overlayed[f]:
156                        if of.startswith(layer):
157                            overlayed.add(of)
158
159            logger.plain('Copying files from %s...' % layer )
160            for root, dirs, files in os.walk(layer):
161                if '.git' in dirs:
162                    dirs.remove('.git')
163                if '.hg' in dirs:
164                    dirs.remove('.hg')
165
166                for f1 in files:
167                    f1full = os.sep.join([root, f1])
168                    if f1full in overlayed:
169                        logger.plain('  Skipping overlayed file %s' % f1full )
170                    else:
171                        ext = os.path.splitext(f1)[1]
172                        if ext != '.bbappend':
173                            fdest = f1full[len(layer):]
174                            fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
175                            bb.utils.mkdirhier(os.path.dirname(fdest))
176                            if os.path.exists(fdest):
177                                if f1 == 'layer.conf' and root.endswith('/conf'):
178                                    logger.plain('  Skipping layer config file %s' % f1full )
179                                    continue
180                                else:
181                                    logger.warning('Overwriting file %s', fdest)
182                            bb.utils.copyfile(f1full, fdest)
183                            if ext == '.bb':
184                                appends = set()
185                                for mc in self.tinfoil.cooker.multiconfigs:
186                                    appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full))
187                                for append in appends:
188                                    if layer_path_match(append):
189                                        logger.plain('  Applying append %s to %s' % (append, fdest))
190                                        self.apply_append(append, fdest)
191                                        applied_appends.append(append)
192
193        # Take care of when some layers are excluded and yet we have included bbappends for those recipes
194        bbappends = set()
195        for mc in self.tinfoil.cooker.multiconfigs:
196            bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends)
197
198        for b in bbappends:
199            (recipename, appendname) = b
200            if appendname not in applied_appends:
201                first_append = None
202                layer = layer_path_match(appendname)
203                if layer:
204                    if first_append:
205                        self.apply_append(appendname, first_append)
206                    else:
207                        fdest = appendname[len(layer):]
208                        fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
209                        bb.utils.mkdirhier(os.path.dirname(fdest))
210                        bb.utils.copyfile(appendname, fdest)
211                        first_append = fdest
212
213        # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
214        # have come from)
215        first_regex = None
216        layerdir = layers[0]
217        for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities:
218            if regex.match(os.path.join(layerdir, 'test')):
219                first_regex = regex
220                break
221
222        if first_regex:
223            # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
224            bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split()
225            bbfiles_layer = []
226            for item in bbfiles:
227                if first_regex.match(item):
228                    newpath = os.path.join(outputdir, item[len(layerdir)+1:])
229                    bbfiles_layer.append(newpath)
230
231            if bbfiles_layer:
232                # Check that all important layer files match BBFILES
233                for root, dirs, files in os.walk(outputdir):
234                    for f1 in files:
235                        ext = os.path.splitext(f1)[1]
236                        if ext in ['.bb', '.bbappend']:
237                            f1full = os.sep.join([root, f1])
238                            entry_found = False
239                            for item in bbfiles_layer:
240                                if fnmatch.fnmatch(f1full, item):
241                                    entry_found = True
242                                    break
243                            if not entry_found:
244                                logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
245
246        self.tinfoil.modified_files()
247
248
249    def get_file_layer(self, filename):
250        layerdir = self.get_file_layerdir(filename)
251        if layerdir:
252            return self.get_layer_name(layerdir)
253        else:
254            return '?'
255
256    def get_file_layerdir(self, filename):
257        layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
258        return self.bbfile_collections.get(layer, None)
259
260    def apply_append(self, appendname, recipename):
261        with open(appendname, 'r') as appendfile:
262            with open(recipename, 'a') as recipefile:
263                recipefile.write('\n')
264                recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
265                recipefile.writelines(appendfile.readlines())
266
267    def register_commands(self, sp):
268        parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False)
269        parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add')
270
271        parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False)
272        parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
273        parser_remove_layer.set_defaults(func=self.do_remove_layer)
274
275        parser_flatten = self.add_command(sp, 'flatten', self.do_flatten)
276        parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)')
277        parser_flatten.add_argument('outputdir', help='Output directory')
278