xref: /openbmc/openbmc/poky/bitbake/lib/bblayers/action.py (revision edff49234e31f23dc79f823473c9e286a21596c1)
1#
2# Copyright BitBake Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7import fnmatch
8import logging
9import os
10import shutil
11import sys
12import tempfile
13
14from bb.cookerdata import findTopdir
15import bb.utils
16
17from bblayers.common import LayerPlugin
18
19logger = logging.getLogger('bitbake-layers')
20
21
22def plugin_init(plugins):
23    return ActionPlugin()
24
25
26class ActionPlugin(LayerPlugin):
27    def do_add_layer(self, args):
28        """Add one or more layers to bblayers.conf."""
29        layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir]
30
31        for layerdir in layerdirs:
32            if not os.path.exists(layerdir):
33                sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir)
34                return 1
35
36            layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
37            if not os.path.exists(layer_conf):
38                sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir)
39                return 1
40
41        bblayers_conf = os.path.join(findTopdir(),'conf', 'bblayers.conf')
42        if not os.path.exists(bblayers_conf):
43            sys.stderr.write("Unable to find bblayers.conf\n")
44            return 1
45
46        # Back up bblayers.conf to tempdir before we add layers
47        tempdir = tempfile.mkdtemp()
48        backup = tempdir + "/bblayers.conf.bak"
49        shutil.copy2(bblayers_conf, backup)
50
51        try:
52            notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
53            if not (args.force or notadded):
54                self.tinfoil.modified_files()
55                try:
56                    self.tinfoil.run_command('parseConfiguration')
57                except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
58                    # Restore the back up copy of bblayers.conf
59                    shutil.copy2(backup, bblayers_conf)
60                    self.tinfoil.modified_files()
61                    bb.fatal("Parse failure with the specified layer added, exiting.")
62                else:
63                    for item in notadded:
64                        sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
65        finally:
66            # Remove the back up copy of bblayers.conf
67            shutil.rmtree(tempdir)
68
69    def do_remove_layer(self, args):
70        """Remove one or more layers from bblayers.conf."""
71        bblayers_conf = os.path.join(findTopdir() ,'conf', 'bblayers.conf')
72        if not os.path.exists(bblayers_conf):
73            sys.stderr.write("Unable to find bblayers.conf\n")
74            return 1
75
76        layerdirs = []
77        for item in args.layerdir:
78            if item.startswith('*'):
79                layerdir = item
80            elif not '/' in item:
81                layerdir = '*/%s' % item
82            else:
83                layerdir = os.path.abspath(item)
84            layerdirs.append(layerdir)
85        (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs)
86        if args.force > 1:
87            return 0
88        self.tinfoil.modified_files()
89        if notremoved:
90            for item in notremoved:
91                sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
92            return 1
93
94    def do_flatten(self, args):
95        """flatten layer configuration into a separate output directory.
96
97Takes the specified layers (or all layers in the current layer
98configuration if none are specified) and builds a "flattened" directory
99containing the contents of all layers, with any overlayed recipes removed
100and bbappends appended to the corresponding recipes. Note that some manual
101cleanup may still be necessary afterwards, in particular:
102
103* where non-recipe files (such as patches) are overwritten (the flatten
104  command will show a warning for these)
105* where anything beyond the normal layer setup has been added to
106  layer.conf (only the lowest priority number layer's layer.conf is used)
107* overridden/appended items from bbappends will need to be tidied up
108* when the flattened layers do not have the same directory structure (the
109  flatten command should show a warning when this will cause a problem)
110
111Warning: if you flatten several layers where another layer is intended to
112be used "inbetween" them (in layer priority order) such that recipes /
113bbappends in the layers interact, and then attempt to use the new output
114layer together with that other layer, you may no longer get the same
115build results (as the layer priority order has effectively changed).
116"""
117        if len(args.layer) == 1:
118            logger.error('If you specify layers to flatten you must specify at least two')
119            return 1
120
121        outputdir = args.outputdir
122        if os.path.exists(outputdir) and os.listdir(outputdir):
123            logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
124            return 1
125
126        layers = self.bblayers
127        if len(args.layer) > 2:
128            layernames = args.layer
129            found_layernames = []
130            found_layerdirs = []
131            for layerdir in layers:
132                layername = self.get_layer_name(layerdir)
133                if layername in layernames:
134                    found_layerdirs.append(layerdir)
135                    found_layernames.append(layername)
136
137            for layername in layernames:
138                if not layername in found_layernames:
139                    logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
140                    return
141            layers = found_layerdirs
142        else:
143            layernames = []
144
145        # Ensure a specified path matches our list of layers
146        def layer_path_match(path):
147            for layerdir in layers:
148                if path.startswith(os.path.join(layerdir, '')):
149                    return layerdir
150            return None
151
152        applied_appends = []
153        for layer in layers:
154            overlayed = set()
155            for mc in self.tinfoil.cooker.multiconfigs:
156                for f in self.tinfoil.cooker.collections[mc].overlayed.keys():
157                    for of in self.tinfoil.cooker.collections[mc].overlayed[f]:
158                        if of.startswith(layer):
159                            overlayed.add(of)
160
161            logger.plain('Copying files from %s...' % layer )
162            for root, dirs, files in os.walk(layer):
163                if '.git' in dirs:
164                    dirs.remove('.git')
165                if '.hg' in dirs:
166                    dirs.remove('.hg')
167
168                for f1 in files:
169                    f1full = os.sep.join([root, f1])
170                    if f1full in overlayed:
171                        logger.plain('  Skipping overlayed file %s' % f1full )
172                    else:
173                        ext = os.path.splitext(f1)[1]
174                        if ext != '.bbappend':
175                            fdest = f1full[len(layer):]
176                            fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
177                            bb.utils.mkdirhier(os.path.dirname(fdest))
178                            if os.path.exists(fdest):
179                                if f1 == 'layer.conf' and root.endswith('/conf'):
180                                    logger.plain('  Skipping layer config file %s' % f1full )
181                                    continue
182                                else:
183                                    logger.warning('Overwriting file %s', fdest)
184                            bb.utils.copyfile(f1full, fdest)
185                            if ext == '.bb':
186                                appends = set()
187                                for mc in self.tinfoil.cooker.multiconfigs:
188                                    appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full))
189                                for append in appends:
190                                    if layer_path_match(append):
191                                        logger.plain('  Applying append %s to %s' % (append, fdest))
192                                        self.apply_append(append, fdest)
193                                        applied_appends.append(append)
194
195        # Take care of when some layers are excluded and yet we have included bbappends for those recipes
196        bbappends = set()
197        for mc in self.tinfoil.cooker.multiconfigs:
198            bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends)
199
200        for b in bbappends:
201            (recipename, appendname) = b
202            if appendname not in applied_appends:
203                first_append = None
204                layer = layer_path_match(appendname)
205                if layer:
206                    if first_append:
207                        self.apply_append(appendname, first_append)
208                    else:
209                        fdest = appendname[len(layer):]
210                        fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
211                        bb.utils.mkdirhier(os.path.dirname(fdest))
212                        bb.utils.copyfile(appendname, fdest)
213                        first_append = fdest
214
215        # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
216        # have come from)
217        first_regex = None
218        layerdir = layers[0]
219        for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities:
220            if regex.match(os.path.join(layerdir, 'test')):
221                first_regex = regex
222                break
223
224        if first_regex:
225            # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
226            bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split()
227            bbfiles_layer = []
228            for item in bbfiles:
229                if first_regex.match(item):
230                    newpath = os.path.join(outputdir, item[len(layerdir)+1:])
231                    bbfiles_layer.append(newpath)
232
233            if bbfiles_layer:
234                # Check that all important layer files match BBFILES
235                for root, dirs, files in os.walk(outputdir):
236                    for f1 in files:
237                        ext = os.path.splitext(f1)[1]
238                        if ext in ['.bb', '.bbappend']:
239                            f1full = os.sep.join([root, f1])
240                            entry_found = False
241                            for item in bbfiles_layer:
242                                if fnmatch.fnmatch(f1full, item):
243                                    entry_found = True
244                                    break
245                            if not entry_found:
246                                logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
247
248        self.tinfoil.modified_files()
249
250
251    def get_file_layer(self, filename):
252        layerdir = self.get_file_layerdir(filename)
253        if layerdir:
254            return self.get_layer_name(layerdir)
255        else:
256            return '?'
257
258    def get_file_layerdir(self, filename):
259        layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
260        return self.bbfile_collections.get(layer, None)
261
262    def apply_append(self, appendname, recipename):
263        with open(appendname, 'r') as appendfile:
264            with open(recipename, 'a') as recipefile:
265                recipefile.write('\n')
266                recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
267                recipefile.writelines(appendfile.readlines())
268
269    def register_commands(self, sp):
270        parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False)
271        parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add')
272
273        parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False)
274        parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
275        parser_remove_layer.set_defaults(func=self.do_remove_layer)
276
277        parser_flatten = self.add_command(sp, 'flatten', self.do_flatten)
278        parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)')
279        parser_flatten.add_argument('outputdir', help='Output directory')
280