1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4
5import fnmatch
6import logging
7import os
8import shutil
9import sys
10import tempfile
11
12import bb.utils
13
14from bblayers.common import LayerPlugin
15
16logger = logging.getLogger('bitbake-layers')
17
18
19def plugin_init(plugins):
20    return ActionPlugin()
21
22
23class ActionPlugin(LayerPlugin):
24    def do_add_layer(self, args):
25        """Add one or more layers to bblayers.conf."""
26        layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir]
27
28        for layerdir in layerdirs:
29            if not os.path.exists(layerdir):
30                sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir)
31                return 1
32
33            layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
34            if not os.path.exists(layer_conf):
35                sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir)
36                return 1
37
38        bblayers_conf = os.path.join('conf', 'bblayers.conf')
39        if not os.path.exists(bblayers_conf):
40            sys.stderr.write("Unable to find bblayers.conf\n")
41            return 1
42
43        # Back up bblayers.conf to tempdir before we add layers
44        tempdir = tempfile.mkdtemp()
45        backup = tempdir + "/bblayers.conf.bak"
46        shutil.copy2(bblayers_conf, backup)
47
48        try:
49            notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
50            if not (args.force or notadded):
51                try:
52                    self.tinfoil.run_command('parseConfiguration')
53                except bb.tinfoil.TinfoilUIException:
54                    # Restore the back up copy of bblayers.conf
55                    shutil.copy2(backup, bblayers_conf)
56                    bb.fatal("Parse failure with the specified layer added")
57                else:
58                    for item in notadded:
59                        sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
60        finally:
61            # Remove the back up copy of bblayers.conf
62            shutil.rmtree(tempdir)
63
64    def do_remove_layer(self, args):
65        """Remove one or more layers from bblayers.conf."""
66        bblayers_conf = os.path.join('conf', 'bblayers.conf')
67        if not os.path.exists(bblayers_conf):
68            sys.stderr.write("Unable to find bblayers.conf\n")
69            return 1
70
71        layerdirs = []
72        for item in args.layerdir:
73            if item.startswith('*'):
74                layerdir = item
75            elif not '/' in item:
76                layerdir = '*/%s' % item
77            else:
78                layerdir = os.path.abspath(item)
79            layerdirs.append(layerdir)
80        (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs)
81        if notremoved:
82            for item in notremoved:
83                sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
84            return 1
85
86    def do_flatten(self, args):
87        """flatten layer configuration into a separate output directory.
88
89Takes the specified layers (or all layers in the current layer
90configuration if none are specified) and builds a "flattened" directory
91containing the contents of all layers, with any overlayed recipes removed
92and bbappends appended to the corresponding recipes. Note that some manual
93cleanup may still be necessary afterwards, in particular:
94
95* where non-recipe files (such as patches) are overwritten (the flatten
96  command will show a warning for these)
97* where anything beyond the normal layer setup has been added to
98  layer.conf (only the lowest priority number layer's layer.conf is used)
99* overridden/appended items from bbappends will need to be tidied up
100* when the flattened layers do not have the same directory structure (the
101  flatten command should show a warning when this will cause a problem)
102
103Warning: if you flatten several layers where another layer is intended to
104be used "inbetween" them (in layer priority order) such that recipes /
105bbappends in the layers interact, and then attempt to use the new output
106layer together with that other layer, you may no longer get the same
107build results (as the layer priority order has effectively changed).
108"""
109        if len(args.layer) == 1:
110            logger.error('If you specify layers to flatten you must specify at least two')
111            return 1
112
113        outputdir = args.outputdir
114        if os.path.exists(outputdir) and os.listdir(outputdir):
115            logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
116            return 1
117
118        layers = self.bblayers
119        if len(args.layer) > 2:
120            layernames = args.layer
121            found_layernames = []
122            found_layerdirs = []
123            for layerdir in layers:
124                layername = self.get_layer_name(layerdir)
125                if layername in layernames:
126                    found_layerdirs.append(layerdir)
127                    found_layernames.append(layername)
128
129            for layername in layernames:
130                if not layername in found_layernames:
131                    logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
132                    return
133            layers = found_layerdirs
134        else:
135            layernames = []
136
137        # Ensure a specified path matches our list of layers
138        def layer_path_match(path):
139            for layerdir in layers:
140                if path.startswith(os.path.join(layerdir, '')):
141                    return layerdir
142            return None
143
144        applied_appends = []
145        for layer in layers:
146            overlayed = []
147            for f in self.tinfoil.cooker.collection.overlayed.keys():
148                for of in self.tinfoil.cooker.collection.overlayed[f]:
149                    if of.startswith(layer):
150                        overlayed.append(of)
151
152            logger.plain('Copying files from %s...' % layer )
153            for root, dirs, files in os.walk(layer):
154                if '.git' in dirs:
155                    dirs.remove('.git')
156                if '.hg' in dirs:
157                    dirs.remove('.hg')
158
159                for f1 in files:
160                    f1full = os.sep.join([root, f1])
161                    if f1full in overlayed:
162                        logger.plain('  Skipping overlayed file %s' % f1full )
163                    else:
164                        ext = os.path.splitext(f1)[1]
165                        if ext != '.bbappend':
166                            fdest = f1full[len(layer):]
167                            fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
168                            bb.utils.mkdirhier(os.path.dirname(fdest))
169                            if os.path.exists(fdest):
170                                if f1 == 'layer.conf' and root.endswith('/conf'):
171                                    logger.plain('  Skipping layer config file %s' % f1full )
172                                    continue
173                                else:
174                                    logger.warning('Overwriting file %s', fdest)
175                            bb.utils.copyfile(f1full, fdest)
176                            if ext == '.bb':
177                                for append in self.tinfoil.cooker.collection.get_file_appends(f1full):
178                                    if layer_path_match(append):
179                                        logger.plain('  Applying append %s to %s' % (append, fdest))
180                                        self.apply_append(append, fdest)
181                                        applied_appends.append(append)
182
183        # Take care of when some layers are excluded and yet we have included bbappends for those recipes
184        for b in self.tinfoil.cooker.collection.bbappends:
185            (recipename, appendname) = b
186            if appendname not in applied_appends:
187                first_append = None
188                layer = layer_path_match(appendname)
189                if layer:
190                    if first_append:
191                        self.apply_append(appendname, first_append)
192                    else:
193                        fdest = appendname[len(layer):]
194                        fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
195                        bb.utils.mkdirhier(os.path.dirname(fdest))
196                        bb.utils.copyfile(appendname, fdest)
197                        first_append = fdest
198
199        # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
200        # have come from)
201        first_regex = None
202        layerdir = layers[0]
203        for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities:
204            if regex.match(os.path.join(layerdir, 'test')):
205                first_regex = regex
206                break
207
208        if first_regex:
209            # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
210            bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split()
211            bbfiles_layer = []
212            for item in bbfiles:
213                if first_regex.match(item):
214                    newpath = os.path.join(outputdir, item[len(layerdir)+1:])
215                    bbfiles_layer.append(newpath)
216
217            if bbfiles_layer:
218                # Check that all important layer files match BBFILES
219                for root, dirs, files in os.walk(outputdir):
220                    for f1 in files:
221                        ext = os.path.splitext(f1)[1]
222                        if ext in ['.bb', '.bbappend']:
223                            f1full = os.sep.join([root, f1])
224                            entry_found = False
225                            for item in bbfiles_layer:
226                                if fnmatch.fnmatch(f1full, item):
227                                    entry_found = True
228                                    break
229                            if not entry_found:
230                                logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
231
232    def get_file_layer(self, filename):
233        layerdir = self.get_file_layerdir(filename)
234        if layerdir:
235            return self.get_layer_name(layerdir)
236        else:
237            return '?'
238
239    def get_file_layerdir(self, filename):
240        layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
241        return self.bbfile_collections.get(layer, None)
242
243    def apply_append(self, appendname, recipename):
244        with open(appendname, 'r') as appendfile:
245            with open(recipename, 'a') as recipefile:
246                recipefile.write('\n')
247                recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
248                recipefile.writelines(appendfile.readlines())
249
250    def register_commands(self, sp):
251        parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False)
252        parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add')
253
254        parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False)
255        parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
256        parser_remove_layer.set_defaults(func=self.do_remove_layer)
257
258        parser_flatten = self.add_command(sp, 'flatten', self.do_flatten)
259        parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)')
260        parser_flatten.add_argument('outputdir', help='Output directory')
261