1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4 5import fnmatch 6import logging 7import os 8import shutil 9import sys 10import tempfile 11 12import bb.utils 13 14from bblayers.common import LayerPlugin 15 16logger = logging.getLogger('bitbake-layers') 17 18 19def plugin_init(plugins): 20 return ActionPlugin() 21 22 23class ActionPlugin(LayerPlugin): 24 def do_add_layer(self, args): 25 """Add one or more layers to bblayers.conf.""" 26 layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir] 27 28 for layerdir in layerdirs: 29 if not os.path.exists(layerdir): 30 sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir) 31 return 1 32 33 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') 34 if not os.path.exists(layer_conf): 35 sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir) 36 return 1 37 38 bblayers_conf = os.path.join('conf', 'bblayers.conf') 39 if not os.path.exists(bblayers_conf): 40 sys.stderr.write("Unable to find bblayers.conf\n") 41 return 1 42 43 # Back up bblayers.conf to tempdir before we add layers 44 tempdir = tempfile.mkdtemp() 45 backup = tempdir + "/bblayers.conf.bak" 46 shutil.copy2(bblayers_conf, backup) 47 48 try: 49 notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None) 50 if not (args.force or notadded): 51 try: 52 self.tinfoil.run_command('parseConfiguration') 53 except bb.tinfoil.TinfoilUIException: 54 # Restore the back up copy of bblayers.conf 55 shutil.copy2(backup, bblayers_conf) 56 bb.fatal("Parse failure with the specified layer added") 57 else: 58 for item in notadded: 59 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item) 60 finally: 61 # Remove the back up copy of bblayers.conf 62 shutil.rmtree(tempdir) 63 64 def do_remove_layer(self, args): 65 """Remove one or more layers from bblayers.conf.""" 66 bblayers_conf = os.path.join('conf', 'bblayers.conf') 67 if not os.path.exists(bblayers_conf): 68 sys.stderr.write("Unable to find bblayers.conf\n") 69 return 1 70 71 layerdirs = [] 72 for item in args.layerdir: 73 if item.startswith('*'): 74 layerdir = item 75 elif not '/' in item: 76 layerdir = '*/%s' % item 77 else: 78 layerdir = os.path.abspath(item) 79 layerdirs.append(layerdir) 80 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs) 81 if notremoved: 82 for item in notremoved: 83 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item) 84 return 1 85 86 def do_flatten(self, args): 87 """flatten layer configuration into a separate output directory. 88 89Takes the specified layers (or all layers in the current layer 90configuration if none are specified) and builds a "flattened" directory 91containing the contents of all layers, with any overlayed recipes removed 92and bbappends appended to the corresponding recipes. Note that some manual 93cleanup may still be necessary afterwards, in particular: 94 95* where non-recipe files (such as patches) are overwritten (the flatten 96 command will show a warning for these) 97* where anything beyond the normal layer setup has been added to 98 layer.conf (only the lowest priority number layer's layer.conf is used) 99* overridden/appended items from bbappends will need to be tidied up 100* when the flattened layers do not have the same directory structure (the 101 flatten command should show a warning when this will cause a problem) 102 103Warning: if you flatten several layers where another layer is intended to 104be used "inbetween" them (in layer priority order) such that recipes / 105bbappends in the layers interact, and then attempt to use the new output 106layer together with that other layer, you may no longer get the same 107build results (as the layer priority order has effectively changed). 108""" 109 if len(args.layer) == 1: 110 logger.error('If you specify layers to flatten you must specify at least two') 111 return 1 112 113 outputdir = args.outputdir 114 if os.path.exists(outputdir) and os.listdir(outputdir): 115 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir) 116 return 1 117 118 layers = self.bblayers 119 if len(args.layer) > 2: 120 layernames = args.layer 121 found_layernames = [] 122 found_layerdirs = [] 123 for layerdir in layers: 124 layername = self.get_layer_name(layerdir) 125 if layername in layernames: 126 found_layerdirs.append(layerdir) 127 found_layernames.append(layername) 128 129 for layername in layernames: 130 if not layername in found_layernames: 131 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0]))) 132 return 133 layers = found_layerdirs 134 else: 135 layernames = [] 136 137 # Ensure a specified path matches our list of layers 138 def layer_path_match(path): 139 for layerdir in layers: 140 if path.startswith(os.path.join(layerdir, '')): 141 return layerdir 142 return None 143 144 applied_appends = [] 145 for layer in layers: 146 overlayed = [] 147 for f in self.tinfoil.cooker.collection.overlayed.keys(): 148 for of in self.tinfoil.cooker.collection.overlayed[f]: 149 if of.startswith(layer): 150 overlayed.append(of) 151 152 logger.plain('Copying files from %s...' % layer ) 153 for root, dirs, files in os.walk(layer): 154 if '.git' in dirs: 155 dirs.remove('.git') 156 if '.hg' in dirs: 157 dirs.remove('.hg') 158 159 for f1 in files: 160 f1full = os.sep.join([root, f1]) 161 if f1full in overlayed: 162 logger.plain(' Skipping overlayed file %s' % f1full ) 163 else: 164 ext = os.path.splitext(f1)[1] 165 if ext != '.bbappend': 166 fdest = f1full[len(layer):] 167 fdest = os.path.normpath(os.sep.join([outputdir,fdest])) 168 bb.utils.mkdirhier(os.path.dirname(fdest)) 169 if os.path.exists(fdest): 170 if f1 == 'layer.conf' and root.endswith('/conf'): 171 logger.plain(' Skipping layer config file %s' % f1full ) 172 continue 173 else: 174 logger.warning('Overwriting file %s', fdest) 175 bb.utils.copyfile(f1full, fdest) 176 if ext == '.bb': 177 for append in self.tinfoil.cooker.collection.get_file_appends(f1full): 178 if layer_path_match(append): 179 logger.plain(' Applying append %s to %s' % (append, fdest)) 180 self.apply_append(append, fdest) 181 applied_appends.append(append) 182 183 # Take care of when some layers are excluded and yet we have included bbappends for those recipes 184 for b in self.tinfoil.cooker.collection.bbappends: 185 (recipename, appendname) = b 186 if appendname not in applied_appends: 187 first_append = None 188 layer = layer_path_match(appendname) 189 if layer: 190 if first_append: 191 self.apply_append(appendname, first_append) 192 else: 193 fdest = appendname[len(layer):] 194 fdest = os.path.normpath(os.sep.join([outputdir,fdest])) 195 bb.utils.mkdirhier(os.path.dirname(fdest)) 196 bb.utils.copyfile(appendname, fdest) 197 first_append = fdest 198 199 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will 200 # have come from) 201 first_regex = None 202 layerdir = layers[0] 203 for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities: 204 if regex.match(os.path.join(layerdir, 'test')): 205 first_regex = regex 206 break 207 208 if first_regex: 209 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file) 210 bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split() 211 bbfiles_layer = [] 212 for item in bbfiles: 213 if first_regex.match(item): 214 newpath = os.path.join(outputdir, item[len(layerdir)+1:]) 215 bbfiles_layer.append(newpath) 216 217 if bbfiles_layer: 218 # Check that all important layer files match BBFILES 219 for root, dirs, files in os.walk(outputdir): 220 for f1 in files: 221 ext = os.path.splitext(f1)[1] 222 if ext in ['.bb', '.bbappend']: 223 f1full = os.sep.join([root, f1]) 224 entry_found = False 225 for item in bbfiles_layer: 226 if fnmatch.fnmatch(f1full, item): 227 entry_found = True 228 break 229 if not entry_found: 230 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full) 231 232 def get_file_layer(self, filename): 233 layerdir = self.get_file_layerdir(filename) 234 if layerdir: 235 return self.get_layer_name(layerdir) 236 else: 237 return '?' 238 239 def get_file_layerdir(self, filename): 240 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data) 241 return self.bbfile_collections.get(layer, None) 242 243 def apply_append(self, appendname, recipename): 244 with open(appendname, 'r') as appendfile: 245 with open(recipename, 'a') as recipefile: 246 recipefile.write('\n') 247 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname)) 248 recipefile.writelines(appendfile.readlines()) 249 250 def register_commands(self, sp): 251 parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False) 252 parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add') 253 254 parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False) 255 parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)') 256 parser_remove_layer.set_defaults(func=self.do_remove_layer) 257 258 parser_flatten = self.add_command(sp, 'flatten', self.do_flatten) 259 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)') 260 parser_flatten.add_argument('outputdir', help='Output directory') 261