1# 2# Copyright BitBake Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import fnmatch 8import logging 9import os 10import shutil 11import sys 12import tempfile 13 14from bb.cookerdata import findTopdir 15import bb.utils 16 17from bblayers.common import LayerPlugin 18 19logger = logging.getLogger('bitbake-layers') 20 21 22def plugin_init(plugins): 23 return ActionPlugin() 24 25 26class ActionPlugin(LayerPlugin): 27 def do_add_layer(self, args): 28 """Add one or more layers to bblayers.conf.""" 29 layerdirs = [os.path.abspath(ldir) for ldir in args.layerdir] 30 31 for layerdir in layerdirs: 32 if not os.path.exists(layerdir): 33 sys.stderr.write("Specified layer directory %s doesn't exist\n" % layerdir) 34 return 1 35 36 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf') 37 if not os.path.exists(layer_conf): 38 sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir) 39 return 1 40 41 bblayers_conf = os.path.join(findTopdir(),'conf', 'bblayers.conf') 42 if not os.path.exists(bblayers_conf): 43 sys.stderr.write("Unable to find bblayers.conf\n") 44 return 1 45 46 # Back up bblayers.conf to tempdir before we add layers 47 tempdir = tempfile.mkdtemp() 48 backup = tempdir + "/bblayers.conf.bak" 49 shutil.copy2(bblayers_conf, backup) 50 51 try: 52 notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None) 53 if not (args.force or notadded): 54 self.tinfoil.modified_files() 55 try: 56 self.tinfoil.run_command('parseConfiguration') 57 except (bb.tinfoil.TinfoilUIException, bb.BBHandledException): 58 # Restore the back up copy of bblayers.conf 59 shutil.copy2(backup, bblayers_conf) 60 self.tinfoil.modified_files() 61 bb.fatal("Parse failure with the specified layer added, exiting.") 62 else: 63 for item in notadded: 64 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item) 65 finally: 66 # Remove the back up copy of bblayers.conf 67 shutil.rmtree(tempdir) 68 69 def do_remove_layer(self, args): 70 """Remove one or more layers from bblayers.conf.""" 71 bblayers_conf = os.path.join(findTopdir() ,'conf', 'bblayers.conf') 72 if not os.path.exists(bblayers_conf): 73 sys.stderr.write("Unable to find bblayers.conf\n") 74 return 1 75 76 layerdirs = [] 77 for item in args.layerdir: 78 if item.startswith('*'): 79 layerdir = item 80 elif not '/' in item: 81 layerdir = '*/%s' % item 82 else: 83 layerdir = os.path.abspath(item) 84 layerdirs.append(layerdir) 85 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs) 86 if args.force > 1: 87 return 0 88 self.tinfoil.modified_files() 89 if notremoved: 90 for item in notremoved: 91 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item) 92 return 1 93 94 def do_flatten(self, args): 95 """flatten layer configuration into a separate output directory. 96 97Takes the specified layers (or all layers in the current layer 98configuration if none are specified) and builds a "flattened" directory 99containing the contents of all layers, with any overlayed recipes removed 100and bbappends appended to the corresponding recipes. Note that some manual 101cleanup may still be necessary afterwards, in particular: 102 103* where non-recipe files (such as patches) are overwritten (the flatten 104 command will show a warning for these) 105* where anything beyond the normal layer setup has been added to 106 layer.conf (only the lowest priority number layer's layer.conf is used) 107* overridden/appended items from bbappends will need to be tidied up 108* when the flattened layers do not have the same directory structure (the 109 flatten command should show a warning when this will cause a problem) 110 111Warning: if you flatten several layers where another layer is intended to 112be used "inbetween" them (in layer priority order) such that recipes / 113bbappends in the layers interact, and then attempt to use the new output 114layer together with that other layer, you may no longer get the same 115build results (as the layer priority order has effectively changed). 116""" 117 if len(args.layer) == 1: 118 logger.error('If you specify layers to flatten you must specify at least two') 119 return 1 120 121 outputdir = args.outputdir 122 if os.path.exists(outputdir) and os.listdir(outputdir): 123 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir) 124 return 1 125 126 layers = self.bblayers 127 if len(args.layer) > 2: 128 layernames = args.layer 129 found_layernames = [] 130 found_layerdirs = [] 131 for layerdir in layers: 132 layername = self.get_layer_name(layerdir) 133 if layername in layernames: 134 found_layerdirs.append(layerdir) 135 found_layernames.append(layername) 136 137 for layername in layernames: 138 if not layername in found_layernames: 139 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0]))) 140 return 141 layers = found_layerdirs 142 else: 143 layernames = [] 144 145 # Ensure a specified path matches our list of layers 146 def layer_path_match(path): 147 for layerdir in layers: 148 if path.startswith(os.path.join(layerdir, '')): 149 return layerdir 150 return None 151 152 applied_appends = [] 153 for layer in layers: 154 overlayed = set() 155 for mc in self.tinfoil.cooker.multiconfigs: 156 for f in self.tinfoil.cooker.collections[mc].overlayed.keys(): 157 for of in self.tinfoil.cooker.collections[mc].overlayed[f]: 158 if of.startswith(layer): 159 overlayed.add(of) 160 161 logger.plain('Copying files from %s...' % layer ) 162 for root, dirs, files in os.walk(layer): 163 if '.git' in dirs: 164 dirs.remove('.git') 165 if '.hg' in dirs: 166 dirs.remove('.hg') 167 168 for f1 in files: 169 f1full = os.sep.join([root, f1]) 170 if f1full in overlayed: 171 logger.plain(' Skipping overlayed file %s' % f1full ) 172 else: 173 ext = os.path.splitext(f1)[1] 174 if ext != '.bbappend': 175 fdest = f1full[len(layer):] 176 fdest = os.path.normpath(os.sep.join([outputdir,fdest])) 177 bb.utils.mkdirhier(os.path.dirname(fdest)) 178 if os.path.exists(fdest): 179 if f1 == 'layer.conf' and root.endswith('/conf'): 180 logger.plain(' Skipping layer config file %s' % f1full ) 181 continue 182 else: 183 logger.warning('Overwriting file %s', fdest) 184 bb.utils.copyfile(f1full, fdest) 185 if ext == '.bb': 186 appends = set() 187 for mc in self.tinfoil.cooker.multiconfigs: 188 appends |= set(self.tinfoil.cooker.collections[mc].get_file_appends(f1full)) 189 for append in appends: 190 if layer_path_match(append): 191 logger.plain(' Applying append %s to %s' % (append, fdest)) 192 self.apply_append(append, fdest) 193 applied_appends.append(append) 194 195 # Take care of when some layers are excluded and yet we have included bbappends for those recipes 196 bbappends = set() 197 for mc in self.tinfoil.cooker.multiconfigs: 198 bbappends |= set(self.tinfoil.cooker.collections[mc].bbappends) 199 200 for b in bbappends: 201 (recipename, appendname) = b 202 if appendname not in applied_appends: 203 first_append = None 204 layer = layer_path_match(appendname) 205 if layer: 206 if first_append: 207 self.apply_append(appendname, first_append) 208 else: 209 fdest = appendname[len(layer):] 210 fdest = os.path.normpath(os.sep.join([outputdir,fdest])) 211 bb.utils.mkdirhier(os.path.dirname(fdest)) 212 bb.utils.copyfile(appendname, fdest) 213 first_append = fdest 214 215 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will 216 # have come from) 217 first_regex = None 218 layerdir = layers[0] 219 for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities: 220 if regex.match(os.path.join(layerdir, 'test')): 221 first_regex = regex 222 break 223 224 if first_regex: 225 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file) 226 bbfiles = str(self.tinfoil.config_data.getVar('BBFILES')).split() 227 bbfiles_layer = [] 228 for item in bbfiles: 229 if first_regex.match(item): 230 newpath = os.path.join(outputdir, item[len(layerdir)+1:]) 231 bbfiles_layer.append(newpath) 232 233 if bbfiles_layer: 234 # Check that all important layer files match BBFILES 235 for root, dirs, files in os.walk(outputdir): 236 for f1 in files: 237 ext = os.path.splitext(f1)[1] 238 if ext in ['.bb', '.bbappend']: 239 f1full = os.sep.join([root, f1]) 240 entry_found = False 241 for item in bbfiles_layer: 242 if fnmatch.fnmatch(f1full, item): 243 entry_found = True 244 break 245 if not entry_found: 246 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full) 247 248 self.tinfoil.modified_files() 249 250 251 def get_file_layer(self, filename): 252 layerdir = self.get_file_layerdir(filename) 253 if layerdir: 254 return self.get_layer_name(layerdir) 255 else: 256 return '?' 257 258 def get_file_layerdir(self, filename): 259 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data) 260 return self.bbfile_collections.get(layer, None) 261 262 def apply_append(self, appendname, recipename): 263 with open(appendname, 'r') as appendfile: 264 with open(recipename, 'a') as recipefile: 265 recipefile.write('\n') 266 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname)) 267 recipefile.writelines(appendfile.readlines()) 268 269 def register_commands(self, sp): 270 parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False) 271 parser_add_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to add') 272 273 parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False) 274 parser_remove_layer.add_argument('layerdir', nargs='+', help='Layer directory/directories to remove (wildcards allowed, enclose in quotes to avoid shell expansion)') 275 parser_remove_layer.set_defaults(func=self.do_remove_layer) 276 277 parser_flatten = self.add_command(sp, 'flatten', self.do_flatten) 278 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)') 279 parser_flatten.add_argument('outputdir', help='Output directory') 280