1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6# This class should provide easy access to the different aspects of the 7# buildsystem such as layers, bitbake location, etc. 8# 9# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers. 10# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports 11# python regular expression, use space as separator, 12# e.g.: ".*-downloads closed-.*" 13# 14 15import stat 16import shutil 17 18def _smart_copy(src, dest): 19 import subprocess 20 # smart_copy will choose the correct function depending on whether the 21 # source is a file or a directory. 22 mode = os.stat(src).st_mode 23 if stat.S_ISDIR(mode): 24 bb.utils.mkdirhier(dest) 25 cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -cf - -C %s -p . \ 26 | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) 27 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) 28 else: 29 shutil.copyfile(src, dest) 30 shutil.copymode(src, dest) 31 32class BuildSystem(object): 33 def __init__(self, context, d): 34 self.d = d 35 self.context = context 36 self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()] 37 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split() 38 self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN') 39 40 def copy_bitbake_and_layers(self, destdir, workspace_name=None): 41 import re 42 # Copy in all metadata layers + bitbake (as repositories) 43 copied_corebase = None 44 layers_copied = [] 45 bb.utils.mkdirhier(destdir) 46 layers = list(self.layerdirs) 47 48 corebase = os.path.abspath(self.d.getVar('COREBASE')) 49 layers.append(corebase) 50 # The bitbake build system uses the meta-skeleton layer as a layout 51 # for common recipies, e.g: the recipetool script to create kernel recipies 52 # Add the meta-skeleton layer to be included as part of the eSDK installation 53 layers.append(os.path.join(corebase, 'meta-skeleton')) 54 55 # Exclude layers 56 for layer_exclude in self.layers_exclude: 57 if layer_exclude in layers: 58 bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude) 59 layers.remove(layer_exclude) 60 61 if self.layers_exclude_pattern: 62 layers_cp = layers[:] 63 for pattern in self.layers_exclude_pattern.split(): 64 for layer in layers_cp: 65 if re.match(pattern, layer): 66 bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer) 67 layers.remove(layer) 68 69 workspace_newname = workspace_name 70 if workspace_newname: 71 layernames = [os.path.basename(layer) for layer in layers] 72 extranum = 0 73 while workspace_newname in layernames: 74 extranum += 1 75 workspace_newname = '%s-%d' % (workspace_name, extranum) 76 77 corebase_files = self.d.getVar('COREBASE_FILES').split() 78 corebase_files = [corebase + '/' +x for x in corebase_files] 79 # Make sure bitbake goes in 80 bitbake_dir = bb.__file__.rsplit('/', 3)[0] 81 corebase_files.append(bitbake_dir) 82 83 for layer in layers: 84 layerconf = os.path.join(layer, 'conf', 'layer.conf') 85 layernewname = os.path.basename(layer) 86 workspace = False 87 if os.path.exists(layerconf): 88 with open(layerconf, 'r') as f: 89 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): 90 if workspace_newname: 91 layernewname = workspace_newname 92 workspace = True 93 else: 94 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) 95 continue 96 97 # If the layer was already under corebase, leave it there 98 # since layers such as meta have issues when moved. 99 layerdestpath = destdir 100 if corebase == os.path.dirname(layer): 101 layerdestpath += '/' + os.path.basename(corebase) 102 # If the layer is located somewhere under the same parent directory 103 # as corebase we keep the layer structure. 104 elif os.path.commonpath([layer, corebase]) == os.path.dirname(corebase): 105 layer_relative = os.path.relpath(layer, os.path.dirname(corebase)) 106 if os.path.dirname(layer_relative) != layernewname: 107 layerdestpath += '/' + os.path.dirname(layer_relative) 108 109 layerdestpath += '/' + layernewname 110 111 layer_relative = os.path.relpath(layerdestpath, 112 destdir) 113 # Treat corebase as special since it typically will contain 114 # build directories or other custom items. 115 if corebase == layer: 116 copied_corebase = layer_relative 117 bb.utils.mkdirhier(layerdestpath) 118 for f in corebase_files: 119 f_basename = os.path.basename(f) 120 destname = os.path.join(layerdestpath, f_basename) 121 _smart_copy(f, destname) 122 else: 123 layers_copied.append(layer_relative) 124 125 if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')): 126 bb.note("Skipping layer %s, already handled" % layer) 127 else: 128 _smart_copy(layer, layerdestpath) 129 130 if workspace: 131 # Make some adjustments original workspace layer 132 # Drop sources (recipe tasks will be locked, so we don't need them) 133 srcdir = os.path.join(layerdestpath, 'sources') 134 if os.path.isdir(srcdir): 135 shutil.rmtree(srcdir) 136 # Drop all bbappends except the one for the image the SDK is being built for 137 # (because of externalsrc, the workspace bbappends will interfere with the 138 # locked signatures if present, and we don't need them anyway) 139 image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend' 140 appenddir = os.path.join(layerdestpath, 'appends') 141 if os.path.isdir(appenddir): 142 for fn in os.listdir(appenddir): 143 if fn == image_bbappend: 144 continue 145 else: 146 os.remove(os.path.join(appenddir, fn)) 147 # Drop README 148 readme = os.path.join(layerdestpath, 'README') 149 if os.path.exists(readme): 150 os.remove(readme) 151 # Filter out comments in layer.conf and change layer name 152 layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf') 153 with open(layerconf, 'r') as f: 154 origlines = f.readlines() 155 with open(layerconf, 'w') as f: 156 for line in origlines: 157 if line.startswith('#'): 158 continue 159 line = line.replace('workspacelayer', workspace_newname) 160 f.write(line) 161 162 # meta-skeleton layer is added as part of the build system 163 # but not as a layer included in the build, therefore it is 164 # not reported to the function caller. 165 for layer in layers_copied: 166 if layer.endswith('/meta-skeleton'): 167 layers_copied.remove(layer) 168 break 169 170 return copied_corebase, layers_copied 171 172def generate_locked_sigs(sigfile, d): 173 bb.utils.mkdirhier(os.path.dirname(sigfile)) 174 depd = d.getVar('BB_TASKDEPDATA', False) 175 tasks = ['%s:%s' % (v[2], v[1]) for v in depd.values()] 176 bb.parse.siggen.dump_lockedsigs(sigfile, tasks) 177 178def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, onlynative, pruned_output): 179 with open(lockedsigs, 'r') as infile: 180 bb.utils.mkdirhier(os.path.dirname(pruned_output)) 181 with open(pruned_output, 'w') as f: 182 invalue = False 183 for line in infile: 184 if invalue: 185 if line.endswith('\\\n'): 186 splitval = line.strip().split(':') 187 if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets: 188 if onlynative: 189 if 'nativesdk' in splitval[0]: 190 f.write(line) 191 else: 192 f.write(line) 193 else: 194 f.write(line) 195 invalue = False 196 elif line.startswith('SIGGEN_LOCKEDSIGS_t'): 197 invalue = True 198 f.write(line) 199 else: 200 invalue = False 201 f.write(line) 202 203def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): 204 merged = {} 205 arch_order = [] 206 otherdata = [] 207 with open(lockedsigs_main, 'r') as f: 208 invalue = None 209 for line in f: 210 if invalue: 211 if line.endswith('\\\n'): 212 merged[invalue].append(line) 213 else: 214 invalue = None 215 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): 216 invalue = line[18:].split('=', 1)[0].rstrip() 217 merged[invalue] = [] 218 arch_order.append(invalue) 219 else: 220 invalue = None 221 otherdata.append(line) 222 223 with open(lockedsigs_extra, 'r') as f: 224 invalue = None 225 tocopy = {} 226 for line in f: 227 if invalue: 228 if line.endswith('\\\n'): 229 if not line in merged[invalue]: 230 target, task = line.strip().split(':')[:2] 231 if not copy_tasks or task in copy_tasks: 232 tocopy[invalue].append(line) 233 merged[invalue].append(line) 234 else: 235 invalue = None 236 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): 237 invalue = line[18:].split('=', 1)[0].rstrip() 238 if not invalue in merged: 239 merged[invalue] = [] 240 arch_order.append(invalue) 241 tocopy[invalue] = [] 242 243 def write_sigs_file(fn, types, sigs): 244 fulltypes = [] 245 bb.utils.mkdirhier(os.path.dirname(fn)) 246 with open(fn, 'w') as f: 247 for typename in types: 248 lines = sigs[typename] 249 if lines: 250 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename) 251 for line in lines: 252 f.write(line) 253 f.write(' "\n') 254 fulltypes.append(typename) 255 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) 256 f.write('\n' + ''.join(otherdata)) 257 258 if copy_output: 259 write_sigs_file(copy_output, list(tocopy.keys()), tocopy) 260 if merged_output: 261 write_sigs_file(merged_output, arch_order, merged) 262 263def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): 264 import shutil 265 bb.note('Generating sstate-cache...') 266 267 nativelsbstring = d.getVar('NATIVELSBSTRING') 268 bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) 269 if fixedlsbstring and nativelsbstring != fixedlsbstring: 270 nativedir = output_sstate_cache + '/' + nativelsbstring 271 if os.path.isdir(nativedir): 272 destdir = os.path.join(output_sstate_cache, fixedlsbstring) 273 for root, _, files in os.walk(nativedir): 274 for fn in files: 275 src = os.path.join(root, fn) 276 dest = os.path.join(destdir, os.path.relpath(src, nativedir)) 277 if os.path.exists(dest): 278 # Already exists, and it'll be the same file, so just delete it 279 os.unlink(src) 280 else: 281 bb.utils.mkdirhier(os.path.dirname(dest)) 282 shutil.move(src, dest) 283 284def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): 285 import subprocess 286 287 bb.note('Generating sstate task list...') 288 289 if not cwd: 290 cwd = os.getcwd() 291 if logfile: 292 logparam = '-l %s' % logfile 293 else: 294 logparam = '' 295 cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) 296 env = dict(d.getVar('BB_ORIGENV', False)) 297 env.pop('BUILDDIR', '') 298 env.pop('BBPATH', '') 299 pathitems = env['PATH'].split(':') 300 env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) 301 bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') 302