1# 2# SPDX-License-Identifier: GPL-2.0-only 3# 4# This class should provide easy access to the different aspects of the 5# buildsystem such as layers, bitbake location, etc. 6# 7# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers. 8# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports 9# python regular expression, use space as separator, 10# e.g.: ".*-downloads closed-.*" 11# 12 13import stat 14import shutil 15 16def _smart_copy(src, dest): 17 import subprocess 18 # smart_copy will choose the correct function depending on whether the 19 # source is a file or a directory. 20 mode = os.stat(src).st_mode 21 if stat.S_ISDIR(mode): 22 bb.utils.mkdirhier(dest) 23 cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \ 24 | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) 25 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) 26 else: 27 shutil.copyfile(src, dest) 28 shutil.copymode(src, dest) 29 30class BuildSystem(object): 31 def __init__(self, context, d): 32 self.d = d 33 self.context = context 34 self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()] 35 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split() 36 self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN') 37 38 def copy_bitbake_and_layers(self, destdir, workspace_name=None): 39 import re 40 # Copy in all metadata layers + bitbake (as repositories) 41 copied_corebase = None 42 layers_copied = [] 43 bb.utils.mkdirhier(destdir) 44 layers = list(self.layerdirs) 45 46 corebase = os.path.abspath(self.d.getVar('COREBASE')) 47 layers.append(corebase) 48 # The bitbake build system uses the meta-skeleton layer as a layout 49 # for common recipies, e.g: the recipetool script to create kernel recipies 50 # Add the meta-skeleton layer to be included as part of the eSDK installation 51 layers.append(os.path.join(corebase, 'meta-skeleton')) 52 53 # Exclude layers 54 for layer_exclude in self.layers_exclude: 55 if layer_exclude in layers: 56 bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude) 57 layers.remove(layer_exclude) 58 59 if self.layers_exclude_pattern: 60 layers_cp = layers[:] 61 for pattern in self.layers_exclude_pattern.split(): 62 for layer in layers_cp: 63 if re.match(pattern, layer): 64 bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer) 65 layers.remove(layer) 66 67 workspace_newname = workspace_name 68 if workspace_newname: 69 layernames = [os.path.basename(layer) for layer in layers] 70 extranum = 0 71 while workspace_newname in layernames: 72 extranum += 1 73 workspace_newname = '%s-%d' % (workspace_name, extranum) 74 75 corebase_files = self.d.getVar('COREBASE_FILES').split() 76 corebase_files = [corebase + '/' +x for x in corebase_files] 77 # Make sure bitbake goes in 78 bitbake_dir = bb.__file__.rsplit('/', 3)[0] 79 corebase_files.append(bitbake_dir) 80 81 for layer in layers: 82 layerconf = os.path.join(layer, 'conf', 'layer.conf') 83 layernewname = os.path.basename(layer) 84 workspace = False 85 if os.path.exists(layerconf): 86 with open(layerconf, 'r') as f: 87 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): 88 if workspace_newname: 89 layernewname = workspace_newname 90 workspace = True 91 else: 92 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) 93 continue 94 95 # If the layer was already under corebase, leave it there 96 # since layers such as meta have issues when moved. 97 layerdestpath = destdir 98 if corebase == os.path.dirname(layer): 99 layerdestpath += '/' + os.path.basename(corebase) 100 # If the layer is located somewhere under the same parent directory 101 # as corebase we keep the layer structure. 102 elif os.path.commonpath([layer, corebase]) == os.path.dirname(corebase): 103 layer_relative = os.path.relpath(layer, os.path.dirname(corebase)) 104 if os.path.dirname(layer_relative) != layernewname: 105 layerdestpath += '/' + os.path.dirname(layer_relative) 106 107 layerdestpath += '/' + layernewname 108 109 layer_relative = os.path.relpath(layerdestpath, 110 destdir) 111 # Treat corebase as special since it typically will contain 112 # build directories or other custom items. 113 if corebase == layer: 114 copied_corebase = layer_relative 115 bb.utils.mkdirhier(layerdestpath) 116 for f in corebase_files: 117 f_basename = os.path.basename(f) 118 destname = os.path.join(layerdestpath, f_basename) 119 _smart_copy(f, destname) 120 else: 121 layers_copied.append(layer_relative) 122 123 if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')): 124 bb.note("Skipping layer %s, already handled" % layer) 125 else: 126 _smart_copy(layer, layerdestpath) 127 128 if workspace: 129 # Make some adjustments original workspace layer 130 # Drop sources (recipe tasks will be locked, so we don't need them) 131 srcdir = os.path.join(layerdestpath, 'sources') 132 if os.path.isdir(srcdir): 133 shutil.rmtree(srcdir) 134 # Drop all bbappends except the one for the image the SDK is being built for 135 # (because of externalsrc, the workspace bbappends will interfere with the 136 # locked signatures if present, and we don't need them anyway) 137 image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend' 138 appenddir = os.path.join(layerdestpath, 'appends') 139 if os.path.isdir(appenddir): 140 for fn in os.listdir(appenddir): 141 if fn == image_bbappend: 142 continue 143 else: 144 os.remove(os.path.join(appenddir, fn)) 145 # Drop README 146 readme = os.path.join(layerdestpath, 'README') 147 if os.path.exists(readme): 148 os.remove(readme) 149 # Filter out comments in layer.conf and change layer name 150 layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf') 151 with open(layerconf, 'r') as f: 152 origlines = f.readlines() 153 with open(layerconf, 'w') as f: 154 for line in origlines: 155 if line.startswith('#'): 156 continue 157 line = line.replace('workspacelayer', workspace_newname) 158 f.write(line) 159 160 # meta-skeleton layer is added as part of the build system 161 # but not as a layer included in the build, therefore it is 162 # not reported to the function caller. 163 for layer in layers_copied: 164 if layer.endswith('/meta-skeleton'): 165 layers_copied.remove(layer) 166 break 167 168 return copied_corebase, layers_copied 169 170def generate_locked_sigs(sigfile, d): 171 bb.utils.mkdirhier(os.path.dirname(sigfile)) 172 depd = d.getVar('BB_TASKDEPDATA', False) 173 tasks = ['%s:%s' % (v[2], v[1]) for v in depd.values()] 174 bb.parse.siggen.dump_lockedsigs(sigfile, tasks) 175 176def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, onlynative, pruned_output): 177 with open(lockedsigs, 'r') as infile: 178 bb.utils.mkdirhier(os.path.dirname(pruned_output)) 179 with open(pruned_output, 'w') as f: 180 invalue = False 181 for line in infile: 182 if invalue: 183 if line.endswith('\\\n'): 184 splitval = line.strip().split(':') 185 if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets: 186 if onlynative: 187 if 'nativesdk' in splitval[0]: 188 f.write(line) 189 else: 190 f.write(line) 191 else: 192 f.write(line) 193 invalue = False 194 elif line.startswith('SIGGEN_LOCKEDSIGS'): 195 invalue = True 196 f.write(line) 197 198def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): 199 merged = {} 200 arch_order = [] 201 with open(lockedsigs_main, 'r') as f: 202 invalue = None 203 for line in f: 204 if invalue: 205 if line.endswith('\\\n'): 206 merged[invalue].append(line) 207 else: 208 invalue = None 209 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): 210 invalue = line[18:].split('=', 1)[0].rstrip() 211 merged[invalue] = [] 212 arch_order.append(invalue) 213 214 with open(lockedsigs_extra, 'r') as f: 215 invalue = None 216 tocopy = {} 217 for line in f: 218 if invalue: 219 if line.endswith('\\\n'): 220 if not line in merged[invalue]: 221 target, task = line.strip().split(':')[:2] 222 if not copy_tasks or task in copy_tasks: 223 tocopy[invalue].append(line) 224 merged[invalue].append(line) 225 else: 226 invalue = None 227 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): 228 invalue = line[18:].split('=', 1)[0].rstrip() 229 if not invalue in merged: 230 merged[invalue] = [] 231 arch_order.append(invalue) 232 tocopy[invalue] = [] 233 234 def write_sigs_file(fn, types, sigs): 235 fulltypes = [] 236 bb.utils.mkdirhier(os.path.dirname(fn)) 237 with open(fn, 'w') as f: 238 for typename in types: 239 lines = sigs[typename] 240 if lines: 241 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename) 242 for line in lines: 243 f.write(line) 244 f.write(' "\n') 245 fulltypes.append(typename) 246 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) 247 248 if copy_output: 249 write_sigs_file(copy_output, list(tocopy.keys()), tocopy) 250 if merged_output: 251 write_sigs_file(merged_output, arch_order, merged) 252 253def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): 254 import shutil 255 bb.note('Generating sstate-cache...') 256 257 nativelsbstring = d.getVar('NATIVELSBSTRING') 258 bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) 259 if fixedlsbstring and nativelsbstring != fixedlsbstring: 260 nativedir = output_sstate_cache + '/' + nativelsbstring 261 if os.path.isdir(nativedir): 262 destdir = os.path.join(output_sstate_cache, fixedlsbstring) 263 for root, _, files in os.walk(nativedir): 264 for fn in files: 265 src = os.path.join(root, fn) 266 dest = os.path.join(destdir, os.path.relpath(src, nativedir)) 267 if os.path.exists(dest): 268 # Already exists, and it'll be the same file, so just delete it 269 os.unlink(src) 270 else: 271 bb.utils.mkdirhier(os.path.dirname(dest)) 272 shutil.move(src, dest) 273 274def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): 275 import subprocess 276 277 bb.note('Generating sstate task list...') 278 279 if not cwd: 280 cwd = os.getcwd() 281 if logfile: 282 logparam = '-l %s' % logfile 283 else: 284 logparam = '' 285 cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) 286 env = dict(d.getVar('BB_ORIGENV', False)) 287 env.pop('BUILDDIR', '') 288 env.pop('BBPATH', '') 289 pathitems = env['PATH'].split(':') 290 env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) 291 bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') 292