1# This class should provide easy access to the different aspects of the 2# buildsystem such as layers, bitbake location, etc. 3# 4# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers. 5# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports 6# python regular expression, use space as separator, 7# e.g.: ".*-downloads closed-.*" 8# 9 10import stat 11import shutil 12 13def _smart_copy(src, dest): 14 import subprocess 15 # smart_copy will choose the correct function depending on whether the 16 # source is a file or a directory. 17 mode = os.stat(src).st_mode 18 if stat.S_ISDIR(mode): 19 bb.utils.mkdirhier(dest) 20 cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \ 21 | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest) 22 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) 23 else: 24 shutil.copyfile(src, dest) 25 shutil.copymode(src, dest) 26 27class BuildSystem(object): 28 def __init__(self, context, d): 29 self.d = d 30 self.context = context 31 self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()] 32 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split() 33 self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN') 34 35 def copy_bitbake_and_layers(self, destdir, workspace_name=None): 36 import re 37 # Copy in all metadata layers + bitbake (as repositories) 38 copied_corebase = None 39 layers_copied = [] 40 bb.utils.mkdirhier(destdir) 41 layers = list(self.layerdirs) 42 43 corebase = os.path.abspath(self.d.getVar('COREBASE')) 44 layers.append(corebase) 45 # The bitbake build system uses the meta-skeleton layer as a layout 46 # for common recipies, e.g: the recipetool script to create kernel recipies 47 # Add the meta-skeleton layer to be included as part of the eSDK installation 48 layers.append(os.path.join(corebase, 'meta-skeleton')) 49 50 # Exclude layers 51 for layer_exclude in self.layers_exclude: 52 if layer_exclude in layers: 53 bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude) 54 layers.remove(layer_exclude) 55 56 if self.layers_exclude_pattern: 57 layers_cp = layers[:] 58 for pattern in self.layers_exclude_pattern.split(): 59 for layer in layers_cp: 60 if re.match(pattern, layer): 61 bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer) 62 layers.remove(layer) 63 64 workspace_newname = workspace_name 65 if workspace_newname: 66 layernames = [os.path.basename(layer) for layer in layers] 67 extranum = 0 68 while workspace_newname in layernames: 69 extranum += 1 70 workspace_newname = '%s-%d' % (workspace_name, extranum) 71 72 corebase_files = self.d.getVar('COREBASE_FILES').split() 73 corebase_files = [corebase + '/' +x for x in corebase_files] 74 # Make sure bitbake goes in 75 bitbake_dir = bb.__file__.rsplit('/', 3)[0] 76 corebase_files.append(bitbake_dir) 77 78 for layer in layers: 79 layerconf = os.path.join(layer, 'conf', 'layer.conf') 80 layernewname = os.path.basename(layer) 81 workspace = False 82 if os.path.exists(layerconf): 83 with open(layerconf, 'r') as f: 84 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): 85 if workspace_newname: 86 layernewname = workspace_newname 87 workspace = True 88 else: 89 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) 90 continue 91 92 # If the layer was already under corebase, leave it there 93 # since layers such as meta have issues when moved. 94 layerdestpath = destdir 95 if corebase == os.path.dirname(layer): 96 layerdestpath += '/' + os.path.basename(corebase) 97 else: 98 layer_relative = os.path.basename(corebase) + '/' + os.path.relpath(layer, corebase) 99 if os.path.dirname(layer_relative) != layernewname: 100 layerdestpath += '/' + os.path.dirname(layer_relative) 101 102 layerdestpath += '/' + layernewname 103 104 layer_relative = os.path.relpath(layerdestpath, 105 destdir) 106 # Treat corebase as special since it typically will contain 107 # build directories or other custom items. 108 if corebase == layer: 109 copied_corebase = layer_relative 110 bb.utils.mkdirhier(layerdestpath) 111 for f in corebase_files: 112 f_basename = os.path.basename(f) 113 destname = os.path.join(layerdestpath, f_basename) 114 _smart_copy(f, destname) 115 else: 116 layers_copied.append(layer_relative) 117 118 if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')): 119 bb.note("Skipping layer %s, already handled" % layer) 120 else: 121 _smart_copy(layer, layerdestpath) 122 123 if workspace: 124 # Make some adjustments original workspace layer 125 # Drop sources (recipe tasks will be locked, so we don't need them) 126 srcdir = os.path.join(layerdestpath, 'sources') 127 if os.path.isdir(srcdir): 128 shutil.rmtree(srcdir) 129 # Drop all bbappends except the one for the image the SDK is being built for 130 # (because of externalsrc, the workspace bbappends will interfere with the 131 # locked signatures if present, and we don't need them anyway) 132 image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend' 133 appenddir = os.path.join(layerdestpath, 'appends') 134 if os.path.isdir(appenddir): 135 for fn in os.listdir(appenddir): 136 if fn == image_bbappend: 137 continue 138 else: 139 os.remove(os.path.join(appenddir, fn)) 140 # Drop README 141 readme = os.path.join(layerdestpath, 'README') 142 if os.path.exists(readme): 143 os.remove(readme) 144 # Filter out comments in layer.conf and change layer name 145 layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf') 146 with open(layerconf, 'r') as f: 147 origlines = f.readlines() 148 with open(layerconf, 'w') as f: 149 for line in origlines: 150 if line.startswith('#'): 151 continue 152 line = line.replace('workspacelayer', workspace_newname) 153 f.write(line) 154 155 # meta-skeleton layer is added as part of the build system 156 # but not as a layer included in the build, therefore it is 157 # not reported to the function caller. 158 for layer in layers_copied: 159 if layer.endswith('/meta-skeleton'): 160 layers_copied.remove(layer) 161 break 162 163 return copied_corebase, layers_copied 164 165def generate_locked_sigs(sigfile, d): 166 bb.utils.mkdirhier(os.path.dirname(sigfile)) 167 depd = d.getVar('BB_TASKDEPDATA', False) 168 tasks = ['%s.%s' % (v[2], v[1]) for v in depd.values()] 169 bb.parse.siggen.dump_lockedsigs(sigfile, tasks) 170 171def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output): 172 with open(lockedsigs, 'r') as infile: 173 bb.utils.mkdirhier(os.path.dirname(pruned_output)) 174 with open(pruned_output, 'w') as f: 175 invalue = False 176 for line in infile: 177 if invalue: 178 if line.endswith('\\\n'): 179 splitval = line.strip().split(':') 180 if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets: 181 f.write(line) 182 else: 183 f.write(line) 184 invalue = False 185 elif line.startswith('SIGGEN_LOCKEDSIGS'): 186 invalue = True 187 f.write(line) 188 189def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None): 190 merged = {} 191 arch_order = [] 192 with open(lockedsigs_main, 'r') as f: 193 invalue = None 194 for line in f: 195 if invalue: 196 if line.endswith('\\\n'): 197 merged[invalue].append(line) 198 else: 199 invalue = None 200 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): 201 invalue = line[18:].split('=', 1)[0].rstrip() 202 merged[invalue] = [] 203 arch_order.append(invalue) 204 205 with open(lockedsigs_extra, 'r') as f: 206 invalue = None 207 tocopy = {} 208 for line in f: 209 if invalue: 210 if line.endswith('\\\n'): 211 if not line in merged[invalue]: 212 target, task = line.strip().split(':')[:2] 213 if not copy_tasks or task in copy_tasks: 214 tocopy[invalue].append(line) 215 merged[invalue].append(line) 216 else: 217 invalue = None 218 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'): 219 invalue = line[18:].split('=', 1)[0].rstrip() 220 if not invalue in merged: 221 merged[invalue] = [] 222 arch_order.append(invalue) 223 tocopy[invalue] = [] 224 225 def write_sigs_file(fn, types, sigs): 226 fulltypes = [] 227 bb.utils.mkdirhier(os.path.dirname(fn)) 228 with open(fn, 'w') as f: 229 for typename in types: 230 lines = sigs[typename] 231 if lines: 232 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename) 233 for line in lines: 234 f.write(line) 235 f.write(' "\n') 236 fulltypes.append(typename) 237 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) 238 239 if copy_output: 240 write_sigs_file(copy_output, list(tocopy.keys()), tocopy) 241 if merged_output: 242 write_sigs_file(merged_output, arch_order, merged) 243 244def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None): 245 import shutil 246 bb.note('Generating sstate-cache...') 247 248 nativelsbstring = d.getVar('NATIVELSBSTRING') 249 bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or '')) 250 if fixedlsbstring and nativelsbstring != fixedlsbstring: 251 nativedir = output_sstate_cache + '/' + nativelsbstring 252 if os.path.isdir(nativedir): 253 destdir = os.path.join(output_sstate_cache, fixedlsbstring) 254 for root, _, files in os.walk(nativedir): 255 for fn in files: 256 src = os.path.join(root, fn) 257 dest = os.path.join(destdir, os.path.relpath(src, nativedir)) 258 if os.path.exists(dest): 259 # Already exists, and it'll be the same file, so just delete it 260 os.unlink(src) 261 else: 262 bb.utils.mkdirhier(os.path.dirname(dest)) 263 shutil.move(src, dest) 264 265def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None): 266 import subprocess 267 268 bb.note('Generating sstate task list...') 269 270 if not cwd: 271 cwd = os.getcwd() 272 if logfile: 273 logparam = '-l %s' % logfile 274 else: 275 logparam = '' 276 cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam) 277 env = dict(d.getVar('BB_ORIGENV', False)) 278 env.pop('BUILDDIR', '') 279 env.pop('BBPATH', '') 280 pathitems = env['PATH'].split(':') 281 env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')]) 282 bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash') 283