1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7# Extensible SDK 8 9inherit populate_sdk_base 10 11# Used to override TOOLCHAIN_HOST_TASK in the eSDK case 12TOOLCHAIN_HOST_TASK_ESDK = " \ 13 meta-environment-extsdk-${MACHINE} \ 14 " 15 16SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0" 17 18SDK_EXT = "" 19SDK_EXT:task-populate-sdk-ext = "-ext" 20 21# Options are full or minimal 22SDK_EXT_TYPE ?= "full" 23SDK_INCLUDE_PKGDATA ?= "0" 24SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}" 25SDK_INCLUDE_NATIVESDK ?= "0" 26SDK_INCLUDE_BUILDTOOLS ?= '1' 27 28SDK_RECRDEP_TASKS ?= "" 29SDK_CUSTOM_TEMPLATECONF ?= "0" 30 31ESDK_LOCALCONF_ALLOW ?= "" 32ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \ 33 BB_NUMBER_THREADS \ 34 BB_NUMBER_PARSE_THREADS \ 35 PARALLEL_MAKE \ 36 PRSERV_HOST \ 37 SSTATE_MIRRORS \ 38 DL_DIR \ 39 SSTATE_DIR \ 40 TMPDIR \ 41 BB_SERVER_TIMEOUT \ 42 " 43ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory icecc" 44SDK_UPDATE_URL ?= "" 45 46SDK_TARGETS ?= "${PN}" 47 48def get_sdk_install_targets(d, images_only=False): 49 sdk_install_targets = '' 50 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal': 51 sdk_install_targets = d.getVar('SDK_TARGETS') 52 53 depd = d.getVar('BB_TASKDEPDATA', False) 54 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 55 tasklist.remove('do_build') 56 for v in depd.values(): 57 if v[1] in tasklist: 58 if v[0] not in sdk_install_targets: 59 sdk_install_targets += ' {}'.format(v[0]) 60 61 if not images_only: 62 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 63 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata' 64 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1': 65 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot' 66 67 return sdk_install_targets 68 69get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA" 70 71OE_INIT_ENV_SCRIPT ?= "oe-init-build-env" 72 73# The files from COREBASE that you want preserved in the COREBASE copied 74# into the sdk. This allows someone to have their own setup scripts in 75# COREBASE be preserved as well as untracked files. 76COREBASE_FILES ?= " \ 77 oe-init-build-env \ 78 scripts \ 79 LICENSE \ 80 .templateconf \ 81" 82 83SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext" 84B:task-populate-sdk-ext = "${SDK_DIR}" 85TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}" 86TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}" 87 88SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest" 89SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest" 90 91python write_target_sdk_ext_manifest () { 92 from oe.sdk import get_extra_sdkinfo 93 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 94 extra_info = get_extra_sdkinfo(sstate_dir) 95 96 target = d.getVar('TARGET_SYS') 97 target_multimach = d.getVar('MULTIMACH_TARGET_SYS') 98 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS') 99 100 pkgs = {} 101 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True) 102 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f: 103 for fn in extra_info['filesizes']: 104 info = fn.split(':') 105 if info[2] in (target, target_multimach, real_target_multimach) \ 106 or info[5] == 'allarch': 107 if not info[1] in pkgs: 108 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 109 pkgs[info[1]] = {} 110} 111python write_host_sdk_ext_manifest () { 112 from oe.sdk import get_extra_sdkinfo 113 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 114 extra_info = get_extra_sdkinfo(sstate_dir) 115 host = d.getVar('BUILD_SYS') 116 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f: 117 for fn in extra_info['filesizes']: 118 info = fn.split(':') 119 if info[2] == host: 120 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 121} 122 123SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; " 124 125SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK" 126 127def clean_esdk_builddir(d, sdkbasepath): 128 """Clean up traces of the fake build for create_filtered_tasklist()""" 129 import shutil 130 cleanpaths = ['cache', 'tmp'] 131 for pth in cleanpaths: 132 fullpth = os.path.join(sdkbasepath, pth) 133 if os.path.isdir(fullpth): 134 shutil.rmtree(fullpth) 135 elif os.path.isfile(fullpth): 136 os.remove(fullpth) 137 138def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath): 139 """ 140 Create a filtered list of tasks. Also double-checks that the build system 141 within the SDK basically works and required sstate artifacts are available. 142 """ 143 import tempfile 144 import shutil 145 import oe.copy_buildsystem 146 147 # Create a temporary build directory that we can pass to the env setup script 148 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak') 149 try: 150 with open(sdkbasepath + '/conf/local.conf', 'a') as f: 151 # Force the use of sstate from the build system 152 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR')) 153 f.write('SSTATE_MIRRORS:forcevariable = "file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n') 154 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it 155 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n') 156 f.write('TCLIBCAPPEND:forcevariable = ""\n') 157 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will 158 # be different and we won't be able to find our native sstate) 159 if not bb.data.inherits_class('uninative', d): 160 f.write('INHERIT:remove = "uninative"\n') 161 162 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake 163 # will not allow in its COREBASE path, so we need to rename the directory temporarily 164 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk' 165 # Delete any existing temp dir 166 try: 167 shutil.rmtree(temp_sdkbasepath) 168 except FileNotFoundError: 169 pass 170 bb.utils.rename(sdkbasepath, temp_sdkbasepath) 171 cmdprefix = '. %s .; ' % conf_initpath 172 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt' 173 try: 174 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile) 175 except bb.process.ExecutionError as e: 176 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip() 177 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout: 178 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n' 179 bb.fatal(msg) 180 bb.utils.rename(temp_sdkbasepath, sdkbasepath) 181 # Clean out residue of running bitbake, which check_sstate_task_list() 182 # will effectively do 183 clean_esdk_builddir(d, sdkbasepath) 184 finally: 185 localconf = sdkbasepath + '/conf/local.conf' 186 if os.path.exists(localconf + '.bak'): 187 os.replace(localconf + '.bak', localconf) 188 189python copy_buildsystem () { 190 import re 191 import shutil 192 import glob 193 import oe.copy_buildsystem 194 195 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT') 196 197 conf_bbpath = '' 198 conf_initpath = '' 199 core_meta_subdir = '' 200 201 # Copy in all metadata layers + bitbake (as repositories) 202 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) 203 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH') 204 205 #check if custome templateconf path is set 206 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF') 207 208 # Determine if we're building a derivative extensible SDK (from devtool build-sdk) 209 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1' 210 if derivative: 211 workspace_name = 'orig-workspace' 212 else: 213 workspace_name = None 214 215 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name) 216 conf_bbpath = os.path.join('layers', corebase, 'bitbake') 217 218 for path in os.listdir(baseoutpath + '/layers'): 219 relpath = os.path.join('layers', path, oe_init_env_script) 220 if os.path.exists(os.path.join(baseoutpath, relpath)): 221 conf_initpath = relpath 222 223 relpath = os.path.join('layers', path, 'scripts', 'devtool') 224 if os.path.exists(os.path.join(baseoutpath, relpath)): 225 scriptrelpath = os.path.dirname(relpath) 226 227 relpath = os.path.join('layers', path, 'meta') 228 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')): 229 core_meta_subdir = relpath 230 231 d.setVar('oe_init_build_env_path', conf_initpath) 232 d.setVar('scriptrelpath', scriptrelpath) 233 234 # Write out config file for devtool 235 import configparser 236 config = configparser.ConfigParser() 237 config.add_section('General') 238 config.set('General', 'bitbake_subdir', conf_bbpath) 239 config.set('General', 'init_path', conf_initpath) 240 config.set('General', 'core_meta_subdir', core_meta_subdir) 241 config.add_section('SDK') 242 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS')) 243 updateurl = d.getVar('SDK_UPDATE_URL') 244 if updateurl: 245 config.set('SDK', 'updateserver', updateurl) 246 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf')) 247 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f: 248 config.write(f) 249 250 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc') 251 with open(unlockedsigs, 'w') as f: 252 pass 253 254 # Create a layer for new recipes / appends 255 bbpath = d.getVar('BBPATH') 256 env = os.environ.copy() 257 env['PYTHONDONTWRITEBYTECODE'] = '1' 258 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--layerseries', d.getVar("LAYERSERIES_CORENAMES"), '--create-only', os.path.join(baseoutpath, 'workspace')], env=env) 259 260 # Create bblayers.conf 261 bb.utils.mkdirhier(baseoutpath + '/conf') 262 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f: 263 f.write('# WARNING: this configuration has been automatically generated and in\n') 264 f.write('# most cases should not be edited. If you need more flexibility than\n') 265 f.write('# this configuration provides, it is strongly suggested that you set\n') 266 f.write('# up a proper instance of the full build system and use that instead.\n\n') 267 268 # LCONF_VERSION may not be set, for example when using meta-poky 269 # so don't error if it isn't found 270 lconf_version = d.getVar('LCONF_VERSION', False) 271 if lconf_version is not None: 272 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version) 273 274 f.write('BBPATH = "$' + '{TOPDIR}"\n') 275 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n') 276 f.write('BBLAYERS := " \\\n') 277 for layerrelpath in sdkbblayers: 278 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath) 279 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n') 280 f.write(' "\n') 281 282 # Copy uninative tarball 283 # For now this is where uninative.bbclass expects the tarball 284 if bb.data.inherits_class('uninative', d): 285 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}') 286 uninative_checksum = bb.utils.sha256_file(uninative_file) 287 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum) 288 bb.utils.mkdirhier(uninative_outdir) 289 shutil.copy(uninative_file, uninative_outdir) 290 291 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split() 292 env_passthrough_values = {} 293 294 # Create local.conf 295 builddir = d.getVar('TOPDIR') 296 if derivative and os.path.exists(builddir + '/conf/site.conf'): 297 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf') 298 if derivative and os.path.exists(builddir + '/conf/auto.conf'): 299 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf') 300 if derivative: 301 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf') 302 else: 303 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split() 304 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split() 305 def handle_var(varname, origvalue, op, newlines): 306 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed): 307 newlines.append('# Removed original setting of %s\n' % varname) 308 return None, op, 0, True 309 else: 310 if varname in env_passthrough: 311 env_passthrough_values[varname] = origvalue 312 return origvalue, op, 0, True 313 varlist = ['[^#=+ ]*'] 314 oldlines = [] 315 if os.path.exists(builddir + '/conf/site.conf'): 316 with open(builddir + '/conf/site.conf', 'r') as f: 317 oldlines += f.readlines() 318 if os.path.exists(builddir + '/conf/auto.conf'): 319 with open(builddir + '/conf/auto.conf', 'r') as f: 320 oldlines += f.readlines() 321 if os.path.exists(builddir + '/conf/local.conf'): 322 with open(builddir + '/conf/local.conf', 'r') as f: 323 oldlines += f.readlines() 324 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) 325 326 with open(baseoutpath + '/conf/local.conf', 'w') as f: 327 f.write('# WARNING: this configuration has been automatically generated and in\n') 328 f.write('# most cases should not be edited. If you need more flexibility than\n') 329 f.write('# this configuration provides, it is strongly suggested that you set\n') 330 f.write('# up a proper instance of the full build system and use that instead.\n\n') 331 for line in newlines: 332 if line.strip() and not line.startswith('#'): 333 f.write(line) 334 # Write a newline just in case there's none at the end of the original 335 f.write('\n') 336 337 f.write('TMPDIR = "${TOPDIR}/tmp"\n') 338 f.write('TCLIBCAPPEND = ""\n') 339 f.write('DL_DIR = "${TOPDIR}/downloads"\n') 340 341 if bb.data.inherits_class('uninative', d): 342 f.write('INHERIT += "%s"\n' % 'uninative') 343 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum)) 344 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) 345 346 # Some classes are not suitable for SDK, remove them from INHERIT 347 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False)) 348 349 # Bypass the default connectivity check if any 350 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') 351 352 # This warning will come out if reverse dependencies for a task 353 # don't have sstate as well as the task itself. We already know 354 # this will be the case for the extensible sdk, so turn off the 355 # warning. 356 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n') 357 358 # Warn if the sigs in the locked-signature file don't match 359 # the sig computed from the metadata. 360 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n') 361 362 # We want to be able to set this without a full reparse 363 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n') 364 365 # Set up which tasks are ignored for run on install 366 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n') 367 368 # Hide the config information from bitbake output (since it's fixed within the SDK) 369 f.write('BUILDCFG_HEADER = ""\n\n') 370 371 # Write METADATA_REVISION 372 f.write('METADATA_REVISION = "%s"\n\n' % d.getVar('METADATA_REVISION')) 373 374 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n') 375 f.write('WITHIN_EXT_SDK = "1"\n\n') 376 377 # Map gcc-dependent uninative sstate cache for installer usage 378 f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n') 379 380 if d.getVar("PRSERV_HOST"): 381 # Override this, we now include PR data, so it should only point ot the local database 382 f.write('PRSERV_HOST = "localhost:0"\n\n') 383 384 # Allow additional config through sdk-extra.conf 385 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) 386 if fn: 387 with open(fn, 'r') as xf: 388 for line in xf: 389 f.write(line) 390 391 # If you define a sdk_extraconf() function then it can contain additional config 392 # (Though this is awkward; sdk-extra.conf should probably be used instead) 393 extraconf = (d.getVar('sdk_extraconf') or '').strip() 394 if extraconf: 395 # Strip off any leading / trailing spaces 396 for line in extraconf.splitlines(): 397 f.write(line.strip() + '\n') 398 399 f.write('require conf/locked-sigs.inc\n') 400 f.write('require conf/unlocked-sigs.inc\n') 401 402 # Copy multiple configurations if they exist in the users config directory 403 if d.getVar('BBMULTICONFIG') is not None: 404 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig')) 405 for mc in d.getVar('BBMULTICONFIG').split(): 406 dest_stub = "/conf/multiconfig/%s.conf" % (mc,) 407 if os.path.exists(builddir + dest_stub): 408 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub) 409 410 cachedir = os.path.join(baseoutpath, 'cache') 411 bb.utils.mkdirhier(cachedir) 412 bb.parse.siggen.copy_unitaskhashes(cachedir) 413 414 # If PR Service is in use, we need to export this as well 415 bb.note('Do we have a pr database?') 416 if d.getVar("PRSERV_HOST"): 417 bb.note('Writing PR database...') 418 # Based on the code in classes/prexport.bbclass 419 import oe.prservice 420 #dump meta info of tables 421 localdata = d.createCopy() 422 localdata.setVar('PRSERV_DUMPOPT_COL', "1") 423 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf')) 424 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc') 425 426 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE'))) 427 428 retval = oe.prservice.prserv_dump_db(localdata) 429 if not retval: 430 bb.error("prexport_handler: export failed!") 431 return 432 (metainfo, datainfo) = retval 433 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True) 434 435 # Use templateconf.cfg file from builddir if exists 436 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1': 437 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg') 438 else: 439 # Write a templateconf.cfg 440 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f: 441 f.write('meta/conf/templates/default\n') 442 os.makedirs(os.path.join(baseoutpath, core_meta_subdir, 'conf/templates/default'), exist_ok=True) 443 444 # Ensure any variables set from the external environment (by way of 445 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration 446 extralines = [] 447 for name, value in env_passthrough_values.items(): 448 actualvalue = d.getVar(name) or '' 449 if value != actualvalue: 450 extralines.append('%s = "%s"\n' % (name, actualvalue)) 451 if extralines: 452 with open(baseoutpath + '/conf/local.conf', 'a') as f: 453 f.write('\n') 454 f.write('# Extra settings from environment:\n') 455 for line in extralines: 456 f.write(line) 457 f.write('\n') 458 459 # Filter the locked signatures file to just the sstate tasks we are interested in 460 excluded_targets = get_sdk_install_targets(d, images_only=True) 461 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 462 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc' 463 #nativesdk-only sigfile to merge into locked-sigs.inc 464 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1') 465 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 466 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc' 467 468 if sdk_include_nativesdk: 469 oe.copy_buildsystem.prune_lockedsigs([], 470 excluded_targets.split(), 471 nativesigfile, 472 True, 473 nativesigfile_pruned) 474 475 oe.copy_buildsystem.merge_lockedsigs([], 476 sigfile, 477 nativesigfile_pruned, 478 sigfile) 479 480 oe.copy_buildsystem.prune_lockedsigs([], 481 excluded_targets.split(), 482 sigfile, 483 False, 484 lockedsigs_pruned) 485 486 sstate_out = baseoutpath + '/sstate-cache' 487 bb.utils.remove(sstate_out, True) 488 489 # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d) 490 fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d) 491 492 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1') 493 sdk_ext_type = d.getVar('SDK_EXT_TYPE') 494 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk: 495 # Create the filtered task list used to generate the sstate cache shipped with the SDK 496 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt' 497 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath) 498 else: 499 tasklistfn = None 500 501 cachedir = os.path.join(baseoutpath, 'cache') 502 bb.utils.mkdirhier(cachedir) 503 bb.parse.siggen.copy_unitaskhashes(cachedir) 504 505 # Add packagedata if enabled 506 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 507 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc' 508 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc' 509 shutil.move(lockedsigs_pruned, lockedsigs_base) 510 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'], 511 lockedsigs_base, 512 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc', 513 lockedsigs_pruned, 514 lockedsigs_copy) 515 516 if sdk_include_toolchain: 517 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc' 518 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc") 519 shutil.move(lockedsigs_pruned, lockedsigs_base) 520 oe.copy_buildsystem.merge_lockedsigs([], 521 lockedsigs_base, 522 lockedsigs_toolchain, 523 lockedsigs_pruned) 524 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain, 525 d.getVar('SSTATE_DIR'), 526 sstate_out, d, 527 fixedlsbstring, 528 filterfile=tasklistfn) 529 530 if sdk_ext_type == 'minimal': 531 if derivative: 532 # Assume the user is not going to set up an additional sstate 533 # mirror, thus we need to copy the additional artifacts (from 534 # workspace recipes) into the derivative SDK 535 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc' 536 if os.path.exists(lockedsigs_orig): 537 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc' 538 oe.copy_buildsystem.merge_lockedsigs(None, 539 lockedsigs_orig, 540 lockedsigs_pruned, 541 None, 542 lockedsigs_extra) 543 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, 544 d.getVar('SSTATE_DIR'), 545 sstate_out, d, 546 fixedlsbstring, 547 filterfile=tasklistfn) 548 else: 549 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, 550 d.getVar('SSTATE_DIR'), 551 sstate_out, d, 552 fixedlsbstring, 553 filterfile=tasklistfn) 554 555 # We don't need sstate do_package files 556 for root, dirs, files in os.walk(sstate_out): 557 for name in files: 558 if name.endswith("_package.tar.zst"): 559 f = os.path.join(root, name) 560 os.remove(f) 561 562 # Write manifest file 563 # Note: at the moment we cannot include the env setup script here to keep 564 # it updated, since it gets modified during SDK installation (see 565 # sdk_ext_postinst() below) thus the checksum we take here would always 566 # be different. 567 manifest_file_list = ['conf/*'] 568 if d.getVar('BBMULTICONFIG') is not None: 569 manifest_file_list.append('conf/multiconfig/*') 570 571 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split() 572 esdk_manifest_excludes_list = [] 573 for exclude_item in esdk_manifest_excludes: 574 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item)) 575 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest') 576 with open(manifest_file, 'w') as f: 577 for item in manifest_file_list: 578 for fn in glob.glob(os.path.join(baseoutpath, item)): 579 if fn == manifest_file or os.path.isdir(fn): 580 continue 581 if fn in esdk_manifest_excludes_list: 582 continue 583 chksum = bb.utils.sha256_file(fn) 584 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath))) 585} 586 587def get_current_buildtools(d): 588 """Get the file name of the current buildtools installer""" 589 import glob 590 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh')) 591 btfiles.sort(key=os.path.getctime) 592 return os.path.basename(btfiles[-1]) 593 594def get_sdk_required_utilities(buildtools_fn, d): 595 """Find required utilities that aren't provided by the buildtools""" 596 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split() 597 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc')) 598 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++')) 599 if buildtools_fn: 600 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn) 601 filelist, _ = bb.process.run('%s -l' % buildtools_installer) 602 else: 603 buildtools_installer = None 604 filelist = "" 605 localdata = bb.data.createCopy(d) 606 localdata.setVar('SDKPATH', '.') 607 sdkpathnative = localdata.getVar('SDKPATHNATIVE') 608 sdkbindirs = [localdata.getVar('bindir_nativesdk'), 609 localdata.getVar('sbindir_nativesdk'), 610 localdata.getVar('base_bindir_nativesdk'), 611 localdata.getVar('base_sbindir_nativesdk')] 612 for line in filelist.splitlines(): 613 splitline = line.split() 614 if len(splitline) > 5: 615 fn = splitline[5] 616 if not fn.startswith('./'): 617 fn = './%s' % fn 618 if fn.startswith(sdkpathnative): 619 relpth = '/' + os.path.relpath(fn, sdkpathnative) 620 for bindir in sdkbindirs: 621 if relpth.startswith(bindir): 622 relpth = os.path.relpath(relpth, bindir) 623 if relpth in sanity_required_utilities: 624 sanity_required_utilities.remove(relpth) 625 break 626 return ' '.join(sanity_required_utilities) 627 628install_tools() { 629 install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk} 630 scripts="devtool recipetool oe-find-native-sysroot runqemu* wic" 631 for script in $scripts; do 632 for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do 633 targetscriptfn="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/$(basename $scriptfn)" 634 test -e ${targetscriptfn} || ln -rs ${scriptfn} ${targetscriptfn} 635 done 636 done 637 # We can't use the same method as above because files in the sysroot won't exist at this point 638 # (they get populated from sstate on installation) 639 unfsd_path="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/unfsd" 640 if [ "${SDK_INCLUDE_TOOLCHAIN}" = "1" -a ! -e $unfsd_path ] ; then 641 binrelpath=${@os.path.relpath(d.getVar('STAGING_BINDIR_NATIVE'), d.getVar('TMPDIR'))} 642 ln -rs ${SDK_OUTPUT}/${SDKPATH}/tmp/$binrelpath/unfsd $unfsd_path 643 fi 644 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase 645 646 # find latest buildtools-tarball and install it 647 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 648 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH} 649 fi 650 651 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH} 652} 653do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True" 654 655sdk_ext_preinst() { 656 # Since bitbake won't run as root it doesn't make sense to try and install 657 # the extensible sdk as root. 658 if [ "`id -u`" = "0" ]; then 659 echo "ERROR: The extensible sdk cannot be installed as root." 660 exit 1 661 fi 662 if ! command -v locale > /dev/null; then 663 echo "ERROR: The installer requires the locale command, please install it first" 664 exit 1 665 fi 666 # Check setting of LC_ALL set above 667 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'` 668 if ! locale -a | grep -q $canonicalised_locale ; then 669 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first" 670 exit 1 671 fi 672 # The relocation script used by buildtools installer requires python 673 if ! command -v python3 > /dev/null; then 674 echo "ERROR: The installer requires python3, please install it first" 675 exit 1 676 fi 677 missing_utils="" 678 for util in ${SDK_REQUIRED_UTILITIES}; do 679 if ! command -v $util > /dev/null; then 680 missing_utils="$missing_utils $util" 681 fi 682 done 683 if [ -n "$missing_utils" ] ; then 684 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils" 685 exit 1 686 fi 687 SDK_EXTENSIBLE="1" 688 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then 689 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache" 690 fi 691} 692SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}" 693 694# FIXME this preparation should be done as part of the SDK construction 695sdk_ext_postinst() { 696 printf "\nExtracting buildtools...\n" 697 cd $target_sdk_dir 698 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}" 699 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 700 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 701 702 # Delete the buildtools tar file since it won't be used again 703 rm -f ./${SDK_BUILDTOOLS_INSTALLER} 704 # We don't need the log either since it succeeded 705 rm -f buildtools.log 706 707 # Make sure when the user sets up the environment, they also get 708 # the buildtools-tarball tools in their path. 709 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script 710 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script 711 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script 712 echo "OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script 713 fi 714 715 # Allow bitbake environment setup to be ran as part of this sdk. 716 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script 717 # Work around runqemu not knowing how to get this information within the eSDK 718 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script 719 720 # A bit of another hack, but we need this in the path only for devtool 721 # so put it at the end of $PATH. 722 echo "export PATH=$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH" >> $env_setup_script 723 724 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script 725 726 # Warn if trying to use external bitbake and the ext SDK together 727 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script 728 729 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 730 printf "Preparing build system...\n" 731 # dash which is /bin/sh on Ubuntu will not preserve the 732 # current working directory when first ran, nor will it set $1 when 733 # sourcing a script. That is why this has to look so ugly. 734 LOGFILE="$target_sdk_dir/preparing_build_system.log" 735 sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 736 fi 737 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then 738 rm $target_sdk_dir/ext-sdk-prepare.py 739 fi 740 echo done 741} 742 743SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}" 744 745SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem; install_tools; " 746 747SDK_INSTALL_TARGETS = "" 748fakeroot python do_populate_sdk_ext() { 749 # FIXME hopefully we can remove this restriction at some point, but uninative 750 # currently forces this upon us 751 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'): 752 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH'))) 753 754 # FIXME hopefully we can remove this restriction at some point, but the eSDK 755 # can only be built for the primary (default) multiconfig 756 if d.getVar('BB_CURRENT_MC') != 'default': 757 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC')) 758 759 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set 760 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}") 761 d.setVar("TOOLCHAIN_TARGET_TASK", "") 762 763 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d)) 764 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1': 765 buildtools_fn = get_current_buildtools(d) 766 else: 767 buildtools_fn = None 768 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d)) 769 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn) 770 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}') 771 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice 772 d.delVar('SDKIMAGE_LINGUAS') 773 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1': 774 generate_nativesdk_lockedsigs(d) 775 populate_sdk_common(d) 776} 777 778def generate_nativesdk_lockedsigs(d): 779 import oe.copy_buildsystem 780 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 781 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 782 783def get_ext_sdk_depends(d): 784 # Note: the deps varflag is a list not a string, so we need to specify expand=False 785 deps = d.getVarFlag('do_image_complete', 'deps', False) 786 pn = d.getVar('PN') 787 deplist = ['%s:%s' % (pn, dep) for dep in deps] 788 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 789 tasklist.append('do_rootfs') 790 for task in tasklist: 791 deplist.extend((d.getVarFlag(task, 'depends') or '').split()) 792 return ' '.join(deplist) 793 794python do_sdk_depends() { 795 # We have to do this separately in its own task so we avoid recursing into 796 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those 797 # into the SDK's sstate-cache 798 import oe.copy_buildsystem 799 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 800 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 801} 802addtask sdk_depends 803 804do_sdk_depends[dirs] = "${WORKDIR}" 805do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot" 806do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}" 807do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}" 808do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}" 809 810do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}" 811 812do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \ 813 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \ 814 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \ 815 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}" 816 817# We must avoid depending on do_build here if rm_work.bbclass is active, 818# because otherwise do_rm_work may run before do_populate_sdk_ext itself. 819# We can't mark do_populate_sdk_ext and do_sdk_depends as having to 820# run before do_rm_work, because then they would also run as part 821# of normal builds. 822do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}" 823 824# Make sure code changes can result in rebuild 825do_populate_sdk_ext[vardeps] += "copy_buildsystem \ 826 sdk_ext_postinst" 827 828# Since any change in the metadata of any layer should cause a rebuild of the 829# sdk(since the layers are put in the sdk) set the task to nostamp so it 830# always runs. 831do_populate_sdk_ext[nostamp] = "1" 832 833SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext" 834 835SSTATETASKS += "do_populate_sdk_ext" 836SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1' 837do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}" 838do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}" 839do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}" 840do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}" 841 842addtask populate_sdk_ext after do_sdk_depends 843