1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7# Extensible SDK 8 9inherit populate_sdk_base 10 11# Used to override TOOLCHAIN_HOST_TASK in the eSDK case 12TOOLCHAIN_HOST_TASK_ESDK = " \ 13 meta-environment-extsdk-${MACHINE} \ 14 " 15 16SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0" 17 18SDK_EXT = "" 19SDK_EXT:task-populate-sdk-ext = "-ext" 20 21# Options are full or minimal 22SDK_EXT_TYPE ?= "full" 23SDK_INCLUDE_PKGDATA ?= "0" 24SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}" 25SDK_INCLUDE_NATIVESDK ?= "0" 26SDK_INCLUDE_BUILDTOOLS ?= '1' 27 28SDK_RECRDEP_TASKS ?= "" 29SDK_CUSTOM_TEMPLATECONF ?= "0" 30 31ESDK_LOCALCONF_ALLOW ?= "" 32ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \ 33 BB_NUMBER_THREADS \ 34 BB_NUMBER_PARSE_THREADS \ 35 PARALLEL_MAKE \ 36 PRSERV_HOST \ 37 SSTATE_MIRRORS \ 38 DL_DIR \ 39 SSTATE_DIR \ 40 TMPDIR \ 41 BB_SERVER_TIMEOUT \ 42 " 43ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory icecc" 44SDK_UPDATE_URL ?= "" 45 46SDK_TARGETS ?= "${PN}" 47 48def get_sdk_install_targets(d, images_only=False): 49 sdk_install_targets = '' 50 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal': 51 sdk_install_targets = d.getVar('SDK_TARGETS') 52 53 depd = d.getVar('BB_TASKDEPDATA', False) 54 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 55 tasklist.remove('do_build') 56 for v in depd.values(): 57 if v[1] in tasklist: 58 if v[0] not in sdk_install_targets: 59 sdk_install_targets += ' {}'.format(v[0]) 60 61 if not images_only: 62 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 63 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata' 64 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1': 65 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot' 66 67 return sdk_install_targets 68 69get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA" 70 71OE_INIT_ENV_SCRIPT ?= "oe-init-build-env" 72 73# The files from COREBASE that you want preserved in the COREBASE copied 74# into the sdk. This allows someone to have their own setup scripts in 75# COREBASE be preserved as well as untracked files. 76COREBASE_FILES ?= " \ 77 oe-init-build-env \ 78 scripts \ 79 LICENSE \ 80 .templateconf \ 81" 82 83SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext" 84B:task-populate-sdk-ext = "${SDK_DIR}" 85TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}" 86TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}" 87 88SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest" 89SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest" 90 91python write_target_sdk_ext_manifest () { 92 from oe.sdk import get_extra_sdkinfo 93 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 94 extra_info = get_extra_sdkinfo(sstate_dir) 95 96 target = d.getVar('TARGET_SYS') 97 target_multimach = d.getVar('MULTIMACH_TARGET_SYS') 98 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS') 99 100 pkgs = {} 101 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True) 102 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f: 103 for fn in extra_info['filesizes']: 104 info = fn.split(':') 105 if info[2] in (target, target_multimach, real_target_multimach) \ 106 or info[5] == 'allarch': 107 if not info[1] in pkgs: 108 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 109 pkgs[info[1]] = {} 110} 111python write_host_sdk_ext_manifest () { 112 from oe.sdk import get_extra_sdkinfo 113 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 114 extra_info = get_extra_sdkinfo(sstate_dir) 115 host = d.getVar('BUILD_SYS') 116 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f: 117 for fn in extra_info['filesizes']: 118 info = fn.split(':') 119 if info[2] == host: 120 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 121} 122 123SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; " 124 125SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK" 126 127def clean_esdk_builddir(d, sdkbasepath): 128 """Clean up traces of the fake build for create_filtered_tasklist()""" 129 import shutil 130 cleanpaths = ['cache', 'tmp'] 131 for pth in cleanpaths: 132 fullpth = os.path.join(sdkbasepath, pth) 133 if os.path.isdir(fullpth): 134 shutil.rmtree(fullpth) 135 elif os.path.isfile(fullpth): 136 os.remove(fullpth) 137 138def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath): 139 """ 140 Create a filtered list of tasks. Also double-checks that the build system 141 within the SDK basically works and required sstate artifacts are available. 142 """ 143 import tempfile 144 import shutil 145 import oe.copy_buildsystem 146 147 # Create a temporary build directory that we can pass to the env setup script 148 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak') 149 try: 150 with open(sdkbasepath + '/conf/local.conf', 'a') as f: 151 # Force the use of sstate from the build system 152 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR')) 153 f.write('SSTATE_MIRRORS:forcevariable = "file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n') 154 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it 155 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n') 156 f.write('TCLIBCAPPEND:forcevariable = ""\n') 157 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will 158 # be different and we won't be able to find our native sstate) 159 if not bb.data.inherits_class('uninative', d): 160 f.write('INHERIT:remove = "uninative"\n') 161 162 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake 163 # will not allow in its COREBASE path, so we need to rename the directory temporarily 164 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk' 165 # Delete any existing temp dir 166 try: 167 shutil.rmtree(temp_sdkbasepath) 168 except FileNotFoundError: 169 pass 170 bb.utils.rename(sdkbasepath, temp_sdkbasepath) 171 cmdprefix = '. %s .; ' % conf_initpath 172 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt' 173 try: 174 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile) 175 except bb.process.ExecutionError as e: 176 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip() 177 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout: 178 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n' 179 bb.fatal(msg) 180 bb.utils.rename(temp_sdkbasepath, sdkbasepath) 181 # Clean out residue of running bitbake, which check_sstate_task_list() 182 # will effectively do 183 clean_esdk_builddir(d, sdkbasepath) 184 finally: 185 localconf = sdkbasepath + '/conf/local.conf' 186 if os.path.exists(localconf + '.bak'): 187 os.replace(localconf + '.bak', localconf) 188 189python copy_buildsystem () { 190 import re 191 import shutil 192 import glob 193 import oe.copy_buildsystem 194 195 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT') 196 197 conf_bbpath = '' 198 conf_initpath = '' 199 core_meta_subdir = '' 200 201 # Copy in all metadata layers + bitbake (as repositories) 202 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) 203 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH') 204 205 #check if custome templateconf path is set 206 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF') 207 208 # Determine if we're building a derivative extensible SDK (from devtool build-sdk) 209 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1' 210 if derivative: 211 workspace_name = 'orig-workspace' 212 else: 213 workspace_name = None 214 215 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name) 216 conf_bbpath = os.path.join('layers', corebase, 'bitbake') 217 218 for path in os.listdir(baseoutpath + '/layers'): 219 relpath = os.path.join('layers', path, oe_init_env_script) 220 if os.path.exists(os.path.join(baseoutpath, relpath)): 221 conf_initpath = relpath 222 223 relpath = os.path.join('layers', path, 'scripts', 'devtool') 224 if os.path.exists(os.path.join(baseoutpath, relpath)): 225 scriptrelpath = os.path.dirname(relpath) 226 227 relpath = os.path.join('layers', path, 'meta') 228 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')): 229 core_meta_subdir = relpath 230 231 d.setVar('oe_init_build_env_path', conf_initpath) 232 d.setVar('scriptrelpath', scriptrelpath) 233 234 # Write out config file for devtool 235 import configparser 236 config = configparser.ConfigParser() 237 config.add_section('General') 238 config.set('General', 'bitbake_subdir', conf_bbpath) 239 config.set('General', 'init_path', conf_initpath) 240 config.set('General', 'core_meta_subdir', core_meta_subdir) 241 config.add_section('SDK') 242 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS')) 243 updateurl = d.getVar('SDK_UPDATE_URL') 244 if updateurl: 245 config.set('SDK', 'updateserver', updateurl) 246 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf')) 247 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f: 248 config.write(f) 249 250 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc') 251 with open(unlockedsigs, 'w') as f: 252 pass 253 254 # Create a layer for new recipes / appends 255 bbpath = d.getVar('BBPATH') 256 env = os.environ.copy() 257 env['PYTHONDONTWRITEBYTECODE'] = '1' 258 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--layerseries', d.getVar("LAYERSERIES_CORENAMES"), '--create-only', os.path.join(baseoutpath, 'workspace')], env=env) 259 260 # Create bblayers.conf 261 bb.utils.mkdirhier(baseoutpath + '/conf') 262 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f: 263 f.write('# WARNING: this configuration has been automatically generated and in\n') 264 f.write('# most cases should not be edited. If you need more flexibility than\n') 265 f.write('# this configuration provides, it is strongly suggested that you set\n') 266 f.write('# up a proper instance of the full build system and use that instead.\n\n') 267 268 # LCONF_VERSION may not be set, for example when using meta-poky 269 # so don't error if it isn't found 270 lconf_version = d.getVar('LCONF_VERSION', False) 271 if lconf_version is not None: 272 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version) 273 274 f.write('BBPATH = "$' + '{TOPDIR}"\n') 275 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n') 276 f.write('BBLAYERS := " \\\n') 277 for layerrelpath in sdkbblayers: 278 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath) 279 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n') 280 f.write(' "\n') 281 282 # Copy uninative tarball 283 # For now this is where uninative.bbclass expects the tarball 284 if bb.data.inherits_class('uninative', d): 285 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}') 286 uninative_checksum = bb.utils.sha256_file(uninative_file) 287 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum) 288 bb.utils.mkdirhier(uninative_outdir) 289 shutil.copy(uninative_file, uninative_outdir) 290 291 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split() 292 env_passthrough_values = {} 293 294 # Create local.conf 295 builddir = d.getVar('TOPDIR') 296 if derivative and os.path.exists(builddir + '/conf/site.conf'): 297 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf') 298 if derivative and os.path.exists(builddir + '/conf/auto.conf'): 299 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf') 300 if derivative: 301 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf') 302 else: 303 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split() 304 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split() 305 def handle_var(varname, origvalue, op, newlines): 306 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed): 307 newlines.append('# Removed original setting of %s\n' % varname) 308 return None, op, 0, True 309 else: 310 if varname in env_passthrough: 311 env_passthrough_values[varname] = origvalue 312 return origvalue, op, 0, True 313 varlist = ['[^#=+ ]*'] 314 oldlines = [] 315 if os.path.exists(builddir + '/conf/site.conf'): 316 with open(builddir + '/conf/site.conf', 'r') as f: 317 oldlines += f.readlines() 318 if os.path.exists(builddir + '/conf/auto.conf'): 319 with open(builddir + '/conf/auto.conf', 'r') as f: 320 oldlines += f.readlines() 321 if os.path.exists(builddir + '/conf/local.conf'): 322 with open(builddir + '/conf/local.conf', 'r') as f: 323 oldlines += f.readlines() 324 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) 325 326 with open(baseoutpath + '/conf/local.conf', 'w') as f: 327 f.write('# WARNING: this configuration has been automatically generated and in\n') 328 f.write('# most cases should not be edited. If you need more flexibility than\n') 329 f.write('# this configuration provides, it is strongly suggested that you set\n') 330 f.write('# up a proper instance of the full build system and use that instead.\n\n') 331 for line in newlines: 332 if line.strip() and not line.startswith('#'): 333 f.write(line) 334 # Write a newline just in case there's none at the end of the original 335 f.write('\n') 336 337 f.write('TMPDIR = "${TOPDIR}/tmp"\n') 338 f.write('TCLIBCAPPEND = ""\n') 339 f.write('DL_DIR = "${TOPDIR}/downloads"\n') 340 341 if bb.data.inherits_class('uninative', d): 342 f.write('INHERIT += "%s"\n' % 'uninative') 343 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum)) 344 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) 345 346 # Some classes are not suitable for SDK, remove them from INHERIT 347 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False)) 348 349 # Bypass the default connectivity check if any 350 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') 351 352 # This warning will come out if reverse dependencies for a task 353 # don't have sstate as well as the task itself. We already know 354 # this will be the case for the extensible sdk, so turn off the 355 # warning. 356 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n') 357 358 # Warn if the sigs in the locked-signature file don't match 359 # the sig computed from the metadata. 360 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n') 361 362 # We want to be able to set this without a full reparse 363 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n') 364 365 # Set up which tasks are ignored for run on install 366 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n') 367 368 # Hide the config information from bitbake output (since it's fixed within the SDK) 369 f.write('BUILDCFG_HEADER = ""\n\n') 370 371 # Write METADATA_REVISION 372 # Needs distro override so it can override the value set in the bbclass code (later than local.conf) 373 f.write('METADATA_REVISION:%s = "%s"\n\n' % (d.getVar('DISTRO'), d.getVar('METADATA_REVISION'))) 374 375 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n') 376 f.write('WITHIN_EXT_SDK = "1"\n\n') 377 378 # Map gcc-dependent uninative sstate cache for installer usage 379 f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n') 380 381 if d.getVar("PRSERV_HOST"): 382 # Override this, we now include PR data, so it should only point ot the local database 383 f.write('PRSERV_HOST = "localhost:0"\n\n') 384 385 # Allow additional config through sdk-extra.conf 386 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) 387 if fn: 388 with open(fn, 'r') as xf: 389 for line in xf: 390 f.write(line) 391 392 # If you define a sdk_extraconf() function then it can contain additional config 393 # (Though this is awkward; sdk-extra.conf should probably be used instead) 394 extraconf = (d.getVar('sdk_extraconf') or '').strip() 395 if extraconf: 396 # Strip off any leading / trailing spaces 397 for line in extraconf.splitlines(): 398 f.write(line.strip() + '\n') 399 400 f.write('require conf/locked-sigs.inc\n') 401 f.write('require conf/unlocked-sigs.inc\n') 402 403 # Copy multiple configurations if they exist in the users config directory 404 if d.getVar('BBMULTICONFIG') is not None: 405 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig')) 406 for mc in d.getVar('BBMULTICONFIG').split(): 407 dest_stub = "/conf/multiconfig/%s.conf" % (mc,) 408 if os.path.exists(builddir + dest_stub): 409 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub) 410 411 cachedir = os.path.join(baseoutpath, 'cache') 412 bb.utils.mkdirhier(cachedir) 413 bb.parse.siggen.copy_unitaskhashes(cachedir) 414 415 # If PR Service is in use, we need to export this as well 416 bb.note('Do we have a pr database?') 417 if d.getVar("PRSERV_HOST"): 418 bb.note('Writing PR database...') 419 # Based on the code in classes/prexport.bbclass 420 import oe.prservice 421 #dump meta info of tables 422 localdata = d.createCopy() 423 localdata.setVar('PRSERV_DUMPOPT_COL', "1") 424 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf')) 425 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc') 426 427 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE'))) 428 429 retval = oe.prservice.prserv_dump_db(localdata) 430 if not retval: 431 bb.error("prexport_handler: export failed!") 432 return 433 (metainfo, datainfo) = retval 434 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True) 435 436 # Use templateconf.cfg file from builddir if exists 437 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1': 438 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg') 439 else: 440 # Write a templateconf.cfg 441 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f: 442 f.write('meta/conf/templates/default\n') 443 os.makedirs(os.path.join(baseoutpath, core_meta_subdir, 'conf/templates/default'), exist_ok=True) 444 445 # Ensure any variables set from the external environment (by way of 446 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration 447 extralines = [] 448 for name, value in env_passthrough_values.items(): 449 actualvalue = d.getVar(name) or '' 450 if value != actualvalue: 451 extralines.append('%s = "%s"\n' % (name, actualvalue)) 452 if extralines: 453 with open(baseoutpath + '/conf/local.conf', 'a') as f: 454 f.write('\n') 455 f.write('# Extra settings from environment:\n') 456 for line in extralines: 457 f.write(line) 458 f.write('\n') 459 460 # Filter the locked signatures file to just the sstate tasks we are interested in 461 excluded_targets = get_sdk_install_targets(d, images_only=True) 462 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 463 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc' 464 #nativesdk-only sigfile to merge into locked-sigs.inc 465 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1') 466 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 467 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc' 468 469 if sdk_include_nativesdk: 470 oe.copy_buildsystem.prune_lockedsigs([], 471 excluded_targets.split(), 472 nativesigfile, 473 True, 474 nativesigfile_pruned) 475 476 oe.copy_buildsystem.merge_lockedsigs([], 477 sigfile, 478 nativesigfile_pruned, 479 sigfile) 480 481 oe.copy_buildsystem.prune_lockedsigs([], 482 excluded_targets.split(), 483 sigfile, 484 False, 485 lockedsigs_pruned) 486 487 sstate_out = baseoutpath + '/sstate-cache' 488 bb.utils.remove(sstate_out, True) 489 490 # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d) 491 fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d) 492 493 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1') 494 sdk_ext_type = d.getVar('SDK_EXT_TYPE') 495 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk: 496 # Create the filtered task list used to generate the sstate cache shipped with the SDK 497 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt' 498 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath) 499 else: 500 tasklistfn = None 501 502 cachedir = os.path.join(baseoutpath, 'cache') 503 bb.utils.mkdirhier(cachedir) 504 bb.parse.siggen.copy_unitaskhashes(cachedir) 505 506 # Add packagedata if enabled 507 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 508 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc' 509 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc' 510 shutil.move(lockedsigs_pruned, lockedsigs_base) 511 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'], 512 lockedsigs_base, 513 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc', 514 lockedsigs_pruned, 515 lockedsigs_copy) 516 517 if sdk_include_toolchain: 518 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc' 519 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc") 520 shutil.move(lockedsigs_pruned, lockedsigs_base) 521 oe.copy_buildsystem.merge_lockedsigs([], 522 lockedsigs_base, 523 lockedsigs_toolchain, 524 lockedsigs_pruned) 525 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain, 526 d.getVar('SSTATE_DIR'), 527 sstate_out, d, 528 fixedlsbstring, 529 filterfile=tasklistfn) 530 531 if sdk_ext_type == 'minimal': 532 if derivative: 533 # Assume the user is not going to set up an additional sstate 534 # mirror, thus we need to copy the additional artifacts (from 535 # workspace recipes) into the derivative SDK 536 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc' 537 if os.path.exists(lockedsigs_orig): 538 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc' 539 oe.copy_buildsystem.merge_lockedsigs(None, 540 lockedsigs_orig, 541 lockedsigs_pruned, 542 None, 543 lockedsigs_extra) 544 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, 545 d.getVar('SSTATE_DIR'), 546 sstate_out, d, 547 fixedlsbstring, 548 filterfile=tasklistfn) 549 else: 550 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, 551 d.getVar('SSTATE_DIR'), 552 sstate_out, d, 553 fixedlsbstring, 554 filterfile=tasklistfn) 555 556 # We don't need sstate do_package files 557 for root, dirs, files in os.walk(sstate_out): 558 for name in files: 559 if name.endswith("_package.tar.zst"): 560 f = os.path.join(root, name) 561 os.remove(f) 562 563 # Write manifest file 564 # Note: at the moment we cannot include the env setup script here to keep 565 # it updated, since it gets modified during SDK installation (see 566 # sdk_ext_postinst() below) thus the checksum we take here would always 567 # be different. 568 manifest_file_list = ['conf/*'] 569 if d.getVar('BBMULTICONFIG') is not None: 570 manifest_file_list.append('conf/multiconfig/*') 571 572 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split() 573 esdk_manifest_excludes_list = [] 574 for exclude_item in esdk_manifest_excludes: 575 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item)) 576 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest') 577 with open(manifest_file, 'w') as f: 578 for item in manifest_file_list: 579 for fn in glob.glob(os.path.join(baseoutpath, item)): 580 if fn == manifest_file or os.path.isdir(fn): 581 continue 582 if fn in esdk_manifest_excludes_list: 583 continue 584 chksum = bb.utils.sha256_file(fn) 585 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath))) 586} 587 588def get_current_buildtools(d): 589 """Get the file name of the current buildtools installer""" 590 import glob 591 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh')) 592 btfiles.sort(key=os.path.getctime) 593 return os.path.basename(btfiles[-1]) 594 595def get_sdk_required_utilities(buildtools_fn, d): 596 """Find required utilities that aren't provided by the buildtools""" 597 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split() 598 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc')) 599 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++')) 600 if buildtools_fn: 601 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn) 602 filelist, _ = bb.process.run('%s -l' % buildtools_installer) 603 else: 604 buildtools_installer = None 605 filelist = "" 606 localdata = bb.data.createCopy(d) 607 localdata.setVar('SDKPATH', '.') 608 sdkpathnative = localdata.getVar('SDKPATHNATIVE') 609 sdkbindirs = [localdata.getVar('bindir_nativesdk'), 610 localdata.getVar('sbindir_nativesdk'), 611 localdata.getVar('base_bindir_nativesdk'), 612 localdata.getVar('base_sbindir_nativesdk')] 613 for line in filelist.splitlines(): 614 splitline = line.split() 615 if len(splitline) > 5: 616 fn = splitline[5] 617 if not fn.startswith('./'): 618 fn = './%s' % fn 619 if fn.startswith(sdkpathnative): 620 relpth = '/' + os.path.relpath(fn, sdkpathnative) 621 for bindir in sdkbindirs: 622 if relpth.startswith(bindir): 623 relpth = os.path.relpath(relpth, bindir) 624 if relpth in sanity_required_utilities: 625 sanity_required_utilities.remove(relpth) 626 break 627 return ' '.join(sanity_required_utilities) 628 629install_tools() { 630 install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk} 631 scripts="devtool recipetool oe-find-native-sysroot runqemu* wic" 632 for script in $scripts; do 633 for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do 634 targetscriptfn="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/$(basename $scriptfn)" 635 test -e ${targetscriptfn} || ln -rs ${scriptfn} ${targetscriptfn} 636 done 637 done 638 # We can't use the same method as above because files in the sysroot won't exist at this point 639 # (they get populated from sstate on installation) 640 unfsd_path="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/unfsd" 641 if [ "${SDK_INCLUDE_TOOLCHAIN}" = "1" -a ! -e $unfsd_path ] ; then 642 binrelpath=${@os.path.relpath(d.getVar('STAGING_BINDIR_NATIVE'), d.getVar('TMPDIR'))} 643 ln -rs ${SDK_OUTPUT}/${SDKPATH}/tmp/$binrelpath/unfsd $unfsd_path 644 fi 645 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase 646 647 # find latest buildtools-tarball and install it 648 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 649 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH} 650 fi 651 652 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH} 653} 654do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True" 655 656sdk_ext_preinst() { 657 # Since bitbake won't run as root it doesn't make sense to try and install 658 # the extensible sdk as root. 659 if [ "`id -u`" = "0" ]; then 660 echo "ERROR: The extensible sdk cannot be installed as root." 661 exit 1 662 fi 663 if ! command -v locale > /dev/null; then 664 echo "ERROR: The installer requires the locale command, please install it first" 665 exit 1 666 fi 667 # Check setting of LC_ALL set above 668 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'` 669 if ! locale -a | grep -q $canonicalised_locale ; then 670 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first" 671 exit 1 672 fi 673 # The relocation script used by buildtools installer requires python 674 if ! command -v python3 > /dev/null; then 675 echo "ERROR: The installer requires python3, please install it first" 676 exit 1 677 fi 678 missing_utils="" 679 for util in ${SDK_REQUIRED_UTILITIES}; do 680 if ! command -v $util > /dev/null; then 681 missing_utils="$missing_utils $util" 682 fi 683 done 684 if [ -n "$missing_utils" ] ; then 685 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils" 686 exit 1 687 fi 688 SDK_EXTENSIBLE="1" 689 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then 690 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache" 691 fi 692} 693SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}" 694 695# FIXME this preparation should be done as part of the SDK construction 696sdk_ext_postinst() { 697 printf "\nExtracting buildtools...\n" 698 cd $target_sdk_dir 699 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}" 700 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 701 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 702 703 # Delete the buildtools tar file since it won't be used again 704 rm -f ./${SDK_BUILDTOOLS_INSTALLER} 705 # We don't need the log either since it succeeded 706 rm -f buildtools.log 707 708 # Make sure when the user sets up the environment, they also get 709 # the buildtools-tarball tools in their path. 710 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script 711 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script 712 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script 713 echo "OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script 714 fi 715 716 # Allow bitbake environment setup to be ran as part of this sdk. 717 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script 718 # Work around runqemu not knowing how to get this information within the eSDK 719 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script 720 721 # A bit of another hack, but we need this in the path only for devtool 722 # so put it at the end of $PATH. 723 echo "export PATH=\"$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH\"" >> $env_setup_script 724 725 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script 726 727 # Warn if trying to use external bitbake and the ext SDK together 728 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script 729 730 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 731 printf "Preparing build system...\n" 732 # dash which is /bin/sh on Ubuntu will not preserve the 733 # current working directory when first ran, nor will it set $1 when 734 # sourcing a script. That is why this has to look so ugly. 735 LOGFILE="$target_sdk_dir/preparing_build_system.log" 736 sh -c ". buildtools/environment-setup* > $LOGFILE 2>&1 && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE 2>&1 && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 737 fi 738 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then 739 rm $target_sdk_dir/ext-sdk-prepare.py 740 fi 741 echo done 742} 743 744SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}" 745 746SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem; install_tools; " 747 748SDK_INSTALL_TARGETS = "" 749fakeroot python do_populate_sdk_ext() { 750 # FIXME hopefully we can remove this restriction at some point, but uninative 751 # currently forces this upon us 752 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'): 753 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH'))) 754 755 # FIXME hopefully we can remove this restriction at some point, but the eSDK 756 # can only be built for the primary (default) multiconfig 757 if d.getVar('BB_CURRENT_MC') != 'default': 758 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC')) 759 760 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set 761 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}") 762 d.setVar("TOOLCHAIN_TARGET_TASK", "") 763 764 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d)) 765 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1': 766 buildtools_fn = get_current_buildtools(d) 767 else: 768 buildtools_fn = None 769 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d)) 770 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn) 771 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}') 772 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice 773 d.delVar('SDKIMAGE_LINGUAS') 774 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1': 775 generate_nativesdk_lockedsigs(d) 776 populate_sdk_common(d) 777} 778 779def generate_nativesdk_lockedsigs(d): 780 import oe.copy_buildsystem 781 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 782 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 783 784def get_ext_sdk_depends(d): 785 # Note: the deps varflag is a list not a string, so we need to specify expand=False 786 deps = d.getVarFlag('do_image_complete', 'deps', False) 787 pn = d.getVar('PN') 788 deplist = ['%s:%s' % (pn, dep) for dep in deps] 789 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 790 tasklist.append('do_rootfs') 791 for task in tasklist: 792 deplist.extend((d.getVarFlag(task, 'depends') or '').split()) 793 return ' '.join(deplist) 794 795python do_sdk_depends() { 796 # We have to do this separately in its own task so we avoid recursing into 797 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those 798 # into the SDK's sstate-cache 799 import oe.copy_buildsystem 800 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 801 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 802} 803addtask sdk_depends 804 805do_sdk_depends[dirs] = "${WORKDIR}" 806do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot" 807do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}" 808do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}" 809do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}" 810 811do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}" 812 813do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \ 814 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \ 815 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \ 816 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}" 817 818# We must avoid depending on do_build here if rm_work.bbclass is active, 819# because otherwise do_rm_work may run before do_populate_sdk_ext itself. 820# We can't mark do_populate_sdk_ext and do_sdk_depends as having to 821# run before do_rm_work, because then they would also run as part 822# of normal builds. 823do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}" 824 825# Make sure code changes can result in rebuild 826do_populate_sdk_ext[vardeps] += "copy_buildsystem \ 827 sdk_ext_postinst" 828 829# Since any change in the metadata of any layer should cause a rebuild of the 830# sdk(since the layers are put in the sdk) set the task to nostamp so it 831# always runs. 832do_populate_sdk_ext[nostamp] = "1" 833 834SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext" 835 836SSTATETASKS += "do_populate_sdk_ext" 837SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1' 838do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}" 839do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}" 840do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}" 841do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}" 842 843addtask populate_sdk_ext after do_sdk_depends 844