1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7# Extensible SDK 8 9inherit populate_sdk_base 10 11# Used to override TOOLCHAIN_HOST_TASK in the eSDK case 12TOOLCHAIN_HOST_TASK_ESDK = " \ 13 meta-environment-extsdk-${MACHINE} \ 14 " 15 16SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0" 17 18SDK_EXT = "" 19SDK_EXT:task-populate-sdk-ext = "-ext" 20 21# Options are full or minimal 22SDK_EXT_TYPE ?= "full" 23SDK_INCLUDE_PKGDATA ?= "0" 24SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}" 25SDK_INCLUDE_NATIVESDK ?= "0" 26SDK_INCLUDE_BUILDTOOLS ?= '1' 27 28SDK_RECRDEP_TASKS ?= "" 29SDK_CUSTOM_TEMPLATECONF ?= "0" 30 31ESDK_LOCALCONF_ALLOW ?= "" 32ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \ 33 BB_NUMBER_THREADS \ 34 BB_NUMBER_PARSE_THREADS \ 35 PARALLEL_MAKE \ 36 PRSERV_HOST \ 37 SSTATE_MIRRORS \ 38 DL_DIR \ 39 SSTATE_DIR \ 40 TMPDIR \ 41 BB_SERVER_TIMEOUT \ 42 " 43ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory icecc" 44SDK_UPDATE_URL ?= "" 45 46SDK_TARGETS ?= "${PN}" 47 48def get_sdk_install_targets(d, images_only=False): 49 sdk_install_targets = '' 50 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal': 51 sdk_install_targets = d.getVar('SDK_TARGETS') 52 53 depd = d.getVar('BB_TASKDEPDATA', False) 54 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 55 tasklist.remove('do_build') 56 for v in depd.values(): 57 if v[1] in tasklist: 58 if v[0] not in sdk_install_targets: 59 sdk_install_targets += ' {}'.format(v[0]) 60 61 if not images_only: 62 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 63 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata' 64 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1': 65 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot' 66 67 return sdk_install_targets 68 69get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA" 70 71OE_INIT_ENV_SCRIPT ?= "oe-init-build-env" 72 73# The files from COREBASE that you want preserved in the COREBASE copied 74# into the sdk. This allows someone to have their own setup scripts in 75# COREBASE be preserved as well as untracked files. 76COREBASE_FILES ?= " \ 77 oe-init-build-env \ 78 scripts \ 79 LICENSE \ 80 .templateconf \ 81" 82 83SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext" 84B:task-populate-sdk-ext = "${SDK_DIR}" 85TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}" 86TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}" 87 88SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest" 89SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest" 90 91python write_target_sdk_ext_manifest () { 92 from oe.sdk import get_extra_sdkinfo 93 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 94 extra_info = get_extra_sdkinfo(sstate_dir) 95 96 target = d.getVar('TARGET_SYS') 97 target_multimach = d.getVar('MULTIMACH_TARGET_SYS') 98 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS') 99 100 pkgs = {} 101 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True) 102 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f: 103 for fn in extra_info['filesizes']: 104 info = fn.split(':') 105 if info[2] in (target, target_multimach, real_target_multimach) \ 106 or info[5] == 'allarch': 107 if not info[1] in pkgs: 108 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 109 pkgs[info[1]] = {} 110} 111python write_host_sdk_ext_manifest () { 112 from oe.sdk import get_extra_sdkinfo 113 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 114 extra_info = get_extra_sdkinfo(sstate_dir) 115 host = d.getVar('BUILD_SYS') 116 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f: 117 for fn in extra_info['filesizes']: 118 info = fn.split(':') 119 if info[2] == host: 120 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 121} 122 123SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest write_host_sdk_ext_manifest" 124 125SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK" 126 127def clean_esdk_builddir(d, sdkbasepath): 128 """Clean up traces of the fake build for create_filtered_tasklist()""" 129 import shutil 130 cleanpaths = ['cache', 'tmp'] 131 for pth in cleanpaths: 132 fullpth = os.path.join(sdkbasepath, pth) 133 if os.path.isdir(fullpth): 134 shutil.rmtree(fullpth) 135 elif os.path.isfile(fullpth): 136 os.remove(fullpth) 137 138def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath): 139 """ 140 Create a filtered list of tasks. Also double-checks that the build system 141 within the SDK basically works and required sstate artifacts are available. 142 """ 143 import tempfile 144 import shutil 145 import oe.copy_buildsystem 146 147 # Create a temporary build directory that we can pass to the env setup script 148 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak') 149 try: 150 with open(sdkbasepath + '/conf/local.conf', 'a') as f: 151 # Force the use of sstate from the build system 152 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR')) 153 f.write('SSTATE_MIRRORS:forcevariable = "file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n') 154 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it 155 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n') 156 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will 157 # be different and we won't be able to find our native sstate) 158 if not bb.data.inherits_class('uninative', d): 159 f.write('INHERIT:remove = "uninative"\n') 160 161 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake 162 # will not allow in its COREBASE path, so we need to rename the directory temporarily 163 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk' 164 # Delete any existing temp dir 165 try: 166 shutil.rmtree(temp_sdkbasepath) 167 except FileNotFoundError: 168 pass 169 bb.utils.rename(sdkbasepath, temp_sdkbasepath) 170 cmdprefix = '. %s .; ' % conf_initpath 171 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt' 172 try: 173 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile) 174 except bb.process.ExecutionError as e: 175 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip() 176 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout: 177 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n' 178 bb.fatal(msg) 179 bb.utils.rename(temp_sdkbasepath, sdkbasepath) 180 # Clean out residue of running bitbake, which check_sstate_task_list() 181 # will effectively do 182 clean_esdk_builddir(d, sdkbasepath) 183 finally: 184 localconf = sdkbasepath + '/conf/local.conf' 185 if os.path.exists(localconf + '.bak'): 186 os.replace(localconf + '.bak', localconf) 187 188def copy_bitbake_and_layers(d, baseoutpath, derivative): 189 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT') 190 191 conf_bbpath = '' 192 conf_initpath = '' 193 core_meta_subdir = '' 194 195 # Copy in all metadata layers + bitbake (as repositories) 196 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) 197 198 if derivative: 199 workspace_name = 'orig-workspace' 200 else: 201 workspace_name = None 202 203 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name) 204 conf_bbpath = os.path.join('layers', corebase, 'bitbake') 205 206 for path in os.listdir(baseoutpath + '/layers'): 207 relpath = os.path.join('layers', path, oe_init_env_script) 208 if os.path.exists(os.path.join(baseoutpath, relpath)): 209 conf_initpath = relpath 210 211 relpath = os.path.join('layers', path, 'scripts', 'esdk-tools', 'devtool') 212 if os.path.exists(os.path.join(baseoutpath, relpath)): 213 esdk_tools_path = os.path.dirname(relpath) 214 215 relpath = os.path.join('layers', path, 'meta') 216 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')): 217 core_meta_subdir = relpath 218 219 d.setVar('oe_init_build_env_path', conf_initpath) 220 d.setVar('esdk_tools_path', esdk_tools_path) 221 222 return (conf_initpath, conf_bbpath, core_meta_subdir, sdkbblayers) 223 224def write_devtool_config(d, baseoutpath, conf_bbpath, conf_initpath, core_meta_subdir): 225 # Write out config file for devtool 226 import configparser 227 config = configparser.ConfigParser() 228 config.add_section('General') 229 config.set('General', 'bitbake_subdir', conf_bbpath) 230 config.set('General', 'init_path', conf_initpath) 231 config.set('General', 'core_meta_subdir', core_meta_subdir) 232 config.add_section('SDK') 233 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS')) 234 updateurl = d.getVar('SDK_UPDATE_URL') 235 if updateurl: 236 config.set('SDK', 'updateserver', updateurl) 237 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf')) 238 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f: 239 config.write(f) 240 241def write_unlocked_sigs(d, baseoutpath): 242 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc') 243 with open(unlockedsigs, 'w') as f: 244 pass 245 246def write_bblayers_conf(d, baseoutpath, sdkbblayers): 247 # Create a layer for new recipes / appends 248 bbpath = d.getVar('BBPATH') 249 env = os.environ.copy() 250 env['PYTHONDONTWRITEBYTECODE'] = '1' 251 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--layerseries', d.getVar("LAYERSERIES_CORENAMES"), '--create-only', os.path.join(baseoutpath, 'workspace')], env=env) 252 253 # Create bblayers.conf 254 bb.utils.mkdirhier(baseoutpath + '/conf') 255 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f: 256 f.write('# WARNING: this configuration has been automatically generated and in\n') 257 f.write('# most cases should not be edited. If you need more flexibility than\n') 258 f.write('# this configuration provides, it is strongly suggested that you set\n') 259 f.write('# up a proper instance of the full build system and use that instead.\n\n') 260 261 # LCONF_VERSION may not be set, for example when using meta-poky 262 # so don't error if it isn't found 263 lconf_version = d.getVar('LCONF_VERSION', False) 264 if lconf_version is not None: 265 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version) 266 267 f.write('BBPATH = "$' + '{TOPDIR}"\n') 268 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n') 269 f.write('BBLAYERS := " \\\n') 270 for layerrelpath in sdkbblayers: 271 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath) 272 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n') 273 f.write(' "\n') 274 275def copy_uninative(d, baseoutpath): 276 import shutil 277 278 uninative_checksum = None 279 280 # Copy uninative tarball 281 # For now this is where uninative.bbclass expects the tarball 282 if bb.data.inherits_class('uninative', d): 283 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}') 284 uninative_checksum = bb.utils.sha256_file(uninative_file) 285 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum) 286 bb.utils.mkdirhier(uninative_outdir) 287 shutil.copy(uninative_file, uninative_outdir) 288 289 return uninative_checksum 290 291def write_local_conf(d, baseoutpath, derivative, core_meta_subdir, uninative_checksum): 292 #check if custome templateconf path is set 293 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF') 294 295 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split() 296 env_passthrough_values = {} 297 298 # Create local.conf 299 builddir = d.getVar('TOPDIR') 300 if derivative and os.path.exists(builddir + '/conf/site.conf'): 301 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf') 302 if derivative and os.path.exists(builddir + '/conf/auto.conf'): 303 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf') 304 if derivative: 305 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf') 306 else: 307 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split() 308 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split() 309 def handle_var(varname, origvalue, op, newlines): 310 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed): 311 newlines.append('# Removed original setting of %s\n' % varname) 312 return None, op, 0, True 313 else: 314 if varname in env_passthrough: 315 env_passthrough_values[varname] = origvalue 316 return origvalue, op, 0, True 317 varlist = ['[^#=+ ]*'] 318 oldlines = [] 319 if os.path.exists(builddir + '/conf/site.conf'): 320 with open(builddir + '/conf/site.conf', 'r') as f: 321 oldlines += f.readlines() 322 if os.path.exists(builddir + '/conf/auto.conf'): 323 with open(builddir + '/conf/auto.conf', 'r') as f: 324 oldlines += f.readlines() 325 if os.path.exists(builddir + '/conf/local.conf'): 326 with open(builddir + '/conf/local.conf', 'r') as f: 327 oldlines += f.readlines() 328 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) 329 330 with open(baseoutpath + '/conf/local.conf', 'w') as f: 331 f.write('# WARNING: this configuration has been automatically generated and in\n') 332 f.write('# most cases should not be edited. If you need more flexibility than\n') 333 f.write('# this configuration provides, it is strongly suggested that you set\n') 334 f.write('# up a proper instance of the full build system and use that instead.\n\n') 335 for line in newlines: 336 if line.strip() and not line.startswith('#'): 337 f.write(line) 338 # Write a newline just in case there's none at the end of the original 339 f.write('\n') 340 341 f.write('TMPDIR = "${TOPDIR}/tmp"\n') 342 f.write('DL_DIR = "${TOPDIR}/downloads"\n') 343 344 if bb.data.inherits_class('uninative', d): 345 f.write('INHERIT += "%s"\n' % 'uninative') 346 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum)) 347 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) 348 349 # Some classes are not suitable for SDK, remove them from INHERIT 350 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False)) 351 352 # Bypass the default connectivity check if any 353 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') 354 355 # This warning will come out if reverse dependencies for a task 356 # don't have sstate as well as the task itself. We already know 357 # this will be the case for the extensible sdk, so turn off the 358 # warning. 359 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n') 360 361 # Warn if the sigs in the locked-signature file don't match 362 # the sig computed from the metadata. 363 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n') 364 365 # We want to be able to set this without a full reparse 366 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n') 367 368 # Set up which tasks are ignored for run on install 369 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n') 370 371 # Hide the config information from bitbake output (since it's fixed within the SDK) 372 f.write('BUILDCFG_HEADER = ""\n\n') 373 374 # Write METADATA_REVISION 375 # Needs distro override so it can override the value set in the bbclass code (later than local.conf) 376 f.write('METADATA_REVISION:%s = "%s"\n\n' % (d.getVar('DISTRO'), d.getVar('METADATA_REVISION'))) 377 378 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n') 379 f.write('WITHIN_EXT_SDK = "1"\n\n') 380 381 # Map gcc-dependent uninative sstate cache for installer usage 382 f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n') 383 384 if d.getVar("PRSERV_HOST"): 385 # Override this, we now include PR data, so it should only point ot the local database 386 f.write('PRSERV_HOST = "localhost:0"\n\n') 387 388 # Allow additional config through sdk-extra.conf 389 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) 390 if fn: 391 with open(fn, 'r') as xf: 392 for line in xf: 393 f.write(line) 394 395 # If you define a sdk_extraconf() function then it can contain additional config 396 # (Though this is awkward; sdk-extra.conf should probably be used instead) 397 extraconf = (d.getVar('sdk_extraconf') or '').strip() 398 if extraconf: 399 # Strip off any leading / trailing spaces 400 for line in extraconf.splitlines(): 401 f.write(line.strip() + '\n') 402 403 f.write('require conf/locked-sigs.inc\n') 404 f.write('require conf/unlocked-sigs.inc\n') 405 406 # Copy multiple configurations if they exist in the users config directory 407 if d.getVar('BBMULTICONFIG') is not None: 408 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig')) 409 for mc in d.getVar('BBMULTICONFIG').split(): 410 dest_stub = "/conf/multiconfig/%s.conf" % (mc,) 411 if os.path.exists(builddir + dest_stub): 412 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub) 413 414 # If PR Service is in use, we need to export this as well 415 bb.note('Do we have a pr database?') 416 if d.getVar("PRSERV_HOST"): 417 bb.note('Writing PR database...') 418 # Based on the code in classes/prexport.bbclass 419 import oe.prservice 420 #dump meta info of tables 421 localdata = d.createCopy() 422 localdata.setVar('PRSERV_DUMPOPT_COL', "1") 423 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf')) 424 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc') 425 426 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE'))) 427 428 retval = oe.prservice.prserv_dump_db(localdata) 429 if not retval: 430 bb.error("prexport_handler: export failed!") 431 return 432 (metainfo, datainfo) = retval 433 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True) 434 435 # Use templateconf.cfg file from builddir if exists 436 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1': 437 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg') 438 else: 439 # Write a templateconf.cfg 440 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f: 441 f.write('meta/conf/templates/default\n') 442 os.makedirs(os.path.join(baseoutpath, core_meta_subdir, 'conf/templates/default'), exist_ok=True) 443 444 # Ensure any variables set from the external environment (by way of 445 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration 446 extralines = [] 447 for name, value in env_passthrough_values.items(): 448 actualvalue = d.getVar(name) or '' 449 if value != actualvalue: 450 extralines.append('%s = "%s"\n' % (name, actualvalue)) 451 if extralines: 452 with open(baseoutpath + '/conf/local.conf', 'a') as f: 453 f.write('\n') 454 f.write('# Extra settings from environment:\n') 455 for line in extralines: 456 f.write(line) 457 f.write('\n') 458 459def prepare_locked_cache(d, baseoutpath, derivative, conf_initpath): 460 import shutil 461 462 # Filter the locked signatures file to just the sstate tasks we are interested in 463 excluded_targets = get_sdk_install_targets(d, images_only=True) 464 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 465 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc' 466 #nativesdk-only sigfile to merge into locked-sigs.inc 467 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1') 468 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 469 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc' 470 471 if sdk_include_nativesdk: 472 oe.copy_buildsystem.prune_lockedsigs([], 473 excluded_targets.split(), 474 nativesigfile, 475 True, 476 nativesigfile_pruned) 477 478 oe.copy_buildsystem.merge_lockedsigs([], 479 sigfile, 480 nativesigfile_pruned, 481 sigfile) 482 483 oe.copy_buildsystem.prune_lockedsigs([], 484 excluded_targets.split(), 485 sigfile, 486 False, 487 lockedsigs_pruned) 488 489 sstate_out = baseoutpath + '/sstate-cache' 490 bb.utils.remove(sstate_out, True) 491 492 # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d) 493 fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d) if bb.data.inherits_class('uninative', d) else "" 494 495 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1') 496 sdk_ext_type = d.getVar('SDK_EXT_TYPE') 497 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk: 498 # Create the filtered task list used to generate the sstate cache shipped with the SDK 499 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt' 500 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath) 501 else: 502 tasklistfn = None 503 504 # Add packagedata if enabled 505 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 506 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc' 507 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc' 508 shutil.move(lockedsigs_pruned, lockedsigs_base) 509 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'], 510 lockedsigs_base, 511 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc', 512 lockedsigs_pruned, 513 lockedsigs_copy) 514 515 if sdk_include_toolchain: 516 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc' 517 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc") 518 shutil.move(lockedsigs_pruned, lockedsigs_base) 519 oe.copy_buildsystem.merge_lockedsigs([], 520 lockedsigs_base, 521 lockedsigs_toolchain, 522 lockedsigs_pruned) 523 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain, 524 d.getVar('SSTATE_DIR'), 525 sstate_out, d, 526 fixedlsbstring, 527 filterfile=tasklistfn) 528 529 if sdk_ext_type == 'minimal': 530 if derivative: 531 # Assume the user is not going to set up an additional sstate 532 # mirror, thus we need to copy the additional artifacts (from 533 # workspace recipes) into the derivative SDK 534 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc' 535 if os.path.exists(lockedsigs_orig): 536 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc' 537 oe.copy_buildsystem.merge_lockedsigs(None, 538 lockedsigs_orig, 539 lockedsigs_pruned, 540 None, 541 lockedsigs_extra) 542 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, 543 d.getVar('SSTATE_DIR'), 544 sstate_out, d, 545 fixedlsbstring, 546 filterfile=tasklistfn) 547 else: 548 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, 549 d.getVar('SSTATE_DIR'), 550 sstate_out, d, 551 fixedlsbstring, 552 filterfile=tasklistfn) 553 554 # We don't need sstate do_package files 555 for root, dirs, files in os.walk(sstate_out): 556 for name in files: 557 if name.endswith("_package.tar.zst"): 558 f = os.path.join(root, name) 559 os.remove(f) 560 561def write_manifest(d, baseoutpath): 562 import glob 563 564 # Write manifest file 565 # Note: at the moment we cannot include the env setup script here to keep 566 # it updated, since it gets modified during SDK installation (see 567 # sdk_ext_postinst() below) thus the checksum we take here would always 568 # be different. 569 manifest_file_list = ['conf/*'] 570 if d.getVar('BBMULTICONFIG') is not None: 571 manifest_file_list.append('conf/multiconfig/*') 572 573 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split() 574 esdk_manifest_excludes_list = [] 575 for exclude_item in esdk_manifest_excludes: 576 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item)) 577 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest') 578 with open(manifest_file, 'w') as f: 579 for item in manifest_file_list: 580 for fn in glob.glob(os.path.join(baseoutpath, item)): 581 if fn == manifest_file or os.path.isdir(fn): 582 continue 583 if fn in esdk_manifest_excludes_list: 584 continue 585 chksum = bb.utils.sha256_file(fn) 586 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath))) 587 588 589python copy_buildsystem () { 590 import oe.copy_buildsystem 591 592 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH') 593 594 # Determine if we're building a derivative extensible SDK (from devtool build-sdk) 595 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1' 596 597 conf_initpath, conf_bbpath, core_meta_subdir, sdkbblayers = copy_bitbake_and_layers(d, baseoutpath, derivative) 598 599 write_devtool_config(d, baseoutpath, conf_bbpath, conf_initpath, core_meta_subdir) 600 601 write_unlocked_sigs(d, baseoutpath) 602 603 write_bblayers_conf(d, baseoutpath, sdkbblayers) 604 605 uninative_checksum = copy_uninative(d, baseoutpath) 606 607 write_local_conf(d, baseoutpath, derivative, core_meta_subdir, uninative_checksum) 608 609 prepare_locked_cache(d, baseoutpath, derivative, conf_initpath) 610 611 write_manifest(d, baseoutpath) 612 613} 614 615def get_current_buildtools(d): 616 """Get the file name of the current buildtools installer""" 617 import glob 618 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh')) 619 btfiles.sort(key=os.path.getctime) 620 return os.path.basename(btfiles[-1]) 621 622def get_sdk_required_utilities(buildtools_fn, d): 623 """Find required utilities that aren't provided by the buildtools""" 624 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split() 625 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc')) 626 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++')) 627 if buildtools_fn: 628 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn) 629 filelist, _ = bb.process.run('%s -l' % buildtools_installer) 630 else: 631 buildtools_installer = None 632 filelist = "" 633 localdata = bb.data.createCopy(d) 634 localdata.setVar('SDKPATH', '.') 635 sdkpathnative = localdata.getVar('SDKPATHNATIVE') 636 sdkbindirs = [localdata.getVar('bindir_nativesdk'), 637 localdata.getVar('sbindir_nativesdk'), 638 localdata.getVar('base_bindir_nativesdk'), 639 localdata.getVar('base_sbindir_nativesdk')] 640 for line in filelist.splitlines(): 641 splitline = line.split() 642 if len(splitline) > 5: 643 fn = splitline[5] 644 if not fn.startswith('./'): 645 fn = './%s' % fn 646 if fn.startswith(sdkpathnative): 647 relpth = '/' + os.path.relpath(fn, sdkpathnative) 648 for bindir in sdkbindirs: 649 if relpth.startswith(bindir): 650 relpth = os.path.relpath(relpth, bindir) 651 if relpth in sanity_required_utilities: 652 sanity_required_utilities.remove(relpth) 653 break 654 return ' '.join(sanity_required_utilities) 655 656install_tools() { 657 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase 658 659 # find latest buildtools-tarball and install it 660 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 661 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH} 662 fi 663 664 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH} 665} 666do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True" 667 668sdk_ext_preinst() { 669 # Since bitbake won't run as root it doesn't make sense to try and install 670 # the extensible sdk as root. 671 if [ "`id -u`" = "0" ]; then 672 echo "ERROR: The extensible sdk cannot be installed as root." 673 exit 1 674 fi 675 if ! command -v locale > /dev/null; then 676 echo "ERROR: The installer requires the locale command, please install it first" 677 exit 1 678 fi 679 # Check setting of LC_ALL set above 680 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'` 681 if ! locale -a | grep -q $canonicalised_locale ; then 682 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first" 683 exit 1 684 fi 685 # The relocation script used by buildtools installer requires python 686 if ! command -v python3 > /dev/null; then 687 echo "ERROR: The installer requires python3, please install it first" 688 exit 1 689 fi 690 missing_utils="" 691 for util in ${SDK_REQUIRED_UTILITIES}; do 692 if ! command -v $util > /dev/null; then 693 missing_utils="$missing_utils $util" 694 fi 695 done 696 if [ -n "$missing_utils" ] ; then 697 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils" 698 exit 1 699 fi 700 SDK_EXTENSIBLE="1" 701 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then 702 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache" 703 fi 704} 705SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}" 706 707# FIXME this preparation should be done as part of the SDK construction 708sdk_ext_postinst() { 709 printf "\nExtracting buildtools...\n" 710 cd $target_sdk_dir 711 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}" 712 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 713 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 714 715 # Delete the buildtools tar file since it won't be used again 716 rm -f ./${SDK_BUILDTOOLS_INSTALLER} 717 # We don't need the log either since it succeeded 718 rm -f buildtools.log 719 720 # Make sure when the user sets up the environment, they also get 721 # the buildtools-tarball tools in their path. 722 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script 723 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script 724 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script 725 echo "export OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script 726 fi 727 728 # Allow bitbake environment setup to be ran as part of this sdk. 729 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script 730 # Work around runqemu not knowing how to get this information within the eSDK 731 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script 732 733 # A bit of another hack, but we need this in the path only for devtool 734 # so put it at the end of $PATH. 735 echo "export PATH=\"$target_sdk_dir/${esdk_tools_path}:\$PATH\"" >> $env_setup_script 736 737 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script 738 739 # Warn if trying to use external bitbake and the ext SDK together 740 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script 741 742 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 743 printf "Preparing build system...\n" 744 # dash which is /bin/sh on Ubuntu will not preserve the 745 # current working directory when first ran, nor will it set $1 when 746 # sourcing a script. That is why this has to look so ugly. 747 LOGFILE="$target_sdk_dir/preparing_build_system.log" 748 sh -c ". buildtools/environment-setup* > $LOGFILE 2>&1 && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE 2>&1 && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 749 fi 750 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then 751 rm $target_sdk_dir/ext-sdk-prepare.py 752 fi 753 echo done 754} 755 756SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}" 757 758SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem install_tools " 759 760SDK_INSTALL_TARGETS = "" 761fakeroot python do_populate_sdk_ext() { 762 # FIXME hopefully we can remove this restriction at some point, but uninative 763 # currently forces this upon us 764 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'): 765 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH'))) 766 767 # FIXME hopefully we can remove this restriction at some point, but the eSDK 768 # can only be built for the primary (default) multiconfig 769 if d.getVar('BB_CURRENT_MC') != 'default': 770 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC')) 771 772 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set 773 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}") 774 d.setVar("TOOLCHAIN_TARGET_TASK", "") 775 776 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d)) 777 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1': 778 buildtools_fn = get_current_buildtools(d) 779 else: 780 buildtools_fn = None 781 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d)) 782 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn) 783 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}') 784 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice 785 d.delVar('SDKIMAGE_LINGUAS') 786 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1': 787 generate_nativesdk_lockedsigs(d) 788 populate_sdk_common(d) 789} 790 791def generate_nativesdk_lockedsigs(d): 792 import oe.copy_buildsystem 793 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 794 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 795 796def get_ext_sdk_depends(d): 797 # Note: the deps varflag is a list not a string, so we need to specify expand=False 798 deps = d.getVarFlag('do_image_complete', 'deps', False) 799 pn = d.getVar('PN') 800 deplist = ['%s:%s' % (pn, dep) for dep in deps] 801 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 802 tasklist.append('do_rootfs') 803 for task in tasklist: 804 deplist.extend((d.getVarFlag(task, 'depends') or '').split()) 805 return ' '.join(deplist) 806 807python do_sdk_depends() { 808 # We have to do this separately in its own task so we avoid recursing into 809 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those 810 # into the SDK's sstate-cache 811 import oe.copy_buildsystem 812 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 813 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 814} 815addtask sdk_depends 816 817do_sdk_depends[dirs] = "${WORKDIR}" 818do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot" 819do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}" 820do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}" 821do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}" 822 823do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}" 824 825do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \ 826 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \ 827 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \ 828 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}" 829 830# We must avoid depending on do_build here if rm_work.bbclass is active, 831# because otherwise do_rm_work may run before do_populate_sdk_ext itself. 832# We can't mark do_populate_sdk_ext and do_sdk_depends as having to 833# run before do_rm_work, because then they would also run as part 834# of normal builds. 835do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}" 836 837# Make sure code changes can result in rebuild 838do_populate_sdk_ext[vardeps] += "copy_buildsystem \ 839 sdk_ext_postinst" 840 841# Since any change in the metadata of any layer should cause a rebuild of the 842# sdk(since the layers are put in the sdk) set the task to nostamp so it 843# always runs. 844do_populate_sdk_ext[nostamp] = "1" 845 846SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext" 847 848SSTATETASKS += "do_populate_sdk_ext" 849SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1' 850do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}" 851do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}" 852do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}" 853do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}" 854 855addtask populate_sdk_ext after do_sdk_depends 856