1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: MIT 5# 6 7# Extensible SDK 8 9inherit populate_sdk_base 10 11# Used to override TOOLCHAIN_HOST_TASK in the eSDK case 12TOOLCHAIN_HOST_TASK_ESDK = " \ 13 meta-environment-extsdk-${MACHINE} \ 14 " 15 16SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0" 17 18SDK_EXT = "" 19SDK_EXT:task-populate-sdk-ext = "-ext" 20 21# Options are full or minimal 22SDK_EXT_TYPE ?= "full" 23SDK_INCLUDE_PKGDATA ?= "0" 24SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}" 25SDK_INCLUDE_NATIVESDK ?= "0" 26SDK_INCLUDE_BUILDTOOLS ?= '1' 27 28SDK_RECRDEP_TASKS ?= "" 29SDK_CUSTOM_TEMPLATECONF ?= "0" 30 31ESDK_LOCALCONF_ALLOW ?= "" 32ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \ 33 BB_NUMBER_THREADS \ 34 BB_NUMBER_PARSE_THREADS \ 35 PARALLEL_MAKE \ 36 PRSERV_HOST \ 37 SSTATE_MIRRORS \ 38 DL_DIR \ 39 SSTATE_DIR \ 40 TMPDIR \ 41 BB_SERVER_TIMEOUT \ 42 " 43ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory icecc" 44SDK_UPDATE_URL ?= "" 45 46SDK_TARGETS ?= "${PN}" 47 48def get_sdk_install_targets(d, images_only=False): 49 sdk_install_targets = '' 50 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal': 51 sdk_install_targets = d.getVar('SDK_TARGETS') 52 53 depd = d.getVar('BB_TASKDEPDATA', False) 54 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 55 tasklist.remove('do_build') 56 for v in depd.values(): 57 if v[1] in tasklist: 58 if v[0] not in sdk_install_targets: 59 sdk_install_targets += ' {}'.format(v[0]) 60 61 if not images_only: 62 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 63 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata' 64 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1': 65 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot' 66 67 return sdk_install_targets 68 69get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA" 70 71OE_INIT_ENV_SCRIPT ?= "oe-init-build-env" 72 73# The files from COREBASE that you want preserved in the COREBASE copied 74# into the sdk. This allows someone to have their own setup scripts in 75# COREBASE be preserved as well as untracked files. 76COREBASE_FILES ?= " \ 77 oe-init-build-env \ 78 scripts \ 79 LICENSE \ 80 .templateconf \ 81" 82 83SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext" 84B:task-populate-sdk-ext = "${SDK_DIR}" 85TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}" 86TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}" 87 88SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest" 89SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest" 90 91python write_target_sdk_ext_manifest () { 92 from oe.sdk import get_extra_sdkinfo 93 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 94 extra_info = get_extra_sdkinfo(sstate_dir) 95 96 target = d.getVar('TARGET_SYS') 97 target_multimach = d.getVar('MULTIMACH_TARGET_SYS') 98 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS') 99 100 pkgs = {} 101 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True) 102 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f: 103 for fn in extra_info['filesizes']: 104 info = fn.split(':') 105 if info[2] in (target, target_multimach, real_target_multimach) \ 106 or info[5] == 'allarch': 107 if not info[1] in pkgs: 108 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 109 pkgs[info[1]] = {} 110} 111python write_host_sdk_ext_manifest () { 112 from oe.sdk import get_extra_sdkinfo 113 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 114 extra_info = get_extra_sdkinfo(sstate_dir) 115 host = d.getVar('BUILD_SYS') 116 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f: 117 for fn in extra_info['filesizes']: 118 info = fn.split(':') 119 if info[2] == host: 120 f.write("%s %s %s\n" % (info[1], info[2], info[3])) 121} 122 123SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest write_host_sdk_ext_manifest" 124 125SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK" 126 127def clean_esdk_builddir(d, sdkbasepath): 128 """Clean up traces of the fake build for create_filtered_tasklist()""" 129 import shutil 130 cleanpaths = ['cache', 'tmp'] 131 for pth in cleanpaths: 132 fullpth = os.path.join(sdkbasepath, pth) 133 if os.path.isdir(fullpth): 134 shutil.rmtree(fullpth) 135 elif os.path.isfile(fullpth): 136 os.remove(fullpth) 137 138def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath): 139 """ 140 Create a filtered list of tasks. Also double-checks that the build system 141 within the SDK basically works and required sstate artifacts are available. 142 """ 143 import tempfile 144 import shutil 145 import oe.copy_buildsystem 146 147 # Create a temporary build directory that we can pass to the env setup script 148 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak') 149 try: 150 with open(sdkbasepath + '/conf/local.conf', 'a') as f: 151 # Force the use of sstate from the build system 152 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR')) 153 f.write('SSTATE_MIRRORS:forcevariable = "file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n') 154 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it 155 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n') 156 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will 157 # be different and we won't be able to find our native sstate) 158 if not bb.data.inherits_class('uninative', d): 159 f.write('INHERIT:remove = "uninative"\n') 160 161 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake 162 # will not allow in its COREBASE path, so we need to rename the directory temporarily 163 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk' 164 # Delete any existing temp dir 165 try: 166 shutil.rmtree(temp_sdkbasepath) 167 except FileNotFoundError: 168 pass 169 bb.utils.rename(sdkbasepath, temp_sdkbasepath) 170 cmdprefix = '. %s .; ' % conf_initpath 171 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt' 172 try: 173 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile) 174 except bb.process.ExecutionError as e: 175 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip() 176 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout: 177 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n' 178 bb.fatal(msg) 179 bb.utils.rename(temp_sdkbasepath, sdkbasepath) 180 # Clean out residue of running bitbake, which check_sstate_task_list() 181 # will effectively do 182 clean_esdk_builddir(d, sdkbasepath) 183 finally: 184 localconf = sdkbasepath + '/conf/local.conf' 185 if os.path.exists(localconf + '.bak'): 186 os.replace(localconf + '.bak', localconf) 187 188def copy_bitbake_and_layers(d, baseoutpath, derivative): 189 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT') 190 191 conf_bbpath = '' 192 conf_initpath = '' 193 core_meta_subdir = '' 194 195 # Copy in all metadata layers + bitbake (as repositories) 196 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) 197 198 if derivative: 199 workspace_name = 'orig-workspace' 200 else: 201 workspace_name = None 202 203 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name) 204 conf_bbpath = os.path.join('layers', corebase, 'bitbake') 205 206 for path in os.listdir(baseoutpath + '/layers'): 207 relpath = os.path.join('layers', path, oe_init_env_script) 208 if os.path.exists(os.path.join(baseoutpath, relpath)): 209 conf_initpath = relpath 210 211 relpath = os.path.join('layers', path, 'scripts', 'esdk-tools', 'devtool') 212 if os.path.exists(os.path.join(baseoutpath, relpath)): 213 esdk_tools_path = os.path.dirname(relpath) 214 215 relpath = os.path.join('layers', path, 'meta') 216 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')): 217 core_meta_subdir = relpath 218 219 d.setVar('oe_init_build_env_path', conf_initpath) 220 d.setVar('esdk_tools_path', esdk_tools_path) 221 222 return (conf_initpath, conf_bbpath, core_meta_subdir, sdkbblayers) 223 224def write_devtool_config(d, baseoutpath, conf_bbpath, conf_initpath, core_meta_subdir): 225 # Write out config file for devtool 226 import configparser 227 config = configparser.ConfigParser() 228 config.add_section('General') 229 config.set('General', 'bitbake_subdir', conf_bbpath) 230 config.set('General', 'init_path', conf_initpath) 231 config.set('General', 'core_meta_subdir', core_meta_subdir) 232 config.add_section('SDK') 233 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS')) 234 updateurl = d.getVar('SDK_UPDATE_URL') 235 if updateurl: 236 config.set('SDK', 'updateserver', updateurl) 237 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf')) 238 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f: 239 config.write(f) 240 241def write_unlocked_sigs(d, baseoutpath): 242 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc') 243 with open(unlockedsigs, 'w') as f: 244 pass 245 246def write_bblayers_conf(d, baseoutpath, sdkbblayers): 247 # Create a layer for new recipes / appends 248 bbpath = d.getVar('BBPATH') 249 env = os.environ.copy() 250 env['PYTHONDONTWRITEBYTECODE'] = '1' 251 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--layerseries', d.getVar("LAYERSERIES_CORENAMES"), '--create-only', os.path.join(baseoutpath, 'workspace')], env=env) 252 253 # Create bblayers.conf 254 bb.utils.mkdirhier(baseoutpath + '/conf') 255 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f: 256 f.write('# WARNING: this configuration has been automatically generated and in\n') 257 f.write('# most cases should not be edited. If you need more flexibility than\n') 258 f.write('# this configuration provides, it is strongly suggested that you set\n') 259 f.write('# up a proper instance of the full build system and use that instead.\n\n') 260 261 # LCONF_VERSION may not be set, for example when using meta-poky 262 # so don't error if it isn't found 263 lconf_version = d.getVar('LCONF_VERSION', False) 264 if lconf_version is not None: 265 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version) 266 267 f.write('BBPATH = "$' + '{TOPDIR}"\n') 268 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n') 269 f.write('BBLAYERS := " \\\n') 270 for layerrelpath in sdkbblayers: 271 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath) 272 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n') 273 f.write(' "\n') 274 275def copy_uninative(d, baseoutpath): 276 import shutil 277 278 uninative_checksum = None 279 280 # Copy uninative tarball 281 # For now this is where uninative.bbclass expects the tarball 282 if bb.data.inherits_class('uninative', d): 283 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}') 284 uninative_checksum = bb.utils.sha256_file(uninative_file) 285 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum) 286 bb.utils.mkdirhier(uninative_outdir) 287 shutil.copy(uninative_file, uninative_outdir) 288 289 return uninative_checksum 290 291def write_local_conf(d, baseoutpath, derivative, core_meta_subdir, uninative_checksum): 292 import shutil 293 294 #check if custome templateconf path is set 295 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF') 296 297 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split() 298 env_passthrough_values = {} 299 300 # Create local.conf 301 builddir = d.getVar('TOPDIR') 302 if derivative and os.path.exists(builddir + '/conf/site.conf'): 303 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf') 304 if derivative and os.path.exists(builddir + '/conf/auto.conf'): 305 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf') 306 if derivative: 307 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf') 308 else: 309 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split() 310 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split() 311 def handle_var(varname, origvalue, op, newlines): 312 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed): 313 newlines.append('# Removed original setting of %s\n' % varname) 314 return None, op, 0, True 315 else: 316 if varname in env_passthrough: 317 env_passthrough_values[varname] = origvalue 318 return origvalue, op, 0, True 319 varlist = ['[^#=+ ]*'] 320 oldlines = [] 321 if os.path.exists(builddir + '/conf/site.conf'): 322 with open(builddir + '/conf/site.conf', 'r') as f: 323 oldlines += f.readlines() 324 if os.path.exists(builddir + '/conf/auto.conf'): 325 with open(builddir + '/conf/auto.conf', 'r') as f: 326 oldlines += f.readlines() 327 if os.path.exists(builddir + '/conf/local.conf'): 328 with open(builddir + '/conf/local.conf', 'r') as f: 329 oldlines += f.readlines() 330 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) 331 332 with open(baseoutpath + '/conf/local.conf', 'w') as f: 333 f.write('# WARNING: this configuration has been automatically generated and in\n') 334 f.write('# most cases should not be edited. If you need more flexibility than\n') 335 f.write('# this configuration provides, it is strongly suggested that you set\n') 336 f.write('# up a proper instance of the full build system and use that instead.\n\n') 337 for line in newlines: 338 if line.strip() and not line.startswith('#'): 339 f.write(line) 340 # Write a newline just in case there's none at the end of the original 341 f.write('\n') 342 343 f.write('TMPDIR = "${TOPDIR}/tmp"\n') 344 f.write('DL_DIR = "${TOPDIR}/downloads"\n') 345 346 if bb.data.inherits_class('uninative', d): 347 f.write('INHERIT += "%s"\n' % 'uninative') 348 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum)) 349 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) 350 351 # Some classes are not suitable for SDK, remove them from INHERIT 352 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False)) 353 354 # Bypass the default connectivity check if any 355 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') 356 357 # This warning will come out if reverse dependencies for a task 358 # don't have sstate as well as the task itself. We already know 359 # this will be the case for the extensible sdk, so turn off the 360 # warning. 361 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n') 362 363 # Warn if the sigs in the locked-signature file don't match 364 # the sig computed from the metadata. 365 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n') 366 367 # We want to be able to set this without a full reparse 368 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n') 369 370 # Set up which tasks are ignored for run on install 371 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n') 372 373 # Hide the config information from bitbake output (since it's fixed within the SDK) 374 f.write('BUILDCFG_HEADER = ""\n\n') 375 376 # Write METADATA_REVISION 377 # Needs distro override so it can override the value set in the bbclass code (later than local.conf) 378 f.write('METADATA_REVISION:%s = "%s"\n\n' % (d.getVar('DISTRO'), d.getVar('METADATA_REVISION'))) 379 380 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n') 381 f.write('WITHIN_EXT_SDK = "1"\n\n') 382 383 # Map gcc-dependent uninative sstate cache for installer usage 384 f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n') 385 386 if d.getVar("PRSERV_HOST"): 387 # Override this, we now include PR data, so it should only point ot the local database 388 f.write('PRSERV_HOST = "localhost:0"\n\n') 389 390 # Allow additional config through sdk-extra.conf 391 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) 392 if fn: 393 with open(fn, 'r') as xf: 394 for line in xf: 395 f.write(line) 396 397 # If you define a sdk_extraconf() function then it can contain additional config 398 # (Though this is awkward; sdk-extra.conf should probably be used instead) 399 extraconf = (d.getVar('sdk_extraconf') or '').strip() 400 if extraconf: 401 # Strip off any leading / trailing spaces 402 for line in extraconf.splitlines(): 403 f.write(line.strip() + '\n') 404 405 f.write('require conf/locked-sigs.inc\n') 406 f.write('require conf/unlocked-sigs.inc\n') 407 408 # Copy multiple configurations if they exist in the users config directory 409 if d.getVar('BBMULTICONFIG') is not None: 410 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig')) 411 for mc in d.getVar('BBMULTICONFIG').split(): 412 dest_stub = "/conf/multiconfig/%s.conf" % (mc,) 413 if os.path.exists(builddir + dest_stub): 414 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub) 415 416 # If PR Service is in use, we need to export this as well 417 bb.note('Do we have a pr database?') 418 if d.getVar("PRSERV_HOST"): 419 bb.note('Writing PR database...') 420 # Based on the code in classes/prexport.bbclass 421 import oe.prservice 422 #dump meta info of tables 423 localdata = d.createCopy() 424 localdata.setVar('PRSERV_DUMPOPT_COL', "1") 425 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf')) 426 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc') 427 428 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE'))) 429 430 retval = oe.prservice.prserv_dump_db(localdata) 431 if not retval: 432 bb.error("prexport_handler: export failed!") 433 return 434 (metainfo, datainfo) = retval 435 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True) 436 437 # Use templateconf.cfg file from builddir if exists 438 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1': 439 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg') 440 else: 441 # Write a templateconf.cfg 442 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f: 443 f.write('meta/conf/templates/default\n') 444 os.makedirs(os.path.join(baseoutpath, core_meta_subdir, 'conf/templates/default'), exist_ok=True) 445 446 # Ensure any variables set from the external environment (by way of 447 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration 448 extralines = [] 449 for name, value in env_passthrough_values.items(): 450 actualvalue = d.getVar(name) or '' 451 if value != actualvalue: 452 extralines.append('%s = "%s"\n' % (name, actualvalue)) 453 if extralines: 454 with open(baseoutpath + '/conf/local.conf', 'a') as f: 455 f.write('\n') 456 f.write('# Extra settings from environment:\n') 457 for line in extralines: 458 f.write(line) 459 f.write('\n') 460 461def prepare_locked_cache(d, baseoutpath, derivative, conf_initpath): 462 import shutil 463 464 # Filter the locked signatures file to just the sstate tasks we are interested in 465 excluded_targets = get_sdk_install_targets(d, images_only=True) 466 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 467 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc' 468 #nativesdk-only sigfile to merge into locked-sigs.inc 469 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1') 470 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 471 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc' 472 473 if sdk_include_nativesdk: 474 oe.copy_buildsystem.prune_lockedsigs([], 475 excluded_targets.split(), 476 nativesigfile, 477 True, 478 nativesigfile_pruned) 479 480 oe.copy_buildsystem.merge_lockedsigs([], 481 sigfile, 482 nativesigfile_pruned, 483 sigfile) 484 485 oe.copy_buildsystem.prune_lockedsigs([], 486 excluded_targets.split(), 487 sigfile, 488 False, 489 lockedsigs_pruned) 490 491 sstate_out = baseoutpath + '/sstate-cache' 492 bb.utils.remove(sstate_out, True) 493 494 # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d) 495 fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d) if bb.data.inherits_class('uninative', d) else "" 496 497 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1') 498 sdk_ext_type = d.getVar('SDK_EXT_TYPE') 499 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk: 500 # Create the filtered task list used to generate the sstate cache shipped with the SDK 501 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt' 502 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath) 503 else: 504 tasklistfn = None 505 506 # Add packagedata if enabled 507 if d.getVar('SDK_INCLUDE_PKGDATA') == '1': 508 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc' 509 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc' 510 shutil.move(lockedsigs_pruned, lockedsigs_base) 511 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'], 512 lockedsigs_base, 513 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc', 514 lockedsigs_pruned, 515 lockedsigs_copy) 516 517 if sdk_include_toolchain: 518 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc' 519 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc") 520 shutil.move(lockedsigs_pruned, lockedsigs_base) 521 oe.copy_buildsystem.merge_lockedsigs([], 522 lockedsigs_base, 523 lockedsigs_toolchain, 524 lockedsigs_pruned) 525 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain, 526 d.getVar('SSTATE_DIR'), 527 sstate_out, d, 528 fixedlsbstring, 529 filterfile=tasklistfn) 530 531 if sdk_ext_type == 'minimal': 532 if derivative: 533 # Assume the user is not going to set up an additional sstate 534 # mirror, thus we need to copy the additional artifacts (from 535 # workspace recipes) into the derivative SDK 536 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc' 537 if os.path.exists(lockedsigs_orig): 538 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc' 539 oe.copy_buildsystem.merge_lockedsigs(None, 540 lockedsigs_orig, 541 lockedsigs_pruned, 542 None, 543 lockedsigs_extra) 544 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, 545 d.getVar('SSTATE_DIR'), 546 sstate_out, d, 547 fixedlsbstring, 548 filterfile=tasklistfn) 549 else: 550 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, 551 d.getVar('SSTATE_DIR'), 552 sstate_out, d, 553 fixedlsbstring, 554 filterfile=tasklistfn) 555 556 # We don't need sstate do_package files 557 for root, dirs, files in os.walk(sstate_out): 558 for name in files: 559 if name.endswith("_package.tar.zst"): 560 f = os.path.join(root, name) 561 os.remove(f) 562 563def write_manifest(d, baseoutpath): 564 import glob 565 566 # Write manifest file 567 # Note: at the moment we cannot include the env setup script here to keep 568 # it updated, since it gets modified during SDK installation (see 569 # sdk_ext_postinst() below) thus the checksum we take here would always 570 # be different. 571 manifest_file_list = ['conf/*'] 572 if d.getVar('BBMULTICONFIG') is not None: 573 manifest_file_list.append('conf/multiconfig/*') 574 575 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split() 576 esdk_manifest_excludes_list = [] 577 for exclude_item in esdk_manifest_excludes: 578 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item)) 579 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest') 580 with open(manifest_file, 'w') as f: 581 for item in manifest_file_list: 582 for fn in glob.glob(os.path.join(baseoutpath, item)): 583 if fn == manifest_file or os.path.isdir(fn): 584 continue 585 if fn in esdk_manifest_excludes_list: 586 continue 587 chksum = bb.utils.sha256_file(fn) 588 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath))) 589 590 591python copy_buildsystem () { 592 import oe.copy_buildsystem 593 594 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH') 595 596 # Determine if we're building a derivative extensible SDK (from devtool build-sdk) 597 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1' 598 599 conf_initpath, conf_bbpath, core_meta_subdir, sdkbblayers = copy_bitbake_and_layers(d, baseoutpath, derivative) 600 601 write_devtool_config(d, baseoutpath, conf_bbpath, conf_initpath, core_meta_subdir) 602 603 write_unlocked_sigs(d, baseoutpath) 604 605 write_bblayers_conf(d, baseoutpath, sdkbblayers) 606 607 uninative_checksum = copy_uninative(d, baseoutpath) 608 609 write_local_conf(d, baseoutpath, derivative, core_meta_subdir, uninative_checksum) 610 611 prepare_locked_cache(d, baseoutpath, derivative, conf_initpath) 612 613 write_manifest(d, baseoutpath) 614 615} 616 617def get_current_buildtools(d): 618 """Get the file name of the current buildtools installer""" 619 import glob 620 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh')) 621 btfiles.sort(key=os.path.getctime) 622 return os.path.basename(btfiles[-1]) 623 624def get_sdk_required_utilities(buildtools_fn, d): 625 """Find required utilities that aren't provided by the buildtools""" 626 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split() 627 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc')) 628 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++')) 629 if buildtools_fn: 630 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn) 631 filelist, _ = bb.process.run('%s -l' % buildtools_installer) 632 else: 633 buildtools_installer = None 634 filelist = "" 635 localdata = bb.data.createCopy(d) 636 localdata.setVar('SDKPATH', '.') 637 sdkpathnative = localdata.getVar('SDKPATHNATIVE') 638 sdkbindirs = [localdata.getVar('bindir_nativesdk'), 639 localdata.getVar('sbindir_nativesdk'), 640 localdata.getVar('base_bindir_nativesdk'), 641 localdata.getVar('base_sbindir_nativesdk')] 642 for line in filelist.splitlines(): 643 splitline = line.split() 644 if len(splitline) > 5: 645 fn = splitline[5] 646 if not fn.startswith('./'): 647 fn = './%s' % fn 648 if fn.startswith(sdkpathnative): 649 relpth = '/' + os.path.relpath(fn, sdkpathnative) 650 for bindir in sdkbindirs: 651 if relpth.startswith(bindir): 652 relpth = os.path.relpath(relpth, bindir) 653 if relpth in sanity_required_utilities: 654 sanity_required_utilities.remove(relpth) 655 break 656 return ' '.join(sanity_required_utilities) 657 658install_tools() { 659 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase 660 661 # find latest buildtools-tarball and install it 662 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 663 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH} 664 fi 665 666 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH} 667} 668do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True" 669 670sdk_ext_preinst() { 671 # Since bitbake won't run as root it doesn't make sense to try and install 672 # the extensible sdk as root. 673 if [ "`id -u`" = "0" ]; then 674 echo "ERROR: The extensible sdk cannot be installed as root." 675 exit 1 676 fi 677 if ! command -v locale > /dev/null; then 678 echo "ERROR: The installer requires the locale command, please install it first" 679 exit 1 680 fi 681 # Check setting of LC_ALL set above 682 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'` 683 if ! locale -a | grep -q $canonicalised_locale ; then 684 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first" 685 exit 1 686 fi 687 # The relocation script used by buildtools installer requires python 688 if ! command -v python3 > /dev/null; then 689 echo "ERROR: The installer requires python3, please install it first" 690 exit 1 691 fi 692 missing_utils="" 693 for util in ${SDK_REQUIRED_UTILITIES}; do 694 if ! command -v $util > /dev/null; then 695 missing_utils="$missing_utils $util" 696 fi 697 done 698 if [ -n "$missing_utils" ] ; then 699 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils" 700 exit 1 701 fi 702 SDK_EXTENSIBLE="1" 703 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then 704 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache" 705 fi 706} 707SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}" 708 709# FIXME this preparation should be done as part of the SDK construction 710sdk_ext_postinst() { 711 printf "\nExtracting buildtools...\n" 712 cd $target_sdk_dir 713 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}" 714 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 715 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 716 717 # Delete the buildtools tar file since it won't be used again 718 rm -f ./${SDK_BUILDTOOLS_INSTALLER} 719 # We don't need the log either since it succeeded 720 rm -f buildtools.log 721 722 # Make sure when the user sets up the environment, they also get 723 # the buildtools-tarball tools in their path. 724 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script 725 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script 726 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script 727 echo "export OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script 728 fi 729 730 # Allow bitbake environment setup to be ran as part of this sdk. 731 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script 732 # Work around runqemu not knowing how to get this information within the eSDK 733 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script 734 735 # A bit of another hack, but we need this in the path only for devtool 736 # so put it at the end of $PATH. 737 echo "export PATH=\"$target_sdk_dir/${esdk_tools_path}:\$PATH\"" >> $env_setup_script 738 739 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script 740 741 # Warn if trying to use external bitbake and the ext SDK together 742 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script 743 744 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then 745 printf "Preparing build system...\n" 746 # dash which is /bin/sh on Ubuntu will not preserve the 747 # current working directory when first ran, nor will it set $1 when 748 # sourcing a script. That is why this has to look so ugly. 749 LOGFILE="$target_sdk_dir/preparing_build_system.log" 750 sh -c ". buildtools/environment-setup* > $LOGFILE 2>&1 && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE 2>&1 && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; } 751 fi 752 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then 753 rm $target_sdk_dir/ext-sdk-prepare.py 754 fi 755 echo done 756} 757 758SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}" 759 760SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem install_tools " 761 762SDK_INSTALL_TARGETS = "" 763fakeroot python do_populate_sdk_ext() { 764 # FIXME hopefully we can remove this restriction at some point, but uninative 765 # currently forces this upon us 766 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'): 767 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH'))) 768 769 # FIXME hopefully we can remove this restriction at some point, but the eSDK 770 # can only be built for the primary (default) multiconfig 771 if d.getVar('BB_CURRENT_MC') != '': 772 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC')) 773 774 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set 775 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}") 776 d.setVar("TOOLCHAIN_TARGET_TASK", "") 777 778 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d)) 779 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1': 780 buildtools_fn = get_current_buildtools(d) 781 else: 782 buildtools_fn = None 783 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d)) 784 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn) 785 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}') 786 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice 787 d.delVar('SDKIMAGE_LINGUAS') 788 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1': 789 generate_nativesdk_lockedsigs(d) 790 populate_sdk_common(d) 791} 792 793def generate_nativesdk_lockedsigs(d): 794 import oe.copy_buildsystem 795 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc' 796 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 797 798def get_ext_sdk_depends(d): 799 # Note: the deps varflag is a list not a string, so we need to specify expand=False 800 deps = d.getVarFlag('do_image_complete', 'deps', False) 801 pn = d.getVar('PN') 802 deplist = ['%s:%s' % (pn, dep) for dep in deps] 803 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d) 804 tasklist.append('do_rootfs') 805 for task in tasklist: 806 deplist.extend((d.getVarFlag(task, 'depends') or '').split()) 807 return ' '.join(deplist) 808 809python do_sdk_depends() { 810 # We have to do this separately in its own task so we avoid recursing into 811 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those 812 # into the SDK's sstate-cache 813 import oe.copy_buildsystem 814 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc' 815 oe.copy_buildsystem.generate_locked_sigs(sigfile, d) 816} 817addtask sdk_depends 818 819do_sdk_depends[dirs] = "${WORKDIR}" 820do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot" 821do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}" 822do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}" 823do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}" 824 825do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}" 826 827do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \ 828 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \ 829 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \ 830 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}" 831 832# We must avoid depending on do_build here if rm_work.bbclass is active, 833# because otherwise do_rm_work may run before do_populate_sdk_ext itself. 834# We can't mark do_populate_sdk_ext and do_sdk_depends as having to 835# run before do_rm_work, because then they would also run as part 836# of normal builds. 837do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}" 838 839# Make sure code changes can result in rebuild 840do_populate_sdk_ext[vardeps] += "copy_buildsystem \ 841 sdk_ext_postinst" 842 843# Since any change in the metadata of any layer should cause a rebuild of the 844# sdk(since the layers are put in the sdk) set the task to nostamp so it 845# always runs. 846do_populate_sdk_ext[nostamp] = "1" 847 848SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext" 849 850SSTATETASKS += "do_populate_sdk_ext" 851SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1' 852do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}" 853do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}" 854do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}" 855do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}" 856 857addtask populate_sdk_ext after do_sdk_depends 858