1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.npm_dev: 149 extracmdopts += ' --npm-dev' 150 if args.no_pypi: 151 extracmdopts += ' --no-pypi' 152 if args.mirrors: 153 extracmdopts += ' --mirrors' 154 if args.srcrev: 155 extracmdopts += ' --srcrev %s' % args.srcrev 156 if args.srcbranch: 157 extracmdopts += ' --srcbranch %s' % args.srcbranch 158 if args.provides: 159 extracmdopts += ' --provides %s' % args.provides 160 161 tempdir = tempfile.mkdtemp(prefix='devtool') 162 try: 163 try: 164 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 165 except bb.process.ExecutionError as e: 166 if e.exitcode == 15: 167 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 168 else: 169 raise DevtoolError('Command \'%s\' failed' % e.command) 170 171 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 172 if recipes: 173 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 174 if recipename in workspace: 175 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 176 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 177 bb.utils.mkdirhier(recipedir) 178 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 179 appendfile = recipe_to_append(recipefile, config) 180 if os.path.exists(appendfile): 181 # This shouldn't be possible, but just in case 182 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 183 if os.path.exists(recipefile): 184 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 185 if tmpsrcdir: 186 srctree = os.path.join(srctreeparent, recipename) 187 if os.path.exists(tmpsrcdir): 188 if os.path.exists(srctree): 189 if os.path.isdir(srctree): 190 try: 191 os.rmdir(srctree) 192 except OSError as e: 193 if e.errno == errno.ENOTEMPTY: 194 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 195 else: 196 raise 197 else: 198 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 199 logger.info('Using default source tree path %s' % srctree) 200 shutil.move(tmpsrcdir, srctree) 201 else: 202 raise DevtoolError('Couldn\'t find source tree created by recipetool') 203 bb.utils.mkdirhier(recipedir) 204 shutil.move(recipes[0], recipefile) 205 # Move any additional files created by recipetool 206 for fn in os.listdir(tempdir): 207 shutil.move(os.path.join(tempdir, fn), recipedir) 208 else: 209 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 210 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 211 if os.path.exists(attic_recipe): 212 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 213 finally: 214 if tmpsrcdir and os.path.exists(tmpsrcdir): 215 shutil.rmtree(tmpsrcdir) 216 shutil.rmtree(tempdir) 217 218 for fn in os.listdir(recipedir): 219 _add_md5(config, recipename, os.path.join(recipedir, fn)) 220 221 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 222 try: 223 try: 224 rd = tinfoil.parse_recipe_file(recipefile, False) 225 except Exception as e: 226 logger.error(str(e)) 227 rd = None 228 if not rd: 229 # Parsing failed. We just created this recipe and we shouldn't 230 # leave it in the workdir or it'll prevent bitbake from starting 231 movefn = '%s.parsefailed' % recipefile 232 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 233 shutil.move(recipefile, movefn) 234 return 1 235 236 if args.fetchuri and not args.no_git: 237 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 238 239 initial_rev = {} 240 if os.path.exists(os.path.join(srctree, '.git')): 241 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 242 initial_rev["."] = stdout.rstrip() 243 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree) 244 for line in stdout.splitlines(): 245 (rev, submodule) = line.split() 246 initial_rev[os.path.relpath(submodule, srctree)] = rev 247 248 if args.src_subdir: 249 srctree = os.path.join(srctree, args.src_subdir) 250 251 bb.utils.mkdirhier(os.path.dirname(appendfile)) 252 with open(appendfile, 'w') as f: 253 f.write('inherit externalsrc\n') 254 f.write('EXTERNALSRC = "%s"\n' % srctree) 255 256 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 257 if b_is_s: 258 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 259 if initial_rev: 260 for key, value in initial_rev.items(): 261 f.write('\n# initial_rev %s: %s\n' % (key, value)) 262 263 if args.binary: 264 f.write('do_install:append() {\n') 265 f.write(' rm -rf ${D}/.git\n') 266 f.write(' rm -f ${D}/singletask.lock\n') 267 f.write('}\n') 268 269 if bb.data.inherits_class('npm', rd): 270 f.write('python do_configure:append() {\n') 271 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 272 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 273 f.write(' bb.utils.remove(lockfile)\n') 274 f.write('}\n') 275 276 # Check if the new layer provides recipes whose priorities have been 277 # overriden by PREFERRED_PROVIDER. 278 recipe_name = rd.getVar('PN') 279 provides = rd.getVar('PROVIDES') 280 # Search every item defined in PROVIDES 281 for recipe_provided in provides.split(): 282 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 283 current_pprovider = rd.getVar(preferred_provider) 284 if current_pprovider and current_pprovider != recipe_name: 285 if args.fixed_setup: 286 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 287 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 288 with open(layerconf_file, 'a') as f: 289 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 290 else: 291 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 292 break 293 294 _add_md5(config, recipename, appendfile) 295 296 check_prerelease_version(rd.getVar('PV'), 'devtool add') 297 298 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 299 300 finally: 301 tinfoil.shutdown() 302 303 return 0 304 305 306def _check_compatible_recipe(pn, d): 307 """Check if the recipe is supported by devtool""" 308 if pn == 'perf': 309 raise DevtoolError("The perf recipe does not actually check out " 310 "source and thus cannot be supported by this tool", 311 4) 312 313 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 314 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 315 316 if bb.data.inherits_class('image', d): 317 raise DevtoolError("The %s recipe is an image, and therefore is not " 318 "supported by this tool" % pn, 4) 319 320 if bb.data.inherits_class('populate_sdk', d): 321 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 322 "supported by this tool" % pn, 4) 323 324 if bb.data.inherits_class('packagegroup', d): 325 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 326 "not supported by this tool" % pn, 4) 327 328 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 329 # Not an incompatibility error per se, so we don't pass the error code 330 raise DevtoolError("externalsrc is currently enabled for the %s " 331 "recipe. This prevents the normal do_patch task " 332 "from working. You will need to disable this " 333 "first." % pn) 334 335def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 336 """Common function for copying a file to the dry run output directory""" 337 relpath = os.path.relpath(dst, base_outdir) 338 if relpath.startswith('..'): 339 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 340 dst = os.path.join(dry_run_outdir, relpath) 341 dst_d = os.path.dirname(dst) 342 if dst_d: 343 bb.utils.mkdirhier(dst_d) 344 # Don't overwrite existing files, otherwise in the case of an upgrade 345 # the dry-run written out recipe will be overwritten with an unmodified 346 # version 347 if not os.path.exists(dst): 348 shutil.copy(src, dst) 349 350def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 351 """Move a file. Creates all the directory components of destination path.""" 352 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 353 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 354 if dry_run_outdir: 355 # We want to copy here, not move 356 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 357 else: 358 dst_d = os.path.dirname(dst) 359 if dst_d: 360 bb.utils.mkdirhier(dst_d) 361 shutil.move(src, dst) 362 363def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 364 """Copy a file. Creates all the directory components of destination path.""" 365 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 366 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 367 if dry_run_outdir: 368 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 369 else: 370 dst_d = os.path.dirname(dst) 371 if dst_d: 372 bb.utils.mkdirhier(dst_d) 373 shutil.copy(src, dst) 374 375def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 376 """List contents of a git treeish""" 377 import bb 378 cmd = ['git', 'ls-tree', '-z', treeish] 379 if recursive: 380 cmd.append('-r') 381 out, _ = bb.process.run(cmd, cwd=repodir) 382 ret = {} 383 if out: 384 for line in out.split('\0'): 385 if line: 386 split = line.split(None, 4) 387 ret[split[3]] = split[0:3] 388 return ret 389 390def _git_exclude_path(srctree, path): 391 """Return pathspec (list of paths) that excludes certain path""" 392 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 393 # we don't catch files that are deleted, for example. A more reliable way 394 # to implement this would be to use "negative pathspecs" which were 395 # introduced in Git v1.9.0. Revisit this when/if the required Git version 396 # becomes greater than that. 397 path = os.path.normpath(path) 398 recurse = True if len(path.split(os.path.sep)) > 1 else False 399 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 400 if path in git_files: 401 git_files.remove(path) 402 return git_files 403 else: 404 return ['.'] 405 406def _ls_tree(directory): 407 """Recursive listing of files in a directory""" 408 ret = [] 409 for root, dirs, files in os.walk(directory): 410 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 411 fname in files]) 412 return ret 413 414 415def extract(args, config, basepath, workspace): 416 """Entry point for the devtool 'extract' subcommand""" 417 import bb 418 419 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 420 if not tinfoil: 421 # Error already shown 422 return 1 423 try: 424 rd = parse_recipe(config, tinfoil, args.recipename, True) 425 if not rd: 426 return 1 427 428 srctree = os.path.abspath(args.srctree) 429 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 430 logger.info('Source tree extracted to %s' % srctree) 431 432 if initial_rev: 433 return 0 434 else: 435 return 1 436 finally: 437 tinfoil.shutdown() 438 439def sync(args, config, basepath, workspace): 440 """Entry point for the devtool 'sync' subcommand""" 441 import bb 442 443 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 444 if not tinfoil: 445 # Error already shown 446 return 1 447 try: 448 rd = parse_recipe(config, tinfoil, args.recipename, True) 449 if not rd: 450 return 1 451 452 srctree = os.path.abspath(args.srctree) 453 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 454 logger.info('Source tree %s synchronized' % srctree) 455 456 if initial_rev: 457 return 0 458 else: 459 return 1 460 finally: 461 tinfoil.shutdown() 462 463def symlink_oelocal_files_srctree(rd, srctree): 464 import oe.patch 465 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 466 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 467 # (otherwise the recipe won't build as expected) 468 local_files_dir = os.path.join(srctree, 'oe-local-files') 469 addfiles = [] 470 for root, _, files in os.walk(local_files_dir): 471 relpth = os.path.relpath(root, local_files_dir) 472 if relpth != '.': 473 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 474 for fn in files: 475 if fn == '.gitignore': 476 continue 477 destpth = os.path.join(srctree, relpth, fn) 478 if os.path.exists(destpth): 479 os.unlink(destpth) 480 if relpth != '.': 481 back_relpth = os.path.relpath(local_files_dir, root) 482 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) 483 else: 484 os.symlink('oe-local-files/%s' % fn, destpth) 485 addfiles.append(os.path.join(relpth, fn)) 486 if addfiles: 487 oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd) 488 489def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 490 """Extract sources of a recipe""" 491 import oe.recipeutils 492 import oe.patch 493 import oe.path 494 495 pn = d.getVar('PN') 496 497 _check_compatible_recipe(pn, d) 498 499 if sync: 500 if not os.path.exists(srctree): 501 raise DevtoolError("output path %s does not exist" % srctree) 502 else: 503 if os.path.exists(srctree): 504 if not os.path.isdir(srctree): 505 raise DevtoolError("output path %s exists and is not a directory" % 506 srctree) 507 elif os.listdir(srctree): 508 raise DevtoolError("output path %s already exists and is " 509 "non-empty" % srctree) 510 511 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 512 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 513 "extract source" % pn, 4) 514 515 if not sync: 516 # Prepare for shutil.move later on 517 bb.utils.mkdirhier(srctree) 518 os.rmdir(srctree) 519 520 extra_overrides = [] 521 if not no_overrides: 522 history = d.varhistory.variable('SRC_URI') 523 for event in history: 524 if not 'flag' in event: 525 if event['op'].startswith((':append[', ':prepend[')): 526 override = event['op'].split('[')[1].split(']')[0] 527 if not override.startswith('pn-'): 528 extra_overrides.append(override) 529 # We want to remove duplicate overrides. If a recipe had multiple 530 # SRC_URI_override += values it would cause mulitple instances of 531 # overrides. This doesn't play nicely with things like creating a 532 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 533 extra_overrides = list(set(extra_overrides)) 534 if extra_overrides: 535 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 536 537 initial_rev = None 538 539 recipefile = d.getVar('FILE') 540 appendfile = recipe_to_append(recipefile, config) 541 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 542 543 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 544 # directory so that: 545 # (a) we pick up all files that get unpacked to the WORKDIR, and 546 # (b) we don't disturb the existing build 547 # However, with recipe-specific sysroots the sysroots for the recipe 548 # will be prepared under WORKDIR, and if we used the system temporary 549 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 550 # our attempts to hardlink files into the recipe-specific sysroots 551 # will fail on systems where /tmp is a different filesystem, and it 552 # would have to fall back to copying the files which is a waste of 553 # time. Put the temp directory under the WORKDIR to prevent that from 554 # being a problem. 555 tempbasedir = d.getVar('WORKDIR') 556 bb.utils.mkdirhier(tempbasedir) 557 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 558 try: 559 tinfoil.logger.setLevel(logging.WARNING) 560 561 # FIXME this results in a cache reload under control of tinfoil, which is fine 562 # except we don't get the knotty progress bar 563 564 if os.path.exists(appendfile): 565 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 566 shutil.copyfile(appendfile, appendbackup) 567 else: 568 appendbackup = None 569 bb.utils.mkdirhier(os.path.dirname(appendfile)) 570 logger.debug('writing append file %s' % appendfile) 571 with open(appendfile, 'a') as f: 572 f.write('###--- _extract_source\n') 573 f.write('deltask do_recipe_qa\n') 574 f.write('deltask do_recipe_qa_setscene\n') 575 f.write('ERROR_QA:remove = "patch-fuzz"\n') 576 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 577 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 578 if not is_kernel_yocto: 579 f.write('PATCHTOOL = "git"\n') 580 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 581 if extra_overrides: 582 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 583 f.write('inherit devtool-source\n') 584 f.write('###--- _extract_source\n') 585 586 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 587 588 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 589 bb.utils.mkdirhier(sstate_manifests) 590 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 591 with open(preservestampfile, 'w') as f: 592 f.write(d.getVar('STAMP')) 593 tinfoil.modified_files() 594 try: 595 if is_kernel_yocto: 596 # We need to generate the kernel config 597 task = 'do_configure' 598 else: 599 task = 'do_patch' 600 601 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 602 logger.info('The %s recipe has %s disabled. Running only ' 603 'do_configure task dependencies' % (pn, task)) 604 605 if 'depends' in d.getVarFlags('do_configure', False): 606 pn = d.getVarFlags('do_configure', False)['depends'] 607 pn = pn.replace('${PV}', d.getVar('PV')) 608 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 609 task = None 610 611 # Run the fetch + unpack tasks 612 res = tinfoil.build_targets(pn, 613 task, 614 handle_events=True) 615 finally: 616 if os.path.exists(preservestampfile): 617 os.remove(preservestampfile) 618 619 if not res: 620 raise DevtoolError('Extracting source for %s failed' % pn) 621 622 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 623 workshareddir = d.getVar('S') 624 if os.path.islink(srctree): 625 os.unlink(srctree) 626 627 os.symlink(workshareddir, srctree) 628 629 # The initial_rev file is created in devtool_post_unpack function that will not be executed if 630 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 631 return True, True 632 633 try: 634 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 635 initial_rev = f.read() 636 637 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 638 srcsubdir = f.read() 639 except FileNotFoundError as e: 640 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 641 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 642 643 # Check if work-shared is empty, if yes 644 # find source and copy to work-shared 645 if is_kernel_yocto: 646 workshareddir = d.getVar('STAGING_KERNEL_DIR') 647 staging_kerVer = get_staging_kver(workshareddir) 648 kernelVersion = d.getVar('LINUX_VERSION') 649 650 # handle dangling symbolic link in work-shared: 651 if os.path.islink(workshareddir): 652 os.unlink(workshareddir) 653 654 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 655 shutil.rmtree(workshareddir) 656 oe.path.copyhardlinktree(srcsubdir, workshareddir) 657 elif not os.path.exists(workshareddir): 658 oe.path.copyhardlinktree(srcsubdir, workshareddir) 659 660 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 661 srctree_localdir = os.path.join(srctree, 'oe-local-files') 662 663 if sync: 664 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 665 666 # Move the oe-local-files directory to srctree. 667 # As oe-local-files is not part of the constructed git tree, 668 # removing it directly during the synchronization might surprise 669 # the user. Instead, we move it to oe-local-files.bak and remind 670 # the user in the log message. 671 if os.path.exists(srctree_localdir + '.bak'): 672 shutil.rmtree(srctree_localdir + '.bak') 673 674 if os.path.exists(srctree_localdir): 675 logger.info('Backing up current local file directory %s' % srctree_localdir) 676 shutil.move(srctree_localdir, srctree_localdir + '.bak') 677 678 if os.path.exists(tempdir_localdir): 679 logger.info('Syncing local source files to srctree...') 680 shutil.copytree(tempdir_localdir, srctree_localdir) 681 else: 682 # Move oe-local-files directory to srctree 683 if os.path.exists(tempdir_localdir): 684 logger.info('Adding local source files to srctree...') 685 shutil.move(tempdir_localdir, srcsubdir) 686 687 shutil.move(srcsubdir, srctree) 688 symlink_oelocal_files_srctree(d, srctree) 689 690 if is_kernel_yocto: 691 logger.info('Copying kernel config to srctree') 692 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 693 694 finally: 695 if appendbackup: 696 shutil.copyfile(appendbackup, appendfile) 697 elif os.path.exists(appendfile): 698 os.remove(appendfile) 699 if keep_temp: 700 logger.info('Preserving temporary directory %s' % tempdir) 701 else: 702 shutil.rmtree(tempdir) 703 return initial_rev, srcsubdir_rel 704 705def _add_md5(config, recipename, filename): 706 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 707 import bb.utils 708 709 def addfile(fn): 710 md5 = bb.utils.md5_file(fn) 711 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 712 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 713 f.seek(0, os.SEEK_SET) 714 if not md5_str in f.read(): 715 f.write(md5_str) 716 717 if os.path.isdir(filename): 718 for root, _, files in os.walk(filename): 719 for f in files: 720 addfile(os.path.join(root, f)) 721 else: 722 addfile(filename) 723 724def _check_preserve(config, recipename): 725 """Check if a file was manually changed and needs to be saved in 'attic' 726 directory""" 727 import bb.utils 728 origfile = os.path.join(config.workspace_path, '.devtool_md5') 729 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 730 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 731 with open(origfile, 'r') as f: 732 with open(newfile, 'w') as tf: 733 for line in f.readlines(): 734 splitline = line.rstrip().split('|') 735 if splitline[0] == recipename: 736 removefile = os.path.join(config.workspace_path, splitline[1]) 737 try: 738 md5 = bb.utils.md5_file(removefile) 739 except IOError as err: 740 if err.errno == 2: 741 # File no longer exists, skip it 742 continue 743 else: 744 raise 745 if splitline[2] != md5: 746 bb.utils.mkdirhier(preservepath) 747 preservefile = os.path.basename(removefile) 748 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 749 shutil.move(removefile, os.path.join(preservepath, preservefile)) 750 else: 751 os.remove(removefile) 752 else: 753 tf.write(line) 754 bb.utils.rename(newfile, origfile) 755 756def get_staging_kver(srcdir): 757 # Kernel version from work-shared 758 kerver = [] 759 staging_kerVer="" 760 if os.path.exists(srcdir) and os.listdir(srcdir): 761 with open(os.path.join(srcdir, "Makefile")) as f: 762 version = [next(f) for x in range(5)][1:4] 763 for word in version: 764 kerver.append(word.split('= ')[1].split('\n')[0]) 765 staging_kerVer = ".".join(kerver) 766 return staging_kerVer 767 768def get_staging_kbranch(srcdir): 769 staging_kbranch = "" 770 if os.path.exists(srcdir) and os.listdir(srcdir): 771 (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir) 772 staging_kbranch = "".join(branch.split('\n')[0]) 773 return staging_kbranch 774 775def get_real_srctree(srctree, s, workdir): 776 # Check that recipe isn't using a shared workdir 777 s = os.path.abspath(s) 778 workdir = os.path.abspath(workdir) 779 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 780 # Handle if S is set to a subdirectory of the source 781 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 782 srctree = os.path.join(srctree, srcsubdir) 783 return srctree 784 785def modify(args, config, basepath, workspace): 786 """Entry point for the devtool 'modify' subcommand""" 787 import bb 788 import oe.recipeutils 789 import oe.patch 790 import oe.path 791 792 if args.recipename in workspace: 793 raise DevtoolError("recipe %s is already in your workspace" % 794 args.recipename) 795 796 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 797 try: 798 rd = parse_recipe(config, tinfoil, args.recipename, True) 799 if not rd: 800 return 1 801 802 pn = rd.getVar('PN') 803 if pn != args.recipename: 804 logger.info('Mapping %s to %s' % (args.recipename, pn)) 805 if pn in workspace: 806 raise DevtoolError("recipe %s is already in your workspace" % 807 pn) 808 809 if args.srctree: 810 srctree = os.path.abspath(args.srctree) 811 else: 812 srctree = get_default_srctree(config, pn) 813 814 if args.no_extract and not os.path.isdir(srctree): 815 raise DevtoolError("--no-extract specified and source path %s does " 816 "not exist or is not a directory" % 817 srctree) 818 819 recipefile = rd.getVar('FILE') 820 appendfile = recipe_to_append(recipefile, config, args.wildcard) 821 if os.path.exists(appendfile): 822 raise DevtoolError("Another variant of recipe %s is already in your " 823 "workspace (only one variant of a recipe can " 824 "currently be worked on at once)" 825 % pn) 826 827 _check_compatible_recipe(pn, rd) 828 829 initial_revs = {} 830 commits = {} 831 check_commits = False 832 833 if bb.data.inherits_class('kernel-yocto', rd): 834 # Current set kernel version 835 kernelVersion = rd.getVar('LINUX_VERSION') 836 srcdir = rd.getVar('STAGING_KERNEL_DIR') 837 kbranch = rd.getVar('KBRANCH') 838 839 staging_kerVer = get_staging_kver(srcdir) 840 staging_kbranch = get_staging_kbranch(srcdir) 841 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 842 oe.path.copyhardlinktree(srcdir, srctree) 843 workdir = rd.getVar('WORKDIR') 844 srcsubdir = rd.getVar('S') 845 localfilesdir = os.path.join(srctree, 'oe-local-files') 846 # Move local source files into separate subdir 847 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 848 local_files = oe.recipeutils.get_recipe_local_files(rd) 849 850 for key in local_files.copy(): 851 if key.endswith('scc'): 852 sccfile = open(local_files[key], 'r') 853 for l in sccfile: 854 line = l.split() 855 if line and line[0] in ('kconf', 'patch'): 856 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 857 if not cfg in local_files.values(): 858 local_files[line[-1]] = cfg 859 shutil.copy2(cfg, workdir) 860 sccfile.close() 861 862 # Ignore local files with subdir={BP} 863 srcabspath = os.path.abspath(srcsubdir) 864 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 865 if local_files: 866 for fname in local_files: 867 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 868 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 869 f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n') 870 871 symlink_oelocal_files_srctree(rd, srctree) 872 873 task = 'do_configure' 874 res = tinfoil.build_targets(pn, task, handle_events=True) 875 876 # Copy .config to workspace 877 kconfpath = rd.getVar('B') 878 logger.info('Copying kernel config to workspace') 879 shutil.copy2(os.path.join(kconfpath, '.config'), srctree) 880 881 # Set this to true, we still need to get initial_rev 882 # by parsing the git repo 883 args.no_extract = True 884 885 if not args.no_extract: 886 initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 887 if not initial_revs["."]: 888 return 1 889 logger.info('Source tree extracted to %s' % srctree) 890 891 if os.path.exists(os.path.join(srctree, '.git')): 892 # Get list of commits since this revision 893 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) 894 commits["."] = stdout.split() 895 check_commits = True 896 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) 897 for line in stdout.splitlines(): 898 (rev, submodule_path) = line.split() 899 submodule = os.path.relpath(submodule_path, srctree) 900 initial_revs[submodule] = rev 901 (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path) 902 commits[submodule] = stdout.split() 903 else: 904 if os.path.exists(os.path.join(srctree, '.git')): 905 # Check if it's a tree previously extracted by us. This is done 906 # by ensuring that devtool-base and args.branch (devtool) exist. 907 # The check_commits logic will cause an exception if either one 908 # of these doesn't exist 909 try: 910 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 911 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 912 except bb.process.ExecutionError: 913 stdout = '' 914 if stdout: 915 check_commits = True 916 for line in stdout.splitlines(): 917 if line.startswith('*'): 918 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 919 initial_revs["."] = stdout.rstrip() 920 if "." not in initial_revs: 921 # Otherwise, just grab the head revision 922 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 923 initial_revs["."] = stdout.rstrip() 924 925 branch_patches = {} 926 if check_commits: 927 # Check if there are override branches 928 (stdout, _) = bb.process.run('git branch', cwd=srctree) 929 branches = [] 930 for line in stdout.rstrip().splitlines(): 931 branchname = line[2:].rstrip() 932 if branchname.startswith(override_branch_prefix): 933 branches.append(branchname) 934 if branches: 935 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 936 branches.insert(0, args.branch) 937 seen_patches = [] 938 for branch in branches: 939 branch_patches[branch] = [] 940 (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree) 941 for sha1 in stdout.splitlines(): 942 notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip()) 943 origpatch = notes.get(oe.patch.GitApplyTree.original_patch) 944 if origpatch and origpatch not in seen_patches: 945 seen_patches.append(origpatch) 946 branch_patches[branch].append(origpatch) 947 948 # Need to grab this here in case the source is within a subdirectory 949 srctreebase = srctree 950 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 951 952 bb.utils.mkdirhier(os.path.dirname(appendfile)) 953 with open(appendfile, 'w') as f: 954 # if not present, add type=git-dependency to the secondary sources 955 # (non local files) so they can be extracted correctly when building a recipe after 956 # doing a devtool modify on it 957 src_uri = rd.getVar('SRC_URI').split() 958 src_uri_append = [] 959 src_uri_remove = [] 960 961 # Assume first entry is main source extracted in ${S} so skip it 962 src_uri = src_uri[1::] 963 964 # Add "type=git-dependency" to all non local sources 965 for url in src_uri: 966 if not url.startswith('file://') and not 'type=' in url: 967 src_uri_remove.append(url) 968 src_uri_append.append('%s;type=git-dependency' % url) 969 970 if src_uri_remove: 971 f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove)) 972 f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append)) 973 974 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 975 # Local files can be modified/tracked in separate subdir under srctree 976 # Mostly useful for packages with S != WORKDIR 977 f.write('FILESPATH:prepend := "%s:"\n' % 978 os.path.join(srctreebase, 'oe-local-files')) 979 f.write('# srctreebase: %s\n' % srctreebase) 980 981 f.write('\ninherit externalsrc\n') 982 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 983 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 984 985 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 986 if b_is_s: 987 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 988 989 if bb.data.inherits_class('kernel', rd): 990 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 991 'do_fetch do_unpack do_kernel_configcheck"\n') 992 f.write('\ndo_patch[noexec] = "1"\n') 993 f.write('\ndo_configure:append() {\n' 994 ' cp ${B}/.config ${S}/.config.baseline\n' 995 ' ln -sfT ${B}/.config ${S}/.config.new\n' 996 '}\n') 997 f.write('\ndo_kernel_configme:prepend() {\n' 998 ' if [ -e ${S}/.config ]; then\n' 999 ' mv ${S}/.config ${S}/.config.old\n' 1000 ' fi\n' 1001 '}\n') 1002 if rd.getVarFlag('do_menuconfig', 'task'): 1003 f.write('\ndo_configure:append() {\n' 1004 ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n' 1005 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' 1006 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' 1007 ' fi\n' 1008 '}\n') 1009 if initial_revs: 1010 for name, rev in initial_revs.items(): 1011 f.write('\n# initial_rev %s: %s\n' % (name, rev)) 1012 if name in commits: 1013 for commit in commits[name]: 1014 f.write('# commit %s: %s\n' % (name, commit)) 1015 if branch_patches: 1016 for branch in branch_patches: 1017 if branch == args.branch: 1018 continue 1019 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 1020 1021 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 1022 1023 _add_md5(config, pn, appendfile) 1024 1025 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 1026 1027 finally: 1028 tinfoil.shutdown() 1029 1030 return 0 1031 1032 1033def rename(args, config, basepath, workspace): 1034 """Entry point for the devtool 'rename' subcommand""" 1035 import bb 1036 import oe.recipeutils 1037 1038 check_workspace_recipe(workspace, args.recipename) 1039 1040 if not (args.newname or args.version): 1041 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 1042 1043 recipefile = workspace[args.recipename]['recipefile'] 1044 if not recipefile: 1045 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1046 1047 if args.newname and args.newname != args.recipename: 1048 reason = oe.recipeutils.validate_pn(args.newname) 1049 if reason: 1050 raise DevtoolError(reason) 1051 newname = args.newname 1052 else: 1053 newname = args.recipename 1054 1055 append = workspace[args.recipename]['bbappend'] 1056 appendfn = os.path.splitext(os.path.basename(append))[0] 1057 splitfn = appendfn.split('_') 1058 if len(splitfn) > 1: 1059 origfnver = appendfn.split('_')[1] 1060 else: 1061 origfnver = '' 1062 1063 recipefilemd5 = None 1064 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1065 try: 1066 rd = parse_recipe(config, tinfoil, args.recipename, True) 1067 if not rd: 1068 return 1 1069 1070 bp = rd.getVar('BP') 1071 bpn = rd.getVar('BPN') 1072 if newname != args.recipename: 1073 localdata = rd.createCopy() 1074 localdata.setVar('PN', newname) 1075 newbpn = localdata.getVar('BPN') 1076 else: 1077 newbpn = bpn 1078 s = rd.getVar('S', False) 1079 src_uri = rd.getVar('SRC_URI', False) 1080 pv = rd.getVar('PV') 1081 1082 # Correct variable values that refer to the upstream source - these 1083 # values must stay the same, so if the name/version are changing then 1084 # we need to fix them up 1085 new_s = s 1086 new_src_uri = src_uri 1087 if newbpn != bpn: 1088 # ${PN} here is technically almost always incorrect, but people do use it 1089 new_s = new_s.replace('${BPN}', bpn) 1090 new_s = new_s.replace('${PN}', bpn) 1091 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1092 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1093 new_src_uri = new_src_uri.replace('${PN}', bpn) 1094 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1095 if args.version and origfnver == pv: 1096 new_s = new_s.replace('${PV}', pv) 1097 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1098 new_src_uri = new_src_uri.replace('${PV}', pv) 1099 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1100 patchfields = {} 1101 if new_s != s: 1102 patchfields['S'] = new_s 1103 if new_src_uri != src_uri: 1104 patchfields['SRC_URI'] = new_src_uri 1105 if patchfields: 1106 recipefilemd5 = bb.utils.md5_file(recipefile) 1107 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1108 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1109 finally: 1110 tinfoil.shutdown() 1111 1112 if args.version: 1113 newver = args.version 1114 else: 1115 newver = origfnver 1116 1117 if newver: 1118 newappend = '%s_%s.bbappend' % (newname, newver) 1119 newfile = '%s_%s.bb' % (newname, newver) 1120 else: 1121 newappend = '%s.bbappend' % newname 1122 newfile = '%s.bb' % newname 1123 1124 oldrecipedir = os.path.dirname(recipefile) 1125 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1126 if oldrecipedir != newrecipedir: 1127 bb.utils.mkdirhier(newrecipedir) 1128 1129 newappend = os.path.join(os.path.dirname(append), newappend) 1130 newfile = os.path.join(newrecipedir, newfile) 1131 1132 # Rename bbappend 1133 logger.info('Renaming %s to %s' % (append, newappend)) 1134 bb.utils.rename(append, newappend) 1135 # Rename recipe file 1136 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1137 bb.utils.rename(recipefile, newfile) 1138 1139 # Rename source tree if it's the default path 1140 appendmd5 = None 1141 if not args.no_srctree: 1142 srctree = workspace[args.recipename]['srctree'] 1143 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1144 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1145 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1146 shutil.move(srctree, newsrctree) 1147 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1148 appendmd5 = bb.utils.md5_file(newappend) 1149 appendlines = [] 1150 with open(newappend, 'r') as f: 1151 for line in f: 1152 appendlines.append(line) 1153 with open(newappend, 'w') as f: 1154 for line in appendlines: 1155 if srctree in line: 1156 line = line.replace(srctree, newsrctree) 1157 f.write(line) 1158 newappendmd5 = bb.utils.md5_file(newappend) 1159 1160 bpndir = None 1161 newbpndir = None 1162 if newbpn != bpn: 1163 bpndir = os.path.join(oldrecipedir, bpn) 1164 if os.path.exists(bpndir): 1165 newbpndir = os.path.join(newrecipedir, newbpn) 1166 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1167 shutil.move(bpndir, newbpndir) 1168 1169 bpdir = None 1170 newbpdir = None 1171 if newver != origfnver or newbpn != bpn: 1172 bpdir = os.path.join(oldrecipedir, bp) 1173 if os.path.exists(bpdir): 1174 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1175 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1176 shutil.move(bpdir, newbpdir) 1177 1178 if oldrecipedir != newrecipedir: 1179 # Move any stray files and delete the old recipe directory 1180 for entry in os.listdir(oldrecipedir): 1181 oldpath = os.path.join(oldrecipedir, entry) 1182 newpath = os.path.join(newrecipedir, entry) 1183 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1184 shutil.move(oldpath, newpath) 1185 os.rmdir(oldrecipedir) 1186 1187 # Now take care of entries in .devtool_md5 1188 md5entries = [] 1189 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1190 for line in f: 1191 md5entries.append(line) 1192 1193 if bpndir and newbpndir: 1194 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1195 else: 1196 relbpndir = None 1197 if bpdir and newbpdir: 1198 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1199 else: 1200 relbpdir = None 1201 1202 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1203 for entry in md5entries: 1204 splitentry = entry.rstrip().split('|') 1205 if len(splitentry) > 2: 1206 if splitentry[0] == args.recipename: 1207 splitentry[0] = newname 1208 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1209 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1210 if appendmd5 and splitentry[2] == appendmd5: 1211 splitentry[2] = newappendmd5 1212 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1213 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1214 if recipefilemd5 and splitentry[2] == recipefilemd5: 1215 splitentry[2] = newrecipefilemd5 1216 elif relbpndir and splitentry[1].startswith(relbpndir): 1217 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1218 elif relbpdir and splitentry[1].startswith(relbpdir): 1219 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1220 entry = '|'.join(splitentry) + '\n' 1221 f.write(entry) 1222 return 0 1223 1224 1225def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1226 """Get initial and update rev of a recipe. These are the start point of the 1227 whole patchset and start point for the patches to be re-generated/updated. 1228 """ 1229 import bb 1230 1231 # Get current branch 1232 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1233 cwd=srctree) 1234 branchname = stdout.rstrip() 1235 1236 # Parse initial rev from recipe if not specified 1237 commits = {} 1238 patches = [] 1239 initial_revs = {} 1240 with open(recipe_path, 'r') as f: 1241 for line in f: 1242 pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$' 1243 match = re.search(pattern, line) 1244 if match: 1245 name = match.group(1) 1246 rev = match.group(2) 1247 if line.startswith('# initial_rev'): 1248 if not (name == "." and initial_rev): 1249 initial_revs[name] = rev 1250 elif line.startswith('# commit') and not force_patch_refresh: 1251 if name not in commits: 1252 commits[name] = [rev] 1253 else: 1254 commits[name].append(rev) 1255 elif line.startswith('# patches_%s:' % branchname): 1256 patches = line.split(':')[-1].strip().split(',') 1257 1258 update_revs = dict(initial_revs) 1259 changed_revs = {} 1260 for name, rev in initial_revs.items(): 1261 # Find first actually changed revision 1262 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1263 rev, cwd=os.path.join(srctree, name)) 1264 newcommits = stdout.split() 1265 if name in commits: 1266 for i in range(min(len(commits[name]), len(newcommits))): 1267 if newcommits[i] == commits[name][i]: 1268 update_revs[name] = commits[name][i] 1269 1270 try: 1271 stdout, _ = bb.process.run('git cherry devtool-patched', 1272 cwd=os.path.join(srctree, name)) 1273 except bb.process.ExecutionError as err: 1274 stdout = None 1275 1276 if stdout is not None and not force_patch_refresh: 1277 for line in stdout.splitlines(): 1278 if line.startswith('+ '): 1279 rev = line.split()[1] 1280 if rev in newcommits: 1281 if name not in changed_revs: 1282 changed_revs[name] = [rev] 1283 else: 1284 changed_revs[name].append(rev) 1285 1286 return initial_revs, update_revs, changed_revs, patches 1287 1288def _remove_file_entries(srcuri, filelist): 1289 """Remove file:// entries from SRC_URI""" 1290 remaining = filelist[:] 1291 entries = [] 1292 for fname in filelist: 1293 basename = os.path.basename(fname) 1294 for i in range(len(srcuri)): 1295 if (srcuri[i].startswith('file://') and 1296 os.path.basename(srcuri[i].split(';')[0]) == basename): 1297 entries.append(srcuri[i]) 1298 remaining.remove(fname) 1299 srcuri.pop(i) 1300 break 1301 return entries, remaining 1302 1303def _replace_srcuri_entry(srcuri, filename, newentry): 1304 """Replace entry corresponding to specified file with a new entry""" 1305 basename = os.path.basename(filename) 1306 for i in range(len(srcuri)): 1307 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1308 srcuri.pop(i) 1309 srcuri.insert(i, newentry) 1310 break 1311 1312def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1313 """Unlink existing patch files""" 1314 1315 dry_run_suffix = ' (dry-run)' if dry_run else '' 1316 1317 for path in files: 1318 if append: 1319 if not destpath: 1320 raise Exception('destpath should be set here') 1321 path = os.path.join(destpath, os.path.basename(path)) 1322 1323 if os.path.exists(path): 1324 if not no_report_remove: 1325 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1326 if not dry_run: 1327 # FIXME "git rm" here would be nice if the file in question is 1328 # tracked 1329 # FIXME there's a chance that this file is referred to by 1330 # another recipe, in which case deleting wouldn't be the 1331 # right thing to do 1332 os.remove(path) 1333 # Remove directory if empty 1334 try: 1335 os.rmdir(os.path.dirname(path)) 1336 except OSError as ose: 1337 if ose.errno != errno.ENOTEMPTY: 1338 raise 1339 1340 1341def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None): 1342 """Export patches from srctree to given location. 1343 Returns three-tuple of dicts: 1344 1. updated - patches that already exist in SRCURI 1345 2. added - new patches that don't exist in SRCURI 1346 3 removed - patches that exist in SRCURI but not in exported patches 1347 In each dict the key is the 'basepath' of the URI and value is: 1348 - for updated and added dicts, a dict with 2 optionnal keys: 1349 - 'path': the absolute path to the existing file in recipe space (if any) 1350 - 'patchdir': the directory in wich the patch should be applied (if any) 1351 - for removed dict, the absolute path to the existing file in recipe space 1352 """ 1353 import oe.recipeutils 1354 from oe.patch import GitApplyTree 1355 updated = OrderedDict() 1356 added = OrderedDict() 1357 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1358 1359 existing_patches = dict((os.path.basename(path), path) for path in 1360 oe.recipeutils.get_recipe_patches(rd)) 1361 logger.debug('Existing patches: %s' % existing_patches) 1362 1363 # Generate patches from Git, exclude local files directory 1364 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1365 GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec) 1366 for dirpath, dirnames, filenames in os.walk(destdir): 1367 new_patches = filenames 1368 reldirpath = os.path.relpath(dirpath, destdir) 1369 for new_patch in new_patches: 1370 # Strip numbering from patch names. If it's a git sequence named patch, 1371 # the numbers might not match up since we are starting from a different 1372 # revision This does assume that people are using unique shortlog 1373 # values, but they ought to be anyway... 1374 new_basename = seqpatch_re.match(new_patch).group(2) 1375 match_name = None 1376 for old_patch in existing_patches: 1377 old_basename = seqpatch_re.match(old_patch).group(2) 1378 old_basename_splitext = os.path.splitext(old_basename) 1379 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1380 old_patch_noext = os.path.splitext(old_patch)[0] 1381 match_name = old_patch_noext 1382 break 1383 elif new_basename == old_basename: 1384 match_name = old_patch 1385 break 1386 if match_name: 1387 # Rename patch files 1388 if new_patch != match_name: 1389 bb.utils.rename(os.path.join(destdir, new_patch), 1390 os.path.join(destdir, match_name)) 1391 # Need to pop it off the list now before checking changed_revs 1392 oldpath = existing_patches.pop(old_patch) 1393 if changed_revs is not None and dirpath in changed_revs: 1394 # Avoid updating patches that have not actually changed 1395 with open(os.path.join(dirpath, match_name), 'r') as f: 1396 firstlineitems = f.readline().split() 1397 # Looking for "From <hash>" line 1398 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1399 if not firstlineitems[1] in changed_revs[dirpath]: 1400 continue 1401 # Recompress if necessary 1402 if oldpath.endswith(('.gz', '.Z')): 1403 bb.process.run(['gzip', match_name], cwd=destdir) 1404 if oldpath.endswith('.gz'): 1405 match_name += '.gz' 1406 else: 1407 match_name += '.Z' 1408 elif oldpath.endswith('.bz2'): 1409 bb.process.run(['bzip2', match_name], cwd=destdir) 1410 match_name += '.bz2' 1411 updated[match_name] = {'path' : oldpath} 1412 if reldirpath != ".": 1413 updated[match_name]['patchdir'] = reldirpath 1414 else: 1415 added[new_patch] = {} 1416 if reldirpath != ".": 1417 added[new_patch]['patchdir'] = reldirpath 1418 1419 return (updated, added, existing_patches) 1420 1421 1422def _create_kconfig_diff(srctree, rd, outfile): 1423 """Create a kconfig fragment""" 1424 # Only update config fragment if both config files exist 1425 orig_config = os.path.join(srctree, '.config.baseline') 1426 new_config = os.path.join(srctree, '.config.new') 1427 if os.path.exists(orig_config) and os.path.exists(new_config): 1428 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1429 '--unchanged-line-format=', orig_config, new_config] 1430 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1431 stderr=subprocess.PIPE) 1432 stdout, stderr = pipe.communicate() 1433 if pipe.returncode == 1: 1434 logger.info("Updating config fragment %s" % outfile) 1435 with open(outfile, 'wb') as fobj: 1436 fobj.write(stdout) 1437 elif pipe.returncode == 0: 1438 logger.info("Would remove config fragment %s" % outfile) 1439 if os.path.exists(outfile): 1440 # Remove fragment file in case of empty diff 1441 logger.info("Removing config fragment %s" % outfile) 1442 os.unlink(outfile) 1443 else: 1444 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1445 return True 1446 return False 1447 1448 1449def _export_local_files(srctree, rd, destdir, srctreebase): 1450 """Copy local files from srctree to given location. 1451 Returns three-tuple of dicts: 1452 1. updated - files that already exist in SRCURI 1453 2. added - new files files that don't exist in SRCURI 1454 3 removed - files that exist in SRCURI but not in exported files 1455 In each dict the key is the 'basepath' of the URI and value is the 1456 absolute path to the existing file in recipe space (if any). 1457 """ 1458 import oe.recipeutils 1459 1460 # Find out local files (SRC_URI files that exist in the "recipe space"). 1461 # Local files that reside in srctree are not included in patch generation. 1462 # Instead they are directly copied over the original source files (in 1463 # recipe space). 1464 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1465 new_set = None 1466 updated = OrderedDict() 1467 added = OrderedDict() 1468 removed = OrderedDict() 1469 1470 # Get current branch and return early with empty lists 1471 # if on one of the override branches 1472 # (local files are provided only for the main branch and processing 1473 # them against lists from recipe overrides will result in mismatches 1474 # and broken modifications to recipes). 1475 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1476 cwd=srctree) 1477 branchname = stdout.rstrip() 1478 if branchname.startswith(override_branch_prefix): 1479 return (updated, added, removed) 1480 1481 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1482 git_files = _git_ls_tree(srctree) 1483 if 'oe-local-files' in git_files: 1484 # If tracked by Git, take the files from srctree HEAD. First get 1485 # the tree object of the directory 1486 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1487 tree = git_files['oe-local-files'][2] 1488 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1489 env=dict(os.environ, GIT_WORK_TREE=destdir, 1490 GIT_INDEX_FILE=tmp_index)) 1491 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1492 elif os.path.isdir(local_files_dir): 1493 # If not tracked by Git, just copy from working copy 1494 new_set = _ls_tree(local_files_dir) 1495 bb.process.run(['cp', '-ax', 1496 os.path.join(local_files_dir, '.'), destdir]) 1497 else: 1498 new_set = [] 1499 1500 # Special handling for kernel config 1501 if bb.data.inherits_class('kernel-yocto', rd): 1502 fragment_fn = 'devtool-fragment.cfg' 1503 fragment_path = os.path.join(destdir, fragment_fn) 1504 if _create_kconfig_diff(srctree, rd, fragment_path): 1505 if os.path.exists(fragment_path): 1506 if fragment_fn not in new_set: 1507 new_set.append(fragment_fn) 1508 # Copy fragment to local-files 1509 if os.path.isdir(local_files_dir): 1510 shutil.copy2(fragment_path, local_files_dir) 1511 else: 1512 if fragment_fn in new_set: 1513 new_set.remove(fragment_fn) 1514 # Remove fragment from local-files 1515 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1516 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1517 1518 # Special handling for cml1, ccmake, etc bbclasses that generated 1519 # configuration fragment files that are consumed as source files 1520 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1521 if bb.data.inherits_class(frag_class, rd): 1522 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1523 if os.path.exists(srcpath): 1524 if frag_name not in new_set: 1525 new_set.append(frag_name) 1526 # copy fragment into destdir 1527 shutil.copy2(srcpath, destdir) 1528 # copy fragment into local files if exists 1529 if os.path.isdir(local_files_dir): 1530 shutil.copy2(srcpath, local_files_dir) 1531 1532 if new_set is not None: 1533 for fname in new_set: 1534 if fname in existing_files: 1535 origpath = existing_files.pop(fname) 1536 workpath = os.path.join(local_files_dir, fname) 1537 if not filecmp.cmp(origpath, workpath): 1538 updated[fname] = origpath 1539 elif fname != '.gitignore': 1540 added[fname] = None 1541 1542 workdir = rd.getVar('WORKDIR') 1543 s = rd.getVar('S') 1544 if not s.endswith(os.sep): 1545 s += os.sep 1546 1547 if workdir != s: 1548 # Handle files where subdir= was specified 1549 for fname in list(existing_files.keys()): 1550 # FIXME handle both subdir starting with BP and not? 1551 fworkpath = os.path.join(workdir, fname) 1552 if fworkpath.startswith(s): 1553 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1554 if os.path.exists(fpath): 1555 origpath = existing_files.pop(fname) 1556 if not filecmp.cmp(origpath, fpath): 1557 updated[fpath] = origpath 1558 1559 removed = existing_files 1560 return (updated, added, removed) 1561 1562 1563def _determine_files_dir(rd): 1564 """Determine the appropriate files directory for a recipe""" 1565 recipedir = rd.getVar('FILE_DIRNAME') 1566 for entry in rd.getVar('FILESPATH').split(':'): 1567 relpth = os.path.relpath(entry, recipedir) 1568 if not os.sep in relpth: 1569 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1570 if os.path.isdir(entry): 1571 return entry 1572 return os.path.join(recipedir, rd.getVar('BPN')) 1573 1574 1575def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1576 """Implement the 'srcrev' mode of update-recipe""" 1577 import bb 1578 import oe.recipeutils 1579 1580 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1581 1582 recipefile = rd.getVar('FILE') 1583 recipedir = os.path.basename(recipefile) 1584 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1585 1586 # Get original SRCREV 1587 old_srcrev = rd.getVar('SRCREV') or '' 1588 if old_srcrev == "INVALID": 1589 raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository') 1590 old_srcrev = {'.': old_srcrev} 1591 1592 # Get HEAD revision 1593 try: 1594 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1595 except bb.process.ExecutionError as err: 1596 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1597 (srctree, err)) 1598 srcrev = stdout.strip() 1599 if len(srcrev) != 40: 1600 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1601 1602 destpath = None 1603 remove_files = [] 1604 patchfields = {} 1605 patchfields['SRCREV'] = srcrev 1606 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1607 srcuri = orig_src_uri.split() 1608 tempdir = tempfile.mkdtemp(prefix='devtool') 1609 update_srcuri = False 1610 appendfile = None 1611 try: 1612 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1613 srctreebase = workspace[recipename]['srctreebase'] 1614 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1615 if not no_remove: 1616 # Find list of existing patches in recipe file 1617 patches_dir = tempfile.mkdtemp(dir=tempdir) 1618 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1619 patches_dir) 1620 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1621 1622 # Remove deleted local files and "overlapping" patches 1623 remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value] 1624 if remove_files: 1625 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1626 update_srcuri = True 1627 1628 if appendlayerdir: 1629 files = dict((os.path.join(local_files_dir, key), val) for 1630 key, val in list(upd_f.items()) + list(new_f.items())) 1631 removevalues = {} 1632 if update_srcuri: 1633 removevalues = {'SRC_URI': removedentries} 1634 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1635 if dry_run_outdir: 1636 logger.info('Creating bbappend (dry-run)') 1637 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1638 rd, appendlayerdir, files, wildcardver=wildcard_version, 1639 extralines=patchfields, removevalues=removevalues, 1640 redirect_output=dry_run_outdir) 1641 else: 1642 files_dir = _determine_files_dir(rd) 1643 for basepath, path in upd_f.items(): 1644 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1645 if os.path.isabs(basepath): 1646 # Original file (probably with subdir pointing inside source tree) 1647 # so we do not want to move it, just copy 1648 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1649 else: 1650 _move_file(os.path.join(local_files_dir, basepath), path, 1651 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1652 update_srcuri= True 1653 for basepath, path in new_f.items(): 1654 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1655 _move_file(os.path.join(local_files_dir, basepath), 1656 os.path.join(files_dir, basepath), 1657 dry_run_outdir=dry_run_outdir, 1658 base_outdir=recipedir) 1659 srcuri.append('file://%s' % basepath) 1660 update_srcuri = True 1661 if update_srcuri: 1662 patchfields['SRC_URI'] = ' '.join(srcuri) 1663 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1664 finally: 1665 shutil.rmtree(tempdir) 1666 if not 'git://' in orig_src_uri: 1667 logger.info('You will need to update SRC_URI within the recipe to ' 1668 'point to a git repository where you have pushed your ' 1669 'changes') 1670 1671 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1672 return True, appendfile, remove_files 1673 1674def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1675 """Implement the 'patch' mode of update-recipe""" 1676 import bb 1677 import oe.recipeutils 1678 1679 recipefile = rd.getVar('FILE') 1680 recipedir = os.path.dirname(recipefile) 1681 append = workspace[recipename]['bbappend'] 1682 if not os.path.exists(append): 1683 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1684 recipename) 1685 srctreebase = workspace[recipename]['srctreebase'] 1686 relpatchdir = os.path.relpath(srctreebase, srctree) 1687 if relpatchdir == '.': 1688 patchdir_params = {} 1689 else: 1690 patchdir_params = {'patchdir': relpatchdir} 1691 1692 def srcuri_entry(basepath, patchdir_params): 1693 if patchdir_params: 1694 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1695 else: 1696 paramstr = '' 1697 return 'file://%s%s' % (basepath, paramstr) 1698 1699 initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1700 if not initial_revs: 1701 raise DevtoolError('Unable to find initial revision - please specify ' 1702 'it with --initial-rev') 1703 1704 appendfile = None 1705 dl_dir = rd.getVar('DL_DIR') 1706 if not dl_dir.endswith('/'): 1707 dl_dir += '/' 1708 1709 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1710 1711 tempdir = tempfile.mkdtemp(prefix='devtool') 1712 try: 1713 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1714 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1715 1716 # Get updated patches from source tree 1717 patches_dir = tempfile.mkdtemp(dir=tempdir) 1718 upd_p, new_p, _ = _export_patches(srctree, rd, update_revs, 1719 patches_dir, changed_revs) 1720 # Get all patches from source tree and check if any should be removed 1721 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1722 _, _, del_p = _export_patches(srctree, rd, initial_revs, 1723 all_patches_dir) 1724 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1725 if filter_patches: 1726 new_p = OrderedDict() 1727 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1728 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) 1729 remove_files = [] 1730 if not no_remove: 1731 # Remove deleted local files and patches 1732 remove_files = list(del_f.values()) + list(del_p.values()) 1733 updatefiles = False 1734 updaterecipe = False 1735 destpath = None 1736 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1737 1738 if appendlayerdir: 1739 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1740 key, val in list(upd_f.items()) + list(new_f.items())) 1741 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1742 key, val in list(upd_p.items()) + list(new_p.items()))) 1743 1744 params = [] 1745 for file, param in files.items(): 1746 patchdir_param = dict(patchdir_params) 1747 patchdir = param.get('patchdir', ".") 1748 if patchdir != "." : 1749 if patchdir_param: 1750 patchdir_param['patchdir'] += patchdir 1751 else: 1752 patchdir_param['patchdir'] = patchdir 1753 params.append(patchdir_param) 1754 1755 if files or remove_files: 1756 removevalues = None 1757 if remove_files: 1758 removedentries, remaining = _remove_file_entries( 1759 srcuri, remove_files) 1760 if removedentries or remaining: 1761 remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for 1762 item in remaining] 1763 removevalues = {'SRC_URI': removedentries + remaining} 1764 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1765 rd, appendlayerdir, files, 1766 wildcardver=wildcard_version, 1767 removevalues=removevalues, 1768 redirect_output=dry_run_outdir, 1769 params=params) 1770 else: 1771 logger.info('No patches or local source files needed updating') 1772 else: 1773 # Update existing files 1774 files_dir = _determine_files_dir(rd) 1775 for basepath, path in upd_f.items(): 1776 logger.info('Updating file %s' % basepath) 1777 if os.path.isabs(basepath): 1778 # Original file (probably with subdir pointing inside source tree) 1779 # so we do not want to move it, just copy 1780 _copy_file(basepath, path, 1781 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1782 else: 1783 _move_file(os.path.join(local_files_dir, basepath), path, 1784 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1785 updatefiles = True 1786 for basepath, param in upd_p.items(): 1787 path = param['path'] 1788 patchdir = param.get('patchdir', ".") 1789 if patchdir != "." : 1790 patchdir_param = dict(patchdir_params) 1791 if patchdir_param: 1792 patchdir_param['patchdir'] += patchdir 1793 else: 1794 patchdir_param['patchdir'] = patchdir 1795 patchfn = os.path.join(patches_dir, patchdir, basepath) 1796 if os.path.dirname(path) + '/' == dl_dir: 1797 # This is a a downloaded patch file - we now need to 1798 # replace the entry in SRC_URI with our local version 1799 logger.info('Replacing remote patch %s with updated local version' % basepath) 1800 path = os.path.join(files_dir, basepath) 1801 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param)) 1802 updaterecipe = True 1803 else: 1804 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1805 _move_file(patchfn, path, 1806 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1807 updatefiles = True 1808 # Add any new files 1809 for basepath, path in new_f.items(): 1810 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1811 _move_file(os.path.join(local_files_dir, basepath), 1812 os.path.join(files_dir, basepath), 1813 dry_run_outdir=dry_run_outdir, 1814 base_outdir=recipedir) 1815 srcuri.append(srcuri_entry(basepath, patchdir_params)) 1816 updaterecipe = True 1817 for basepath, param in new_p.items(): 1818 patchdir = param.get('patchdir', ".") 1819 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1820 _move_file(os.path.join(patches_dir, patchdir, basepath), 1821 os.path.join(files_dir, basepath), 1822 dry_run_outdir=dry_run_outdir, 1823 base_outdir=recipedir) 1824 params = dict(patchdir_params) 1825 if patchdir != "." : 1826 if params: 1827 params['patchdir'] += patchdir 1828 else: 1829 params['patchdir'] = patchdir 1830 1831 srcuri.append(srcuri_entry(basepath, params)) 1832 updaterecipe = True 1833 # Update recipe, if needed 1834 if _remove_file_entries(srcuri, remove_files)[0]: 1835 updaterecipe = True 1836 if updaterecipe: 1837 if not dry_run_outdir: 1838 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1839 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1840 {'SRC_URI': ' '.join(srcuri)}, 1841 redirect_output=dry_run_outdir) 1842 elif not updatefiles: 1843 # Neither patches nor recipe were updated 1844 logger.info('No patches or files need updating') 1845 return False, None, [] 1846 finally: 1847 shutil.rmtree(tempdir) 1848 1849 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1850 return True, appendfile, remove_files 1851 1852def _guess_recipe_update_mode(srctree, rdata): 1853 """Guess the recipe update mode to use""" 1854 src_uri = (rdata.getVar('SRC_URI') or '').split() 1855 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1856 if not git_uris: 1857 return 'patch' 1858 # Just use the first URI for now 1859 uri = git_uris[0] 1860 # Check remote branch 1861 params = bb.fetch.decodeurl(uri)[5] 1862 upstr_branch = params['branch'] if 'branch' in params else 'master' 1863 # Check if current branch HEAD is found in upstream branch 1864 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1865 head_rev = stdout.rstrip() 1866 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1867 cwd=srctree) 1868 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1869 if 'origin/' + upstr_branch in remote_brs: 1870 return 'srcrev' 1871 1872 return 'patch' 1873 1874def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1875 srctree = workspace[recipename]['srctree'] 1876 if mode == 'auto': 1877 mode = _guess_recipe_update_mode(srctree, rd) 1878 1879 override_branches = [] 1880 mainbranch = None 1881 startbranch = None 1882 if not no_overrides: 1883 stdout, _ = bb.process.run('git branch', cwd=srctree) 1884 other_branches = [] 1885 for line in stdout.splitlines(): 1886 branchname = line[2:] 1887 if line.startswith('* '): 1888 if 'HEAD' in line: 1889 raise DevtoolError('Detached HEAD - please check out a branch, e.g., "devtool"') 1890 startbranch = branchname 1891 if branchname.startswith(override_branch_prefix): 1892 override_branches.append(branchname) 1893 else: 1894 other_branches.append(branchname) 1895 1896 if override_branches: 1897 logger.debug('_update_recipe: override branches: %s' % override_branches) 1898 logger.debug('_update_recipe: other branches: %s' % other_branches) 1899 if startbranch.startswith(override_branch_prefix): 1900 if len(other_branches) == 1: 1901 mainbranch = other_branches[1] 1902 else: 1903 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1904 else: 1905 mainbranch = startbranch 1906 1907 checkedout = None 1908 anyupdated = False 1909 appendfile = None 1910 allremoved = [] 1911 if override_branches: 1912 logger.info('Handling main branch (%s)...' % mainbranch) 1913 if startbranch != mainbranch: 1914 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1915 checkedout = mainbranch 1916 try: 1917 branchlist = [mainbranch] + override_branches 1918 for branch in branchlist: 1919 crd = bb.data.createCopy(rd) 1920 if branch != mainbranch: 1921 logger.info('Handling branch %s...' % branch) 1922 override = branch[len(override_branch_prefix):] 1923 crd.appendVar('OVERRIDES', ':%s' % override) 1924 bb.process.run('git checkout %s' % branch, cwd=srctree) 1925 checkedout = branch 1926 1927 if mode == 'srcrev': 1928 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1929 elif mode == 'patch': 1930 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1931 else: 1932 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1933 if updated: 1934 anyupdated = True 1935 if appendf: 1936 appendfile = appendf 1937 allremoved.extend(removed) 1938 finally: 1939 if startbranch and checkedout != startbranch: 1940 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1941 1942 return anyupdated, appendfile, allremoved 1943 1944def update_recipe(args, config, basepath, workspace): 1945 """Entry point for the devtool 'update-recipe' subcommand""" 1946 check_workspace_recipe(workspace, args.recipename) 1947 1948 if args.append: 1949 if not os.path.exists(args.append): 1950 raise DevtoolError('bbappend destination layer directory "%s" ' 1951 'does not exist' % args.append) 1952 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1953 raise DevtoolError('conf/layer.conf not found in bbappend ' 1954 'destination layer "%s"' % args.append) 1955 1956 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1957 try: 1958 1959 rd = parse_recipe(config, tinfoil, args.recipename, True) 1960 if not rd: 1961 return 1 1962 1963 dry_run_output = None 1964 dry_run_outdir = None 1965 if args.dry_run: 1966 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1967 dry_run_outdir = dry_run_output.name 1968 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1969 1970 if updated: 1971 rf = rd.getVar('FILE') 1972 if rf.startswith(config.workspace_path): 1973 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1974 finally: 1975 tinfoil.shutdown() 1976 1977 return 0 1978 1979 1980def status(args, config, basepath, workspace): 1981 """Entry point for the devtool 'status' subcommand""" 1982 if workspace: 1983 for recipe, value in sorted(workspace.items()): 1984 recipefile = value['recipefile'] 1985 if recipefile: 1986 recipestr = ' (%s)' % recipefile 1987 else: 1988 recipestr = '' 1989 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1990 else: 1991 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1992 return 0 1993 1994 1995def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1996 """Reset one or more recipes""" 1997 import oe.path 1998 1999 def clean_preferred_provider(pn, layerconf_path): 2000 """Remove PREFERRED_PROVIDER from layer.conf'""" 2001 import re 2002 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 2003 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 2004 pprovider_found = False 2005 with open(layerconf_file, 'r') as f: 2006 lines = f.readlines() 2007 with open(new_layerconf_file, 'a') as nf: 2008 for line in lines: 2009 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 2010 if not re.match(pprovider_exp, line): 2011 nf.write(line) 2012 else: 2013 pprovider_found = True 2014 if pprovider_found: 2015 shutil.move(new_layerconf_file, layerconf_file) 2016 else: 2017 os.remove(new_layerconf_file) 2018 2019 if recipes and not no_clean: 2020 if len(recipes) == 1: 2021 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 2022 else: 2023 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 2024 # If the recipe file itself was created in the workspace, and 2025 # it uses BBCLASSEXTEND, then we need to also clean the other 2026 # variants 2027 targets = [] 2028 for recipe in recipes: 2029 targets.append(recipe) 2030 recipefile = workspace[recipe]['recipefile'] 2031 if recipefile and os.path.exists(recipefile): 2032 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 2033 try: 2034 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 2035 except bb.process.ExecutionError as e: 2036 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 2037 'wish, you may specify -n/--no-clean to ' 2038 'skip running this command when resetting' % 2039 (e.command, e.stdout)) 2040 2041 for pn in recipes: 2042 _check_preserve(config, pn) 2043 2044 appendfile = workspace[pn]['bbappend'] 2045 if os.path.exists(appendfile): 2046 # This shouldn't happen, but is possible if devtool errored out prior to 2047 # writing the md5 file. We need to delete this here or the recipe won't 2048 # actually be reset 2049 os.remove(appendfile) 2050 2051 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 2052 def preservedir(origdir): 2053 if os.path.exists(origdir): 2054 for root, dirs, files in os.walk(origdir): 2055 for fn in files: 2056 logger.warning('Preserving %s in %s' % (fn, preservepath)) 2057 _move_file(os.path.join(origdir, fn), 2058 os.path.join(preservepath, fn)) 2059 for dn in dirs: 2060 preservedir(os.path.join(root, dn)) 2061 os.rmdir(origdir) 2062 2063 recipefile = workspace[pn]['recipefile'] 2064 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 2065 # This should always be true if recipefile is set, but just in case 2066 preservedir(os.path.dirname(recipefile)) 2067 # We don't automatically create this dir next to appends, but the user can 2068 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 2069 2070 srctreebase = workspace[pn]['srctreebase'] 2071 if os.path.isdir(srctreebase): 2072 if os.listdir(srctreebase): 2073 if remove_work: 2074 logger.info('-r argument used on %s, removing source tree.' 2075 ' You will lose any unsaved work' %pn) 2076 shutil.rmtree(srctreebase) 2077 else: 2078 # We don't want to risk wiping out any work in progress 2079 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): 2080 from datetime import datetime 2081 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S"))) 2082 logger.info('Preserving source tree in %s\nIf you no ' 2083 'longer need it then please delete it manually.\n' 2084 'It is also possible to reuse it via devtool source tree argument.' 2085 % preservesrc) 2086 bb.utils.mkdirhier(os.path.dirname(preservesrc)) 2087 shutil.move(srctreebase, preservesrc) 2088 else: 2089 logger.info('Leaving source tree %s as-is; if you no ' 2090 'longer need it then please delete it manually' 2091 % srctreebase) 2092 else: 2093 # This is unlikely, but if it's empty we can just remove it 2094 os.rmdir(srctreebase) 2095 2096 clean_preferred_provider(pn, config.workspace_path) 2097 2098def reset(args, config, basepath, workspace): 2099 """Entry point for the devtool 'reset' subcommand""" 2100 import bb 2101 import shutil 2102 2103 recipes = "" 2104 2105 if args.recipename: 2106 if args.all: 2107 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 2108 else: 2109 for recipe in args.recipename: 2110 check_workspace_recipe(workspace, recipe, checksrc=False) 2111 elif not args.all: 2112 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2113 "reset all recipes") 2114 if args.all: 2115 recipes = list(workspace.keys()) 2116 else: 2117 recipes = args.recipename 2118 2119 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2120 2121 return 0 2122 2123 2124def _get_layer(layername, d): 2125 """Determine the base layer path for the specified layer name/path""" 2126 layerdirs = d.getVar('BBLAYERS').split() 2127 layers = {} # {basename: layer_paths} 2128 for p in layerdirs: 2129 bn = os.path.basename(p) 2130 if bn not in layers: 2131 layers[bn] = [p] 2132 else: 2133 layers[bn].append(p) 2134 # Provide some shortcuts 2135 if layername.lower() in ['oe-core', 'openembedded-core']: 2136 layername = 'meta' 2137 layer_paths = layers.get(layername, None) 2138 if not layer_paths: 2139 return os.path.abspath(layername) 2140 elif len(layer_paths) == 1: 2141 return os.path.abspath(layer_paths[0]) 2142 else: 2143 # multiple layers having the same base name 2144 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 2145 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 2146 return os.path.abspath(layer_paths[0]) 2147 2148 2149def finish(args, config, basepath, workspace): 2150 """Entry point for the devtool 'finish' subcommand""" 2151 import bb 2152 import oe.recipeutils 2153 2154 check_workspace_recipe(workspace, args.recipename) 2155 2156 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2157 2158 # Grab the equivalent of COREBASE without having to initialise tinfoil 2159 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2160 2161 srctree = workspace[args.recipename]['srctree'] 2162 check_git_repo_op(srctree, [corebasedir]) 2163 dirty = check_git_repo_dirty(srctree) 2164 if dirty: 2165 if args.force: 2166 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2167 else: 2168 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2169 2170 no_clean = args.no_clean 2171 remove_work=args.remove_work 2172 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2173 try: 2174 rd = parse_recipe(config, tinfoil, args.recipename, True) 2175 if not rd: 2176 return 1 2177 2178 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2179 recipefile = rd.getVar('FILE') 2180 recipedir = os.path.dirname(recipefile) 2181 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2182 2183 if not os.path.isdir(destlayerdir): 2184 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2185 2186 if os.path.abspath(destlayerdir) == config.workspace_path: 2187 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2188 2189 # If it's an upgrade, grab the original path 2190 origpath = None 2191 origfilelist = None 2192 append = workspace[args.recipename]['bbappend'] 2193 with open(append, 'r') as f: 2194 for line in f: 2195 if line.startswith('# original_path:'): 2196 origpath = line.split(':')[1].strip() 2197 elif line.startswith('# original_files:'): 2198 origfilelist = line.split(':')[1].split() 2199 2200 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2201 2202 if origlayerdir == config.workspace_path: 2203 # Recipe file itself is in workspace, update it there first 2204 appendlayerdir = None 2205 origrelpath = None 2206 if origpath: 2207 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2208 if origlayerpath: 2209 origrelpath = os.path.relpath(origpath, origlayerpath) 2210 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2211 if not destpath: 2212 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2213 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2214 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2215 if not os.path.abspath(destlayerbasedir) in layerdirs: 2216 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2217 2218 elif destlayerdir == origlayerdir: 2219 # Same layer, update the original recipe 2220 appendlayerdir = None 2221 destpath = None 2222 else: 2223 # Create/update a bbappend in the specified layer 2224 appendlayerdir = destlayerdir 2225 destpath = None 2226 2227 # Actually update the recipe / bbappend 2228 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2229 dry_run_output = None 2230 dry_run_outdir = None 2231 if args.dry_run: 2232 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2233 dry_run_outdir = dry_run_output.name 2234 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2235 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2236 2237 # Remove any old files in the case of an upgrade 2238 if removing_original: 2239 for fn in origfilelist: 2240 fnp = os.path.join(origpath, fn) 2241 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2242 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2243 if not args.dry_run: 2244 try: 2245 os.remove(fnp) 2246 except FileNotFoundError: 2247 pass 2248 2249 if origlayerdir == config.workspace_path and destpath: 2250 # Recipe file itself is in the workspace - need to move it and any 2251 # associated files to the specified layer 2252 no_clean = True 2253 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2254 for root, _, files in os.walk(recipedir): 2255 for fn in files: 2256 srcpath = os.path.join(root, fn) 2257 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2258 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2259 destfp = os.path.join(destdir, fn) 2260 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2261 2262 if dry_run_outdir: 2263 import difflib 2264 comparelist = [] 2265 for root, _, files in os.walk(dry_run_outdir): 2266 for fn in files: 2267 outf = os.path.join(root, fn) 2268 relf = os.path.relpath(outf, dry_run_outdir) 2269 logger.debug('dry-run: output file %s' % relf) 2270 if fn.endswith('.bb'): 2271 if origfilelist and origpath and destpath: 2272 # Need to match this up with the pre-upgrade recipe file 2273 for origf in origfilelist: 2274 if origf.endswith('.bb'): 2275 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2276 outf, 2277 os.path.abspath(os.path.join(destpath, relf)))) 2278 break 2279 else: 2280 # Compare to the existing recipe 2281 comparelist.append((recipefile, outf, recipefile)) 2282 elif fn.endswith('.bbappend'): 2283 if appendfile: 2284 if os.path.exists(appendfile): 2285 comparelist.append((appendfile, outf, appendfile)) 2286 else: 2287 comparelist.append((None, outf, appendfile)) 2288 else: 2289 if destpath: 2290 recipedest = destpath 2291 elif appendfile: 2292 recipedest = os.path.dirname(appendfile) 2293 else: 2294 recipedest = os.path.dirname(recipefile) 2295 destfp = os.path.join(recipedest, relf) 2296 if os.path.exists(destfp): 2297 comparelist.append((destfp, outf, destfp)) 2298 output = '' 2299 for oldfile, newfile, newfileshow in comparelist: 2300 if oldfile: 2301 with open(oldfile, 'r') as f: 2302 oldlines = f.readlines() 2303 else: 2304 oldfile = '/dev/null' 2305 oldlines = [] 2306 with open(newfile, 'r') as f: 2307 newlines = f.readlines() 2308 if not newfileshow: 2309 newfileshow = newfile 2310 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2311 difflines = list(diff) 2312 if difflines: 2313 output += ''.join(difflines) 2314 if output: 2315 logger.info('Diff of changed files:\n%s' % output) 2316 finally: 2317 tinfoil.shutdown() 2318 2319 # Everything else has succeeded, we can now reset 2320 if args.dry_run: 2321 logger.info('Resetting recipe (dry-run)') 2322 else: 2323 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2324 2325 return 0 2326 2327 2328def get_default_srctree(config, recipename=''): 2329 """Get the default srctree path""" 2330 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2331 if recipename: 2332 return os.path.join(srctreeparent, 'sources', recipename) 2333 else: 2334 return os.path.join(srctreeparent, 'sources') 2335 2336def register_commands(subparsers, context): 2337 """Register devtool subcommands from this plugin""" 2338 2339 defsrctree = get_default_srctree(context.config) 2340 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2341 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2342 group='starting', order=100) 2343 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2344 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2345 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2346 group = parser_add.add_mutually_exclusive_group() 2347 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2348 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2349 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2350 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2351 parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true") 2352 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2353 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2354 group = parser_add.add_mutually_exclusive_group() 2355 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2356 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2357 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2358 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2359 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2360 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2361 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2362 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2363 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2364 2365 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2366 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2367 group='starting', order=90) 2368 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2369 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2370 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2371 group = parser_modify.add_mutually_exclusive_group() 2372 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2373 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2374 group = parser_modify.add_mutually_exclusive_group() 2375 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2376 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2377 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2378 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2379 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2380 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2381 2382 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2383 description='Extracts the source for an existing recipe', 2384 group='advanced') 2385 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2386 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2387 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2388 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2389 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2390 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2391 2392 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2393 description='Synchronize the previously extracted source tree for an existing recipe', 2394 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2395 group='advanced') 2396 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2397 parser_sync.add_argument('srctree', help='Path to the source tree') 2398 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2399 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2400 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2401 2402 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2403 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2404 group='working', order=10) 2405 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2406 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2407 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2408 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2409 parser_rename.set_defaults(func=rename) 2410 2411 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2412 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2413 group='working', order=-90) 2414 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2415 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2416 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2417 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2418 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2419 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2420 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2421 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2422 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2423 parser_update_recipe.set_defaults(func=update_recipe) 2424 2425 parser_status = subparsers.add_parser('status', help='Show workspace status', 2426 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2427 group='info', order=100) 2428 parser_status.set_defaults(func=status) 2429 2430 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2431 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2432 group='working', order=-100) 2433 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2434 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2435 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2436 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2437 parser_reset.set_defaults(func=reset) 2438 2439 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2440 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2441 group='working', order=-100) 2442 parser_finish.add_argument('recipename', help='Recipe to finish') 2443 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2444 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2445 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2446 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2447 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2448 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2449 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2450 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2451 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2452 parser_finish.set_defaults(func=finish) 2453