1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21from collections import OrderedDict 22from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 23from devtool import parse_recipe 24 25logger = logging.getLogger('devtool') 26 27override_branch_prefix = 'devtool-override-' 28 29 30def add(args, config, basepath, workspace): 31 """Entry point for the devtool 'add' subcommand""" 32 import bb 33 import oe.recipeutils 34 35 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 36 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 37 38 # These are positional arguments, but because we're nice, allow 39 # specifying e.g. source tree without name, or fetch URI without name or 40 # source tree (if we can detect that that is what the user meant) 41 if scriptutils.is_src_url(args.recipename): 42 if not args.fetchuri: 43 if args.fetch: 44 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 45 args.fetchuri = args.recipename 46 args.recipename = '' 47 elif scriptutils.is_src_url(args.srctree): 48 if not args.fetchuri: 49 if args.fetch: 50 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 51 args.fetchuri = args.srctree 52 args.srctree = '' 53 elif args.recipename and not args.srctree: 54 if os.sep in args.recipename: 55 args.srctree = args.recipename 56 args.recipename = None 57 elif os.path.isdir(args.recipename): 58 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 59 60 if not args.fetchuri: 61 if args.srcrev: 62 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 63 if args.srcbranch: 64 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 65 66 if args.srctree and os.path.isfile(args.srctree): 67 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 68 args.srctree = '' 69 70 if args.fetch: 71 if args.fetchuri: 72 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 73 else: 74 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 75 args.fetchuri = args.fetch 76 77 if args.recipename: 78 if args.recipename in workspace: 79 raise DevtoolError("recipe %s is already in your workspace" % 80 args.recipename) 81 reason = oe.recipeutils.validate_pn(args.recipename) 82 if reason: 83 raise DevtoolError(reason) 84 85 if args.srctree: 86 srctree = os.path.abspath(args.srctree) 87 srctreeparent = None 88 tmpsrcdir = None 89 else: 90 srctree = None 91 srctreeparent = get_default_srctree(config) 92 bb.utils.mkdirhier(srctreeparent) 93 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 94 95 if srctree and os.path.exists(srctree): 96 if args.fetchuri: 97 if not os.path.isdir(srctree): 98 raise DevtoolError("Cannot fetch into source tree path %s as " 99 "it exists and is not a directory" % 100 srctree) 101 elif os.listdir(srctree): 102 raise DevtoolError("Cannot fetch into source tree path %s as " 103 "it already exists and is non-empty" % 104 srctree) 105 elif not args.fetchuri: 106 if args.srctree: 107 raise DevtoolError("Specified source tree %s could not be found" % 108 args.srctree) 109 elif srctree: 110 raise DevtoolError("No source tree exists at default path %s - " 111 "either create and populate this directory, " 112 "or specify a path to a source tree, or a " 113 "URI to fetch source from" % srctree) 114 else: 115 raise DevtoolError("You must either specify a source tree " 116 "or a URI to fetch source from") 117 118 if args.version: 119 if '_' in args.version or ' ' in args.version: 120 raise DevtoolError('Invalid version string "%s"' % args.version) 121 122 if args.color == 'auto' and sys.stdout.isatty(): 123 color = 'always' 124 else: 125 color = args.color 126 extracmdopts = '' 127 if args.fetchuri: 128 source = args.fetchuri 129 if srctree: 130 extracmdopts += ' -x %s' % srctree 131 else: 132 extracmdopts += ' -x %s' % tmpsrcdir 133 else: 134 source = srctree 135 if args.recipename: 136 extracmdopts += ' -N %s' % args.recipename 137 if args.version: 138 extracmdopts += ' -V %s' % args.version 139 if args.binary: 140 extracmdopts += ' -b' 141 if args.also_native: 142 extracmdopts += ' --also-native' 143 if args.src_subdir: 144 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 145 if args.autorev: 146 extracmdopts += ' -a' 147 if args.npm_dev: 148 extracmdopts += ' --npm-dev' 149 if args.no_pypi: 150 extracmdopts += ' --no-pypi' 151 if args.mirrors: 152 extracmdopts += ' --mirrors' 153 if args.srcrev: 154 extracmdopts += ' --srcrev %s' % args.srcrev 155 if args.srcbranch: 156 extracmdopts += ' --srcbranch %s' % args.srcbranch 157 if args.provides: 158 extracmdopts += ' --provides %s' % args.provides 159 160 tempdir = tempfile.mkdtemp(prefix='devtool') 161 try: 162 try: 163 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 164 except bb.process.ExecutionError as e: 165 if e.exitcode == 15: 166 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 167 else: 168 raise DevtoolError('Command \'%s\' failed' % e.command) 169 170 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 171 if recipes: 172 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 173 if recipename in workspace: 174 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 175 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 176 bb.utils.mkdirhier(recipedir) 177 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 178 appendfile = recipe_to_append(recipefile, config) 179 if os.path.exists(appendfile): 180 # This shouldn't be possible, but just in case 181 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 182 if os.path.exists(recipefile): 183 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 184 if tmpsrcdir: 185 srctree = os.path.join(srctreeparent, recipename) 186 if os.path.exists(tmpsrcdir): 187 if os.path.exists(srctree): 188 if os.path.isdir(srctree): 189 try: 190 os.rmdir(srctree) 191 except OSError as e: 192 if e.errno == errno.ENOTEMPTY: 193 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 194 else: 195 raise 196 else: 197 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 198 logger.info('Using default source tree path %s' % srctree) 199 shutil.move(tmpsrcdir, srctree) 200 else: 201 raise DevtoolError('Couldn\'t find source tree created by recipetool') 202 bb.utils.mkdirhier(recipedir) 203 shutil.move(recipes[0], recipefile) 204 # Move any additional files created by recipetool 205 for fn in os.listdir(tempdir): 206 shutil.move(os.path.join(tempdir, fn), recipedir) 207 else: 208 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 209 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 210 if os.path.exists(attic_recipe): 211 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 212 finally: 213 if tmpsrcdir and os.path.exists(tmpsrcdir): 214 shutil.rmtree(tmpsrcdir) 215 shutil.rmtree(tempdir) 216 217 for fn in os.listdir(recipedir): 218 _add_md5(config, recipename, os.path.join(recipedir, fn)) 219 220 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 221 try: 222 try: 223 rd = tinfoil.parse_recipe_file(recipefile, False) 224 except Exception as e: 225 logger.error(str(e)) 226 rd = None 227 if not rd: 228 # Parsing failed. We just created this recipe and we shouldn't 229 # leave it in the workdir or it'll prevent bitbake from starting 230 movefn = '%s.parsefailed' % recipefile 231 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 232 shutil.move(recipefile, movefn) 233 return 1 234 235 if args.fetchuri and not args.no_git: 236 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 237 238 initial_rev = {} 239 if os.path.exists(os.path.join(srctree, '.git')): 240 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 241 initial_rev["."] = stdout.rstrip() 242 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree) 243 for line in stdout.splitlines(): 244 (rev, submodule) = line.split() 245 initial_rev[os.path.relpath(submodule, srctree)] = rev 246 247 if args.src_subdir: 248 srctree = os.path.join(srctree, args.src_subdir) 249 250 bb.utils.mkdirhier(os.path.dirname(appendfile)) 251 with open(appendfile, 'w') as f: 252 f.write('inherit externalsrc\n') 253 f.write('EXTERNALSRC = "%s"\n' % srctree) 254 255 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 256 if b_is_s: 257 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 258 if initial_rev: 259 for key, value in initial_rev.items(): 260 f.write('\n# initial_rev %s: %s\n' % (key, value)) 261 262 if args.binary: 263 f.write('do_install:append() {\n') 264 f.write(' rm -rf ${D}/.git\n') 265 f.write(' rm -f ${D}/singletask.lock\n') 266 f.write('}\n') 267 268 if bb.data.inherits_class('npm', rd): 269 f.write('python do_configure:append() {\n') 270 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 271 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 272 f.write(' bb.utils.remove(lockfile)\n') 273 f.write('}\n') 274 275 # Check if the new layer provides recipes whose priorities have been 276 # overriden by PREFERRED_PROVIDER. 277 recipe_name = rd.getVar('PN') 278 provides = rd.getVar('PROVIDES') 279 # Search every item defined in PROVIDES 280 for recipe_provided in provides.split(): 281 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 282 current_pprovider = rd.getVar(preferred_provider) 283 if current_pprovider and current_pprovider != recipe_name: 284 if args.fixed_setup: 285 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 286 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 287 with open(layerconf_file, 'a') as f: 288 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 289 else: 290 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 291 break 292 293 _add_md5(config, recipename, appendfile) 294 295 check_prerelease_version(rd.getVar('PV'), 'devtool add') 296 297 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 298 299 finally: 300 tinfoil.shutdown() 301 302 return 0 303 304 305def _check_compatible_recipe(pn, d): 306 """Check if the recipe is supported by devtool""" 307 if pn == 'perf': 308 raise DevtoolError("The perf recipe does not actually check out " 309 "source and thus cannot be supported by this tool", 310 4) 311 312 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 313 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 314 315 if bb.data.inherits_class('image', d): 316 raise DevtoolError("The %s recipe is an image, and therefore is not " 317 "supported by this tool" % pn, 4) 318 319 if bb.data.inherits_class('populate_sdk', d): 320 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 321 "supported by this tool" % pn, 4) 322 323 if bb.data.inherits_class('packagegroup', d): 324 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 325 "not supported by this tool" % pn, 4) 326 327 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 328 # Not an incompatibility error per se, so we don't pass the error code 329 raise DevtoolError("externalsrc is currently enabled for the %s " 330 "recipe. This prevents the normal do_patch task " 331 "from working. You will need to disable this " 332 "first." % pn) 333 334def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 335 """Common function for copying a file to the dry run output directory""" 336 relpath = os.path.relpath(dst, base_outdir) 337 if relpath.startswith('..'): 338 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 339 dst = os.path.join(dry_run_outdir, relpath) 340 dst_d = os.path.dirname(dst) 341 if dst_d: 342 bb.utils.mkdirhier(dst_d) 343 # Don't overwrite existing files, otherwise in the case of an upgrade 344 # the dry-run written out recipe will be overwritten with an unmodified 345 # version 346 if not os.path.exists(dst): 347 shutil.copy(src, dst) 348 349def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 350 """Move a file. Creates all the directory components of destination path.""" 351 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 352 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 353 if dry_run_outdir: 354 # We want to copy here, not move 355 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 356 else: 357 dst_d = os.path.dirname(dst) 358 if dst_d: 359 bb.utils.mkdirhier(dst_d) 360 shutil.move(src, dst) 361 362def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 363 """Copy a file. Creates all the directory components of destination path.""" 364 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 365 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 366 if dry_run_outdir: 367 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 368 else: 369 dst_d = os.path.dirname(dst) 370 if dst_d: 371 bb.utils.mkdirhier(dst_d) 372 shutil.copy(src, dst) 373 374def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 375 """List contents of a git treeish""" 376 import bb 377 cmd = ['git', 'ls-tree', '-z', treeish] 378 if recursive: 379 cmd.append('-r') 380 out, _ = bb.process.run(cmd, cwd=repodir) 381 ret = {} 382 if out: 383 for line in out.split('\0'): 384 if line: 385 split = line.split(None, 4) 386 ret[split[3]] = split[0:3] 387 return ret 388 389def _git_modified(repodir): 390 """List the difference between HEAD and the index""" 391 import bb 392 cmd = ['git', 'status', '--porcelain'] 393 out, _ = bb.process.run(cmd, cwd=repodir) 394 ret = [] 395 if out: 396 for line in out.split("\n"): 397 if line and not line.startswith('??'): 398 ret.append(line[3:]) 399 return ret 400 401 402def _git_exclude_path(srctree, path): 403 """Return pathspec (list of paths) that excludes certain path""" 404 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 405 # we don't catch files that are deleted, for example. A more reliable way 406 # to implement this would be to use "negative pathspecs" which were 407 # introduced in Git v1.9.0. Revisit this when/if the required Git version 408 # becomes greater than that. 409 path = os.path.normpath(path) 410 recurse = True if len(path.split(os.path.sep)) > 1 else False 411 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 412 if path in git_files: 413 git_files.remove(path) 414 return git_files 415 else: 416 return ['.'] 417 418def _ls_tree(directory): 419 """Recursive listing of files in a directory""" 420 ret = [] 421 for root, dirs, files in os.walk(directory): 422 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 423 fname in files]) 424 return ret 425 426 427def extract(args, config, basepath, workspace): 428 """Entry point for the devtool 'extract' subcommand""" 429 import bb 430 431 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 432 if not tinfoil: 433 # Error already shown 434 return 1 435 try: 436 rd = parse_recipe(config, tinfoil, args.recipename, True) 437 if not rd: 438 return 1 439 440 srctree = os.path.abspath(args.srctree) 441 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 442 logger.info('Source tree extracted to %s' % srctree) 443 444 if initial_rev: 445 return 0 446 else: 447 return 1 448 finally: 449 tinfoil.shutdown() 450 451def sync(args, config, basepath, workspace): 452 """Entry point for the devtool 'sync' subcommand""" 453 import bb 454 455 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 456 if not tinfoil: 457 # Error already shown 458 return 1 459 try: 460 rd = parse_recipe(config, tinfoil, args.recipename, True) 461 if not rd: 462 return 1 463 464 srctree = os.path.abspath(args.srctree) 465 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 466 logger.info('Source tree %s synchronized' % srctree) 467 468 if initial_rev: 469 return 0 470 else: 471 return 1 472 finally: 473 tinfoil.shutdown() 474 475def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 476 """Extract sources of a recipe""" 477 import oe.recipeutils 478 import oe.patch 479 import oe.path 480 481 pn = d.getVar('PN') 482 483 _check_compatible_recipe(pn, d) 484 485 if sync: 486 if not os.path.exists(srctree): 487 raise DevtoolError("output path %s does not exist" % srctree) 488 else: 489 if os.path.exists(srctree): 490 if not os.path.isdir(srctree): 491 raise DevtoolError("output path %s exists and is not a directory" % 492 srctree) 493 elif os.listdir(srctree): 494 raise DevtoolError("output path %s already exists and is " 495 "non-empty" % srctree) 496 497 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 498 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 499 "extract source" % pn, 4) 500 501 if not sync: 502 # Prepare for shutil.move later on 503 bb.utils.mkdirhier(srctree) 504 os.rmdir(srctree) 505 506 extra_overrides = [] 507 if not no_overrides: 508 history = d.varhistory.variable('SRC_URI') 509 for event in history: 510 if not 'flag' in event: 511 if event['op'].startswith((':append[', ':prepend[')): 512 override = event['op'].split('[')[1].split(']')[0] 513 if not override.startswith('pn-'): 514 extra_overrides.append(override) 515 # We want to remove duplicate overrides. If a recipe had multiple 516 # SRC_URI_override += values it would cause mulitple instances of 517 # overrides. This doesn't play nicely with things like creating a 518 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 519 extra_overrides = list(set(extra_overrides)) 520 if extra_overrides: 521 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 522 523 initial_rev = None 524 525 recipefile = d.getVar('FILE') 526 appendfile = recipe_to_append(recipefile, config) 527 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 528 529 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 530 # directory so that: 531 # (a) we pick up all files that get unpacked to the WORKDIR, and 532 # (b) we don't disturb the existing build 533 # However, with recipe-specific sysroots the sysroots for the recipe 534 # will be prepared under WORKDIR, and if we used the system temporary 535 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 536 # our attempts to hardlink files into the recipe-specific sysroots 537 # will fail on systems where /tmp is a different filesystem, and it 538 # would have to fall back to copying the files which is a waste of 539 # time. Put the temp directory under the WORKDIR to prevent that from 540 # being a problem. 541 tempbasedir = d.getVar('WORKDIR') 542 bb.utils.mkdirhier(tempbasedir) 543 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 544 try: 545 tinfoil.logger.setLevel(logging.WARNING) 546 547 # FIXME this results in a cache reload under control of tinfoil, which is fine 548 # except we don't get the knotty progress bar 549 550 if os.path.exists(appendfile): 551 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 552 shutil.copyfile(appendfile, appendbackup) 553 else: 554 appendbackup = None 555 bb.utils.mkdirhier(os.path.dirname(appendfile)) 556 logger.debug('writing append file %s' % appendfile) 557 with open(appendfile, 'a') as f: 558 f.write('###--- _extract_source\n') 559 f.write('deltask do_recipe_qa\n') 560 f.write('deltask do_recipe_qa_setscene\n') 561 f.write('ERROR_QA:remove = "patch-fuzz"\n') 562 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 563 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 564 if not is_kernel_yocto: 565 f.write('PATCHTOOL = "git"\n') 566 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 567 if extra_overrides: 568 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 569 f.write('inherit devtool-source\n') 570 f.write('###--- _extract_source\n') 571 572 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 573 574 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 575 bb.utils.mkdirhier(sstate_manifests) 576 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 577 with open(preservestampfile, 'w') as f: 578 f.write(d.getVar('STAMP')) 579 tinfoil.modified_files() 580 try: 581 if is_kernel_yocto: 582 # We need to generate the kernel config 583 task = 'do_configure' 584 else: 585 task = 'do_patch' 586 587 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 588 logger.info('The %s recipe has %s disabled. Running only ' 589 'do_configure task dependencies' % (pn, task)) 590 591 if 'depends' in d.getVarFlags('do_configure', False): 592 pn = d.getVarFlags('do_configure', False)['depends'] 593 pn = pn.replace('${PV}', d.getVar('PV')) 594 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 595 task = None 596 597 # Run the fetch + unpack tasks 598 res = tinfoil.build_targets(pn, 599 task, 600 handle_events=True) 601 finally: 602 if os.path.exists(preservestampfile): 603 os.remove(preservestampfile) 604 605 if not res: 606 raise DevtoolError('Extracting source for %s failed' % pn) 607 608 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 609 workshareddir = d.getVar('S') 610 if os.path.islink(srctree): 611 os.unlink(srctree) 612 613 os.symlink(workshareddir, srctree) 614 615 # The initial_rev file is created in devtool_post_unpack function that will not be executed if 616 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 617 return True, True 618 619 try: 620 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 621 initial_rev = f.read() 622 623 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 624 srcsubdir = f.read() 625 except FileNotFoundError as e: 626 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 627 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 628 629 # Check if work-shared is empty, if yes 630 # find source and copy to work-shared 631 if is_kernel_yocto: 632 workshareddir = d.getVar('STAGING_KERNEL_DIR') 633 staging_kerVer = get_staging_kver(workshareddir) 634 kernelVersion = d.getVar('LINUX_VERSION') 635 636 # handle dangling symbolic link in work-shared: 637 if os.path.islink(workshareddir): 638 os.unlink(workshareddir) 639 640 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 641 shutil.rmtree(workshareddir) 642 oe.path.copyhardlinktree(srcsubdir, workshareddir) 643 elif not os.path.exists(workshareddir): 644 oe.path.copyhardlinktree(srcsubdir, workshareddir) 645 646 if sync: 647 try: 648 logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch)) 649 bb.process.run('git branch -f ' + devbranch + '.bak', cwd=srctree) 650 651 # Use git fetch to update the source with the current recipe 652 # To be able to update the currently checked out branch with 653 # possibly new history (no fast-forward) git needs to be told 654 # that's ok 655 logger.info('Syncing source files including patches to git branch: %s' % devbranch) 656 bb.process.run('git fetch --update-head-ok --force file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 657 except bb.process.ExecutionError as e: 658 raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e)) 659 660 else: 661 shutil.move(srcsubdir, srctree) 662 663 if is_kernel_yocto: 664 logger.info('Copying kernel config to srctree') 665 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 666 667 finally: 668 if appendbackup: 669 shutil.copyfile(appendbackup, appendfile) 670 elif os.path.exists(appendfile): 671 os.remove(appendfile) 672 if keep_temp: 673 logger.info('Preserving temporary directory %s' % tempdir) 674 else: 675 shutil.rmtree(tempdir) 676 return initial_rev, srcsubdir_rel 677 678def _add_md5(config, recipename, filename): 679 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 680 import bb.utils 681 682 def addfile(fn): 683 md5 = bb.utils.md5_file(fn) 684 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 685 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 686 f.seek(0, os.SEEK_SET) 687 if not md5_str in f.read(): 688 f.write(md5_str) 689 690 if os.path.isdir(filename): 691 for root, _, files in os.walk(filename): 692 for f in files: 693 addfile(os.path.join(root, f)) 694 else: 695 addfile(filename) 696 697def _check_preserve(config, recipename): 698 """Check if a file was manually changed and needs to be saved in 'attic' 699 directory""" 700 import bb.utils 701 origfile = os.path.join(config.workspace_path, '.devtool_md5') 702 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 703 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 704 with open(origfile, 'r') as f: 705 with open(newfile, 'w') as tf: 706 for line in f.readlines(): 707 splitline = line.rstrip().split('|') 708 if splitline[0] == recipename: 709 removefile = os.path.join(config.workspace_path, splitline[1]) 710 try: 711 md5 = bb.utils.md5_file(removefile) 712 except IOError as err: 713 if err.errno == 2: 714 # File no longer exists, skip it 715 continue 716 else: 717 raise 718 if splitline[2] != md5: 719 bb.utils.mkdirhier(preservepath) 720 preservefile = os.path.basename(removefile) 721 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 722 shutil.move(removefile, os.path.join(preservepath, preservefile)) 723 else: 724 os.remove(removefile) 725 else: 726 tf.write(line) 727 bb.utils.rename(newfile, origfile) 728 729def get_staging_kver(srcdir): 730 # Kernel version from work-shared 731 kerver = [] 732 staging_kerVer="" 733 if os.path.exists(srcdir) and os.listdir(srcdir): 734 with open(os.path.join(srcdir, "Makefile")) as f: 735 version = [next(f) for x in range(5)][1:4] 736 for word in version: 737 kerver.append(word.split('= ')[1].split('\n')[0]) 738 staging_kerVer = ".".join(kerver) 739 return staging_kerVer 740 741def get_staging_kbranch(srcdir): 742 staging_kbranch = "" 743 if os.path.exists(srcdir) and os.listdir(srcdir): 744 (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir) 745 staging_kbranch = "".join(branch.split('\n')[0]) 746 return staging_kbranch 747 748def get_real_srctree(srctree, s, workdir): 749 # Check that recipe isn't using a shared workdir 750 s = os.path.abspath(s) 751 workdir = os.path.abspath(workdir) 752 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 753 # Handle if S is set to a subdirectory of the source 754 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 755 srctree = os.path.join(srctree, srcsubdir) 756 return srctree 757 758def modify(args, config, basepath, workspace): 759 """Entry point for the devtool 'modify' subcommand""" 760 import bb 761 import oe.recipeutils 762 import oe.patch 763 import oe.path 764 765 if args.recipename in workspace: 766 raise DevtoolError("recipe %s is already in your workspace" % 767 args.recipename) 768 769 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 770 try: 771 rd = parse_recipe(config, tinfoil, args.recipename, True) 772 if not rd: 773 return 1 774 775 pn = rd.getVar('PN') 776 if pn != args.recipename: 777 logger.info('Mapping %s to %s' % (args.recipename, pn)) 778 if pn in workspace: 779 raise DevtoolError("recipe %s is already in your workspace" % 780 pn) 781 782 if args.srctree: 783 srctree = os.path.abspath(args.srctree) 784 else: 785 srctree = get_default_srctree(config, pn) 786 787 if args.no_extract and not os.path.isdir(srctree): 788 raise DevtoolError("--no-extract specified and source path %s does " 789 "not exist or is not a directory" % 790 srctree) 791 792 recipefile = rd.getVar('FILE') 793 appendfile = recipe_to_append(recipefile, config, args.wildcard) 794 if os.path.exists(appendfile): 795 raise DevtoolError("Another variant of recipe %s is already in your " 796 "workspace (only one variant of a recipe can " 797 "currently be worked on at once)" 798 % pn) 799 800 _check_compatible_recipe(pn, rd) 801 802 initial_revs = {} 803 commits = {} 804 check_commits = False 805 806 if bb.data.inherits_class('kernel-yocto', rd): 807 # Current set kernel version 808 kernelVersion = rd.getVar('LINUX_VERSION') 809 srcdir = rd.getVar('STAGING_KERNEL_DIR') 810 kbranch = rd.getVar('KBRANCH') 811 812 staging_kerVer = get_staging_kver(srcdir) 813 staging_kbranch = get_staging_kbranch(srcdir) 814 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 815 oe.path.copyhardlinktree(srcdir, srctree) 816 unpackdir = rd.getVar('UNPACKDIR') 817 srcsubdir = rd.getVar('S') 818 819 # Add locally copied files to gitignore as we add back to the metadata directly 820 local_files = oe.recipeutils.get_recipe_local_files(rd) 821 srcabspath = os.path.abspath(srcsubdir) 822 local_files = [fname for fname in local_files if 823 os.path.exists(os.path.join(unpackdir, fname)) and 824 srcabspath == unpackdir] 825 if local_files: 826 with open(os.path.join(srctree, '.gitignore'), 'a+') as f: 827 f.write('# Ignore local files, by default. Remove following lines' 828 'if you want to commit the directory to Git\n') 829 for fname in local_files: 830 f.write('%s\n' % fname) 831 832 task = 'do_configure' 833 res = tinfoil.build_targets(pn, task, handle_events=True) 834 835 # Copy .config to workspace 836 kconfpath = rd.getVar('B') 837 logger.info('Copying kernel config to workspace') 838 shutil.copy2(os.path.join(kconfpath, '.config'), srctree) 839 840 # Set this to true, we still need to get initial_rev 841 # by parsing the git repo 842 args.no_extract = True 843 844 if not args.no_extract: 845 initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 846 if not initial_revs["."]: 847 return 1 848 logger.info('Source tree extracted to %s' % srctree) 849 850 if os.path.exists(os.path.join(srctree, '.git')): 851 # Get list of commits since this revision 852 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) 853 commits["."] = stdout.split() 854 check_commits = True 855 try: 856 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) 857 except bb.process.ExecutionError: 858 stdout = "" 859 for line in stdout.splitlines(): 860 (rev, submodule_path) = line.split() 861 submodule = os.path.relpath(submodule_path, srctree) 862 initial_revs[submodule] = rev 863 (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path) 864 commits[submodule] = stdout.split() 865 else: 866 if os.path.exists(os.path.join(srctree, '.git')): 867 # Check if it's a tree previously extracted by us. This is done 868 # by ensuring that devtool-base and args.branch (devtool) exist. 869 # The check_commits logic will cause an exception if either one 870 # of these doesn't exist 871 try: 872 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 873 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 874 except bb.process.ExecutionError: 875 stdout = '' 876 if stdout: 877 check_commits = True 878 for line in stdout.splitlines(): 879 if line.startswith('*'): 880 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 881 initial_revs["."] = stdout.rstrip() 882 if "." not in initial_revs: 883 # Otherwise, just grab the head revision 884 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 885 initial_revs["."] = stdout.rstrip() 886 887 branch_patches = {} 888 if check_commits: 889 # Check if there are override branches 890 (stdout, _) = bb.process.run('git branch', cwd=srctree) 891 branches = [] 892 for line in stdout.rstrip().splitlines(): 893 branchname = line[2:].rstrip() 894 if branchname.startswith(override_branch_prefix): 895 branches.append(branchname) 896 if branches: 897 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 898 branches.insert(0, args.branch) 899 seen_patches = [] 900 for branch in branches: 901 branch_patches[branch] = [] 902 (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree) 903 for sha1 in stdout.splitlines(): 904 notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip()) 905 origpatch = notes.get(oe.patch.GitApplyTree.original_patch) 906 if origpatch and origpatch not in seen_patches: 907 seen_patches.append(origpatch) 908 branch_patches[branch].append(origpatch) 909 910 # Need to grab this here in case the source is within a subdirectory 911 srctreebase = srctree 912 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 913 914 bb.utils.mkdirhier(os.path.dirname(appendfile)) 915 with open(appendfile, 'w') as f: 916 # if not present, add type=git-dependency to the secondary sources 917 # (non local files) so they can be extracted correctly when building a recipe after 918 # doing a devtool modify on it 919 src_uri = rd.getVar('SRC_URI').split() 920 src_uri_append = [] 921 src_uri_remove = [] 922 923 # Assume first entry is main source extracted in ${S} so skip it 924 src_uri = src_uri[1::] 925 926 # Add "type=git-dependency" to all non local sources 927 for url in src_uri: 928 if not url.startswith('file://') and not 'type=' in url: 929 src_uri_remove.append(url) 930 src_uri_append.append('%s;type=git-dependency' % url) 931 932 if src_uri_remove: 933 f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove)) 934 f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append)) 935 936 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 937 # Local files can be modified/tracked in separate subdir under srctree 938 # Mostly useful for packages with S != WORKDIR 939 f.write('FILESPATH:prepend := "%s:"\n' % 940 os.path.join(srctreebase, 'oe-local-files')) 941 f.write('# srctreebase: %s\n' % srctreebase) 942 943 f.write('\ninherit externalsrc\n') 944 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 945 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 946 947 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 948 if b_is_s: 949 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 950 951 if bb.data.inherits_class('kernel', rd): 952 f.write('\ndo_kernel_configme:prepend() {\n' 953 ' if [ -e ${S}/.config ]; then\n' 954 ' mv ${S}/.config ${S}/.config.old\n' 955 ' fi\n' 956 '}\n') 957 if rd.getVarFlag('do_menuconfig', 'task'): 958 f.write('\ndo_configure:append() {\n' 959 ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n' 960 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' 961 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' 962 ' fi\n' 963 '}\n') 964 if initial_revs: 965 for name, rev in initial_revs.items(): 966 f.write('\n# initial_rev %s: %s\n' % (name, rev)) 967 if name in commits: 968 for commit in commits[name]: 969 f.write('# commit %s: %s\n' % (name, commit)) 970 if branch_patches: 971 for branch in branch_patches: 972 if branch == args.branch: 973 continue 974 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 975 976 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 977 978 _add_md5(config, pn, appendfile) 979 980 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 981 982 finally: 983 tinfoil.shutdown() 984 985 return 0 986 987 988def rename(args, config, basepath, workspace): 989 """Entry point for the devtool 'rename' subcommand""" 990 import bb 991 import oe.recipeutils 992 993 check_workspace_recipe(workspace, args.recipename) 994 995 if not (args.newname or args.version): 996 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 997 998 recipefile = workspace[args.recipename]['recipefile'] 999 if not recipefile: 1000 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1001 1002 if args.newname and args.newname != args.recipename: 1003 reason = oe.recipeutils.validate_pn(args.newname) 1004 if reason: 1005 raise DevtoolError(reason) 1006 newname = args.newname 1007 else: 1008 newname = args.recipename 1009 1010 append = workspace[args.recipename]['bbappend'] 1011 appendfn = os.path.splitext(os.path.basename(append))[0] 1012 splitfn = appendfn.split('_') 1013 if len(splitfn) > 1: 1014 origfnver = appendfn.split('_')[1] 1015 else: 1016 origfnver = '' 1017 1018 recipefilemd5 = None 1019 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1020 try: 1021 rd = parse_recipe(config, tinfoil, args.recipename, True) 1022 if not rd: 1023 return 1 1024 1025 bp = rd.getVar('BP') 1026 bpn = rd.getVar('BPN') 1027 if newname != args.recipename: 1028 localdata = rd.createCopy() 1029 localdata.setVar('PN', newname) 1030 newbpn = localdata.getVar('BPN') 1031 else: 1032 newbpn = bpn 1033 s = rd.getVar('S', False) 1034 src_uri = rd.getVar('SRC_URI', False) 1035 pv = rd.getVar('PV') 1036 1037 # Correct variable values that refer to the upstream source - these 1038 # values must stay the same, so if the name/version are changing then 1039 # we need to fix them up 1040 new_s = s 1041 new_src_uri = src_uri 1042 if newbpn != bpn: 1043 # ${PN} here is technically almost always incorrect, but people do use it 1044 new_s = new_s.replace('${BPN}', bpn) 1045 new_s = new_s.replace('${PN}', bpn) 1046 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1047 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1048 new_src_uri = new_src_uri.replace('${PN}', bpn) 1049 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1050 if args.version and origfnver == pv: 1051 new_s = new_s.replace('${PV}', pv) 1052 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1053 new_src_uri = new_src_uri.replace('${PV}', pv) 1054 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1055 patchfields = {} 1056 if new_s != s: 1057 patchfields['S'] = new_s 1058 if new_src_uri != src_uri: 1059 patchfields['SRC_URI'] = new_src_uri 1060 if patchfields: 1061 recipefilemd5 = bb.utils.md5_file(recipefile) 1062 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1063 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1064 finally: 1065 tinfoil.shutdown() 1066 1067 if args.version: 1068 newver = args.version 1069 else: 1070 newver = origfnver 1071 1072 if newver: 1073 newappend = '%s_%s.bbappend' % (newname, newver) 1074 newfile = '%s_%s.bb' % (newname, newver) 1075 else: 1076 newappend = '%s.bbappend' % newname 1077 newfile = '%s.bb' % newname 1078 1079 oldrecipedir = os.path.dirname(recipefile) 1080 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1081 if oldrecipedir != newrecipedir: 1082 bb.utils.mkdirhier(newrecipedir) 1083 1084 newappend = os.path.join(os.path.dirname(append), newappend) 1085 newfile = os.path.join(newrecipedir, newfile) 1086 1087 # Rename bbappend 1088 logger.info('Renaming %s to %s' % (append, newappend)) 1089 bb.utils.rename(append, newappend) 1090 # Rename recipe file 1091 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1092 bb.utils.rename(recipefile, newfile) 1093 1094 # Rename source tree if it's the default path 1095 appendmd5 = None 1096 if not args.no_srctree: 1097 srctree = workspace[args.recipename]['srctree'] 1098 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1099 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1100 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1101 shutil.move(srctree, newsrctree) 1102 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1103 appendmd5 = bb.utils.md5_file(newappend) 1104 appendlines = [] 1105 with open(newappend, 'r') as f: 1106 for line in f: 1107 appendlines.append(line) 1108 with open(newappend, 'w') as f: 1109 for line in appendlines: 1110 if srctree in line: 1111 line = line.replace(srctree, newsrctree) 1112 f.write(line) 1113 newappendmd5 = bb.utils.md5_file(newappend) 1114 1115 bpndir = None 1116 newbpndir = None 1117 if newbpn != bpn: 1118 bpndir = os.path.join(oldrecipedir, bpn) 1119 if os.path.exists(bpndir): 1120 newbpndir = os.path.join(newrecipedir, newbpn) 1121 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1122 shutil.move(bpndir, newbpndir) 1123 1124 bpdir = None 1125 newbpdir = None 1126 if newver != origfnver or newbpn != bpn: 1127 bpdir = os.path.join(oldrecipedir, bp) 1128 if os.path.exists(bpdir): 1129 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1130 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1131 shutil.move(bpdir, newbpdir) 1132 1133 if oldrecipedir != newrecipedir: 1134 # Move any stray files and delete the old recipe directory 1135 for entry in os.listdir(oldrecipedir): 1136 oldpath = os.path.join(oldrecipedir, entry) 1137 newpath = os.path.join(newrecipedir, entry) 1138 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1139 shutil.move(oldpath, newpath) 1140 os.rmdir(oldrecipedir) 1141 1142 # Now take care of entries in .devtool_md5 1143 md5entries = [] 1144 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1145 for line in f: 1146 md5entries.append(line) 1147 1148 if bpndir and newbpndir: 1149 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1150 else: 1151 relbpndir = None 1152 if bpdir and newbpdir: 1153 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1154 else: 1155 relbpdir = None 1156 1157 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1158 for entry in md5entries: 1159 splitentry = entry.rstrip().split('|') 1160 if len(splitentry) > 2: 1161 if splitentry[0] == args.recipename: 1162 splitentry[0] = newname 1163 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1164 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1165 if appendmd5 and splitentry[2] == appendmd5: 1166 splitentry[2] = newappendmd5 1167 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1168 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1169 if recipefilemd5 and splitentry[2] == recipefilemd5: 1170 splitentry[2] = newrecipefilemd5 1171 elif relbpndir and splitentry[1].startswith(relbpndir): 1172 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1173 elif relbpdir and splitentry[1].startswith(relbpdir): 1174 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1175 entry = '|'.join(splitentry) + '\n' 1176 f.write(entry) 1177 return 0 1178 1179 1180def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1181 """Get initial and update rev of a recipe. These are the start point of the 1182 whole patchset and start point for the patches to be re-generated/updated. 1183 """ 1184 import bb 1185 1186 # Get current branch 1187 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1188 cwd=srctree) 1189 branchname = stdout.rstrip() 1190 1191 # Parse initial rev from recipe if not specified 1192 commits = {} 1193 patches = [] 1194 initial_revs = {} 1195 with open(recipe_path, 'r') as f: 1196 for line in f: 1197 pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$' 1198 match = re.search(pattern, line) 1199 if match: 1200 name = match.group(1) 1201 rev = match.group(2) 1202 if line.startswith('# initial_rev'): 1203 if not (name == "." and initial_rev): 1204 initial_revs[name] = rev 1205 elif line.startswith('# commit') and not force_patch_refresh: 1206 if name not in commits: 1207 commits[name] = [rev] 1208 else: 1209 commits[name].append(rev) 1210 elif line.startswith('# patches_%s:' % branchname): 1211 patches = line.split(':')[-1].strip().split(',') 1212 1213 update_revs = dict(initial_revs) 1214 changed_revs = {} 1215 for name, rev in initial_revs.items(): 1216 # Find first actually changed revision 1217 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1218 rev, cwd=os.path.join(srctree, name)) 1219 newcommits = stdout.split() 1220 if name in commits: 1221 for i in range(min(len(commits[name]), len(newcommits))): 1222 if newcommits[i] == commits[name][i]: 1223 update_revs[name] = commits[name][i] 1224 1225 try: 1226 stdout, _ = bb.process.run('git cherry devtool-patched', 1227 cwd=os.path.join(srctree, name)) 1228 except bb.process.ExecutionError as err: 1229 stdout = None 1230 1231 if stdout is not None and not force_patch_refresh: 1232 for line in stdout.splitlines(): 1233 if line.startswith('+ '): 1234 rev = line.split()[1] 1235 if rev in newcommits: 1236 if name not in changed_revs: 1237 changed_revs[name] = [rev] 1238 else: 1239 changed_revs[name].append(rev) 1240 1241 return initial_revs, update_revs, changed_revs, patches 1242 1243def _remove_file_entries(srcuri, filelist): 1244 """Remove file:// entries from SRC_URI""" 1245 remaining = filelist[:] 1246 entries = [] 1247 for fname in filelist: 1248 basename = os.path.basename(fname) 1249 for i in range(len(srcuri)): 1250 if (srcuri[i].startswith('file://') and 1251 os.path.basename(srcuri[i].split(';')[0]) == basename): 1252 entries.append(srcuri[i]) 1253 remaining.remove(fname) 1254 srcuri.pop(i) 1255 break 1256 return entries, remaining 1257 1258def _replace_srcuri_entry(srcuri, filename, newentry): 1259 """Replace entry corresponding to specified file with a new entry""" 1260 basename = os.path.basename(filename) 1261 for i in range(len(srcuri)): 1262 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1263 srcuri.pop(i) 1264 srcuri.insert(i, newentry) 1265 break 1266 1267def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1268 """Unlink existing patch files""" 1269 1270 dry_run_suffix = ' (dry-run)' if dry_run else '' 1271 1272 for path in files: 1273 if append: 1274 if not destpath: 1275 raise Exception('destpath should be set here') 1276 path = os.path.join(destpath, os.path.basename(path)) 1277 1278 if os.path.exists(path): 1279 if not no_report_remove: 1280 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1281 if not dry_run: 1282 # FIXME "git rm" here would be nice if the file in question is 1283 # tracked 1284 # FIXME there's a chance that this file is referred to by 1285 # another recipe, in which case deleting wouldn't be the 1286 # right thing to do 1287 os.remove(path) 1288 # Remove directory if empty 1289 try: 1290 os.rmdir(os.path.dirname(path)) 1291 except OSError as ose: 1292 if ose.errno != errno.ENOTEMPTY: 1293 raise 1294 1295 1296def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None): 1297 """Export patches from srctree to given location. 1298 Returns three-tuple of dicts: 1299 1. updated - patches that already exist in SRCURI 1300 2. added - new patches that don't exist in SRCURI 1301 3 removed - patches that exist in SRCURI but not in exported patches 1302 In each dict the key is the 'basepath' of the URI and value is: 1303 - for updated and added dicts, a dict with 2 optionnal keys: 1304 - 'path': the absolute path to the existing file in recipe space (if any) 1305 - 'patchdir': the directory in wich the patch should be applied (if any) 1306 - for removed dict, the absolute path to the existing file in recipe space 1307 """ 1308 import oe.recipeutils 1309 from oe.patch import GitApplyTree 1310 updated = OrderedDict() 1311 added = OrderedDict() 1312 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1313 1314 existing_patches = dict((os.path.basename(path), path) for path in 1315 oe.recipeutils.get_recipe_patches(rd)) 1316 logger.debug('Existing patches: %s' % existing_patches) 1317 1318 # Generate patches from Git, exclude local files directory 1319 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1320 GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec) 1321 for dirpath, dirnames, filenames in os.walk(destdir): 1322 new_patches = filenames 1323 reldirpath = os.path.relpath(dirpath, destdir) 1324 for new_patch in new_patches: 1325 # Strip numbering from patch names. If it's a git sequence named patch, 1326 # the numbers might not match up since we are starting from a different 1327 # revision This does assume that people are using unique shortlog 1328 # values, but they ought to be anyway... 1329 new_basename = seqpatch_re.match(new_patch).group(2) 1330 match_name = None 1331 for old_patch in existing_patches: 1332 old_basename = seqpatch_re.match(old_patch).group(2) 1333 old_basename_splitext = os.path.splitext(old_basename) 1334 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1335 old_patch_noext = os.path.splitext(old_patch)[0] 1336 match_name = old_patch_noext 1337 break 1338 elif new_basename == old_basename: 1339 match_name = old_patch 1340 break 1341 if match_name: 1342 # Rename patch files 1343 if new_patch != match_name: 1344 bb.utils.rename(os.path.join(destdir, new_patch), 1345 os.path.join(destdir, match_name)) 1346 # Need to pop it off the list now before checking changed_revs 1347 oldpath = existing_patches.pop(old_patch) 1348 if changed_revs is not None and dirpath in changed_revs: 1349 # Avoid updating patches that have not actually changed 1350 with open(os.path.join(dirpath, match_name), 'r') as f: 1351 firstlineitems = f.readline().split() 1352 # Looking for "From <hash>" line 1353 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1354 if not firstlineitems[1] in changed_revs[dirpath]: 1355 continue 1356 # Recompress if necessary 1357 if oldpath.endswith(('.gz', '.Z')): 1358 bb.process.run(['gzip', match_name], cwd=destdir) 1359 if oldpath.endswith('.gz'): 1360 match_name += '.gz' 1361 else: 1362 match_name += '.Z' 1363 elif oldpath.endswith('.bz2'): 1364 bb.process.run(['bzip2', match_name], cwd=destdir) 1365 match_name += '.bz2' 1366 updated[match_name] = {'path' : oldpath} 1367 if reldirpath != ".": 1368 updated[match_name]['patchdir'] = reldirpath 1369 else: 1370 added[new_patch] = {} 1371 if reldirpath != ".": 1372 added[new_patch]['patchdir'] = reldirpath 1373 1374 return (updated, added, existing_patches) 1375 1376 1377def _create_kconfig_diff(srctree, rd, outfile): 1378 """Create a kconfig fragment""" 1379 # Only update config fragment if both config files exist 1380 orig_config = os.path.join(srctree, '.config.baseline') 1381 new_config = os.path.join(srctree, '.config.new') 1382 if os.path.exists(orig_config) and os.path.exists(new_config): 1383 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1384 '--unchanged-line-format=', orig_config, new_config] 1385 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1386 stderr=subprocess.PIPE) 1387 stdout, stderr = pipe.communicate() 1388 if pipe.returncode == 1: 1389 logger.info("Updating config fragment %s" % outfile) 1390 with open(outfile, 'wb') as fobj: 1391 fobj.write(stdout) 1392 elif pipe.returncode == 0: 1393 logger.info("Would remove config fragment %s" % outfile) 1394 if os.path.exists(outfile): 1395 # Remove fragment file in case of empty diff 1396 logger.info("Removing config fragment %s" % outfile) 1397 os.unlink(outfile) 1398 else: 1399 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1400 return True 1401 return False 1402 1403 1404def _export_local_files(srctree, rd, destdir, srctreebase): 1405 """Copy local files from srctree to given location. 1406 Returns three-tuple of dicts: 1407 1. updated - files that already exist in SRCURI 1408 2. added - new files files that don't exist in SRCURI 1409 3 removed - files that exist in SRCURI but not in exported files 1410 In each dict the key is the 'basepath' of the URI and value is: 1411 - for updated and added dicts, a dict with 1 optionnal key: 1412 - 'path': the absolute path to the existing file in recipe space (if any) 1413 - for removed dict, the absolute path to the existing file in recipe space 1414 """ 1415 import oe.recipeutils 1416 1417 # Find out local files (SRC_URI files that exist in the "recipe space"). 1418 # Local files that reside in srctree are not included in patch generation. 1419 # Instead they are directly copied over the original source files (in 1420 # recipe space). 1421 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1422 1423 new_set = None 1424 updated = OrderedDict() 1425 added = OrderedDict() 1426 removed = OrderedDict() 1427 1428 # Get current branch and return early with empty lists 1429 # if on one of the override branches 1430 # (local files are provided only for the main branch and processing 1431 # them against lists from recipe overrides will result in mismatches 1432 # and broken modifications to recipes). 1433 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1434 cwd=srctree) 1435 branchname = stdout.rstrip() 1436 if branchname.startswith(override_branch_prefix): 1437 return (updated, added, removed) 1438 1439 files = _git_modified(srctree) 1440 #if not files: 1441 # files = _ls_tree(srctree) 1442 for f in files: 1443 fullfile = os.path.join(srctree, f) 1444 if os.path.exists(os.path.join(fullfile, ".git")): 1445 # submodules handled elsewhere 1446 continue 1447 if f not in existing_files: 1448 added[f] = {} 1449 if os.path.isdir(os.path.join(srctree, f)): 1450 shutil.copytree(fullfile, os.path.join(destdir, f)) 1451 else: 1452 shutil.copy2(fullfile, os.path.join(destdir, f)) 1453 elif not os.path.exists(fullfile): 1454 removed[f] = existing_files[f] 1455 elif f in existing_files: 1456 updated[f] = {'path' : existing_files[f]} 1457 if os.path.isdir(os.path.join(srctree, f)): 1458 shutil.copytree(fullfile, os.path.join(destdir, f)) 1459 else: 1460 shutil.copy2(fullfile, os.path.join(destdir, f)) 1461 1462 # Special handling for kernel config 1463 if bb.data.inherits_class('kernel-yocto', rd): 1464 fragment_fn = 'devtool-fragment.cfg' 1465 fragment_path = os.path.join(destdir, fragment_fn) 1466 if _create_kconfig_diff(srctree, rd, fragment_path): 1467 if os.path.exists(fragment_path): 1468 if fragment_fn in removed: 1469 del removed[fragment_fn] 1470 if fragment_fn not in updated and fragment_fn not in added: 1471 added[fragment_fn] = {} 1472 else: 1473 if fragment_fn in updated: 1474 revoved[fragment_fn] = updated[fragment_fn] 1475 del updated[fragment_fn] 1476 1477 # Special handling for cml1, ccmake, etc bbclasses that generated 1478 # configuration fragment files that are consumed as source files 1479 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1480 if bb.data.inherits_class(frag_class, rd): 1481 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1482 if os.path.exists(srcpath): 1483 if frag_name in removed: 1484 del removed[frag_name] 1485 if frag_name not in updated: 1486 added[frag_name] = {} 1487 # copy fragment into destdir 1488 shutil.copy2(srcpath, destdir) 1489 1490 return (updated, added, removed) 1491 1492 1493def _determine_files_dir(rd): 1494 """Determine the appropriate files directory for a recipe""" 1495 recipedir = rd.getVar('FILE_DIRNAME') 1496 for entry in rd.getVar('FILESPATH').split(':'): 1497 relpth = os.path.relpath(entry, recipedir) 1498 if not os.sep in relpth: 1499 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1500 if os.path.isdir(entry): 1501 return entry 1502 return os.path.join(recipedir, rd.getVar('BPN')) 1503 1504 1505def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1506 """Implement the 'srcrev' mode of update-recipe""" 1507 import bb 1508 import oe.recipeutils 1509 1510 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1511 1512 recipefile = rd.getVar('FILE') 1513 recipedir = os.path.basename(recipefile) 1514 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1515 1516 # Get original SRCREV 1517 old_srcrev = rd.getVar('SRCREV') or '' 1518 if old_srcrev == "INVALID": 1519 raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository') 1520 old_srcrev = {'.': old_srcrev} 1521 1522 # Get HEAD revision 1523 try: 1524 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1525 except bb.process.ExecutionError as err: 1526 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1527 (srctree, err)) 1528 srcrev = stdout.strip() 1529 if len(srcrev) != 40: 1530 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1531 1532 destpath = None 1533 remove_files = [] 1534 patchfields = {} 1535 patchfields['SRCREV'] = srcrev 1536 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1537 srcuri = orig_src_uri.split() 1538 tempdir = tempfile.mkdtemp(prefix='devtool') 1539 update_srcuri = False 1540 appendfile = None 1541 try: 1542 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1543 srctreebase = workspace[recipename]['srctreebase'] 1544 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1545 if not no_remove: 1546 # Find list of existing patches in recipe file 1547 patches_dir = tempfile.mkdtemp(dir=tempdir) 1548 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1549 patches_dir) 1550 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1551 1552 # Remove deleted local files and "overlapping" patches 1553 remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value] 1554 if remove_files: 1555 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1556 update_srcuri = True 1557 1558 if appendlayerdir: 1559 files = dict((os.path.join(local_files_dir, key), val) for 1560 key, val in list(upd_f.items()) + list(new_f.items())) 1561 removevalues = {} 1562 if update_srcuri: 1563 removevalues = {'SRC_URI': removedentries} 1564 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1565 if dry_run_outdir: 1566 logger.info('Creating bbappend (dry-run)') 1567 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1568 rd, appendlayerdir, files, wildcardver=wildcard_version, 1569 extralines=patchfields, removevalues=removevalues, 1570 redirect_output=dry_run_outdir) 1571 else: 1572 files_dir = _determine_files_dir(rd) 1573 for basepath, param in upd_f.items(): 1574 path = param['path'] 1575 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1576 if os.path.isabs(basepath): 1577 # Original file (probably with subdir pointing inside source tree) 1578 # so we do not want to move it, just copy 1579 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1580 else: 1581 _move_file(os.path.join(local_files_dir, basepath), path, 1582 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1583 update_srcuri= True 1584 for basepath, param in new_f.items(): 1585 path = param['path'] 1586 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1587 _move_file(os.path.join(local_files_dir, basepath), 1588 os.path.join(files_dir, basepath), 1589 dry_run_outdir=dry_run_outdir, 1590 base_outdir=recipedir) 1591 srcuri.append('file://%s' % basepath) 1592 update_srcuri = True 1593 if update_srcuri: 1594 patchfields['SRC_URI'] = ' '.join(srcuri) 1595 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1596 finally: 1597 shutil.rmtree(tempdir) 1598 if not 'git://' in orig_src_uri: 1599 logger.info('You will need to update SRC_URI within the recipe to ' 1600 'point to a git repository where you have pushed your ' 1601 'changes') 1602 1603 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1604 return True, appendfile, remove_files 1605 1606def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1607 """Implement the 'patch' mode of update-recipe""" 1608 import bb 1609 import oe.recipeutils 1610 1611 recipefile = rd.getVar('FILE') 1612 recipedir = os.path.dirname(recipefile) 1613 append = workspace[recipename]['bbappend'] 1614 if not os.path.exists(append): 1615 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1616 recipename) 1617 srctreebase = workspace[recipename]['srctreebase'] 1618 relpatchdir = os.path.relpath(srctreebase, srctree) 1619 if relpatchdir == '.': 1620 patchdir_params = {} 1621 else: 1622 patchdir_params = {'patchdir': relpatchdir} 1623 1624 def srcuri_entry(basepath, patchdir_params): 1625 if patchdir_params: 1626 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1627 else: 1628 paramstr = '' 1629 return 'file://%s%s' % (basepath, paramstr) 1630 1631 initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1632 if not initial_revs: 1633 raise DevtoolError('Unable to find initial revision - please specify ' 1634 'it with --initial-rev') 1635 1636 appendfile = None 1637 dl_dir = rd.getVar('DL_DIR') 1638 if not dl_dir.endswith('/'): 1639 dl_dir += '/' 1640 1641 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1642 1643 tempdir = tempfile.mkdtemp(prefix='devtool') 1644 try: 1645 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1646 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1647 1648 # Get updated patches from source tree 1649 patches_dir = tempfile.mkdtemp(dir=tempdir) 1650 upd_p, new_p, _ = _export_patches(srctree, rd, update_revs, 1651 patches_dir, changed_revs) 1652 # Get all patches from source tree and check if any should be removed 1653 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1654 _, _, del_p = _export_patches(srctree, rd, initial_revs, 1655 all_patches_dir) 1656 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1657 if filter_patches: 1658 new_p = OrderedDict() 1659 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1660 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) 1661 remove_files = [] 1662 if not no_remove: 1663 # Remove deleted local files and patches 1664 remove_files = list(del_f.values()) + list(del_p.values()) 1665 updatefiles = False 1666 updaterecipe = False 1667 destpath = None 1668 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1669 1670 if appendlayerdir: 1671 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1672 key, val in list(upd_f.items()) + list(new_f.items())) 1673 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1674 key, val in list(upd_p.items()) + list(new_p.items()))) 1675 1676 params = [] 1677 for file, param in files.items(): 1678 patchdir_param = dict(patchdir_params) 1679 patchdir = param.get('patchdir', ".") 1680 if patchdir != "." : 1681 if patchdir_param: 1682 patchdir_param['patchdir'] += patchdir 1683 else: 1684 patchdir_param['patchdir'] = patchdir 1685 params.append(patchdir_param) 1686 1687 if files or remove_files: 1688 removevalues = None 1689 if remove_files: 1690 removedentries, remaining = _remove_file_entries( 1691 srcuri, remove_files) 1692 if removedentries or remaining: 1693 remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for 1694 item in remaining] 1695 removevalues = {'SRC_URI': removedentries + remaining} 1696 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1697 rd, appendlayerdir, files, 1698 wildcardver=wildcard_version, 1699 removevalues=removevalues, 1700 redirect_output=dry_run_outdir, 1701 params=params) 1702 else: 1703 logger.info('No patches or local source files needed updating') 1704 else: 1705 # Update existing files 1706 files_dir = _determine_files_dir(rd) 1707 for basepath, param in upd_f.items(): 1708 path = param['path'] 1709 logger.info('Updating file %s' % basepath) 1710 if os.path.isabs(basepath): 1711 # Original file (probably with subdir pointing inside source tree) 1712 # so we do not want to move it, just copy 1713 _copy_file(basepath, path, 1714 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1715 else: 1716 _move_file(os.path.join(local_files_dir, basepath), path, 1717 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1718 updatefiles = True 1719 for basepath, param in upd_p.items(): 1720 path = param['path'] 1721 patchdir = param.get('patchdir', ".") 1722 if patchdir != "." : 1723 patchdir_param = dict(patchdir_params) 1724 if patchdir_param: 1725 patchdir_param['patchdir'] += patchdir 1726 else: 1727 patchdir_param['patchdir'] = patchdir 1728 patchfn = os.path.join(patches_dir, patchdir, basepath) 1729 if os.path.dirname(path) + '/' == dl_dir: 1730 # This is a a downloaded patch file - we now need to 1731 # replace the entry in SRC_URI with our local version 1732 logger.info('Replacing remote patch %s with updated local version' % basepath) 1733 path = os.path.join(files_dir, basepath) 1734 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param)) 1735 updaterecipe = True 1736 else: 1737 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1738 _move_file(patchfn, path, 1739 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1740 updatefiles = True 1741 # Add any new files 1742 for basepath, param in new_f.items(): 1743 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1744 _move_file(os.path.join(local_files_dir, basepath), 1745 os.path.join(files_dir, basepath), 1746 dry_run_outdir=dry_run_outdir, 1747 base_outdir=recipedir) 1748 srcuri.append(srcuri_entry(basepath, patchdir_params)) 1749 updaterecipe = True 1750 for basepath, param in new_p.items(): 1751 patchdir = param.get('patchdir', ".") 1752 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1753 _move_file(os.path.join(patches_dir, patchdir, basepath), 1754 os.path.join(files_dir, basepath), 1755 dry_run_outdir=dry_run_outdir, 1756 base_outdir=recipedir) 1757 params = dict(patchdir_params) 1758 if patchdir != "." : 1759 if params: 1760 params['patchdir'] += patchdir 1761 else: 1762 params['patchdir'] = patchdir 1763 1764 srcuri.append(srcuri_entry(basepath, params)) 1765 updaterecipe = True 1766 # Update recipe, if needed 1767 if _remove_file_entries(srcuri, remove_files)[0]: 1768 updaterecipe = True 1769 if updaterecipe: 1770 if not dry_run_outdir: 1771 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1772 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1773 {'SRC_URI': ' '.join(srcuri)}, 1774 redirect_output=dry_run_outdir) 1775 elif not updatefiles: 1776 # Neither patches nor recipe were updated 1777 logger.info('No patches or files need updating') 1778 return False, None, [] 1779 finally: 1780 shutil.rmtree(tempdir) 1781 1782 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1783 return True, appendfile, remove_files 1784 1785def _guess_recipe_update_mode(srctree, rdata): 1786 """Guess the recipe update mode to use""" 1787 src_uri = (rdata.getVar('SRC_URI') or '').split() 1788 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1789 if not git_uris: 1790 return 'patch' 1791 # Just use the first URI for now 1792 uri = git_uris[0] 1793 # Check remote branch 1794 params = bb.fetch.decodeurl(uri)[5] 1795 upstr_branch = params['branch'] if 'branch' in params else 'master' 1796 # Check if current branch HEAD is found in upstream branch 1797 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1798 head_rev = stdout.rstrip() 1799 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1800 cwd=srctree) 1801 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1802 if 'origin/' + upstr_branch in remote_brs: 1803 return 'srcrev' 1804 1805 return 'patch' 1806 1807def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1808 srctree = workspace[recipename]['srctree'] 1809 if mode == 'auto': 1810 mode = _guess_recipe_update_mode(srctree, rd) 1811 1812 override_branches = [] 1813 mainbranch = None 1814 startbranch = None 1815 if not no_overrides: 1816 stdout, _ = bb.process.run('git branch', cwd=srctree) 1817 other_branches = [] 1818 for line in stdout.splitlines(): 1819 branchname = line[2:] 1820 if line.startswith('* '): 1821 if 'HEAD' in line: 1822 raise DevtoolError('Detached HEAD - please check out a branch, e.g., "devtool"') 1823 startbranch = branchname 1824 if branchname.startswith(override_branch_prefix): 1825 override_branches.append(branchname) 1826 else: 1827 other_branches.append(branchname) 1828 1829 if override_branches: 1830 logger.debug('_update_recipe: override branches: %s' % override_branches) 1831 logger.debug('_update_recipe: other branches: %s' % other_branches) 1832 if startbranch.startswith(override_branch_prefix): 1833 if len(other_branches) == 1: 1834 mainbranch = other_branches[1] 1835 else: 1836 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1837 else: 1838 mainbranch = startbranch 1839 1840 checkedout = None 1841 anyupdated = False 1842 appendfile = None 1843 allremoved = [] 1844 if override_branches: 1845 logger.info('Handling main branch (%s)...' % mainbranch) 1846 if startbranch != mainbranch: 1847 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1848 checkedout = mainbranch 1849 try: 1850 branchlist = [mainbranch] + override_branches 1851 for branch in branchlist: 1852 crd = bb.data.createCopy(rd) 1853 if branch != mainbranch: 1854 logger.info('Handling branch %s...' % branch) 1855 override = branch[len(override_branch_prefix):] 1856 crd.appendVar('OVERRIDES', ':%s' % override) 1857 bb.process.run('git checkout %s' % branch, cwd=srctree) 1858 checkedout = branch 1859 1860 if mode == 'srcrev': 1861 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1862 elif mode == 'patch': 1863 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1864 else: 1865 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1866 if updated: 1867 anyupdated = True 1868 if appendf: 1869 appendfile = appendf 1870 allremoved.extend(removed) 1871 finally: 1872 if startbranch and checkedout != startbranch: 1873 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1874 1875 return anyupdated, appendfile, allremoved 1876 1877def update_recipe(args, config, basepath, workspace): 1878 """Entry point for the devtool 'update-recipe' subcommand""" 1879 check_workspace_recipe(workspace, args.recipename) 1880 1881 if args.append: 1882 if not os.path.exists(args.append): 1883 raise DevtoolError('bbappend destination layer directory "%s" ' 1884 'does not exist' % args.append) 1885 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1886 raise DevtoolError('conf/layer.conf not found in bbappend ' 1887 'destination layer "%s"' % args.append) 1888 1889 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1890 try: 1891 1892 rd = parse_recipe(config, tinfoil, args.recipename, True) 1893 if not rd: 1894 return 1 1895 1896 dry_run_output = None 1897 dry_run_outdir = None 1898 if args.dry_run: 1899 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1900 dry_run_outdir = dry_run_output.name 1901 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1902 1903 if updated: 1904 rf = rd.getVar('FILE') 1905 if rf.startswith(config.workspace_path): 1906 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1907 finally: 1908 tinfoil.shutdown() 1909 1910 return 0 1911 1912 1913def status(args, config, basepath, workspace): 1914 """Entry point for the devtool 'status' subcommand""" 1915 if workspace: 1916 for recipe, value in sorted(workspace.items()): 1917 recipefile = value['recipefile'] 1918 if recipefile: 1919 recipestr = ' (%s)' % recipefile 1920 else: 1921 recipestr = '' 1922 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1923 else: 1924 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1925 return 0 1926 1927 1928def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1929 """Reset one or more recipes""" 1930 import oe.path 1931 1932 def clean_preferred_provider(pn, layerconf_path): 1933 """Remove PREFERRED_PROVIDER from layer.conf'""" 1934 import re 1935 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1936 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1937 pprovider_found = False 1938 with open(layerconf_file, 'r') as f: 1939 lines = f.readlines() 1940 with open(new_layerconf_file, 'a') as nf: 1941 for line in lines: 1942 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1943 if not re.match(pprovider_exp, line): 1944 nf.write(line) 1945 else: 1946 pprovider_found = True 1947 if pprovider_found: 1948 shutil.move(new_layerconf_file, layerconf_file) 1949 else: 1950 os.remove(new_layerconf_file) 1951 1952 if recipes and not no_clean: 1953 if len(recipes) == 1: 1954 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 1955 else: 1956 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 1957 # If the recipe file itself was created in the workspace, and 1958 # it uses BBCLASSEXTEND, then we need to also clean the other 1959 # variants 1960 targets = [] 1961 for recipe in recipes: 1962 targets.append(recipe) 1963 recipefile = workspace[recipe]['recipefile'] 1964 if recipefile and os.path.exists(recipefile): 1965 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 1966 try: 1967 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 1968 except bb.process.ExecutionError as e: 1969 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 1970 'wish, you may specify -n/--no-clean to ' 1971 'skip running this command when resetting' % 1972 (e.command, e.stdout)) 1973 1974 for pn in recipes: 1975 _check_preserve(config, pn) 1976 1977 appendfile = workspace[pn]['bbappend'] 1978 if os.path.exists(appendfile): 1979 # This shouldn't happen, but is possible if devtool errored out prior to 1980 # writing the md5 file. We need to delete this here or the recipe won't 1981 # actually be reset 1982 os.remove(appendfile) 1983 1984 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 1985 def preservedir(origdir): 1986 if os.path.exists(origdir): 1987 for root, dirs, files in os.walk(origdir): 1988 for fn in files: 1989 logger.warning('Preserving %s in %s' % (fn, preservepath)) 1990 _move_file(os.path.join(origdir, fn), 1991 os.path.join(preservepath, fn)) 1992 for dn in dirs: 1993 preservedir(os.path.join(root, dn)) 1994 os.rmdir(origdir) 1995 1996 recipefile = workspace[pn]['recipefile'] 1997 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 1998 # This should always be true if recipefile is set, but just in case 1999 preservedir(os.path.dirname(recipefile)) 2000 # We don't automatically create this dir next to appends, but the user can 2001 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 2002 2003 srctreebase = workspace[pn]['srctreebase'] 2004 if os.path.isdir(srctreebase): 2005 if os.listdir(srctreebase): 2006 if remove_work: 2007 logger.info('-r argument used on %s, removing source tree.' 2008 ' You will lose any unsaved work' %pn) 2009 shutil.rmtree(srctreebase) 2010 else: 2011 # We don't want to risk wiping out any work in progress 2012 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): 2013 from datetime import datetime 2014 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S"))) 2015 logger.info('Preserving source tree in %s\nIf you no ' 2016 'longer need it then please delete it manually.\n' 2017 'It is also possible to reuse it via devtool source tree argument.' 2018 % preservesrc) 2019 bb.utils.mkdirhier(os.path.dirname(preservesrc)) 2020 shutil.move(srctreebase, preservesrc) 2021 else: 2022 logger.info('Leaving source tree %s as-is; if you no ' 2023 'longer need it then please delete it manually' 2024 % srctreebase) 2025 else: 2026 # This is unlikely, but if it's empty we can just remove it 2027 os.rmdir(srctreebase) 2028 2029 clean_preferred_provider(pn, config.workspace_path) 2030 2031def reset(args, config, basepath, workspace): 2032 """Entry point for the devtool 'reset' subcommand""" 2033 import bb 2034 import shutil 2035 2036 recipes = "" 2037 2038 if args.recipename: 2039 if args.all: 2040 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 2041 else: 2042 for recipe in args.recipename: 2043 check_workspace_recipe(workspace, recipe, checksrc=False) 2044 elif not args.all: 2045 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2046 "reset all recipes") 2047 if args.all: 2048 recipes = list(workspace.keys()) 2049 else: 2050 recipes = args.recipename 2051 2052 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2053 2054 return 0 2055 2056 2057def _get_layer(layername, d): 2058 """Determine the base layer path for the specified layer name/path""" 2059 layerdirs = d.getVar('BBLAYERS').split() 2060 layers = {} # {basename: layer_paths} 2061 for p in layerdirs: 2062 bn = os.path.basename(p) 2063 if bn not in layers: 2064 layers[bn] = [p] 2065 else: 2066 layers[bn].append(p) 2067 # Provide some shortcuts 2068 if layername.lower() in ['oe-core', 'openembedded-core']: 2069 layername = 'meta' 2070 layer_paths = layers.get(layername, None) 2071 if not layer_paths: 2072 return os.path.abspath(layername) 2073 elif len(layer_paths) == 1: 2074 return os.path.abspath(layer_paths[0]) 2075 else: 2076 # multiple layers having the same base name 2077 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 2078 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 2079 return os.path.abspath(layer_paths[0]) 2080 2081 2082def finish(args, config, basepath, workspace): 2083 """Entry point for the devtool 'finish' subcommand""" 2084 import bb 2085 import oe.recipeutils 2086 2087 check_workspace_recipe(workspace, args.recipename) 2088 2089 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2090 2091 # Grab the equivalent of COREBASE without having to initialise tinfoil 2092 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2093 2094 srctree = workspace[args.recipename]['srctree'] 2095 check_git_repo_op(srctree, [corebasedir]) 2096 dirty = check_git_repo_dirty(srctree) 2097 if dirty: 2098 if args.force: 2099 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2100 else: 2101 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2102 2103 no_clean = args.no_clean 2104 remove_work=args.remove_work 2105 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2106 try: 2107 rd = parse_recipe(config, tinfoil, args.recipename, True) 2108 if not rd: 2109 return 1 2110 2111 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2112 recipefile = rd.getVar('FILE') 2113 recipedir = os.path.dirname(recipefile) 2114 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2115 2116 if not os.path.isdir(destlayerdir): 2117 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2118 2119 if os.path.abspath(destlayerdir) == config.workspace_path: 2120 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2121 2122 # If it's an upgrade, grab the original path 2123 origpath = None 2124 origfilelist = None 2125 append = workspace[args.recipename]['bbappend'] 2126 with open(append, 'r') as f: 2127 for line in f: 2128 if line.startswith('# original_path:'): 2129 origpath = line.split(':')[1].strip() 2130 elif line.startswith('# original_files:'): 2131 origfilelist = line.split(':')[1].split() 2132 2133 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2134 2135 if origlayerdir == config.workspace_path: 2136 # Recipe file itself is in workspace, update it there first 2137 appendlayerdir = None 2138 origrelpath = None 2139 if origpath: 2140 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2141 if origlayerpath: 2142 origrelpath = os.path.relpath(origpath, origlayerpath) 2143 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2144 if not destpath: 2145 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2146 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2147 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2148 if not os.path.abspath(destlayerbasedir) in layerdirs: 2149 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2150 2151 elif destlayerdir == origlayerdir: 2152 # Same layer, update the original recipe 2153 appendlayerdir = None 2154 destpath = None 2155 else: 2156 # Create/update a bbappend in the specified layer 2157 appendlayerdir = destlayerdir 2158 destpath = None 2159 2160 # Actually update the recipe / bbappend 2161 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2162 dry_run_output = None 2163 dry_run_outdir = None 2164 if args.dry_run: 2165 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2166 dry_run_outdir = dry_run_output.name 2167 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2168 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2169 2170 # Remove any old files in the case of an upgrade 2171 if removing_original: 2172 for fn in origfilelist: 2173 fnp = os.path.join(origpath, fn) 2174 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2175 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2176 if not args.dry_run: 2177 try: 2178 os.remove(fnp) 2179 except FileNotFoundError: 2180 pass 2181 2182 if origlayerdir == config.workspace_path and destpath: 2183 # Recipe file itself is in the workspace - need to move it and any 2184 # associated files to the specified layer 2185 no_clean = True 2186 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2187 for root, _, files in os.walk(recipedir): 2188 for fn in files: 2189 srcpath = os.path.join(root, fn) 2190 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2191 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2192 destfp = os.path.join(destdir, fn) 2193 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2194 2195 if dry_run_outdir: 2196 import difflib 2197 comparelist = [] 2198 for root, _, files in os.walk(dry_run_outdir): 2199 for fn in files: 2200 outf = os.path.join(root, fn) 2201 relf = os.path.relpath(outf, dry_run_outdir) 2202 logger.debug('dry-run: output file %s' % relf) 2203 if fn.endswith('.bb'): 2204 if origfilelist and origpath and destpath: 2205 # Need to match this up with the pre-upgrade recipe file 2206 for origf in origfilelist: 2207 if origf.endswith('.bb'): 2208 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2209 outf, 2210 os.path.abspath(os.path.join(destpath, relf)))) 2211 break 2212 else: 2213 # Compare to the existing recipe 2214 comparelist.append((recipefile, outf, recipefile)) 2215 elif fn.endswith('.bbappend'): 2216 if appendfile: 2217 if os.path.exists(appendfile): 2218 comparelist.append((appendfile, outf, appendfile)) 2219 else: 2220 comparelist.append((None, outf, appendfile)) 2221 else: 2222 if destpath: 2223 recipedest = destpath 2224 elif appendfile: 2225 recipedest = os.path.dirname(appendfile) 2226 else: 2227 recipedest = os.path.dirname(recipefile) 2228 destfp = os.path.join(recipedest, relf) 2229 if os.path.exists(destfp): 2230 comparelist.append((destfp, outf, destfp)) 2231 output = '' 2232 for oldfile, newfile, newfileshow in comparelist: 2233 if oldfile: 2234 with open(oldfile, 'r') as f: 2235 oldlines = f.readlines() 2236 else: 2237 oldfile = '/dev/null' 2238 oldlines = [] 2239 with open(newfile, 'r') as f: 2240 newlines = f.readlines() 2241 if not newfileshow: 2242 newfileshow = newfile 2243 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2244 difflines = list(diff) 2245 if difflines: 2246 output += ''.join(difflines) 2247 if output: 2248 logger.info('Diff of changed files:\n%s' % output) 2249 finally: 2250 tinfoil.shutdown() 2251 2252 # Everything else has succeeded, we can now reset 2253 if args.dry_run: 2254 logger.info('Resetting recipe (dry-run)') 2255 else: 2256 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2257 2258 return 0 2259 2260 2261def get_default_srctree(config, recipename=''): 2262 """Get the default srctree path""" 2263 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2264 if recipename: 2265 return os.path.join(srctreeparent, 'sources', recipename) 2266 else: 2267 return os.path.join(srctreeparent, 'sources') 2268 2269def register_commands(subparsers, context): 2270 """Register devtool subcommands from this plugin""" 2271 2272 defsrctree = get_default_srctree(context.config) 2273 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2274 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2275 group='starting', order=100) 2276 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2277 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2278 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2279 group = parser_add.add_mutually_exclusive_group() 2280 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2281 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2282 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2283 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2284 parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true") 2285 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2286 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2287 group = parser_add.add_mutually_exclusive_group() 2288 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2289 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2290 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2291 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2292 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2293 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2294 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2295 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2296 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2297 2298 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2299 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2300 group='starting', order=90) 2301 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2302 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2303 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2304 group = parser_modify.add_mutually_exclusive_group() 2305 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2306 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2307 group = parser_modify.add_mutually_exclusive_group() 2308 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2309 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2310 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2311 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2312 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2313 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2314 2315 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2316 description='Extracts the source for an existing recipe', 2317 group='advanced') 2318 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2319 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2320 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2321 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2322 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2323 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2324 2325 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2326 description='Synchronize the previously extracted source tree for an existing recipe', 2327 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2328 group='advanced') 2329 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2330 parser_sync.add_argument('srctree', help='Path to the source tree') 2331 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2332 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2333 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2334 2335 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2336 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2337 group='working', order=10) 2338 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2339 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2340 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2341 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2342 parser_rename.set_defaults(func=rename) 2343 2344 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2345 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2346 group='working', order=-90) 2347 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2348 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2349 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2350 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2351 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2352 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2353 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2354 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2355 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2356 parser_update_recipe.set_defaults(func=update_recipe) 2357 2358 parser_status = subparsers.add_parser('status', help='Show workspace status', 2359 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2360 group='info', order=100) 2361 parser_status.set_defaults(func=status) 2362 2363 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2364 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2365 group='working', order=-100) 2366 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2367 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2368 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2369 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2370 parser_reset.set_defaults(func=reset) 2371 2372 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2373 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2374 group='working', order=-100) 2375 parser_finish.add_argument('recipename', help='Recipe to finish') 2376 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2377 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2378 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2379 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2380 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2381 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2382 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2383 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2384 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2385 parser_finish.set_defaults(func=finish) 2386