1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.npm_dev: 149 extracmdopts += ' --npm-dev' 150 if args.mirrors: 151 extracmdopts += ' --mirrors' 152 if args.srcrev: 153 extracmdopts += ' --srcrev %s' % args.srcrev 154 if args.srcbranch: 155 extracmdopts += ' --srcbranch %s' % args.srcbranch 156 if args.provides: 157 extracmdopts += ' --provides %s' % args.provides 158 159 tempdir = tempfile.mkdtemp(prefix='devtool') 160 try: 161 try: 162 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 163 except bb.process.ExecutionError as e: 164 if e.exitcode == 15: 165 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 166 else: 167 raise DevtoolError('Command \'%s\' failed' % e.command) 168 169 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 170 if recipes: 171 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 172 if recipename in workspace: 173 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 174 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 175 bb.utils.mkdirhier(recipedir) 176 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 177 appendfile = recipe_to_append(recipefile, config) 178 if os.path.exists(appendfile): 179 # This shouldn't be possible, but just in case 180 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 181 if os.path.exists(recipefile): 182 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 183 if tmpsrcdir: 184 srctree = os.path.join(srctreeparent, recipename) 185 if os.path.exists(tmpsrcdir): 186 if os.path.exists(srctree): 187 if os.path.isdir(srctree): 188 try: 189 os.rmdir(srctree) 190 except OSError as e: 191 if e.errno == errno.ENOTEMPTY: 192 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 193 else: 194 raise 195 else: 196 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 197 logger.info('Using default source tree path %s' % srctree) 198 shutil.move(tmpsrcdir, srctree) 199 else: 200 raise DevtoolError('Couldn\'t find source tree created by recipetool') 201 bb.utils.mkdirhier(recipedir) 202 shutil.move(recipes[0], recipefile) 203 # Move any additional files created by recipetool 204 for fn in os.listdir(tempdir): 205 shutil.move(os.path.join(tempdir, fn), recipedir) 206 else: 207 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 208 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 209 if os.path.exists(attic_recipe): 210 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 211 finally: 212 if tmpsrcdir and os.path.exists(tmpsrcdir): 213 shutil.rmtree(tmpsrcdir) 214 shutil.rmtree(tempdir) 215 216 for fn in os.listdir(recipedir): 217 _add_md5(config, recipename, os.path.join(recipedir, fn)) 218 219 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 220 try: 221 try: 222 rd = tinfoil.parse_recipe_file(recipefile, False) 223 except Exception as e: 224 logger.error(str(e)) 225 rd = None 226 if not rd: 227 # Parsing failed. We just created this recipe and we shouldn't 228 # leave it in the workdir or it'll prevent bitbake from starting 229 movefn = '%s.parsefailed' % recipefile 230 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 231 shutil.move(recipefile, movefn) 232 return 1 233 234 if args.fetchuri and not args.no_git: 235 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 236 237 initial_rev = None 238 if os.path.exists(os.path.join(srctree, '.git')): 239 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 240 initial_rev = stdout.rstrip() 241 242 if args.src_subdir: 243 srctree = os.path.join(srctree, args.src_subdir) 244 245 bb.utils.mkdirhier(os.path.dirname(appendfile)) 246 with open(appendfile, 'w') as f: 247 f.write('inherit externalsrc\n') 248 f.write('EXTERNALSRC = "%s"\n' % srctree) 249 250 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 251 if b_is_s: 252 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 253 if initial_rev: 254 f.write('\n# initial_rev: %s\n' % initial_rev) 255 256 if args.binary: 257 f.write('do_install_append() {\n') 258 f.write(' rm -rf ${D}/.git\n') 259 f.write(' rm -f ${D}/singletask.lock\n') 260 f.write('}\n') 261 262 if bb.data.inherits_class('npm', rd): 263 f.write('python do_configure_append() {\n') 264 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 265 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 266 f.write(' bb.utils.remove(lockfile)\n') 267 f.write('}\n') 268 269 # Check if the new layer provides recipes whose priorities have been 270 # overriden by PREFERRED_PROVIDER. 271 recipe_name = rd.getVar('PN') 272 provides = rd.getVar('PROVIDES') 273 # Search every item defined in PROVIDES 274 for recipe_provided in provides.split(): 275 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 276 current_pprovider = rd.getVar(preferred_provider) 277 if current_pprovider and current_pprovider != recipe_name: 278 if args.fixed_setup: 279 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 280 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 281 with open(layerconf_file, 'a') as f: 282 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 283 else: 284 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 285 break 286 287 _add_md5(config, recipename, appendfile) 288 289 check_prerelease_version(rd.getVar('PV'), 'devtool add') 290 291 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 292 293 finally: 294 tinfoil.shutdown() 295 296 return 0 297 298 299def _check_compatible_recipe(pn, d): 300 """Check if the recipe is supported by devtool""" 301 if pn == 'perf': 302 raise DevtoolError("The perf recipe does not actually check out " 303 "source and thus cannot be supported by this tool", 304 4) 305 306 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 307 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 308 309 if bb.data.inherits_class('image', d): 310 raise DevtoolError("The %s recipe is an image, and therefore is not " 311 "supported by this tool" % pn, 4) 312 313 if bb.data.inherits_class('populate_sdk', d): 314 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 315 "supported by this tool" % pn, 4) 316 317 if bb.data.inherits_class('packagegroup', d): 318 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 319 "not supported by this tool" % pn, 4) 320 321 if bb.data.inherits_class('meta', d): 322 raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " 323 "not supported by this tool" % pn, 4) 324 325 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 326 # Not an incompatibility error per se, so we don't pass the error code 327 raise DevtoolError("externalsrc is currently enabled for the %s " 328 "recipe. This prevents the normal do_patch task " 329 "from working. You will need to disable this " 330 "first." % pn) 331 332def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 333 """Common function for copying a file to the dry run output directory""" 334 relpath = os.path.relpath(dst, base_outdir) 335 if relpath.startswith('..'): 336 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 337 dst = os.path.join(dry_run_outdir, relpath) 338 dst_d = os.path.dirname(dst) 339 if dst_d: 340 bb.utils.mkdirhier(dst_d) 341 # Don't overwrite existing files, otherwise in the case of an upgrade 342 # the dry-run written out recipe will be overwritten with an unmodified 343 # version 344 if not os.path.exists(dst): 345 shutil.copy(src, dst) 346 347def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 348 """Move a file. Creates all the directory components of destination path.""" 349 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 350 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 351 if dry_run_outdir: 352 # We want to copy here, not move 353 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 354 else: 355 dst_d = os.path.dirname(dst) 356 if dst_d: 357 bb.utils.mkdirhier(dst_d) 358 shutil.move(src, dst) 359 360def _copy_file(src, dst, dry_run_outdir=None): 361 """Copy a file. Creates all the directory components of destination path.""" 362 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 363 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 364 if dry_run_outdir: 365 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 366 else: 367 dst_d = os.path.dirname(dst) 368 if dst_d: 369 bb.utils.mkdirhier(dst_d) 370 shutil.copy(src, dst) 371 372def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 373 """List contents of a git treeish""" 374 import bb 375 cmd = ['git', 'ls-tree', '-z', treeish] 376 if recursive: 377 cmd.append('-r') 378 out, _ = bb.process.run(cmd, cwd=repodir) 379 ret = {} 380 if out: 381 for line in out.split('\0'): 382 if line: 383 split = line.split(None, 4) 384 ret[split[3]] = split[0:3] 385 return ret 386 387def _git_exclude_path(srctree, path): 388 """Return pathspec (list of paths) that excludes certain path""" 389 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 390 # we don't catch files that are deleted, for example. A more reliable way 391 # to implement this would be to use "negative pathspecs" which were 392 # introduced in Git v1.9.0. Revisit this when/if the required Git version 393 # becomes greater than that. 394 path = os.path.normpath(path) 395 recurse = True if len(path.split(os.path.sep)) > 1 else False 396 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 397 if path in git_files: 398 git_files.remove(path) 399 return git_files 400 else: 401 return ['.'] 402 403def _ls_tree(directory): 404 """Recursive listing of files in a directory""" 405 ret = [] 406 for root, dirs, files in os.walk(directory): 407 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 408 fname in files]) 409 return ret 410 411 412def extract(args, config, basepath, workspace): 413 """Entry point for the devtool 'extract' subcommand""" 414 import bb 415 416 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 417 if not tinfoil: 418 # Error already shown 419 return 1 420 try: 421 rd = parse_recipe(config, tinfoil, args.recipename, True) 422 if not rd: 423 return 1 424 425 srctree = os.path.abspath(args.srctree) 426 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 427 logger.info('Source tree extracted to %s' % srctree) 428 429 if initial_rev: 430 return 0 431 else: 432 return 1 433 finally: 434 tinfoil.shutdown() 435 436def sync(args, config, basepath, workspace): 437 """Entry point for the devtool 'sync' subcommand""" 438 import bb 439 440 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 441 if not tinfoil: 442 # Error already shown 443 return 1 444 try: 445 rd = parse_recipe(config, tinfoil, args.recipename, True) 446 if not rd: 447 return 1 448 449 srctree = os.path.abspath(args.srctree) 450 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 451 logger.info('Source tree %s synchronized' % srctree) 452 453 if initial_rev: 454 return 0 455 else: 456 return 1 457 finally: 458 tinfoil.shutdown() 459 460def symlink_oelocal_files_srctree(rd,srctree): 461 import oe.patch 462 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 463 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 464 # (otherwise the recipe won't build as expected) 465 local_files_dir = os.path.join(srctree, 'oe-local-files') 466 addfiles = [] 467 for root, _, files in os.walk(local_files_dir): 468 relpth = os.path.relpath(root, local_files_dir) 469 if relpth != '.': 470 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 471 for fn in files: 472 if fn == '.gitignore': 473 continue 474 destpth = os.path.join(srctree, relpth, fn) 475 if os.path.exists(destpth): 476 os.unlink(destpth) 477 os.symlink('oe-local-files/%s' % fn, destpth) 478 addfiles.append(os.path.join(relpth, fn)) 479 if addfiles: 480 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 481 useroptions = [] 482 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 483 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 484 485 486def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 487 """Extract sources of a recipe""" 488 import oe.recipeutils 489 import oe.patch 490 import oe.path 491 492 pn = d.getVar('PN') 493 494 _check_compatible_recipe(pn, d) 495 496 if sync: 497 if not os.path.exists(srctree): 498 raise DevtoolError("output path %s does not exist" % srctree) 499 else: 500 if os.path.exists(srctree): 501 if not os.path.isdir(srctree): 502 raise DevtoolError("output path %s exists and is not a directory" % 503 srctree) 504 elif os.listdir(srctree): 505 raise DevtoolError("output path %s already exists and is " 506 "non-empty" % srctree) 507 508 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 509 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 510 "extract source" % pn, 4) 511 512 if not sync: 513 # Prepare for shutil.move later on 514 bb.utils.mkdirhier(srctree) 515 os.rmdir(srctree) 516 517 extra_overrides = [] 518 if not no_overrides: 519 history = d.varhistory.variable('SRC_URI') 520 for event in history: 521 if not 'flag' in event: 522 if event['op'].startswith(('_append[', '_prepend[')): 523 extra_overrides.append(event['op'].split('[')[1].split(']')[0]) 524 # We want to remove duplicate overrides. If a recipe had multiple 525 # SRC_URI_override += values it would cause mulitple instances of 526 # overrides. This doesn't play nicely with things like creating a 527 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 528 extra_overrides = list(set(extra_overrides)) 529 if extra_overrides: 530 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 531 532 initial_rev = None 533 534 appendexisted = False 535 recipefile = d.getVar('FILE') 536 appendfile = recipe_to_append(recipefile, config) 537 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 538 539 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 540 # directory so that: 541 # (a) we pick up all files that get unpacked to the WORKDIR, and 542 # (b) we don't disturb the existing build 543 # However, with recipe-specific sysroots the sysroots for the recipe 544 # will be prepared under WORKDIR, and if we used the system temporary 545 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 546 # our attempts to hardlink files into the recipe-specific sysroots 547 # will fail on systems where /tmp is a different filesystem, and it 548 # would have to fall back to copying the files which is a waste of 549 # time. Put the temp directory under the WORKDIR to prevent that from 550 # being a problem. 551 tempbasedir = d.getVar('WORKDIR') 552 bb.utils.mkdirhier(tempbasedir) 553 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 554 try: 555 tinfoil.logger.setLevel(logging.WARNING) 556 557 # FIXME this results in a cache reload under control of tinfoil, which is fine 558 # except we don't get the knotty progress bar 559 560 if os.path.exists(appendfile): 561 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 562 shutil.copyfile(appendfile, appendbackup) 563 else: 564 appendbackup = None 565 bb.utils.mkdirhier(os.path.dirname(appendfile)) 566 logger.debug('writing append file %s' % appendfile) 567 with open(appendfile, 'a') as f: 568 f.write('###--- _extract_source\n') 569 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 570 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 571 if not is_kernel_yocto: 572 f.write('PATCHTOOL = "git"\n') 573 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 574 if extra_overrides: 575 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 576 f.write('inherit devtool-source\n') 577 f.write('###--- _extract_source\n') 578 579 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 580 581 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 582 bb.utils.mkdirhier(sstate_manifests) 583 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 584 with open(preservestampfile, 'w') as f: 585 f.write(d.getVar('STAMP')) 586 try: 587 if is_kernel_yocto: 588 # We need to generate the kernel config 589 task = 'do_configure' 590 else: 591 task = 'do_patch' 592 593 # Run the fetch + unpack tasks 594 res = tinfoil.build_targets(pn, 595 task, 596 handle_events=True) 597 finally: 598 if os.path.exists(preservestampfile): 599 os.remove(preservestampfile) 600 601 if not res: 602 raise DevtoolError('Extracting source for %s failed' % pn) 603 604 try: 605 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 606 initial_rev = f.read() 607 608 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 609 srcsubdir = f.read() 610 except FileNotFoundError as e: 611 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 612 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 613 614 # Check if work-shared is empty, if yes 615 # find source and copy to work-shared 616 if is_kernel_yocto: 617 workshareddir = d.getVar('STAGING_KERNEL_DIR') 618 staging_kerVer = get_staging_kver(workshareddir) 619 kernelVersion = d.getVar('LINUX_VERSION') 620 621 # handle dangling symbolic link in work-shared: 622 if os.path.islink(workshareddir): 623 os.unlink(workshareddir) 624 625 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 626 shutil.rmtree(workshareddir) 627 oe.path.copyhardlinktree(srcsubdir,workshareddir) 628 elif not os.path.exists(workshareddir): 629 oe.path.copyhardlinktree(srcsubdir,workshareddir) 630 631 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 632 srctree_localdir = os.path.join(srctree, 'oe-local-files') 633 634 if sync: 635 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 636 637 # Move oe-local-files directory to srctree 638 # As the oe-local-files is not part of the constructed git tree, 639 # remove them directly during the synchrounizating might surprise 640 # the users. Instead, we move it to oe-local-files.bak and remind 641 # user in the log message. 642 if os.path.exists(srctree_localdir + '.bak'): 643 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 644 645 if os.path.exists(srctree_localdir): 646 logger.info('Backing up current local file directory %s' % srctree_localdir) 647 shutil.move(srctree_localdir, srctree_localdir + '.bak') 648 649 if os.path.exists(tempdir_localdir): 650 logger.info('Syncing local source files to srctree...') 651 shutil.copytree(tempdir_localdir, srctree_localdir) 652 else: 653 # Move oe-local-files directory to srctree 654 if os.path.exists(tempdir_localdir): 655 logger.info('Adding local source files to srctree...') 656 shutil.move(tempdir_localdir, srcsubdir) 657 658 shutil.move(srcsubdir, srctree) 659 symlink_oelocal_files_srctree(d,srctree) 660 661 if is_kernel_yocto: 662 logger.info('Copying kernel config to srctree') 663 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 664 665 finally: 666 if appendbackup: 667 shutil.copyfile(appendbackup, appendfile) 668 elif os.path.exists(appendfile): 669 os.remove(appendfile) 670 if keep_temp: 671 logger.info('Preserving temporary directory %s' % tempdir) 672 else: 673 shutil.rmtree(tempdir) 674 return initial_rev, srcsubdir_rel 675 676def _add_md5(config, recipename, filename): 677 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 678 import bb.utils 679 680 def addfile(fn): 681 md5 = bb.utils.md5_file(fn) 682 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 683 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 684 f.seek(0, os.SEEK_SET) 685 if not md5_str in f.read(): 686 f.write(md5_str) 687 688 if os.path.isdir(filename): 689 for root, _, files in os.walk(filename): 690 for f in files: 691 addfile(os.path.join(root, f)) 692 else: 693 addfile(filename) 694 695def _check_preserve(config, recipename): 696 """Check if a file was manually changed and needs to be saved in 'attic' 697 directory""" 698 import bb.utils 699 origfile = os.path.join(config.workspace_path, '.devtool_md5') 700 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 701 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 702 with open(origfile, 'r') as f: 703 with open(newfile, 'w') as tf: 704 for line in f.readlines(): 705 splitline = line.rstrip().split('|') 706 if splitline[0] == recipename: 707 removefile = os.path.join(config.workspace_path, splitline[1]) 708 try: 709 md5 = bb.utils.md5_file(removefile) 710 except IOError as err: 711 if err.errno == 2: 712 # File no longer exists, skip it 713 continue 714 else: 715 raise 716 if splitline[2] != md5: 717 bb.utils.mkdirhier(preservepath) 718 preservefile = os.path.basename(removefile) 719 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 720 shutil.move(removefile, os.path.join(preservepath, preservefile)) 721 else: 722 os.remove(removefile) 723 else: 724 tf.write(line) 725 os.rename(newfile, origfile) 726 727def get_staging_kver(srcdir): 728 # Kernel version from work-shared 729 kerver = [] 730 staging_kerVer="" 731 if os.path.exists(srcdir) and os.listdir(srcdir): 732 with open(os.path.join(srcdir,"Makefile")) as f: 733 version = [next(f) for x in range(5)][1:4] 734 for word in version: 735 kerver.append(word.split('= ')[1].split('\n')[0]) 736 staging_kerVer = ".".join(kerver) 737 return staging_kerVer 738 739def get_staging_kbranch(srcdir): 740 staging_kbranch = "" 741 if os.path.exists(srcdir) and os.listdir(srcdir): 742 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) 743 staging_kbranch = "".join(branch.split('\n')[0]) 744 return staging_kbranch 745 746def modify(args, config, basepath, workspace): 747 """Entry point for the devtool 'modify' subcommand""" 748 import bb 749 import oe.recipeutils 750 import oe.patch 751 import oe.path 752 753 if args.recipename in workspace: 754 raise DevtoolError("recipe %s is already in your workspace" % 755 args.recipename) 756 757 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 758 try: 759 rd = parse_recipe(config, tinfoil, args.recipename, True) 760 if not rd: 761 return 1 762 763 pn = rd.getVar('PN') 764 if pn != args.recipename: 765 logger.info('Mapping %s to %s' % (args.recipename, pn)) 766 if pn in workspace: 767 raise DevtoolError("recipe %s is already in your workspace" % 768 pn) 769 770 if args.srctree: 771 srctree = os.path.abspath(args.srctree) 772 else: 773 srctree = get_default_srctree(config, pn) 774 775 if args.no_extract and not os.path.isdir(srctree): 776 raise DevtoolError("--no-extract specified and source path %s does " 777 "not exist or is not a directory" % 778 srctree) 779 780 recipefile = rd.getVar('FILE') 781 appendfile = recipe_to_append(recipefile, config, args.wildcard) 782 if os.path.exists(appendfile): 783 raise DevtoolError("Another variant of recipe %s is already in your " 784 "workspace (only one variant of a recipe can " 785 "currently be worked on at once)" 786 % pn) 787 788 _check_compatible_recipe(pn, rd) 789 790 initial_rev = None 791 commits = [] 792 check_commits = False 793 794 if bb.data.inherits_class('kernel-yocto', rd): 795 # Current set kernel version 796 kernelVersion = rd.getVar('LINUX_VERSION') 797 srcdir = rd.getVar('STAGING_KERNEL_DIR') 798 kbranch = rd.getVar('KBRANCH') 799 800 staging_kerVer = get_staging_kver(srcdir) 801 staging_kbranch = get_staging_kbranch(srcdir) 802 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 803 oe.path.copyhardlinktree(srcdir,srctree) 804 workdir = rd.getVar('WORKDIR') 805 srcsubdir = rd.getVar('S') 806 localfilesdir = os.path.join(srctree,'oe-local-files') 807 # Move local source files into separate subdir 808 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 809 local_files = oe.recipeutils.get_recipe_local_files(rd) 810 811 for key in local_files.copy(): 812 if key.endswith('scc'): 813 sccfile = open(local_files[key], 'r') 814 for l in sccfile: 815 line = l.split() 816 if line and line[0] in ('kconf', 'patch'): 817 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 818 if not cfg in local_files.values(): 819 local_files[line[-1]] = cfg 820 shutil.copy2(cfg, workdir) 821 sccfile.close() 822 823 # Ignore local files with subdir={BP} 824 srcabspath = os.path.abspath(srcsubdir) 825 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 826 if local_files: 827 for fname in local_files: 828 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 829 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 830 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 831 832 symlink_oelocal_files_srctree(rd,srctree) 833 834 task = 'do_configure' 835 res = tinfoil.build_targets(pn, task, handle_events=True) 836 837 # Copy .config to workspace 838 kconfpath = rd.getVar('B') 839 logger.info('Copying kernel config to workspace') 840 shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 841 842 # Set this to true, we still need to get initial_rev 843 # by parsing the git repo 844 args.no_extract = True 845 846 if not args.no_extract: 847 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 848 if not initial_rev: 849 return 1 850 logger.info('Source tree extracted to %s' % srctree) 851 # Get list of commits since this revision 852 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) 853 commits = stdout.split() 854 check_commits = True 855 else: 856 if os.path.exists(os.path.join(srctree, '.git')): 857 # Check if it's a tree previously extracted by us. This is done 858 # by ensuring that devtool-base and args.branch (devtool) exist. 859 # The check_commits logic will cause an exception if either one 860 # of these doesn't exist 861 try: 862 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 863 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 864 except bb.process.ExecutionError: 865 stdout = '' 866 if stdout: 867 check_commits = True 868 for line in stdout.splitlines(): 869 if line.startswith('*'): 870 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 871 initial_rev = stdout.rstrip() 872 if not initial_rev: 873 # Otherwise, just grab the head revision 874 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 875 initial_rev = stdout.rstrip() 876 877 branch_patches = {} 878 if check_commits: 879 # Check if there are override branches 880 (stdout, _) = bb.process.run('git branch', cwd=srctree) 881 branches = [] 882 for line in stdout.rstrip().splitlines(): 883 branchname = line[2:].rstrip() 884 if branchname.startswith(override_branch_prefix): 885 branches.append(branchname) 886 if branches: 887 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 888 branches.insert(0, args.branch) 889 seen_patches = [] 890 for branch in branches: 891 branch_patches[branch] = [] 892 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 893 for line in stdout.splitlines(): 894 line = line.strip() 895 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 896 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 897 if not origpatch in seen_patches: 898 seen_patches.append(origpatch) 899 branch_patches[branch].append(origpatch) 900 901 # Need to grab this here in case the source is within a subdirectory 902 srctreebase = srctree 903 904 # Check that recipe isn't using a shared workdir 905 s = os.path.abspath(rd.getVar('S')) 906 workdir = os.path.abspath(rd.getVar('WORKDIR')) 907 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 908 # Handle if S is set to a subdirectory of the source 909 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 910 srctree = os.path.join(srctree, srcsubdir) 911 912 bb.utils.mkdirhier(os.path.dirname(appendfile)) 913 with open(appendfile, 'w') as f: 914 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n') 915 # Local files can be modified/tracked in separate subdir under srctree 916 # Mostly useful for packages with S != WORKDIR 917 f.write('FILESPATH_prepend := "%s:"\n' % 918 os.path.join(srctreebase, 'oe-local-files')) 919 f.write('# srctreebase: %s\n' % srctreebase) 920 921 f.write('\ninherit externalsrc\n') 922 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 923 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) 924 925 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 926 if b_is_s: 927 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) 928 929 if bb.data.inherits_class('kernel', rd): 930 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 931 'do_fetch do_unpack do_kernel_configme do_kernel_configcheck"\n') 932 f.write('\ndo_patch[noexec] = "1"\n') 933 f.write('\ndo_configure_append() {\n' 934 ' cp ${B}/.config ${S}/.config.baseline\n' 935 ' ln -sfT ${B}/.config ${S}/.config.new\n' 936 '}\n') 937 if rd.getVarFlag('do_menuconfig','task'): 938 f.write('\ndo_configure_append() {\n' 939 ' if [ ! ${DEVTOOL_DISABLE_MENUCONFIG} ]; then\n' 940 ' cp ${B}/.config ${S}/.config.baseline\n' 941 ' ln -sfT ${B}/.config ${S}/.config.new\n' 942 ' fi\n' 943 '}\n') 944 if initial_rev: 945 f.write('\n# initial_rev: %s\n' % initial_rev) 946 for commit in commits: 947 f.write('# commit: %s\n' % commit) 948 if branch_patches: 949 for branch in branch_patches: 950 if branch == args.branch: 951 continue 952 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 953 954 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 955 956 _add_md5(config, pn, appendfile) 957 958 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 959 960 finally: 961 tinfoil.shutdown() 962 963 return 0 964 965 966def rename(args, config, basepath, workspace): 967 """Entry point for the devtool 'rename' subcommand""" 968 import bb 969 import oe.recipeutils 970 971 check_workspace_recipe(workspace, args.recipename) 972 973 if not (args.newname or args.version): 974 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 975 976 recipefile = workspace[args.recipename]['recipefile'] 977 if not recipefile: 978 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 979 980 if args.newname and args.newname != args.recipename: 981 reason = oe.recipeutils.validate_pn(args.newname) 982 if reason: 983 raise DevtoolError(reason) 984 newname = args.newname 985 else: 986 newname = args.recipename 987 988 append = workspace[args.recipename]['bbappend'] 989 appendfn = os.path.splitext(os.path.basename(append))[0] 990 splitfn = appendfn.split('_') 991 if len(splitfn) > 1: 992 origfnver = appendfn.split('_')[1] 993 else: 994 origfnver = '' 995 996 recipefilemd5 = None 997 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 998 try: 999 rd = parse_recipe(config, tinfoil, args.recipename, True) 1000 if not rd: 1001 return 1 1002 1003 bp = rd.getVar('BP') 1004 bpn = rd.getVar('BPN') 1005 if newname != args.recipename: 1006 localdata = rd.createCopy() 1007 localdata.setVar('PN', newname) 1008 newbpn = localdata.getVar('BPN') 1009 else: 1010 newbpn = bpn 1011 s = rd.getVar('S', False) 1012 src_uri = rd.getVar('SRC_URI', False) 1013 pv = rd.getVar('PV') 1014 1015 # Correct variable values that refer to the upstream source - these 1016 # values must stay the same, so if the name/version are changing then 1017 # we need to fix them up 1018 new_s = s 1019 new_src_uri = src_uri 1020 if newbpn != bpn: 1021 # ${PN} here is technically almost always incorrect, but people do use it 1022 new_s = new_s.replace('${BPN}', bpn) 1023 new_s = new_s.replace('${PN}', bpn) 1024 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1025 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1026 new_src_uri = new_src_uri.replace('${PN}', bpn) 1027 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1028 if args.version and origfnver == pv: 1029 new_s = new_s.replace('${PV}', pv) 1030 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1031 new_src_uri = new_src_uri.replace('${PV}', pv) 1032 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1033 patchfields = {} 1034 if new_s != s: 1035 patchfields['S'] = new_s 1036 if new_src_uri != src_uri: 1037 patchfields['SRC_URI'] = new_src_uri 1038 if patchfields: 1039 recipefilemd5 = bb.utils.md5_file(recipefile) 1040 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1041 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1042 finally: 1043 tinfoil.shutdown() 1044 1045 if args.version: 1046 newver = args.version 1047 else: 1048 newver = origfnver 1049 1050 if newver: 1051 newappend = '%s_%s.bbappend' % (newname, newver) 1052 newfile = '%s_%s.bb' % (newname, newver) 1053 else: 1054 newappend = '%s.bbappend' % newname 1055 newfile = '%s.bb' % newname 1056 1057 oldrecipedir = os.path.dirname(recipefile) 1058 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1059 if oldrecipedir != newrecipedir: 1060 bb.utils.mkdirhier(newrecipedir) 1061 1062 newappend = os.path.join(os.path.dirname(append), newappend) 1063 newfile = os.path.join(newrecipedir, newfile) 1064 1065 # Rename bbappend 1066 logger.info('Renaming %s to %s' % (append, newappend)) 1067 os.rename(append, newappend) 1068 # Rename recipe file 1069 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1070 os.rename(recipefile, newfile) 1071 1072 # Rename source tree if it's the default path 1073 appendmd5 = None 1074 if not args.no_srctree: 1075 srctree = workspace[args.recipename]['srctree'] 1076 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1077 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1078 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1079 shutil.move(srctree, newsrctree) 1080 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1081 appendmd5 = bb.utils.md5_file(newappend) 1082 appendlines = [] 1083 with open(newappend, 'r') as f: 1084 for line in f: 1085 appendlines.append(line) 1086 with open(newappend, 'w') as f: 1087 for line in appendlines: 1088 if srctree in line: 1089 line = line.replace(srctree, newsrctree) 1090 f.write(line) 1091 newappendmd5 = bb.utils.md5_file(newappend) 1092 1093 bpndir = None 1094 newbpndir = None 1095 if newbpn != bpn: 1096 bpndir = os.path.join(oldrecipedir, bpn) 1097 if os.path.exists(bpndir): 1098 newbpndir = os.path.join(newrecipedir, newbpn) 1099 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1100 shutil.move(bpndir, newbpndir) 1101 1102 bpdir = None 1103 newbpdir = None 1104 if newver != origfnver or newbpn != bpn: 1105 bpdir = os.path.join(oldrecipedir, bp) 1106 if os.path.exists(bpdir): 1107 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1108 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1109 shutil.move(bpdir, newbpdir) 1110 1111 if oldrecipedir != newrecipedir: 1112 # Move any stray files and delete the old recipe directory 1113 for entry in os.listdir(oldrecipedir): 1114 oldpath = os.path.join(oldrecipedir, entry) 1115 newpath = os.path.join(newrecipedir, entry) 1116 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1117 shutil.move(oldpath, newpath) 1118 os.rmdir(oldrecipedir) 1119 1120 # Now take care of entries in .devtool_md5 1121 md5entries = [] 1122 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1123 for line in f: 1124 md5entries.append(line) 1125 1126 if bpndir and newbpndir: 1127 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1128 else: 1129 relbpndir = None 1130 if bpdir and newbpdir: 1131 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1132 else: 1133 relbpdir = None 1134 1135 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1136 for entry in md5entries: 1137 splitentry = entry.rstrip().split('|') 1138 if len(splitentry) > 2: 1139 if splitentry[0] == args.recipename: 1140 splitentry[0] = newname 1141 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1142 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1143 if appendmd5 and splitentry[2] == appendmd5: 1144 splitentry[2] = newappendmd5 1145 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1146 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1147 if recipefilemd5 and splitentry[2] == recipefilemd5: 1148 splitentry[2] = newrecipefilemd5 1149 elif relbpndir and splitentry[1].startswith(relbpndir): 1150 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1151 elif relbpdir and splitentry[1].startswith(relbpdir): 1152 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1153 entry = '|'.join(splitentry) + '\n' 1154 f.write(entry) 1155 return 0 1156 1157 1158def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1159 """Get initial and update rev of a recipe. These are the start point of the 1160 whole patchset and start point for the patches to be re-generated/updated. 1161 """ 1162 import bb 1163 1164 # Get current branch 1165 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1166 cwd=srctree) 1167 branchname = stdout.rstrip() 1168 1169 # Parse initial rev from recipe if not specified 1170 commits = [] 1171 patches = [] 1172 with open(recipe_path, 'r') as f: 1173 for line in f: 1174 if line.startswith('# initial_rev:'): 1175 if not initial_rev: 1176 initial_rev = line.split(':')[-1].strip() 1177 elif line.startswith('# commit:') and not force_patch_refresh: 1178 commits.append(line.split(':')[-1].strip()) 1179 elif line.startswith('# patches_%s:' % branchname): 1180 patches = line.split(':')[-1].strip().split(',') 1181 1182 update_rev = initial_rev 1183 changed_revs = None 1184 if initial_rev: 1185 # Find first actually changed revision 1186 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1187 initial_rev, cwd=srctree) 1188 newcommits = stdout.split() 1189 for i in range(min(len(commits), len(newcommits))): 1190 if newcommits[i] == commits[i]: 1191 update_rev = commits[i] 1192 1193 try: 1194 stdout, _ = bb.process.run('git cherry devtool-patched', 1195 cwd=srctree) 1196 except bb.process.ExecutionError as err: 1197 stdout = None 1198 1199 if stdout is not None and not force_patch_refresh: 1200 changed_revs = [] 1201 for line in stdout.splitlines(): 1202 if line.startswith('+ '): 1203 rev = line.split()[1] 1204 if rev in newcommits: 1205 changed_revs.append(rev) 1206 1207 return initial_rev, update_rev, changed_revs, patches 1208 1209def _remove_file_entries(srcuri, filelist): 1210 """Remove file:// entries from SRC_URI""" 1211 remaining = filelist[:] 1212 entries = [] 1213 for fname in filelist: 1214 basename = os.path.basename(fname) 1215 for i in range(len(srcuri)): 1216 if (srcuri[i].startswith('file://') and 1217 os.path.basename(srcuri[i].split(';')[0]) == basename): 1218 entries.append(srcuri[i]) 1219 remaining.remove(fname) 1220 srcuri.pop(i) 1221 break 1222 return entries, remaining 1223 1224def _replace_srcuri_entry(srcuri, filename, newentry): 1225 """Replace entry corresponding to specified file with a new entry""" 1226 basename = os.path.basename(filename) 1227 for i in range(len(srcuri)): 1228 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1229 srcuri.pop(i) 1230 srcuri.insert(i, newentry) 1231 break 1232 1233def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1234 """Unlink existing patch files""" 1235 1236 dry_run_suffix = ' (dry-run)' if dry_run else '' 1237 1238 for path in files: 1239 if append: 1240 if not destpath: 1241 raise Exception('destpath should be set here') 1242 path = os.path.join(destpath, os.path.basename(path)) 1243 1244 if os.path.exists(path): 1245 if not no_report_remove: 1246 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1247 if not dry_run: 1248 # FIXME "git rm" here would be nice if the file in question is 1249 # tracked 1250 # FIXME there's a chance that this file is referred to by 1251 # another recipe, in which case deleting wouldn't be the 1252 # right thing to do 1253 os.remove(path) 1254 # Remove directory if empty 1255 try: 1256 os.rmdir(os.path.dirname(path)) 1257 except OSError as ose: 1258 if ose.errno != errno.ENOTEMPTY: 1259 raise 1260 1261 1262def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1263 """Export patches from srctree to given location. 1264 Returns three-tuple of dicts: 1265 1. updated - patches that already exist in SRCURI 1266 2. added - new patches that don't exist in SRCURI 1267 3 removed - patches that exist in SRCURI but not in exported patches 1268 In each dict the key is the 'basepath' of the URI and value is the 1269 absolute path to the existing file in recipe space (if any). 1270 """ 1271 import oe.recipeutils 1272 from oe.patch import GitApplyTree 1273 updated = OrderedDict() 1274 added = OrderedDict() 1275 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1276 1277 existing_patches = dict((os.path.basename(path), path) for path in 1278 oe.recipeutils.get_recipe_patches(rd)) 1279 logger.debug('Existing patches: %s' % existing_patches) 1280 1281 # Generate patches from Git, exclude local files directory 1282 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1283 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) 1284 1285 new_patches = sorted(os.listdir(destdir)) 1286 for new_patch in new_patches: 1287 # Strip numbering from patch names. If it's a git sequence named patch, 1288 # the numbers might not match up since we are starting from a different 1289 # revision This does assume that people are using unique shortlog 1290 # values, but they ought to be anyway... 1291 new_basename = seqpatch_re.match(new_patch).group(2) 1292 match_name = None 1293 for old_patch in existing_patches: 1294 old_basename = seqpatch_re.match(old_patch).group(2) 1295 old_basename_splitext = os.path.splitext(old_basename) 1296 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1297 old_patch_noext = os.path.splitext(old_patch)[0] 1298 match_name = old_patch_noext 1299 break 1300 elif new_basename == old_basename: 1301 match_name = old_patch 1302 break 1303 if match_name: 1304 # Rename patch files 1305 if new_patch != match_name: 1306 os.rename(os.path.join(destdir, new_patch), 1307 os.path.join(destdir, match_name)) 1308 # Need to pop it off the list now before checking changed_revs 1309 oldpath = existing_patches.pop(old_patch) 1310 if changed_revs is not None: 1311 # Avoid updating patches that have not actually changed 1312 with open(os.path.join(destdir, match_name), 'r') as f: 1313 firstlineitems = f.readline().split() 1314 # Looking for "From <hash>" line 1315 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1316 if not firstlineitems[1] in changed_revs: 1317 continue 1318 # Recompress if necessary 1319 if oldpath.endswith(('.gz', '.Z')): 1320 bb.process.run(['gzip', match_name], cwd=destdir) 1321 if oldpath.endswith('.gz'): 1322 match_name += '.gz' 1323 else: 1324 match_name += '.Z' 1325 elif oldpath.endswith('.bz2'): 1326 bb.process.run(['bzip2', match_name], cwd=destdir) 1327 match_name += '.bz2' 1328 updated[match_name] = oldpath 1329 else: 1330 added[new_patch] = None 1331 return (updated, added, existing_patches) 1332 1333 1334def _create_kconfig_diff(srctree, rd, outfile): 1335 """Create a kconfig fragment""" 1336 # Only update config fragment if both config files exist 1337 orig_config = os.path.join(srctree, '.config.baseline') 1338 new_config = os.path.join(srctree, '.config.new') 1339 if os.path.exists(orig_config) and os.path.exists(new_config): 1340 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1341 '--unchanged-line-format=', orig_config, new_config] 1342 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1343 stderr=subprocess.PIPE) 1344 stdout, stderr = pipe.communicate() 1345 if pipe.returncode == 1: 1346 logger.info("Updating config fragment %s" % outfile) 1347 with open(outfile, 'wb') as fobj: 1348 fobj.write(stdout) 1349 elif pipe.returncode == 0: 1350 logger.info("Would remove config fragment %s" % outfile) 1351 if os.path.exists(outfile): 1352 # Remove fragment file in case of empty diff 1353 logger.info("Removing config fragment %s" % outfile) 1354 os.unlink(outfile) 1355 else: 1356 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1357 return True 1358 return False 1359 1360 1361def _export_local_files(srctree, rd, destdir, srctreebase): 1362 """Copy local files from srctree to given location. 1363 Returns three-tuple of dicts: 1364 1. updated - files that already exist in SRCURI 1365 2. added - new files files that don't exist in SRCURI 1366 3 removed - files that exist in SRCURI but not in exported files 1367 In each dict the key is the 'basepath' of the URI and value is the 1368 absolute path to the existing file in recipe space (if any). 1369 """ 1370 import oe.recipeutils 1371 1372 # Find out local files (SRC_URI files that exist in the "recipe space"). 1373 # Local files that reside in srctree are not included in patch generation. 1374 # Instead they are directly copied over the original source files (in 1375 # recipe space). 1376 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1377 new_set = None 1378 updated = OrderedDict() 1379 added = OrderedDict() 1380 removed = OrderedDict() 1381 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1382 git_files = _git_ls_tree(srctree) 1383 if 'oe-local-files' in git_files: 1384 # If tracked by Git, take the files from srctree HEAD. First get 1385 # the tree object of the directory 1386 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1387 tree = git_files['oe-local-files'][2] 1388 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1389 env=dict(os.environ, GIT_WORK_TREE=destdir, 1390 GIT_INDEX_FILE=tmp_index)) 1391 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1392 elif os.path.isdir(local_files_dir): 1393 # If not tracked by Git, just copy from working copy 1394 new_set = _ls_tree(local_files_dir) 1395 bb.process.run(['cp', '-ax', 1396 os.path.join(local_files_dir, '.'), destdir]) 1397 else: 1398 new_set = [] 1399 1400 # Special handling for kernel config 1401 if bb.data.inherits_class('kernel-yocto', rd): 1402 fragment_fn = 'devtool-fragment.cfg' 1403 fragment_path = os.path.join(destdir, fragment_fn) 1404 if _create_kconfig_diff(srctree, rd, fragment_path): 1405 if os.path.exists(fragment_path): 1406 if fragment_fn not in new_set: 1407 new_set.append(fragment_fn) 1408 # Copy fragment to local-files 1409 if os.path.isdir(local_files_dir): 1410 shutil.copy2(fragment_path, local_files_dir) 1411 else: 1412 if fragment_fn in new_set: 1413 new_set.remove(fragment_fn) 1414 # Remove fragment from local-files 1415 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1416 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1417 1418 # Special handling for cml1, ccmake, etc bbclasses that generated 1419 # configuration fragment files that are consumed as source files 1420 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1421 if bb.data.inherits_class(frag_class, rd): 1422 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1423 if os.path.exists(srcpath): 1424 if frag_name not in new_set: 1425 new_set.append(frag_name) 1426 # copy fragment into destdir 1427 shutil.copy2(srcpath, destdir) 1428 # copy fragment into local files if exists 1429 if os.path.isdir(local_files_dir): 1430 shutil.copy2(srcpath, local_files_dir) 1431 1432 if new_set is not None: 1433 for fname in new_set: 1434 if fname in existing_files: 1435 origpath = existing_files.pop(fname) 1436 workpath = os.path.join(local_files_dir, fname) 1437 if not filecmp.cmp(origpath, workpath): 1438 updated[fname] = origpath 1439 elif fname != '.gitignore': 1440 added[fname] = None 1441 1442 workdir = rd.getVar('WORKDIR') 1443 s = rd.getVar('S') 1444 if not s.endswith(os.sep): 1445 s += os.sep 1446 1447 if workdir != s: 1448 # Handle files where subdir= was specified 1449 for fname in list(existing_files.keys()): 1450 # FIXME handle both subdir starting with BP and not? 1451 fworkpath = os.path.join(workdir, fname) 1452 if fworkpath.startswith(s): 1453 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1454 if os.path.exists(fpath): 1455 origpath = existing_files.pop(fname) 1456 if not filecmp.cmp(origpath, fpath): 1457 updated[fpath] = origpath 1458 1459 removed = existing_files 1460 return (updated, added, removed) 1461 1462 1463def _determine_files_dir(rd): 1464 """Determine the appropriate files directory for a recipe""" 1465 recipedir = rd.getVar('FILE_DIRNAME') 1466 for entry in rd.getVar('FILESPATH').split(':'): 1467 relpth = os.path.relpath(entry, recipedir) 1468 if not os.sep in relpth: 1469 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1470 if os.path.isdir(entry): 1471 return entry 1472 return os.path.join(recipedir, rd.getVar('BPN')) 1473 1474 1475def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1476 """Implement the 'srcrev' mode of update-recipe""" 1477 import bb 1478 import oe.recipeutils 1479 1480 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1481 1482 recipefile = rd.getVar('FILE') 1483 recipedir = os.path.basename(recipefile) 1484 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1485 1486 # Get HEAD revision 1487 try: 1488 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1489 except bb.process.ExecutionError as err: 1490 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1491 (srctree, err)) 1492 srcrev = stdout.strip() 1493 if len(srcrev) != 40: 1494 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1495 1496 destpath = None 1497 remove_files = [] 1498 patchfields = {} 1499 patchfields['SRCREV'] = srcrev 1500 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1501 srcuri = orig_src_uri.split() 1502 tempdir = tempfile.mkdtemp(prefix='devtool') 1503 update_srcuri = False 1504 appendfile = None 1505 try: 1506 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1507 srctreebase = workspace[recipename]['srctreebase'] 1508 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1509 if not no_remove: 1510 # Find list of existing patches in recipe file 1511 patches_dir = tempfile.mkdtemp(dir=tempdir) 1512 old_srcrev = rd.getVar('SRCREV') or '' 1513 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1514 patches_dir) 1515 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1516 1517 # Remove deleted local files and "overlapping" patches 1518 remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) 1519 if remove_files: 1520 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1521 update_srcuri = True 1522 1523 if appendlayerdir: 1524 files = dict((os.path.join(local_files_dir, key), val) for 1525 key, val in list(upd_f.items()) + list(new_f.items())) 1526 removevalues = {} 1527 if update_srcuri: 1528 removevalues = {'SRC_URI': removedentries} 1529 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1530 if dry_run_outdir: 1531 logger.info('Creating bbappend (dry-run)') 1532 else: 1533 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1534 rd, appendlayerdir, files, wildcardver=wildcard_version, 1535 extralines=patchfields, removevalues=removevalues, 1536 redirect_output=dry_run_outdir) 1537 else: 1538 files_dir = _determine_files_dir(rd) 1539 for basepath, path in upd_f.items(): 1540 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1541 if os.path.isabs(basepath): 1542 # Original file (probably with subdir pointing inside source tree) 1543 # so we do not want to move it, just copy 1544 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1545 else: 1546 _move_file(os.path.join(local_files_dir, basepath), path, 1547 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1548 update_srcuri= True 1549 for basepath, path in new_f.items(): 1550 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1551 _move_file(os.path.join(local_files_dir, basepath), 1552 os.path.join(files_dir, basepath), 1553 dry_run_outdir=dry_run_outdir, 1554 base_outdir=recipedir) 1555 srcuri.append('file://%s' % basepath) 1556 update_srcuri = True 1557 if update_srcuri: 1558 patchfields['SRC_URI'] = ' '.join(srcuri) 1559 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1560 finally: 1561 shutil.rmtree(tempdir) 1562 if not 'git://' in orig_src_uri: 1563 logger.info('You will need to update SRC_URI within the recipe to ' 1564 'point to a git repository where you have pushed your ' 1565 'changes') 1566 1567 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1568 return True, appendfile, remove_files 1569 1570def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1571 """Implement the 'patch' mode of update-recipe""" 1572 import bb 1573 import oe.recipeutils 1574 1575 recipefile = rd.getVar('FILE') 1576 recipedir = os.path.dirname(recipefile) 1577 append = workspace[recipename]['bbappend'] 1578 if not os.path.exists(append): 1579 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1580 recipename) 1581 1582 initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1583 if not initial_rev: 1584 raise DevtoolError('Unable to find initial revision - please specify ' 1585 'it with --initial-rev') 1586 1587 appendfile = None 1588 dl_dir = rd.getVar('DL_DIR') 1589 if not dl_dir.endswith('/'): 1590 dl_dir += '/' 1591 1592 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1593 1594 tempdir = tempfile.mkdtemp(prefix='devtool') 1595 try: 1596 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1597 if filter_patches: 1598 upd_f = {} 1599 new_f = {} 1600 del_f = {} 1601 else: 1602 srctreebase = workspace[recipename]['srctreebase'] 1603 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1604 1605 remove_files = [] 1606 if not no_remove: 1607 # Get all patches from source tree and check if any should be removed 1608 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1609 _, _, del_p = _export_patches(srctree, rd, initial_rev, 1610 all_patches_dir) 1611 # Remove deleted local files and patches 1612 remove_files = list(del_f.values()) + list(del_p.values()) 1613 1614 # Get updated patches from source tree 1615 patches_dir = tempfile.mkdtemp(dir=tempdir) 1616 upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, 1617 patches_dir, changed_revs) 1618 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1619 if filter_patches: 1620 new_p = OrderedDict() 1621 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1622 remove_files = [f for f in remove_files if f in filter_patches] 1623 updatefiles = False 1624 updaterecipe = False 1625 destpath = None 1626 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1627 if appendlayerdir: 1628 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1629 key, val in list(upd_f.items()) + list(new_f.items())) 1630 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1631 key, val in list(upd_p.items()) + list(new_p.items()))) 1632 if files or remove_files: 1633 removevalues = None 1634 if remove_files: 1635 removedentries, remaining = _remove_file_entries( 1636 srcuri, remove_files) 1637 if removedentries or remaining: 1638 remaining = ['file://' + os.path.basename(item) for 1639 item in remaining] 1640 removevalues = {'SRC_URI': removedentries + remaining} 1641 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1642 rd, appendlayerdir, files, 1643 wildcardver=wildcard_version, 1644 removevalues=removevalues, 1645 redirect_output=dry_run_outdir) 1646 else: 1647 logger.info('No patches or local source files needed updating') 1648 else: 1649 # Update existing files 1650 files_dir = _determine_files_dir(rd) 1651 for basepath, path in upd_f.items(): 1652 logger.info('Updating file %s' % basepath) 1653 if os.path.isabs(basepath): 1654 # Original file (probably with subdir pointing inside source tree) 1655 # so we do not want to move it, just copy 1656 _copy_file(basepath, path, 1657 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1658 else: 1659 _move_file(os.path.join(local_files_dir, basepath), path, 1660 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1661 updatefiles = True 1662 for basepath, path in upd_p.items(): 1663 patchfn = os.path.join(patches_dir, basepath) 1664 if os.path.dirname(path) + '/' == dl_dir: 1665 # This is a a downloaded patch file - we now need to 1666 # replace the entry in SRC_URI with our local version 1667 logger.info('Replacing remote patch %s with updated local version' % basepath) 1668 path = os.path.join(files_dir, basepath) 1669 _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) 1670 updaterecipe = True 1671 else: 1672 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1673 _move_file(patchfn, path, 1674 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1675 updatefiles = True 1676 # Add any new files 1677 for basepath, path in new_f.items(): 1678 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1679 _move_file(os.path.join(local_files_dir, basepath), 1680 os.path.join(files_dir, basepath), 1681 dry_run_outdir=dry_run_outdir, 1682 base_outdir=recipedir) 1683 srcuri.append('file://%s' % basepath) 1684 updaterecipe = True 1685 for basepath, path in new_p.items(): 1686 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1687 _move_file(os.path.join(patches_dir, basepath), 1688 os.path.join(files_dir, basepath), 1689 dry_run_outdir=dry_run_outdir, 1690 base_outdir=recipedir) 1691 srcuri.append('file://%s' % basepath) 1692 updaterecipe = True 1693 # Update recipe, if needed 1694 if _remove_file_entries(srcuri, remove_files)[0]: 1695 updaterecipe = True 1696 if updaterecipe: 1697 if not dry_run_outdir: 1698 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1699 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1700 {'SRC_URI': ' '.join(srcuri)}, 1701 redirect_output=dry_run_outdir) 1702 elif not updatefiles: 1703 # Neither patches nor recipe were updated 1704 logger.info('No patches or files need updating') 1705 return False, None, [] 1706 finally: 1707 shutil.rmtree(tempdir) 1708 1709 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1710 return True, appendfile, remove_files 1711 1712def _guess_recipe_update_mode(srctree, rdata): 1713 """Guess the recipe update mode to use""" 1714 src_uri = (rdata.getVar('SRC_URI') or '').split() 1715 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1716 if not git_uris: 1717 return 'patch' 1718 # Just use the first URI for now 1719 uri = git_uris[0] 1720 # Check remote branch 1721 params = bb.fetch.decodeurl(uri)[5] 1722 upstr_branch = params['branch'] if 'branch' in params else 'master' 1723 # Check if current branch HEAD is found in upstream branch 1724 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1725 head_rev = stdout.rstrip() 1726 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1727 cwd=srctree) 1728 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1729 if 'origin/' + upstr_branch in remote_brs: 1730 return 'srcrev' 1731 1732 return 'patch' 1733 1734def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1735 srctree = workspace[recipename]['srctree'] 1736 if mode == 'auto': 1737 mode = _guess_recipe_update_mode(srctree, rd) 1738 1739 override_branches = [] 1740 mainbranch = None 1741 startbranch = None 1742 if not no_overrides: 1743 stdout, _ = bb.process.run('git branch', cwd=srctree) 1744 other_branches = [] 1745 for line in stdout.splitlines(): 1746 branchname = line[2:] 1747 if line.startswith('* '): 1748 startbranch = branchname 1749 if branchname.startswith(override_branch_prefix): 1750 override_branches.append(branchname) 1751 else: 1752 other_branches.append(branchname) 1753 1754 if override_branches: 1755 logger.debug('_update_recipe: override branches: %s' % override_branches) 1756 logger.debug('_update_recipe: other branches: %s' % other_branches) 1757 if startbranch.startswith(override_branch_prefix): 1758 if len(other_branches) == 1: 1759 mainbranch = other_branches[1] 1760 else: 1761 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1762 else: 1763 mainbranch = startbranch 1764 1765 checkedout = None 1766 anyupdated = False 1767 appendfile = None 1768 allremoved = [] 1769 if override_branches: 1770 logger.info('Handling main branch (%s)...' % mainbranch) 1771 if startbranch != mainbranch: 1772 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1773 checkedout = mainbranch 1774 try: 1775 branchlist = [mainbranch] + override_branches 1776 for branch in branchlist: 1777 crd = bb.data.createCopy(rd) 1778 if branch != mainbranch: 1779 logger.info('Handling branch %s...' % branch) 1780 override = branch[len(override_branch_prefix):] 1781 crd.appendVar('OVERRIDES', ':%s' % override) 1782 bb.process.run('git checkout %s' % branch, cwd=srctree) 1783 checkedout = branch 1784 1785 if mode == 'srcrev': 1786 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1787 elif mode == 'patch': 1788 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1789 else: 1790 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1791 if updated: 1792 anyupdated = True 1793 if appendf: 1794 appendfile = appendf 1795 allremoved.extend(removed) 1796 finally: 1797 if startbranch and checkedout != startbranch: 1798 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1799 1800 return anyupdated, appendfile, allremoved 1801 1802def update_recipe(args, config, basepath, workspace): 1803 """Entry point for the devtool 'update-recipe' subcommand""" 1804 check_workspace_recipe(workspace, args.recipename) 1805 1806 if args.append: 1807 if not os.path.exists(args.append): 1808 raise DevtoolError('bbappend destination layer directory "%s" ' 1809 'does not exist' % args.append) 1810 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1811 raise DevtoolError('conf/layer.conf not found in bbappend ' 1812 'destination layer "%s"' % args.append) 1813 1814 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1815 try: 1816 1817 rd = parse_recipe(config, tinfoil, args.recipename, True) 1818 if not rd: 1819 return 1 1820 1821 dry_run_output = None 1822 dry_run_outdir = None 1823 if args.dry_run: 1824 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1825 dry_run_outdir = dry_run_output.name 1826 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1827 1828 if updated: 1829 rf = rd.getVar('FILE') 1830 if rf.startswith(config.workspace_path): 1831 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1832 finally: 1833 tinfoil.shutdown() 1834 1835 return 0 1836 1837 1838def status(args, config, basepath, workspace): 1839 """Entry point for the devtool 'status' subcommand""" 1840 if workspace: 1841 for recipe, value in sorted(workspace.items()): 1842 recipefile = value['recipefile'] 1843 if recipefile: 1844 recipestr = ' (%s)' % recipefile 1845 else: 1846 recipestr = '' 1847 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1848 else: 1849 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1850 return 0 1851 1852 1853def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1854 """Reset one or more recipes""" 1855 import oe.path 1856 1857 def clean_preferred_provider(pn, layerconf_path): 1858 """Remove PREFERRED_PROVIDER from layer.conf'""" 1859 import re 1860 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1861 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1862 pprovider_found = False 1863 with open(layerconf_file, 'r') as f: 1864 lines = f.readlines() 1865 with open(new_layerconf_file, 'a') as nf: 1866 for line in lines: 1867 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1868 if not re.match(pprovider_exp, line): 1869 nf.write(line) 1870 else: 1871 pprovider_found = True 1872 if pprovider_found: 1873 shutil.move(new_layerconf_file, layerconf_file) 1874 else: 1875 os.remove(new_layerconf_file) 1876 1877 if recipes and not no_clean: 1878 if len(recipes) == 1: 1879 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 1880 else: 1881 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 1882 # If the recipe file itself was created in the workspace, and 1883 # it uses BBCLASSEXTEND, then we need to also clean the other 1884 # variants 1885 targets = [] 1886 for recipe in recipes: 1887 targets.append(recipe) 1888 recipefile = workspace[recipe]['recipefile'] 1889 if recipefile and os.path.exists(recipefile): 1890 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 1891 try: 1892 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 1893 except bb.process.ExecutionError as e: 1894 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 1895 'wish, you may specify -n/--no-clean to ' 1896 'skip running this command when resetting' % 1897 (e.command, e.stdout)) 1898 1899 for pn in recipes: 1900 _check_preserve(config, pn) 1901 1902 appendfile = workspace[pn]['bbappend'] 1903 if os.path.exists(appendfile): 1904 # This shouldn't happen, but is possible if devtool errored out prior to 1905 # writing the md5 file. We need to delete this here or the recipe won't 1906 # actually be reset 1907 os.remove(appendfile) 1908 1909 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 1910 def preservedir(origdir): 1911 if os.path.exists(origdir): 1912 for root, dirs, files in os.walk(origdir): 1913 for fn in files: 1914 logger.warning('Preserving %s in %s' % (fn, preservepath)) 1915 _move_file(os.path.join(origdir, fn), 1916 os.path.join(preservepath, fn)) 1917 for dn in dirs: 1918 preservedir(os.path.join(root, dn)) 1919 os.rmdir(origdir) 1920 1921 recipefile = workspace[pn]['recipefile'] 1922 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 1923 # This should always be true if recipefile is set, but just in case 1924 preservedir(os.path.dirname(recipefile)) 1925 # We don't automatically create this dir next to appends, but the user can 1926 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 1927 1928 srctreebase = workspace[pn]['srctreebase'] 1929 if os.path.isdir(srctreebase): 1930 if os.listdir(srctreebase): 1931 if remove_work: 1932 logger.info('-r argument used on %s, removing source tree.' 1933 ' You will lose any unsaved work' %pn) 1934 shutil.rmtree(srctreebase) 1935 else: 1936 # We don't want to risk wiping out any work in progress 1937 logger.info('Leaving source tree %s as-is; if you no ' 1938 'longer need it then please delete it manually' 1939 % srctreebase) 1940 else: 1941 # This is unlikely, but if it's empty we can just remove it 1942 os.rmdir(srctreebase) 1943 1944 clean_preferred_provider(pn, config.workspace_path) 1945 1946def reset(args, config, basepath, workspace): 1947 """Entry point for the devtool 'reset' subcommand""" 1948 import bb 1949 import shutil 1950 1951 recipes = "" 1952 1953 if args.recipename: 1954 if args.all: 1955 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 1956 else: 1957 for recipe in args.recipename: 1958 check_workspace_recipe(workspace, recipe, checksrc=False) 1959 elif not args.all: 1960 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 1961 "reset all recipes") 1962 if args.all: 1963 recipes = list(workspace.keys()) 1964 else: 1965 recipes = args.recipename 1966 1967 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 1968 1969 return 0 1970 1971 1972def _get_layer(layername, d): 1973 """Determine the base layer path for the specified layer name/path""" 1974 layerdirs = d.getVar('BBLAYERS').split() 1975 layers = {} # {basename: layer_paths} 1976 for p in layerdirs: 1977 bn = os.path.basename(p) 1978 if bn not in layers: 1979 layers[bn] = [p] 1980 else: 1981 layers[bn].append(p) 1982 # Provide some shortcuts 1983 if layername.lower() in ['oe-core', 'openembedded-core']: 1984 layername = 'meta' 1985 layer_paths = layers.get(layername, None) 1986 if not layer_paths: 1987 return os.path.abspath(layername) 1988 elif len(layer_paths) == 1: 1989 return os.path.abspath(layer_paths[0]) 1990 else: 1991 # multiple layers having the same base name 1992 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 1993 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 1994 return os.path.abspath(layer_paths[0]) 1995 1996 1997def finish(args, config, basepath, workspace): 1998 """Entry point for the devtool 'finish' subcommand""" 1999 import bb 2000 import oe.recipeutils 2001 2002 check_workspace_recipe(workspace, args.recipename) 2003 2004 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2005 2006 # Grab the equivalent of COREBASE without having to initialise tinfoil 2007 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2008 2009 srctree = workspace[args.recipename]['srctree'] 2010 check_git_repo_op(srctree, [corebasedir]) 2011 dirty = check_git_repo_dirty(srctree) 2012 if dirty: 2013 if args.force: 2014 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2015 else: 2016 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2017 2018 no_clean = args.no_clean 2019 remove_work=args.remove_work 2020 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2021 try: 2022 rd = parse_recipe(config, tinfoil, args.recipename, True) 2023 if not rd: 2024 return 1 2025 2026 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2027 recipefile = rd.getVar('FILE') 2028 recipedir = os.path.dirname(recipefile) 2029 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2030 2031 if not os.path.isdir(destlayerdir): 2032 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2033 2034 if os.path.abspath(destlayerdir) == config.workspace_path: 2035 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2036 2037 # If it's an upgrade, grab the original path 2038 origpath = None 2039 origfilelist = None 2040 append = workspace[args.recipename]['bbappend'] 2041 with open(append, 'r') as f: 2042 for line in f: 2043 if line.startswith('# original_path:'): 2044 origpath = line.split(':')[1].strip() 2045 elif line.startswith('# original_files:'): 2046 origfilelist = line.split(':')[1].split() 2047 2048 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2049 2050 if origlayerdir == config.workspace_path: 2051 # Recipe file itself is in workspace, update it there first 2052 appendlayerdir = None 2053 origrelpath = None 2054 if origpath: 2055 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2056 if origlayerpath: 2057 origrelpath = os.path.relpath(origpath, origlayerpath) 2058 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2059 if not destpath: 2060 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2061 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2062 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2063 if not os.path.abspath(destlayerbasedir) in layerdirs: 2064 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2065 2066 elif destlayerdir == origlayerdir: 2067 # Same layer, update the original recipe 2068 appendlayerdir = None 2069 destpath = None 2070 else: 2071 # Create/update a bbappend in the specified layer 2072 appendlayerdir = destlayerdir 2073 destpath = None 2074 2075 # Actually update the recipe / bbappend 2076 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2077 dry_run_output = None 2078 dry_run_outdir = None 2079 if args.dry_run: 2080 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2081 dry_run_outdir = dry_run_output.name 2082 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2083 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2084 2085 # Remove any old files in the case of an upgrade 2086 if removing_original: 2087 for fn in origfilelist: 2088 fnp = os.path.join(origpath, fn) 2089 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2090 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2091 if not args.dry_run: 2092 try: 2093 os.remove(fnp) 2094 except FileNotFoundError: 2095 pass 2096 2097 if origlayerdir == config.workspace_path and destpath: 2098 # Recipe file itself is in the workspace - need to move it and any 2099 # associated files to the specified layer 2100 no_clean = True 2101 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2102 for root, _, files in os.walk(recipedir): 2103 for fn in files: 2104 srcpath = os.path.join(root, fn) 2105 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2106 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2107 destfp = os.path.join(destdir, fn) 2108 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2109 2110 if dry_run_outdir: 2111 import difflib 2112 comparelist = [] 2113 for root, _, files in os.walk(dry_run_outdir): 2114 for fn in files: 2115 outf = os.path.join(root, fn) 2116 relf = os.path.relpath(outf, dry_run_outdir) 2117 logger.debug('dry-run: output file %s' % relf) 2118 if fn.endswith('.bb'): 2119 if origfilelist and origpath and destpath: 2120 # Need to match this up with the pre-upgrade recipe file 2121 for origf in origfilelist: 2122 if origf.endswith('.bb'): 2123 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2124 outf, 2125 os.path.abspath(os.path.join(destpath, relf)))) 2126 break 2127 else: 2128 # Compare to the existing recipe 2129 comparelist.append((recipefile, outf, recipefile)) 2130 elif fn.endswith('.bbappend'): 2131 if appendfile: 2132 if os.path.exists(appendfile): 2133 comparelist.append((appendfile, outf, appendfile)) 2134 else: 2135 comparelist.append((None, outf, appendfile)) 2136 else: 2137 if destpath: 2138 recipedest = destpath 2139 elif appendfile: 2140 recipedest = os.path.dirname(appendfile) 2141 else: 2142 recipedest = os.path.dirname(recipefile) 2143 destfp = os.path.join(recipedest, relf) 2144 if os.path.exists(destfp): 2145 comparelist.append((destfp, outf, destfp)) 2146 output = '' 2147 for oldfile, newfile, newfileshow in comparelist: 2148 if oldfile: 2149 with open(oldfile, 'r') as f: 2150 oldlines = f.readlines() 2151 else: 2152 oldfile = '/dev/null' 2153 oldlines = [] 2154 with open(newfile, 'r') as f: 2155 newlines = f.readlines() 2156 if not newfileshow: 2157 newfileshow = newfile 2158 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2159 difflines = list(diff) 2160 if difflines: 2161 output += ''.join(difflines) 2162 if output: 2163 logger.info('Diff of changed files:\n%s' % output) 2164 finally: 2165 tinfoil.shutdown() 2166 2167 # Everything else has succeeded, we can now reset 2168 if args.dry_run: 2169 logger.info('Resetting recipe (dry-run)') 2170 else: 2171 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2172 2173 return 0 2174 2175 2176def get_default_srctree(config, recipename=''): 2177 """Get the default srctree path""" 2178 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2179 if recipename: 2180 return os.path.join(srctreeparent, 'sources', recipename) 2181 else: 2182 return os.path.join(srctreeparent, 'sources') 2183 2184def register_commands(subparsers, context): 2185 """Register devtool subcommands from this plugin""" 2186 2187 defsrctree = get_default_srctree(context.config) 2188 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2189 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2190 group='starting', order=100) 2191 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2192 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2193 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2194 group = parser_add.add_mutually_exclusive_group() 2195 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2196 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2197 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2198 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2199 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2200 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2201 group = parser_add.add_mutually_exclusive_group() 2202 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2203 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2204 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2205 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2206 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2207 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2208 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2209 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2210 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2211 2212 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2213 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2214 group='starting', order=90) 2215 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2216 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2217 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2218 group = parser_modify.add_mutually_exclusive_group() 2219 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2220 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2221 group = parser_modify.add_mutually_exclusive_group() 2222 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2223 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2224 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2225 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2226 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2227 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2228 2229 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2230 description='Extracts the source for an existing recipe', 2231 group='advanced') 2232 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2233 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2234 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2235 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2236 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2237 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2238 2239 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2240 description='Synchronize the previously extracted source tree for an existing recipe', 2241 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2242 group='advanced') 2243 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2244 parser_sync.add_argument('srctree', help='Path to the source tree') 2245 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2246 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2247 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2248 2249 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2250 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2251 group='working', order=10) 2252 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2253 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2254 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2255 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2256 parser_rename.set_defaults(func=rename) 2257 2258 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2259 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2260 group='working', order=-90) 2261 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2262 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2263 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2264 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2265 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2266 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2267 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2268 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2269 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2270 parser_update_recipe.set_defaults(func=update_recipe) 2271 2272 parser_status = subparsers.add_parser('status', help='Show workspace status', 2273 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2274 group='info', order=100) 2275 parser_status.set_defaults(func=status) 2276 2277 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2278 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2279 group='working', order=-100) 2280 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2281 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2282 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2283 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2284 parser_reset.set_defaults(func=reset) 2285 2286 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2287 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2288 group='working', order=-100) 2289 parser_finish.add_argument('recipename', help='Recipe to finish') 2290 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2291 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2292 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2293 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2294 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2295 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2296 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2297 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2298 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2299 parser_finish.set_defaults(func=finish) 2300