1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.fetch_dev: 149 extracmdopts += ' --fetch-dev' 150 if args.mirrors: 151 extracmdopts += ' --mirrors' 152 if args.srcrev: 153 extracmdopts += ' --srcrev %s' % args.srcrev 154 if args.srcbranch: 155 extracmdopts += ' --srcbranch %s' % args.srcbranch 156 if args.provides: 157 extracmdopts += ' --provides %s' % args.provides 158 159 tempdir = tempfile.mkdtemp(prefix='devtool') 160 try: 161 try: 162 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 163 except bb.process.ExecutionError as e: 164 if e.exitcode == 15: 165 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 166 else: 167 raise DevtoolError('Command \'%s\' failed' % e.command) 168 169 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 170 if recipes: 171 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 172 if recipename in workspace: 173 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 174 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 175 bb.utils.mkdirhier(recipedir) 176 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 177 appendfile = recipe_to_append(recipefile, config) 178 if os.path.exists(appendfile): 179 # This shouldn't be possible, but just in case 180 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 181 if os.path.exists(recipefile): 182 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 183 if tmpsrcdir: 184 srctree = os.path.join(srctreeparent, recipename) 185 if os.path.exists(tmpsrcdir): 186 if os.path.exists(srctree): 187 if os.path.isdir(srctree): 188 try: 189 os.rmdir(srctree) 190 except OSError as e: 191 if e.errno == errno.ENOTEMPTY: 192 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 193 else: 194 raise 195 else: 196 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 197 logger.info('Using default source tree path %s' % srctree) 198 shutil.move(tmpsrcdir, srctree) 199 else: 200 raise DevtoolError('Couldn\'t find source tree created by recipetool') 201 bb.utils.mkdirhier(recipedir) 202 shutil.move(recipes[0], recipefile) 203 # Move any additional files created by recipetool 204 for fn in os.listdir(tempdir): 205 shutil.move(os.path.join(tempdir, fn), recipedir) 206 else: 207 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 208 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 209 if os.path.exists(attic_recipe): 210 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 211 finally: 212 if tmpsrcdir and os.path.exists(tmpsrcdir): 213 shutil.rmtree(tmpsrcdir) 214 shutil.rmtree(tempdir) 215 216 for fn in os.listdir(recipedir): 217 _add_md5(config, recipename, os.path.join(recipedir, fn)) 218 219 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 220 try: 221 try: 222 rd = tinfoil.parse_recipe_file(recipefile, False) 223 except Exception as e: 224 logger.error(str(e)) 225 rd = None 226 if not rd: 227 # Parsing failed. We just created this recipe and we shouldn't 228 # leave it in the workdir or it'll prevent bitbake from starting 229 movefn = '%s.parsefailed' % recipefile 230 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 231 shutil.move(recipefile, movefn) 232 return 1 233 234 if args.fetchuri and not args.no_git: 235 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 236 237 initial_rev = None 238 if os.path.exists(os.path.join(srctree, '.git')): 239 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 240 initial_rev = stdout.rstrip() 241 242 if args.src_subdir: 243 srctree = os.path.join(srctree, args.src_subdir) 244 245 bb.utils.mkdirhier(os.path.dirname(appendfile)) 246 with open(appendfile, 'w') as f: 247 f.write('inherit externalsrc\n') 248 f.write('EXTERNALSRC = "%s"\n' % srctree) 249 250 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 251 if b_is_s: 252 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 253 if initial_rev: 254 f.write('\n# initial_rev: %s\n' % initial_rev) 255 256 if args.binary: 257 f.write('do_install_append() {\n') 258 f.write(' rm -rf ${D}/.git\n') 259 f.write(' rm -f ${D}/singletask.lock\n') 260 f.write('}\n') 261 262 if bb.data.inherits_class('npm', rd): 263 f.write('do_install_append() {\n') 264 f.write(' # Remove files added to source dir by devtool/externalsrc\n') 265 f.write(' rm -f ${NPM_INSTALLDIR}/singletask.lock\n') 266 f.write(' rm -rf ${NPM_INSTALLDIR}/.git\n') 267 f.write(' rm -rf ${NPM_INSTALLDIR}/oe-local-files\n') 268 f.write(' for symlink in ${EXTERNALSRC_SYMLINKS} ; do\n') 269 f.write(' rm -f ${NPM_INSTALLDIR}/${symlink%%:*}\n') 270 f.write(' done\n') 271 f.write('}\n') 272 273 # Check if the new layer provides recipes whose priorities have been 274 # overriden by PREFERRED_PROVIDER. 275 recipe_name = rd.getVar('PN') 276 provides = rd.getVar('PROVIDES') 277 # Search every item defined in PROVIDES 278 for recipe_provided in provides.split(): 279 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 280 current_pprovider = rd.getVar(preferred_provider) 281 if current_pprovider and current_pprovider != recipe_name: 282 if args.fixed_setup: 283 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 284 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 285 with open(layerconf_file, 'a') as f: 286 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 287 else: 288 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 289 break 290 291 _add_md5(config, recipename, appendfile) 292 293 check_prerelease_version(rd.getVar('PV'), 'devtool add') 294 295 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 296 297 finally: 298 tinfoil.shutdown() 299 300 return 0 301 302 303def _check_compatible_recipe(pn, d): 304 """Check if the recipe is supported by devtool""" 305 if pn == 'perf': 306 raise DevtoolError("The perf recipe does not actually check out " 307 "source and thus cannot be supported by this tool", 308 4) 309 310 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 311 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 312 313 if bb.data.inherits_class('image', d): 314 raise DevtoolError("The %s recipe is an image, and therefore is not " 315 "supported by this tool" % pn, 4) 316 317 if bb.data.inherits_class('populate_sdk', d): 318 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 319 "supported by this tool" % pn, 4) 320 321 if bb.data.inherits_class('packagegroup', d): 322 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 323 "not supported by this tool" % pn, 4) 324 325 if bb.data.inherits_class('meta', d): 326 raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " 327 "not supported by this tool" % pn, 4) 328 329 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 330 # Not an incompatibility error per se, so we don't pass the error code 331 raise DevtoolError("externalsrc is currently enabled for the %s " 332 "recipe. This prevents the normal do_patch task " 333 "from working. You will need to disable this " 334 "first." % pn) 335 336def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 337 """Common function for copying a file to the dry run output directory""" 338 relpath = os.path.relpath(dst, base_outdir) 339 if relpath.startswith('..'): 340 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 341 dst = os.path.join(dry_run_outdir, relpath) 342 dst_d = os.path.dirname(dst) 343 if dst_d: 344 bb.utils.mkdirhier(dst_d) 345 # Don't overwrite existing files, otherwise in the case of an upgrade 346 # the dry-run written out recipe will be overwritten with an unmodified 347 # version 348 if not os.path.exists(dst): 349 shutil.copy(src, dst) 350 351def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 352 """Move a file. Creates all the directory components of destination path.""" 353 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 354 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 355 if dry_run_outdir: 356 # We want to copy here, not move 357 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 358 else: 359 dst_d = os.path.dirname(dst) 360 if dst_d: 361 bb.utils.mkdirhier(dst_d) 362 shutil.move(src, dst) 363 364def _copy_file(src, dst, dry_run_outdir=None): 365 """Copy a file. Creates all the directory components of destination path.""" 366 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 367 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 368 if dry_run_outdir: 369 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 370 else: 371 dst_d = os.path.dirname(dst) 372 if dst_d: 373 bb.utils.mkdirhier(dst_d) 374 shutil.copy(src, dst) 375 376def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 377 """List contents of a git treeish""" 378 import bb 379 cmd = ['git', 'ls-tree', '-z', treeish] 380 if recursive: 381 cmd.append('-r') 382 out, _ = bb.process.run(cmd, cwd=repodir) 383 ret = {} 384 if out: 385 for line in out.split('\0'): 386 if line: 387 split = line.split(None, 4) 388 ret[split[3]] = split[0:3] 389 return ret 390 391def _git_exclude_path(srctree, path): 392 """Return pathspec (list of paths) that excludes certain path""" 393 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 394 # we don't catch files that are deleted, for example. A more reliable way 395 # to implement this would be to use "negative pathspecs" which were 396 # introduced in Git v1.9.0. Revisit this when/if the required Git version 397 # becomes greater than that. 398 path = os.path.normpath(path) 399 recurse = True if len(path.split(os.path.sep)) > 1 else False 400 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 401 if path in git_files: 402 git_files.remove(path) 403 return git_files 404 else: 405 return ['.'] 406 407def _ls_tree(directory): 408 """Recursive listing of files in a directory""" 409 ret = [] 410 for root, dirs, files in os.walk(directory): 411 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 412 fname in files]) 413 return ret 414 415 416def extract(args, config, basepath, workspace): 417 """Entry point for the devtool 'extract' subcommand""" 418 import bb 419 420 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 421 if not tinfoil: 422 # Error already shown 423 return 1 424 try: 425 rd = parse_recipe(config, tinfoil, args.recipename, True) 426 if not rd: 427 return 1 428 429 srctree = os.path.abspath(args.srctree) 430 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 431 logger.info('Source tree extracted to %s' % srctree) 432 433 if initial_rev: 434 return 0 435 else: 436 return 1 437 finally: 438 tinfoil.shutdown() 439 440def sync(args, config, basepath, workspace): 441 """Entry point for the devtool 'sync' subcommand""" 442 import bb 443 444 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 445 if not tinfoil: 446 # Error already shown 447 return 1 448 try: 449 rd = parse_recipe(config, tinfoil, args.recipename, True) 450 if not rd: 451 return 1 452 453 srctree = os.path.abspath(args.srctree) 454 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 455 logger.info('Source tree %s synchronized' % srctree) 456 457 if initial_rev: 458 return 0 459 else: 460 return 1 461 finally: 462 tinfoil.shutdown() 463 464def symlink_oelocal_files_srctree(rd,srctree): 465 import oe.patch 466 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 467 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 468 # (otherwise the recipe won't build as expected) 469 local_files_dir = os.path.join(srctree, 'oe-local-files') 470 addfiles = [] 471 for root, _, files in os.walk(local_files_dir): 472 relpth = os.path.relpath(root, local_files_dir) 473 if relpth != '.': 474 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 475 for fn in files: 476 if fn == '.gitignore': 477 continue 478 destpth = os.path.join(srctree, relpth, fn) 479 if os.path.exists(destpth): 480 os.unlink(destpth) 481 os.symlink('oe-local-files/%s' % fn, destpth) 482 addfiles.append(os.path.join(relpth, fn)) 483 if addfiles: 484 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 485 useroptions = [] 486 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 487 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 488 489 490def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 491 """Extract sources of a recipe""" 492 import oe.recipeutils 493 import oe.patch 494 import oe.path 495 496 pn = d.getVar('PN') 497 498 _check_compatible_recipe(pn, d) 499 500 if sync: 501 if not os.path.exists(srctree): 502 raise DevtoolError("output path %s does not exist" % srctree) 503 else: 504 if os.path.exists(srctree): 505 if not os.path.isdir(srctree): 506 raise DevtoolError("output path %s exists and is not a directory" % 507 srctree) 508 elif os.listdir(srctree): 509 raise DevtoolError("output path %s already exists and is " 510 "non-empty" % srctree) 511 512 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 513 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 514 "extract source" % pn, 4) 515 516 if not sync: 517 # Prepare for shutil.move later on 518 bb.utils.mkdirhier(srctree) 519 os.rmdir(srctree) 520 521 extra_overrides = [] 522 if not no_overrides: 523 history = d.varhistory.variable('SRC_URI') 524 for event in history: 525 if not 'flag' in event: 526 if event['op'].startswith(('_append[', '_prepend[')): 527 extra_overrides.append(event['op'].split('[')[1].split(']')[0]) 528 # We want to remove duplicate overrides. If a recipe had multiple 529 # SRC_URI_override += values it would cause mulitple instances of 530 # overrides. This doesn't play nicely with things like creating a 531 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 532 extra_overrides = list(set(extra_overrides)) 533 if extra_overrides: 534 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 535 536 initial_rev = None 537 538 appendexisted = False 539 recipefile = d.getVar('FILE') 540 appendfile = recipe_to_append(recipefile, config) 541 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 542 543 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 544 # directory so that: 545 # (a) we pick up all files that get unpacked to the WORKDIR, and 546 # (b) we don't disturb the existing build 547 # However, with recipe-specific sysroots the sysroots for the recipe 548 # will be prepared under WORKDIR, and if we used the system temporary 549 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 550 # our attempts to hardlink files into the recipe-specific sysroots 551 # will fail on systems where /tmp is a different filesystem, and it 552 # would have to fall back to copying the files which is a waste of 553 # time. Put the temp directory under the WORKDIR to prevent that from 554 # being a problem. 555 tempbasedir = d.getVar('WORKDIR') 556 bb.utils.mkdirhier(tempbasedir) 557 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 558 try: 559 tinfoil.logger.setLevel(logging.WARNING) 560 561 # FIXME this results in a cache reload under control of tinfoil, which is fine 562 # except we don't get the knotty progress bar 563 564 if os.path.exists(appendfile): 565 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 566 shutil.copyfile(appendfile, appendbackup) 567 else: 568 appendbackup = None 569 bb.utils.mkdirhier(os.path.dirname(appendfile)) 570 logger.debug('writing append file %s' % appendfile) 571 with open(appendfile, 'a') as f: 572 f.write('###--- _extract_source\n') 573 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 574 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 575 if not is_kernel_yocto: 576 f.write('PATCHTOOL = "git"\n') 577 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 578 if extra_overrides: 579 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 580 f.write('inherit devtool-source\n') 581 f.write('###--- _extract_source\n') 582 583 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 584 585 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 586 bb.utils.mkdirhier(sstate_manifests) 587 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 588 with open(preservestampfile, 'w') as f: 589 f.write(d.getVar('STAMP')) 590 try: 591 if is_kernel_yocto: 592 # We need to generate the kernel config 593 task = 'do_configure' 594 else: 595 task = 'do_patch' 596 597 # Run the fetch + unpack tasks 598 res = tinfoil.build_targets(pn, 599 task, 600 handle_events=True) 601 finally: 602 if os.path.exists(preservestampfile): 603 os.remove(preservestampfile) 604 605 if not res: 606 raise DevtoolError('Extracting source for %s failed' % pn) 607 608 try: 609 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 610 initial_rev = f.read() 611 612 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 613 srcsubdir = f.read() 614 except FileNotFoundError as e: 615 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 616 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 617 618 # Check if work-shared is empty, if yes 619 # find source and copy to work-shared 620 if is_kernel_yocto: 621 workshareddir = d.getVar('STAGING_KERNEL_DIR') 622 staging_kerVer = get_staging_kver(workshareddir) 623 kernelVersion = d.getVar('LINUX_VERSION') 624 625 # handle dangling symbolic link in work-shared: 626 if os.path.islink(workshareddir): 627 os.unlink(workshareddir) 628 629 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 630 shutil.rmtree(workshareddir) 631 oe.path.copyhardlinktree(srcsubdir,workshareddir) 632 elif not os.path.exists(workshareddir): 633 oe.path.copyhardlinktree(srcsubdir,workshareddir) 634 635 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 636 srctree_localdir = os.path.join(srctree, 'oe-local-files') 637 638 if sync: 639 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 640 641 # Move oe-local-files directory to srctree 642 # As the oe-local-files is not part of the constructed git tree, 643 # remove them directly during the synchrounizating might surprise 644 # the users. Instead, we move it to oe-local-files.bak and remind 645 # user in the log message. 646 if os.path.exists(srctree_localdir + '.bak'): 647 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 648 649 if os.path.exists(srctree_localdir): 650 logger.info('Backing up current local file directory %s' % srctree_localdir) 651 shutil.move(srctree_localdir, srctree_localdir + '.bak') 652 653 if os.path.exists(tempdir_localdir): 654 logger.info('Syncing local source files to srctree...') 655 shutil.copytree(tempdir_localdir, srctree_localdir) 656 else: 657 # Move oe-local-files directory to srctree 658 if os.path.exists(tempdir_localdir): 659 logger.info('Adding local source files to srctree...') 660 shutil.move(tempdir_localdir, srcsubdir) 661 662 shutil.move(srcsubdir, srctree) 663 symlink_oelocal_files_srctree(d,srctree) 664 665 if is_kernel_yocto: 666 logger.info('Copying kernel config to srctree') 667 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 668 669 finally: 670 if appendbackup: 671 shutil.copyfile(appendbackup, appendfile) 672 elif os.path.exists(appendfile): 673 os.remove(appendfile) 674 if keep_temp: 675 logger.info('Preserving temporary directory %s' % tempdir) 676 else: 677 shutil.rmtree(tempdir) 678 return initial_rev, srcsubdir_rel 679 680def _add_md5(config, recipename, filename): 681 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 682 import bb.utils 683 684 def addfile(fn): 685 md5 = bb.utils.md5_file(fn) 686 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 687 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 688 f.seek(0, os.SEEK_SET) 689 if not md5_str in f.read(): 690 f.write(md5_str) 691 692 if os.path.isdir(filename): 693 for root, _, files in os.walk(filename): 694 for f in files: 695 addfile(os.path.join(root, f)) 696 else: 697 addfile(filename) 698 699def _check_preserve(config, recipename): 700 """Check if a file was manually changed and needs to be saved in 'attic' 701 directory""" 702 import bb.utils 703 origfile = os.path.join(config.workspace_path, '.devtool_md5') 704 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 705 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 706 with open(origfile, 'r') as f: 707 with open(newfile, 'w') as tf: 708 for line in f.readlines(): 709 splitline = line.rstrip().split('|') 710 if splitline[0] == recipename: 711 removefile = os.path.join(config.workspace_path, splitline[1]) 712 try: 713 md5 = bb.utils.md5_file(removefile) 714 except IOError as err: 715 if err.errno == 2: 716 # File no longer exists, skip it 717 continue 718 else: 719 raise 720 if splitline[2] != md5: 721 bb.utils.mkdirhier(preservepath) 722 preservefile = os.path.basename(removefile) 723 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 724 shutil.move(removefile, os.path.join(preservepath, preservefile)) 725 else: 726 os.remove(removefile) 727 else: 728 tf.write(line) 729 os.rename(newfile, origfile) 730 731def get_staging_kver(srcdir): 732 # Kernel version from work-shared 733 kerver = [] 734 staging_kerVer="" 735 if os.path.exists(srcdir) and os.listdir(srcdir): 736 with open(os.path.join(srcdir,"Makefile")) as f: 737 version = [next(f) for x in range(5)][1:4] 738 for word in version: 739 kerver.append(word.split('= ')[1].split('\n')[0]) 740 staging_kerVer = ".".join(kerver) 741 return staging_kerVer 742 743def get_staging_kbranch(srcdir): 744 staging_kbranch = "" 745 if os.path.exists(srcdir) and os.listdir(srcdir): 746 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) 747 staging_kbranch = "".join(branch.split('\n')[0]) 748 return staging_kbranch 749 750def modify(args, config, basepath, workspace): 751 """Entry point for the devtool 'modify' subcommand""" 752 import bb 753 import oe.recipeutils 754 import oe.patch 755 import oe.path 756 757 if args.recipename in workspace: 758 raise DevtoolError("recipe %s is already in your workspace" % 759 args.recipename) 760 761 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 762 try: 763 rd = parse_recipe(config, tinfoil, args.recipename, True) 764 if not rd: 765 return 1 766 767 pn = rd.getVar('PN') 768 if pn != args.recipename: 769 logger.info('Mapping %s to %s' % (args.recipename, pn)) 770 if pn in workspace: 771 raise DevtoolError("recipe %s is already in your workspace" % 772 pn) 773 774 if args.srctree: 775 srctree = os.path.abspath(args.srctree) 776 else: 777 srctree = get_default_srctree(config, pn) 778 779 if args.no_extract and not os.path.isdir(srctree): 780 raise DevtoolError("--no-extract specified and source path %s does " 781 "not exist or is not a directory" % 782 srctree) 783 784 recipefile = rd.getVar('FILE') 785 appendfile = recipe_to_append(recipefile, config, args.wildcard) 786 if os.path.exists(appendfile): 787 raise DevtoolError("Another variant of recipe %s is already in your " 788 "workspace (only one variant of a recipe can " 789 "currently be worked on at once)" 790 % pn) 791 792 _check_compatible_recipe(pn, rd) 793 794 initial_rev = None 795 commits = [] 796 check_commits = False 797 798 if bb.data.inherits_class('kernel-yocto', rd): 799 # Current set kernel version 800 kernelVersion = rd.getVar('LINUX_VERSION') 801 srcdir = rd.getVar('STAGING_KERNEL_DIR') 802 kbranch = rd.getVar('KBRANCH') 803 804 staging_kerVer = get_staging_kver(srcdir) 805 staging_kbranch = get_staging_kbranch(srcdir) 806 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 807 oe.path.copyhardlinktree(srcdir,srctree) 808 workdir = rd.getVar('WORKDIR') 809 srcsubdir = rd.getVar('S') 810 localfilesdir = os.path.join(srctree,'oe-local-files') 811 # Move local source files into separate subdir 812 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 813 local_files = oe.recipeutils.get_recipe_local_files(rd) 814 815 for key in local_files.copy(): 816 if key.endswith('scc'): 817 sccfile = open(local_files[key], 'r') 818 for l in sccfile: 819 line = l.split() 820 if line and line[0] in ('kconf', 'patch'): 821 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 822 if not cfg in local_files.values(): 823 local_files[line[-1]] = cfg 824 shutil.copy2(cfg, workdir) 825 sccfile.close() 826 827 # Ignore local files with subdir={BP} 828 srcabspath = os.path.abspath(srcsubdir) 829 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 830 if local_files: 831 for fname in local_files: 832 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 833 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 834 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 835 836 symlink_oelocal_files_srctree(rd,srctree) 837 838 task = 'do_configure' 839 res = tinfoil.build_targets(pn, task, handle_events=True) 840 841 # Copy .config to workspace 842 kconfpath = rd.getVar('B') 843 logger.info('Copying kernel config to workspace') 844 shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 845 846 # Set this to true, we still need to get initial_rev 847 # by parsing the git repo 848 args.no_extract = True 849 850 if not args.no_extract: 851 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 852 if not initial_rev: 853 return 1 854 logger.info('Source tree extracted to %s' % srctree) 855 # Get list of commits since this revision 856 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) 857 commits = stdout.split() 858 check_commits = True 859 else: 860 if os.path.exists(os.path.join(srctree, '.git')): 861 # Check if it's a tree previously extracted by us. This is done 862 # by ensuring that devtool-base and args.branch (devtool) exist. 863 # The check_commits logic will cause an exception if either one 864 # of these doesn't exist 865 try: 866 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 867 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 868 except bb.process.ExecutionError: 869 stdout = '' 870 if stdout: 871 check_commits = True 872 for line in stdout.splitlines(): 873 if line.startswith('*'): 874 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 875 initial_rev = stdout.rstrip() 876 if not initial_rev: 877 # Otherwise, just grab the head revision 878 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 879 initial_rev = stdout.rstrip() 880 881 branch_patches = {} 882 if check_commits: 883 # Check if there are override branches 884 (stdout, _) = bb.process.run('git branch', cwd=srctree) 885 branches = [] 886 for line in stdout.rstrip().splitlines(): 887 branchname = line[2:].rstrip() 888 if branchname.startswith(override_branch_prefix): 889 branches.append(branchname) 890 if branches: 891 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 892 branches.insert(0, args.branch) 893 seen_patches = [] 894 for branch in branches: 895 branch_patches[branch] = [] 896 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 897 for line in stdout.splitlines(): 898 line = line.strip() 899 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 900 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 901 if not origpatch in seen_patches: 902 seen_patches.append(origpatch) 903 branch_patches[branch].append(origpatch) 904 905 # Need to grab this here in case the source is within a subdirectory 906 srctreebase = srctree 907 908 # Check that recipe isn't using a shared workdir 909 s = os.path.abspath(rd.getVar('S')) 910 workdir = os.path.abspath(rd.getVar('WORKDIR')) 911 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 912 # Handle if S is set to a subdirectory of the source 913 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 914 srctree = os.path.join(srctree, srcsubdir) 915 916 bb.utils.mkdirhier(os.path.dirname(appendfile)) 917 with open(appendfile, 'w') as f: 918 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n') 919 # Local files can be modified/tracked in separate subdir under srctree 920 # Mostly useful for packages with S != WORKDIR 921 f.write('FILESPATH_prepend := "%s:"\n' % 922 os.path.join(srctreebase, 'oe-local-files')) 923 f.write('# srctreebase: %s\n' % srctreebase) 924 925 f.write('\ninherit externalsrc\n') 926 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 927 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) 928 929 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 930 if b_is_s: 931 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) 932 933 if bb.data.inherits_class('kernel', rd): 934 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 935 'do_fetch do_unpack do_kernel_configme do_kernel_configcheck"\n') 936 f.write('\ndo_patch[noexec] = "1"\n') 937 f.write('\ndo_configure_append() {\n' 938 ' cp ${B}/.config ${S}/.config.baseline\n' 939 ' ln -sfT ${B}/.config ${S}/.config.new\n' 940 '}\n') 941 if rd.getVarFlag('do_menuconfig','task'): 942 f.write('\ndo_configure_append() {\n' 943 ' cp ${B}/.config ${S}/.config.baseline\n' 944 ' ln -sfT ${B}/.config ${S}/.config.new\n' 945 '}\n') 946 if initial_rev: 947 f.write('\n# initial_rev: %s\n' % initial_rev) 948 for commit in commits: 949 f.write('# commit: %s\n' % commit) 950 if branch_patches: 951 for branch in branch_patches: 952 if branch == args.branch: 953 continue 954 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 955 956 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 957 958 _add_md5(config, pn, appendfile) 959 960 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 961 962 finally: 963 tinfoil.shutdown() 964 965 return 0 966 967 968def rename(args, config, basepath, workspace): 969 """Entry point for the devtool 'rename' subcommand""" 970 import bb 971 import oe.recipeutils 972 973 check_workspace_recipe(workspace, args.recipename) 974 975 if not (args.newname or args.version): 976 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 977 978 recipefile = workspace[args.recipename]['recipefile'] 979 if not recipefile: 980 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 981 982 if args.newname and args.newname != args.recipename: 983 reason = oe.recipeutils.validate_pn(args.newname) 984 if reason: 985 raise DevtoolError(reason) 986 newname = args.newname 987 else: 988 newname = args.recipename 989 990 append = workspace[args.recipename]['bbappend'] 991 appendfn = os.path.splitext(os.path.basename(append))[0] 992 splitfn = appendfn.split('_') 993 if len(splitfn) > 1: 994 origfnver = appendfn.split('_')[1] 995 else: 996 origfnver = '' 997 998 recipefilemd5 = None 999 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1000 try: 1001 rd = parse_recipe(config, tinfoil, args.recipename, True) 1002 if not rd: 1003 return 1 1004 1005 bp = rd.getVar('BP') 1006 bpn = rd.getVar('BPN') 1007 if newname != args.recipename: 1008 localdata = rd.createCopy() 1009 localdata.setVar('PN', newname) 1010 newbpn = localdata.getVar('BPN') 1011 else: 1012 newbpn = bpn 1013 s = rd.getVar('S', False) 1014 src_uri = rd.getVar('SRC_URI', False) 1015 pv = rd.getVar('PV') 1016 1017 # Correct variable values that refer to the upstream source - these 1018 # values must stay the same, so if the name/version are changing then 1019 # we need to fix them up 1020 new_s = s 1021 new_src_uri = src_uri 1022 if newbpn != bpn: 1023 # ${PN} here is technically almost always incorrect, but people do use it 1024 new_s = new_s.replace('${BPN}', bpn) 1025 new_s = new_s.replace('${PN}', bpn) 1026 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1027 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1028 new_src_uri = new_src_uri.replace('${PN}', bpn) 1029 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1030 if args.version and origfnver == pv: 1031 new_s = new_s.replace('${PV}', pv) 1032 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1033 new_src_uri = new_src_uri.replace('${PV}', pv) 1034 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1035 patchfields = {} 1036 if new_s != s: 1037 patchfields['S'] = new_s 1038 if new_src_uri != src_uri: 1039 patchfields['SRC_URI'] = new_src_uri 1040 if patchfields: 1041 recipefilemd5 = bb.utils.md5_file(recipefile) 1042 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1043 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1044 finally: 1045 tinfoil.shutdown() 1046 1047 if args.version: 1048 newver = args.version 1049 else: 1050 newver = origfnver 1051 1052 if newver: 1053 newappend = '%s_%s.bbappend' % (newname, newver) 1054 newfile = '%s_%s.bb' % (newname, newver) 1055 else: 1056 newappend = '%s.bbappend' % newname 1057 newfile = '%s.bb' % newname 1058 1059 oldrecipedir = os.path.dirname(recipefile) 1060 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1061 if oldrecipedir != newrecipedir: 1062 bb.utils.mkdirhier(newrecipedir) 1063 1064 newappend = os.path.join(os.path.dirname(append), newappend) 1065 newfile = os.path.join(newrecipedir, newfile) 1066 1067 # Rename bbappend 1068 logger.info('Renaming %s to %s' % (append, newappend)) 1069 os.rename(append, newappend) 1070 # Rename recipe file 1071 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1072 os.rename(recipefile, newfile) 1073 1074 # Rename source tree if it's the default path 1075 appendmd5 = None 1076 if not args.no_srctree: 1077 srctree = workspace[args.recipename]['srctree'] 1078 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1079 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1080 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1081 shutil.move(srctree, newsrctree) 1082 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1083 appendmd5 = bb.utils.md5_file(newappend) 1084 appendlines = [] 1085 with open(newappend, 'r') as f: 1086 for line in f: 1087 appendlines.append(line) 1088 with open(newappend, 'w') as f: 1089 for line in appendlines: 1090 if srctree in line: 1091 line = line.replace(srctree, newsrctree) 1092 f.write(line) 1093 newappendmd5 = bb.utils.md5_file(newappend) 1094 1095 bpndir = None 1096 newbpndir = None 1097 if newbpn != bpn: 1098 bpndir = os.path.join(oldrecipedir, bpn) 1099 if os.path.exists(bpndir): 1100 newbpndir = os.path.join(newrecipedir, newbpn) 1101 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1102 shutil.move(bpndir, newbpndir) 1103 1104 bpdir = None 1105 newbpdir = None 1106 if newver != origfnver or newbpn != bpn: 1107 bpdir = os.path.join(oldrecipedir, bp) 1108 if os.path.exists(bpdir): 1109 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1110 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1111 shutil.move(bpdir, newbpdir) 1112 1113 if oldrecipedir != newrecipedir: 1114 # Move any stray files and delete the old recipe directory 1115 for entry in os.listdir(oldrecipedir): 1116 oldpath = os.path.join(oldrecipedir, entry) 1117 newpath = os.path.join(newrecipedir, entry) 1118 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1119 shutil.move(oldpath, newpath) 1120 os.rmdir(oldrecipedir) 1121 1122 # Now take care of entries in .devtool_md5 1123 md5entries = [] 1124 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1125 for line in f: 1126 md5entries.append(line) 1127 1128 if bpndir and newbpndir: 1129 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1130 else: 1131 relbpndir = None 1132 if bpdir and newbpdir: 1133 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1134 else: 1135 relbpdir = None 1136 1137 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1138 for entry in md5entries: 1139 splitentry = entry.rstrip().split('|') 1140 if len(splitentry) > 2: 1141 if splitentry[0] == args.recipename: 1142 splitentry[0] = newname 1143 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1144 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1145 if appendmd5 and splitentry[2] == appendmd5: 1146 splitentry[2] = newappendmd5 1147 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1148 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1149 if recipefilemd5 and splitentry[2] == recipefilemd5: 1150 splitentry[2] = newrecipefilemd5 1151 elif relbpndir and splitentry[1].startswith(relbpndir): 1152 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1153 elif relbpdir and splitentry[1].startswith(relbpdir): 1154 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1155 entry = '|'.join(splitentry) + '\n' 1156 f.write(entry) 1157 return 0 1158 1159 1160def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1161 """Get initial and update rev of a recipe. These are the start point of the 1162 whole patchset and start point for the patches to be re-generated/updated. 1163 """ 1164 import bb 1165 1166 # Get current branch 1167 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1168 cwd=srctree) 1169 branchname = stdout.rstrip() 1170 1171 # Parse initial rev from recipe if not specified 1172 commits = [] 1173 patches = [] 1174 with open(recipe_path, 'r') as f: 1175 for line in f: 1176 if line.startswith('# initial_rev:'): 1177 if not initial_rev: 1178 initial_rev = line.split(':')[-1].strip() 1179 elif line.startswith('# commit:') and not force_patch_refresh: 1180 commits.append(line.split(':')[-1].strip()) 1181 elif line.startswith('# patches_%s:' % branchname): 1182 patches = line.split(':')[-1].strip().split(',') 1183 1184 update_rev = initial_rev 1185 changed_revs = None 1186 if initial_rev: 1187 # Find first actually changed revision 1188 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1189 initial_rev, cwd=srctree) 1190 newcommits = stdout.split() 1191 for i in range(min(len(commits), len(newcommits))): 1192 if newcommits[i] == commits[i]: 1193 update_rev = commits[i] 1194 1195 try: 1196 stdout, _ = bb.process.run('git cherry devtool-patched', 1197 cwd=srctree) 1198 except bb.process.ExecutionError as err: 1199 stdout = None 1200 1201 if stdout is not None and not force_patch_refresh: 1202 changed_revs = [] 1203 for line in stdout.splitlines(): 1204 if line.startswith('+ '): 1205 rev = line.split()[1] 1206 if rev in newcommits: 1207 changed_revs.append(rev) 1208 1209 return initial_rev, update_rev, changed_revs, patches 1210 1211def _remove_file_entries(srcuri, filelist): 1212 """Remove file:// entries from SRC_URI""" 1213 remaining = filelist[:] 1214 entries = [] 1215 for fname in filelist: 1216 basename = os.path.basename(fname) 1217 for i in range(len(srcuri)): 1218 if (srcuri[i].startswith('file://') and 1219 os.path.basename(srcuri[i].split(';')[0]) == basename): 1220 entries.append(srcuri[i]) 1221 remaining.remove(fname) 1222 srcuri.pop(i) 1223 break 1224 return entries, remaining 1225 1226def _replace_srcuri_entry(srcuri, filename, newentry): 1227 """Replace entry corresponding to specified file with a new entry""" 1228 basename = os.path.basename(filename) 1229 for i in range(len(srcuri)): 1230 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1231 srcuri.pop(i) 1232 srcuri.insert(i, newentry) 1233 break 1234 1235def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1236 """Unlink existing patch files""" 1237 1238 dry_run_suffix = ' (dry-run)' if dry_run else '' 1239 1240 for path in files: 1241 if append: 1242 if not destpath: 1243 raise Exception('destpath should be set here') 1244 path = os.path.join(destpath, os.path.basename(path)) 1245 1246 if os.path.exists(path): 1247 if not no_report_remove: 1248 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1249 if not dry_run: 1250 # FIXME "git rm" here would be nice if the file in question is 1251 # tracked 1252 # FIXME there's a chance that this file is referred to by 1253 # another recipe, in which case deleting wouldn't be the 1254 # right thing to do 1255 os.remove(path) 1256 # Remove directory if empty 1257 try: 1258 os.rmdir(os.path.dirname(path)) 1259 except OSError as ose: 1260 if ose.errno != errno.ENOTEMPTY: 1261 raise 1262 1263 1264def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1265 """Export patches from srctree to given location. 1266 Returns three-tuple of dicts: 1267 1. updated - patches that already exist in SRCURI 1268 2. added - new patches that don't exist in SRCURI 1269 3 removed - patches that exist in SRCURI but not in exported patches 1270 In each dict the key is the 'basepath' of the URI and value is the 1271 absolute path to the existing file in recipe space (if any). 1272 """ 1273 import oe.recipeutils 1274 from oe.patch import GitApplyTree 1275 updated = OrderedDict() 1276 added = OrderedDict() 1277 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1278 1279 existing_patches = dict((os.path.basename(path), path) for path in 1280 oe.recipeutils.get_recipe_patches(rd)) 1281 logger.debug('Existing patches: %s' % existing_patches) 1282 1283 # Generate patches from Git, exclude local files directory 1284 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1285 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) 1286 1287 new_patches = sorted(os.listdir(destdir)) 1288 for new_patch in new_patches: 1289 # Strip numbering from patch names. If it's a git sequence named patch, 1290 # the numbers might not match up since we are starting from a different 1291 # revision This does assume that people are using unique shortlog 1292 # values, but they ought to be anyway... 1293 new_basename = seqpatch_re.match(new_patch).group(2) 1294 match_name = None 1295 for old_patch in existing_patches: 1296 old_basename = seqpatch_re.match(old_patch).group(2) 1297 old_basename_splitext = os.path.splitext(old_basename) 1298 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1299 old_patch_noext = os.path.splitext(old_patch)[0] 1300 match_name = old_patch_noext 1301 break 1302 elif new_basename == old_basename: 1303 match_name = old_patch 1304 break 1305 if match_name: 1306 # Rename patch files 1307 if new_patch != match_name: 1308 os.rename(os.path.join(destdir, new_patch), 1309 os.path.join(destdir, match_name)) 1310 # Need to pop it off the list now before checking changed_revs 1311 oldpath = existing_patches.pop(old_patch) 1312 if changed_revs is not None: 1313 # Avoid updating patches that have not actually changed 1314 with open(os.path.join(destdir, match_name), 'r') as f: 1315 firstlineitems = f.readline().split() 1316 # Looking for "From <hash>" line 1317 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1318 if not firstlineitems[1] in changed_revs: 1319 continue 1320 # Recompress if necessary 1321 if oldpath.endswith(('.gz', '.Z')): 1322 bb.process.run(['gzip', match_name], cwd=destdir) 1323 if oldpath.endswith('.gz'): 1324 match_name += '.gz' 1325 else: 1326 match_name += '.Z' 1327 elif oldpath.endswith('.bz2'): 1328 bb.process.run(['bzip2', match_name], cwd=destdir) 1329 match_name += '.bz2' 1330 updated[match_name] = oldpath 1331 else: 1332 added[new_patch] = None 1333 return (updated, added, existing_patches) 1334 1335 1336def _create_kconfig_diff(srctree, rd, outfile): 1337 """Create a kconfig fragment""" 1338 # Only update config fragment if both config files exist 1339 orig_config = os.path.join(srctree, '.config.baseline') 1340 new_config = os.path.join(srctree, '.config.new') 1341 if os.path.exists(orig_config) and os.path.exists(new_config): 1342 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1343 '--unchanged-line-format=', orig_config, new_config] 1344 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1345 stderr=subprocess.PIPE) 1346 stdout, stderr = pipe.communicate() 1347 if pipe.returncode == 1: 1348 logger.info("Updating config fragment %s" % outfile) 1349 with open(outfile, 'wb') as fobj: 1350 fobj.write(stdout) 1351 elif pipe.returncode == 0: 1352 logger.info("Would remove config fragment %s" % outfile) 1353 if os.path.exists(outfile): 1354 # Remove fragment file in case of empty diff 1355 logger.info("Removing config fragment %s" % outfile) 1356 os.unlink(outfile) 1357 else: 1358 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1359 return True 1360 return False 1361 1362 1363def _export_local_files(srctree, rd, destdir, srctreebase): 1364 """Copy local files from srctree to given location. 1365 Returns three-tuple of dicts: 1366 1. updated - files that already exist in SRCURI 1367 2. added - new files files that don't exist in SRCURI 1368 3 removed - files that exist in SRCURI but not in exported files 1369 In each dict the key is the 'basepath' of the URI and value is the 1370 absolute path to the existing file in recipe space (if any). 1371 """ 1372 import oe.recipeutils 1373 1374 # Find out local files (SRC_URI files that exist in the "recipe space"). 1375 # Local files that reside in srctree are not included in patch generation. 1376 # Instead they are directly copied over the original source files (in 1377 # recipe space). 1378 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1379 new_set = None 1380 updated = OrderedDict() 1381 added = OrderedDict() 1382 removed = OrderedDict() 1383 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1384 git_files = _git_ls_tree(srctree) 1385 if 'oe-local-files' in git_files: 1386 # If tracked by Git, take the files from srctree HEAD. First get 1387 # the tree object of the directory 1388 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1389 tree = git_files['oe-local-files'][2] 1390 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1391 env=dict(os.environ, GIT_WORK_TREE=destdir, 1392 GIT_INDEX_FILE=tmp_index)) 1393 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1394 elif os.path.isdir(local_files_dir): 1395 # If not tracked by Git, just copy from working copy 1396 new_set = _ls_tree(local_files_dir) 1397 bb.process.run(['cp', '-ax', 1398 os.path.join(local_files_dir, '.'), destdir]) 1399 else: 1400 new_set = [] 1401 1402 # Special handling for kernel config 1403 if bb.data.inherits_class('kernel-yocto', rd): 1404 fragment_fn = 'devtool-fragment.cfg' 1405 fragment_path = os.path.join(destdir, fragment_fn) 1406 if _create_kconfig_diff(srctree, rd, fragment_path): 1407 if os.path.exists(fragment_path): 1408 if fragment_fn not in new_set: 1409 new_set.append(fragment_fn) 1410 # Copy fragment to local-files 1411 if os.path.isdir(local_files_dir): 1412 shutil.copy2(fragment_path, local_files_dir) 1413 else: 1414 if fragment_fn in new_set: 1415 new_set.remove(fragment_fn) 1416 # Remove fragment from local-files 1417 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1418 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1419 1420 # Special handling for cml1, ccmake, etc bbclasses that generated 1421 # configuration fragment files that are consumed as source files 1422 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1423 if bb.data.inherits_class(frag_class, rd): 1424 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1425 if os.path.exists(srcpath): 1426 if frag_name not in new_set: 1427 new_set.append(frag_name) 1428 # copy fragment into destdir 1429 shutil.copy2(srcpath, destdir) 1430 # copy fragment into local files if exists 1431 if os.path.isdir(local_files_dir): 1432 shutil.copy2(srcpath, local_files_dir) 1433 1434 if new_set is not None: 1435 for fname in new_set: 1436 if fname in existing_files: 1437 origpath = existing_files.pop(fname) 1438 workpath = os.path.join(local_files_dir, fname) 1439 if not filecmp.cmp(origpath, workpath): 1440 updated[fname] = origpath 1441 elif fname != '.gitignore': 1442 added[fname] = None 1443 1444 workdir = rd.getVar('WORKDIR') 1445 s = rd.getVar('S') 1446 if not s.endswith(os.sep): 1447 s += os.sep 1448 1449 if workdir != s: 1450 # Handle files where subdir= was specified 1451 for fname in list(existing_files.keys()): 1452 # FIXME handle both subdir starting with BP and not? 1453 fworkpath = os.path.join(workdir, fname) 1454 if fworkpath.startswith(s): 1455 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1456 if os.path.exists(fpath): 1457 origpath = existing_files.pop(fname) 1458 if not filecmp.cmp(origpath, fpath): 1459 updated[fpath] = origpath 1460 1461 removed = existing_files 1462 return (updated, added, removed) 1463 1464 1465def _determine_files_dir(rd): 1466 """Determine the appropriate files directory for a recipe""" 1467 recipedir = rd.getVar('FILE_DIRNAME') 1468 for entry in rd.getVar('FILESPATH').split(':'): 1469 relpth = os.path.relpath(entry, recipedir) 1470 if not os.sep in relpth: 1471 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1472 if os.path.isdir(entry): 1473 return entry 1474 return os.path.join(recipedir, rd.getVar('BPN')) 1475 1476 1477def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1478 """Implement the 'srcrev' mode of update-recipe""" 1479 import bb 1480 import oe.recipeutils 1481 1482 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1483 1484 recipefile = rd.getVar('FILE') 1485 recipedir = os.path.basename(recipefile) 1486 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1487 1488 # Get HEAD revision 1489 try: 1490 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1491 except bb.process.ExecutionError as err: 1492 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1493 (srctree, err)) 1494 srcrev = stdout.strip() 1495 if len(srcrev) != 40: 1496 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1497 1498 destpath = None 1499 remove_files = [] 1500 patchfields = {} 1501 patchfields['SRCREV'] = srcrev 1502 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1503 srcuri = orig_src_uri.split() 1504 tempdir = tempfile.mkdtemp(prefix='devtool') 1505 update_srcuri = False 1506 appendfile = None 1507 try: 1508 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1509 srctreebase = workspace[recipename]['srctreebase'] 1510 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1511 if not no_remove: 1512 # Find list of existing patches in recipe file 1513 patches_dir = tempfile.mkdtemp(dir=tempdir) 1514 old_srcrev = rd.getVar('SRCREV') or '' 1515 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1516 patches_dir) 1517 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1518 1519 # Remove deleted local files and "overlapping" patches 1520 remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) 1521 if remove_files: 1522 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1523 update_srcuri = True 1524 1525 if appendlayerdir: 1526 files = dict((os.path.join(local_files_dir, key), val) for 1527 key, val in list(upd_f.items()) + list(new_f.items())) 1528 removevalues = {} 1529 if update_srcuri: 1530 removevalues = {'SRC_URI': removedentries} 1531 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1532 if dry_run_outdir: 1533 logger.info('Creating bbappend (dry-run)') 1534 else: 1535 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1536 rd, appendlayerdir, files, wildcardver=wildcard_version, 1537 extralines=patchfields, removevalues=removevalues, 1538 redirect_output=dry_run_outdir) 1539 else: 1540 files_dir = _determine_files_dir(rd) 1541 for basepath, path in upd_f.items(): 1542 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1543 if os.path.isabs(basepath): 1544 # Original file (probably with subdir pointing inside source tree) 1545 # so we do not want to move it, just copy 1546 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1547 else: 1548 _move_file(os.path.join(local_files_dir, basepath), path, 1549 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1550 update_srcuri= True 1551 for basepath, path in new_f.items(): 1552 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1553 _move_file(os.path.join(local_files_dir, basepath), 1554 os.path.join(files_dir, basepath), 1555 dry_run_outdir=dry_run_outdir, 1556 base_outdir=recipedir) 1557 srcuri.append('file://%s' % basepath) 1558 update_srcuri = True 1559 if update_srcuri: 1560 patchfields['SRC_URI'] = ' '.join(srcuri) 1561 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1562 finally: 1563 shutil.rmtree(tempdir) 1564 if not 'git://' in orig_src_uri: 1565 logger.info('You will need to update SRC_URI within the recipe to ' 1566 'point to a git repository where you have pushed your ' 1567 'changes') 1568 1569 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1570 return True, appendfile, remove_files 1571 1572def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1573 """Implement the 'patch' mode of update-recipe""" 1574 import bb 1575 import oe.recipeutils 1576 1577 recipefile = rd.getVar('FILE') 1578 recipedir = os.path.dirname(recipefile) 1579 append = workspace[recipename]['bbappend'] 1580 if not os.path.exists(append): 1581 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1582 recipename) 1583 1584 initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1585 if not initial_rev: 1586 raise DevtoolError('Unable to find initial revision - please specify ' 1587 'it with --initial-rev') 1588 1589 appendfile = None 1590 dl_dir = rd.getVar('DL_DIR') 1591 if not dl_dir.endswith('/'): 1592 dl_dir += '/' 1593 1594 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1595 1596 tempdir = tempfile.mkdtemp(prefix='devtool') 1597 try: 1598 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1599 if filter_patches: 1600 upd_f = {} 1601 new_f = {} 1602 del_f = {} 1603 else: 1604 srctreebase = workspace[recipename]['srctreebase'] 1605 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1606 1607 remove_files = [] 1608 if not no_remove: 1609 # Get all patches from source tree and check if any should be removed 1610 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1611 _, _, del_p = _export_patches(srctree, rd, initial_rev, 1612 all_patches_dir) 1613 # Remove deleted local files and patches 1614 remove_files = list(del_f.values()) + list(del_p.values()) 1615 1616 # Get updated patches from source tree 1617 patches_dir = tempfile.mkdtemp(dir=tempdir) 1618 upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, 1619 patches_dir, changed_revs) 1620 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1621 if filter_patches: 1622 new_p = OrderedDict() 1623 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1624 remove_files = [f for f in remove_files if f in filter_patches] 1625 updatefiles = False 1626 updaterecipe = False 1627 destpath = None 1628 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1629 if appendlayerdir: 1630 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1631 key, val in list(upd_f.items()) + list(new_f.items())) 1632 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1633 key, val in list(upd_p.items()) + list(new_p.items()))) 1634 if files or remove_files: 1635 removevalues = None 1636 if remove_files: 1637 removedentries, remaining = _remove_file_entries( 1638 srcuri, remove_files) 1639 if removedentries or remaining: 1640 remaining = ['file://' + os.path.basename(item) for 1641 item in remaining] 1642 removevalues = {'SRC_URI': removedentries + remaining} 1643 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1644 rd, appendlayerdir, files, 1645 wildcardver=wildcard_version, 1646 removevalues=removevalues, 1647 redirect_output=dry_run_outdir) 1648 else: 1649 logger.info('No patches or local source files needed updating') 1650 else: 1651 # Update existing files 1652 files_dir = _determine_files_dir(rd) 1653 for basepath, path in upd_f.items(): 1654 logger.info('Updating file %s' % basepath) 1655 if os.path.isabs(basepath): 1656 # Original file (probably with subdir pointing inside source tree) 1657 # so we do not want to move it, just copy 1658 _copy_file(basepath, path, 1659 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1660 else: 1661 _move_file(os.path.join(local_files_dir, basepath), path, 1662 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1663 updatefiles = True 1664 for basepath, path in upd_p.items(): 1665 patchfn = os.path.join(patches_dir, basepath) 1666 if os.path.dirname(path) + '/' == dl_dir: 1667 # This is a a downloaded patch file - we now need to 1668 # replace the entry in SRC_URI with our local version 1669 logger.info('Replacing remote patch %s with updated local version' % basepath) 1670 path = os.path.join(files_dir, basepath) 1671 _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) 1672 updaterecipe = True 1673 else: 1674 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1675 _move_file(patchfn, path, 1676 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1677 updatefiles = True 1678 # Add any new files 1679 for basepath, path in new_f.items(): 1680 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1681 _move_file(os.path.join(local_files_dir, basepath), 1682 os.path.join(files_dir, basepath), 1683 dry_run_outdir=dry_run_outdir, 1684 base_outdir=recipedir) 1685 srcuri.append('file://%s' % basepath) 1686 updaterecipe = True 1687 for basepath, path in new_p.items(): 1688 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1689 _move_file(os.path.join(patches_dir, basepath), 1690 os.path.join(files_dir, basepath), 1691 dry_run_outdir=dry_run_outdir, 1692 base_outdir=recipedir) 1693 srcuri.append('file://%s' % basepath) 1694 updaterecipe = True 1695 # Update recipe, if needed 1696 if _remove_file_entries(srcuri, remove_files)[0]: 1697 updaterecipe = True 1698 if updaterecipe: 1699 if not dry_run_outdir: 1700 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1701 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1702 {'SRC_URI': ' '.join(srcuri)}, 1703 redirect_output=dry_run_outdir) 1704 elif not updatefiles: 1705 # Neither patches nor recipe were updated 1706 logger.info('No patches or files need updating') 1707 return False, None, [] 1708 finally: 1709 shutil.rmtree(tempdir) 1710 1711 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1712 return True, appendfile, remove_files 1713 1714def _guess_recipe_update_mode(srctree, rdata): 1715 """Guess the recipe update mode to use""" 1716 src_uri = (rdata.getVar('SRC_URI', False) or '').split() 1717 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1718 if not git_uris: 1719 return 'patch' 1720 # Just use the first URI for now 1721 uri = git_uris[0] 1722 # Check remote branch 1723 params = bb.fetch.decodeurl(uri)[5] 1724 upstr_branch = params['branch'] if 'branch' in params else 'master' 1725 # Check if current branch HEAD is found in upstream branch 1726 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1727 head_rev = stdout.rstrip() 1728 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1729 cwd=srctree) 1730 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1731 if 'origin/' + upstr_branch in remote_brs: 1732 return 'srcrev' 1733 1734 return 'patch' 1735 1736def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1737 srctree = workspace[recipename]['srctree'] 1738 if mode == 'auto': 1739 mode = _guess_recipe_update_mode(srctree, rd) 1740 1741 override_branches = [] 1742 mainbranch = None 1743 startbranch = None 1744 if not no_overrides: 1745 stdout, _ = bb.process.run('git branch', cwd=srctree) 1746 other_branches = [] 1747 for line in stdout.splitlines(): 1748 branchname = line[2:] 1749 if line.startswith('* '): 1750 startbranch = branchname 1751 if branchname.startswith(override_branch_prefix): 1752 override_branches.append(branchname) 1753 else: 1754 other_branches.append(branchname) 1755 1756 if override_branches: 1757 logger.debug('_update_recipe: override branches: %s' % override_branches) 1758 logger.debug('_update_recipe: other branches: %s' % other_branches) 1759 if startbranch.startswith(override_branch_prefix): 1760 if len(other_branches) == 1: 1761 mainbranch = other_branches[1] 1762 else: 1763 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1764 else: 1765 mainbranch = startbranch 1766 1767 checkedout = None 1768 anyupdated = False 1769 appendfile = None 1770 allremoved = [] 1771 if override_branches: 1772 logger.info('Handling main branch (%s)...' % mainbranch) 1773 if startbranch != mainbranch: 1774 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1775 checkedout = mainbranch 1776 try: 1777 branchlist = [mainbranch] + override_branches 1778 for branch in branchlist: 1779 crd = bb.data.createCopy(rd) 1780 if branch != mainbranch: 1781 logger.info('Handling branch %s...' % branch) 1782 override = branch[len(override_branch_prefix):] 1783 crd.appendVar('OVERRIDES', ':%s' % override) 1784 bb.process.run('git checkout %s' % branch, cwd=srctree) 1785 checkedout = branch 1786 1787 if mode == 'srcrev': 1788 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1789 elif mode == 'patch': 1790 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1791 else: 1792 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1793 if updated: 1794 anyupdated = True 1795 if appendf: 1796 appendfile = appendf 1797 allremoved.extend(removed) 1798 finally: 1799 if startbranch and checkedout != startbranch: 1800 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1801 1802 return anyupdated, appendfile, allremoved 1803 1804def update_recipe(args, config, basepath, workspace): 1805 """Entry point for the devtool 'update-recipe' subcommand""" 1806 check_workspace_recipe(workspace, args.recipename) 1807 1808 if args.append: 1809 if not os.path.exists(args.append): 1810 raise DevtoolError('bbappend destination layer directory "%s" ' 1811 'does not exist' % args.append) 1812 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1813 raise DevtoolError('conf/layer.conf not found in bbappend ' 1814 'destination layer "%s"' % args.append) 1815 1816 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1817 try: 1818 1819 rd = parse_recipe(config, tinfoil, args.recipename, True) 1820 if not rd: 1821 return 1 1822 1823 dry_run_output = None 1824 dry_run_outdir = None 1825 if args.dry_run: 1826 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1827 dry_run_outdir = dry_run_output.name 1828 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1829 1830 if updated: 1831 rf = rd.getVar('FILE') 1832 if rf.startswith(config.workspace_path): 1833 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1834 finally: 1835 tinfoil.shutdown() 1836 1837 return 0 1838 1839 1840def status(args, config, basepath, workspace): 1841 """Entry point for the devtool 'status' subcommand""" 1842 if workspace: 1843 for recipe, value in sorted(workspace.items()): 1844 recipefile = value['recipefile'] 1845 if recipefile: 1846 recipestr = ' (%s)' % recipefile 1847 else: 1848 recipestr = '' 1849 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1850 else: 1851 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1852 return 0 1853 1854 1855def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1856 """Reset one or more recipes""" 1857 import oe.path 1858 1859 def clean_preferred_provider(pn, layerconf_path): 1860 """Remove PREFERRED_PROVIDER from layer.conf'""" 1861 import re 1862 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1863 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1864 pprovider_found = False 1865 with open(layerconf_file, 'r') as f: 1866 lines = f.readlines() 1867 with open(new_layerconf_file, 'a') as nf: 1868 for line in lines: 1869 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1870 if not re.match(pprovider_exp, line): 1871 nf.write(line) 1872 else: 1873 pprovider_found = True 1874 if pprovider_found: 1875 shutil.move(new_layerconf_file, layerconf_file) 1876 else: 1877 os.remove(new_layerconf_file) 1878 1879 if recipes and not no_clean: 1880 if len(recipes) == 1: 1881 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 1882 else: 1883 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 1884 # If the recipe file itself was created in the workspace, and 1885 # it uses BBCLASSEXTEND, then we need to also clean the other 1886 # variants 1887 targets = [] 1888 for recipe in recipes: 1889 targets.append(recipe) 1890 recipefile = workspace[recipe]['recipefile'] 1891 if recipefile and os.path.exists(recipefile): 1892 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 1893 try: 1894 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 1895 except bb.process.ExecutionError as e: 1896 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 1897 'wish, you may specify -n/--no-clean to ' 1898 'skip running this command when resetting' % 1899 (e.command, e.stdout)) 1900 1901 for pn in recipes: 1902 _check_preserve(config, pn) 1903 1904 appendfile = workspace[pn]['bbappend'] 1905 if os.path.exists(appendfile): 1906 # This shouldn't happen, but is possible if devtool errored out prior to 1907 # writing the md5 file. We need to delete this here or the recipe won't 1908 # actually be reset 1909 os.remove(appendfile) 1910 1911 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 1912 def preservedir(origdir): 1913 if os.path.exists(origdir): 1914 for root, dirs, files in os.walk(origdir): 1915 for fn in files: 1916 logger.warning('Preserving %s in %s' % (fn, preservepath)) 1917 _move_file(os.path.join(origdir, fn), 1918 os.path.join(preservepath, fn)) 1919 for dn in dirs: 1920 preservedir(os.path.join(root, dn)) 1921 os.rmdir(origdir) 1922 1923 recipefile = workspace[pn]['recipefile'] 1924 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 1925 # This should always be true if recipefile is set, but just in case 1926 preservedir(os.path.dirname(recipefile)) 1927 # We don't automatically create this dir next to appends, but the user can 1928 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 1929 1930 srctreebase = workspace[pn]['srctreebase'] 1931 if os.path.isdir(srctreebase): 1932 if os.listdir(srctreebase): 1933 if remove_work: 1934 logger.info('-r argument used on %s, removing source tree.' 1935 ' You will lose any unsaved work' %pn) 1936 shutil.rmtree(srctreebase) 1937 else: 1938 # We don't want to risk wiping out any work in progress 1939 logger.info('Leaving source tree %s as-is; if you no ' 1940 'longer need it then please delete it manually' 1941 % srctreebase) 1942 else: 1943 # This is unlikely, but if it's empty we can just remove it 1944 os.rmdir(srctreebase) 1945 1946 clean_preferred_provider(pn, config.workspace_path) 1947 1948def reset(args, config, basepath, workspace): 1949 """Entry point for the devtool 'reset' subcommand""" 1950 import bb 1951 import shutil 1952 1953 recipes = "" 1954 1955 if args.recipename: 1956 if args.all: 1957 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 1958 else: 1959 for recipe in args.recipename: 1960 check_workspace_recipe(workspace, recipe, checksrc=False) 1961 elif not args.all: 1962 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 1963 "reset all recipes") 1964 if args.all: 1965 recipes = list(workspace.keys()) 1966 else: 1967 recipes = args.recipename 1968 1969 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 1970 1971 return 0 1972 1973 1974def _get_layer(layername, d): 1975 """Determine the base layer path for the specified layer name/path""" 1976 layerdirs = d.getVar('BBLAYERS').split() 1977 layers = {} # {basename: layer_paths} 1978 for p in layerdirs: 1979 bn = os.path.basename(p) 1980 if bn not in layers: 1981 layers[bn] = [p] 1982 else: 1983 layers[bn].append(p) 1984 # Provide some shortcuts 1985 if layername.lower() in ['oe-core', 'openembedded-core']: 1986 layername = 'meta' 1987 layer_paths = layers.get(layername, None) 1988 if not layer_paths: 1989 return os.path.abspath(layername) 1990 elif len(layer_paths) == 1: 1991 return os.path.abspath(layer_paths[0]) 1992 else: 1993 # multiple layers having the same base name 1994 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 1995 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 1996 return os.path.abspath(layer_paths[0]) 1997 1998 1999def finish(args, config, basepath, workspace): 2000 """Entry point for the devtool 'finish' subcommand""" 2001 import bb 2002 import oe.recipeutils 2003 2004 check_workspace_recipe(workspace, args.recipename) 2005 2006 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2007 2008 # Grab the equivalent of COREBASE without having to initialise tinfoil 2009 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2010 2011 srctree = workspace[args.recipename]['srctree'] 2012 check_git_repo_op(srctree, [corebasedir]) 2013 dirty = check_git_repo_dirty(srctree) 2014 if dirty: 2015 if args.force: 2016 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2017 else: 2018 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2019 2020 no_clean = args.no_clean 2021 remove_work=args.remove_work 2022 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2023 try: 2024 rd = parse_recipe(config, tinfoil, args.recipename, True) 2025 if not rd: 2026 return 1 2027 2028 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2029 recipefile = rd.getVar('FILE') 2030 recipedir = os.path.dirname(recipefile) 2031 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2032 2033 if not os.path.isdir(destlayerdir): 2034 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2035 2036 if os.path.abspath(destlayerdir) == config.workspace_path: 2037 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2038 2039 # If it's an upgrade, grab the original path 2040 origpath = None 2041 origfilelist = None 2042 append = workspace[args.recipename]['bbappend'] 2043 with open(append, 'r') as f: 2044 for line in f: 2045 if line.startswith('# original_path:'): 2046 origpath = line.split(':')[1].strip() 2047 elif line.startswith('# original_files:'): 2048 origfilelist = line.split(':')[1].split() 2049 2050 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2051 2052 if origlayerdir == config.workspace_path: 2053 # Recipe file itself is in workspace, update it there first 2054 appendlayerdir = None 2055 origrelpath = None 2056 if origpath: 2057 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2058 if origlayerpath: 2059 origrelpath = os.path.relpath(origpath, origlayerpath) 2060 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2061 if not destpath: 2062 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2063 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2064 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2065 if not os.path.abspath(destlayerbasedir) in layerdirs: 2066 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2067 2068 elif destlayerdir == origlayerdir: 2069 # Same layer, update the original recipe 2070 appendlayerdir = None 2071 destpath = None 2072 else: 2073 # Create/update a bbappend in the specified layer 2074 appendlayerdir = destlayerdir 2075 destpath = None 2076 2077 # Actually update the recipe / bbappend 2078 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2079 dry_run_output = None 2080 dry_run_outdir = None 2081 if args.dry_run: 2082 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2083 dry_run_outdir = dry_run_output.name 2084 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2085 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2086 2087 # Remove any old files in the case of an upgrade 2088 if removing_original: 2089 for fn in origfilelist: 2090 fnp = os.path.join(origpath, fn) 2091 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2092 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2093 if not args.dry_run: 2094 try: 2095 os.remove(fnp) 2096 except FileNotFoundError: 2097 pass 2098 2099 if origlayerdir == config.workspace_path and destpath: 2100 # Recipe file itself is in the workspace - need to move it and any 2101 # associated files to the specified layer 2102 no_clean = True 2103 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2104 for root, _, files in os.walk(recipedir): 2105 for fn in files: 2106 srcpath = os.path.join(root, fn) 2107 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2108 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2109 destfp = os.path.join(destdir, fn) 2110 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2111 2112 if dry_run_outdir: 2113 import difflib 2114 comparelist = [] 2115 for root, _, files in os.walk(dry_run_outdir): 2116 for fn in files: 2117 outf = os.path.join(root, fn) 2118 relf = os.path.relpath(outf, dry_run_outdir) 2119 logger.debug('dry-run: output file %s' % relf) 2120 if fn.endswith('.bb'): 2121 if origfilelist and origpath and destpath: 2122 # Need to match this up with the pre-upgrade recipe file 2123 for origf in origfilelist: 2124 if origf.endswith('.bb'): 2125 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2126 outf, 2127 os.path.abspath(os.path.join(destpath, relf)))) 2128 break 2129 else: 2130 # Compare to the existing recipe 2131 comparelist.append((recipefile, outf, recipefile)) 2132 elif fn.endswith('.bbappend'): 2133 if appendfile: 2134 if os.path.exists(appendfile): 2135 comparelist.append((appendfile, outf, appendfile)) 2136 else: 2137 comparelist.append((None, outf, appendfile)) 2138 else: 2139 if destpath: 2140 recipedest = destpath 2141 elif appendfile: 2142 recipedest = os.path.dirname(appendfile) 2143 else: 2144 recipedest = os.path.dirname(recipefile) 2145 destfp = os.path.join(recipedest, relf) 2146 if os.path.exists(destfp): 2147 comparelist.append((destfp, outf, destfp)) 2148 output = '' 2149 for oldfile, newfile, newfileshow in comparelist: 2150 if oldfile: 2151 with open(oldfile, 'r') as f: 2152 oldlines = f.readlines() 2153 else: 2154 oldfile = '/dev/null' 2155 oldlines = [] 2156 with open(newfile, 'r') as f: 2157 newlines = f.readlines() 2158 if not newfileshow: 2159 newfileshow = newfile 2160 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2161 difflines = list(diff) 2162 if difflines: 2163 output += ''.join(difflines) 2164 if output: 2165 logger.info('Diff of changed files:\n%s' % output) 2166 finally: 2167 tinfoil.shutdown() 2168 2169 # Everything else has succeeded, we can now reset 2170 if args.dry_run: 2171 logger.info('Resetting recipe (dry-run)') 2172 else: 2173 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2174 2175 return 0 2176 2177 2178def get_default_srctree(config, recipename=''): 2179 """Get the default srctree path""" 2180 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2181 if recipename: 2182 return os.path.join(srctreeparent, 'sources', recipename) 2183 else: 2184 return os.path.join(srctreeparent, 'sources') 2185 2186def register_commands(subparsers, context): 2187 """Register devtool subcommands from this plugin""" 2188 2189 defsrctree = get_default_srctree(context.config) 2190 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2191 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2192 group='starting', order=100) 2193 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2194 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2195 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2196 group = parser_add.add_mutually_exclusive_group() 2197 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2198 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2199 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2200 parser_add.add_argument('--fetch-dev', help='For npm, also fetch devDependencies', action="store_true") 2201 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2202 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2203 group = parser_add.add_mutually_exclusive_group() 2204 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2205 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2206 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2207 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2208 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2209 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2210 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2211 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2212 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2213 2214 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2215 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2216 group='starting', order=90) 2217 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2218 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2219 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2220 group = parser_modify.add_mutually_exclusive_group() 2221 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2222 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2223 group = parser_modify.add_mutually_exclusive_group() 2224 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2225 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2226 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2227 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2228 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2229 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2230 2231 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2232 description='Extracts the source for an existing recipe', 2233 group='advanced') 2234 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2235 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2236 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2237 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2238 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2239 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2240 2241 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2242 description='Synchronize the previously extracted source tree for an existing recipe', 2243 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2244 group='advanced') 2245 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2246 parser_sync.add_argument('srctree', help='Path to the source tree') 2247 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2248 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2249 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2250 2251 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2252 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2253 group='working', order=10) 2254 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2255 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2256 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2257 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2258 parser_rename.set_defaults(func=rename) 2259 2260 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2261 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2262 group='working', order=-90) 2263 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2264 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2265 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2266 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2267 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2268 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2269 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2270 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2271 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2272 parser_update_recipe.set_defaults(func=update_recipe) 2273 2274 parser_status = subparsers.add_parser('status', help='Show workspace status', 2275 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2276 group='info', order=100) 2277 parser_status.set_defaults(func=status) 2278 2279 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2280 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2281 group='working', order=-100) 2282 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2283 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2284 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2285 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2286 parser_reset.set_defaults(func=reset) 2287 2288 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2289 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2290 group='working', order=-100) 2291 parser_finish.add_argument('recipename', help='Recipe to finish') 2292 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2293 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2294 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2295 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2296 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2297 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2298 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2299 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2300 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2301 parser_finish.set_defaults(func=finish) 2302