1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.npm_dev: 149 extracmdopts += ' --npm-dev' 150 if args.mirrors: 151 extracmdopts += ' --mirrors' 152 if args.srcrev: 153 extracmdopts += ' --srcrev %s' % args.srcrev 154 if args.srcbranch: 155 extracmdopts += ' --srcbranch %s' % args.srcbranch 156 if args.provides: 157 extracmdopts += ' --provides %s' % args.provides 158 159 tempdir = tempfile.mkdtemp(prefix='devtool') 160 try: 161 try: 162 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 163 except bb.process.ExecutionError as e: 164 if e.exitcode == 15: 165 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 166 else: 167 raise DevtoolError('Command \'%s\' failed' % e.command) 168 169 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 170 if recipes: 171 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 172 if recipename in workspace: 173 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 174 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 175 bb.utils.mkdirhier(recipedir) 176 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 177 appendfile = recipe_to_append(recipefile, config) 178 if os.path.exists(appendfile): 179 # This shouldn't be possible, but just in case 180 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 181 if os.path.exists(recipefile): 182 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 183 if tmpsrcdir: 184 srctree = os.path.join(srctreeparent, recipename) 185 if os.path.exists(tmpsrcdir): 186 if os.path.exists(srctree): 187 if os.path.isdir(srctree): 188 try: 189 os.rmdir(srctree) 190 except OSError as e: 191 if e.errno == errno.ENOTEMPTY: 192 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 193 else: 194 raise 195 else: 196 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 197 logger.info('Using default source tree path %s' % srctree) 198 shutil.move(tmpsrcdir, srctree) 199 else: 200 raise DevtoolError('Couldn\'t find source tree created by recipetool') 201 bb.utils.mkdirhier(recipedir) 202 shutil.move(recipes[0], recipefile) 203 # Move any additional files created by recipetool 204 for fn in os.listdir(tempdir): 205 shutil.move(os.path.join(tempdir, fn), recipedir) 206 else: 207 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 208 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 209 if os.path.exists(attic_recipe): 210 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 211 finally: 212 if tmpsrcdir and os.path.exists(tmpsrcdir): 213 shutil.rmtree(tmpsrcdir) 214 shutil.rmtree(tempdir) 215 216 for fn in os.listdir(recipedir): 217 _add_md5(config, recipename, os.path.join(recipedir, fn)) 218 219 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 220 try: 221 try: 222 rd = tinfoil.parse_recipe_file(recipefile, False) 223 except Exception as e: 224 logger.error(str(e)) 225 rd = None 226 if not rd: 227 # Parsing failed. We just created this recipe and we shouldn't 228 # leave it in the workdir or it'll prevent bitbake from starting 229 movefn = '%s.parsefailed' % recipefile 230 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 231 shutil.move(recipefile, movefn) 232 return 1 233 234 if args.fetchuri and not args.no_git: 235 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 236 237 initial_rev = None 238 if os.path.exists(os.path.join(srctree, '.git')): 239 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 240 initial_rev = stdout.rstrip() 241 242 if args.src_subdir: 243 srctree = os.path.join(srctree, args.src_subdir) 244 245 bb.utils.mkdirhier(os.path.dirname(appendfile)) 246 with open(appendfile, 'w') as f: 247 f.write('inherit externalsrc\n') 248 f.write('EXTERNALSRC = "%s"\n' % srctree) 249 250 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 251 if b_is_s: 252 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 253 if initial_rev: 254 f.write('\n# initial_rev: %s\n' % initial_rev) 255 256 if args.binary: 257 f.write('do_install:append() {\n') 258 f.write(' rm -rf ${D}/.git\n') 259 f.write(' rm -f ${D}/singletask.lock\n') 260 f.write('}\n') 261 262 if bb.data.inherits_class('npm', rd): 263 f.write('python do_configure:append() {\n') 264 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 265 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 266 f.write(' bb.utils.remove(lockfile)\n') 267 f.write('}\n') 268 269 # Check if the new layer provides recipes whose priorities have been 270 # overriden by PREFERRED_PROVIDER. 271 recipe_name = rd.getVar('PN') 272 provides = rd.getVar('PROVIDES') 273 # Search every item defined in PROVIDES 274 for recipe_provided in provides.split(): 275 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 276 current_pprovider = rd.getVar(preferred_provider) 277 if current_pprovider and current_pprovider != recipe_name: 278 if args.fixed_setup: 279 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 280 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 281 with open(layerconf_file, 'a') as f: 282 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 283 else: 284 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 285 break 286 287 _add_md5(config, recipename, appendfile) 288 289 check_prerelease_version(rd.getVar('PV'), 'devtool add') 290 291 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 292 293 finally: 294 tinfoil.shutdown() 295 296 return 0 297 298 299def _check_compatible_recipe(pn, d): 300 """Check if the recipe is supported by devtool""" 301 if pn == 'perf': 302 raise DevtoolError("The perf recipe does not actually check out " 303 "source and thus cannot be supported by this tool", 304 4) 305 306 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 307 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 308 309 if bb.data.inherits_class('image', d): 310 raise DevtoolError("The %s recipe is an image, and therefore is not " 311 "supported by this tool" % pn, 4) 312 313 if bb.data.inherits_class('populate_sdk', d): 314 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 315 "supported by this tool" % pn, 4) 316 317 if bb.data.inherits_class('packagegroup', d): 318 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 319 "not supported by this tool" % pn, 4) 320 321 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 322 # Not an incompatibility error per se, so we don't pass the error code 323 raise DevtoolError("externalsrc is currently enabled for the %s " 324 "recipe. This prevents the normal do_patch task " 325 "from working. You will need to disable this " 326 "first." % pn) 327 328def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 329 """Common function for copying a file to the dry run output directory""" 330 relpath = os.path.relpath(dst, base_outdir) 331 if relpath.startswith('..'): 332 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 333 dst = os.path.join(dry_run_outdir, relpath) 334 dst_d = os.path.dirname(dst) 335 if dst_d: 336 bb.utils.mkdirhier(dst_d) 337 # Don't overwrite existing files, otherwise in the case of an upgrade 338 # the dry-run written out recipe will be overwritten with an unmodified 339 # version 340 if not os.path.exists(dst): 341 shutil.copy(src, dst) 342 343def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 344 """Move a file. Creates all the directory components of destination path.""" 345 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 346 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 347 if dry_run_outdir: 348 # We want to copy here, not move 349 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 350 else: 351 dst_d = os.path.dirname(dst) 352 if dst_d: 353 bb.utils.mkdirhier(dst_d) 354 shutil.move(src, dst) 355 356def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 357 """Copy a file. Creates all the directory components of destination path.""" 358 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 359 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 360 if dry_run_outdir: 361 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 362 else: 363 dst_d = os.path.dirname(dst) 364 if dst_d: 365 bb.utils.mkdirhier(dst_d) 366 shutil.copy(src, dst) 367 368def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 369 """List contents of a git treeish""" 370 import bb 371 cmd = ['git', 'ls-tree', '-z', treeish] 372 if recursive: 373 cmd.append('-r') 374 out, _ = bb.process.run(cmd, cwd=repodir) 375 ret = {} 376 if out: 377 for line in out.split('\0'): 378 if line: 379 split = line.split(None, 4) 380 ret[split[3]] = split[0:3] 381 return ret 382 383def _git_exclude_path(srctree, path): 384 """Return pathspec (list of paths) that excludes certain path""" 385 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 386 # we don't catch files that are deleted, for example. A more reliable way 387 # to implement this would be to use "negative pathspecs" which were 388 # introduced in Git v1.9.0. Revisit this when/if the required Git version 389 # becomes greater than that. 390 path = os.path.normpath(path) 391 recurse = True if len(path.split(os.path.sep)) > 1 else False 392 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 393 if path in git_files: 394 git_files.remove(path) 395 return git_files 396 else: 397 return ['.'] 398 399def _ls_tree(directory): 400 """Recursive listing of files in a directory""" 401 ret = [] 402 for root, dirs, files in os.walk(directory): 403 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 404 fname in files]) 405 return ret 406 407 408def extract(args, config, basepath, workspace): 409 """Entry point for the devtool 'extract' subcommand""" 410 import bb 411 412 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 413 if not tinfoil: 414 # Error already shown 415 return 1 416 try: 417 rd = parse_recipe(config, tinfoil, args.recipename, True) 418 if not rd: 419 return 1 420 421 srctree = os.path.abspath(args.srctree) 422 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 423 logger.info('Source tree extracted to %s' % srctree) 424 425 if initial_rev: 426 return 0 427 else: 428 return 1 429 finally: 430 tinfoil.shutdown() 431 432def sync(args, config, basepath, workspace): 433 """Entry point for the devtool 'sync' subcommand""" 434 import bb 435 436 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 437 if not tinfoil: 438 # Error already shown 439 return 1 440 try: 441 rd = parse_recipe(config, tinfoil, args.recipename, True) 442 if not rd: 443 return 1 444 445 srctree = os.path.abspath(args.srctree) 446 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 447 logger.info('Source tree %s synchronized' % srctree) 448 449 if initial_rev: 450 return 0 451 else: 452 return 1 453 finally: 454 tinfoil.shutdown() 455 456def symlink_oelocal_files_srctree(rd,srctree): 457 import oe.patch 458 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 459 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 460 # (otherwise the recipe won't build as expected) 461 local_files_dir = os.path.join(srctree, 'oe-local-files') 462 addfiles = [] 463 for root, _, files in os.walk(local_files_dir): 464 relpth = os.path.relpath(root, local_files_dir) 465 if relpth != '.': 466 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 467 for fn in files: 468 if fn == '.gitignore': 469 continue 470 destpth = os.path.join(srctree, relpth, fn) 471 if os.path.exists(destpth): 472 os.unlink(destpth) 473 if relpth != '.': 474 back_relpth = os.path.relpath(local_files_dir, root) 475 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) 476 else: 477 os.symlink('oe-local-files/%s' % fn, destpth) 478 addfiles.append(os.path.join(relpth, fn)) 479 if addfiles: 480 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 481 useroptions = [] 482 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 483 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 484 485 486def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 487 """Extract sources of a recipe""" 488 import oe.recipeutils 489 import oe.patch 490 import oe.path 491 492 pn = d.getVar('PN') 493 494 _check_compatible_recipe(pn, d) 495 496 if sync: 497 if not os.path.exists(srctree): 498 raise DevtoolError("output path %s does not exist" % srctree) 499 else: 500 if os.path.exists(srctree): 501 if not os.path.isdir(srctree): 502 raise DevtoolError("output path %s exists and is not a directory" % 503 srctree) 504 elif os.listdir(srctree): 505 raise DevtoolError("output path %s already exists and is " 506 "non-empty" % srctree) 507 508 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 509 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 510 "extract source" % pn, 4) 511 512 if not sync: 513 # Prepare for shutil.move later on 514 bb.utils.mkdirhier(srctree) 515 os.rmdir(srctree) 516 517 extra_overrides = [] 518 if not no_overrides: 519 history = d.varhistory.variable('SRC_URI') 520 for event in history: 521 if not 'flag' in event: 522 if event['op'].startswith((':append[', ':prepend[')): 523 override = event['op'].split('[')[1].split(']')[0] 524 if not override.startswith('pn-'): 525 extra_overrides.append(override) 526 # We want to remove duplicate overrides. If a recipe had multiple 527 # SRC_URI_override += values it would cause mulitple instances of 528 # overrides. This doesn't play nicely with things like creating a 529 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 530 extra_overrides = list(set(extra_overrides)) 531 if extra_overrides: 532 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 533 534 initial_rev = None 535 536 recipefile = d.getVar('FILE') 537 appendfile = recipe_to_append(recipefile, config) 538 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 539 540 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 541 # directory so that: 542 # (a) we pick up all files that get unpacked to the WORKDIR, and 543 # (b) we don't disturb the existing build 544 # However, with recipe-specific sysroots the sysroots for the recipe 545 # will be prepared under WORKDIR, and if we used the system temporary 546 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 547 # our attempts to hardlink files into the recipe-specific sysroots 548 # will fail on systems where /tmp is a different filesystem, and it 549 # would have to fall back to copying the files which is a waste of 550 # time. Put the temp directory under the WORKDIR to prevent that from 551 # being a problem. 552 tempbasedir = d.getVar('WORKDIR') 553 bb.utils.mkdirhier(tempbasedir) 554 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 555 try: 556 tinfoil.logger.setLevel(logging.WARNING) 557 558 # FIXME this results in a cache reload under control of tinfoil, which is fine 559 # except we don't get the knotty progress bar 560 561 if os.path.exists(appendfile): 562 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 563 shutil.copyfile(appendfile, appendbackup) 564 else: 565 appendbackup = None 566 bb.utils.mkdirhier(os.path.dirname(appendfile)) 567 logger.debug('writing append file %s' % appendfile) 568 with open(appendfile, 'a') as f: 569 f.write('###--- _extract_source\n') 570 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 571 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 572 if not is_kernel_yocto: 573 f.write('PATCHTOOL = "git"\n') 574 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 575 if extra_overrides: 576 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 577 f.write('inherit devtool-source\n') 578 f.write('###--- _extract_source\n') 579 580 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 581 582 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 583 bb.utils.mkdirhier(sstate_manifests) 584 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 585 with open(preservestampfile, 'w') as f: 586 f.write(d.getVar('STAMP')) 587 try: 588 if is_kernel_yocto: 589 # We need to generate the kernel config 590 task = 'do_configure' 591 else: 592 task = 'do_patch' 593 594 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 595 logger.info('The %s recipe has %s disabled. Running only ' 596 'do_configure task dependencies' % (pn, task)) 597 598 if 'depends' in d.getVarFlags('do_configure', False): 599 pn = d.getVarFlags('do_configure', False)['depends'] 600 pn = pn.replace('${PV}', d.getVar('PV')) 601 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 602 task = None 603 604 # Run the fetch + unpack tasks 605 res = tinfoil.build_targets(pn, 606 task, 607 handle_events=True) 608 finally: 609 if os.path.exists(preservestampfile): 610 os.remove(preservestampfile) 611 612 if not res: 613 raise DevtoolError('Extracting source for %s failed' % pn) 614 615 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 616 workshareddir = d.getVar('S') 617 if os.path.islink(srctree): 618 os.unlink(srctree) 619 620 os.symlink(workshareddir, srctree) 621 622 # The initial_rev file is created in devtool_post_unpack function that will not be executed if 623 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 624 return True, True 625 626 try: 627 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 628 initial_rev = f.read() 629 630 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 631 srcsubdir = f.read() 632 except FileNotFoundError as e: 633 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 634 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 635 636 # Check if work-shared is empty, if yes 637 # find source and copy to work-shared 638 if is_kernel_yocto: 639 workshareddir = d.getVar('STAGING_KERNEL_DIR') 640 staging_kerVer = get_staging_kver(workshareddir) 641 kernelVersion = d.getVar('LINUX_VERSION') 642 643 # handle dangling symbolic link in work-shared: 644 if os.path.islink(workshareddir): 645 os.unlink(workshareddir) 646 647 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 648 shutil.rmtree(workshareddir) 649 oe.path.copyhardlinktree(srcsubdir,workshareddir) 650 elif not os.path.exists(workshareddir): 651 oe.path.copyhardlinktree(srcsubdir,workshareddir) 652 653 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 654 srctree_localdir = os.path.join(srctree, 'oe-local-files') 655 656 if sync: 657 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 658 659 # Move oe-local-files directory to srctree 660 # As the oe-local-files is not part of the constructed git tree, 661 # remove them directly during the synchrounizating might surprise 662 # the users. Instead, we move it to oe-local-files.bak and remind 663 # user in the log message. 664 if os.path.exists(srctree_localdir + '.bak'): 665 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 666 667 if os.path.exists(srctree_localdir): 668 logger.info('Backing up current local file directory %s' % srctree_localdir) 669 shutil.move(srctree_localdir, srctree_localdir + '.bak') 670 671 if os.path.exists(tempdir_localdir): 672 logger.info('Syncing local source files to srctree...') 673 shutil.copytree(tempdir_localdir, srctree_localdir) 674 else: 675 # Move oe-local-files directory to srctree 676 if os.path.exists(tempdir_localdir): 677 logger.info('Adding local source files to srctree...') 678 shutil.move(tempdir_localdir, srcsubdir) 679 680 shutil.move(srcsubdir, srctree) 681 symlink_oelocal_files_srctree(d,srctree) 682 683 if is_kernel_yocto: 684 logger.info('Copying kernel config to srctree') 685 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 686 687 finally: 688 if appendbackup: 689 shutil.copyfile(appendbackup, appendfile) 690 elif os.path.exists(appendfile): 691 os.remove(appendfile) 692 if keep_temp: 693 logger.info('Preserving temporary directory %s' % tempdir) 694 else: 695 shutil.rmtree(tempdir) 696 return initial_rev, srcsubdir_rel 697 698def _add_md5(config, recipename, filename): 699 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 700 import bb.utils 701 702 def addfile(fn): 703 md5 = bb.utils.md5_file(fn) 704 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 705 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 706 f.seek(0, os.SEEK_SET) 707 if not md5_str in f.read(): 708 f.write(md5_str) 709 710 if os.path.isdir(filename): 711 for root, _, files in os.walk(filename): 712 for f in files: 713 addfile(os.path.join(root, f)) 714 else: 715 addfile(filename) 716 717def _check_preserve(config, recipename): 718 """Check if a file was manually changed and needs to be saved in 'attic' 719 directory""" 720 import bb.utils 721 origfile = os.path.join(config.workspace_path, '.devtool_md5') 722 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 723 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 724 with open(origfile, 'r') as f: 725 with open(newfile, 'w') as tf: 726 for line in f.readlines(): 727 splitline = line.rstrip().split('|') 728 if splitline[0] == recipename: 729 removefile = os.path.join(config.workspace_path, splitline[1]) 730 try: 731 md5 = bb.utils.md5_file(removefile) 732 except IOError as err: 733 if err.errno == 2: 734 # File no longer exists, skip it 735 continue 736 else: 737 raise 738 if splitline[2] != md5: 739 bb.utils.mkdirhier(preservepath) 740 preservefile = os.path.basename(removefile) 741 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 742 shutil.move(removefile, os.path.join(preservepath, preservefile)) 743 else: 744 os.remove(removefile) 745 else: 746 tf.write(line) 747 bb.utils.rename(newfile, origfile) 748 749def get_staging_kver(srcdir): 750 # Kernel version from work-shared 751 kerver = [] 752 staging_kerVer="" 753 if os.path.exists(srcdir) and os.listdir(srcdir): 754 with open(os.path.join(srcdir,"Makefile")) as f: 755 version = [next(f) for x in range(5)][1:4] 756 for word in version: 757 kerver.append(word.split('= ')[1].split('\n')[0]) 758 staging_kerVer = ".".join(kerver) 759 return staging_kerVer 760 761def get_staging_kbranch(srcdir): 762 staging_kbranch = "" 763 if os.path.exists(srcdir) and os.listdir(srcdir): 764 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) 765 staging_kbranch = "".join(branch.split('\n')[0]) 766 return staging_kbranch 767 768def modify(args, config, basepath, workspace): 769 """Entry point for the devtool 'modify' subcommand""" 770 import bb 771 import oe.recipeutils 772 import oe.patch 773 import oe.path 774 775 if args.recipename in workspace: 776 raise DevtoolError("recipe %s is already in your workspace" % 777 args.recipename) 778 779 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 780 try: 781 rd = parse_recipe(config, tinfoil, args.recipename, True) 782 if not rd: 783 return 1 784 785 pn = rd.getVar('PN') 786 if pn != args.recipename: 787 logger.info('Mapping %s to %s' % (args.recipename, pn)) 788 if pn in workspace: 789 raise DevtoolError("recipe %s is already in your workspace" % 790 pn) 791 792 if args.srctree: 793 srctree = os.path.abspath(args.srctree) 794 else: 795 srctree = get_default_srctree(config, pn) 796 797 if args.no_extract and not os.path.isdir(srctree): 798 raise DevtoolError("--no-extract specified and source path %s does " 799 "not exist or is not a directory" % 800 srctree) 801 802 recipefile = rd.getVar('FILE') 803 appendfile = recipe_to_append(recipefile, config, args.wildcard) 804 if os.path.exists(appendfile): 805 raise DevtoolError("Another variant of recipe %s is already in your " 806 "workspace (only one variant of a recipe can " 807 "currently be worked on at once)" 808 % pn) 809 810 _check_compatible_recipe(pn, rd) 811 812 initial_rev = None 813 commits = [] 814 check_commits = False 815 816 if bb.data.inherits_class('kernel-yocto', rd): 817 # Current set kernel version 818 kernelVersion = rd.getVar('LINUX_VERSION') 819 srcdir = rd.getVar('STAGING_KERNEL_DIR') 820 kbranch = rd.getVar('KBRANCH') 821 822 staging_kerVer = get_staging_kver(srcdir) 823 staging_kbranch = get_staging_kbranch(srcdir) 824 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 825 oe.path.copyhardlinktree(srcdir,srctree) 826 workdir = rd.getVar('WORKDIR') 827 srcsubdir = rd.getVar('S') 828 localfilesdir = os.path.join(srctree,'oe-local-files') 829 # Move local source files into separate subdir 830 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 831 local_files = oe.recipeutils.get_recipe_local_files(rd) 832 833 for key in local_files.copy(): 834 if key.endswith('scc'): 835 sccfile = open(local_files[key], 'r') 836 for l in sccfile: 837 line = l.split() 838 if line and line[0] in ('kconf', 'patch'): 839 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 840 if not cfg in local_files.values(): 841 local_files[line[-1]] = cfg 842 shutil.copy2(cfg, workdir) 843 sccfile.close() 844 845 # Ignore local files with subdir={BP} 846 srcabspath = os.path.abspath(srcsubdir) 847 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 848 if local_files: 849 for fname in local_files: 850 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 851 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 852 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 853 854 symlink_oelocal_files_srctree(rd,srctree) 855 856 task = 'do_configure' 857 res = tinfoil.build_targets(pn, task, handle_events=True) 858 859 # Copy .config to workspace 860 kconfpath = rd.getVar('B') 861 logger.info('Copying kernel config to workspace') 862 shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 863 864 # Set this to true, we still need to get initial_rev 865 # by parsing the git repo 866 args.no_extract = True 867 868 if not args.no_extract: 869 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 870 if not initial_rev: 871 return 1 872 logger.info('Source tree extracted to %s' % srctree) 873 if os.path.exists(os.path.join(srctree, '.git')): 874 # Get list of commits since this revision 875 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) 876 commits = stdout.split() 877 check_commits = True 878 else: 879 if os.path.exists(os.path.join(srctree, '.git')): 880 # Check if it's a tree previously extracted by us. This is done 881 # by ensuring that devtool-base and args.branch (devtool) exist. 882 # The check_commits logic will cause an exception if either one 883 # of these doesn't exist 884 try: 885 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 886 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 887 except bb.process.ExecutionError: 888 stdout = '' 889 if stdout: 890 check_commits = True 891 for line in stdout.splitlines(): 892 if line.startswith('*'): 893 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 894 initial_rev = stdout.rstrip() 895 if not initial_rev: 896 # Otherwise, just grab the head revision 897 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 898 initial_rev = stdout.rstrip() 899 900 branch_patches = {} 901 if check_commits: 902 # Check if there are override branches 903 (stdout, _) = bb.process.run('git branch', cwd=srctree) 904 branches = [] 905 for line in stdout.rstrip().splitlines(): 906 branchname = line[2:].rstrip() 907 if branchname.startswith(override_branch_prefix): 908 branches.append(branchname) 909 if branches: 910 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 911 branches.insert(0, args.branch) 912 seen_patches = [] 913 for branch in branches: 914 branch_patches[branch] = [] 915 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 916 for line in stdout.splitlines(): 917 line = line.strip() 918 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 919 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 920 if not origpatch in seen_patches: 921 seen_patches.append(origpatch) 922 branch_patches[branch].append(origpatch) 923 924 # Need to grab this here in case the source is within a subdirectory 925 srctreebase = srctree 926 927 # Check that recipe isn't using a shared workdir 928 s = os.path.abspath(rd.getVar('S')) 929 workdir = os.path.abspath(rd.getVar('WORKDIR')) 930 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 931 # Handle if S is set to a subdirectory of the source 932 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 933 srctree = os.path.join(srctree, srcsubdir) 934 935 bb.utils.mkdirhier(os.path.dirname(appendfile)) 936 with open(appendfile, 'w') as f: 937 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 938 # Local files can be modified/tracked in separate subdir under srctree 939 # Mostly useful for packages with S != WORKDIR 940 f.write('FILESPATH:prepend := "%s:"\n' % 941 os.path.join(srctreebase, 'oe-local-files')) 942 f.write('# srctreebase: %s\n' % srctreebase) 943 944 f.write('\ninherit externalsrc\n') 945 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 946 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 947 948 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 949 if b_is_s: 950 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 951 952 if bb.data.inherits_class('kernel', rd): 953 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 954 'do_fetch do_unpack do_kernel_configcheck"\n') 955 f.write('\ndo_patch[noexec] = "1"\n') 956 f.write('\ndo_configure:append() {\n' 957 ' cp ${B}/.config ${S}/.config.baseline\n' 958 ' ln -sfT ${B}/.config ${S}/.config.new\n' 959 '}\n') 960 f.write('\ndo_kernel_configme:prepend() {\n' 961 ' if [ -e ${S}/.config ]; then\n' 962 ' mv ${S}/.config ${S}/.config.old\n' 963 ' fi\n' 964 '}\n') 965 if rd.getVarFlag('do_menuconfig','task'): 966 f.write('\ndo_configure:append() {\n' 967 ' if [ ! ${DEVTOOL_DISABLE_MENUCONFIG} ]; then\n' 968 ' cp ${B}/.config ${S}/.config.baseline\n' 969 ' ln -sfT ${B}/.config ${S}/.config.new\n' 970 ' fi\n' 971 '}\n') 972 if initial_rev: 973 f.write('\n# initial_rev: %s\n' % initial_rev) 974 for commit in commits: 975 f.write('# commit: %s\n' % commit) 976 if branch_patches: 977 for branch in branch_patches: 978 if branch == args.branch: 979 continue 980 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 981 982 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 983 984 _add_md5(config, pn, appendfile) 985 986 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 987 988 finally: 989 tinfoil.shutdown() 990 991 return 0 992 993 994def rename(args, config, basepath, workspace): 995 """Entry point for the devtool 'rename' subcommand""" 996 import bb 997 import oe.recipeutils 998 999 check_workspace_recipe(workspace, args.recipename) 1000 1001 if not (args.newname or args.version): 1002 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 1003 1004 recipefile = workspace[args.recipename]['recipefile'] 1005 if not recipefile: 1006 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1007 1008 if args.newname and args.newname != args.recipename: 1009 reason = oe.recipeutils.validate_pn(args.newname) 1010 if reason: 1011 raise DevtoolError(reason) 1012 newname = args.newname 1013 else: 1014 newname = args.recipename 1015 1016 append = workspace[args.recipename]['bbappend'] 1017 appendfn = os.path.splitext(os.path.basename(append))[0] 1018 splitfn = appendfn.split('_') 1019 if len(splitfn) > 1: 1020 origfnver = appendfn.split('_')[1] 1021 else: 1022 origfnver = '' 1023 1024 recipefilemd5 = None 1025 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1026 try: 1027 rd = parse_recipe(config, tinfoil, args.recipename, True) 1028 if not rd: 1029 return 1 1030 1031 bp = rd.getVar('BP') 1032 bpn = rd.getVar('BPN') 1033 if newname != args.recipename: 1034 localdata = rd.createCopy() 1035 localdata.setVar('PN', newname) 1036 newbpn = localdata.getVar('BPN') 1037 else: 1038 newbpn = bpn 1039 s = rd.getVar('S', False) 1040 src_uri = rd.getVar('SRC_URI', False) 1041 pv = rd.getVar('PV') 1042 1043 # Correct variable values that refer to the upstream source - these 1044 # values must stay the same, so if the name/version are changing then 1045 # we need to fix them up 1046 new_s = s 1047 new_src_uri = src_uri 1048 if newbpn != bpn: 1049 # ${PN} here is technically almost always incorrect, but people do use it 1050 new_s = new_s.replace('${BPN}', bpn) 1051 new_s = new_s.replace('${PN}', bpn) 1052 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1053 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1054 new_src_uri = new_src_uri.replace('${PN}', bpn) 1055 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1056 if args.version and origfnver == pv: 1057 new_s = new_s.replace('${PV}', pv) 1058 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1059 new_src_uri = new_src_uri.replace('${PV}', pv) 1060 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1061 patchfields = {} 1062 if new_s != s: 1063 patchfields['S'] = new_s 1064 if new_src_uri != src_uri: 1065 patchfields['SRC_URI'] = new_src_uri 1066 if patchfields: 1067 recipefilemd5 = bb.utils.md5_file(recipefile) 1068 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1069 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1070 finally: 1071 tinfoil.shutdown() 1072 1073 if args.version: 1074 newver = args.version 1075 else: 1076 newver = origfnver 1077 1078 if newver: 1079 newappend = '%s_%s.bbappend' % (newname, newver) 1080 newfile = '%s_%s.bb' % (newname, newver) 1081 else: 1082 newappend = '%s.bbappend' % newname 1083 newfile = '%s.bb' % newname 1084 1085 oldrecipedir = os.path.dirname(recipefile) 1086 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1087 if oldrecipedir != newrecipedir: 1088 bb.utils.mkdirhier(newrecipedir) 1089 1090 newappend = os.path.join(os.path.dirname(append), newappend) 1091 newfile = os.path.join(newrecipedir, newfile) 1092 1093 # Rename bbappend 1094 logger.info('Renaming %s to %s' % (append, newappend)) 1095 bb.utils.rename(append, newappend) 1096 # Rename recipe file 1097 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1098 bb.utils.rename(recipefile, newfile) 1099 1100 # Rename source tree if it's the default path 1101 appendmd5 = None 1102 if not args.no_srctree: 1103 srctree = workspace[args.recipename]['srctree'] 1104 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1105 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1106 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1107 shutil.move(srctree, newsrctree) 1108 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1109 appendmd5 = bb.utils.md5_file(newappend) 1110 appendlines = [] 1111 with open(newappend, 'r') as f: 1112 for line in f: 1113 appendlines.append(line) 1114 with open(newappend, 'w') as f: 1115 for line in appendlines: 1116 if srctree in line: 1117 line = line.replace(srctree, newsrctree) 1118 f.write(line) 1119 newappendmd5 = bb.utils.md5_file(newappend) 1120 1121 bpndir = None 1122 newbpndir = None 1123 if newbpn != bpn: 1124 bpndir = os.path.join(oldrecipedir, bpn) 1125 if os.path.exists(bpndir): 1126 newbpndir = os.path.join(newrecipedir, newbpn) 1127 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1128 shutil.move(bpndir, newbpndir) 1129 1130 bpdir = None 1131 newbpdir = None 1132 if newver != origfnver or newbpn != bpn: 1133 bpdir = os.path.join(oldrecipedir, bp) 1134 if os.path.exists(bpdir): 1135 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1136 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1137 shutil.move(bpdir, newbpdir) 1138 1139 if oldrecipedir != newrecipedir: 1140 # Move any stray files and delete the old recipe directory 1141 for entry in os.listdir(oldrecipedir): 1142 oldpath = os.path.join(oldrecipedir, entry) 1143 newpath = os.path.join(newrecipedir, entry) 1144 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1145 shutil.move(oldpath, newpath) 1146 os.rmdir(oldrecipedir) 1147 1148 # Now take care of entries in .devtool_md5 1149 md5entries = [] 1150 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1151 for line in f: 1152 md5entries.append(line) 1153 1154 if bpndir and newbpndir: 1155 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1156 else: 1157 relbpndir = None 1158 if bpdir and newbpdir: 1159 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1160 else: 1161 relbpdir = None 1162 1163 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1164 for entry in md5entries: 1165 splitentry = entry.rstrip().split('|') 1166 if len(splitentry) > 2: 1167 if splitentry[0] == args.recipename: 1168 splitentry[0] = newname 1169 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1170 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1171 if appendmd5 and splitentry[2] == appendmd5: 1172 splitentry[2] = newappendmd5 1173 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1174 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1175 if recipefilemd5 and splitentry[2] == recipefilemd5: 1176 splitentry[2] = newrecipefilemd5 1177 elif relbpndir and splitentry[1].startswith(relbpndir): 1178 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1179 elif relbpdir and splitentry[1].startswith(relbpdir): 1180 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1181 entry = '|'.join(splitentry) + '\n' 1182 f.write(entry) 1183 return 0 1184 1185 1186def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1187 """Get initial and update rev of a recipe. These are the start point of the 1188 whole patchset and start point for the patches to be re-generated/updated. 1189 """ 1190 import bb 1191 1192 # Get current branch 1193 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1194 cwd=srctree) 1195 branchname = stdout.rstrip() 1196 1197 # Parse initial rev from recipe if not specified 1198 commits = [] 1199 patches = [] 1200 with open(recipe_path, 'r') as f: 1201 for line in f: 1202 if line.startswith('# initial_rev:'): 1203 if not initial_rev: 1204 initial_rev = line.split(':')[-1].strip() 1205 elif line.startswith('# commit:') and not force_patch_refresh: 1206 commits.append(line.split(':')[-1].strip()) 1207 elif line.startswith('# patches_%s:' % branchname): 1208 patches = line.split(':')[-1].strip().split(',') 1209 1210 update_rev = initial_rev 1211 changed_revs = None 1212 if initial_rev: 1213 # Find first actually changed revision 1214 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1215 initial_rev, cwd=srctree) 1216 newcommits = stdout.split() 1217 for i in range(min(len(commits), len(newcommits))): 1218 if newcommits[i] == commits[i]: 1219 update_rev = commits[i] 1220 1221 try: 1222 stdout, _ = bb.process.run('git cherry devtool-patched', 1223 cwd=srctree) 1224 except bb.process.ExecutionError as err: 1225 stdout = None 1226 1227 if stdout is not None and not force_patch_refresh: 1228 changed_revs = [] 1229 for line in stdout.splitlines(): 1230 if line.startswith('+ '): 1231 rev = line.split()[1] 1232 if rev in newcommits: 1233 changed_revs.append(rev) 1234 1235 return initial_rev, update_rev, changed_revs, patches 1236 1237def _remove_file_entries(srcuri, filelist): 1238 """Remove file:// entries from SRC_URI""" 1239 remaining = filelist[:] 1240 entries = [] 1241 for fname in filelist: 1242 basename = os.path.basename(fname) 1243 for i in range(len(srcuri)): 1244 if (srcuri[i].startswith('file://') and 1245 os.path.basename(srcuri[i].split(';')[0]) == basename): 1246 entries.append(srcuri[i]) 1247 remaining.remove(fname) 1248 srcuri.pop(i) 1249 break 1250 return entries, remaining 1251 1252def _replace_srcuri_entry(srcuri, filename, newentry): 1253 """Replace entry corresponding to specified file with a new entry""" 1254 basename = os.path.basename(filename) 1255 for i in range(len(srcuri)): 1256 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1257 srcuri.pop(i) 1258 srcuri.insert(i, newentry) 1259 break 1260 1261def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1262 """Unlink existing patch files""" 1263 1264 dry_run_suffix = ' (dry-run)' if dry_run else '' 1265 1266 for path in files: 1267 if append: 1268 if not destpath: 1269 raise Exception('destpath should be set here') 1270 path = os.path.join(destpath, os.path.basename(path)) 1271 1272 if os.path.exists(path): 1273 if not no_report_remove: 1274 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1275 if not dry_run: 1276 # FIXME "git rm" here would be nice if the file in question is 1277 # tracked 1278 # FIXME there's a chance that this file is referred to by 1279 # another recipe, in which case deleting wouldn't be the 1280 # right thing to do 1281 os.remove(path) 1282 # Remove directory if empty 1283 try: 1284 os.rmdir(os.path.dirname(path)) 1285 except OSError as ose: 1286 if ose.errno != errno.ENOTEMPTY: 1287 raise 1288 1289 1290def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1291 """Export patches from srctree to given location. 1292 Returns three-tuple of dicts: 1293 1. updated - patches that already exist in SRCURI 1294 2. added - new patches that don't exist in SRCURI 1295 3 removed - patches that exist in SRCURI but not in exported patches 1296 In each dict the key is the 'basepath' of the URI and value is the 1297 absolute path to the existing file in recipe space (if any). 1298 """ 1299 import oe.recipeutils 1300 from oe.patch import GitApplyTree 1301 updated = OrderedDict() 1302 added = OrderedDict() 1303 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1304 1305 existing_patches = dict((os.path.basename(path), path) for path in 1306 oe.recipeutils.get_recipe_patches(rd)) 1307 logger.debug('Existing patches: %s' % existing_patches) 1308 1309 # Generate patches from Git, exclude local files directory 1310 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1311 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) 1312 1313 new_patches = sorted(os.listdir(destdir)) 1314 for new_patch in new_patches: 1315 # Strip numbering from patch names. If it's a git sequence named patch, 1316 # the numbers might not match up since we are starting from a different 1317 # revision This does assume that people are using unique shortlog 1318 # values, but they ought to be anyway... 1319 new_basename = seqpatch_re.match(new_patch).group(2) 1320 match_name = None 1321 for old_patch in existing_patches: 1322 old_basename = seqpatch_re.match(old_patch).group(2) 1323 old_basename_splitext = os.path.splitext(old_basename) 1324 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1325 old_patch_noext = os.path.splitext(old_patch)[0] 1326 match_name = old_patch_noext 1327 break 1328 elif new_basename == old_basename: 1329 match_name = old_patch 1330 break 1331 if match_name: 1332 # Rename patch files 1333 if new_patch != match_name: 1334 bb.utils.rename(os.path.join(destdir, new_patch), 1335 os.path.join(destdir, match_name)) 1336 # Need to pop it off the list now before checking changed_revs 1337 oldpath = existing_patches.pop(old_patch) 1338 if changed_revs is not None: 1339 # Avoid updating patches that have not actually changed 1340 with open(os.path.join(destdir, match_name), 'r') as f: 1341 firstlineitems = f.readline().split() 1342 # Looking for "From <hash>" line 1343 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1344 if not firstlineitems[1] in changed_revs: 1345 continue 1346 # Recompress if necessary 1347 if oldpath.endswith(('.gz', '.Z')): 1348 bb.process.run(['gzip', match_name], cwd=destdir) 1349 if oldpath.endswith('.gz'): 1350 match_name += '.gz' 1351 else: 1352 match_name += '.Z' 1353 elif oldpath.endswith('.bz2'): 1354 bb.process.run(['bzip2', match_name], cwd=destdir) 1355 match_name += '.bz2' 1356 updated[match_name] = oldpath 1357 else: 1358 added[new_patch] = None 1359 return (updated, added, existing_patches) 1360 1361 1362def _create_kconfig_diff(srctree, rd, outfile): 1363 """Create a kconfig fragment""" 1364 # Only update config fragment if both config files exist 1365 orig_config = os.path.join(srctree, '.config.baseline') 1366 new_config = os.path.join(srctree, '.config.new') 1367 if os.path.exists(orig_config) and os.path.exists(new_config): 1368 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1369 '--unchanged-line-format=', orig_config, new_config] 1370 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1371 stderr=subprocess.PIPE) 1372 stdout, stderr = pipe.communicate() 1373 if pipe.returncode == 1: 1374 logger.info("Updating config fragment %s" % outfile) 1375 with open(outfile, 'wb') as fobj: 1376 fobj.write(stdout) 1377 elif pipe.returncode == 0: 1378 logger.info("Would remove config fragment %s" % outfile) 1379 if os.path.exists(outfile): 1380 # Remove fragment file in case of empty diff 1381 logger.info("Removing config fragment %s" % outfile) 1382 os.unlink(outfile) 1383 else: 1384 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1385 return True 1386 return False 1387 1388 1389def _export_local_files(srctree, rd, destdir, srctreebase): 1390 """Copy local files from srctree to given location. 1391 Returns three-tuple of dicts: 1392 1. updated - files that already exist in SRCURI 1393 2. added - new files files that don't exist in SRCURI 1394 3 removed - files that exist in SRCURI but not in exported files 1395 In each dict the key is the 'basepath' of the URI and value is the 1396 absolute path to the existing file in recipe space (if any). 1397 """ 1398 import oe.recipeutils 1399 1400 # Find out local files (SRC_URI files that exist in the "recipe space"). 1401 # Local files that reside in srctree are not included in patch generation. 1402 # Instead they are directly copied over the original source files (in 1403 # recipe space). 1404 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1405 new_set = None 1406 updated = OrderedDict() 1407 added = OrderedDict() 1408 removed = OrderedDict() 1409 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1410 git_files = _git_ls_tree(srctree) 1411 if 'oe-local-files' in git_files: 1412 # If tracked by Git, take the files from srctree HEAD. First get 1413 # the tree object of the directory 1414 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1415 tree = git_files['oe-local-files'][2] 1416 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1417 env=dict(os.environ, GIT_WORK_TREE=destdir, 1418 GIT_INDEX_FILE=tmp_index)) 1419 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1420 elif os.path.isdir(local_files_dir): 1421 # If not tracked by Git, just copy from working copy 1422 new_set = _ls_tree(local_files_dir) 1423 bb.process.run(['cp', '-ax', 1424 os.path.join(local_files_dir, '.'), destdir]) 1425 else: 1426 new_set = [] 1427 1428 # Special handling for kernel config 1429 if bb.data.inherits_class('kernel-yocto', rd): 1430 fragment_fn = 'devtool-fragment.cfg' 1431 fragment_path = os.path.join(destdir, fragment_fn) 1432 if _create_kconfig_diff(srctree, rd, fragment_path): 1433 if os.path.exists(fragment_path): 1434 if fragment_fn not in new_set: 1435 new_set.append(fragment_fn) 1436 # Copy fragment to local-files 1437 if os.path.isdir(local_files_dir): 1438 shutil.copy2(fragment_path, local_files_dir) 1439 else: 1440 if fragment_fn in new_set: 1441 new_set.remove(fragment_fn) 1442 # Remove fragment from local-files 1443 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1444 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1445 1446 # Special handling for cml1, ccmake, etc bbclasses that generated 1447 # configuration fragment files that are consumed as source files 1448 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1449 if bb.data.inherits_class(frag_class, rd): 1450 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1451 if os.path.exists(srcpath): 1452 if frag_name not in new_set: 1453 new_set.append(frag_name) 1454 # copy fragment into destdir 1455 shutil.copy2(srcpath, destdir) 1456 # copy fragment into local files if exists 1457 if os.path.isdir(local_files_dir): 1458 shutil.copy2(srcpath, local_files_dir) 1459 1460 if new_set is not None: 1461 for fname in new_set: 1462 if fname in existing_files: 1463 origpath = existing_files.pop(fname) 1464 workpath = os.path.join(local_files_dir, fname) 1465 if not filecmp.cmp(origpath, workpath): 1466 updated[fname] = origpath 1467 elif fname != '.gitignore': 1468 added[fname] = None 1469 1470 workdir = rd.getVar('WORKDIR') 1471 s = rd.getVar('S') 1472 if not s.endswith(os.sep): 1473 s += os.sep 1474 1475 if workdir != s: 1476 # Handle files where subdir= was specified 1477 for fname in list(existing_files.keys()): 1478 # FIXME handle both subdir starting with BP and not? 1479 fworkpath = os.path.join(workdir, fname) 1480 if fworkpath.startswith(s): 1481 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1482 if os.path.exists(fpath): 1483 origpath = existing_files.pop(fname) 1484 if not filecmp.cmp(origpath, fpath): 1485 updated[fpath] = origpath 1486 1487 removed = existing_files 1488 return (updated, added, removed) 1489 1490 1491def _determine_files_dir(rd): 1492 """Determine the appropriate files directory for a recipe""" 1493 recipedir = rd.getVar('FILE_DIRNAME') 1494 for entry in rd.getVar('FILESPATH').split(':'): 1495 relpth = os.path.relpath(entry, recipedir) 1496 if not os.sep in relpth: 1497 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1498 if os.path.isdir(entry): 1499 return entry 1500 return os.path.join(recipedir, rd.getVar('BPN')) 1501 1502 1503def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1504 """Implement the 'srcrev' mode of update-recipe""" 1505 import bb 1506 import oe.recipeutils 1507 1508 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1509 1510 recipefile = rd.getVar('FILE') 1511 recipedir = os.path.basename(recipefile) 1512 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1513 1514 # Get HEAD revision 1515 try: 1516 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1517 except bb.process.ExecutionError as err: 1518 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1519 (srctree, err)) 1520 srcrev = stdout.strip() 1521 if len(srcrev) != 40: 1522 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1523 1524 destpath = None 1525 remove_files = [] 1526 patchfields = {} 1527 patchfields['SRCREV'] = srcrev 1528 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1529 srcuri = orig_src_uri.split() 1530 tempdir = tempfile.mkdtemp(prefix='devtool') 1531 update_srcuri = False 1532 appendfile = None 1533 try: 1534 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1535 srctreebase = workspace[recipename]['srctreebase'] 1536 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1537 if not no_remove: 1538 # Find list of existing patches in recipe file 1539 patches_dir = tempfile.mkdtemp(dir=tempdir) 1540 old_srcrev = rd.getVar('SRCREV') or '' 1541 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1542 patches_dir) 1543 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1544 1545 # Remove deleted local files and "overlapping" patches 1546 remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) 1547 if remove_files: 1548 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1549 update_srcuri = True 1550 1551 if appendlayerdir: 1552 files = dict((os.path.join(local_files_dir, key), val) for 1553 key, val in list(upd_f.items()) + list(new_f.items())) 1554 removevalues = {} 1555 if update_srcuri: 1556 removevalues = {'SRC_URI': removedentries} 1557 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1558 if dry_run_outdir: 1559 logger.info('Creating bbappend (dry-run)') 1560 else: 1561 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1562 rd, appendlayerdir, files, wildcardver=wildcard_version, 1563 extralines=patchfields, removevalues=removevalues, 1564 redirect_output=dry_run_outdir) 1565 else: 1566 files_dir = _determine_files_dir(rd) 1567 for basepath, path in upd_f.items(): 1568 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1569 if os.path.isabs(basepath): 1570 # Original file (probably with subdir pointing inside source tree) 1571 # so we do not want to move it, just copy 1572 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1573 else: 1574 _move_file(os.path.join(local_files_dir, basepath), path, 1575 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1576 update_srcuri= True 1577 for basepath, path in new_f.items(): 1578 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1579 _move_file(os.path.join(local_files_dir, basepath), 1580 os.path.join(files_dir, basepath), 1581 dry_run_outdir=dry_run_outdir, 1582 base_outdir=recipedir) 1583 srcuri.append('file://%s' % basepath) 1584 update_srcuri = True 1585 if update_srcuri: 1586 patchfields['SRC_URI'] = ' '.join(srcuri) 1587 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1588 finally: 1589 shutil.rmtree(tempdir) 1590 if not 'git://' in orig_src_uri: 1591 logger.info('You will need to update SRC_URI within the recipe to ' 1592 'point to a git repository where you have pushed your ' 1593 'changes') 1594 1595 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1596 return True, appendfile, remove_files 1597 1598def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1599 """Implement the 'patch' mode of update-recipe""" 1600 import bb 1601 import oe.recipeutils 1602 1603 recipefile = rd.getVar('FILE') 1604 recipedir = os.path.dirname(recipefile) 1605 append = workspace[recipename]['bbappend'] 1606 if not os.path.exists(append): 1607 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1608 recipename) 1609 srctreebase = workspace[recipename]['srctreebase'] 1610 relpatchdir = os.path.relpath(srctreebase, srctree) 1611 if relpatchdir == '.': 1612 patchdir_params = {} 1613 else: 1614 patchdir_params = {'patchdir': relpatchdir} 1615 1616 def srcuri_entry(fname): 1617 if patchdir_params: 1618 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1619 else: 1620 paramstr = '' 1621 return 'file://%s%s' % (basepath, paramstr) 1622 1623 initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1624 if not initial_rev: 1625 raise DevtoolError('Unable to find initial revision - please specify ' 1626 'it with --initial-rev') 1627 1628 appendfile = None 1629 dl_dir = rd.getVar('DL_DIR') 1630 if not dl_dir.endswith('/'): 1631 dl_dir += '/' 1632 1633 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1634 1635 tempdir = tempfile.mkdtemp(prefix='devtool') 1636 try: 1637 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1638 if filter_patches: 1639 upd_f = {} 1640 new_f = {} 1641 del_f = {} 1642 else: 1643 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1644 1645 remove_files = [] 1646 if not no_remove: 1647 # Get all patches from source tree and check if any should be removed 1648 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1649 _, _, del_p = _export_patches(srctree, rd, initial_rev, 1650 all_patches_dir) 1651 # Remove deleted local files and patches 1652 remove_files = list(del_f.values()) + list(del_p.values()) 1653 1654 # Get updated patches from source tree 1655 patches_dir = tempfile.mkdtemp(dir=tempdir) 1656 upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, 1657 patches_dir, changed_revs) 1658 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1659 if filter_patches: 1660 new_p = OrderedDict() 1661 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1662 remove_files = [f for f in remove_files if f in filter_patches] 1663 updatefiles = False 1664 updaterecipe = False 1665 destpath = None 1666 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1667 if appendlayerdir: 1668 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1669 key, val in list(upd_f.items()) + list(new_f.items())) 1670 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1671 key, val in list(upd_p.items()) + list(new_p.items()))) 1672 if files or remove_files: 1673 removevalues = None 1674 if remove_files: 1675 removedentries, remaining = _remove_file_entries( 1676 srcuri, remove_files) 1677 if removedentries or remaining: 1678 remaining = [srcuri_entry(os.path.basename(item)) for 1679 item in remaining] 1680 removevalues = {'SRC_URI': removedentries + remaining} 1681 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1682 rd, appendlayerdir, files, 1683 wildcardver=wildcard_version, 1684 removevalues=removevalues, 1685 redirect_output=dry_run_outdir, 1686 params=[patchdir_params] * len(files)) 1687 else: 1688 logger.info('No patches or local source files needed updating') 1689 else: 1690 # Update existing files 1691 files_dir = _determine_files_dir(rd) 1692 for basepath, path in upd_f.items(): 1693 logger.info('Updating file %s' % basepath) 1694 if os.path.isabs(basepath): 1695 # Original file (probably with subdir pointing inside source tree) 1696 # so we do not want to move it, just copy 1697 _copy_file(basepath, path, 1698 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1699 else: 1700 _move_file(os.path.join(local_files_dir, basepath), path, 1701 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1702 updatefiles = True 1703 for basepath, path in upd_p.items(): 1704 patchfn = os.path.join(patches_dir, basepath) 1705 if os.path.dirname(path) + '/' == dl_dir: 1706 # This is a a downloaded patch file - we now need to 1707 # replace the entry in SRC_URI with our local version 1708 logger.info('Replacing remote patch %s with updated local version' % basepath) 1709 path = os.path.join(files_dir, basepath) 1710 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath)) 1711 updaterecipe = True 1712 else: 1713 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1714 _move_file(patchfn, path, 1715 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1716 updatefiles = True 1717 # Add any new files 1718 for basepath, path in new_f.items(): 1719 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1720 _move_file(os.path.join(local_files_dir, basepath), 1721 os.path.join(files_dir, basepath), 1722 dry_run_outdir=dry_run_outdir, 1723 base_outdir=recipedir) 1724 srcuri.append(srcuri_entry(basepath)) 1725 updaterecipe = True 1726 for basepath, path in new_p.items(): 1727 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1728 _move_file(os.path.join(patches_dir, basepath), 1729 os.path.join(files_dir, basepath), 1730 dry_run_outdir=dry_run_outdir, 1731 base_outdir=recipedir) 1732 srcuri.append(srcuri_entry(basepath)) 1733 updaterecipe = True 1734 # Update recipe, if needed 1735 if _remove_file_entries(srcuri, remove_files)[0]: 1736 updaterecipe = True 1737 if updaterecipe: 1738 if not dry_run_outdir: 1739 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1740 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1741 {'SRC_URI': ' '.join(srcuri)}, 1742 redirect_output=dry_run_outdir) 1743 elif not updatefiles: 1744 # Neither patches nor recipe were updated 1745 logger.info('No patches or files need updating') 1746 return False, None, [] 1747 finally: 1748 shutil.rmtree(tempdir) 1749 1750 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1751 return True, appendfile, remove_files 1752 1753def _guess_recipe_update_mode(srctree, rdata): 1754 """Guess the recipe update mode to use""" 1755 src_uri = (rdata.getVar('SRC_URI') or '').split() 1756 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1757 if not git_uris: 1758 return 'patch' 1759 # Just use the first URI for now 1760 uri = git_uris[0] 1761 # Check remote branch 1762 params = bb.fetch.decodeurl(uri)[5] 1763 upstr_branch = params['branch'] if 'branch' in params else 'master' 1764 # Check if current branch HEAD is found in upstream branch 1765 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1766 head_rev = stdout.rstrip() 1767 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1768 cwd=srctree) 1769 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1770 if 'origin/' + upstr_branch in remote_brs: 1771 return 'srcrev' 1772 1773 return 'patch' 1774 1775def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1776 srctree = workspace[recipename]['srctree'] 1777 if mode == 'auto': 1778 mode = _guess_recipe_update_mode(srctree, rd) 1779 1780 override_branches = [] 1781 mainbranch = None 1782 startbranch = None 1783 if not no_overrides: 1784 stdout, _ = bb.process.run('git branch', cwd=srctree) 1785 other_branches = [] 1786 for line in stdout.splitlines(): 1787 branchname = line[2:] 1788 if line.startswith('* '): 1789 startbranch = branchname 1790 if branchname.startswith(override_branch_prefix): 1791 override_branches.append(branchname) 1792 else: 1793 other_branches.append(branchname) 1794 1795 if override_branches: 1796 logger.debug('_update_recipe: override branches: %s' % override_branches) 1797 logger.debug('_update_recipe: other branches: %s' % other_branches) 1798 if startbranch.startswith(override_branch_prefix): 1799 if len(other_branches) == 1: 1800 mainbranch = other_branches[1] 1801 else: 1802 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1803 else: 1804 mainbranch = startbranch 1805 1806 checkedout = None 1807 anyupdated = False 1808 appendfile = None 1809 allremoved = [] 1810 if override_branches: 1811 logger.info('Handling main branch (%s)...' % mainbranch) 1812 if startbranch != mainbranch: 1813 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1814 checkedout = mainbranch 1815 try: 1816 branchlist = [mainbranch] + override_branches 1817 for branch in branchlist: 1818 crd = bb.data.createCopy(rd) 1819 if branch != mainbranch: 1820 logger.info('Handling branch %s...' % branch) 1821 override = branch[len(override_branch_prefix):] 1822 crd.appendVar('OVERRIDES', ':%s' % override) 1823 bb.process.run('git checkout %s' % branch, cwd=srctree) 1824 checkedout = branch 1825 1826 if mode == 'srcrev': 1827 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1828 elif mode == 'patch': 1829 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1830 else: 1831 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1832 if updated: 1833 anyupdated = True 1834 if appendf: 1835 appendfile = appendf 1836 allremoved.extend(removed) 1837 finally: 1838 if startbranch and checkedout != startbranch: 1839 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1840 1841 return anyupdated, appendfile, allremoved 1842 1843def update_recipe(args, config, basepath, workspace): 1844 """Entry point for the devtool 'update-recipe' subcommand""" 1845 check_workspace_recipe(workspace, args.recipename) 1846 1847 if args.append: 1848 if not os.path.exists(args.append): 1849 raise DevtoolError('bbappend destination layer directory "%s" ' 1850 'does not exist' % args.append) 1851 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1852 raise DevtoolError('conf/layer.conf not found in bbappend ' 1853 'destination layer "%s"' % args.append) 1854 1855 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1856 try: 1857 1858 rd = parse_recipe(config, tinfoil, args.recipename, True) 1859 if not rd: 1860 return 1 1861 1862 dry_run_output = None 1863 dry_run_outdir = None 1864 if args.dry_run: 1865 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1866 dry_run_outdir = dry_run_output.name 1867 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1868 1869 if updated: 1870 rf = rd.getVar('FILE') 1871 if rf.startswith(config.workspace_path): 1872 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1873 finally: 1874 tinfoil.shutdown() 1875 1876 return 0 1877 1878 1879def status(args, config, basepath, workspace): 1880 """Entry point for the devtool 'status' subcommand""" 1881 if workspace: 1882 for recipe, value in sorted(workspace.items()): 1883 recipefile = value['recipefile'] 1884 if recipefile: 1885 recipestr = ' (%s)' % recipefile 1886 else: 1887 recipestr = '' 1888 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1889 else: 1890 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1891 return 0 1892 1893 1894def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1895 """Reset one or more recipes""" 1896 import oe.path 1897 1898 def clean_preferred_provider(pn, layerconf_path): 1899 """Remove PREFERRED_PROVIDER from layer.conf'""" 1900 import re 1901 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1902 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1903 pprovider_found = False 1904 with open(layerconf_file, 'r') as f: 1905 lines = f.readlines() 1906 with open(new_layerconf_file, 'a') as nf: 1907 for line in lines: 1908 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1909 if not re.match(pprovider_exp, line): 1910 nf.write(line) 1911 else: 1912 pprovider_found = True 1913 if pprovider_found: 1914 shutil.move(new_layerconf_file, layerconf_file) 1915 else: 1916 os.remove(new_layerconf_file) 1917 1918 if recipes and not no_clean: 1919 if len(recipes) == 1: 1920 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 1921 else: 1922 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 1923 # If the recipe file itself was created in the workspace, and 1924 # it uses BBCLASSEXTEND, then we need to also clean the other 1925 # variants 1926 targets = [] 1927 for recipe in recipes: 1928 targets.append(recipe) 1929 recipefile = workspace[recipe]['recipefile'] 1930 if recipefile and os.path.exists(recipefile): 1931 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 1932 try: 1933 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 1934 except bb.process.ExecutionError as e: 1935 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 1936 'wish, you may specify -n/--no-clean to ' 1937 'skip running this command when resetting' % 1938 (e.command, e.stdout)) 1939 1940 for pn in recipes: 1941 _check_preserve(config, pn) 1942 1943 appendfile = workspace[pn]['bbappend'] 1944 if os.path.exists(appendfile): 1945 # This shouldn't happen, but is possible if devtool errored out prior to 1946 # writing the md5 file. We need to delete this here or the recipe won't 1947 # actually be reset 1948 os.remove(appendfile) 1949 1950 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 1951 def preservedir(origdir): 1952 if os.path.exists(origdir): 1953 for root, dirs, files in os.walk(origdir): 1954 for fn in files: 1955 logger.warning('Preserving %s in %s' % (fn, preservepath)) 1956 _move_file(os.path.join(origdir, fn), 1957 os.path.join(preservepath, fn)) 1958 for dn in dirs: 1959 preservedir(os.path.join(root, dn)) 1960 os.rmdir(origdir) 1961 1962 recipefile = workspace[pn]['recipefile'] 1963 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 1964 # This should always be true if recipefile is set, but just in case 1965 preservedir(os.path.dirname(recipefile)) 1966 # We don't automatically create this dir next to appends, but the user can 1967 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 1968 1969 srctreebase = workspace[pn]['srctreebase'] 1970 if os.path.isdir(srctreebase): 1971 if os.listdir(srctreebase): 1972 if remove_work: 1973 logger.info('-r argument used on %s, removing source tree.' 1974 ' You will lose any unsaved work' %pn) 1975 shutil.rmtree(srctreebase) 1976 else: 1977 # We don't want to risk wiping out any work in progress 1978 logger.info('Leaving source tree %s as-is; if you no ' 1979 'longer need it then please delete it manually' 1980 % srctreebase) 1981 else: 1982 # This is unlikely, but if it's empty we can just remove it 1983 os.rmdir(srctreebase) 1984 1985 clean_preferred_provider(pn, config.workspace_path) 1986 1987def reset(args, config, basepath, workspace): 1988 """Entry point for the devtool 'reset' subcommand""" 1989 import bb 1990 import shutil 1991 1992 recipes = "" 1993 1994 if args.recipename: 1995 if args.all: 1996 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 1997 else: 1998 for recipe in args.recipename: 1999 check_workspace_recipe(workspace, recipe, checksrc=False) 2000 elif not args.all: 2001 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2002 "reset all recipes") 2003 if args.all: 2004 recipes = list(workspace.keys()) 2005 else: 2006 recipes = args.recipename 2007 2008 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2009 2010 return 0 2011 2012 2013def _get_layer(layername, d): 2014 """Determine the base layer path for the specified layer name/path""" 2015 layerdirs = d.getVar('BBLAYERS').split() 2016 layers = {} # {basename: layer_paths} 2017 for p in layerdirs: 2018 bn = os.path.basename(p) 2019 if bn not in layers: 2020 layers[bn] = [p] 2021 else: 2022 layers[bn].append(p) 2023 # Provide some shortcuts 2024 if layername.lower() in ['oe-core', 'openembedded-core']: 2025 layername = 'meta' 2026 layer_paths = layers.get(layername, None) 2027 if not layer_paths: 2028 return os.path.abspath(layername) 2029 elif len(layer_paths) == 1: 2030 return os.path.abspath(layer_paths[0]) 2031 else: 2032 # multiple layers having the same base name 2033 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 2034 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 2035 return os.path.abspath(layer_paths[0]) 2036 2037 2038def finish(args, config, basepath, workspace): 2039 """Entry point for the devtool 'finish' subcommand""" 2040 import bb 2041 import oe.recipeutils 2042 2043 check_workspace_recipe(workspace, args.recipename) 2044 2045 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2046 2047 # Grab the equivalent of COREBASE without having to initialise tinfoil 2048 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2049 2050 srctree = workspace[args.recipename]['srctree'] 2051 check_git_repo_op(srctree, [corebasedir]) 2052 dirty = check_git_repo_dirty(srctree) 2053 if dirty: 2054 if args.force: 2055 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2056 else: 2057 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2058 2059 no_clean = args.no_clean 2060 remove_work=args.remove_work 2061 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2062 try: 2063 rd = parse_recipe(config, tinfoil, args.recipename, True) 2064 if not rd: 2065 return 1 2066 2067 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2068 recipefile = rd.getVar('FILE') 2069 recipedir = os.path.dirname(recipefile) 2070 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2071 2072 if not os.path.isdir(destlayerdir): 2073 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2074 2075 if os.path.abspath(destlayerdir) == config.workspace_path: 2076 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2077 2078 # If it's an upgrade, grab the original path 2079 origpath = None 2080 origfilelist = None 2081 append = workspace[args.recipename]['bbappend'] 2082 with open(append, 'r') as f: 2083 for line in f: 2084 if line.startswith('# original_path:'): 2085 origpath = line.split(':')[1].strip() 2086 elif line.startswith('# original_files:'): 2087 origfilelist = line.split(':')[1].split() 2088 2089 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2090 2091 if origlayerdir == config.workspace_path: 2092 # Recipe file itself is in workspace, update it there first 2093 appendlayerdir = None 2094 origrelpath = None 2095 if origpath: 2096 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2097 if origlayerpath: 2098 origrelpath = os.path.relpath(origpath, origlayerpath) 2099 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2100 if not destpath: 2101 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2102 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2103 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2104 if not os.path.abspath(destlayerbasedir) in layerdirs: 2105 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2106 2107 elif destlayerdir == origlayerdir: 2108 # Same layer, update the original recipe 2109 appendlayerdir = None 2110 destpath = None 2111 else: 2112 # Create/update a bbappend in the specified layer 2113 appendlayerdir = destlayerdir 2114 destpath = None 2115 2116 # Actually update the recipe / bbappend 2117 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2118 dry_run_output = None 2119 dry_run_outdir = None 2120 if args.dry_run: 2121 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2122 dry_run_outdir = dry_run_output.name 2123 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2124 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2125 2126 # Remove any old files in the case of an upgrade 2127 if removing_original: 2128 for fn in origfilelist: 2129 fnp = os.path.join(origpath, fn) 2130 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2131 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2132 if not args.dry_run: 2133 try: 2134 os.remove(fnp) 2135 except FileNotFoundError: 2136 pass 2137 2138 if origlayerdir == config.workspace_path and destpath: 2139 # Recipe file itself is in the workspace - need to move it and any 2140 # associated files to the specified layer 2141 no_clean = True 2142 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2143 for root, _, files in os.walk(recipedir): 2144 for fn in files: 2145 srcpath = os.path.join(root, fn) 2146 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2147 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2148 destfp = os.path.join(destdir, fn) 2149 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2150 2151 if dry_run_outdir: 2152 import difflib 2153 comparelist = [] 2154 for root, _, files in os.walk(dry_run_outdir): 2155 for fn in files: 2156 outf = os.path.join(root, fn) 2157 relf = os.path.relpath(outf, dry_run_outdir) 2158 logger.debug('dry-run: output file %s' % relf) 2159 if fn.endswith('.bb'): 2160 if origfilelist and origpath and destpath: 2161 # Need to match this up with the pre-upgrade recipe file 2162 for origf in origfilelist: 2163 if origf.endswith('.bb'): 2164 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2165 outf, 2166 os.path.abspath(os.path.join(destpath, relf)))) 2167 break 2168 else: 2169 # Compare to the existing recipe 2170 comparelist.append((recipefile, outf, recipefile)) 2171 elif fn.endswith('.bbappend'): 2172 if appendfile: 2173 if os.path.exists(appendfile): 2174 comparelist.append((appendfile, outf, appendfile)) 2175 else: 2176 comparelist.append((None, outf, appendfile)) 2177 else: 2178 if destpath: 2179 recipedest = destpath 2180 elif appendfile: 2181 recipedest = os.path.dirname(appendfile) 2182 else: 2183 recipedest = os.path.dirname(recipefile) 2184 destfp = os.path.join(recipedest, relf) 2185 if os.path.exists(destfp): 2186 comparelist.append((destfp, outf, destfp)) 2187 output = '' 2188 for oldfile, newfile, newfileshow in comparelist: 2189 if oldfile: 2190 with open(oldfile, 'r') as f: 2191 oldlines = f.readlines() 2192 else: 2193 oldfile = '/dev/null' 2194 oldlines = [] 2195 with open(newfile, 'r') as f: 2196 newlines = f.readlines() 2197 if not newfileshow: 2198 newfileshow = newfile 2199 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2200 difflines = list(diff) 2201 if difflines: 2202 output += ''.join(difflines) 2203 if output: 2204 logger.info('Diff of changed files:\n%s' % output) 2205 finally: 2206 tinfoil.shutdown() 2207 2208 # Everything else has succeeded, we can now reset 2209 if args.dry_run: 2210 logger.info('Resetting recipe (dry-run)') 2211 else: 2212 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2213 2214 return 0 2215 2216 2217def get_default_srctree(config, recipename=''): 2218 """Get the default srctree path""" 2219 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2220 if recipename: 2221 return os.path.join(srctreeparent, 'sources', recipename) 2222 else: 2223 return os.path.join(srctreeparent, 'sources') 2224 2225def register_commands(subparsers, context): 2226 """Register devtool subcommands from this plugin""" 2227 2228 defsrctree = get_default_srctree(context.config) 2229 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2230 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2231 group='starting', order=100) 2232 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2233 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2234 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2235 group = parser_add.add_mutually_exclusive_group() 2236 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2237 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2238 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2239 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2240 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2241 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2242 group = parser_add.add_mutually_exclusive_group() 2243 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2244 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2245 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2246 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2247 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2248 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2249 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2250 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2251 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2252 2253 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2254 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2255 group='starting', order=90) 2256 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2257 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2258 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2259 group = parser_modify.add_mutually_exclusive_group() 2260 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2261 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2262 group = parser_modify.add_mutually_exclusive_group() 2263 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2264 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2265 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2266 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2267 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2268 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2269 2270 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2271 description='Extracts the source for an existing recipe', 2272 group='advanced') 2273 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2274 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2275 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2276 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2277 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2278 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2279 2280 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2281 description='Synchronize the previously extracted source tree for an existing recipe', 2282 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2283 group='advanced') 2284 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2285 parser_sync.add_argument('srctree', help='Path to the source tree') 2286 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2287 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2288 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2289 2290 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2291 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2292 group='working', order=10) 2293 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2294 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2295 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2296 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2297 parser_rename.set_defaults(func=rename) 2298 2299 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2300 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2301 group='working', order=-90) 2302 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2303 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2304 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2305 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2306 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2307 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2308 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2309 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2310 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2311 parser_update_recipe.set_defaults(func=update_recipe) 2312 2313 parser_status = subparsers.add_parser('status', help='Show workspace status', 2314 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2315 group='info', order=100) 2316 parser_status.set_defaults(func=status) 2317 2318 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2319 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2320 group='working', order=-100) 2321 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2322 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2323 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2324 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2325 parser_reset.set_defaults(func=reset) 2326 2327 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2328 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2329 group='working', order=-100) 2330 parser_finish.add_argument('recipename', help='Recipe to finish') 2331 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2332 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2333 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2334 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2335 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2336 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2337 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2338 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2339 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2340 parser_finish.set_defaults(func=finish) 2341