1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.npm_dev: 149 extracmdopts += ' --npm-dev' 150 if args.mirrors: 151 extracmdopts += ' --mirrors' 152 if args.srcrev: 153 extracmdopts += ' --srcrev %s' % args.srcrev 154 if args.srcbranch: 155 extracmdopts += ' --srcbranch %s' % args.srcbranch 156 if args.provides: 157 extracmdopts += ' --provides %s' % args.provides 158 159 tempdir = tempfile.mkdtemp(prefix='devtool') 160 try: 161 try: 162 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 163 except bb.process.ExecutionError as e: 164 if e.exitcode == 15: 165 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 166 else: 167 raise DevtoolError('Command \'%s\' failed' % e.command) 168 169 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 170 if recipes: 171 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 172 if recipename in workspace: 173 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 174 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 175 bb.utils.mkdirhier(recipedir) 176 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 177 appendfile = recipe_to_append(recipefile, config) 178 if os.path.exists(appendfile): 179 # This shouldn't be possible, but just in case 180 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 181 if os.path.exists(recipefile): 182 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 183 if tmpsrcdir: 184 srctree = os.path.join(srctreeparent, recipename) 185 if os.path.exists(tmpsrcdir): 186 if os.path.exists(srctree): 187 if os.path.isdir(srctree): 188 try: 189 os.rmdir(srctree) 190 except OSError as e: 191 if e.errno == errno.ENOTEMPTY: 192 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 193 else: 194 raise 195 else: 196 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 197 logger.info('Using default source tree path %s' % srctree) 198 shutil.move(tmpsrcdir, srctree) 199 else: 200 raise DevtoolError('Couldn\'t find source tree created by recipetool') 201 bb.utils.mkdirhier(recipedir) 202 shutil.move(recipes[0], recipefile) 203 # Move any additional files created by recipetool 204 for fn in os.listdir(tempdir): 205 shutil.move(os.path.join(tempdir, fn), recipedir) 206 else: 207 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 208 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 209 if os.path.exists(attic_recipe): 210 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 211 finally: 212 if tmpsrcdir and os.path.exists(tmpsrcdir): 213 shutil.rmtree(tmpsrcdir) 214 shutil.rmtree(tempdir) 215 216 for fn in os.listdir(recipedir): 217 _add_md5(config, recipename, os.path.join(recipedir, fn)) 218 219 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 220 try: 221 try: 222 rd = tinfoil.parse_recipe_file(recipefile, False) 223 except Exception as e: 224 logger.error(str(e)) 225 rd = None 226 if not rd: 227 # Parsing failed. We just created this recipe and we shouldn't 228 # leave it in the workdir or it'll prevent bitbake from starting 229 movefn = '%s.parsefailed' % recipefile 230 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 231 shutil.move(recipefile, movefn) 232 return 1 233 234 if args.fetchuri and not args.no_git: 235 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 236 237 initial_rev = None 238 if os.path.exists(os.path.join(srctree, '.git')): 239 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 240 initial_rev = stdout.rstrip() 241 242 if args.src_subdir: 243 srctree = os.path.join(srctree, args.src_subdir) 244 245 bb.utils.mkdirhier(os.path.dirname(appendfile)) 246 with open(appendfile, 'w') as f: 247 f.write('inherit externalsrc\n') 248 f.write('EXTERNALSRC = "%s"\n' % srctree) 249 250 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 251 if b_is_s: 252 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 253 if initial_rev: 254 f.write('\n# initial_rev: %s\n' % initial_rev) 255 256 if args.binary: 257 f.write('do_install:append() {\n') 258 f.write(' rm -rf ${D}/.git\n') 259 f.write(' rm -f ${D}/singletask.lock\n') 260 f.write('}\n') 261 262 if bb.data.inherits_class('npm', rd): 263 f.write('python do_configure:append() {\n') 264 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 265 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 266 f.write(' bb.utils.remove(lockfile)\n') 267 f.write('}\n') 268 269 # Check if the new layer provides recipes whose priorities have been 270 # overriden by PREFERRED_PROVIDER. 271 recipe_name = rd.getVar('PN') 272 provides = rd.getVar('PROVIDES') 273 # Search every item defined in PROVIDES 274 for recipe_provided in provides.split(): 275 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 276 current_pprovider = rd.getVar(preferred_provider) 277 if current_pprovider and current_pprovider != recipe_name: 278 if args.fixed_setup: 279 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 280 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 281 with open(layerconf_file, 'a') as f: 282 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 283 else: 284 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 285 break 286 287 _add_md5(config, recipename, appendfile) 288 289 check_prerelease_version(rd.getVar('PV'), 'devtool add') 290 291 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 292 293 finally: 294 tinfoil.shutdown() 295 296 return 0 297 298 299def _check_compatible_recipe(pn, d): 300 """Check if the recipe is supported by devtool""" 301 if pn == 'perf': 302 raise DevtoolError("The perf recipe does not actually check out " 303 "source and thus cannot be supported by this tool", 304 4) 305 306 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 307 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 308 309 if bb.data.inherits_class('image', d): 310 raise DevtoolError("The %s recipe is an image, and therefore is not " 311 "supported by this tool" % pn, 4) 312 313 if bb.data.inherits_class('populate_sdk', d): 314 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 315 "supported by this tool" % pn, 4) 316 317 if bb.data.inherits_class('packagegroup', d): 318 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 319 "not supported by this tool" % pn, 4) 320 321 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 322 # Not an incompatibility error per se, so we don't pass the error code 323 raise DevtoolError("externalsrc is currently enabled for the %s " 324 "recipe. This prevents the normal do_patch task " 325 "from working. You will need to disable this " 326 "first." % pn) 327 328def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 329 """Common function for copying a file to the dry run output directory""" 330 relpath = os.path.relpath(dst, base_outdir) 331 if relpath.startswith('..'): 332 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 333 dst = os.path.join(dry_run_outdir, relpath) 334 dst_d = os.path.dirname(dst) 335 if dst_d: 336 bb.utils.mkdirhier(dst_d) 337 # Don't overwrite existing files, otherwise in the case of an upgrade 338 # the dry-run written out recipe will be overwritten with an unmodified 339 # version 340 if not os.path.exists(dst): 341 shutil.copy(src, dst) 342 343def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 344 """Move a file. Creates all the directory components of destination path.""" 345 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 346 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 347 if dry_run_outdir: 348 # We want to copy here, not move 349 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 350 else: 351 dst_d = os.path.dirname(dst) 352 if dst_d: 353 bb.utils.mkdirhier(dst_d) 354 shutil.move(src, dst) 355 356def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 357 """Copy a file. Creates all the directory components of destination path.""" 358 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 359 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 360 if dry_run_outdir: 361 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 362 else: 363 dst_d = os.path.dirname(dst) 364 if dst_d: 365 bb.utils.mkdirhier(dst_d) 366 shutil.copy(src, dst) 367 368def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 369 """List contents of a git treeish""" 370 import bb 371 cmd = ['git', 'ls-tree', '-z', treeish] 372 if recursive: 373 cmd.append('-r') 374 out, _ = bb.process.run(cmd, cwd=repodir) 375 ret = {} 376 if out: 377 for line in out.split('\0'): 378 if line: 379 split = line.split(None, 4) 380 ret[split[3]] = split[0:3] 381 return ret 382 383def _git_exclude_path(srctree, path): 384 """Return pathspec (list of paths) that excludes certain path""" 385 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 386 # we don't catch files that are deleted, for example. A more reliable way 387 # to implement this would be to use "negative pathspecs" which were 388 # introduced in Git v1.9.0. Revisit this when/if the required Git version 389 # becomes greater than that. 390 path = os.path.normpath(path) 391 recurse = True if len(path.split(os.path.sep)) > 1 else False 392 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 393 if path in git_files: 394 git_files.remove(path) 395 return git_files 396 else: 397 return ['.'] 398 399def _ls_tree(directory): 400 """Recursive listing of files in a directory""" 401 ret = [] 402 for root, dirs, files in os.walk(directory): 403 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 404 fname in files]) 405 return ret 406 407 408def extract(args, config, basepath, workspace): 409 """Entry point for the devtool 'extract' subcommand""" 410 import bb 411 412 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 413 if not tinfoil: 414 # Error already shown 415 return 1 416 try: 417 rd = parse_recipe(config, tinfoil, args.recipename, True) 418 if not rd: 419 return 1 420 421 srctree = os.path.abspath(args.srctree) 422 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 423 logger.info('Source tree extracted to %s' % srctree) 424 425 if initial_rev: 426 return 0 427 else: 428 return 1 429 finally: 430 tinfoil.shutdown() 431 432def sync(args, config, basepath, workspace): 433 """Entry point for the devtool 'sync' subcommand""" 434 import bb 435 436 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 437 if not tinfoil: 438 # Error already shown 439 return 1 440 try: 441 rd = parse_recipe(config, tinfoil, args.recipename, True) 442 if not rd: 443 return 1 444 445 srctree = os.path.abspath(args.srctree) 446 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 447 logger.info('Source tree %s synchronized' % srctree) 448 449 if initial_rev: 450 return 0 451 else: 452 return 1 453 finally: 454 tinfoil.shutdown() 455 456def symlink_oelocal_files_srctree(rd,srctree): 457 import oe.patch 458 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 459 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 460 # (otherwise the recipe won't build as expected) 461 local_files_dir = os.path.join(srctree, 'oe-local-files') 462 addfiles = [] 463 for root, _, files in os.walk(local_files_dir): 464 relpth = os.path.relpath(root, local_files_dir) 465 if relpth != '.': 466 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 467 for fn in files: 468 if fn == '.gitignore': 469 continue 470 destpth = os.path.join(srctree, relpth, fn) 471 if os.path.exists(destpth): 472 os.unlink(destpth) 473 if relpth != '.': 474 back_relpth = os.path.relpath(local_files_dir, root) 475 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) 476 else: 477 os.symlink('oe-local-files/%s' % fn, destpth) 478 addfiles.append(os.path.join(relpth, fn)) 479 if addfiles: 480 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 481 useroptions = [] 482 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 483 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 484 485 486def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 487 """Extract sources of a recipe""" 488 import oe.recipeutils 489 import oe.patch 490 import oe.path 491 492 pn = d.getVar('PN') 493 494 _check_compatible_recipe(pn, d) 495 496 if sync: 497 if not os.path.exists(srctree): 498 raise DevtoolError("output path %s does not exist" % srctree) 499 else: 500 if os.path.exists(srctree): 501 if not os.path.isdir(srctree): 502 raise DevtoolError("output path %s exists and is not a directory" % 503 srctree) 504 elif os.listdir(srctree): 505 raise DevtoolError("output path %s already exists and is " 506 "non-empty" % srctree) 507 508 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 509 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 510 "extract source" % pn, 4) 511 512 if not sync: 513 # Prepare for shutil.move later on 514 bb.utils.mkdirhier(srctree) 515 os.rmdir(srctree) 516 517 extra_overrides = [] 518 if not no_overrides: 519 history = d.varhistory.variable('SRC_URI') 520 for event in history: 521 if not 'flag' in event: 522 if event['op'].startswith((':append[', ':prepend[')): 523 override = event['op'].split('[')[1].split(']')[0] 524 if not override.startswith('pn-'): 525 extra_overrides.append(override) 526 # We want to remove duplicate overrides. If a recipe had multiple 527 # SRC_URI_override += values it would cause mulitple instances of 528 # overrides. This doesn't play nicely with things like creating a 529 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 530 extra_overrides = list(set(extra_overrides)) 531 if extra_overrides: 532 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 533 534 initial_rev = None 535 536 recipefile = d.getVar('FILE') 537 appendfile = recipe_to_append(recipefile, config) 538 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 539 540 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 541 # directory so that: 542 # (a) we pick up all files that get unpacked to the WORKDIR, and 543 # (b) we don't disturb the existing build 544 # However, with recipe-specific sysroots the sysroots for the recipe 545 # will be prepared under WORKDIR, and if we used the system temporary 546 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 547 # our attempts to hardlink files into the recipe-specific sysroots 548 # will fail on systems where /tmp is a different filesystem, and it 549 # would have to fall back to copying the files which is a waste of 550 # time. Put the temp directory under the WORKDIR to prevent that from 551 # being a problem. 552 tempbasedir = d.getVar('WORKDIR') 553 bb.utils.mkdirhier(tempbasedir) 554 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 555 try: 556 tinfoil.logger.setLevel(logging.WARNING) 557 558 # FIXME this results in a cache reload under control of tinfoil, which is fine 559 # except we don't get the knotty progress bar 560 561 if os.path.exists(appendfile): 562 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 563 shutil.copyfile(appendfile, appendbackup) 564 else: 565 appendbackup = None 566 bb.utils.mkdirhier(os.path.dirname(appendfile)) 567 logger.debug('writing append file %s' % appendfile) 568 with open(appendfile, 'a') as f: 569 f.write('###--- _extract_source\n') 570 f.write('deltask do_recipe_qa\n') 571 f.write('deltask do_recipe_qa_setscene\n') 572 f.write('ERROR_QA:remove = "patch-fuzz"\n') 573 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 574 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 575 if not is_kernel_yocto: 576 f.write('PATCHTOOL = "git"\n') 577 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 578 if extra_overrides: 579 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 580 f.write('inherit devtool-source\n') 581 f.write('###--- _extract_source\n') 582 583 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 584 585 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 586 bb.utils.mkdirhier(sstate_manifests) 587 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 588 with open(preservestampfile, 'w') as f: 589 f.write(d.getVar('STAMP')) 590 try: 591 if is_kernel_yocto: 592 # We need to generate the kernel config 593 task = 'do_configure' 594 else: 595 task = 'do_patch' 596 597 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 598 logger.info('The %s recipe has %s disabled. Running only ' 599 'do_configure task dependencies' % (pn, task)) 600 601 if 'depends' in d.getVarFlags('do_configure', False): 602 pn = d.getVarFlags('do_configure', False)['depends'] 603 pn = pn.replace('${PV}', d.getVar('PV')) 604 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 605 task = None 606 607 # Run the fetch + unpack tasks 608 res = tinfoil.build_targets(pn, 609 task, 610 handle_events=True) 611 finally: 612 if os.path.exists(preservestampfile): 613 os.remove(preservestampfile) 614 615 if not res: 616 raise DevtoolError('Extracting source for %s failed' % pn) 617 618 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 619 workshareddir = d.getVar('S') 620 if os.path.islink(srctree): 621 os.unlink(srctree) 622 623 os.symlink(workshareddir, srctree) 624 625 # The initial_rev file is created in devtool_post_unpack function that will not be executed if 626 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 627 return True, True 628 629 try: 630 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 631 initial_rev = f.read() 632 633 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 634 srcsubdir = f.read() 635 except FileNotFoundError as e: 636 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 637 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 638 639 # Check if work-shared is empty, if yes 640 # find source and copy to work-shared 641 if is_kernel_yocto: 642 workshareddir = d.getVar('STAGING_KERNEL_DIR') 643 staging_kerVer = get_staging_kver(workshareddir) 644 kernelVersion = d.getVar('LINUX_VERSION') 645 646 # handle dangling symbolic link in work-shared: 647 if os.path.islink(workshareddir): 648 os.unlink(workshareddir) 649 650 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 651 shutil.rmtree(workshareddir) 652 oe.path.copyhardlinktree(srcsubdir,workshareddir) 653 elif not os.path.exists(workshareddir): 654 oe.path.copyhardlinktree(srcsubdir,workshareddir) 655 656 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 657 srctree_localdir = os.path.join(srctree, 'oe-local-files') 658 659 if sync: 660 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 661 662 # Move oe-local-files directory to srctree 663 # As the oe-local-files is not part of the constructed git tree, 664 # remove them directly during the synchrounizating might surprise 665 # the users. Instead, we move it to oe-local-files.bak and remind 666 # user in the log message. 667 if os.path.exists(srctree_localdir + '.bak'): 668 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 669 670 if os.path.exists(srctree_localdir): 671 logger.info('Backing up current local file directory %s' % srctree_localdir) 672 shutil.move(srctree_localdir, srctree_localdir + '.bak') 673 674 if os.path.exists(tempdir_localdir): 675 logger.info('Syncing local source files to srctree...') 676 shutil.copytree(tempdir_localdir, srctree_localdir) 677 else: 678 # Move oe-local-files directory to srctree 679 if os.path.exists(tempdir_localdir): 680 logger.info('Adding local source files to srctree...') 681 shutil.move(tempdir_localdir, srcsubdir) 682 683 shutil.move(srcsubdir, srctree) 684 symlink_oelocal_files_srctree(d,srctree) 685 686 if is_kernel_yocto: 687 logger.info('Copying kernel config to srctree') 688 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 689 690 finally: 691 if appendbackup: 692 shutil.copyfile(appendbackup, appendfile) 693 elif os.path.exists(appendfile): 694 os.remove(appendfile) 695 if keep_temp: 696 logger.info('Preserving temporary directory %s' % tempdir) 697 else: 698 shutil.rmtree(tempdir) 699 return initial_rev, srcsubdir_rel 700 701def _add_md5(config, recipename, filename): 702 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 703 import bb.utils 704 705 def addfile(fn): 706 md5 = bb.utils.md5_file(fn) 707 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 708 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 709 f.seek(0, os.SEEK_SET) 710 if not md5_str in f.read(): 711 f.write(md5_str) 712 713 if os.path.isdir(filename): 714 for root, _, files in os.walk(filename): 715 for f in files: 716 addfile(os.path.join(root, f)) 717 else: 718 addfile(filename) 719 720def _check_preserve(config, recipename): 721 """Check if a file was manually changed and needs to be saved in 'attic' 722 directory""" 723 import bb.utils 724 origfile = os.path.join(config.workspace_path, '.devtool_md5') 725 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 726 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 727 with open(origfile, 'r') as f: 728 with open(newfile, 'w') as tf: 729 for line in f.readlines(): 730 splitline = line.rstrip().split('|') 731 if splitline[0] == recipename: 732 removefile = os.path.join(config.workspace_path, splitline[1]) 733 try: 734 md5 = bb.utils.md5_file(removefile) 735 except IOError as err: 736 if err.errno == 2: 737 # File no longer exists, skip it 738 continue 739 else: 740 raise 741 if splitline[2] != md5: 742 bb.utils.mkdirhier(preservepath) 743 preservefile = os.path.basename(removefile) 744 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 745 shutil.move(removefile, os.path.join(preservepath, preservefile)) 746 else: 747 os.remove(removefile) 748 else: 749 tf.write(line) 750 bb.utils.rename(newfile, origfile) 751 752def get_staging_kver(srcdir): 753 # Kernel version from work-shared 754 kerver = [] 755 staging_kerVer="" 756 if os.path.exists(srcdir) and os.listdir(srcdir): 757 with open(os.path.join(srcdir,"Makefile")) as f: 758 version = [next(f) for x in range(5)][1:4] 759 for word in version: 760 kerver.append(word.split('= ')[1].split('\n')[0]) 761 staging_kerVer = ".".join(kerver) 762 return staging_kerVer 763 764def get_staging_kbranch(srcdir): 765 staging_kbranch = "" 766 if os.path.exists(srcdir) and os.listdir(srcdir): 767 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) 768 staging_kbranch = "".join(branch.split('\n')[0]) 769 return staging_kbranch 770 771def get_real_srctree(srctree, s, workdir): 772 # Check that recipe isn't using a shared workdir 773 s = os.path.abspath(s) 774 workdir = os.path.abspath(workdir) 775 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 776 # Handle if S is set to a subdirectory of the source 777 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 778 srctree = os.path.join(srctree, srcsubdir) 779 return srctree 780 781def modify(args, config, basepath, workspace): 782 """Entry point for the devtool 'modify' subcommand""" 783 import bb 784 import oe.recipeutils 785 import oe.patch 786 import oe.path 787 788 if args.recipename in workspace: 789 raise DevtoolError("recipe %s is already in your workspace" % 790 args.recipename) 791 792 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 793 try: 794 rd = parse_recipe(config, tinfoil, args.recipename, True) 795 if not rd: 796 return 1 797 798 pn = rd.getVar('PN') 799 if pn != args.recipename: 800 logger.info('Mapping %s to %s' % (args.recipename, pn)) 801 if pn in workspace: 802 raise DevtoolError("recipe %s is already in your workspace" % 803 pn) 804 805 if args.srctree: 806 srctree = os.path.abspath(args.srctree) 807 else: 808 srctree = get_default_srctree(config, pn) 809 810 if args.no_extract and not os.path.isdir(srctree): 811 raise DevtoolError("--no-extract specified and source path %s does " 812 "not exist or is not a directory" % 813 srctree) 814 815 recipefile = rd.getVar('FILE') 816 appendfile = recipe_to_append(recipefile, config, args.wildcard) 817 if os.path.exists(appendfile): 818 raise DevtoolError("Another variant of recipe %s is already in your " 819 "workspace (only one variant of a recipe can " 820 "currently be worked on at once)" 821 % pn) 822 823 _check_compatible_recipe(pn, rd) 824 825 initial_rev = None 826 commits = [] 827 check_commits = False 828 829 if bb.data.inherits_class('kernel-yocto', rd): 830 # Current set kernel version 831 kernelVersion = rd.getVar('LINUX_VERSION') 832 srcdir = rd.getVar('STAGING_KERNEL_DIR') 833 kbranch = rd.getVar('KBRANCH') 834 835 staging_kerVer = get_staging_kver(srcdir) 836 staging_kbranch = get_staging_kbranch(srcdir) 837 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 838 oe.path.copyhardlinktree(srcdir,srctree) 839 workdir = rd.getVar('WORKDIR') 840 srcsubdir = rd.getVar('S') 841 localfilesdir = os.path.join(srctree,'oe-local-files') 842 # Move local source files into separate subdir 843 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 844 local_files = oe.recipeutils.get_recipe_local_files(rd) 845 846 for key in local_files.copy(): 847 if key.endswith('scc'): 848 sccfile = open(local_files[key], 'r') 849 for l in sccfile: 850 line = l.split() 851 if line and line[0] in ('kconf', 'patch'): 852 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 853 if not cfg in local_files.values(): 854 local_files[line[-1]] = cfg 855 shutil.copy2(cfg, workdir) 856 sccfile.close() 857 858 # Ignore local files with subdir={BP} 859 srcabspath = os.path.abspath(srcsubdir) 860 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 861 if local_files: 862 for fname in local_files: 863 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 864 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 865 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 866 867 symlink_oelocal_files_srctree(rd,srctree) 868 869 task = 'do_configure' 870 res = tinfoil.build_targets(pn, task, handle_events=True) 871 872 # Copy .config to workspace 873 kconfpath = rd.getVar('B') 874 logger.info('Copying kernel config to workspace') 875 shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 876 877 # Set this to true, we still need to get initial_rev 878 # by parsing the git repo 879 args.no_extract = True 880 881 if not args.no_extract: 882 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 883 if not initial_rev: 884 return 1 885 logger.info('Source tree extracted to %s' % srctree) 886 if os.path.exists(os.path.join(srctree, '.git')): 887 # Get list of commits since this revision 888 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) 889 commits = stdout.split() 890 check_commits = True 891 else: 892 if os.path.exists(os.path.join(srctree, '.git')): 893 # Check if it's a tree previously extracted by us. This is done 894 # by ensuring that devtool-base and args.branch (devtool) exist. 895 # The check_commits logic will cause an exception if either one 896 # of these doesn't exist 897 try: 898 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 899 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 900 except bb.process.ExecutionError: 901 stdout = '' 902 if stdout: 903 check_commits = True 904 for line in stdout.splitlines(): 905 if line.startswith('*'): 906 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 907 initial_rev = stdout.rstrip() 908 if not initial_rev: 909 # Otherwise, just grab the head revision 910 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 911 initial_rev = stdout.rstrip() 912 913 branch_patches = {} 914 if check_commits: 915 # Check if there are override branches 916 (stdout, _) = bb.process.run('git branch', cwd=srctree) 917 branches = [] 918 for line in stdout.rstrip().splitlines(): 919 branchname = line[2:].rstrip() 920 if branchname.startswith(override_branch_prefix): 921 branches.append(branchname) 922 if branches: 923 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 924 branches.insert(0, args.branch) 925 seen_patches = [] 926 for branch in branches: 927 branch_patches[branch] = [] 928 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 929 for line in stdout.splitlines(): 930 line = line.strip() 931 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 932 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 933 if not origpatch in seen_patches: 934 seen_patches.append(origpatch) 935 branch_patches[branch].append(origpatch) 936 937 # Need to grab this here in case the source is within a subdirectory 938 srctreebase = srctree 939 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 940 941 bb.utils.mkdirhier(os.path.dirname(appendfile)) 942 with open(appendfile, 'w') as f: 943 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 944 # Local files can be modified/tracked in separate subdir under srctree 945 # Mostly useful for packages with S != WORKDIR 946 f.write('FILESPATH:prepend := "%s:"\n' % 947 os.path.join(srctreebase, 'oe-local-files')) 948 f.write('# srctreebase: %s\n' % srctreebase) 949 950 f.write('\ninherit externalsrc\n') 951 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 952 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 953 954 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 955 if b_is_s: 956 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 957 958 if bb.data.inherits_class('kernel', rd): 959 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 960 'do_fetch do_unpack do_kernel_configcheck"\n') 961 f.write('\ndo_patch[noexec] = "1"\n') 962 f.write('\ndo_configure:append() {\n' 963 ' cp ${B}/.config ${S}/.config.baseline\n' 964 ' ln -sfT ${B}/.config ${S}/.config.new\n' 965 '}\n') 966 f.write('\ndo_kernel_configme:prepend() {\n' 967 ' if [ -e ${S}/.config ]; then\n' 968 ' mv ${S}/.config ${S}/.config.old\n' 969 ' fi\n' 970 '}\n') 971 if rd.getVarFlag('do_menuconfig','task'): 972 f.write('\ndo_configure:append() {\n' 973 ' if [ ${@ oe.types.boolean(\'${KCONFIG_CONFIG_ENABLE_MENUCONFIG}\') } = True ]; then\n' 974 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' 975 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' 976 ' fi\n' 977 '}\n') 978 if initial_rev: 979 f.write('\n# initial_rev: %s\n' % initial_rev) 980 for commit in commits: 981 f.write('# commit: %s\n' % commit) 982 if branch_patches: 983 for branch in branch_patches: 984 if branch == args.branch: 985 continue 986 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 987 988 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 989 990 _add_md5(config, pn, appendfile) 991 992 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 993 994 finally: 995 tinfoil.shutdown() 996 997 return 0 998 999 1000def rename(args, config, basepath, workspace): 1001 """Entry point for the devtool 'rename' subcommand""" 1002 import bb 1003 import oe.recipeutils 1004 1005 check_workspace_recipe(workspace, args.recipename) 1006 1007 if not (args.newname or args.version): 1008 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 1009 1010 recipefile = workspace[args.recipename]['recipefile'] 1011 if not recipefile: 1012 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1013 1014 if args.newname and args.newname != args.recipename: 1015 reason = oe.recipeutils.validate_pn(args.newname) 1016 if reason: 1017 raise DevtoolError(reason) 1018 newname = args.newname 1019 else: 1020 newname = args.recipename 1021 1022 append = workspace[args.recipename]['bbappend'] 1023 appendfn = os.path.splitext(os.path.basename(append))[0] 1024 splitfn = appendfn.split('_') 1025 if len(splitfn) > 1: 1026 origfnver = appendfn.split('_')[1] 1027 else: 1028 origfnver = '' 1029 1030 recipefilemd5 = None 1031 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1032 try: 1033 rd = parse_recipe(config, tinfoil, args.recipename, True) 1034 if not rd: 1035 return 1 1036 1037 bp = rd.getVar('BP') 1038 bpn = rd.getVar('BPN') 1039 if newname != args.recipename: 1040 localdata = rd.createCopy() 1041 localdata.setVar('PN', newname) 1042 newbpn = localdata.getVar('BPN') 1043 else: 1044 newbpn = bpn 1045 s = rd.getVar('S', False) 1046 src_uri = rd.getVar('SRC_URI', False) 1047 pv = rd.getVar('PV') 1048 1049 # Correct variable values that refer to the upstream source - these 1050 # values must stay the same, so if the name/version are changing then 1051 # we need to fix them up 1052 new_s = s 1053 new_src_uri = src_uri 1054 if newbpn != bpn: 1055 # ${PN} here is technically almost always incorrect, but people do use it 1056 new_s = new_s.replace('${BPN}', bpn) 1057 new_s = new_s.replace('${PN}', bpn) 1058 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1059 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1060 new_src_uri = new_src_uri.replace('${PN}', bpn) 1061 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1062 if args.version and origfnver == pv: 1063 new_s = new_s.replace('${PV}', pv) 1064 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1065 new_src_uri = new_src_uri.replace('${PV}', pv) 1066 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1067 patchfields = {} 1068 if new_s != s: 1069 patchfields['S'] = new_s 1070 if new_src_uri != src_uri: 1071 patchfields['SRC_URI'] = new_src_uri 1072 if patchfields: 1073 recipefilemd5 = bb.utils.md5_file(recipefile) 1074 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1075 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1076 finally: 1077 tinfoil.shutdown() 1078 1079 if args.version: 1080 newver = args.version 1081 else: 1082 newver = origfnver 1083 1084 if newver: 1085 newappend = '%s_%s.bbappend' % (newname, newver) 1086 newfile = '%s_%s.bb' % (newname, newver) 1087 else: 1088 newappend = '%s.bbappend' % newname 1089 newfile = '%s.bb' % newname 1090 1091 oldrecipedir = os.path.dirname(recipefile) 1092 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1093 if oldrecipedir != newrecipedir: 1094 bb.utils.mkdirhier(newrecipedir) 1095 1096 newappend = os.path.join(os.path.dirname(append), newappend) 1097 newfile = os.path.join(newrecipedir, newfile) 1098 1099 # Rename bbappend 1100 logger.info('Renaming %s to %s' % (append, newappend)) 1101 bb.utils.rename(append, newappend) 1102 # Rename recipe file 1103 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1104 bb.utils.rename(recipefile, newfile) 1105 1106 # Rename source tree if it's the default path 1107 appendmd5 = None 1108 if not args.no_srctree: 1109 srctree = workspace[args.recipename]['srctree'] 1110 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1111 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1112 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1113 shutil.move(srctree, newsrctree) 1114 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1115 appendmd5 = bb.utils.md5_file(newappend) 1116 appendlines = [] 1117 with open(newappend, 'r') as f: 1118 for line in f: 1119 appendlines.append(line) 1120 with open(newappend, 'w') as f: 1121 for line in appendlines: 1122 if srctree in line: 1123 line = line.replace(srctree, newsrctree) 1124 f.write(line) 1125 newappendmd5 = bb.utils.md5_file(newappend) 1126 1127 bpndir = None 1128 newbpndir = None 1129 if newbpn != bpn: 1130 bpndir = os.path.join(oldrecipedir, bpn) 1131 if os.path.exists(bpndir): 1132 newbpndir = os.path.join(newrecipedir, newbpn) 1133 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1134 shutil.move(bpndir, newbpndir) 1135 1136 bpdir = None 1137 newbpdir = None 1138 if newver != origfnver or newbpn != bpn: 1139 bpdir = os.path.join(oldrecipedir, bp) 1140 if os.path.exists(bpdir): 1141 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1142 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1143 shutil.move(bpdir, newbpdir) 1144 1145 if oldrecipedir != newrecipedir: 1146 # Move any stray files and delete the old recipe directory 1147 for entry in os.listdir(oldrecipedir): 1148 oldpath = os.path.join(oldrecipedir, entry) 1149 newpath = os.path.join(newrecipedir, entry) 1150 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1151 shutil.move(oldpath, newpath) 1152 os.rmdir(oldrecipedir) 1153 1154 # Now take care of entries in .devtool_md5 1155 md5entries = [] 1156 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1157 for line in f: 1158 md5entries.append(line) 1159 1160 if bpndir and newbpndir: 1161 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1162 else: 1163 relbpndir = None 1164 if bpdir and newbpdir: 1165 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1166 else: 1167 relbpdir = None 1168 1169 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1170 for entry in md5entries: 1171 splitentry = entry.rstrip().split('|') 1172 if len(splitentry) > 2: 1173 if splitentry[0] == args.recipename: 1174 splitentry[0] = newname 1175 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1176 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1177 if appendmd5 and splitentry[2] == appendmd5: 1178 splitentry[2] = newappendmd5 1179 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1180 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1181 if recipefilemd5 and splitentry[2] == recipefilemd5: 1182 splitentry[2] = newrecipefilemd5 1183 elif relbpndir and splitentry[1].startswith(relbpndir): 1184 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1185 elif relbpdir and splitentry[1].startswith(relbpdir): 1186 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1187 entry = '|'.join(splitentry) + '\n' 1188 f.write(entry) 1189 return 0 1190 1191 1192def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1193 """Get initial and update rev of a recipe. These are the start point of the 1194 whole patchset and start point for the patches to be re-generated/updated. 1195 """ 1196 import bb 1197 1198 # Get current branch 1199 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1200 cwd=srctree) 1201 branchname = stdout.rstrip() 1202 1203 # Parse initial rev from recipe if not specified 1204 commits = [] 1205 patches = [] 1206 with open(recipe_path, 'r') as f: 1207 for line in f: 1208 if line.startswith('# initial_rev:'): 1209 if not initial_rev: 1210 initial_rev = line.split(':')[-1].strip() 1211 elif line.startswith('# commit:') and not force_patch_refresh: 1212 commits.append(line.split(':')[-1].strip()) 1213 elif line.startswith('# patches_%s:' % branchname): 1214 patches = line.split(':')[-1].strip().split(',') 1215 1216 update_rev = initial_rev 1217 changed_revs = None 1218 if initial_rev: 1219 # Find first actually changed revision 1220 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1221 initial_rev, cwd=srctree) 1222 newcommits = stdout.split() 1223 for i in range(min(len(commits), len(newcommits))): 1224 if newcommits[i] == commits[i]: 1225 update_rev = commits[i] 1226 1227 try: 1228 stdout, _ = bb.process.run('git cherry devtool-patched', 1229 cwd=srctree) 1230 except bb.process.ExecutionError as err: 1231 stdout = None 1232 1233 if stdout is not None and not force_patch_refresh: 1234 changed_revs = [] 1235 for line in stdout.splitlines(): 1236 if line.startswith('+ '): 1237 rev = line.split()[1] 1238 if rev in newcommits: 1239 changed_revs.append(rev) 1240 1241 return initial_rev, update_rev, changed_revs, patches 1242 1243def _remove_file_entries(srcuri, filelist): 1244 """Remove file:// entries from SRC_URI""" 1245 remaining = filelist[:] 1246 entries = [] 1247 for fname in filelist: 1248 basename = os.path.basename(fname) 1249 for i in range(len(srcuri)): 1250 if (srcuri[i].startswith('file://') and 1251 os.path.basename(srcuri[i].split(';')[0]) == basename): 1252 entries.append(srcuri[i]) 1253 remaining.remove(fname) 1254 srcuri.pop(i) 1255 break 1256 return entries, remaining 1257 1258def _replace_srcuri_entry(srcuri, filename, newentry): 1259 """Replace entry corresponding to specified file with a new entry""" 1260 basename = os.path.basename(filename) 1261 for i in range(len(srcuri)): 1262 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1263 srcuri.pop(i) 1264 srcuri.insert(i, newentry) 1265 break 1266 1267def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1268 """Unlink existing patch files""" 1269 1270 dry_run_suffix = ' (dry-run)' if dry_run else '' 1271 1272 for path in files: 1273 if append: 1274 if not destpath: 1275 raise Exception('destpath should be set here') 1276 path = os.path.join(destpath, os.path.basename(path)) 1277 1278 if os.path.exists(path): 1279 if not no_report_remove: 1280 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1281 if not dry_run: 1282 # FIXME "git rm" here would be nice if the file in question is 1283 # tracked 1284 # FIXME there's a chance that this file is referred to by 1285 # another recipe, in which case deleting wouldn't be the 1286 # right thing to do 1287 os.remove(path) 1288 # Remove directory if empty 1289 try: 1290 os.rmdir(os.path.dirname(path)) 1291 except OSError as ose: 1292 if ose.errno != errno.ENOTEMPTY: 1293 raise 1294 1295 1296def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1297 """Export patches from srctree to given location. 1298 Returns three-tuple of dicts: 1299 1. updated - patches that already exist in SRCURI 1300 2. added - new patches that don't exist in SRCURI 1301 3 removed - patches that exist in SRCURI but not in exported patches 1302 In each dict the key is the 'basepath' of the URI and value is the 1303 absolute path to the existing file in recipe space (if any). 1304 """ 1305 import oe.recipeutils 1306 from oe.patch import GitApplyTree 1307 updated = OrderedDict() 1308 added = OrderedDict() 1309 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1310 1311 existing_patches = dict((os.path.basename(path), path) for path in 1312 oe.recipeutils.get_recipe_patches(rd)) 1313 logger.debug('Existing patches: %s' % existing_patches) 1314 1315 # Generate patches from Git, exclude local files directory 1316 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1317 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) 1318 1319 new_patches = sorted(os.listdir(destdir)) 1320 for new_patch in new_patches: 1321 # Strip numbering from patch names. If it's a git sequence named patch, 1322 # the numbers might not match up since we are starting from a different 1323 # revision This does assume that people are using unique shortlog 1324 # values, but they ought to be anyway... 1325 new_basename = seqpatch_re.match(new_patch).group(2) 1326 match_name = None 1327 for old_patch in existing_patches: 1328 old_basename = seqpatch_re.match(old_patch).group(2) 1329 old_basename_splitext = os.path.splitext(old_basename) 1330 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1331 old_patch_noext = os.path.splitext(old_patch)[0] 1332 match_name = old_patch_noext 1333 break 1334 elif new_basename == old_basename: 1335 match_name = old_patch 1336 break 1337 if match_name: 1338 # Rename patch files 1339 if new_patch != match_name: 1340 bb.utils.rename(os.path.join(destdir, new_patch), 1341 os.path.join(destdir, match_name)) 1342 # Need to pop it off the list now before checking changed_revs 1343 oldpath = existing_patches.pop(old_patch) 1344 if changed_revs is not None: 1345 # Avoid updating patches that have not actually changed 1346 with open(os.path.join(destdir, match_name), 'r') as f: 1347 firstlineitems = f.readline().split() 1348 # Looking for "From <hash>" line 1349 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1350 if not firstlineitems[1] in changed_revs: 1351 continue 1352 # Recompress if necessary 1353 if oldpath.endswith(('.gz', '.Z')): 1354 bb.process.run(['gzip', match_name], cwd=destdir) 1355 if oldpath.endswith('.gz'): 1356 match_name += '.gz' 1357 else: 1358 match_name += '.Z' 1359 elif oldpath.endswith('.bz2'): 1360 bb.process.run(['bzip2', match_name], cwd=destdir) 1361 match_name += '.bz2' 1362 updated[match_name] = oldpath 1363 else: 1364 added[new_patch] = None 1365 return (updated, added, existing_patches) 1366 1367 1368def _create_kconfig_diff(srctree, rd, outfile): 1369 """Create a kconfig fragment""" 1370 # Only update config fragment if both config files exist 1371 orig_config = os.path.join(srctree, '.config.baseline') 1372 new_config = os.path.join(srctree, '.config.new') 1373 if os.path.exists(orig_config) and os.path.exists(new_config): 1374 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1375 '--unchanged-line-format=', orig_config, new_config] 1376 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1377 stderr=subprocess.PIPE) 1378 stdout, stderr = pipe.communicate() 1379 if pipe.returncode == 1: 1380 logger.info("Updating config fragment %s" % outfile) 1381 with open(outfile, 'wb') as fobj: 1382 fobj.write(stdout) 1383 elif pipe.returncode == 0: 1384 logger.info("Would remove config fragment %s" % outfile) 1385 if os.path.exists(outfile): 1386 # Remove fragment file in case of empty diff 1387 logger.info("Removing config fragment %s" % outfile) 1388 os.unlink(outfile) 1389 else: 1390 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1391 return True 1392 return False 1393 1394 1395def _export_local_files(srctree, rd, destdir, srctreebase): 1396 """Copy local files from srctree to given location. 1397 Returns three-tuple of dicts: 1398 1. updated - files that already exist in SRCURI 1399 2. added - new files files that don't exist in SRCURI 1400 3 removed - files that exist in SRCURI but not in exported files 1401 In each dict the key is the 'basepath' of the URI and value is the 1402 absolute path to the existing file in recipe space (if any). 1403 """ 1404 import oe.recipeutils 1405 1406 # Find out local files (SRC_URI files that exist in the "recipe space"). 1407 # Local files that reside in srctree are not included in patch generation. 1408 # Instead they are directly copied over the original source files (in 1409 # recipe space). 1410 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1411 new_set = None 1412 updated = OrderedDict() 1413 added = OrderedDict() 1414 removed = OrderedDict() 1415 1416 # Get current branch and return early with empty lists 1417 # if on one of the override branches 1418 # (local files are provided only for the main branch and processing 1419 # them against lists from recipe overrides will result in mismatches 1420 # and broken modifications to recipes). 1421 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1422 cwd=srctree) 1423 branchname = stdout.rstrip() 1424 if branchname.startswith(override_branch_prefix): 1425 return (updated, added, removed) 1426 1427 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1428 git_files = _git_ls_tree(srctree) 1429 if 'oe-local-files' in git_files: 1430 # If tracked by Git, take the files from srctree HEAD. First get 1431 # the tree object of the directory 1432 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1433 tree = git_files['oe-local-files'][2] 1434 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1435 env=dict(os.environ, GIT_WORK_TREE=destdir, 1436 GIT_INDEX_FILE=tmp_index)) 1437 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1438 elif os.path.isdir(local_files_dir): 1439 # If not tracked by Git, just copy from working copy 1440 new_set = _ls_tree(local_files_dir) 1441 bb.process.run(['cp', '-ax', 1442 os.path.join(local_files_dir, '.'), destdir]) 1443 else: 1444 new_set = [] 1445 1446 # Special handling for kernel config 1447 if bb.data.inherits_class('kernel-yocto', rd): 1448 fragment_fn = 'devtool-fragment.cfg' 1449 fragment_path = os.path.join(destdir, fragment_fn) 1450 if _create_kconfig_diff(srctree, rd, fragment_path): 1451 if os.path.exists(fragment_path): 1452 if fragment_fn not in new_set: 1453 new_set.append(fragment_fn) 1454 # Copy fragment to local-files 1455 if os.path.isdir(local_files_dir): 1456 shutil.copy2(fragment_path, local_files_dir) 1457 else: 1458 if fragment_fn in new_set: 1459 new_set.remove(fragment_fn) 1460 # Remove fragment from local-files 1461 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1462 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1463 1464 # Special handling for cml1, ccmake, etc bbclasses that generated 1465 # configuration fragment files that are consumed as source files 1466 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1467 if bb.data.inherits_class(frag_class, rd): 1468 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1469 if os.path.exists(srcpath): 1470 if frag_name not in new_set: 1471 new_set.append(frag_name) 1472 # copy fragment into destdir 1473 shutil.copy2(srcpath, destdir) 1474 # copy fragment into local files if exists 1475 if os.path.isdir(local_files_dir): 1476 shutil.copy2(srcpath, local_files_dir) 1477 1478 if new_set is not None: 1479 for fname in new_set: 1480 if fname in existing_files: 1481 origpath = existing_files.pop(fname) 1482 workpath = os.path.join(local_files_dir, fname) 1483 if not filecmp.cmp(origpath, workpath): 1484 updated[fname] = origpath 1485 elif fname != '.gitignore': 1486 added[fname] = None 1487 1488 workdir = rd.getVar('WORKDIR') 1489 s = rd.getVar('S') 1490 if not s.endswith(os.sep): 1491 s += os.sep 1492 1493 if workdir != s: 1494 # Handle files where subdir= was specified 1495 for fname in list(existing_files.keys()): 1496 # FIXME handle both subdir starting with BP and not? 1497 fworkpath = os.path.join(workdir, fname) 1498 if fworkpath.startswith(s): 1499 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1500 if os.path.exists(fpath): 1501 origpath = existing_files.pop(fname) 1502 if not filecmp.cmp(origpath, fpath): 1503 updated[fpath] = origpath 1504 1505 removed = existing_files 1506 return (updated, added, removed) 1507 1508 1509def _determine_files_dir(rd): 1510 """Determine the appropriate files directory for a recipe""" 1511 recipedir = rd.getVar('FILE_DIRNAME') 1512 for entry in rd.getVar('FILESPATH').split(':'): 1513 relpth = os.path.relpath(entry, recipedir) 1514 if not os.sep in relpth: 1515 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1516 if os.path.isdir(entry): 1517 return entry 1518 return os.path.join(recipedir, rd.getVar('BPN')) 1519 1520 1521def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1522 """Implement the 'srcrev' mode of update-recipe""" 1523 import bb 1524 import oe.recipeutils 1525 1526 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1527 1528 recipefile = rd.getVar('FILE') 1529 recipedir = os.path.basename(recipefile) 1530 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1531 1532 # Get HEAD revision 1533 try: 1534 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1535 except bb.process.ExecutionError as err: 1536 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1537 (srctree, err)) 1538 srcrev = stdout.strip() 1539 if len(srcrev) != 40: 1540 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1541 1542 destpath = None 1543 remove_files = [] 1544 patchfields = {} 1545 patchfields['SRCREV'] = srcrev 1546 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1547 srcuri = orig_src_uri.split() 1548 tempdir = tempfile.mkdtemp(prefix='devtool') 1549 update_srcuri = False 1550 appendfile = None 1551 try: 1552 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1553 srctreebase = workspace[recipename]['srctreebase'] 1554 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1555 if not no_remove: 1556 # Find list of existing patches in recipe file 1557 patches_dir = tempfile.mkdtemp(dir=tempdir) 1558 old_srcrev = rd.getVar('SRCREV') or '' 1559 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1560 patches_dir) 1561 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1562 1563 # Remove deleted local files and "overlapping" patches 1564 remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) 1565 if remove_files: 1566 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1567 update_srcuri = True 1568 1569 if appendlayerdir: 1570 files = dict((os.path.join(local_files_dir, key), val) for 1571 key, val in list(upd_f.items()) + list(new_f.items())) 1572 removevalues = {} 1573 if update_srcuri: 1574 removevalues = {'SRC_URI': removedentries} 1575 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1576 if dry_run_outdir: 1577 logger.info('Creating bbappend (dry-run)') 1578 else: 1579 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1580 rd, appendlayerdir, files, wildcardver=wildcard_version, 1581 extralines=patchfields, removevalues=removevalues, 1582 redirect_output=dry_run_outdir) 1583 else: 1584 files_dir = _determine_files_dir(rd) 1585 for basepath, path in upd_f.items(): 1586 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1587 if os.path.isabs(basepath): 1588 # Original file (probably with subdir pointing inside source tree) 1589 # so we do not want to move it, just copy 1590 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1591 else: 1592 _move_file(os.path.join(local_files_dir, basepath), path, 1593 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1594 update_srcuri= True 1595 for basepath, path in new_f.items(): 1596 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1597 _move_file(os.path.join(local_files_dir, basepath), 1598 os.path.join(files_dir, basepath), 1599 dry_run_outdir=dry_run_outdir, 1600 base_outdir=recipedir) 1601 srcuri.append('file://%s' % basepath) 1602 update_srcuri = True 1603 if update_srcuri: 1604 patchfields['SRC_URI'] = ' '.join(srcuri) 1605 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1606 finally: 1607 shutil.rmtree(tempdir) 1608 if not 'git://' in orig_src_uri: 1609 logger.info('You will need to update SRC_URI within the recipe to ' 1610 'point to a git repository where you have pushed your ' 1611 'changes') 1612 1613 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1614 return True, appendfile, remove_files 1615 1616def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1617 """Implement the 'patch' mode of update-recipe""" 1618 import bb 1619 import oe.recipeutils 1620 1621 recipefile = rd.getVar('FILE') 1622 recipedir = os.path.dirname(recipefile) 1623 append = workspace[recipename]['bbappend'] 1624 if not os.path.exists(append): 1625 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1626 recipename) 1627 srctreebase = workspace[recipename]['srctreebase'] 1628 relpatchdir = os.path.relpath(srctreebase, srctree) 1629 if relpatchdir == '.': 1630 patchdir_params = {} 1631 else: 1632 patchdir_params = {'patchdir': relpatchdir} 1633 1634 def srcuri_entry(basepath): 1635 if patchdir_params: 1636 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1637 else: 1638 paramstr = '' 1639 return 'file://%s%s' % (basepath, paramstr) 1640 1641 initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1642 if not initial_rev: 1643 raise DevtoolError('Unable to find initial revision - please specify ' 1644 'it with --initial-rev') 1645 1646 appendfile = None 1647 dl_dir = rd.getVar('DL_DIR') 1648 if not dl_dir.endswith('/'): 1649 dl_dir += '/' 1650 1651 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1652 1653 tempdir = tempfile.mkdtemp(prefix='devtool') 1654 try: 1655 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1656 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1657 1658 # Get updated patches from source tree 1659 patches_dir = tempfile.mkdtemp(dir=tempdir) 1660 upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, 1661 patches_dir, changed_revs) 1662 # Get all patches from source tree and check if any should be removed 1663 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1664 _, _, del_p = _export_patches(srctree, rd, initial_rev, 1665 all_patches_dir) 1666 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1667 if filter_patches: 1668 new_p = OrderedDict() 1669 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1670 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) 1671 remove_files = [] 1672 if not no_remove: 1673 # Remove deleted local files and patches 1674 remove_files = list(del_f.values()) + list(del_p.values()) 1675 updatefiles = False 1676 updaterecipe = False 1677 destpath = None 1678 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1679 if appendlayerdir: 1680 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1681 key, val in list(upd_f.items()) + list(new_f.items())) 1682 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1683 key, val in list(upd_p.items()) + list(new_p.items()))) 1684 if files or remove_files: 1685 removevalues = None 1686 if remove_files: 1687 removedentries, remaining = _remove_file_entries( 1688 srcuri, remove_files) 1689 if removedentries or remaining: 1690 remaining = [srcuri_entry(os.path.basename(item)) for 1691 item in remaining] 1692 removevalues = {'SRC_URI': removedentries + remaining} 1693 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1694 rd, appendlayerdir, files, 1695 wildcardver=wildcard_version, 1696 removevalues=removevalues, 1697 redirect_output=dry_run_outdir, 1698 params=[patchdir_params] * len(files)) 1699 else: 1700 logger.info('No patches or local source files needed updating') 1701 else: 1702 # Update existing files 1703 files_dir = _determine_files_dir(rd) 1704 for basepath, path in upd_f.items(): 1705 logger.info('Updating file %s' % basepath) 1706 if os.path.isabs(basepath): 1707 # Original file (probably with subdir pointing inside source tree) 1708 # so we do not want to move it, just copy 1709 _copy_file(basepath, path, 1710 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1711 else: 1712 _move_file(os.path.join(local_files_dir, basepath), path, 1713 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1714 updatefiles = True 1715 for basepath, path in upd_p.items(): 1716 patchfn = os.path.join(patches_dir, basepath) 1717 if os.path.dirname(path) + '/' == dl_dir: 1718 # This is a a downloaded patch file - we now need to 1719 # replace the entry in SRC_URI with our local version 1720 logger.info('Replacing remote patch %s with updated local version' % basepath) 1721 path = os.path.join(files_dir, basepath) 1722 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath)) 1723 updaterecipe = True 1724 else: 1725 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1726 _move_file(patchfn, path, 1727 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1728 updatefiles = True 1729 # Add any new files 1730 for basepath, path in new_f.items(): 1731 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1732 _move_file(os.path.join(local_files_dir, basepath), 1733 os.path.join(files_dir, basepath), 1734 dry_run_outdir=dry_run_outdir, 1735 base_outdir=recipedir) 1736 srcuri.append(srcuri_entry(basepath)) 1737 updaterecipe = True 1738 for basepath, path in new_p.items(): 1739 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1740 _move_file(os.path.join(patches_dir, basepath), 1741 os.path.join(files_dir, basepath), 1742 dry_run_outdir=dry_run_outdir, 1743 base_outdir=recipedir) 1744 srcuri.append(srcuri_entry(basepath)) 1745 updaterecipe = True 1746 # Update recipe, if needed 1747 if _remove_file_entries(srcuri, remove_files)[0]: 1748 updaterecipe = True 1749 if updaterecipe: 1750 if not dry_run_outdir: 1751 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1752 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1753 {'SRC_URI': ' '.join(srcuri)}, 1754 redirect_output=dry_run_outdir) 1755 elif not updatefiles: 1756 # Neither patches nor recipe were updated 1757 logger.info('No patches or files need updating') 1758 return False, None, [] 1759 finally: 1760 shutil.rmtree(tempdir) 1761 1762 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1763 return True, appendfile, remove_files 1764 1765def _guess_recipe_update_mode(srctree, rdata): 1766 """Guess the recipe update mode to use""" 1767 src_uri = (rdata.getVar('SRC_URI') or '').split() 1768 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1769 if not git_uris: 1770 return 'patch' 1771 # Just use the first URI for now 1772 uri = git_uris[0] 1773 # Check remote branch 1774 params = bb.fetch.decodeurl(uri)[5] 1775 upstr_branch = params['branch'] if 'branch' in params else 'master' 1776 # Check if current branch HEAD is found in upstream branch 1777 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1778 head_rev = stdout.rstrip() 1779 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1780 cwd=srctree) 1781 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1782 if 'origin/' + upstr_branch in remote_brs: 1783 return 'srcrev' 1784 1785 return 'patch' 1786 1787def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1788 srctree = workspace[recipename]['srctree'] 1789 if mode == 'auto': 1790 mode = _guess_recipe_update_mode(srctree, rd) 1791 1792 override_branches = [] 1793 mainbranch = None 1794 startbranch = None 1795 if not no_overrides: 1796 stdout, _ = bb.process.run('git branch', cwd=srctree) 1797 other_branches = [] 1798 for line in stdout.splitlines(): 1799 branchname = line[2:] 1800 if line.startswith('* '): 1801 startbranch = branchname 1802 if branchname.startswith(override_branch_prefix): 1803 override_branches.append(branchname) 1804 else: 1805 other_branches.append(branchname) 1806 1807 if override_branches: 1808 logger.debug('_update_recipe: override branches: %s' % override_branches) 1809 logger.debug('_update_recipe: other branches: %s' % other_branches) 1810 if startbranch.startswith(override_branch_prefix): 1811 if len(other_branches) == 1: 1812 mainbranch = other_branches[1] 1813 else: 1814 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1815 else: 1816 mainbranch = startbranch 1817 1818 checkedout = None 1819 anyupdated = False 1820 appendfile = None 1821 allremoved = [] 1822 if override_branches: 1823 logger.info('Handling main branch (%s)...' % mainbranch) 1824 if startbranch != mainbranch: 1825 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1826 checkedout = mainbranch 1827 try: 1828 branchlist = [mainbranch] + override_branches 1829 for branch in branchlist: 1830 crd = bb.data.createCopy(rd) 1831 if branch != mainbranch: 1832 logger.info('Handling branch %s...' % branch) 1833 override = branch[len(override_branch_prefix):] 1834 crd.appendVar('OVERRIDES', ':%s' % override) 1835 bb.process.run('git checkout %s' % branch, cwd=srctree) 1836 checkedout = branch 1837 1838 if mode == 'srcrev': 1839 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1840 elif mode == 'patch': 1841 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1842 else: 1843 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1844 if updated: 1845 anyupdated = True 1846 if appendf: 1847 appendfile = appendf 1848 allremoved.extend(removed) 1849 finally: 1850 if startbranch and checkedout != startbranch: 1851 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1852 1853 return anyupdated, appendfile, allremoved 1854 1855def update_recipe(args, config, basepath, workspace): 1856 """Entry point for the devtool 'update-recipe' subcommand""" 1857 check_workspace_recipe(workspace, args.recipename) 1858 1859 if args.append: 1860 if not os.path.exists(args.append): 1861 raise DevtoolError('bbappend destination layer directory "%s" ' 1862 'does not exist' % args.append) 1863 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1864 raise DevtoolError('conf/layer.conf not found in bbappend ' 1865 'destination layer "%s"' % args.append) 1866 1867 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1868 try: 1869 1870 rd = parse_recipe(config, tinfoil, args.recipename, True) 1871 if not rd: 1872 return 1 1873 1874 dry_run_output = None 1875 dry_run_outdir = None 1876 if args.dry_run: 1877 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1878 dry_run_outdir = dry_run_output.name 1879 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1880 1881 if updated: 1882 rf = rd.getVar('FILE') 1883 if rf.startswith(config.workspace_path): 1884 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1885 finally: 1886 tinfoil.shutdown() 1887 1888 return 0 1889 1890 1891def status(args, config, basepath, workspace): 1892 """Entry point for the devtool 'status' subcommand""" 1893 if workspace: 1894 for recipe, value in sorted(workspace.items()): 1895 recipefile = value['recipefile'] 1896 if recipefile: 1897 recipestr = ' (%s)' % recipefile 1898 else: 1899 recipestr = '' 1900 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1901 else: 1902 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1903 return 0 1904 1905 1906def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1907 """Reset one or more recipes""" 1908 import oe.path 1909 1910 def clean_preferred_provider(pn, layerconf_path): 1911 """Remove PREFERRED_PROVIDER from layer.conf'""" 1912 import re 1913 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1914 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1915 pprovider_found = False 1916 with open(layerconf_file, 'r') as f: 1917 lines = f.readlines() 1918 with open(new_layerconf_file, 'a') as nf: 1919 for line in lines: 1920 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1921 if not re.match(pprovider_exp, line): 1922 nf.write(line) 1923 else: 1924 pprovider_found = True 1925 if pprovider_found: 1926 shutil.move(new_layerconf_file, layerconf_file) 1927 else: 1928 os.remove(new_layerconf_file) 1929 1930 if recipes and not no_clean: 1931 if len(recipes) == 1: 1932 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 1933 else: 1934 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 1935 # If the recipe file itself was created in the workspace, and 1936 # it uses BBCLASSEXTEND, then we need to also clean the other 1937 # variants 1938 targets = [] 1939 for recipe in recipes: 1940 targets.append(recipe) 1941 recipefile = workspace[recipe]['recipefile'] 1942 if recipefile and os.path.exists(recipefile): 1943 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 1944 try: 1945 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 1946 except bb.process.ExecutionError as e: 1947 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 1948 'wish, you may specify -n/--no-clean to ' 1949 'skip running this command when resetting' % 1950 (e.command, e.stdout)) 1951 1952 for pn in recipes: 1953 _check_preserve(config, pn) 1954 1955 appendfile = workspace[pn]['bbappend'] 1956 if os.path.exists(appendfile): 1957 # This shouldn't happen, but is possible if devtool errored out prior to 1958 # writing the md5 file. We need to delete this here or the recipe won't 1959 # actually be reset 1960 os.remove(appendfile) 1961 1962 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 1963 def preservedir(origdir): 1964 if os.path.exists(origdir): 1965 for root, dirs, files in os.walk(origdir): 1966 for fn in files: 1967 logger.warning('Preserving %s in %s' % (fn, preservepath)) 1968 _move_file(os.path.join(origdir, fn), 1969 os.path.join(preservepath, fn)) 1970 for dn in dirs: 1971 preservedir(os.path.join(root, dn)) 1972 os.rmdir(origdir) 1973 1974 recipefile = workspace[pn]['recipefile'] 1975 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 1976 # This should always be true if recipefile is set, but just in case 1977 preservedir(os.path.dirname(recipefile)) 1978 # We don't automatically create this dir next to appends, but the user can 1979 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 1980 1981 srctreebase = workspace[pn]['srctreebase'] 1982 if os.path.isdir(srctreebase): 1983 if os.listdir(srctreebase): 1984 if remove_work: 1985 logger.info('-r argument used on %s, removing source tree.' 1986 ' You will lose any unsaved work' %pn) 1987 shutil.rmtree(srctreebase) 1988 else: 1989 # We don't want to risk wiping out any work in progress 1990 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): 1991 from datetime import datetime 1992 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn,datetime.now().strftime("%Y%m%d%H%M%S"))) 1993 logger.info('Preserving source tree in %s\nIf you no ' 1994 'longer need it then please delete it manually.\n' 1995 'It is also possible to reuse it via devtool source tree argument.' 1996 % preservesrc) 1997 bb.utils.mkdirhier(os.path.dirname(preservesrc)) 1998 shutil.move(srctreebase, preservesrc) 1999 else: 2000 logger.info('Leaving source tree %s as-is; if you no ' 2001 'longer need it then please delete it manually' 2002 % srctreebase) 2003 else: 2004 # This is unlikely, but if it's empty we can just remove it 2005 os.rmdir(srctreebase) 2006 2007 clean_preferred_provider(pn, config.workspace_path) 2008 2009def reset(args, config, basepath, workspace): 2010 """Entry point for the devtool 'reset' subcommand""" 2011 import bb 2012 import shutil 2013 2014 recipes = "" 2015 2016 if args.recipename: 2017 if args.all: 2018 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 2019 else: 2020 for recipe in args.recipename: 2021 check_workspace_recipe(workspace, recipe, checksrc=False) 2022 elif not args.all: 2023 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2024 "reset all recipes") 2025 if args.all: 2026 recipes = list(workspace.keys()) 2027 else: 2028 recipes = args.recipename 2029 2030 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2031 2032 return 0 2033 2034 2035def _get_layer(layername, d): 2036 """Determine the base layer path for the specified layer name/path""" 2037 layerdirs = d.getVar('BBLAYERS').split() 2038 layers = {} # {basename: layer_paths} 2039 for p in layerdirs: 2040 bn = os.path.basename(p) 2041 if bn not in layers: 2042 layers[bn] = [p] 2043 else: 2044 layers[bn].append(p) 2045 # Provide some shortcuts 2046 if layername.lower() in ['oe-core', 'openembedded-core']: 2047 layername = 'meta' 2048 layer_paths = layers.get(layername, None) 2049 if not layer_paths: 2050 return os.path.abspath(layername) 2051 elif len(layer_paths) == 1: 2052 return os.path.abspath(layer_paths[0]) 2053 else: 2054 # multiple layers having the same base name 2055 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 2056 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 2057 return os.path.abspath(layer_paths[0]) 2058 2059 2060def finish(args, config, basepath, workspace): 2061 """Entry point for the devtool 'finish' subcommand""" 2062 import bb 2063 import oe.recipeutils 2064 2065 check_workspace_recipe(workspace, args.recipename) 2066 2067 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2068 2069 # Grab the equivalent of COREBASE without having to initialise tinfoil 2070 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2071 2072 srctree = workspace[args.recipename]['srctree'] 2073 check_git_repo_op(srctree, [corebasedir]) 2074 dirty = check_git_repo_dirty(srctree) 2075 if dirty: 2076 if args.force: 2077 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2078 else: 2079 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2080 2081 no_clean = args.no_clean 2082 remove_work=args.remove_work 2083 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2084 try: 2085 rd = parse_recipe(config, tinfoil, args.recipename, True) 2086 if not rd: 2087 return 1 2088 2089 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2090 recipefile = rd.getVar('FILE') 2091 recipedir = os.path.dirname(recipefile) 2092 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2093 2094 if not os.path.isdir(destlayerdir): 2095 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2096 2097 if os.path.abspath(destlayerdir) == config.workspace_path: 2098 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2099 2100 # If it's an upgrade, grab the original path 2101 origpath = None 2102 origfilelist = None 2103 append = workspace[args.recipename]['bbappend'] 2104 with open(append, 'r') as f: 2105 for line in f: 2106 if line.startswith('# original_path:'): 2107 origpath = line.split(':')[1].strip() 2108 elif line.startswith('# original_files:'): 2109 origfilelist = line.split(':')[1].split() 2110 2111 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2112 2113 if origlayerdir == config.workspace_path: 2114 # Recipe file itself is in workspace, update it there first 2115 appendlayerdir = None 2116 origrelpath = None 2117 if origpath: 2118 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2119 if origlayerpath: 2120 origrelpath = os.path.relpath(origpath, origlayerpath) 2121 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2122 if not destpath: 2123 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2124 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2125 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2126 if not os.path.abspath(destlayerbasedir) in layerdirs: 2127 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2128 2129 elif destlayerdir == origlayerdir: 2130 # Same layer, update the original recipe 2131 appendlayerdir = None 2132 destpath = None 2133 else: 2134 # Create/update a bbappend in the specified layer 2135 appendlayerdir = destlayerdir 2136 destpath = None 2137 2138 # Actually update the recipe / bbappend 2139 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2140 dry_run_output = None 2141 dry_run_outdir = None 2142 if args.dry_run: 2143 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2144 dry_run_outdir = dry_run_output.name 2145 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2146 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2147 2148 # Remove any old files in the case of an upgrade 2149 if removing_original: 2150 for fn in origfilelist: 2151 fnp = os.path.join(origpath, fn) 2152 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2153 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2154 if not args.dry_run: 2155 try: 2156 os.remove(fnp) 2157 except FileNotFoundError: 2158 pass 2159 2160 if origlayerdir == config.workspace_path and destpath: 2161 # Recipe file itself is in the workspace - need to move it and any 2162 # associated files to the specified layer 2163 no_clean = True 2164 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2165 for root, _, files in os.walk(recipedir): 2166 for fn in files: 2167 srcpath = os.path.join(root, fn) 2168 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2169 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2170 destfp = os.path.join(destdir, fn) 2171 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2172 2173 if dry_run_outdir: 2174 import difflib 2175 comparelist = [] 2176 for root, _, files in os.walk(dry_run_outdir): 2177 for fn in files: 2178 outf = os.path.join(root, fn) 2179 relf = os.path.relpath(outf, dry_run_outdir) 2180 logger.debug('dry-run: output file %s' % relf) 2181 if fn.endswith('.bb'): 2182 if origfilelist and origpath and destpath: 2183 # Need to match this up with the pre-upgrade recipe file 2184 for origf in origfilelist: 2185 if origf.endswith('.bb'): 2186 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2187 outf, 2188 os.path.abspath(os.path.join(destpath, relf)))) 2189 break 2190 else: 2191 # Compare to the existing recipe 2192 comparelist.append((recipefile, outf, recipefile)) 2193 elif fn.endswith('.bbappend'): 2194 if appendfile: 2195 if os.path.exists(appendfile): 2196 comparelist.append((appendfile, outf, appendfile)) 2197 else: 2198 comparelist.append((None, outf, appendfile)) 2199 else: 2200 if destpath: 2201 recipedest = destpath 2202 elif appendfile: 2203 recipedest = os.path.dirname(appendfile) 2204 else: 2205 recipedest = os.path.dirname(recipefile) 2206 destfp = os.path.join(recipedest, relf) 2207 if os.path.exists(destfp): 2208 comparelist.append((destfp, outf, destfp)) 2209 output = '' 2210 for oldfile, newfile, newfileshow in comparelist: 2211 if oldfile: 2212 with open(oldfile, 'r') as f: 2213 oldlines = f.readlines() 2214 else: 2215 oldfile = '/dev/null' 2216 oldlines = [] 2217 with open(newfile, 'r') as f: 2218 newlines = f.readlines() 2219 if not newfileshow: 2220 newfileshow = newfile 2221 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2222 difflines = list(diff) 2223 if difflines: 2224 output += ''.join(difflines) 2225 if output: 2226 logger.info('Diff of changed files:\n%s' % output) 2227 finally: 2228 tinfoil.shutdown() 2229 2230 # Everything else has succeeded, we can now reset 2231 if args.dry_run: 2232 logger.info('Resetting recipe (dry-run)') 2233 else: 2234 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2235 2236 return 0 2237 2238 2239def get_default_srctree(config, recipename=''): 2240 """Get the default srctree path""" 2241 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2242 if recipename: 2243 return os.path.join(srctreeparent, 'sources', recipename) 2244 else: 2245 return os.path.join(srctreeparent, 'sources') 2246 2247def register_commands(subparsers, context): 2248 """Register devtool subcommands from this plugin""" 2249 2250 defsrctree = get_default_srctree(context.config) 2251 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2252 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2253 group='starting', order=100) 2254 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2255 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2256 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2257 group = parser_add.add_mutually_exclusive_group() 2258 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2259 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2260 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2261 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2262 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2263 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2264 group = parser_add.add_mutually_exclusive_group() 2265 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2266 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2267 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2268 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2269 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2270 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2271 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2272 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2273 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2274 2275 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2276 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2277 group='starting', order=90) 2278 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2279 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2280 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2281 group = parser_modify.add_mutually_exclusive_group() 2282 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2283 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2284 group = parser_modify.add_mutually_exclusive_group() 2285 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2286 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2287 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2288 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2289 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2290 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2291 2292 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2293 description='Extracts the source for an existing recipe', 2294 group='advanced') 2295 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2296 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2297 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2298 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2299 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2300 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2301 2302 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2303 description='Synchronize the previously extracted source tree for an existing recipe', 2304 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2305 group='advanced') 2306 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2307 parser_sync.add_argument('srctree', help='Path to the source tree') 2308 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2309 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2310 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2311 2312 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2313 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2314 group='working', order=10) 2315 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2316 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2317 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2318 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2319 parser_rename.set_defaults(func=rename) 2320 2321 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2322 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2323 group='working', order=-90) 2324 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2325 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2326 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2327 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2328 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2329 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2330 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2331 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2332 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2333 parser_update_recipe.set_defaults(func=update_recipe) 2334 2335 parser_status = subparsers.add_parser('status', help='Show workspace status', 2336 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2337 group='info', order=100) 2338 parser_status.set_defaults(func=status) 2339 2340 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2341 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2342 group='working', order=-100) 2343 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2344 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2345 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2346 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2347 parser_reset.set_defaults(func=reset) 2348 2349 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2350 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2351 group='working', order=-100) 2352 parser_finish.add_argument('recipename', help='Recipe to finish') 2353 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2354 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2355 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2356 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2357 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2358 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2359 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2360 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2361 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2362 parser_finish.set_defaults(func=finish) 2363