1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.npm_dev: 149 extracmdopts += ' --npm-dev' 150 if args.no_pypi: 151 extracmdopts += ' --no-pypi' 152 if args.mirrors: 153 extracmdopts += ' --mirrors' 154 if args.srcrev: 155 extracmdopts += ' --srcrev %s' % args.srcrev 156 if args.srcbranch: 157 extracmdopts += ' --srcbranch %s' % args.srcbranch 158 if args.provides: 159 extracmdopts += ' --provides %s' % args.provides 160 161 tempdir = tempfile.mkdtemp(prefix='devtool') 162 try: 163 try: 164 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 165 except bb.process.ExecutionError as e: 166 if e.exitcode == 15: 167 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 168 else: 169 raise DevtoolError('Command \'%s\' failed' % e.command) 170 171 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 172 if recipes: 173 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 174 if recipename in workspace: 175 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 176 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 177 bb.utils.mkdirhier(recipedir) 178 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 179 appendfile = recipe_to_append(recipefile, config) 180 if os.path.exists(appendfile): 181 # This shouldn't be possible, but just in case 182 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 183 if os.path.exists(recipefile): 184 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 185 if tmpsrcdir: 186 srctree = os.path.join(srctreeparent, recipename) 187 if os.path.exists(tmpsrcdir): 188 if os.path.exists(srctree): 189 if os.path.isdir(srctree): 190 try: 191 os.rmdir(srctree) 192 except OSError as e: 193 if e.errno == errno.ENOTEMPTY: 194 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 195 else: 196 raise 197 else: 198 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 199 logger.info('Using default source tree path %s' % srctree) 200 shutil.move(tmpsrcdir, srctree) 201 else: 202 raise DevtoolError('Couldn\'t find source tree created by recipetool') 203 bb.utils.mkdirhier(recipedir) 204 shutil.move(recipes[0], recipefile) 205 # Move any additional files created by recipetool 206 for fn in os.listdir(tempdir): 207 shutil.move(os.path.join(tempdir, fn), recipedir) 208 else: 209 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 210 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 211 if os.path.exists(attic_recipe): 212 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 213 finally: 214 if tmpsrcdir and os.path.exists(tmpsrcdir): 215 shutil.rmtree(tmpsrcdir) 216 shutil.rmtree(tempdir) 217 218 for fn in os.listdir(recipedir): 219 _add_md5(config, recipename, os.path.join(recipedir, fn)) 220 221 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 222 try: 223 try: 224 rd = tinfoil.parse_recipe_file(recipefile, False) 225 except Exception as e: 226 logger.error(str(e)) 227 rd = None 228 if not rd: 229 # Parsing failed. We just created this recipe and we shouldn't 230 # leave it in the workdir or it'll prevent bitbake from starting 231 movefn = '%s.parsefailed' % recipefile 232 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 233 shutil.move(recipefile, movefn) 234 return 1 235 236 if args.fetchuri and not args.no_git: 237 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 238 239 initial_rev = {} 240 if os.path.exists(os.path.join(srctree, '.git')): 241 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 242 initial_rev["."] = stdout.rstrip() 243 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree) 244 for line in stdout.splitlines(): 245 (rev, submodule) = line.split() 246 initial_rev[os.path.relpath(submodule, srctree)] = rev 247 248 if args.src_subdir: 249 srctree = os.path.join(srctree, args.src_subdir) 250 251 bb.utils.mkdirhier(os.path.dirname(appendfile)) 252 with open(appendfile, 'w') as f: 253 f.write('inherit externalsrc\n') 254 f.write('EXTERNALSRC = "%s"\n' % srctree) 255 256 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 257 if b_is_s: 258 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 259 if initial_rev: 260 for key, value in initial_rev.items(): 261 f.write('\n# initial_rev %s: %s\n' % (key, value)) 262 263 if args.binary: 264 f.write('do_install:append() {\n') 265 f.write(' rm -rf ${D}/.git\n') 266 f.write(' rm -f ${D}/singletask.lock\n') 267 f.write('}\n') 268 269 if bb.data.inherits_class('npm', rd): 270 f.write('python do_configure:append() {\n') 271 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 272 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 273 f.write(' bb.utils.remove(lockfile)\n') 274 f.write('}\n') 275 276 # Check if the new layer provides recipes whose priorities have been 277 # overriden by PREFERRED_PROVIDER. 278 recipe_name = rd.getVar('PN') 279 provides = rd.getVar('PROVIDES') 280 # Search every item defined in PROVIDES 281 for recipe_provided in provides.split(): 282 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 283 current_pprovider = rd.getVar(preferred_provider) 284 if current_pprovider and current_pprovider != recipe_name: 285 if args.fixed_setup: 286 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 287 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 288 with open(layerconf_file, 'a') as f: 289 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 290 else: 291 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 292 break 293 294 _add_md5(config, recipename, appendfile) 295 296 check_prerelease_version(rd.getVar('PV'), 'devtool add') 297 298 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 299 300 finally: 301 tinfoil.shutdown() 302 303 return 0 304 305 306def _check_compatible_recipe(pn, d): 307 """Check if the recipe is supported by devtool""" 308 if pn == 'perf': 309 raise DevtoolError("The perf recipe does not actually check out " 310 "source and thus cannot be supported by this tool", 311 4) 312 313 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 314 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 315 316 if bb.data.inherits_class('image', d): 317 raise DevtoolError("The %s recipe is an image, and therefore is not " 318 "supported by this tool" % pn, 4) 319 320 if bb.data.inherits_class('populate_sdk', d): 321 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 322 "supported by this tool" % pn, 4) 323 324 if bb.data.inherits_class('packagegroup', d): 325 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 326 "not supported by this tool" % pn, 4) 327 328 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 329 # Not an incompatibility error per se, so we don't pass the error code 330 raise DevtoolError("externalsrc is currently enabled for the %s " 331 "recipe. This prevents the normal do_patch task " 332 "from working. You will need to disable this " 333 "first." % pn) 334 335def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 336 """Common function for copying a file to the dry run output directory""" 337 relpath = os.path.relpath(dst, base_outdir) 338 if relpath.startswith('..'): 339 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 340 dst = os.path.join(dry_run_outdir, relpath) 341 dst_d = os.path.dirname(dst) 342 if dst_d: 343 bb.utils.mkdirhier(dst_d) 344 # Don't overwrite existing files, otherwise in the case of an upgrade 345 # the dry-run written out recipe will be overwritten with an unmodified 346 # version 347 if not os.path.exists(dst): 348 shutil.copy(src, dst) 349 350def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 351 """Move a file. Creates all the directory components of destination path.""" 352 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 353 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 354 if dry_run_outdir: 355 # We want to copy here, not move 356 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 357 else: 358 dst_d = os.path.dirname(dst) 359 if dst_d: 360 bb.utils.mkdirhier(dst_d) 361 shutil.move(src, dst) 362 363def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 364 """Copy a file. Creates all the directory components of destination path.""" 365 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 366 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 367 if dry_run_outdir: 368 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 369 else: 370 dst_d = os.path.dirname(dst) 371 if dst_d: 372 bb.utils.mkdirhier(dst_d) 373 shutil.copy(src, dst) 374 375def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 376 """List contents of a git treeish""" 377 import bb 378 cmd = ['git', 'ls-tree', '-z', treeish] 379 if recursive: 380 cmd.append('-r') 381 out, _ = bb.process.run(cmd, cwd=repodir) 382 ret = {} 383 if out: 384 for line in out.split('\0'): 385 if line: 386 split = line.split(None, 4) 387 ret[split[3]] = split[0:3] 388 return ret 389 390def _git_exclude_path(srctree, path): 391 """Return pathspec (list of paths) that excludes certain path""" 392 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 393 # we don't catch files that are deleted, for example. A more reliable way 394 # to implement this would be to use "negative pathspecs" which were 395 # introduced in Git v1.9.0. Revisit this when/if the required Git version 396 # becomes greater than that. 397 path = os.path.normpath(path) 398 recurse = True if len(path.split(os.path.sep)) > 1 else False 399 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 400 if path in git_files: 401 git_files.remove(path) 402 return git_files 403 else: 404 return ['.'] 405 406def _ls_tree(directory): 407 """Recursive listing of files in a directory""" 408 ret = [] 409 for root, dirs, files in os.walk(directory): 410 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 411 fname in files]) 412 return ret 413 414 415def extract(args, config, basepath, workspace): 416 """Entry point for the devtool 'extract' subcommand""" 417 import bb 418 419 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 420 if not tinfoil: 421 # Error already shown 422 return 1 423 try: 424 rd = parse_recipe(config, tinfoil, args.recipename, True) 425 if not rd: 426 return 1 427 428 srctree = os.path.abspath(args.srctree) 429 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 430 logger.info('Source tree extracted to %s' % srctree) 431 432 if initial_rev: 433 return 0 434 else: 435 return 1 436 finally: 437 tinfoil.shutdown() 438 439def sync(args, config, basepath, workspace): 440 """Entry point for the devtool 'sync' subcommand""" 441 import bb 442 443 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 444 if not tinfoil: 445 # Error already shown 446 return 1 447 try: 448 rd = parse_recipe(config, tinfoil, args.recipename, True) 449 if not rd: 450 return 1 451 452 srctree = os.path.abspath(args.srctree) 453 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 454 logger.info('Source tree %s synchronized' % srctree) 455 456 if initial_rev: 457 return 0 458 else: 459 return 1 460 finally: 461 tinfoil.shutdown() 462 463def symlink_oelocal_files_srctree(rd, srctree): 464 import oe.patch 465 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 466 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 467 # (otherwise the recipe won't build as expected) 468 local_files_dir = os.path.join(srctree, 'oe-local-files') 469 addfiles = [] 470 for root, _, files in os.walk(local_files_dir): 471 relpth = os.path.relpath(root, local_files_dir) 472 if relpth != '.': 473 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 474 for fn in files: 475 if fn == '.gitignore': 476 continue 477 destpth = os.path.join(srctree, relpth, fn) 478 if os.path.exists(destpth): 479 os.unlink(destpth) 480 if relpth != '.': 481 back_relpth = os.path.relpath(local_files_dir, root) 482 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) 483 else: 484 os.symlink('oe-local-files/%s' % fn, destpth) 485 addfiles.append(os.path.join(relpth, fn)) 486 if addfiles: 487 oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd) 488 489def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 490 """Extract sources of a recipe""" 491 import oe.recipeutils 492 import oe.patch 493 import oe.path 494 495 pn = d.getVar('PN') 496 497 _check_compatible_recipe(pn, d) 498 499 if sync: 500 if not os.path.exists(srctree): 501 raise DevtoolError("output path %s does not exist" % srctree) 502 else: 503 if os.path.exists(srctree): 504 if not os.path.isdir(srctree): 505 raise DevtoolError("output path %s exists and is not a directory" % 506 srctree) 507 elif os.listdir(srctree): 508 raise DevtoolError("output path %s already exists and is " 509 "non-empty" % srctree) 510 511 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 512 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 513 "extract source" % pn, 4) 514 515 if not sync: 516 # Prepare for shutil.move later on 517 bb.utils.mkdirhier(srctree) 518 os.rmdir(srctree) 519 520 extra_overrides = [] 521 if not no_overrides: 522 history = d.varhistory.variable('SRC_URI') 523 for event in history: 524 if not 'flag' in event: 525 if event['op'].startswith((':append[', ':prepend[')): 526 override = event['op'].split('[')[1].split(']')[0] 527 if not override.startswith('pn-'): 528 extra_overrides.append(override) 529 # We want to remove duplicate overrides. If a recipe had multiple 530 # SRC_URI_override += values it would cause mulitple instances of 531 # overrides. This doesn't play nicely with things like creating a 532 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 533 extra_overrides = list(set(extra_overrides)) 534 if extra_overrides: 535 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 536 537 initial_rev = None 538 539 recipefile = d.getVar('FILE') 540 appendfile = recipe_to_append(recipefile, config) 541 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 542 543 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 544 # directory so that: 545 # (a) we pick up all files that get unpacked to the WORKDIR, and 546 # (b) we don't disturb the existing build 547 # However, with recipe-specific sysroots the sysroots for the recipe 548 # will be prepared under WORKDIR, and if we used the system temporary 549 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 550 # our attempts to hardlink files into the recipe-specific sysroots 551 # will fail on systems where /tmp is a different filesystem, and it 552 # would have to fall back to copying the files which is a waste of 553 # time. Put the temp directory under the WORKDIR to prevent that from 554 # being a problem. 555 tempbasedir = d.getVar('WORKDIR') 556 bb.utils.mkdirhier(tempbasedir) 557 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 558 try: 559 tinfoil.logger.setLevel(logging.WARNING) 560 561 # FIXME this results in a cache reload under control of tinfoil, which is fine 562 # except we don't get the knotty progress bar 563 564 if os.path.exists(appendfile): 565 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 566 shutil.copyfile(appendfile, appendbackup) 567 else: 568 appendbackup = None 569 bb.utils.mkdirhier(os.path.dirname(appendfile)) 570 logger.debug('writing append file %s' % appendfile) 571 with open(appendfile, 'a') as f: 572 f.write('###--- _extract_source\n') 573 f.write('deltask do_recipe_qa\n') 574 f.write('deltask do_recipe_qa_setscene\n') 575 f.write('ERROR_QA:remove = "patch-fuzz"\n') 576 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 577 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 578 if not is_kernel_yocto: 579 f.write('PATCHTOOL = "git"\n') 580 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 581 if extra_overrides: 582 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 583 f.write('inherit devtool-source\n') 584 f.write('###--- _extract_source\n') 585 586 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 587 588 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 589 bb.utils.mkdirhier(sstate_manifests) 590 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 591 with open(preservestampfile, 'w') as f: 592 f.write(d.getVar('STAMP')) 593 tinfoil.modified_files() 594 try: 595 if is_kernel_yocto: 596 # We need to generate the kernel config 597 task = 'do_configure' 598 else: 599 task = 'do_patch' 600 601 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 602 logger.info('The %s recipe has %s disabled. Running only ' 603 'do_configure task dependencies' % (pn, task)) 604 605 if 'depends' in d.getVarFlags('do_configure', False): 606 pn = d.getVarFlags('do_configure', False)['depends'] 607 pn = pn.replace('${PV}', d.getVar('PV')) 608 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 609 task = None 610 611 # Run the fetch + unpack tasks 612 res = tinfoil.build_targets(pn, 613 task, 614 handle_events=True) 615 finally: 616 if os.path.exists(preservestampfile): 617 os.remove(preservestampfile) 618 619 if not res: 620 raise DevtoolError('Extracting source for %s failed' % pn) 621 622 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 623 workshareddir = d.getVar('S') 624 if os.path.islink(srctree): 625 os.unlink(srctree) 626 627 os.symlink(workshareddir, srctree) 628 629 # The initial_rev file is created in devtool_post_unpack function that will not be executed if 630 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 631 return True, True 632 633 try: 634 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 635 initial_rev = f.read() 636 637 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 638 srcsubdir = f.read() 639 except FileNotFoundError as e: 640 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 641 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 642 643 # Check if work-shared is empty, if yes 644 # find source and copy to work-shared 645 if is_kernel_yocto: 646 workshareddir = d.getVar('STAGING_KERNEL_DIR') 647 staging_kerVer = get_staging_kver(workshareddir) 648 kernelVersion = d.getVar('LINUX_VERSION') 649 650 # handle dangling symbolic link in work-shared: 651 if os.path.islink(workshareddir): 652 os.unlink(workshareddir) 653 654 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 655 shutil.rmtree(workshareddir) 656 oe.path.copyhardlinktree(srcsubdir, workshareddir) 657 elif not os.path.exists(workshareddir): 658 oe.path.copyhardlinktree(srcsubdir, workshareddir) 659 660 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 661 srctree_localdir = os.path.join(srctree, 'oe-local-files') 662 663 if sync: 664 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 665 666 # Move the oe-local-files directory to srctree. 667 # As oe-local-files is not part of the constructed git tree, 668 # removing it directly during the synchronization might surprise 669 # the user. Instead, we move it to oe-local-files.bak and remind 670 # the user in the log message. 671 if os.path.exists(srctree_localdir + '.bak'): 672 shutil.rmtree(srctree_localdir + '.bak') 673 674 if os.path.exists(srctree_localdir): 675 logger.info('Backing up current local file directory %s' % srctree_localdir) 676 shutil.move(srctree_localdir, srctree_localdir + '.bak') 677 678 if os.path.exists(tempdir_localdir): 679 logger.info('Syncing local source files to srctree...') 680 shutil.copytree(tempdir_localdir, srctree_localdir) 681 else: 682 # Move oe-local-files directory to srctree 683 if os.path.exists(tempdir_localdir): 684 logger.info('Adding local source files to srctree...') 685 shutil.move(tempdir_localdir, srcsubdir) 686 687 shutil.move(srcsubdir, srctree) 688 symlink_oelocal_files_srctree(d, srctree) 689 690 if is_kernel_yocto: 691 logger.info('Copying kernel config to srctree') 692 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 693 694 finally: 695 if appendbackup: 696 shutil.copyfile(appendbackup, appendfile) 697 elif os.path.exists(appendfile): 698 os.remove(appendfile) 699 if keep_temp: 700 logger.info('Preserving temporary directory %s' % tempdir) 701 else: 702 shutil.rmtree(tempdir) 703 return initial_rev, srcsubdir_rel 704 705def _add_md5(config, recipename, filename): 706 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 707 import bb.utils 708 709 def addfile(fn): 710 md5 = bb.utils.md5_file(fn) 711 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 712 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 713 f.seek(0, os.SEEK_SET) 714 if not md5_str in f.read(): 715 f.write(md5_str) 716 717 if os.path.isdir(filename): 718 for root, _, files in os.walk(filename): 719 for f in files: 720 addfile(os.path.join(root, f)) 721 else: 722 addfile(filename) 723 724def _check_preserve(config, recipename): 725 """Check if a file was manually changed and needs to be saved in 'attic' 726 directory""" 727 import bb.utils 728 origfile = os.path.join(config.workspace_path, '.devtool_md5') 729 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 730 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 731 with open(origfile, 'r') as f: 732 with open(newfile, 'w') as tf: 733 for line in f.readlines(): 734 splitline = line.rstrip().split('|') 735 if splitline[0] == recipename: 736 removefile = os.path.join(config.workspace_path, splitline[1]) 737 try: 738 md5 = bb.utils.md5_file(removefile) 739 except IOError as err: 740 if err.errno == 2: 741 # File no longer exists, skip it 742 continue 743 else: 744 raise 745 if splitline[2] != md5: 746 bb.utils.mkdirhier(preservepath) 747 preservefile = os.path.basename(removefile) 748 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 749 shutil.move(removefile, os.path.join(preservepath, preservefile)) 750 else: 751 os.remove(removefile) 752 else: 753 tf.write(line) 754 bb.utils.rename(newfile, origfile) 755 756def get_staging_kver(srcdir): 757 # Kernel version from work-shared 758 kerver = [] 759 staging_kerVer="" 760 if os.path.exists(srcdir) and os.listdir(srcdir): 761 with open(os.path.join(srcdir, "Makefile")) as f: 762 version = [next(f) for x in range(5)][1:4] 763 for word in version: 764 kerver.append(word.split('= ')[1].split('\n')[0]) 765 staging_kerVer = ".".join(kerver) 766 return staging_kerVer 767 768def get_staging_kbranch(srcdir): 769 staging_kbranch = "" 770 if os.path.exists(srcdir) and os.listdir(srcdir): 771 (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir) 772 staging_kbranch = "".join(branch.split('\n')[0]) 773 return staging_kbranch 774 775def get_real_srctree(srctree, s, workdir): 776 # Check that recipe isn't using a shared workdir 777 s = os.path.abspath(s) 778 workdir = os.path.abspath(workdir) 779 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 780 # Handle if S is set to a subdirectory of the source 781 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 782 srctree = os.path.join(srctree, srcsubdir) 783 return srctree 784 785def modify(args, config, basepath, workspace): 786 """Entry point for the devtool 'modify' subcommand""" 787 import bb 788 import oe.recipeutils 789 import oe.patch 790 import oe.path 791 792 if args.recipename in workspace: 793 raise DevtoolError("recipe %s is already in your workspace" % 794 args.recipename) 795 796 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 797 try: 798 rd = parse_recipe(config, tinfoil, args.recipename, True) 799 if not rd: 800 return 1 801 802 pn = rd.getVar('PN') 803 if pn != args.recipename: 804 logger.info('Mapping %s to %s' % (args.recipename, pn)) 805 if pn in workspace: 806 raise DevtoolError("recipe %s is already in your workspace" % 807 pn) 808 809 if args.srctree: 810 srctree = os.path.abspath(args.srctree) 811 else: 812 srctree = get_default_srctree(config, pn) 813 814 if args.no_extract and not os.path.isdir(srctree): 815 raise DevtoolError("--no-extract specified and source path %s does " 816 "not exist or is not a directory" % 817 srctree) 818 819 recipefile = rd.getVar('FILE') 820 appendfile = recipe_to_append(recipefile, config, args.wildcard) 821 if os.path.exists(appendfile): 822 raise DevtoolError("Another variant of recipe %s is already in your " 823 "workspace (only one variant of a recipe can " 824 "currently be worked on at once)" 825 % pn) 826 827 _check_compatible_recipe(pn, rd) 828 829 initial_revs = {} 830 commits = {} 831 check_commits = False 832 833 if bb.data.inherits_class('kernel-yocto', rd): 834 # Current set kernel version 835 kernelVersion = rd.getVar('LINUX_VERSION') 836 srcdir = rd.getVar('STAGING_KERNEL_DIR') 837 kbranch = rd.getVar('KBRANCH') 838 839 staging_kerVer = get_staging_kver(srcdir) 840 staging_kbranch = get_staging_kbranch(srcdir) 841 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 842 oe.path.copyhardlinktree(srcdir, srctree) 843 workdir = rd.getVar('WORKDIR') 844 srcsubdir = rd.getVar('S') 845 localfilesdir = os.path.join(srctree, 'oe-local-files') 846 # Move local source files into separate subdir 847 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 848 local_files = oe.recipeutils.get_recipe_local_files(rd) 849 850 for key in local_files.copy(): 851 if key.endswith('scc'): 852 sccfile = open(local_files[key], 'r') 853 for l in sccfile: 854 line = l.split() 855 if line and line[0] in ('kconf', 'patch'): 856 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 857 if not cfg in local_files.values(): 858 local_files[line[-1]] = cfg 859 shutil.copy2(cfg, workdir) 860 sccfile.close() 861 862 # Ignore local files with subdir={BP} 863 srcabspath = os.path.abspath(srcsubdir) 864 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 865 if local_files: 866 for fname in local_files: 867 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 868 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 869 f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n') 870 871 symlink_oelocal_files_srctree(rd, srctree) 872 873 task = 'do_configure' 874 res = tinfoil.build_targets(pn, task, handle_events=True) 875 876 # Copy .config to workspace 877 kconfpath = rd.getVar('B') 878 logger.info('Copying kernel config to workspace') 879 shutil.copy2(os.path.join(kconfpath, '.config'), srctree) 880 881 # Set this to true, we still need to get initial_rev 882 # by parsing the git repo 883 args.no_extract = True 884 885 if not args.no_extract: 886 initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 887 if not initial_revs["."]: 888 return 1 889 logger.info('Source tree extracted to %s' % srctree) 890 891 if os.path.exists(os.path.join(srctree, '.git')): 892 # Get list of commits since this revision 893 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) 894 commits["."] = stdout.split() 895 check_commits = True 896 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) 897 for line in stdout.splitlines(): 898 (rev, submodule_path) = line.split() 899 submodule = os.path.relpath(submodule_path, srctree) 900 initial_revs[submodule] = rev 901 (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path) 902 commits[submodule] = stdout.split() 903 else: 904 if os.path.exists(os.path.join(srctree, '.git')): 905 # Check if it's a tree previously extracted by us. This is done 906 # by ensuring that devtool-base and args.branch (devtool) exist. 907 # The check_commits logic will cause an exception if either one 908 # of these doesn't exist 909 try: 910 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 911 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 912 except bb.process.ExecutionError: 913 stdout = '' 914 if stdout: 915 check_commits = True 916 for line in stdout.splitlines(): 917 if line.startswith('*'): 918 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 919 initial_revs["."] = stdout.rstrip() 920 if "." not in initial_revs: 921 # Otherwise, just grab the head revision 922 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 923 initial_revs["."] = stdout.rstrip() 924 925 branch_patches = {} 926 if check_commits: 927 # Check if there are override branches 928 (stdout, _) = bb.process.run('git branch', cwd=srctree) 929 branches = [] 930 for line in stdout.rstrip().splitlines(): 931 branchname = line[2:].rstrip() 932 if branchname.startswith(override_branch_prefix): 933 branches.append(branchname) 934 if branches: 935 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 936 branches.insert(0, args.branch) 937 seen_patches = [] 938 for branch in branches: 939 branch_patches[branch] = [] 940 (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree) 941 for sha1 in stdout.splitlines(): 942 notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip()) 943 origpatch = notes.get(oe.patch.GitApplyTree.original_patch) 944 if origpatch and origpatch not in seen_patches: 945 seen_patches.append(origpatch) 946 branch_patches[branch].append(origpatch) 947 948 # Need to grab this here in case the source is within a subdirectory 949 srctreebase = srctree 950 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 951 952 bb.utils.mkdirhier(os.path.dirname(appendfile)) 953 with open(appendfile, 'w') as f: 954 # if not present, add type=git-dependency to the secondary sources 955 # (non local files) so they can be extracted correctly when building a recipe after 956 # doing a devtool modify on it 957 src_uri = rd.getVar('SRC_URI').split() 958 src_uri_append = [] 959 src_uri_remove = [] 960 961 # Assume first entry is main source extracted in ${S} so skip it 962 src_uri = src_uri[1::] 963 964 # Add "type=git-dependency" to all non local sources 965 for url in src_uri: 966 if not url.startswith('file://') and not 'type=' in url: 967 src_uri_remove.append(url) 968 src_uri_append.append('%s;type=git-dependency' % url) 969 970 if src_uri_remove: 971 f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove)) 972 f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append)) 973 974 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 975 # Local files can be modified/tracked in separate subdir under srctree 976 # Mostly useful for packages with S != WORKDIR 977 f.write('FILESPATH:prepend := "%s:"\n' % 978 os.path.join(srctreebase, 'oe-local-files')) 979 f.write('# srctreebase: %s\n' % srctreebase) 980 981 f.write('\ninherit externalsrc\n') 982 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 983 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 984 985 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 986 if b_is_s: 987 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 988 989 if bb.data.inherits_class('kernel', rd): 990 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 991 'do_fetch do_unpack do_kernel_configcheck"\n') 992 f.write('\ndo_patch[noexec] = "1"\n') 993 f.write('\ndo_configure:append() {\n' 994 ' cp ${B}/.config ${S}/.config.baseline\n' 995 ' ln -sfT ${B}/.config ${S}/.config.new\n' 996 '}\n') 997 f.write('\ndo_kernel_configme:prepend() {\n' 998 ' if [ -e ${S}/.config ]; then\n' 999 ' mv ${S}/.config ${S}/.config.old\n' 1000 ' fi\n' 1001 '}\n') 1002 if rd.getVarFlag('do_menuconfig', 'task'): 1003 f.write('\ndo_configure:append() {\n' 1004 ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n' 1005 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' 1006 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' 1007 ' fi\n' 1008 '}\n') 1009 if initial_revs: 1010 for name, rev in initial_revs.items(): 1011 f.write('\n# initial_rev %s: %s\n' % (name, rev)) 1012 if name in commits: 1013 for commit in commits[name]: 1014 f.write('# commit %s: %s\n' % (name, commit)) 1015 if branch_patches: 1016 for branch in branch_patches: 1017 if branch == args.branch: 1018 continue 1019 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 1020 1021 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 1022 1023 _add_md5(config, pn, appendfile) 1024 1025 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 1026 1027 finally: 1028 tinfoil.shutdown() 1029 1030 return 0 1031 1032 1033def rename(args, config, basepath, workspace): 1034 """Entry point for the devtool 'rename' subcommand""" 1035 import bb 1036 import oe.recipeutils 1037 1038 check_workspace_recipe(workspace, args.recipename) 1039 1040 if not (args.newname or args.version): 1041 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 1042 1043 recipefile = workspace[args.recipename]['recipefile'] 1044 if not recipefile: 1045 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1046 1047 if args.newname and args.newname != args.recipename: 1048 reason = oe.recipeutils.validate_pn(args.newname) 1049 if reason: 1050 raise DevtoolError(reason) 1051 newname = args.newname 1052 else: 1053 newname = args.recipename 1054 1055 append = workspace[args.recipename]['bbappend'] 1056 appendfn = os.path.splitext(os.path.basename(append))[0] 1057 splitfn = appendfn.split('_') 1058 if len(splitfn) > 1: 1059 origfnver = appendfn.split('_')[1] 1060 else: 1061 origfnver = '' 1062 1063 recipefilemd5 = None 1064 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1065 try: 1066 rd = parse_recipe(config, tinfoil, args.recipename, True) 1067 if not rd: 1068 return 1 1069 1070 bp = rd.getVar('BP') 1071 bpn = rd.getVar('BPN') 1072 if newname != args.recipename: 1073 localdata = rd.createCopy() 1074 localdata.setVar('PN', newname) 1075 newbpn = localdata.getVar('BPN') 1076 else: 1077 newbpn = bpn 1078 s = rd.getVar('S', False) 1079 src_uri = rd.getVar('SRC_URI', False) 1080 pv = rd.getVar('PV') 1081 1082 # Correct variable values that refer to the upstream source - these 1083 # values must stay the same, so if the name/version are changing then 1084 # we need to fix them up 1085 new_s = s 1086 new_src_uri = src_uri 1087 if newbpn != bpn: 1088 # ${PN} here is technically almost always incorrect, but people do use it 1089 new_s = new_s.replace('${BPN}', bpn) 1090 new_s = new_s.replace('${PN}', bpn) 1091 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1092 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1093 new_src_uri = new_src_uri.replace('${PN}', bpn) 1094 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1095 if args.version and origfnver == pv: 1096 new_s = new_s.replace('${PV}', pv) 1097 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1098 new_src_uri = new_src_uri.replace('${PV}', pv) 1099 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1100 patchfields = {} 1101 if new_s != s: 1102 patchfields['S'] = new_s 1103 if new_src_uri != src_uri: 1104 patchfields['SRC_URI'] = new_src_uri 1105 if patchfields: 1106 recipefilemd5 = bb.utils.md5_file(recipefile) 1107 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1108 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1109 finally: 1110 tinfoil.shutdown() 1111 1112 if args.version: 1113 newver = args.version 1114 else: 1115 newver = origfnver 1116 1117 if newver: 1118 newappend = '%s_%s.bbappend' % (newname, newver) 1119 newfile = '%s_%s.bb' % (newname, newver) 1120 else: 1121 newappend = '%s.bbappend' % newname 1122 newfile = '%s.bb' % newname 1123 1124 oldrecipedir = os.path.dirname(recipefile) 1125 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1126 if oldrecipedir != newrecipedir: 1127 bb.utils.mkdirhier(newrecipedir) 1128 1129 newappend = os.path.join(os.path.dirname(append), newappend) 1130 newfile = os.path.join(newrecipedir, newfile) 1131 1132 # Rename bbappend 1133 logger.info('Renaming %s to %s' % (append, newappend)) 1134 bb.utils.rename(append, newappend) 1135 # Rename recipe file 1136 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1137 bb.utils.rename(recipefile, newfile) 1138 1139 # Rename source tree if it's the default path 1140 appendmd5 = None 1141 if not args.no_srctree: 1142 srctree = workspace[args.recipename]['srctree'] 1143 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1144 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1145 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1146 shutil.move(srctree, newsrctree) 1147 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1148 appendmd5 = bb.utils.md5_file(newappend) 1149 appendlines = [] 1150 with open(newappend, 'r') as f: 1151 for line in f: 1152 appendlines.append(line) 1153 with open(newappend, 'w') as f: 1154 for line in appendlines: 1155 if srctree in line: 1156 line = line.replace(srctree, newsrctree) 1157 f.write(line) 1158 newappendmd5 = bb.utils.md5_file(newappend) 1159 1160 bpndir = None 1161 newbpndir = None 1162 if newbpn != bpn: 1163 bpndir = os.path.join(oldrecipedir, bpn) 1164 if os.path.exists(bpndir): 1165 newbpndir = os.path.join(newrecipedir, newbpn) 1166 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1167 shutil.move(bpndir, newbpndir) 1168 1169 bpdir = None 1170 newbpdir = None 1171 if newver != origfnver or newbpn != bpn: 1172 bpdir = os.path.join(oldrecipedir, bp) 1173 if os.path.exists(bpdir): 1174 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1175 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1176 shutil.move(bpdir, newbpdir) 1177 1178 if oldrecipedir != newrecipedir: 1179 # Move any stray files and delete the old recipe directory 1180 for entry in os.listdir(oldrecipedir): 1181 oldpath = os.path.join(oldrecipedir, entry) 1182 newpath = os.path.join(newrecipedir, entry) 1183 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1184 shutil.move(oldpath, newpath) 1185 os.rmdir(oldrecipedir) 1186 1187 # Now take care of entries in .devtool_md5 1188 md5entries = [] 1189 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1190 for line in f: 1191 md5entries.append(line) 1192 1193 if bpndir and newbpndir: 1194 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1195 else: 1196 relbpndir = None 1197 if bpdir and newbpdir: 1198 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1199 else: 1200 relbpdir = None 1201 1202 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1203 for entry in md5entries: 1204 splitentry = entry.rstrip().split('|') 1205 if len(splitentry) > 2: 1206 if splitentry[0] == args.recipename: 1207 splitentry[0] = newname 1208 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1209 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1210 if appendmd5 and splitentry[2] == appendmd5: 1211 splitentry[2] = newappendmd5 1212 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1213 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1214 if recipefilemd5 and splitentry[2] == recipefilemd5: 1215 splitentry[2] = newrecipefilemd5 1216 elif relbpndir and splitentry[1].startswith(relbpndir): 1217 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1218 elif relbpdir and splitentry[1].startswith(relbpdir): 1219 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1220 entry = '|'.join(splitentry) + '\n' 1221 f.write(entry) 1222 return 0 1223 1224 1225def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1226 """Get initial and update rev of a recipe. These are the start point of the 1227 whole patchset and start point for the patches to be re-generated/updated. 1228 """ 1229 import bb 1230 1231 # Get current branch 1232 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1233 cwd=srctree) 1234 branchname = stdout.rstrip() 1235 1236 # Parse initial rev from recipe if not specified 1237 commits = {} 1238 patches = [] 1239 initial_revs = {} 1240 with open(recipe_path, 'r') as f: 1241 for line in f: 1242 pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$' 1243 match = re.search(pattern, line) 1244 if match: 1245 name = match.group(1) 1246 rev = match.group(2) 1247 if line.startswith('# initial_rev'): 1248 if not (name == "." and initial_rev): 1249 initial_revs[name] = rev 1250 elif line.startswith('# commit') and not force_patch_refresh: 1251 if name not in commits: 1252 commits[name] = [rev] 1253 else: 1254 commits[name].append(rev) 1255 elif line.startswith('# patches_%s:' % branchname): 1256 patches = line.split(':')[-1].strip().split(',') 1257 1258 update_revs = dict(initial_revs) 1259 changed_revs = {} 1260 for name, rev in initial_revs.items(): 1261 # Find first actually changed revision 1262 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1263 rev, cwd=os.path.join(srctree, name)) 1264 newcommits = stdout.split() 1265 if name in commits: 1266 for i in range(min(len(commits[name]), len(newcommits))): 1267 if newcommits[i] == commits[name][i]: 1268 update_revs[name] = commits[name][i] 1269 1270 try: 1271 stdout, _ = bb.process.run('git cherry devtool-patched', 1272 cwd=os.path.join(srctree, name)) 1273 except bb.process.ExecutionError as err: 1274 stdout = None 1275 1276 if stdout is not None and not force_patch_refresh: 1277 for line in stdout.splitlines(): 1278 if line.startswith('+ '): 1279 rev = line.split()[1] 1280 if rev in newcommits: 1281 if name not in changed_revs: 1282 changed_revs[name] = [rev] 1283 else: 1284 changed_revs[name].append(rev) 1285 1286 return initial_revs, update_revs, changed_revs, patches 1287 1288def _remove_file_entries(srcuri, filelist): 1289 """Remove file:// entries from SRC_URI""" 1290 remaining = filelist[:] 1291 entries = [] 1292 for fname in filelist: 1293 basename = os.path.basename(fname) 1294 for i in range(len(srcuri)): 1295 if (srcuri[i].startswith('file://') and 1296 os.path.basename(srcuri[i].split(';')[0]) == basename): 1297 entries.append(srcuri[i]) 1298 remaining.remove(fname) 1299 srcuri.pop(i) 1300 break 1301 return entries, remaining 1302 1303def _replace_srcuri_entry(srcuri, filename, newentry): 1304 """Replace entry corresponding to specified file with a new entry""" 1305 basename = os.path.basename(filename) 1306 for i in range(len(srcuri)): 1307 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1308 srcuri.pop(i) 1309 srcuri.insert(i, newentry) 1310 break 1311 1312def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1313 """Unlink existing patch files""" 1314 1315 dry_run_suffix = ' (dry-run)' if dry_run else '' 1316 1317 for path in files: 1318 if append: 1319 if not destpath: 1320 raise Exception('destpath should be set here') 1321 path = os.path.join(destpath, os.path.basename(path)) 1322 1323 if os.path.exists(path): 1324 if not no_report_remove: 1325 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1326 if not dry_run: 1327 # FIXME "git rm" here would be nice if the file in question is 1328 # tracked 1329 # FIXME there's a chance that this file is referred to by 1330 # another recipe, in which case deleting wouldn't be the 1331 # right thing to do 1332 os.remove(path) 1333 # Remove directory if empty 1334 try: 1335 os.rmdir(os.path.dirname(path)) 1336 except OSError as ose: 1337 if ose.errno != errno.ENOTEMPTY: 1338 raise 1339 1340 1341def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None): 1342 """Export patches from srctree to given location. 1343 Returns three-tuple of dicts: 1344 1. updated - patches that already exist in SRCURI 1345 2. added - new patches that don't exist in SRCURI 1346 3 removed - patches that exist in SRCURI but not in exported patches 1347 In each dict the key is the 'basepath' of the URI and value is: 1348 - for updated and added dicts, a dict with 2 optionnal keys: 1349 - 'path': the absolute path to the existing file in recipe space (if any) 1350 - 'patchdir': the directory in wich the patch should be applied (if any) 1351 - for removed dict, the absolute path to the existing file in recipe space 1352 """ 1353 import oe.recipeutils 1354 from oe.patch import GitApplyTree 1355 updated = OrderedDict() 1356 added = OrderedDict() 1357 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1358 1359 existing_patches = dict((os.path.basename(path), path) for path in 1360 oe.recipeutils.get_recipe_patches(rd)) 1361 logger.debug('Existing patches: %s' % existing_patches) 1362 1363 # Generate patches from Git, exclude local files directory 1364 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1365 GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec) 1366 for dirpath, dirnames, filenames in os.walk(destdir): 1367 new_patches = filenames 1368 reldirpath = os.path.relpath(dirpath, destdir) 1369 for new_patch in new_patches: 1370 # Strip numbering from patch names. If it's a git sequence named patch, 1371 # the numbers might not match up since we are starting from a different 1372 # revision This does assume that people are using unique shortlog 1373 # values, but they ought to be anyway... 1374 new_basename = seqpatch_re.match(new_patch).group(2) 1375 match_name = None 1376 for old_patch in existing_patches: 1377 old_basename = seqpatch_re.match(old_patch).group(2) 1378 old_basename_splitext = os.path.splitext(old_basename) 1379 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1380 old_patch_noext = os.path.splitext(old_patch)[0] 1381 match_name = old_patch_noext 1382 break 1383 elif new_basename == old_basename: 1384 match_name = old_patch 1385 break 1386 if match_name: 1387 # Rename patch files 1388 if new_patch != match_name: 1389 bb.utils.rename(os.path.join(destdir, new_patch), 1390 os.path.join(destdir, match_name)) 1391 # Need to pop it off the list now before checking changed_revs 1392 oldpath = existing_patches.pop(old_patch) 1393 if changed_revs is not None and dirpath in changed_revs: 1394 # Avoid updating patches that have not actually changed 1395 with open(os.path.join(dirpath, match_name), 'r') as f: 1396 firstlineitems = f.readline().split() 1397 # Looking for "From <hash>" line 1398 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1399 if not firstlineitems[1] in changed_revs[dirpath]: 1400 continue 1401 # Recompress if necessary 1402 if oldpath.endswith(('.gz', '.Z')): 1403 bb.process.run(['gzip', match_name], cwd=destdir) 1404 if oldpath.endswith('.gz'): 1405 match_name += '.gz' 1406 else: 1407 match_name += '.Z' 1408 elif oldpath.endswith('.bz2'): 1409 bb.process.run(['bzip2', match_name], cwd=destdir) 1410 match_name += '.bz2' 1411 updated[match_name] = {'path' : oldpath} 1412 if reldirpath != ".": 1413 updated[match_name]['patchdir'] = reldirpath 1414 else: 1415 added[new_patch] = {} 1416 if reldirpath != ".": 1417 added[new_patch]['patchdir'] = reldirpath 1418 1419 return (updated, added, existing_patches) 1420 1421 1422def _create_kconfig_diff(srctree, rd, outfile): 1423 """Create a kconfig fragment""" 1424 # Only update config fragment if both config files exist 1425 orig_config = os.path.join(srctree, '.config.baseline') 1426 new_config = os.path.join(srctree, '.config.new') 1427 if os.path.exists(orig_config) and os.path.exists(new_config): 1428 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1429 '--unchanged-line-format=', orig_config, new_config] 1430 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1431 stderr=subprocess.PIPE) 1432 stdout, stderr = pipe.communicate() 1433 if pipe.returncode == 1: 1434 logger.info("Updating config fragment %s" % outfile) 1435 with open(outfile, 'wb') as fobj: 1436 fobj.write(stdout) 1437 elif pipe.returncode == 0: 1438 logger.info("Would remove config fragment %s" % outfile) 1439 if os.path.exists(outfile): 1440 # Remove fragment file in case of empty diff 1441 logger.info("Removing config fragment %s" % outfile) 1442 os.unlink(outfile) 1443 else: 1444 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1445 return True 1446 return False 1447 1448 1449def _export_local_files(srctree, rd, destdir, srctreebase): 1450 """Copy local files from srctree to given location. 1451 Returns three-tuple of dicts: 1452 1. updated - files that already exist in SRCURI 1453 2. added - new files files that don't exist in SRCURI 1454 3 removed - files that exist in SRCURI but not in exported files 1455 In each dict the key is the 'basepath' of the URI and value is the 1456 absolute path to the existing file in recipe space (if any). 1457 """ 1458 import oe.recipeutils 1459 1460 # Find out local files (SRC_URI files that exist in the "recipe space"). 1461 # Local files that reside in srctree are not included in patch generation. 1462 # Instead they are directly copied over the original source files (in 1463 # recipe space). 1464 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1465 new_set = None 1466 updated = OrderedDict() 1467 added = OrderedDict() 1468 removed = OrderedDict() 1469 1470 # Get current branch and return early with empty lists 1471 # if on one of the override branches 1472 # (local files are provided only for the main branch and processing 1473 # them against lists from recipe overrides will result in mismatches 1474 # and broken modifications to recipes). 1475 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1476 cwd=srctree) 1477 branchname = stdout.rstrip() 1478 if branchname.startswith(override_branch_prefix): 1479 return (updated, added, removed) 1480 1481 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1482 git_files = _git_ls_tree(srctree) 1483 if 'oe-local-files' in git_files: 1484 # If tracked by Git, take the files from srctree HEAD. First get 1485 # the tree object of the directory 1486 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1487 tree = git_files['oe-local-files'][2] 1488 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1489 env=dict(os.environ, GIT_WORK_TREE=destdir, 1490 GIT_INDEX_FILE=tmp_index)) 1491 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1492 elif os.path.isdir(local_files_dir): 1493 # If not tracked by Git, just copy from working copy 1494 new_set = _ls_tree(local_files_dir) 1495 bb.process.run(['cp', '-ax', 1496 os.path.join(local_files_dir, '.'), destdir]) 1497 else: 1498 new_set = [] 1499 1500 # Special handling for kernel config 1501 if bb.data.inherits_class('kernel-yocto', rd): 1502 fragment_fn = 'devtool-fragment.cfg' 1503 fragment_path = os.path.join(destdir, fragment_fn) 1504 if _create_kconfig_diff(srctree, rd, fragment_path): 1505 if os.path.exists(fragment_path): 1506 if fragment_fn not in new_set: 1507 new_set.append(fragment_fn) 1508 # Copy fragment to local-files 1509 if os.path.isdir(local_files_dir): 1510 shutil.copy2(fragment_path, local_files_dir) 1511 else: 1512 if fragment_fn in new_set: 1513 new_set.remove(fragment_fn) 1514 # Remove fragment from local-files 1515 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1516 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1517 1518 # Special handling for cml1, ccmake, etc bbclasses that generated 1519 # configuration fragment files that are consumed as source files 1520 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1521 if bb.data.inherits_class(frag_class, rd): 1522 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1523 if os.path.exists(srcpath): 1524 if frag_name not in new_set: 1525 new_set.append(frag_name) 1526 # copy fragment into destdir 1527 shutil.copy2(srcpath, destdir) 1528 # copy fragment into local files if exists 1529 if os.path.isdir(local_files_dir): 1530 shutil.copy2(srcpath, local_files_dir) 1531 1532 if new_set is not None: 1533 for fname in new_set: 1534 if fname in existing_files: 1535 origpath = existing_files.pop(fname) 1536 workpath = os.path.join(local_files_dir, fname) 1537 if not filecmp.cmp(origpath, workpath): 1538 updated[fname] = origpath 1539 elif fname != '.gitignore': 1540 added[fname] = None 1541 1542 workdir = rd.getVar('WORKDIR') 1543 s = rd.getVar('S') 1544 if not s.endswith(os.sep): 1545 s += os.sep 1546 1547 if workdir != s: 1548 # Handle files where subdir= was specified 1549 for fname in list(existing_files.keys()): 1550 # FIXME handle both subdir starting with BP and not? 1551 fworkpath = os.path.join(workdir, fname) 1552 if fworkpath.startswith(s): 1553 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1554 if os.path.exists(fpath): 1555 origpath = existing_files.pop(fname) 1556 if not filecmp.cmp(origpath, fpath): 1557 updated[fpath] = origpath 1558 1559 removed = existing_files 1560 return (updated, added, removed) 1561 1562 1563def _determine_files_dir(rd): 1564 """Determine the appropriate files directory for a recipe""" 1565 recipedir = rd.getVar('FILE_DIRNAME') 1566 for entry in rd.getVar('FILESPATH').split(':'): 1567 relpth = os.path.relpath(entry, recipedir) 1568 if not os.sep in relpth: 1569 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1570 if os.path.isdir(entry): 1571 return entry 1572 return os.path.join(recipedir, rd.getVar('BPN')) 1573 1574 1575def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1576 """Implement the 'srcrev' mode of update-recipe""" 1577 import bb 1578 import oe.recipeutils 1579 1580 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1581 1582 recipefile = rd.getVar('FILE') 1583 recipedir = os.path.basename(recipefile) 1584 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1585 1586 # Get original SRCREV 1587 old_srcrev = rd.getVar('SRCREV') or '' 1588 if old_srcrev == "INVALID": 1589 raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository') 1590 old_srcrev = {'.': old_srcrev} 1591 1592 # Get HEAD revision 1593 try: 1594 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1595 except bb.process.ExecutionError as err: 1596 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1597 (srctree, err)) 1598 srcrev = stdout.strip() 1599 if len(srcrev) != 40: 1600 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1601 1602 destpath = None 1603 remove_files = [] 1604 patchfields = {} 1605 patchfields['SRCREV'] = srcrev 1606 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1607 srcuri = orig_src_uri.split() 1608 tempdir = tempfile.mkdtemp(prefix='devtool') 1609 update_srcuri = False 1610 appendfile = None 1611 try: 1612 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1613 srctreebase = workspace[recipename]['srctreebase'] 1614 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1615 if not no_remove: 1616 # Find list of existing patches in recipe file 1617 patches_dir = tempfile.mkdtemp(dir=tempdir) 1618 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1619 patches_dir) 1620 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1621 1622 # Remove deleted local files and "overlapping" patches 1623 remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value] 1624 if remove_files: 1625 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1626 update_srcuri = True 1627 1628 if appendlayerdir: 1629 files = dict((os.path.join(local_files_dir, key), val) for 1630 key, val in list(upd_f.items()) + list(new_f.items())) 1631 removevalues = {} 1632 if update_srcuri: 1633 removevalues = {'SRC_URI': removedentries} 1634 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1635 if dry_run_outdir: 1636 logger.info('Creating bbappend (dry-run)') 1637 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1638 rd, appendlayerdir, files, wildcardver=wildcard_version, 1639 extralines=patchfields, removevalues=removevalues, 1640 redirect_output=dry_run_outdir) 1641 else: 1642 files_dir = _determine_files_dir(rd) 1643 for basepath, path in upd_f.items(): 1644 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1645 if os.path.isabs(basepath): 1646 # Original file (probably with subdir pointing inside source tree) 1647 # so we do not want to move it, just copy 1648 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1649 else: 1650 _move_file(os.path.join(local_files_dir, basepath), path, 1651 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1652 update_srcuri= True 1653 for basepath, path in new_f.items(): 1654 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1655 _move_file(os.path.join(local_files_dir, basepath), 1656 os.path.join(files_dir, basepath), 1657 dry_run_outdir=dry_run_outdir, 1658 base_outdir=recipedir) 1659 srcuri.append('file://%s' % basepath) 1660 update_srcuri = True 1661 if update_srcuri: 1662 patchfields['SRC_URI'] = ' '.join(srcuri) 1663 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1664 finally: 1665 shutil.rmtree(tempdir) 1666 if not 'git://' in orig_src_uri: 1667 logger.info('You will need to update SRC_URI within the recipe to ' 1668 'point to a git repository where you have pushed your ' 1669 'changes') 1670 1671 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1672 return True, appendfile, remove_files 1673 1674def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1675 """Implement the 'patch' mode of update-recipe""" 1676 import bb 1677 import oe.recipeutils 1678 1679 recipefile = rd.getVar('FILE') 1680 recipedir = os.path.dirname(recipefile) 1681 append = workspace[recipename]['bbappend'] 1682 if not os.path.exists(append): 1683 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1684 recipename) 1685 srctreebase = workspace[recipename]['srctreebase'] 1686 relpatchdir = os.path.relpath(srctreebase, srctree) 1687 if relpatchdir == '.': 1688 patchdir_params = {} 1689 else: 1690 patchdir_params = {'patchdir': relpatchdir} 1691 1692 def srcuri_entry(basepath, patchdir_params): 1693 if patchdir_params: 1694 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1695 else: 1696 paramstr = '' 1697 return 'file://%s%s' % (basepath, paramstr) 1698 1699 initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1700 if not initial_revs: 1701 raise DevtoolError('Unable to find initial revision - please specify ' 1702 'it with --initial-rev') 1703 1704 appendfile = None 1705 dl_dir = rd.getVar('DL_DIR') 1706 if not dl_dir.endswith('/'): 1707 dl_dir += '/' 1708 1709 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1710 1711 tempdir = tempfile.mkdtemp(prefix='devtool') 1712 try: 1713 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1714 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1715 1716 # Get updated patches from source tree 1717 patches_dir = tempfile.mkdtemp(dir=tempdir) 1718 upd_p, new_p, _ = _export_patches(srctree, rd, update_revs, 1719 patches_dir, changed_revs) 1720 # Get all patches from source tree and check if any should be removed 1721 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1722 _, _, del_p = _export_patches(srctree, rd, initial_revs, 1723 all_patches_dir) 1724 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1725 if filter_patches: 1726 new_p = OrderedDict() 1727 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1728 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) 1729 remove_files = [] 1730 if not no_remove: 1731 # Remove deleted local files and patches 1732 remove_files = list(del_f.values()) + list(del_p.values()) 1733 updatefiles = False 1734 updaterecipe = False 1735 destpath = None 1736 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1737 1738 if appendlayerdir: 1739 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1740 key, val in list(upd_f.items()) + list(new_f.items())) 1741 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1742 key, val in list(upd_p.items()) + list(new_p.items()))) 1743 1744 params = [] 1745 for file, param in files.items(): 1746 patchdir_param = dict(patchdir_params) 1747 patchdir = param.get('patchdir', ".") 1748 if patchdir != "." : 1749 if patchdir_param: 1750 patchdir_param['patchdir'] += patchdir 1751 else: 1752 patchdir_param['patchdir'] = patchdir 1753 params.append(patchdir_param) 1754 1755 if files or remove_files: 1756 removevalues = None 1757 if remove_files: 1758 removedentries, remaining = _remove_file_entries( 1759 srcuri, remove_files) 1760 if removedentries or remaining: 1761 remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for 1762 item in remaining] 1763 removevalues = {'SRC_URI': removedentries + remaining} 1764 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1765 rd, appendlayerdir, files, 1766 wildcardver=wildcard_version, 1767 removevalues=removevalues, 1768 redirect_output=dry_run_outdir, 1769 params=params) 1770 else: 1771 logger.info('No patches or local source files needed updating') 1772 else: 1773 # Update existing files 1774 files_dir = _determine_files_dir(rd) 1775 for basepath, path in upd_f.items(): 1776 logger.info('Updating file %s' % basepath) 1777 if os.path.isabs(basepath): 1778 # Original file (probably with subdir pointing inside source tree) 1779 # so we do not want to move it, just copy 1780 _copy_file(basepath, path, 1781 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1782 else: 1783 _move_file(os.path.join(local_files_dir, basepath), path, 1784 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1785 updatefiles = True 1786 for basepath, param in upd_p.items(): 1787 path = param['path'] 1788 patchdir = param.get('patchdir', ".") 1789 if patchdir != "." : 1790 patchdir_param = dict(patchdir_params) 1791 if patchdir_param: 1792 patchdir_param['patchdir'] += patchdir 1793 else: 1794 patchdir_param['patchdir'] = patchdir 1795 patchfn = os.path.join(patches_dir, patchdir, basepath) 1796 if os.path.dirname(path) + '/' == dl_dir: 1797 # This is a a downloaded patch file - we now need to 1798 # replace the entry in SRC_URI with our local version 1799 logger.info('Replacing remote patch %s with updated local version' % basepath) 1800 path = os.path.join(files_dir, basepath) 1801 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param)) 1802 updaterecipe = True 1803 else: 1804 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1805 _move_file(patchfn, path, 1806 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1807 updatefiles = True 1808 # Add any new files 1809 for basepath, path in new_f.items(): 1810 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1811 _move_file(os.path.join(local_files_dir, basepath), 1812 os.path.join(files_dir, basepath), 1813 dry_run_outdir=dry_run_outdir, 1814 base_outdir=recipedir) 1815 srcuri.append(srcuri_entry(basepath, patchdir_params)) 1816 updaterecipe = True 1817 for basepath, param in new_p.items(): 1818 patchdir = param.get('patchdir', ".") 1819 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1820 _move_file(os.path.join(patches_dir, patchdir, basepath), 1821 os.path.join(files_dir, basepath), 1822 dry_run_outdir=dry_run_outdir, 1823 base_outdir=recipedir) 1824 params = dict(patchdir_params) 1825 if patchdir != "." : 1826 if params: 1827 params['patchdir'] += patchdir 1828 else: 1829 params['patchdir'] = patchdir 1830 1831 srcuri.append(srcuri_entry(basepath, params)) 1832 updaterecipe = True 1833 # Update recipe, if needed 1834 if _remove_file_entries(srcuri, remove_files)[0]: 1835 updaterecipe = True 1836 if updaterecipe: 1837 if not dry_run_outdir: 1838 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1839 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1840 {'SRC_URI': ' '.join(srcuri)}, 1841 redirect_output=dry_run_outdir) 1842 elif not updatefiles: 1843 # Neither patches nor recipe were updated 1844 logger.info('No patches or files need updating') 1845 return False, None, [] 1846 finally: 1847 shutil.rmtree(tempdir) 1848 1849 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1850 return True, appendfile, remove_files 1851 1852def _guess_recipe_update_mode(srctree, rdata): 1853 """Guess the recipe update mode to use""" 1854 src_uri = (rdata.getVar('SRC_URI') or '').split() 1855 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1856 if not git_uris: 1857 return 'patch' 1858 # Just use the first URI for now 1859 uri = git_uris[0] 1860 # Check remote branch 1861 params = bb.fetch.decodeurl(uri)[5] 1862 upstr_branch = params['branch'] if 'branch' in params else 'master' 1863 # Check if current branch HEAD is found in upstream branch 1864 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1865 head_rev = stdout.rstrip() 1866 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1867 cwd=srctree) 1868 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1869 if 'origin/' + upstr_branch in remote_brs: 1870 return 'srcrev' 1871 1872 return 'patch' 1873 1874def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1875 srctree = workspace[recipename]['srctree'] 1876 if mode == 'auto': 1877 mode = _guess_recipe_update_mode(srctree, rd) 1878 1879 override_branches = [] 1880 mainbranch = None 1881 startbranch = None 1882 if not no_overrides: 1883 stdout, _ = bb.process.run('git branch', cwd=srctree) 1884 other_branches = [] 1885 for line in stdout.splitlines(): 1886 branchname = line[2:] 1887 if line.startswith('* '): 1888 startbranch = branchname 1889 if branchname.startswith(override_branch_prefix): 1890 override_branches.append(branchname) 1891 else: 1892 other_branches.append(branchname) 1893 1894 if override_branches: 1895 logger.debug('_update_recipe: override branches: %s' % override_branches) 1896 logger.debug('_update_recipe: other branches: %s' % other_branches) 1897 if startbranch.startswith(override_branch_prefix): 1898 if len(other_branches) == 1: 1899 mainbranch = other_branches[1] 1900 else: 1901 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1902 else: 1903 mainbranch = startbranch 1904 1905 checkedout = None 1906 anyupdated = False 1907 appendfile = None 1908 allremoved = [] 1909 if override_branches: 1910 logger.info('Handling main branch (%s)...' % mainbranch) 1911 if startbranch != mainbranch: 1912 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1913 checkedout = mainbranch 1914 try: 1915 branchlist = [mainbranch] + override_branches 1916 for branch in branchlist: 1917 crd = bb.data.createCopy(rd) 1918 if branch != mainbranch: 1919 logger.info('Handling branch %s...' % branch) 1920 override = branch[len(override_branch_prefix):] 1921 crd.appendVar('OVERRIDES', ':%s' % override) 1922 bb.process.run('git checkout %s' % branch, cwd=srctree) 1923 checkedout = branch 1924 1925 if mode == 'srcrev': 1926 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1927 elif mode == 'patch': 1928 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1929 else: 1930 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1931 if updated: 1932 anyupdated = True 1933 if appendf: 1934 appendfile = appendf 1935 allremoved.extend(removed) 1936 finally: 1937 if startbranch and checkedout != startbranch: 1938 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1939 1940 return anyupdated, appendfile, allremoved 1941 1942def update_recipe(args, config, basepath, workspace): 1943 """Entry point for the devtool 'update-recipe' subcommand""" 1944 check_workspace_recipe(workspace, args.recipename) 1945 1946 if args.append: 1947 if not os.path.exists(args.append): 1948 raise DevtoolError('bbappend destination layer directory "%s" ' 1949 'does not exist' % args.append) 1950 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1951 raise DevtoolError('conf/layer.conf not found in bbappend ' 1952 'destination layer "%s"' % args.append) 1953 1954 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1955 try: 1956 1957 rd = parse_recipe(config, tinfoil, args.recipename, True) 1958 if not rd: 1959 return 1 1960 1961 dry_run_output = None 1962 dry_run_outdir = None 1963 if args.dry_run: 1964 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1965 dry_run_outdir = dry_run_output.name 1966 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1967 1968 if updated: 1969 rf = rd.getVar('FILE') 1970 if rf.startswith(config.workspace_path): 1971 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1972 finally: 1973 tinfoil.shutdown() 1974 1975 return 0 1976 1977 1978def status(args, config, basepath, workspace): 1979 """Entry point for the devtool 'status' subcommand""" 1980 if workspace: 1981 for recipe, value in sorted(workspace.items()): 1982 recipefile = value['recipefile'] 1983 if recipefile: 1984 recipestr = ' (%s)' % recipefile 1985 else: 1986 recipestr = '' 1987 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1988 else: 1989 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1990 return 0 1991 1992 1993def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1994 """Reset one or more recipes""" 1995 import oe.path 1996 1997 def clean_preferred_provider(pn, layerconf_path): 1998 """Remove PREFERRED_PROVIDER from layer.conf'""" 1999 import re 2000 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 2001 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 2002 pprovider_found = False 2003 with open(layerconf_file, 'r') as f: 2004 lines = f.readlines() 2005 with open(new_layerconf_file, 'a') as nf: 2006 for line in lines: 2007 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 2008 if not re.match(pprovider_exp, line): 2009 nf.write(line) 2010 else: 2011 pprovider_found = True 2012 if pprovider_found: 2013 shutil.move(new_layerconf_file, layerconf_file) 2014 else: 2015 os.remove(new_layerconf_file) 2016 2017 if recipes and not no_clean: 2018 if len(recipes) == 1: 2019 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 2020 else: 2021 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 2022 # If the recipe file itself was created in the workspace, and 2023 # it uses BBCLASSEXTEND, then we need to also clean the other 2024 # variants 2025 targets = [] 2026 for recipe in recipes: 2027 targets.append(recipe) 2028 recipefile = workspace[recipe]['recipefile'] 2029 if recipefile and os.path.exists(recipefile): 2030 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 2031 try: 2032 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 2033 except bb.process.ExecutionError as e: 2034 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 2035 'wish, you may specify -n/--no-clean to ' 2036 'skip running this command when resetting' % 2037 (e.command, e.stdout)) 2038 2039 for pn in recipes: 2040 _check_preserve(config, pn) 2041 2042 appendfile = workspace[pn]['bbappend'] 2043 if os.path.exists(appendfile): 2044 # This shouldn't happen, but is possible if devtool errored out prior to 2045 # writing the md5 file. We need to delete this here or the recipe won't 2046 # actually be reset 2047 os.remove(appendfile) 2048 2049 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 2050 def preservedir(origdir): 2051 if os.path.exists(origdir): 2052 for root, dirs, files in os.walk(origdir): 2053 for fn in files: 2054 logger.warning('Preserving %s in %s' % (fn, preservepath)) 2055 _move_file(os.path.join(origdir, fn), 2056 os.path.join(preservepath, fn)) 2057 for dn in dirs: 2058 preservedir(os.path.join(root, dn)) 2059 os.rmdir(origdir) 2060 2061 recipefile = workspace[pn]['recipefile'] 2062 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 2063 # This should always be true if recipefile is set, but just in case 2064 preservedir(os.path.dirname(recipefile)) 2065 # We don't automatically create this dir next to appends, but the user can 2066 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 2067 2068 srctreebase = workspace[pn]['srctreebase'] 2069 if os.path.isdir(srctreebase): 2070 if os.listdir(srctreebase): 2071 if remove_work: 2072 logger.info('-r argument used on %s, removing source tree.' 2073 ' You will lose any unsaved work' %pn) 2074 shutil.rmtree(srctreebase) 2075 else: 2076 # We don't want to risk wiping out any work in progress 2077 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): 2078 from datetime import datetime 2079 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S"))) 2080 logger.info('Preserving source tree in %s\nIf you no ' 2081 'longer need it then please delete it manually.\n' 2082 'It is also possible to reuse it via devtool source tree argument.' 2083 % preservesrc) 2084 bb.utils.mkdirhier(os.path.dirname(preservesrc)) 2085 shutil.move(srctreebase, preservesrc) 2086 else: 2087 logger.info('Leaving source tree %s as-is; if you no ' 2088 'longer need it then please delete it manually' 2089 % srctreebase) 2090 else: 2091 # This is unlikely, but if it's empty we can just remove it 2092 os.rmdir(srctreebase) 2093 2094 clean_preferred_provider(pn, config.workspace_path) 2095 2096def reset(args, config, basepath, workspace): 2097 """Entry point for the devtool 'reset' subcommand""" 2098 import bb 2099 import shutil 2100 2101 recipes = "" 2102 2103 if args.recipename: 2104 if args.all: 2105 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 2106 else: 2107 for recipe in args.recipename: 2108 check_workspace_recipe(workspace, recipe, checksrc=False) 2109 elif not args.all: 2110 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2111 "reset all recipes") 2112 if args.all: 2113 recipes = list(workspace.keys()) 2114 else: 2115 recipes = args.recipename 2116 2117 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2118 2119 return 0 2120 2121 2122def _get_layer(layername, d): 2123 """Determine the base layer path for the specified layer name/path""" 2124 layerdirs = d.getVar('BBLAYERS').split() 2125 layers = {} # {basename: layer_paths} 2126 for p in layerdirs: 2127 bn = os.path.basename(p) 2128 if bn not in layers: 2129 layers[bn] = [p] 2130 else: 2131 layers[bn].append(p) 2132 # Provide some shortcuts 2133 if layername.lower() in ['oe-core', 'openembedded-core']: 2134 layername = 'meta' 2135 layer_paths = layers.get(layername, None) 2136 if not layer_paths: 2137 return os.path.abspath(layername) 2138 elif len(layer_paths) == 1: 2139 return os.path.abspath(layer_paths[0]) 2140 else: 2141 # multiple layers having the same base name 2142 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 2143 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 2144 return os.path.abspath(layer_paths[0]) 2145 2146 2147def finish(args, config, basepath, workspace): 2148 """Entry point for the devtool 'finish' subcommand""" 2149 import bb 2150 import oe.recipeutils 2151 2152 check_workspace_recipe(workspace, args.recipename) 2153 2154 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2155 2156 # Grab the equivalent of COREBASE without having to initialise tinfoil 2157 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2158 2159 srctree = workspace[args.recipename]['srctree'] 2160 check_git_repo_op(srctree, [corebasedir]) 2161 dirty = check_git_repo_dirty(srctree) 2162 if dirty: 2163 if args.force: 2164 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2165 else: 2166 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2167 2168 no_clean = args.no_clean 2169 remove_work=args.remove_work 2170 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2171 try: 2172 rd = parse_recipe(config, tinfoil, args.recipename, True) 2173 if not rd: 2174 return 1 2175 2176 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2177 recipefile = rd.getVar('FILE') 2178 recipedir = os.path.dirname(recipefile) 2179 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2180 2181 if not os.path.isdir(destlayerdir): 2182 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2183 2184 if os.path.abspath(destlayerdir) == config.workspace_path: 2185 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2186 2187 # If it's an upgrade, grab the original path 2188 origpath = None 2189 origfilelist = None 2190 append = workspace[args.recipename]['bbappend'] 2191 with open(append, 'r') as f: 2192 for line in f: 2193 if line.startswith('# original_path:'): 2194 origpath = line.split(':')[1].strip() 2195 elif line.startswith('# original_files:'): 2196 origfilelist = line.split(':')[1].split() 2197 2198 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2199 2200 if origlayerdir == config.workspace_path: 2201 # Recipe file itself is in workspace, update it there first 2202 appendlayerdir = None 2203 origrelpath = None 2204 if origpath: 2205 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2206 if origlayerpath: 2207 origrelpath = os.path.relpath(origpath, origlayerpath) 2208 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2209 if not destpath: 2210 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2211 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2212 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2213 if not os.path.abspath(destlayerbasedir) in layerdirs: 2214 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2215 2216 elif destlayerdir == origlayerdir: 2217 # Same layer, update the original recipe 2218 appendlayerdir = None 2219 destpath = None 2220 else: 2221 # Create/update a bbappend in the specified layer 2222 appendlayerdir = destlayerdir 2223 destpath = None 2224 2225 # Actually update the recipe / bbappend 2226 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2227 dry_run_output = None 2228 dry_run_outdir = None 2229 if args.dry_run: 2230 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2231 dry_run_outdir = dry_run_output.name 2232 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2233 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2234 2235 # Remove any old files in the case of an upgrade 2236 if removing_original: 2237 for fn in origfilelist: 2238 fnp = os.path.join(origpath, fn) 2239 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2240 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2241 if not args.dry_run: 2242 try: 2243 os.remove(fnp) 2244 except FileNotFoundError: 2245 pass 2246 2247 if origlayerdir == config.workspace_path and destpath: 2248 # Recipe file itself is in the workspace - need to move it and any 2249 # associated files to the specified layer 2250 no_clean = True 2251 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2252 for root, _, files in os.walk(recipedir): 2253 for fn in files: 2254 srcpath = os.path.join(root, fn) 2255 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2256 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2257 destfp = os.path.join(destdir, fn) 2258 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2259 2260 if dry_run_outdir: 2261 import difflib 2262 comparelist = [] 2263 for root, _, files in os.walk(dry_run_outdir): 2264 for fn in files: 2265 outf = os.path.join(root, fn) 2266 relf = os.path.relpath(outf, dry_run_outdir) 2267 logger.debug('dry-run: output file %s' % relf) 2268 if fn.endswith('.bb'): 2269 if origfilelist and origpath and destpath: 2270 # Need to match this up with the pre-upgrade recipe file 2271 for origf in origfilelist: 2272 if origf.endswith('.bb'): 2273 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2274 outf, 2275 os.path.abspath(os.path.join(destpath, relf)))) 2276 break 2277 else: 2278 # Compare to the existing recipe 2279 comparelist.append((recipefile, outf, recipefile)) 2280 elif fn.endswith('.bbappend'): 2281 if appendfile: 2282 if os.path.exists(appendfile): 2283 comparelist.append((appendfile, outf, appendfile)) 2284 else: 2285 comparelist.append((None, outf, appendfile)) 2286 else: 2287 if destpath: 2288 recipedest = destpath 2289 elif appendfile: 2290 recipedest = os.path.dirname(appendfile) 2291 else: 2292 recipedest = os.path.dirname(recipefile) 2293 destfp = os.path.join(recipedest, relf) 2294 if os.path.exists(destfp): 2295 comparelist.append((destfp, outf, destfp)) 2296 output = '' 2297 for oldfile, newfile, newfileshow in comparelist: 2298 if oldfile: 2299 with open(oldfile, 'r') as f: 2300 oldlines = f.readlines() 2301 else: 2302 oldfile = '/dev/null' 2303 oldlines = [] 2304 with open(newfile, 'r') as f: 2305 newlines = f.readlines() 2306 if not newfileshow: 2307 newfileshow = newfile 2308 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2309 difflines = list(diff) 2310 if difflines: 2311 output += ''.join(difflines) 2312 if output: 2313 logger.info('Diff of changed files:\n%s' % output) 2314 finally: 2315 tinfoil.shutdown() 2316 2317 # Everything else has succeeded, we can now reset 2318 if args.dry_run: 2319 logger.info('Resetting recipe (dry-run)') 2320 else: 2321 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2322 2323 return 0 2324 2325 2326def get_default_srctree(config, recipename=''): 2327 """Get the default srctree path""" 2328 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2329 if recipename: 2330 return os.path.join(srctreeparent, 'sources', recipename) 2331 else: 2332 return os.path.join(srctreeparent, 'sources') 2333 2334def register_commands(subparsers, context): 2335 """Register devtool subcommands from this plugin""" 2336 2337 defsrctree = get_default_srctree(context.config) 2338 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2339 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2340 group='starting', order=100) 2341 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2342 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2343 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2344 group = parser_add.add_mutually_exclusive_group() 2345 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2346 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2347 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2348 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2349 parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true") 2350 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2351 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2352 group = parser_add.add_mutually_exclusive_group() 2353 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2354 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2355 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2356 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2357 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2358 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2359 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2360 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2361 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2362 2363 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2364 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2365 group='starting', order=90) 2366 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2367 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2368 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2369 group = parser_modify.add_mutually_exclusive_group() 2370 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2371 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2372 group = parser_modify.add_mutually_exclusive_group() 2373 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2374 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2375 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2376 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2377 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2378 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2379 2380 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2381 description='Extracts the source for an existing recipe', 2382 group='advanced') 2383 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2384 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2385 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2386 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2387 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2388 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2389 2390 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2391 description='Synchronize the previously extracted source tree for an existing recipe', 2392 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2393 group='advanced') 2394 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2395 parser_sync.add_argument('srctree', help='Path to the source tree') 2396 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2397 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2398 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2399 2400 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2401 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2402 group='working', order=10) 2403 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2404 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2405 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2406 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2407 parser_rename.set_defaults(func=rename) 2408 2409 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2410 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2411 group='working', order=-90) 2412 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2413 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2414 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2415 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2416 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2417 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2418 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2419 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2420 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2421 parser_update_recipe.set_defaults(func=update_recipe) 2422 2423 parser_status = subparsers.add_parser('status', help='Show workspace status', 2424 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2425 group='info', order=100) 2426 parser_status.set_defaults(func=status) 2427 2428 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2429 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2430 group='working', order=-100) 2431 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2432 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2433 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2434 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2435 parser_reset.set_defaults(func=reset) 2436 2437 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2438 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2439 group='working', order=-100) 2440 parser_finish.add_argument('recipename', help='Recipe to finish') 2441 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2442 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2443 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2444 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2445 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2446 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2447 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2448 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2449 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2450 parser_finish.set_defaults(func=finish) 2451