1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.npm_dev: 149 extracmdopts += ' --npm-dev' 150 if args.no_pypi: 151 extracmdopts += ' --no-pypi' 152 if args.mirrors: 153 extracmdopts += ' --mirrors' 154 if args.srcrev: 155 extracmdopts += ' --srcrev %s' % args.srcrev 156 if args.srcbranch: 157 extracmdopts += ' --srcbranch %s' % args.srcbranch 158 if args.provides: 159 extracmdopts += ' --provides %s' % args.provides 160 161 tempdir = tempfile.mkdtemp(prefix='devtool') 162 try: 163 try: 164 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 165 except bb.process.ExecutionError as e: 166 if e.exitcode == 15: 167 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 168 else: 169 raise DevtoolError('Command \'%s\' failed' % e.command) 170 171 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 172 if recipes: 173 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 174 if recipename in workspace: 175 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 176 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 177 bb.utils.mkdirhier(recipedir) 178 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 179 appendfile = recipe_to_append(recipefile, config) 180 if os.path.exists(appendfile): 181 # This shouldn't be possible, but just in case 182 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 183 if os.path.exists(recipefile): 184 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 185 if tmpsrcdir: 186 srctree = os.path.join(srctreeparent, recipename) 187 if os.path.exists(tmpsrcdir): 188 if os.path.exists(srctree): 189 if os.path.isdir(srctree): 190 try: 191 os.rmdir(srctree) 192 except OSError as e: 193 if e.errno == errno.ENOTEMPTY: 194 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 195 else: 196 raise 197 else: 198 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 199 logger.info('Using default source tree path %s' % srctree) 200 shutil.move(tmpsrcdir, srctree) 201 else: 202 raise DevtoolError('Couldn\'t find source tree created by recipetool') 203 bb.utils.mkdirhier(recipedir) 204 shutil.move(recipes[0], recipefile) 205 # Move any additional files created by recipetool 206 for fn in os.listdir(tempdir): 207 shutil.move(os.path.join(tempdir, fn), recipedir) 208 else: 209 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 210 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 211 if os.path.exists(attic_recipe): 212 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 213 finally: 214 if tmpsrcdir and os.path.exists(tmpsrcdir): 215 shutil.rmtree(tmpsrcdir) 216 shutil.rmtree(tempdir) 217 218 for fn in os.listdir(recipedir): 219 _add_md5(config, recipename, os.path.join(recipedir, fn)) 220 221 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 222 try: 223 try: 224 rd = tinfoil.parse_recipe_file(recipefile, False) 225 except Exception as e: 226 logger.error(str(e)) 227 rd = None 228 if not rd: 229 # Parsing failed. We just created this recipe and we shouldn't 230 # leave it in the workdir or it'll prevent bitbake from starting 231 movefn = '%s.parsefailed' % recipefile 232 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 233 shutil.move(recipefile, movefn) 234 return 1 235 236 if args.fetchuri and not args.no_git: 237 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 238 239 initial_rev = {} 240 if os.path.exists(os.path.join(srctree, '.git')): 241 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 242 initial_rev["."] = stdout.rstrip() 243 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree) 244 for line in stdout.splitlines(): 245 (rev, submodule) = line.split() 246 initial_rev[os.path.relpath(submodule, srctree)] = rev 247 248 if args.src_subdir: 249 srctree = os.path.join(srctree, args.src_subdir) 250 251 bb.utils.mkdirhier(os.path.dirname(appendfile)) 252 with open(appendfile, 'w') as f: 253 f.write('inherit externalsrc\n') 254 f.write('EXTERNALSRC = "%s"\n' % srctree) 255 256 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 257 if b_is_s: 258 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 259 if initial_rev: 260 for key, value in initial_rev.items(): 261 f.write('\n# initial_rev %s: %s\n' % (key, value)) 262 263 if args.binary: 264 f.write('do_install:append() {\n') 265 f.write(' rm -rf ${D}/.git\n') 266 f.write(' rm -f ${D}/singletask.lock\n') 267 f.write('}\n') 268 269 if bb.data.inherits_class('npm', rd): 270 f.write('python do_configure:append() {\n') 271 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 272 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 273 f.write(' bb.utils.remove(lockfile)\n') 274 f.write('}\n') 275 276 # Check if the new layer provides recipes whose priorities have been 277 # overriden by PREFERRED_PROVIDER. 278 recipe_name = rd.getVar('PN') 279 provides = rd.getVar('PROVIDES') 280 # Search every item defined in PROVIDES 281 for recipe_provided in provides.split(): 282 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 283 current_pprovider = rd.getVar(preferred_provider) 284 if current_pprovider and current_pprovider != recipe_name: 285 if args.fixed_setup: 286 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 287 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 288 with open(layerconf_file, 'a') as f: 289 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 290 else: 291 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 292 break 293 294 _add_md5(config, recipename, appendfile) 295 296 check_prerelease_version(rd.getVar('PV'), 'devtool add') 297 298 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 299 300 finally: 301 tinfoil.shutdown() 302 303 return 0 304 305 306def _check_compatible_recipe(pn, d): 307 """Check if the recipe is supported by devtool""" 308 if pn == 'perf': 309 raise DevtoolError("The perf recipe does not actually check out " 310 "source and thus cannot be supported by this tool", 311 4) 312 313 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 314 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 315 316 if bb.data.inherits_class('image', d): 317 raise DevtoolError("The %s recipe is an image, and therefore is not " 318 "supported by this tool" % pn, 4) 319 320 if bb.data.inherits_class('populate_sdk', d): 321 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 322 "supported by this tool" % pn, 4) 323 324 if bb.data.inherits_class('packagegroup', d): 325 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 326 "not supported by this tool" % pn, 4) 327 328 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 329 # Not an incompatibility error per se, so we don't pass the error code 330 raise DevtoolError("externalsrc is currently enabled for the %s " 331 "recipe. This prevents the normal do_patch task " 332 "from working. You will need to disable this " 333 "first." % pn) 334 335def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 336 """Common function for copying a file to the dry run output directory""" 337 relpath = os.path.relpath(dst, base_outdir) 338 if relpath.startswith('..'): 339 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 340 dst = os.path.join(dry_run_outdir, relpath) 341 dst_d = os.path.dirname(dst) 342 if dst_d: 343 bb.utils.mkdirhier(dst_d) 344 # Don't overwrite existing files, otherwise in the case of an upgrade 345 # the dry-run written out recipe will be overwritten with an unmodified 346 # version 347 if not os.path.exists(dst): 348 shutil.copy(src, dst) 349 350def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 351 """Move a file. Creates all the directory components of destination path.""" 352 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 353 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 354 if dry_run_outdir: 355 # We want to copy here, not move 356 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 357 else: 358 dst_d = os.path.dirname(dst) 359 if dst_d: 360 bb.utils.mkdirhier(dst_d) 361 shutil.move(src, dst) 362 363def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 364 """Copy a file. Creates all the directory components of destination path.""" 365 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 366 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 367 if dry_run_outdir: 368 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 369 else: 370 dst_d = os.path.dirname(dst) 371 if dst_d: 372 bb.utils.mkdirhier(dst_d) 373 shutil.copy(src, dst) 374 375def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 376 """List contents of a git treeish""" 377 import bb 378 cmd = ['git', 'ls-tree', '-z', treeish] 379 if recursive: 380 cmd.append('-r') 381 out, _ = bb.process.run(cmd, cwd=repodir) 382 ret = {} 383 if out: 384 for line in out.split('\0'): 385 if line: 386 split = line.split(None, 4) 387 ret[split[3]] = split[0:3] 388 return ret 389 390def _git_exclude_path(srctree, path): 391 """Return pathspec (list of paths) that excludes certain path""" 392 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 393 # we don't catch files that are deleted, for example. A more reliable way 394 # to implement this would be to use "negative pathspecs" which were 395 # introduced in Git v1.9.0. Revisit this when/if the required Git version 396 # becomes greater than that. 397 path = os.path.normpath(path) 398 recurse = True if len(path.split(os.path.sep)) > 1 else False 399 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 400 if path in git_files: 401 git_files.remove(path) 402 return git_files 403 else: 404 return ['.'] 405 406def _ls_tree(directory): 407 """Recursive listing of files in a directory""" 408 ret = [] 409 for root, dirs, files in os.walk(directory): 410 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 411 fname in files]) 412 return ret 413 414 415def extract(args, config, basepath, workspace): 416 """Entry point for the devtool 'extract' subcommand""" 417 import bb 418 419 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 420 if not tinfoil: 421 # Error already shown 422 return 1 423 try: 424 rd = parse_recipe(config, tinfoil, args.recipename, True) 425 if not rd: 426 return 1 427 428 srctree = os.path.abspath(args.srctree) 429 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 430 logger.info('Source tree extracted to %s' % srctree) 431 432 if initial_rev: 433 return 0 434 else: 435 return 1 436 finally: 437 tinfoil.shutdown() 438 439def sync(args, config, basepath, workspace): 440 """Entry point for the devtool 'sync' subcommand""" 441 import bb 442 443 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 444 if not tinfoil: 445 # Error already shown 446 return 1 447 try: 448 rd = parse_recipe(config, tinfoil, args.recipename, True) 449 if not rd: 450 return 1 451 452 srctree = os.path.abspath(args.srctree) 453 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 454 logger.info('Source tree %s synchronized' % srctree) 455 456 if initial_rev: 457 return 0 458 else: 459 return 1 460 finally: 461 tinfoil.shutdown() 462 463def symlink_oelocal_files_srctree(rd,srctree): 464 import oe.patch 465 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 466 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 467 # (otherwise the recipe won't build as expected) 468 local_files_dir = os.path.join(srctree, 'oe-local-files') 469 addfiles = [] 470 for root, _, files in os.walk(local_files_dir): 471 relpth = os.path.relpath(root, local_files_dir) 472 if relpth != '.': 473 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 474 for fn in files: 475 if fn == '.gitignore': 476 continue 477 destpth = os.path.join(srctree, relpth, fn) 478 if os.path.exists(destpth): 479 os.unlink(destpth) 480 if relpth != '.': 481 back_relpth = os.path.relpath(local_files_dir, root) 482 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) 483 else: 484 os.symlink('oe-local-files/%s' % fn, destpth) 485 addfiles.append(os.path.join(relpth, fn)) 486 if addfiles: 487 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 488 useroptions = [] 489 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 490 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 491 492 493def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 494 """Extract sources of a recipe""" 495 import oe.recipeutils 496 import oe.patch 497 import oe.path 498 499 pn = d.getVar('PN') 500 501 _check_compatible_recipe(pn, d) 502 503 if sync: 504 if not os.path.exists(srctree): 505 raise DevtoolError("output path %s does not exist" % srctree) 506 else: 507 if os.path.exists(srctree): 508 if not os.path.isdir(srctree): 509 raise DevtoolError("output path %s exists and is not a directory" % 510 srctree) 511 elif os.listdir(srctree): 512 raise DevtoolError("output path %s already exists and is " 513 "non-empty" % srctree) 514 515 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 516 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 517 "extract source" % pn, 4) 518 519 if not sync: 520 # Prepare for shutil.move later on 521 bb.utils.mkdirhier(srctree) 522 os.rmdir(srctree) 523 524 extra_overrides = [] 525 if not no_overrides: 526 history = d.varhistory.variable('SRC_URI') 527 for event in history: 528 if not 'flag' in event: 529 if event['op'].startswith((':append[', ':prepend[')): 530 override = event['op'].split('[')[1].split(']')[0] 531 if not override.startswith('pn-'): 532 extra_overrides.append(override) 533 # We want to remove duplicate overrides. If a recipe had multiple 534 # SRC_URI_override += values it would cause mulitple instances of 535 # overrides. This doesn't play nicely with things like creating a 536 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 537 extra_overrides = list(set(extra_overrides)) 538 if extra_overrides: 539 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 540 541 initial_rev = None 542 543 recipefile = d.getVar('FILE') 544 appendfile = recipe_to_append(recipefile, config) 545 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 546 547 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 548 # directory so that: 549 # (a) we pick up all files that get unpacked to the WORKDIR, and 550 # (b) we don't disturb the existing build 551 # However, with recipe-specific sysroots the sysroots for the recipe 552 # will be prepared under WORKDIR, and if we used the system temporary 553 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 554 # our attempts to hardlink files into the recipe-specific sysroots 555 # will fail on systems where /tmp is a different filesystem, and it 556 # would have to fall back to copying the files which is a waste of 557 # time. Put the temp directory under the WORKDIR to prevent that from 558 # being a problem. 559 tempbasedir = d.getVar('WORKDIR') 560 bb.utils.mkdirhier(tempbasedir) 561 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 562 try: 563 tinfoil.logger.setLevel(logging.WARNING) 564 565 # FIXME this results in a cache reload under control of tinfoil, which is fine 566 # except we don't get the knotty progress bar 567 568 if os.path.exists(appendfile): 569 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 570 shutil.copyfile(appendfile, appendbackup) 571 else: 572 appendbackup = None 573 bb.utils.mkdirhier(os.path.dirname(appendfile)) 574 logger.debug('writing append file %s' % appendfile) 575 with open(appendfile, 'a') as f: 576 f.write('###--- _extract_source\n') 577 f.write('deltask do_recipe_qa\n') 578 f.write('deltask do_recipe_qa_setscene\n') 579 f.write('ERROR_QA:remove = "patch-fuzz"\n') 580 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 581 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 582 if not is_kernel_yocto: 583 f.write('PATCHTOOL = "git"\n') 584 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 585 if extra_overrides: 586 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 587 f.write('inherit devtool-source\n') 588 f.write('###--- _extract_source\n') 589 590 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 591 592 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 593 bb.utils.mkdirhier(sstate_manifests) 594 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 595 with open(preservestampfile, 'w') as f: 596 f.write(d.getVar('STAMP')) 597 tinfoil.modified_files() 598 try: 599 if is_kernel_yocto: 600 # We need to generate the kernel config 601 task = 'do_configure' 602 else: 603 task = 'do_patch' 604 605 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 606 logger.info('The %s recipe has %s disabled. Running only ' 607 'do_configure task dependencies' % (pn, task)) 608 609 if 'depends' in d.getVarFlags('do_configure', False): 610 pn = d.getVarFlags('do_configure', False)['depends'] 611 pn = pn.replace('${PV}', d.getVar('PV')) 612 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 613 task = None 614 615 # Run the fetch + unpack tasks 616 res = tinfoil.build_targets(pn, 617 task, 618 handle_events=True) 619 finally: 620 if os.path.exists(preservestampfile): 621 os.remove(preservestampfile) 622 623 if not res: 624 raise DevtoolError('Extracting source for %s failed' % pn) 625 626 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 627 workshareddir = d.getVar('S') 628 if os.path.islink(srctree): 629 os.unlink(srctree) 630 631 os.symlink(workshareddir, srctree) 632 633 # The initial_rev file is created in devtool_post_unpack function that will not be executed if 634 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 635 return True, True 636 637 try: 638 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 639 initial_rev = f.read() 640 641 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 642 srcsubdir = f.read() 643 except FileNotFoundError as e: 644 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 645 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 646 647 # Check if work-shared is empty, if yes 648 # find source and copy to work-shared 649 if is_kernel_yocto: 650 workshareddir = d.getVar('STAGING_KERNEL_DIR') 651 staging_kerVer = get_staging_kver(workshareddir) 652 kernelVersion = d.getVar('LINUX_VERSION') 653 654 # handle dangling symbolic link in work-shared: 655 if os.path.islink(workshareddir): 656 os.unlink(workshareddir) 657 658 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 659 shutil.rmtree(workshareddir) 660 oe.path.copyhardlinktree(srcsubdir,workshareddir) 661 elif not os.path.exists(workshareddir): 662 oe.path.copyhardlinktree(srcsubdir,workshareddir) 663 664 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 665 srctree_localdir = os.path.join(srctree, 'oe-local-files') 666 667 if sync: 668 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 669 670 # Move oe-local-files directory to srctree 671 # As the oe-local-files is not part of the constructed git tree, 672 # remove them directly during the synchrounizating might surprise 673 # the users. Instead, we move it to oe-local-files.bak and remind 674 # user in the log message. 675 if os.path.exists(srctree_localdir + '.bak'): 676 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 677 678 if os.path.exists(srctree_localdir): 679 logger.info('Backing up current local file directory %s' % srctree_localdir) 680 shutil.move(srctree_localdir, srctree_localdir + '.bak') 681 682 if os.path.exists(tempdir_localdir): 683 logger.info('Syncing local source files to srctree...') 684 shutil.copytree(tempdir_localdir, srctree_localdir) 685 else: 686 # Move oe-local-files directory to srctree 687 if os.path.exists(tempdir_localdir): 688 logger.info('Adding local source files to srctree...') 689 shutil.move(tempdir_localdir, srcsubdir) 690 691 shutil.move(srcsubdir, srctree) 692 symlink_oelocal_files_srctree(d,srctree) 693 694 if is_kernel_yocto: 695 logger.info('Copying kernel config to srctree') 696 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 697 698 finally: 699 if appendbackup: 700 shutil.copyfile(appendbackup, appendfile) 701 elif os.path.exists(appendfile): 702 os.remove(appendfile) 703 if keep_temp: 704 logger.info('Preserving temporary directory %s' % tempdir) 705 else: 706 shutil.rmtree(tempdir) 707 return initial_rev, srcsubdir_rel 708 709def _add_md5(config, recipename, filename): 710 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 711 import bb.utils 712 713 def addfile(fn): 714 md5 = bb.utils.md5_file(fn) 715 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 716 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 717 f.seek(0, os.SEEK_SET) 718 if not md5_str in f.read(): 719 f.write(md5_str) 720 721 if os.path.isdir(filename): 722 for root, _, files in os.walk(filename): 723 for f in files: 724 addfile(os.path.join(root, f)) 725 else: 726 addfile(filename) 727 728def _check_preserve(config, recipename): 729 """Check if a file was manually changed and needs to be saved in 'attic' 730 directory""" 731 import bb.utils 732 origfile = os.path.join(config.workspace_path, '.devtool_md5') 733 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 734 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 735 with open(origfile, 'r') as f: 736 with open(newfile, 'w') as tf: 737 for line in f.readlines(): 738 splitline = line.rstrip().split('|') 739 if splitline[0] == recipename: 740 removefile = os.path.join(config.workspace_path, splitline[1]) 741 try: 742 md5 = bb.utils.md5_file(removefile) 743 except IOError as err: 744 if err.errno == 2: 745 # File no longer exists, skip it 746 continue 747 else: 748 raise 749 if splitline[2] != md5: 750 bb.utils.mkdirhier(preservepath) 751 preservefile = os.path.basename(removefile) 752 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 753 shutil.move(removefile, os.path.join(preservepath, preservefile)) 754 else: 755 os.remove(removefile) 756 else: 757 tf.write(line) 758 bb.utils.rename(newfile, origfile) 759 760def get_staging_kver(srcdir): 761 # Kernel version from work-shared 762 kerver = [] 763 staging_kerVer="" 764 if os.path.exists(srcdir) and os.listdir(srcdir): 765 with open(os.path.join(srcdir,"Makefile")) as f: 766 version = [next(f) for x in range(5)][1:4] 767 for word in version: 768 kerver.append(word.split('= ')[1].split('\n')[0]) 769 staging_kerVer = ".".join(kerver) 770 return staging_kerVer 771 772def get_staging_kbranch(srcdir): 773 staging_kbranch = "" 774 if os.path.exists(srcdir) and os.listdir(srcdir): 775 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) 776 staging_kbranch = "".join(branch.split('\n')[0]) 777 return staging_kbranch 778 779def get_real_srctree(srctree, s, workdir): 780 # Check that recipe isn't using a shared workdir 781 s = os.path.abspath(s) 782 workdir = os.path.abspath(workdir) 783 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 784 # Handle if S is set to a subdirectory of the source 785 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 786 srctree = os.path.join(srctree, srcsubdir) 787 return srctree 788 789def modify(args, config, basepath, workspace): 790 """Entry point for the devtool 'modify' subcommand""" 791 import bb 792 import oe.recipeutils 793 import oe.patch 794 import oe.path 795 796 if args.recipename in workspace: 797 raise DevtoolError("recipe %s is already in your workspace" % 798 args.recipename) 799 800 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 801 try: 802 rd = parse_recipe(config, tinfoil, args.recipename, True) 803 if not rd: 804 return 1 805 806 pn = rd.getVar('PN') 807 if pn != args.recipename: 808 logger.info('Mapping %s to %s' % (args.recipename, pn)) 809 if pn in workspace: 810 raise DevtoolError("recipe %s is already in your workspace" % 811 pn) 812 813 if args.srctree: 814 srctree = os.path.abspath(args.srctree) 815 else: 816 srctree = get_default_srctree(config, pn) 817 818 if args.no_extract and not os.path.isdir(srctree): 819 raise DevtoolError("--no-extract specified and source path %s does " 820 "not exist or is not a directory" % 821 srctree) 822 823 recipefile = rd.getVar('FILE') 824 appendfile = recipe_to_append(recipefile, config, args.wildcard) 825 if os.path.exists(appendfile): 826 raise DevtoolError("Another variant of recipe %s is already in your " 827 "workspace (only one variant of a recipe can " 828 "currently be worked on at once)" 829 % pn) 830 831 _check_compatible_recipe(pn, rd) 832 833 initial_revs = {} 834 commits = {} 835 check_commits = False 836 837 if bb.data.inherits_class('kernel-yocto', rd): 838 # Current set kernel version 839 kernelVersion = rd.getVar('LINUX_VERSION') 840 srcdir = rd.getVar('STAGING_KERNEL_DIR') 841 kbranch = rd.getVar('KBRANCH') 842 843 staging_kerVer = get_staging_kver(srcdir) 844 staging_kbranch = get_staging_kbranch(srcdir) 845 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 846 oe.path.copyhardlinktree(srcdir,srctree) 847 workdir = rd.getVar('WORKDIR') 848 srcsubdir = rd.getVar('S') 849 localfilesdir = os.path.join(srctree,'oe-local-files') 850 # Move local source files into separate subdir 851 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 852 local_files = oe.recipeutils.get_recipe_local_files(rd) 853 854 for key in local_files.copy(): 855 if key.endswith('scc'): 856 sccfile = open(local_files[key], 'r') 857 for l in sccfile: 858 line = l.split() 859 if line and line[0] in ('kconf', 'patch'): 860 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 861 if not cfg in local_files.values(): 862 local_files[line[-1]] = cfg 863 shutil.copy2(cfg, workdir) 864 sccfile.close() 865 866 # Ignore local files with subdir={BP} 867 srcabspath = os.path.abspath(srcsubdir) 868 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 869 if local_files: 870 for fname in local_files: 871 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 872 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 873 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 874 875 symlink_oelocal_files_srctree(rd,srctree) 876 877 task = 'do_configure' 878 res = tinfoil.build_targets(pn, task, handle_events=True) 879 880 # Copy .config to workspace 881 kconfpath = rd.getVar('B') 882 logger.info('Copying kernel config to workspace') 883 shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 884 885 # Set this to true, we still need to get initial_rev 886 # by parsing the git repo 887 args.no_extract = True 888 889 if not args.no_extract: 890 initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 891 if not initial_revs["."]: 892 return 1 893 logger.info('Source tree extracted to %s' % srctree) 894 895 if os.path.exists(os.path.join(srctree, '.git')): 896 # Get list of commits since this revision 897 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) 898 commits["."] = stdout.split() 899 check_commits = True 900 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) 901 for line in stdout.splitlines(): 902 (rev, submodule_path) = line.split() 903 submodule = os.path.relpath(submodule_path, srctree) 904 initial_revs[submodule] = rev 905 (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path) 906 commits[submodule] = stdout.split() 907 else: 908 if os.path.exists(os.path.join(srctree, '.git')): 909 # Check if it's a tree previously extracted by us. This is done 910 # by ensuring that devtool-base and args.branch (devtool) exist. 911 # The check_commits logic will cause an exception if either one 912 # of these doesn't exist 913 try: 914 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 915 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 916 except bb.process.ExecutionError: 917 stdout = '' 918 if stdout: 919 check_commits = True 920 for line in stdout.splitlines(): 921 if line.startswith('*'): 922 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 923 initial_revs["."] = stdout.rstrip() 924 if not initial_revs["."]: 925 # Otherwise, just grab the head revision 926 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 927 initial_revs["."] = stdout.rstrip() 928 929 branch_patches = {} 930 if check_commits: 931 # Check if there are override branches 932 (stdout, _) = bb.process.run('git branch', cwd=srctree) 933 branches = [] 934 for line in stdout.rstrip().splitlines(): 935 branchname = line[2:].rstrip() 936 if branchname.startswith(override_branch_prefix): 937 branches.append(branchname) 938 if branches: 939 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 940 branches.insert(0, args.branch) 941 seen_patches = [] 942 for branch in branches: 943 branch_patches[branch] = [] 944 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 945 for line in stdout.splitlines(): 946 line = line.strip() 947 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 948 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 949 if not origpatch in seen_patches: 950 seen_patches.append(origpatch) 951 branch_patches[branch].append(origpatch) 952 953 # Need to grab this here in case the source is within a subdirectory 954 srctreebase = srctree 955 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 956 957 bb.utils.mkdirhier(os.path.dirname(appendfile)) 958 with open(appendfile, 'w') as f: 959 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 960 # Local files can be modified/tracked in separate subdir under srctree 961 # Mostly useful for packages with S != WORKDIR 962 f.write('FILESPATH:prepend := "%s:"\n' % 963 os.path.join(srctreebase, 'oe-local-files')) 964 f.write('# srctreebase: %s\n' % srctreebase) 965 966 f.write('\ninherit externalsrc\n') 967 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 968 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 969 970 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 971 if b_is_s: 972 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 973 974 if bb.data.inherits_class('kernel', rd): 975 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 976 'do_fetch do_unpack do_kernel_configcheck"\n') 977 f.write('\ndo_patch[noexec] = "1"\n') 978 f.write('\ndo_configure:append() {\n' 979 ' cp ${B}/.config ${S}/.config.baseline\n' 980 ' ln -sfT ${B}/.config ${S}/.config.new\n' 981 '}\n') 982 f.write('\ndo_kernel_configme:prepend() {\n' 983 ' if [ -e ${S}/.config ]; then\n' 984 ' mv ${S}/.config ${S}/.config.old\n' 985 ' fi\n' 986 '}\n') 987 if rd.getVarFlag('do_menuconfig','task'): 988 f.write('\ndo_configure:append() {\n' 989 ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n' 990 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' 991 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' 992 ' fi\n' 993 '}\n') 994 if initial_revs: 995 for name, rev in initial_revs.items(): 996 f.write('\n# initial_rev %s: %s\n' % (name, rev)) 997 if name in commits: 998 for commit in commits[name]: 999 f.write('# commit %s: %s\n' % (name, commit)) 1000 if branch_patches: 1001 for branch in branch_patches: 1002 if branch == args.branch: 1003 continue 1004 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 1005 1006 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 1007 1008 _add_md5(config, pn, appendfile) 1009 1010 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 1011 1012 finally: 1013 tinfoil.shutdown() 1014 1015 return 0 1016 1017 1018def rename(args, config, basepath, workspace): 1019 """Entry point for the devtool 'rename' subcommand""" 1020 import bb 1021 import oe.recipeutils 1022 1023 check_workspace_recipe(workspace, args.recipename) 1024 1025 if not (args.newname or args.version): 1026 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 1027 1028 recipefile = workspace[args.recipename]['recipefile'] 1029 if not recipefile: 1030 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1031 1032 if args.newname and args.newname != args.recipename: 1033 reason = oe.recipeutils.validate_pn(args.newname) 1034 if reason: 1035 raise DevtoolError(reason) 1036 newname = args.newname 1037 else: 1038 newname = args.recipename 1039 1040 append = workspace[args.recipename]['bbappend'] 1041 appendfn = os.path.splitext(os.path.basename(append))[0] 1042 splitfn = appendfn.split('_') 1043 if len(splitfn) > 1: 1044 origfnver = appendfn.split('_')[1] 1045 else: 1046 origfnver = '' 1047 1048 recipefilemd5 = None 1049 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1050 try: 1051 rd = parse_recipe(config, tinfoil, args.recipename, True) 1052 if not rd: 1053 return 1 1054 1055 bp = rd.getVar('BP') 1056 bpn = rd.getVar('BPN') 1057 if newname != args.recipename: 1058 localdata = rd.createCopy() 1059 localdata.setVar('PN', newname) 1060 newbpn = localdata.getVar('BPN') 1061 else: 1062 newbpn = bpn 1063 s = rd.getVar('S', False) 1064 src_uri = rd.getVar('SRC_URI', False) 1065 pv = rd.getVar('PV') 1066 1067 # Correct variable values that refer to the upstream source - these 1068 # values must stay the same, so if the name/version are changing then 1069 # we need to fix them up 1070 new_s = s 1071 new_src_uri = src_uri 1072 if newbpn != bpn: 1073 # ${PN} here is technically almost always incorrect, but people do use it 1074 new_s = new_s.replace('${BPN}', bpn) 1075 new_s = new_s.replace('${PN}', bpn) 1076 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1077 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1078 new_src_uri = new_src_uri.replace('${PN}', bpn) 1079 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1080 if args.version and origfnver == pv: 1081 new_s = new_s.replace('${PV}', pv) 1082 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1083 new_src_uri = new_src_uri.replace('${PV}', pv) 1084 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1085 patchfields = {} 1086 if new_s != s: 1087 patchfields['S'] = new_s 1088 if new_src_uri != src_uri: 1089 patchfields['SRC_URI'] = new_src_uri 1090 if patchfields: 1091 recipefilemd5 = bb.utils.md5_file(recipefile) 1092 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1093 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1094 finally: 1095 tinfoil.shutdown() 1096 1097 if args.version: 1098 newver = args.version 1099 else: 1100 newver = origfnver 1101 1102 if newver: 1103 newappend = '%s_%s.bbappend' % (newname, newver) 1104 newfile = '%s_%s.bb' % (newname, newver) 1105 else: 1106 newappend = '%s.bbappend' % newname 1107 newfile = '%s.bb' % newname 1108 1109 oldrecipedir = os.path.dirname(recipefile) 1110 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1111 if oldrecipedir != newrecipedir: 1112 bb.utils.mkdirhier(newrecipedir) 1113 1114 newappend = os.path.join(os.path.dirname(append), newappend) 1115 newfile = os.path.join(newrecipedir, newfile) 1116 1117 # Rename bbappend 1118 logger.info('Renaming %s to %s' % (append, newappend)) 1119 bb.utils.rename(append, newappend) 1120 # Rename recipe file 1121 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1122 bb.utils.rename(recipefile, newfile) 1123 1124 # Rename source tree if it's the default path 1125 appendmd5 = None 1126 if not args.no_srctree: 1127 srctree = workspace[args.recipename]['srctree'] 1128 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1129 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1130 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1131 shutil.move(srctree, newsrctree) 1132 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1133 appendmd5 = bb.utils.md5_file(newappend) 1134 appendlines = [] 1135 with open(newappend, 'r') as f: 1136 for line in f: 1137 appendlines.append(line) 1138 with open(newappend, 'w') as f: 1139 for line in appendlines: 1140 if srctree in line: 1141 line = line.replace(srctree, newsrctree) 1142 f.write(line) 1143 newappendmd5 = bb.utils.md5_file(newappend) 1144 1145 bpndir = None 1146 newbpndir = None 1147 if newbpn != bpn: 1148 bpndir = os.path.join(oldrecipedir, bpn) 1149 if os.path.exists(bpndir): 1150 newbpndir = os.path.join(newrecipedir, newbpn) 1151 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1152 shutil.move(bpndir, newbpndir) 1153 1154 bpdir = None 1155 newbpdir = None 1156 if newver != origfnver or newbpn != bpn: 1157 bpdir = os.path.join(oldrecipedir, bp) 1158 if os.path.exists(bpdir): 1159 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1160 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1161 shutil.move(bpdir, newbpdir) 1162 1163 if oldrecipedir != newrecipedir: 1164 # Move any stray files and delete the old recipe directory 1165 for entry in os.listdir(oldrecipedir): 1166 oldpath = os.path.join(oldrecipedir, entry) 1167 newpath = os.path.join(newrecipedir, entry) 1168 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1169 shutil.move(oldpath, newpath) 1170 os.rmdir(oldrecipedir) 1171 1172 # Now take care of entries in .devtool_md5 1173 md5entries = [] 1174 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1175 for line in f: 1176 md5entries.append(line) 1177 1178 if bpndir and newbpndir: 1179 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1180 else: 1181 relbpndir = None 1182 if bpdir and newbpdir: 1183 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1184 else: 1185 relbpdir = None 1186 1187 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1188 for entry in md5entries: 1189 splitentry = entry.rstrip().split('|') 1190 if len(splitentry) > 2: 1191 if splitentry[0] == args.recipename: 1192 splitentry[0] = newname 1193 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1194 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1195 if appendmd5 and splitentry[2] == appendmd5: 1196 splitentry[2] = newappendmd5 1197 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1198 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1199 if recipefilemd5 and splitentry[2] == recipefilemd5: 1200 splitentry[2] = newrecipefilemd5 1201 elif relbpndir and splitentry[1].startswith(relbpndir): 1202 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1203 elif relbpdir and splitentry[1].startswith(relbpdir): 1204 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1205 entry = '|'.join(splitentry) + '\n' 1206 f.write(entry) 1207 return 0 1208 1209 1210def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1211 """Get initial and update rev of a recipe. These are the start point of the 1212 whole patchset and start point for the patches to be re-generated/updated. 1213 """ 1214 import bb 1215 1216 # Get current branch 1217 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1218 cwd=srctree) 1219 branchname = stdout.rstrip() 1220 1221 # Parse initial rev from recipe if not specified 1222 commits = {} 1223 patches = [] 1224 initial_revs = {} 1225 with open(recipe_path, 'r') as f: 1226 for line in f: 1227 pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$' 1228 match = re.search(pattern, line) 1229 if match: 1230 name = match.group(1) 1231 rev = match.group(2) 1232 if line.startswith('# initial_rev'): 1233 if not (name == "." and initial_rev): 1234 initial_revs[name] = rev 1235 elif line.startswith('# commit') and not force_patch_refresh: 1236 if name not in commits: 1237 commits[name] = [rev] 1238 else: 1239 commits[name].append(rev) 1240 elif line.startswith('# patches_%s:' % branchname): 1241 patches = line.split(':')[-1].strip().split(',') 1242 1243 update_revs = dict(initial_revs) 1244 changed_revs = {} 1245 for name, rev in initial_revs.items(): 1246 # Find first actually changed revision 1247 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1248 rev, cwd=os.path.join(srctree, name)) 1249 newcommits = stdout.split() 1250 if name in commits: 1251 for i in range(min(len(commits[name]), len(newcommits))): 1252 if newcommits[i] == commits[name][i]: 1253 update_revs[name] = commits[name][i] 1254 1255 try: 1256 stdout, _ = bb.process.run('git cherry devtool-patched', 1257 cwd=os.path.join(srctree, name)) 1258 except bb.process.ExecutionError as err: 1259 stdout = None 1260 1261 if stdout is not None and not force_patch_refresh: 1262 for line in stdout.splitlines(): 1263 if line.startswith('+ '): 1264 rev = line.split()[1] 1265 if rev in newcommits: 1266 if name not in changed_revs: 1267 changed_revs[name] = [rev] 1268 else: 1269 changed_revs[name].append(rev) 1270 1271 return initial_revs, update_revs, changed_revs, patches 1272 1273def _remove_file_entries(srcuri, filelist): 1274 """Remove file:// entries from SRC_URI""" 1275 remaining = filelist[:] 1276 entries = [] 1277 for fname in filelist: 1278 basename = os.path.basename(fname) 1279 for i in range(len(srcuri)): 1280 if (srcuri[i].startswith('file://') and 1281 os.path.basename(srcuri[i].split(';')[0]) == basename): 1282 entries.append(srcuri[i]) 1283 remaining.remove(fname) 1284 srcuri.pop(i) 1285 break 1286 return entries, remaining 1287 1288def _replace_srcuri_entry(srcuri, filename, newentry): 1289 """Replace entry corresponding to specified file with a new entry""" 1290 basename = os.path.basename(filename) 1291 for i in range(len(srcuri)): 1292 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1293 srcuri.pop(i) 1294 srcuri.insert(i, newentry) 1295 break 1296 1297def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1298 """Unlink existing patch files""" 1299 1300 dry_run_suffix = ' (dry-run)' if dry_run else '' 1301 1302 for path in files: 1303 if append: 1304 if not destpath: 1305 raise Exception('destpath should be set here') 1306 path = os.path.join(destpath, os.path.basename(path)) 1307 1308 if os.path.exists(path): 1309 if not no_report_remove: 1310 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1311 if not dry_run: 1312 # FIXME "git rm" here would be nice if the file in question is 1313 # tracked 1314 # FIXME there's a chance that this file is referred to by 1315 # another recipe, in which case deleting wouldn't be the 1316 # right thing to do 1317 os.remove(path) 1318 # Remove directory if empty 1319 try: 1320 os.rmdir(os.path.dirname(path)) 1321 except OSError as ose: 1322 if ose.errno != errno.ENOTEMPTY: 1323 raise 1324 1325 1326def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None): 1327 """Export patches from srctree to given location. 1328 Returns three-tuple of dicts: 1329 1. updated - patches that already exist in SRCURI 1330 2. added - new patches that don't exist in SRCURI 1331 3 removed - patches that exist in SRCURI but not in exported patches 1332 In each dict the key is the 'basepath' of the URI and value is: 1333 - for updated and added dicts, a dict with 2 optionnal keys: 1334 - 'path': the absolute path to the existing file in recipe space (if any) 1335 - 'patchdir': the directory in wich the patch should be applied (if any) 1336 - for removed dict, the absolute path to the existing file in recipe space 1337 """ 1338 import oe.recipeutils 1339 from oe.patch import GitApplyTree 1340 updated = OrderedDict() 1341 added = OrderedDict() 1342 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1343 1344 existing_patches = dict((os.path.basename(path), path) for path in 1345 oe.recipeutils.get_recipe_patches(rd)) 1346 logger.debug('Existing patches: %s' % existing_patches) 1347 1348 # Generate patches from Git, exclude local files directory 1349 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1350 GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec) 1351 for dirpath, dirnames, filenames in os.walk(destdir): 1352 new_patches = filenames 1353 reldirpath = os.path.relpath(dirpath, destdir) 1354 for new_patch in new_patches: 1355 # Strip numbering from patch names. If it's a git sequence named patch, 1356 # the numbers might not match up since we are starting from a different 1357 # revision This does assume that people are using unique shortlog 1358 # values, but they ought to be anyway... 1359 new_basename = seqpatch_re.match(new_patch).group(2) 1360 match_name = None 1361 for old_patch in existing_patches: 1362 old_basename = seqpatch_re.match(old_patch).group(2) 1363 old_basename_splitext = os.path.splitext(old_basename) 1364 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1365 old_patch_noext = os.path.splitext(old_patch)[0] 1366 match_name = old_patch_noext 1367 break 1368 elif new_basename == old_basename: 1369 match_name = old_patch 1370 break 1371 if match_name: 1372 # Rename patch files 1373 if new_patch != match_name: 1374 bb.utils.rename(os.path.join(destdir, new_patch), 1375 os.path.join(destdir, match_name)) 1376 # Need to pop it off the list now before checking changed_revs 1377 oldpath = existing_patches.pop(old_patch) 1378 if changed_revs is not None and dirpath in changed_revs: 1379 # Avoid updating patches that have not actually changed 1380 with open(os.path.join(dirpath, match_name), 'r') as f: 1381 firstlineitems = f.readline().split() 1382 # Looking for "From <hash>" line 1383 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1384 if not firstlineitems[1] in changed_revs[dirpath]: 1385 continue 1386 # Recompress if necessary 1387 if oldpath.endswith(('.gz', '.Z')): 1388 bb.process.run(['gzip', match_name], cwd=destdir) 1389 if oldpath.endswith('.gz'): 1390 match_name += '.gz' 1391 else: 1392 match_name += '.Z' 1393 elif oldpath.endswith('.bz2'): 1394 bb.process.run(['bzip2', match_name], cwd=destdir) 1395 match_name += '.bz2' 1396 updated[match_name] = {'path' : oldpath} 1397 if reldirpath != ".": 1398 updated[match_name]['patchdir'] = reldirpath 1399 else: 1400 added[new_patch] = {} 1401 if reldirpath != ".": 1402 added[new_patch]['patchdir'] = reldirpath 1403 1404 return (updated, added, existing_patches) 1405 1406 1407def _create_kconfig_diff(srctree, rd, outfile): 1408 """Create a kconfig fragment""" 1409 # Only update config fragment if both config files exist 1410 orig_config = os.path.join(srctree, '.config.baseline') 1411 new_config = os.path.join(srctree, '.config.new') 1412 if os.path.exists(orig_config) and os.path.exists(new_config): 1413 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1414 '--unchanged-line-format=', orig_config, new_config] 1415 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1416 stderr=subprocess.PIPE) 1417 stdout, stderr = pipe.communicate() 1418 if pipe.returncode == 1: 1419 logger.info("Updating config fragment %s" % outfile) 1420 with open(outfile, 'wb') as fobj: 1421 fobj.write(stdout) 1422 elif pipe.returncode == 0: 1423 logger.info("Would remove config fragment %s" % outfile) 1424 if os.path.exists(outfile): 1425 # Remove fragment file in case of empty diff 1426 logger.info("Removing config fragment %s" % outfile) 1427 os.unlink(outfile) 1428 else: 1429 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1430 return True 1431 return False 1432 1433 1434def _export_local_files(srctree, rd, destdir, srctreebase): 1435 """Copy local files from srctree to given location. 1436 Returns three-tuple of dicts: 1437 1. updated - files that already exist in SRCURI 1438 2. added - new files files that don't exist in SRCURI 1439 3 removed - files that exist in SRCURI but not in exported files 1440 In each dict the key is the 'basepath' of the URI and value is the 1441 absolute path to the existing file in recipe space (if any). 1442 """ 1443 import oe.recipeutils 1444 1445 # Find out local files (SRC_URI files that exist in the "recipe space"). 1446 # Local files that reside in srctree are not included in patch generation. 1447 # Instead they are directly copied over the original source files (in 1448 # recipe space). 1449 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1450 new_set = None 1451 updated = OrderedDict() 1452 added = OrderedDict() 1453 removed = OrderedDict() 1454 1455 # Get current branch and return early with empty lists 1456 # if on one of the override branches 1457 # (local files are provided only for the main branch and processing 1458 # them against lists from recipe overrides will result in mismatches 1459 # and broken modifications to recipes). 1460 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1461 cwd=srctree) 1462 branchname = stdout.rstrip() 1463 if branchname.startswith(override_branch_prefix): 1464 return (updated, added, removed) 1465 1466 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1467 git_files = _git_ls_tree(srctree) 1468 if 'oe-local-files' in git_files: 1469 # If tracked by Git, take the files from srctree HEAD. First get 1470 # the tree object of the directory 1471 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1472 tree = git_files['oe-local-files'][2] 1473 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1474 env=dict(os.environ, GIT_WORK_TREE=destdir, 1475 GIT_INDEX_FILE=tmp_index)) 1476 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1477 elif os.path.isdir(local_files_dir): 1478 # If not tracked by Git, just copy from working copy 1479 new_set = _ls_tree(local_files_dir) 1480 bb.process.run(['cp', '-ax', 1481 os.path.join(local_files_dir, '.'), destdir]) 1482 else: 1483 new_set = [] 1484 1485 # Special handling for kernel config 1486 if bb.data.inherits_class('kernel-yocto', rd): 1487 fragment_fn = 'devtool-fragment.cfg' 1488 fragment_path = os.path.join(destdir, fragment_fn) 1489 if _create_kconfig_diff(srctree, rd, fragment_path): 1490 if os.path.exists(fragment_path): 1491 if fragment_fn not in new_set: 1492 new_set.append(fragment_fn) 1493 # Copy fragment to local-files 1494 if os.path.isdir(local_files_dir): 1495 shutil.copy2(fragment_path, local_files_dir) 1496 else: 1497 if fragment_fn in new_set: 1498 new_set.remove(fragment_fn) 1499 # Remove fragment from local-files 1500 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1501 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1502 1503 # Special handling for cml1, ccmake, etc bbclasses that generated 1504 # configuration fragment files that are consumed as source files 1505 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1506 if bb.data.inherits_class(frag_class, rd): 1507 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1508 if os.path.exists(srcpath): 1509 if frag_name not in new_set: 1510 new_set.append(frag_name) 1511 # copy fragment into destdir 1512 shutil.copy2(srcpath, destdir) 1513 # copy fragment into local files if exists 1514 if os.path.isdir(local_files_dir): 1515 shutil.copy2(srcpath, local_files_dir) 1516 1517 if new_set is not None: 1518 for fname in new_set: 1519 if fname in existing_files: 1520 origpath = existing_files.pop(fname) 1521 workpath = os.path.join(local_files_dir, fname) 1522 if not filecmp.cmp(origpath, workpath): 1523 updated[fname] = origpath 1524 elif fname != '.gitignore': 1525 added[fname] = None 1526 1527 workdir = rd.getVar('WORKDIR') 1528 s = rd.getVar('S') 1529 if not s.endswith(os.sep): 1530 s += os.sep 1531 1532 if workdir != s: 1533 # Handle files where subdir= was specified 1534 for fname in list(existing_files.keys()): 1535 # FIXME handle both subdir starting with BP and not? 1536 fworkpath = os.path.join(workdir, fname) 1537 if fworkpath.startswith(s): 1538 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1539 if os.path.exists(fpath): 1540 origpath = existing_files.pop(fname) 1541 if not filecmp.cmp(origpath, fpath): 1542 updated[fpath] = origpath 1543 1544 removed = existing_files 1545 return (updated, added, removed) 1546 1547 1548def _determine_files_dir(rd): 1549 """Determine the appropriate files directory for a recipe""" 1550 recipedir = rd.getVar('FILE_DIRNAME') 1551 for entry in rd.getVar('FILESPATH').split(':'): 1552 relpth = os.path.relpath(entry, recipedir) 1553 if not os.sep in relpth: 1554 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1555 if os.path.isdir(entry): 1556 return entry 1557 return os.path.join(recipedir, rd.getVar('BPN')) 1558 1559 1560def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1561 """Implement the 'srcrev' mode of update-recipe""" 1562 import bb 1563 import oe.recipeutils 1564 1565 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1566 1567 recipefile = rd.getVar('FILE') 1568 recipedir = os.path.basename(recipefile) 1569 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1570 1571 # Get original SRCREV 1572 old_srcrev = rd.getVar('SRCREV') or '' 1573 if old_srcrev == "INVALID": 1574 raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository') 1575 old_srcrev = {'.': old_srcrev} 1576 1577 # Get HEAD revision 1578 try: 1579 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1580 except bb.process.ExecutionError as err: 1581 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1582 (srctree, err)) 1583 srcrev = stdout.strip() 1584 if len(srcrev) != 40: 1585 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1586 1587 destpath = None 1588 remove_files = [] 1589 patchfields = {} 1590 patchfields['SRCREV'] = srcrev 1591 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1592 srcuri = orig_src_uri.split() 1593 tempdir = tempfile.mkdtemp(prefix='devtool') 1594 update_srcuri = False 1595 appendfile = None 1596 try: 1597 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1598 srctreebase = workspace[recipename]['srctreebase'] 1599 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1600 if not no_remove: 1601 # Find list of existing patches in recipe file 1602 patches_dir = tempfile.mkdtemp(dir=tempdir) 1603 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1604 patches_dir) 1605 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1606 1607 # Remove deleted local files and "overlapping" patches 1608 remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value] 1609 if remove_files: 1610 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1611 update_srcuri = True 1612 1613 if appendlayerdir: 1614 files = dict((os.path.join(local_files_dir, key), val) for 1615 key, val in list(upd_f.items()) + list(new_f.items())) 1616 removevalues = {} 1617 if update_srcuri: 1618 removevalues = {'SRC_URI': removedentries} 1619 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1620 if dry_run_outdir: 1621 logger.info('Creating bbappend (dry-run)') 1622 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1623 rd, appendlayerdir, files, wildcardver=wildcard_version, 1624 extralines=patchfields, removevalues=removevalues, 1625 redirect_output=dry_run_outdir) 1626 else: 1627 files_dir = _determine_files_dir(rd) 1628 for basepath, path in upd_f.items(): 1629 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1630 if os.path.isabs(basepath): 1631 # Original file (probably with subdir pointing inside source tree) 1632 # so we do not want to move it, just copy 1633 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1634 else: 1635 _move_file(os.path.join(local_files_dir, basepath), path, 1636 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1637 update_srcuri= True 1638 for basepath, path in new_f.items(): 1639 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1640 _move_file(os.path.join(local_files_dir, basepath), 1641 os.path.join(files_dir, basepath), 1642 dry_run_outdir=dry_run_outdir, 1643 base_outdir=recipedir) 1644 srcuri.append('file://%s' % basepath) 1645 update_srcuri = True 1646 if update_srcuri: 1647 patchfields['SRC_URI'] = ' '.join(srcuri) 1648 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1649 finally: 1650 shutil.rmtree(tempdir) 1651 if not 'git://' in orig_src_uri: 1652 logger.info('You will need to update SRC_URI within the recipe to ' 1653 'point to a git repository where you have pushed your ' 1654 'changes') 1655 1656 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1657 return True, appendfile, remove_files 1658 1659def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1660 """Implement the 'patch' mode of update-recipe""" 1661 import bb 1662 import oe.recipeutils 1663 1664 recipefile = rd.getVar('FILE') 1665 recipedir = os.path.dirname(recipefile) 1666 append = workspace[recipename]['bbappend'] 1667 if not os.path.exists(append): 1668 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1669 recipename) 1670 srctreebase = workspace[recipename]['srctreebase'] 1671 relpatchdir = os.path.relpath(srctreebase, srctree) 1672 if relpatchdir == '.': 1673 patchdir_params = {} 1674 else: 1675 patchdir_params = {'patchdir': relpatchdir} 1676 1677 def srcuri_entry(basepath, patchdir_params): 1678 if patchdir_params: 1679 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1680 else: 1681 paramstr = '' 1682 return 'file://%s%s' % (basepath, paramstr) 1683 1684 initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1685 if not initial_revs: 1686 raise DevtoolError('Unable to find initial revision - please specify ' 1687 'it with --initial-rev') 1688 1689 appendfile = None 1690 dl_dir = rd.getVar('DL_DIR') 1691 if not dl_dir.endswith('/'): 1692 dl_dir += '/' 1693 1694 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1695 1696 tempdir = tempfile.mkdtemp(prefix='devtool') 1697 try: 1698 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1699 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1700 1701 # Get updated patches from source tree 1702 patches_dir = tempfile.mkdtemp(dir=tempdir) 1703 upd_p, new_p, _ = _export_patches(srctree, rd, update_revs, 1704 patches_dir, changed_revs) 1705 # Get all patches from source tree and check if any should be removed 1706 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1707 _, _, del_p = _export_patches(srctree, rd, initial_revs, 1708 all_patches_dir) 1709 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1710 if filter_patches: 1711 new_p = OrderedDict() 1712 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1713 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) 1714 remove_files = [] 1715 if not no_remove: 1716 # Remove deleted local files and patches 1717 remove_files = list(del_f.values()) + list(del_p.values()) 1718 updatefiles = False 1719 updaterecipe = False 1720 destpath = None 1721 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1722 1723 if appendlayerdir: 1724 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1725 key, val in list(upd_f.items()) + list(new_f.items())) 1726 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1727 key, val in list(upd_p.items()) + list(new_p.items()))) 1728 1729 params = [] 1730 for file, param in files.items(): 1731 patchdir_param = dict(patchdir_params) 1732 patchdir = param.get('patchdir', ".") 1733 if patchdir != "." : 1734 if patchdir_param: 1735 patchdir_param['patchdir'] += patchdir 1736 else: 1737 patchdir_param['patchdir'] = patchdir 1738 params.append(patchdir_param) 1739 1740 if files or remove_files: 1741 removevalues = None 1742 if remove_files: 1743 removedentries, remaining = _remove_file_entries( 1744 srcuri, remove_files) 1745 if removedentries or remaining: 1746 remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for 1747 item in remaining] 1748 removevalues = {'SRC_URI': removedentries + remaining} 1749 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1750 rd, appendlayerdir, files, 1751 wildcardver=wildcard_version, 1752 removevalues=removevalues, 1753 redirect_output=dry_run_outdir, 1754 params=params) 1755 else: 1756 logger.info('No patches or local source files needed updating') 1757 else: 1758 # Update existing files 1759 files_dir = _determine_files_dir(rd) 1760 for basepath, path in upd_f.items(): 1761 logger.info('Updating file %s' % basepath) 1762 if os.path.isabs(basepath): 1763 # Original file (probably with subdir pointing inside source tree) 1764 # so we do not want to move it, just copy 1765 _copy_file(basepath, path, 1766 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1767 else: 1768 _move_file(os.path.join(local_files_dir, basepath), path, 1769 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1770 updatefiles = True 1771 for basepath, param in upd_p.items(): 1772 path = param['path'] 1773 patchdir = param.get('patchdir', ".") 1774 if patchdir != "." : 1775 patchdir_param = dict(patchdir_params) 1776 if patchdir_param: 1777 patchdir_param['patchdir'] += patchdir 1778 else: 1779 patchdir_param['patchdir'] = patchdir 1780 patchfn = os.path.join(patches_dir, patchdir, basepath) 1781 if os.path.dirname(path) + '/' == dl_dir: 1782 # This is a a downloaded patch file - we now need to 1783 # replace the entry in SRC_URI with our local version 1784 logger.info('Replacing remote patch %s with updated local version' % basepath) 1785 path = os.path.join(files_dir, basepath) 1786 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param)) 1787 updaterecipe = True 1788 else: 1789 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1790 _move_file(patchfn, path, 1791 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1792 updatefiles = True 1793 # Add any new files 1794 for basepath, path in new_f.items(): 1795 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1796 _move_file(os.path.join(local_files_dir, basepath), 1797 os.path.join(files_dir, basepath), 1798 dry_run_outdir=dry_run_outdir, 1799 base_outdir=recipedir) 1800 srcuri.append(srcuri_entry(basepath, patchdir_params)) 1801 updaterecipe = True 1802 for basepath, param in new_p.items(): 1803 patchdir = param.get('patchdir', ".") 1804 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1805 _move_file(os.path.join(patches_dir, patchdir, basepath), 1806 os.path.join(files_dir, basepath), 1807 dry_run_outdir=dry_run_outdir, 1808 base_outdir=recipedir) 1809 params = dict(patchdir_params) 1810 if patchdir != "." : 1811 if params: 1812 params['patchdir'] += patchdir 1813 else: 1814 params['patchdir'] = patchdir 1815 1816 srcuri.append(srcuri_entry(basepath, params)) 1817 updaterecipe = True 1818 # Update recipe, if needed 1819 if _remove_file_entries(srcuri, remove_files)[0]: 1820 updaterecipe = True 1821 if updaterecipe: 1822 if not dry_run_outdir: 1823 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1824 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1825 {'SRC_URI': ' '.join(srcuri)}, 1826 redirect_output=dry_run_outdir) 1827 elif not updatefiles: 1828 # Neither patches nor recipe were updated 1829 logger.info('No patches or files need updating') 1830 return False, None, [] 1831 finally: 1832 shutil.rmtree(tempdir) 1833 1834 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1835 return True, appendfile, remove_files 1836 1837def _guess_recipe_update_mode(srctree, rdata): 1838 """Guess the recipe update mode to use""" 1839 src_uri = (rdata.getVar('SRC_URI') or '').split() 1840 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1841 if not git_uris: 1842 return 'patch' 1843 # Just use the first URI for now 1844 uri = git_uris[0] 1845 # Check remote branch 1846 params = bb.fetch.decodeurl(uri)[5] 1847 upstr_branch = params['branch'] if 'branch' in params else 'master' 1848 # Check if current branch HEAD is found in upstream branch 1849 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1850 head_rev = stdout.rstrip() 1851 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1852 cwd=srctree) 1853 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1854 if 'origin/' + upstr_branch in remote_brs: 1855 return 'srcrev' 1856 1857 return 'patch' 1858 1859def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1860 srctree = workspace[recipename]['srctree'] 1861 if mode == 'auto': 1862 mode = _guess_recipe_update_mode(srctree, rd) 1863 1864 override_branches = [] 1865 mainbranch = None 1866 startbranch = None 1867 if not no_overrides: 1868 stdout, _ = bb.process.run('git branch', cwd=srctree) 1869 other_branches = [] 1870 for line in stdout.splitlines(): 1871 branchname = line[2:] 1872 if line.startswith('* '): 1873 startbranch = branchname 1874 if branchname.startswith(override_branch_prefix): 1875 override_branches.append(branchname) 1876 else: 1877 other_branches.append(branchname) 1878 1879 if override_branches: 1880 logger.debug('_update_recipe: override branches: %s' % override_branches) 1881 logger.debug('_update_recipe: other branches: %s' % other_branches) 1882 if startbranch.startswith(override_branch_prefix): 1883 if len(other_branches) == 1: 1884 mainbranch = other_branches[1] 1885 else: 1886 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1887 else: 1888 mainbranch = startbranch 1889 1890 checkedout = None 1891 anyupdated = False 1892 appendfile = None 1893 allremoved = [] 1894 if override_branches: 1895 logger.info('Handling main branch (%s)...' % mainbranch) 1896 if startbranch != mainbranch: 1897 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1898 checkedout = mainbranch 1899 try: 1900 branchlist = [mainbranch] + override_branches 1901 for branch in branchlist: 1902 crd = bb.data.createCopy(rd) 1903 if branch != mainbranch: 1904 logger.info('Handling branch %s...' % branch) 1905 override = branch[len(override_branch_prefix):] 1906 crd.appendVar('OVERRIDES', ':%s' % override) 1907 bb.process.run('git checkout %s' % branch, cwd=srctree) 1908 checkedout = branch 1909 1910 if mode == 'srcrev': 1911 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1912 elif mode == 'patch': 1913 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1914 else: 1915 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1916 if updated: 1917 anyupdated = True 1918 if appendf: 1919 appendfile = appendf 1920 allremoved.extend(removed) 1921 finally: 1922 if startbranch and checkedout != startbranch: 1923 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1924 1925 return anyupdated, appendfile, allremoved 1926 1927def update_recipe(args, config, basepath, workspace): 1928 """Entry point for the devtool 'update-recipe' subcommand""" 1929 check_workspace_recipe(workspace, args.recipename) 1930 1931 if args.append: 1932 if not os.path.exists(args.append): 1933 raise DevtoolError('bbappend destination layer directory "%s" ' 1934 'does not exist' % args.append) 1935 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1936 raise DevtoolError('conf/layer.conf not found in bbappend ' 1937 'destination layer "%s"' % args.append) 1938 1939 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1940 try: 1941 1942 rd = parse_recipe(config, tinfoil, args.recipename, True) 1943 if not rd: 1944 return 1 1945 1946 dry_run_output = None 1947 dry_run_outdir = None 1948 if args.dry_run: 1949 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1950 dry_run_outdir = dry_run_output.name 1951 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1952 1953 if updated: 1954 rf = rd.getVar('FILE') 1955 if rf.startswith(config.workspace_path): 1956 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1957 finally: 1958 tinfoil.shutdown() 1959 1960 return 0 1961 1962 1963def status(args, config, basepath, workspace): 1964 """Entry point for the devtool 'status' subcommand""" 1965 if workspace: 1966 for recipe, value in sorted(workspace.items()): 1967 recipefile = value['recipefile'] 1968 if recipefile: 1969 recipestr = ' (%s)' % recipefile 1970 else: 1971 recipestr = '' 1972 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1973 else: 1974 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1975 return 0 1976 1977 1978def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1979 """Reset one or more recipes""" 1980 import oe.path 1981 1982 def clean_preferred_provider(pn, layerconf_path): 1983 """Remove PREFERRED_PROVIDER from layer.conf'""" 1984 import re 1985 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1986 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1987 pprovider_found = False 1988 with open(layerconf_file, 'r') as f: 1989 lines = f.readlines() 1990 with open(new_layerconf_file, 'a') as nf: 1991 for line in lines: 1992 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1993 if not re.match(pprovider_exp, line): 1994 nf.write(line) 1995 else: 1996 pprovider_found = True 1997 if pprovider_found: 1998 shutil.move(new_layerconf_file, layerconf_file) 1999 else: 2000 os.remove(new_layerconf_file) 2001 2002 if recipes and not no_clean: 2003 if len(recipes) == 1: 2004 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 2005 else: 2006 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 2007 # If the recipe file itself was created in the workspace, and 2008 # it uses BBCLASSEXTEND, then we need to also clean the other 2009 # variants 2010 targets = [] 2011 for recipe in recipes: 2012 targets.append(recipe) 2013 recipefile = workspace[recipe]['recipefile'] 2014 if recipefile and os.path.exists(recipefile): 2015 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 2016 try: 2017 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 2018 except bb.process.ExecutionError as e: 2019 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 2020 'wish, you may specify -n/--no-clean to ' 2021 'skip running this command when resetting' % 2022 (e.command, e.stdout)) 2023 2024 for pn in recipes: 2025 _check_preserve(config, pn) 2026 2027 appendfile = workspace[pn]['bbappend'] 2028 if os.path.exists(appendfile): 2029 # This shouldn't happen, but is possible if devtool errored out prior to 2030 # writing the md5 file. We need to delete this here or the recipe won't 2031 # actually be reset 2032 os.remove(appendfile) 2033 2034 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 2035 def preservedir(origdir): 2036 if os.path.exists(origdir): 2037 for root, dirs, files in os.walk(origdir): 2038 for fn in files: 2039 logger.warning('Preserving %s in %s' % (fn, preservepath)) 2040 _move_file(os.path.join(origdir, fn), 2041 os.path.join(preservepath, fn)) 2042 for dn in dirs: 2043 preservedir(os.path.join(root, dn)) 2044 os.rmdir(origdir) 2045 2046 recipefile = workspace[pn]['recipefile'] 2047 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 2048 # This should always be true if recipefile is set, but just in case 2049 preservedir(os.path.dirname(recipefile)) 2050 # We don't automatically create this dir next to appends, but the user can 2051 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 2052 2053 srctreebase = workspace[pn]['srctreebase'] 2054 if os.path.isdir(srctreebase): 2055 if os.listdir(srctreebase): 2056 if remove_work: 2057 logger.info('-r argument used on %s, removing source tree.' 2058 ' You will lose any unsaved work' %pn) 2059 shutil.rmtree(srctreebase) 2060 else: 2061 # We don't want to risk wiping out any work in progress 2062 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): 2063 from datetime import datetime 2064 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn,datetime.now().strftime("%Y%m%d%H%M%S"))) 2065 logger.info('Preserving source tree in %s\nIf you no ' 2066 'longer need it then please delete it manually.\n' 2067 'It is also possible to reuse it via devtool source tree argument.' 2068 % preservesrc) 2069 bb.utils.mkdirhier(os.path.dirname(preservesrc)) 2070 shutil.move(srctreebase, preservesrc) 2071 else: 2072 logger.info('Leaving source tree %s as-is; if you no ' 2073 'longer need it then please delete it manually' 2074 % srctreebase) 2075 else: 2076 # This is unlikely, but if it's empty we can just remove it 2077 os.rmdir(srctreebase) 2078 2079 clean_preferred_provider(pn, config.workspace_path) 2080 2081def reset(args, config, basepath, workspace): 2082 """Entry point for the devtool 'reset' subcommand""" 2083 import bb 2084 import shutil 2085 2086 recipes = "" 2087 2088 if args.recipename: 2089 if args.all: 2090 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 2091 else: 2092 for recipe in args.recipename: 2093 check_workspace_recipe(workspace, recipe, checksrc=False) 2094 elif not args.all: 2095 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2096 "reset all recipes") 2097 if args.all: 2098 recipes = list(workspace.keys()) 2099 else: 2100 recipes = args.recipename 2101 2102 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2103 2104 return 0 2105 2106 2107def _get_layer(layername, d): 2108 """Determine the base layer path for the specified layer name/path""" 2109 layerdirs = d.getVar('BBLAYERS').split() 2110 layers = {} # {basename: layer_paths} 2111 for p in layerdirs: 2112 bn = os.path.basename(p) 2113 if bn not in layers: 2114 layers[bn] = [p] 2115 else: 2116 layers[bn].append(p) 2117 # Provide some shortcuts 2118 if layername.lower() in ['oe-core', 'openembedded-core']: 2119 layername = 'meta' 2120 layer_paths = layers.get(layername, None) 2121 if not layer_paths: 2122 return os.path.abspath(layername) 2123 elif len(layer_paths) == 1: 2124 return os.path.abspath(layer_paths[0]) 2125 else: 2126 # multiple layers having the same base name 2127 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 2128 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 2129 return os.path.abspath(layer_paths[0]) 2130 2131 2132def finish(args, config, basepath, workspace): 2133 """Entry point for the devtool 'finish' subcommand""" 2134 import bb 2135 import oe.recipeutils 2136 2137 check_workspace_recipe(workspace, args.recipename) 2138 2139 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2140 2141 # Grab the equivalent of COREBASE without having to initialise tinfoil 2142 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2143 2144 srctree = workspace[args.recipename]['srctree'] 2145 check_git_repo_op(srctree, [corebasedir]) 2146 dirty = check_git_repo_dirty(srctree) 2147 if dirty: 2148 if args.force: 2149 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2150 else: 2151 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2152 2153 no_clean = args.no_clean 2154 remove_work=args.remove_work 2155 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2156 try: 2157 rd = parse_recipe(config, tinfoil, args.recipename, True) 2158 if not rd: 2159 return 1 2160 2161 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2162 recipefile = rd.getVar('FILE') 2163 recipedir = os.path.dirname(recipefile) 2164 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2165 2166 if not os.path.isdir(destlayerdir): 2167 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2168 2169 if os.path.abspath(destlayerdir) == config.workspace_path: 2170 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2171 2172 # If it's an upgrade, grab the original path 2173 origpath = None 2174 origfilelist = None 2175 append = workspace[args.recipename]['bbappend'] 2176 with open(append, 'r') as f: 2177 for line in f: 2178 if line.startswith('# original_path:'): 2179 origpath = line.split(':')[1].strip() 2180 elif line.startswith('# original_files:'): 2181 origfilelist = line.split(':')[1].split() 2182 2183 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2184 2185 if origlayerdir == config.workspace_path: 2186 # Recipe file itself is in workspace, update it there first 2187 appendlayerdir = None 2188 origrelpath = None 2189 if origpath: 2190 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2191 if origlayerpath: 2192 origrelpath = os.path.relpath(origpath, origlayerpath) 2193 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2194 if not destpath: 2195 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2196 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2197 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2198 if not os.path.abspath(destlayerbasedir) in layerdirs: 2199 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2200 2201 elif destlayerdir == origlayerdir: 2202 # Same layer, update the original recipe 2203 appendlayerdir = None 2204 destpath = None 2205 else: 2206 # Create/update a bbappend in the specified layer 2207 appendlayerdir = destlayerdir 2208 destpath = None 2209 2210 # Actually update the recipe / bbappend 2211 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2212 dry_run_output = None 2213 dry_run_outdir = None 2214 if args.dry_run: 2215 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2216 dry_run_outdir = dry_run_output.name 2217 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2218 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2219 2220 # Remove any old files in the case of an upgrade 2221 if removing_original: 2222 for fn in origfilelist: 2223 fnp = os.path.join(origpath, fn) 2224 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2225 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2226 if not args.dry_run: 2227 try: 2228 os.remove(fnp) 2229 except FileNotFoundError: 2230 pass 2231 2232 if origlayerdir == config.workspace_path and destpath: 2233 # Recipe file itself is in the workspace - need to move it and any 2234 # associated files to the specified layer 2235 no_clean = True 2236 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2237 for root, _, files in os.walk(recipedir): 2238 for fn in files: 2239 srcpath = os.path.join(root, fn) 2240 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2241 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2242 destfp = os.path.join(destdir, fn) 2243 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2244 2245 if dry_run_outdir: 2246 import difflib 2247 comparelist = [] 2248 for root, _, files in os.walk(dry_run_outdir): 2249 for fn in files: 2250 outf = os.path.join(root, fn) 2251 relf = os.path.relpath(outf, dry_run_outdir) 2252 logger.debug('dry-run: output file %s' % relf) 2253 if fn.endswith('.bb'): 2254 if origfilelist and origpath and destpath: 2255 # Need to match this up with the pre-upgrade recipe file 2256 for origf in origfilelist: 2257 if origf.endswith('.bb'): 2258 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2259 outf, 2260 os.path.abspath(os.path.join(destpath, relf)))) 2261 break 2262 else: 2263 # Compare to the existing recipe 2264 comparelist.append((recipefile, outf, recipefile)) 2265 elif fn.endswith('.bbappend'): 2266 if appendfile: 2267 if os.path.exists(appendfile): 2268 comparelist.append((appendfile, outf, appendfile)) 2269 else: 2270 comparelist.append((None, outf, appendfile)) 2271 else: 2272 if destpath: 2273 recipedest = destpath 2274 elif appendfile: 2275 recipedest = os.path.dirname(appendfile) 2276 else: 2277 recipedest = os.path.dirname(recipefile) 2278 destfp = os.path.join(recipedest, relf) 2279 if os.path.exists(destfp): 2280 comparelist.append((destfp, outf, destfp)) 2281 output = '' 2282 for oldfile, newfile, newfileshow in comparelist: 2283 if oldfile: 2284 with open(oldfile, 'r') as f: 2285 oldlines = f.readlines() 2286 else: 2287 oldfile = '/dev/null' 2288 oldlines = [] 2289 with open(newfile, 'r') as f: 2290 newlines = f.readlines() 2291 if not newfileshow: 2292 newfileshow = newfile 2293 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2294 difflines = list(diff) 2295 if difflines: 2296 output += ''.join(difflines) 2297 if output: 2298 logger.info('Diff of changed files:\n%s' % output) 2299 finally: 2300 tinfoil.shutdown() 2301 2302 # Everything else has succeeded, we can now reset 2303 if args.dry_run: 2304 logger.info('Resetting recipe (dry-run)') 2305 else: 2306 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2307 2308 return 0 2309 2310 2311def get_default_srctree(config, recipename=''): 2312 """Get the default srctree path""" 2313 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2314 if recipename: 2315 return os.path.join(srctreeparent, 'sources', recipename) 2316 else: 2317 return os.path.join(srctreeparent, 'sources') 2318 2319def register_commands(subparsers, context): 2320 """Register devtool subcommands from this plugin""" 2321 2322 defsrctree = get_default_srctree(context.config) 2323 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2324 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2325 group='starting', order=100) 2326 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2327 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2328 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2329 group = parser_add.add_mutually_exclusive_group() 2330 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2331 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2332 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2333 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2334 parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true") 2335 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2336 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2337 group = parser_add.add_mutually_exclusive_group() 2338 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2339 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2340 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2341 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2342 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2343 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2344 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2345 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2346 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2347 2348 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2349 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2350 group='starting', order=90) 2351 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2352 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2353 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2354 group = parser_modify.add_mutually_exclusive_group() 2355 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2356 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2357 group = parser_modify.add_mutually_exclusive_group() 2358 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2359 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2360 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2361 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2362 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2363 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2364 2365 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2366 description='Extracts the source for an existing recipe', 2367 group='advanced') 2368 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2369 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2370 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2371 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2372 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2373 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2374 2375 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2376 description='Synchronize the previously extracted source tree for an existing recipe', 2377 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2378 group='advanced') 2379 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2380 parser_sync.add_argument('srctree', help='Path to the source tree') 2381 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2382 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2383 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2384 2385 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2386 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2387 group='working', order=10) 2388 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2389 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2390 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2391 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2392 parser_rename.set_defaults(func=rename) 2393 2394 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2395 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2396 group='working', order=-90) 2397 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2398 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2399 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2400 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2401 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2402 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2403 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2404 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2405 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2406 parser_update_recipe.set_defaults(func=update_recipe) 2407 2408 parser_status = subparsers.add_parser('status', help='Show workspace status', 2409 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2410 group='info', order=100) 2411 parser_status.set_defaults(func=status) 2412 2413 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2414 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2415 group='working', order=-100) 2416 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2417 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2418 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2419 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2420 parser_reset.set_defaults(func=reset) 2421 2422 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2423 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2424 group='working', order=-100) 2425 parser_finish.add_argument('recipename', help='Recipe to finish') 2426 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2427 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2428 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2429 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2430 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2431 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2432 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2433 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2434 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2435 parser_finish.set_defaults(func=finish) 2436