1# Development tool - standard commands plugin 2# 3# Copyright (C) 2014-2017 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7"""Devtool standard plugins""" 8 9import os 10import sys 11import re 12import shutil 13import subprocess 14import tempfile 15import logging 16import argparse 17import argparse_oe 18import scriptutils 19import errno 20import glob 21import filecmp 22from collections import OrderedDict 23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24from devtool import parse_recipe 25 26logger = logging.getLogger('devtool') 27 28override_branch_prefix = 'devtool-override-' 29 30 31def add(args, config, basepath, workspace): 32 """Entry point for the devtool 'add' subcommand""" 33 import bb 34 import oe.recipeutils 35 36 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38 39 # These are positional arguments, but because we're nice, allow 40 # specifying e.g. source tree without name, or fetch URI without name or 41 # source tree (if we can detect that that is what the user meant) 42 if scriptutils.is_src_url(args.recipename): 43 if not args.fetchuri: 44 if args.fetch: 45 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46 args.fetchuri = args.recipename 47 args.recipename = '' 48 elif scriptutils.is_src_url(args.srctree): 49 if not args.fetchuri: 50 if args.fetch: 51 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52 args.fetchuri = args.srctree 53 args.srctree = '' 54 elif args.recipename and not args.srctree: 55 if os.sep in args.recipename: 56 args.srctree = args.recipename 57 args.recipename = None 58 elif os.path.isdir(args.recipename): 59 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60 61 if not args.fetchuri: 62 if args.srcrev: 63 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64 if args.srcbranch: 65 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66 67 if args.srctree and os.path.isfile(args.srctree): 68 args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69 args.srctree = '' 70 71 if args.fetch: 72 if args.fetchuri: 73 raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74 else: 75 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76 args.fetchuri = args.fetch 77 78 if args.recipename: 79 if args.recipename in workspace: 80 raise DevtoolError("recipe %s is already in your workspace" % 81 args.recipename) 82 reason = oe.recipeutils.validate_pn(args.recipename) 83 if reason: 84 raise DevtoolError(reason) 85 86 if args.srctree: 87 srctree = os.path.abspath(args.srctree) 88 srctreeparent = None 89 tmpsrcdir = None 90 else: 91 srctree = None 92 srctreeparent = get_default_srctree(config) 93 bb.utils.mkdirhier(srctreeparent) 94 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95 96 if srctree and os.path.exists(srctree): 97 if args.fetchuri: 98 if not os.path.isdir(srctree): 99 raise DevtoolError("Cannot fetch into source tree path %s as " 100 "it exists and is not a directory" % 101 srctree) 102 elif os.listdir(srctree): 103 raise DevtoolError("Cannot fetch into source tree path %s as " 104 "it already exists and is non-empty" % 105 srctree) 106 elif not args.fetchuri: 107 if args.srctree: 108 raise DevtoolError("Specified source tree %s could not be found" % 109 args.srctree) 110 elif srctree: 111 raise DevtoolError("No source tree exists at default path %s - " 112 "either create and populate this directory, " 113 "or specify a path to a source tree, or a " 114 "URI to fetch source from" % srctree) 115 else: 116 raise DevtoolError("You must either specify a source tree " 117 "or a URI to fetch source from") 118 119 if args.version: 120 if '_' in args.version or ' ' in args.version: 121 raise DevtoolError('Invalid version string "%s"' % args.version) 122 123 if args.color == 'auto' and sys.stdout.isatty(): 124 color = 'always' 125 else: 126 color = args.color 127 extracmdopts = '' 128 if args.fetchuri: 129 source = args.fetchuri 130 if srctree: 131 extracmdopts += ' -x %s' % srctree 132 else: 133 extracmdopts += ' -x %s' % tmpsrcdir 134 else: 135 source = srctree 136 if args.recipename: 137 extracmdopts += ' -N %s' % args.recipename 138 if args.version: 139 extracmdopts += ' -V %s' % args.version 140 if args.binary: 141 extracmdopts += ' -b' 142 if args.also_native: 143 extracmdopts += ' --also-native' 144 if args.src_subdir: 145 extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146 if args.autorev: 147 extracmdopts += ' -a' 148 if args.npm_dev: 149 extracmdopts += ' --npm-dev' 150 if args.mirrors: 151 extracmdopts += ' --mirrors' 152 if args.srcrev: 153 extracmdopts += ' --srcrev %s' % args.srcrev 154 if args.srcbranch: 155 extracmdopts += ' --srcbranch %s' % args.srcbranch 156 if args.provides: 157 extracmdopts += ' --provides %s' % args.provides 158 159 tempdir = tempfile.mkdtemp(prefix='devtool') 160 try: 161 try: 162 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 163 except bb.process.ExecutionError as e: 164 if e.exitcode == 15: 165 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 166 else: 167 raise DevtoolError('Command \'%s\' failed' % e.command) 168 169 recipes = glob.glob(os.path.join(tempdir, '*.bb')) 170 if recipes: 171 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 172 if recipename in workspace: 173 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 174 recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 175 bb.utils.mkdirhier(recipedir) 176 recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 177 appendfile = recipe_to_append(recipefile, config) 178 if os.path.exists(appendfile): 179 # This shouldn't be possible, but just in case 180 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 181 if os.path.exists(recipefile): 182 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 183 if tmpsrcdir: 184 srctree = os.path.join(srctreeparent, recipename) 185 if os.path.exists(tmpsrcdir): 186 if os.path.exists(srctree): 187 if os.path.isdir(srctree): 188 try: 189 os.rmdir(srctree) 190 except OSError as e: 191 if e.errno == errno.ENOTEMPTY: 192 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 193 else: 194 raise 195 else: 196 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 197 logger.info('Using default source tree path %s' % srctree) 198 shutil.move(tmpsrcdir, srctree) 199 else: 200 raise DevtoolError('Couldn\'t find source tree created by recipetool') 201 bb.utils.mkdirhier(recipedir) 202 shutil.move(recipes[0], recipefile) 203 # Move any additional files created by recipetool 204 for fn in os.listdir(tempdir): 205 shutil.move(os.path.join(tempdir, fn), recipedir) 206 else: 207 raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 208 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 209 if os.path.exists(attic_recipe): 210 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 211 finally: 212 if tmpsrcdir and os.path.exists(tmpsrcdir): 213 shutil.rmtree(tmpsrcdir) 214 shutil.rmtree(tempdir) 215 216 for fn in os.listdir(recipedir): 217 _add_md5(config, recipename, os.path.join(recipedir, fn)) 218 219 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 220 try: 221 try: 222 rd = tinfoil.parse_recipe_file(recipefile, False) 223 except Exception as e: 224 logger.error(str(e)) 225 rd = None 226 if not rd: 227 # Parsing failed. We just created this recipe and we shouldn't 228 # leave it in the workdir or it'll prevent bitbake from starting 229 movefn = '%s.parsefailed' % recipefile 230 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 231 shutil.move(recipefile, movefn) 232 return 1 233 234 if args.fetchuri and not args.no_git: 235 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 236 237 initial_rev = None 238 if os.path.exists(os.path.join(srctree, '.git')): 239 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 240 initial_rev = stdout.rstrip() 241 242 if args.src_subdir: 243 srctree = os.path.join(srctree, args.src_subdir) 244 245 bb.utils.mkdirhier(os.path.dirname(appendfile)) 246 with open(appendfile, 'w') as f: 247 f.write('inherit externalsrc\n') 248 f.write('EXTERNALSRC = "%s"\n' % srctree) 249 250 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 251 if b_is_s: 252 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 253 if initial_rev: 254 f.write('\n# initial_rev: %s\n' % initial_rev) 255 256 if args.binary: 257 f.write('do_install:append() {\n') 258 f.write(' rm -rf ${D}/.git\n') 259 f.write(' rm -f ${D}/singletask.lock\n') 260 f.write('}\n') 261 262 if bb.data.inherits_class('npm', rd): 263 f.write('python do_configure:append() {\n') 264 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 265 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 266 f.write(' bb.utils.remove(lockfile)\n') 267 f.write('}\n') 268 269 # Check if the new layer provides recipes whose priorities have been 270 # overriden by PREFERRED_PROVIDER. 271 recipe_name = rd.getVar('PN') 272 provides = rd.getVar('PROVIDES') 273 # Search every item defined in PROVIDES 274 for recipe_provided in provides.split(): 275 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 276 current_pprovider = rd.getVar(preferred_provider) 277 if current_pprovider and current_pprovider != recipe_name: 278 if args.fixed_setup: 279 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 280 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 281 with open(layerconf_file, 'a') as f: 282 f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 283 else: 284 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 285 break 286 287 _add_md5(config, recipename, appendfile) 288 289 check_prerelease_version(rd.getVar('PV'), 'devtool add') 290 291 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 292 293 finally: 294 tinfoil.shutdown() 295 296 return 0 297 298 299def _check_compatible_recipe(pn, d): 300 """Check if the recipe is supported by devtool""" 301 if pn == 'perf': 302 raise DevtoolError("The perf recipe does not actually check out " 303 "source and thus cannot be supported by this tool", 304 4) 305 306 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 307 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 308 309 if bb.data.inherits_class('image', d): 310 raise DevtoolError("The %s recipe is an image, and therefore is not " 311 "supported by this tool" % pn, 4) 312 313 if bb.data.inherits_class('populate_sdk', d): 314 raise DevtoolError("The %s recipe is an SDK, and therefore is not " 315 "supported by this tool" % pn, 4) 316 317 if bb.data.inherits_class('packagegroup', d): 318 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 319 "not supported by this tool" % pn, 4) 320 321 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 322 # Not an incompatibility error per se, so we don't pass the error code 323 raise DevtoolError("externalsrc is currently enabled for the %s " 324 "recipe. This prevents the normal do_patch task " 325 "from working. You will need to disable this " 326 "first." % pn) 327 328def _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 329 """Common function for copying a file to the dry run output directory""" 330 relpath = os.path.relpath(dst, base_outdir) 331 if relpath.startswith('..'): 332 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 333 dst = os.path.join(dry_run_outdir, relpath) 334 dst_d = os.path.dirname(dst) 335 if dst_d: 336 bb.utils.mkdirhier(dst_d) 337 # Don't overwrite existing files, otherwise in the case of an upgrade 338 # the dry-run written out recipe will be overwritten with an unmodified 339 # version 340 if not os.path.exists(dst): 341 shutil.copy(src, dst) 342 343def _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 344 """Move a file. Creates all the directory components of destination path.""" 345 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 346 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 347 if dry_run_outdir: 348 # We want to copy here, not move 349 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 350 else: 351 dst_d = os.path.dirname(dst) 352 if dst_d: 353 bb.utils.mkdirhier(dst_d) 354 shutil.move(src, dst) 355 356def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 357 """Copy a file. Creates all the directory components of destination path.""" 358 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 359 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 360 if dry_run_outdir: 361 _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 362 else: 363 dst_d = os.path.dirname(dst) 364 if dst_d: 365 bb.utils.mkdirhier(dst_d) 366 shutil.copy(src, dst) 367 368def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 369 """List contents of a git treeish""" 370 import bb 371 cmd = ['git', 'ls-tree', '-z', treeish] 372 if recursive: 373 cmd.append('-r') 374 out, _ = bb.process.run(cmd, cwd=repodir) 375 ret = {} 376 if out: 377 for line in out.split('\0'): 378 if line: 379 split = line.split(None, 4) 380 ret[split[3]] = split[0:3] 381 return ret 382 383def _git_exclude_path(srctree, path): 384 """Return pathspec (list of paths) that excludes certain path""" 385 # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 386 # we don't catch files that are deleted, for example. A more reliable way 387 # to implement this would be to use "negative pathspecs" which were 388 # introduced in Git v1.9.0. Revisit this when/if the required Git version 389 # becomes greater than that. 390 path = os.path.normpath(path) 391 recurse = True if len(path.split(os.path.sep)) > 1 else False 392 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 393 if path in git_files: 394 git_files.remove(path) 395 return git_files 396 else: 397 return ['.'] 398 399def _ls_tree(directory): 400 """Recursive listing of files in a directory""" 401 ret = [] 402 for root, dirs, files in os.walk(directory): 403 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 404 fname in files]) 405 return ret 406 407 408def extract(args, config, basepath, workspace): 409 """Entry point for the devtool 'extract' subcommand""" 410 import bb 411 412 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 413 if not tinfoil: 414 # Error already shown 415 return 1 416 try: 417 rd = parse_recipe(config, tinfoil, args.recipename, True) 418 if not rd: 419 return 1 420 421 srctree = os.path.abspath(args.srctree) 422 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 423 logger.info('Source tree extracted to %s' % srctree) 424 425 if initial_rev: 426 return 0 427 else: 428 return 1 429 finally: 430 tinfoil.shutdown() 431 432def sync(args, config, basepath, workspace): 433 """Entry point for the devtool 'sync' subcommand""" 434 import bb 435 436 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 437 if not tinfoil: 438 # Error already shown 439 return 1 440 try: 441 rd = parse_recipe(config, tinfoil, args.recipename, True) 442 if not rd: 443 return 1 444 445 srctree = os.path.abspath(args.srctree) 446 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 447 logger.info('Source tree %s synchronized' % srctree) 448 449 if initial_rev: 450 return 0 451 else: 452 return 1 453 finally: 454 tinfoil.shutdown() 455 456def symlink_oelocal_files_srctree(rd,srctree): 457 import oe.patch 458 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 459 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 460 # (otherwise the recipe won't build as expected) 461 local_files_dir = os.path.join(srctree, 'oe-local-files') 462 addfiles = [] 463 for root, _, files in os.walk(local_files_dir): 464 relpth = os.path.relpath(root, local_files_dir) 465 if relpth != '.': 466 bb.utils.mkdirhier(os.path.join(srctree, relpth)) 467 for fn in files: 468 if fn == '.gitignore': 469 continue 470 destpth = os.path.join(srctree, relpth, fn) 471 if os.path.exists(destpth): 472 os.unlink(destpth) 473 if relpth != '.': 474 back_relpth = os.path.relpath(local_files_dir, root) 475 os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) 476 else: 477 os.symlink('oe-local-files/%s' % fn, destpth) 478 addfiles.append(os.path.join(relpth, fn)) 479 if addfiles: 480 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 481 useroptions = [] 482 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 483 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 484 485 486def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 487 """Extract sources of a recipe""" 488 import oe.recipeutils 489 import oe.patch 490 import oe.path 491 492 pn = d.getVar('PN') 493 494 _check_compatible_recipe(pn, d) 495 496 if sync: 497 if not os.path.exists(srctree): 498 raise DevtoolError("output path %s does not exist" % srctree) 499 else: 500 if os.path.exists(srctree): 501 if not os.path.isdir(srctree): 502 raise DevtoolError("output path %s exists and is not a directory" % 503 srctree) 504 elif os.listdir(srctree): 505 raise DevtoolError("output path %s already exists and is " 506 "non-empty" % srctree) 507 508 if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 509 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 510 "extract source" % pn, 4) 511 512 if not sync: 513 # Prepare for shutil.move later on 514 bb.utils.mkdirhier(srctree) 515 os.rmdir(srctree) 516 517 extra_overrides = [] 518 if not no_overrides: 519 history = d.varhistory.variable('SRC_URI') 520 for event in history: 521 if not 'flag' in event: 522 if event['op'].startswith((':append[', ':prepend[')): 523 override = event['op'].split('[')[1].split(']')[0] 524 if not override.startswith('pn-'): 525 extra_overrides.append(override) 526 # We want to remove duplicate overrides. If a recipe had multiple 527 # SRC_URI_override += values it would cause mulitple instances of 528 # overrides. This doesn't play nicely with things like creating a 529 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 530 extra_overrides = list(set(extra_overrides)) 531 if extra_overrides: 532 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 533 534 initial_rev = None 535 536 recipefile = d.getVar('FILE') 537 appendfile = recipe_to_append(recipefile, config) 538 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 539 540 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 541 # directory so that: 542 # (a) we pick up all files that get unpacked to the WORKDIR, and 543 # (b) we don't disturb the existing build 544 # However, with recipe-specific sysroots the sysroots for the recipe 545 # will be prepared under WORKDIR, and if we used the system temporary 546 # directory (i.e. usually /tmp) as used by mkdtemp by default, then 547 # our attempts to hardlink files into the recipe-specific sysroots 548 # will fail on systems where /tmp is a different filesystem, and it 549 # would have to fall back to copying the files which is a waste of 550 # time. Put the temp directory under the WORKDIR to prevent that from 551 # being a problem. 552 tempbasedir = d.getVar('WORKDIR') 553 bb.utils.mkdirhier(tempbasedir) 554 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 555 try: 556 tinfoil.logger.setLevel(logging.WARNING) 557 558 # FIXME this results in a cache reload under control of tinfoil, which is fine 559 # except we don't get the knotty progress bar 560 561 if os.path.exists(appendfile): 562 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 563 shutil.copyfile(appendfile, appendbackup) 564 else: 565 appendbackup = None 566 bb.utils.mkdirhier(os.path.dirname(appendfile)) 567 logger.debug('writing append file %s' % appendfile) 568 with open(appendfile, 'a') as f: 569 f.write('###--- _extract_source\n') 570 f.write('ERROR_QA:remove = "patch-fuzz"\n') 571 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 572 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 573 if not is_kernel_yocto: 574 f.write('PATCHTOOL = "git"\n') 575 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 576 if extra_overrides: 577 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 578 f.write('inherit devtool-source\n') 579 f.write('###--- _extract_source\n') 580 581 update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 582 583 sstate_manifests = d.getVar('SSTATE_MANIFESTS') 584 bb.utils.mkdirhier(sstate_manifests) 585 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 586 with open(preservestampfile, 'w') as f: 587 f.write(d.getVar('STAMP')) 588 try: 589 if is_kernel_yocto: 590 # We need to generate the kernel config 591 task = 'do_configure' 592 else: 593 task = 'do_patch' 594 595 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 596 logger.info('The %s recipe has %s disabled. Running only ' 597 'do_configure task dependencies' % (pn, task)) 598 599 if 'depends' in d.getVarFlags('do_configure', False): 600 pn = d.getVarFlags('do_configure', False)['depends'] 601 pn = pn.replace('${PV}', d.getVar('PV')) 602 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 603 task = None 604 605 # Run the fetch + unpack tasks 606 res = tinfoil.build_targets(pn, 607 task, 608 handle_events=True) 609 finally: 610 if os.path.exists(preservestampfile): 611 os.remove(preservestampfile) 612 613 if not res: 614 raise DevtoolError('Extracting source for %s failed' % pn) 615 616 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 617 workshareddir = d.getVar('S') 618 if os.path.islink(srctree): 619 os.unlink(srctree) 620 621 os.symlink(workshareddir, srctree) 622 623 # The initial_rev file is created in devtool_post_unpack function that will not be executed if 624 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 625 return True, True 626 627 try: 628 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 629 initial_rev = f.read() 630 631 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 632 srcsubdir = f.read() 633 except FileNotFoundError as e: 634 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 635 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 636 637 # Check if work-shared is empty, if yes 638 # find source and copy to work-shared 639 if is_kernel_yocto: 640 workshareddir = d.getVar('STAGING_KERNEL_DIR') 641 staging_kerVer = get_staging_kver(workshareddir) 642 kernelVersion = d.getVar('LINUX_VERSION') 643 644 # handle dangling symbolic link in work-shared: 645 if os.path.islink(workshareddir): 646 os.unlink(workshareddir) 647 648 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 649 shutil.rmtree(workshareddir) 650 oe.path.copyhardlinktree(srcsubdir,workshareddir) 651 elif not os.path.exists(workshareddir): 652 oe.path.copyhardlinktree(srcsubdir,workshareddir) 653 654 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 655 srctree_localdir = os.path.join(srctree, 'oe-local-files') 656 657 if sync: 658 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 659 660 # Move oe-local-files directory to srctree 661 # As the oe-local-files is not part of the constructed git tree, 662 # remove them directly during the synchrounizating might surprise 663 # the users. Instead, we move it to oe-local-files.bak and remind 664 # user in the log message. 665 if os.path.exists(srctree_localdir + '.bak'): 666 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 667 668 if os.path.exists(srctree_localdir): 669 logger.info('Backing up current local file directory %s' % srctree_localdir) 670 shutil.move(srctree_localdir, srctree_localdir + '.bak') 671 672 if os.path.exists(tempdir_localdir): 673 logger.info('Syncing local source files to srctree...') 674 shutil.copytree(tempdir_localdir, srctree_localdir) 675 else: 676 # Move oe-local-files directory to srctree 677 if os.path.exists(tempdir_localdir): 678 logger.info('Adding local source files to srctree...') 679 shutil.move(tempdir_localdir, srcsubdir) 680 681 shutil.move(srcsubdir, srctree) 682 symlink_oelocal_files_srctree(d,srctree) 683 684 if is_kernel_yocto: 685 logger.info('Copying kernel config to srctree') 686 shutil.copy2(os.path.join(tempdir, '.config'), srctree) 687 688 finally: 689 if appendbackup: 690 shutil.copyfile(appendbackup, appendfile) 691 elif os.path.exists(appendfile): 692 os.remove(appendfile) 693 if keep_temp: 694 logger.info('Preserving temporary directory %s' % tempdir) 695 else: 696 shutil.rmtree(tempdir) 697 return initial_rev, srcsubdir_rel 698 699def _add_md5(config, recipename, filename): 700 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 701 import bb.utils 702 703 def addfile(fn): 704 md5 = bb.utils.md5_file(fn) 705 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 706 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 707 f.seek(0, os.SEEK_SET) 708 if not md5_str in f.read(): 709 f.write(md5_str) 710 711 if os.path.isdir(filename): 712 for root, _, files in os.walk(filename): 713 for f in files: 714 addfile(os.path.join(root, f)) 715 else: 716 addfile(filename) 717 718def _check_preserve(config, recipename): 719 """Check if a file was manually changed and needs to be saved in 'attic' 720 directory""" 721 import bb.utils 722 origfile = os.path.join(config.workspace_path, '.devtool_md5') 723 newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 724 preservepath = os.path.join(config.workspace_path, 'attic', recipename) 725 with open(origfile, 'r') as f: 726 with open(newfile, 'w') as tf: 727 for line in f.readlines(): 728 splitline = line.rstrip().split('|') 729 if splitline[0] == recipename: 730 removefile = os.path.join(config.workspace_path, splitline[1]) 731 try: 732 md5 = bb.utils.md5_file(removefile) 733 except IOError as err: 734 if err.errno == 2: 735 # File no longer exists, skip it 736 continue 737 else: 738 raise 739 if splitline[2] != md5: 740 bb.utils.mkdirhier(preservepath) 741 preservefile = os.path.basename(removefile) 742 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 743 shutil.move(removefile, os.path.join(preservepath, preservefile)) 744 else: 745 os.remove(removefile) 746 else: 747 tf.write(line) 748 bb.utils.rename(newfile, origfile) 749 750def get_staging_kver(srcdir): 751 # Kernel version from work-shared 752 kerver = [] 753 staging_kerVer="" 754 if os.path.exists(srcdir) and os.listdir(srcdir): 755 with open(os.path.join(srcdir,"Makefile")) as f: 756 version = [next(f) for x in range(5)][1:4] 757 for word in version: 758 kerver.append(word.split('= ')[1].split('\n')[0]) 759 staging_kerVer = ".".join(kerver) 760 return staging_kerVer 761 762def get_staging_kbranch(srcdir): 763 staging_kbranch = "" 764 if os.path.exists(srcdir) and os.listdir(srcdir): 765 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) 766 staging_kbranch = "".join(branch.split('\n')[0]) 767 return staging_kbranch 768 769def get_real_srctree(srctree, s, workdir): 770 # Check that recipe isn't using a shared workdir 771 s = os.path.abspath(s) 772 workdir = os.path.abspath(workdir) 773 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 774 # Handle if S is set to a subdirectory of the source 775 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 776 srctree = os.path.join(srctree, srcsubdir) 777 return srctree 778 779def modify(args, config, basepath, workspace): 780 """Entry point for the devtool 'modify' subcommand""" 781 import bb 782 import oe.recipeutils 783 import oe.patch 784 import oe.path 785 786 if args.recipename in workspace: 787 raise DevtoolError("recipe %s is already in your workspace" % 788 args.recipename) 789 790 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 791 try: 792 rd = parse_recipe(config, tinfoil, args.recipename, True) 793 if not rd: 794 return 1 795 796 pn = rd.getVar('PN') 797 if pn != args.recipename: 798 logger.info('Mapping %s to %s' % (args.recipename, pn)) 799 if pn in workspace: 800 raise DevtoolError("recipe %s is already in your workspace" % 801 pn) 802 803 if args.srctree: 804 srctree = os.path.abspath(args.srctree) 805 else: 806 srctree = get_default_srctree(config, pn) 807 808 if args.no_extract and not os.path.isdir(srctree): 809 raise DevtoolError("--no-extract specified and source path %s does " 810 "not exist or is not a directory" % 811 srctree) 812 813 recipefile = rd.getVar('FILE') 814 appendfile = recipe_to_append(recipefile, config, args.wildcard) 815 if os.path.exists(appendfile): 816 raise DevtoolError("Another variant of recipe %s is already in your " 817 "workspace (only one variant of a recipe can " 818 "currently be worked on at once)" 819 % pn) 820 821 _check_compatible_recipe(pn, rd) 822 823 initial_rev = None 824 commits = [] 825 check_commits = False 826 827 if bb.data.inherits_class('kernel-yocto', rd): 828 # Current set kernel version 829 kernelVersion = rd.getVar('LINUX_VERSION') 830 srcdir = rd.getVar('STAGING_KERNEL_DIR') 831 kbranch = rd.getVar('KBRANCH') 832 833 staging_kerVer = get_staging_kver(srcdir) 834 staging_kbranch = get_staging_kbranch(srcdir) 835 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 836 oe.path.copyhardlinktree(srcdir,srctree) 837 workdir = rd.getVar('WORKDIR') 838 srcsubdir = rd.getVar('S') 839 localfilesdir = os.path.join(srctree,'oe-local-files') 840 # Move local source files into separate subdir 841 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 842 local_files = oe.recipeutils.get_recipe_local_files(rd) 843 844 for key in local_files.copy(): 845 if key.endswith('scc'): 846 sccfile = open(local_files[key], 'r') 847 for l in sccfile: 848 line = l.split() 849 if line and line[0] in ('kconf', 'patch'): 850 cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 851 if not cfg in local_files.values(): 852 local_files[line[-1]] = cfg 853 shutil.copy2(cfg, workdir) 854 sccfile.close() 855 856 # Ignore local files with subdir={BP} 857 srcabspath = os.path.abspath(srcsubdir) 858 local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 859 if local_files: 860 for fname in local_files: 861 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 862 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 863 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 864 865 symlink_oelocal_files_srctree(rd,srctree) 866 867 task = 'do_configure' 868 res = tinfoil.build_targets(pn, task, handle_events=True) 869 870 # Copy .config to workspace 871 kconfpath = rd.getVar('B') 872 logger.info('Copying kernel config to workspace') 873 shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 874 875 # Set this to true, we still need to get initial_rev 876 # by parsing the git repo 877 args.no_extract = True 878 879 if not args.no_extract: 880 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 881 if not initial_rev: 882 return 1 883 logger.info('Source tree extracted to %s' % srctree) 884 if os.path.exists(os.path.join(srctree, '.git')): 885 # Get list of commits since this revision 886 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) 887 commits = stdout.split() 888 check_commits = True 889 else: 890 if os.path.exists(os.path.join(srctree, '.git')): 891 # Check if it's a tree previously extracted by us. This is done 892 # by ensuring that devtool-base and args.branch (devtool) exist. 893 # The check_commits logic will cause an exception if either one 894 # of these doesn't exist 895 try: 896 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 897 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 898 except bb.process.ExecutionError: 899 stdout = '' 900 if stdout: 901 check_commits = True 902 for line in stdout.splitlines(): 903 if line.startswith('*'): 904 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 905 initial_rev = stdout.rstrip() 906 if not initial_rev: 907 # Otherwise, just grab the head revision 908 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 909 initial_rev = stdout.rstrip() 910 911 branch_patches = {} 912 if check_commits: 913 # Check if there are override branches 914 (stdout, _) = bb.process.run('git branch', cwd=srctree) 915 branches = [] 916 for line in stdout.rstrip().splitlines(): 917 branchname = line[2:].rstrip() 918 if branchname.startswith(override_branch_prefix): 919 branches.append(branchname) 920 if branches: 921 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 922 branches.insert(0, args.branch) 923 seen_patches = [] 924 for branch in branches: 925 branch_patches[branch] = [] 926 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 927 for line in stdout.splitlines(): 928 line = line.strip() 929 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 930 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 931 if not origpatch in seen_patches: 932 seen_patches.append(origpatch) 933 branch_patches[branch].append(origpatch) 934 935 # Need to grab this here in case the source is within a subdirectory 936 srctreebase = srctree 937 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 938 939 bb.utils.mkdirhier(os.path.dirname(appendfile)) 940 with open(appendfile, 'w') as f: 941 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 942 # Local files can be modified/tracked in separate subdir under srctree 943 # Mostly useful for packages with S != WORKDIR 944 f.write('FILESPATH:prepend := "%s:"\n' % 945 os.path.join(srctreebase, 'oe-local-files')) 946 f.write('# srctreebase: %s\n' % srctreebase) 947 948 f.write('\ninherit externalsrc\n') 949 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 950 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 951 952 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 953 if b_is_s: 954 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 955 956 if bb.data.inherits_class('kernel', rd): 957 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 958 'do_fetch do_unpack do_kernel_configcheck"\n') 959 f.write('\ndo_patch[noexec] = "1"\n') 960 f.write('\ndo_configure:append() {\n' 961 ' cp ${B}/.config ${S}/.config.baseline\n' 962 ' ln -sfT ${B}/.config ${S}/.config.new\n' 963 '}\n') 964 f.write('\ndo_kernel_configme:prepend() {\n' 965 ' if [ -e ${S}/.config ]; then\n' 966 ' mv ${S}/.config ${S}/.config.old\n' 967 ' fi\n' 968 '}\n') 969 if rd.getVarFlag('do_menuconfig','task'): 970 f.write('\ndo_configure:append() {\n' 971 ' if [ ${@ oe.types.boolean(\'${KCONFIG_CONFIG_ENABLE_MENUCONFIG}\') } = True ]; then\n' 972 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' 973 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' 974 ' fi\n' 975 '}\n') 976 if initial_rev: 977 f.write('\n# initial_rev: %s\n' % initial_rev) 978 for commit in commits: 979 f.write('# commit: %s\n' % commit) 980 if branch_patches: 981 for branch in branch_patches: 982 if branch == args.branch: 983 continue 984 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 985 986 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 987 988 _add_md5(config, pn, appendfile) 989 990 logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 991 992 finally: 993 tinfoil.shutdown() 994 995 return 0 996 997 998def rename(args, config, basepath, workspace): 999 """Entry point for the devtool 'rename' subcommand""" 1000 import bb 1001 import oe.recipeutils 1002 1003 check_workspace_recipe(workspace, args.recipename) 1004 1005 if not (args.newname or args.version): 1006 raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 1007 1008 recipefile = workspace[args.recipename]['recipefile'] 1009 if not recipefile: 1010 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1011 1012 if args.newname and args.newname != args.recipename: 1013 reason = oe.recipeutils.validate_pn(args.newname) 1014 if reason: 1015 raise DevtoolError(reason) 1016 newname = args.newname 1017 else: 1018 newname = args.recipename 1019 1020 append = workspace[args.recipename]['bbappend'] 1021 appendfn = os.path.splitext(os.path.basename(append))[0] 1022 splitfn = appendfn.split('_') 1023 if len(splitfn) > 1: 1024 origfnver = appendfn.split('_')[1] 1025 else: 1026 origfnver = '' 1027 1028 recipefilemd5 = None 1029 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1030 try: 1031 rd = parse_recipe(config, tinfoil, args.recipename, True) 1032 if not rd: 1033 return 1 1034 1035 bp = rd.getVar('BP') 1036 bpn = rd.getVar('BPN') 1037 if newname != args.recipename: 1038 localdata = rd.createCopy() 1039 localdata.setVar('PN', newname) 1040 newbpn = localdata.getVar('BPN') 1041 else: 1042 newbpn = bpn 1043 s = rd.getVar('S', False) 1044 src_uri = rd.getVar('SRC_URI', False) 1045 pv = rd.getVar('PV') 1046 1047 # Correct variable values that refer to the upstream source - these 1048 # values must stay the same, so if the name/version are changing then 1049 # we need to fix them up 1050 new_s = s 1051 new_src_uri = src_uri 1052 if newbpn != bpn: 1053 # ${PN} here is technically almost always incorrect, but people do use it 1054 new_s = new_s.replace('${BPN}', bpn) 1055 new_s = new_s.replace('${PN}', bpn) 1056 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1057 new_src_uri = new_src_uri.replace('${BPN}', bpn) 1058 new_src_uri = new_src_uri.replace('${PN}', bpn) 1059 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1060 if args.version and origfnver == pv: 1061 new_s = new_s.replace('${PV}', pv) 1062 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1063 new_src_uri = new_src_uri.replace('${PV}', pv) 1064 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1065 patchfields = {} 1066 if new_s != s: 1067 patchfields['S'] = new_s 1068 if new_src_uri != src_uri: 1069 patchfields['SRC_URI'] = new_src_uri 1070 if patchfields: 1071 recipefilemd5 = bb.utils.md5_file(recipefile) 1072 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1073 newrecipefilemd5 = bb.utils.md5_file(recipefile) 1074 finally: 1075 tinfoil.shutdown() 1076 1077 if args.version: 1078 newver = args.version 1079 else: 1080 newver = origfnver 1081 1082 if newver: 1083 newappend = '%s_%s.bbappend' % (newname, newver) 1084 newfile = '%s_%s.bb' % (newname, newver) 1085 else: 1086 newappend = '%s.bbappend' % newname 1087 newfile = '%s.bb' % newname 1088 1089 oldrecipedir = os.path.dirname(recipefile) 1090 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1091 if oldrecipedir != newrecipedir: 1092 bb.utils.mkdirhier(newrecipedir) 1093 1094 newappend = os.path.join(os.path.dirname(append), newappend) 1095 newfile = os.path.join(newrecipedir, newfile) 1096 1097 # Rename bbappend 1098 logger.info('Renaming %s to %s' % (append, newappend)) 1099 bb.utils.rename(append, newappend) 1100 # Rename recipe file 1101 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1102 bb.utils.rename(recipefile, newfile) 1103 1104 # Rename source tree if it's the default path 1105 appendmd5 = None 1106 if not args.no_srctree: 1107 srctree = workspace[args.recipename]['srctree'] 1108 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1109 newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1110 logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1111 shutil.move(srctree, newsrctree) 1112 # Correct any references (basically EXTERNALSRC*) in the .bbappend 1113 appendmd5 = bb.utils.md5_file(newappend) 1114 appendlines = [] 1115 with open(newappend, 'r') as f: 1116 for line in f: 1117 appendlines.append(line) 1118 with open(newappend, 'w') as f: 1119 for line in appendlines: 1120 if srctree in line: 1121 line = line.replace(srctree, newsrctree) 1122 f.write(line) 1123 newappendmd5 = bb.utils.md5_file(newappend) 1124 1125 bpndir = None 1126 newbpndir = None 1127 if newbpn != bpn: 1128 bpndir = os.path.join(oldrecipedir, bpn) 1129 if os.path.exists(bpndir): 1130 newbpndir = os.path.join(newrecipedir, newbpn) 1131 logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1132 shutil.move(bpndir, newbpndir) 1133 1134 bpdir = None 1135 newbpdir = None 1136 if newver != origfnver or newbpn != bpn: 1137 bpdir = os.path.join(oldrecipedir, bp) 1138 if os.path.exists(bpdir): 1139 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1140 logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1141 shutil.move(bpdir, newbpdir) 1142 1143 if oldrecipedir != newrecipedir: 1144 # Move any stray files and delete the old recipe directory 1145 for entry in os.listdir(oldrecipedir): 1146 oldpath = os.path.join(oldrecipedir, entry) 1147 newpath = os.path.join(newrecipedir, entry) 1148 logger.info('Renaming %s to %s' % (oldpath, newpath)) 1149 shutil.move(oldpath, newpath) 1150 os.rmdir(oldrecipedir) 1151 1152 # Now take care of entries in .devtool_md5 1153 md5entries = [] 1154 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1155 for line in f: 1156 md5entries.append(line) 1157 1158 if bpndir and newbpndir: 1159 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1160 else: 1161 relbpndir = None 1162 if bpdir and newbpdir: 1163 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1164 else: 1165 relbpdir = None 1166 1167 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1168 for entry in md5entries: 1169 splitentry = entry.rstrip().split('|') 1170 if len(splitentry) > 2: 1171 if splitentry[0] == args.recipename: 1172 splitentry[0] = newname 1173 if splitentry[1] == os.path.relpath(append, config.workspace_path): 1174 splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1175 if appendmd5 and splitentry[2] == appendmd5: 1176 splitentry[2] = newappendmd5 1177 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1178 splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1179 if recipefilemd5 and splitentry[2] == recipefilemd5: 1180 splitentry[2] = newrecipefilemd5 1181 elif relbpndir and splitentry[1].startswith(relbpndir): 1182 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1183 elif relbpdir and splitentry[1].startswith(relbpdir): 1184 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1185 entry = '|'.join(splitentry) + '\n' 1186 f.write(entry) 1187 return 0 1188 1189 1190def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1191 """Get initial and update rev of a recipe. These are the start point of the 1192 whole patchset and start point for the patches to be re-generated/updated. 1193 """ 1194 import bb 1195 1196 # Get current branch 1197 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1198 cwd=srctree) 1199 branchname = stdout.rstrip() 1200 1201 # Parse initial rev from recipe if not specified 1202 commits = [] 1203 patches = [] 1204 with open(recipe_path, 'r') as f: 1205 for line in f: 1206 if line.startswith('# initial_rev:'): 1207 if not initial_rev: 1208 initial_rev = line.split(':')[-1].strip() 1209 elif line.startswith('# commit:') and not force_patch_refresh: 1210 commits.append(line.split(':')[-1].strip()) 1211 elif line.startswith('# patches_%s:' % branchname): 1212 patches = line.split(':')[-1].strip().split(',') 1213 1214 update_rev = initial_rev 1215 changed_revs = None 1216 if initial_rev: 1217 # Find first actually changed revision 1218 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1219 initial_rev, cwd=srctree) 1220 newcommits = stdout.split() 1221 for i in range(min(len(commits), len(newcommits))): 1222 if newcommits[i] == commits[i]: 1223 update_rev = commits[i] 1224 1225 try: 1226 stdout, _ = bb.process.run('git cherry devtool-patched', 1227 cwd=srctree) 1228 except bb.process.ExecutionError as err: 1229 stdout = None 1230 1231 if stdout is not None and not force_patch_refresh: 1232 changed_revs = [] 1233 for line in stdout.splitlines(): 1234 if line.startswith('+ '): 1235 rev = line.split()[1] 1236 if rev in newcommits: 1237 changed_revs.append(rev) 1238 1239 return initial_rev, update_rev, changed_revs, patches 1240 1241def _remove_file_entries(srcuri, filelist): 1242 """Remove file:// entries from SRC_URI""" 1243 remaining = filelist[:] 1244 entries = [] 1245 for fname in filelist: 1246 basename = os.path.basename(fname) 1247 for i in range(len(srcuri)): 1248 if (srcuri[i].startswith('file://') and 1249 os.path.basename(srcuri[i].split(';')[0]) == basename): 1250 entries.append(srcuri[i]) 1251 remaining.remove(fname) 1252 srcuri.pop(i) 1253 break 1254 return entries, remaining 1255 1256def _replace_srcuri_entry(srcuri, filename, newentry): 1257 """Replace entry corresponding to specified file with a new entry""" 1258 basename = os.path.basename(filename) 1259 for i in range(len(srcuri)): 1260 if os.path.basename(srcuri[i].split(';')[0]) == basename: 1261 srcuri.pop(i) 1262 srcuri.insert(i, newentry) 1263 break 1264 1265def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1266 """Unlink existing patch files""" 1267 1268 dry_run_suffix = ' (dry-run)' if dry_run else '' 1269 1270 for path in files: 1271 if append: 1272 if not destpath: 1273 raise Exception('destpath should be set here') 1274 path = os.path.join(destpath, os.path.basename(path)) 1275 1276 if os.path.exists(path): 1277 if not no_report_remove: 1278 logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1279 if not dry_run: 1280 # FIXME "git rm" here would be nice if the file in question is 1281 # tracked 1282 # FIXME there's a chance that this file is referred to by 1283 # another recipe, in which case deleting wouldn't be the 1284 # right thing to do 1285 os.remove(path) 1286 # Remove directory if empty 1287 try: 1288 os.rmdir(os.path.dirname(path)) 1289 except OSError as ose: 1290 if ose.errno != errno.ENOTEMPTY: 1291 raise 1292 1293 1294def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1295 """Export patches from srctree to given location. 1296 Returns three-tuple of dicts: 1297 1. updated - patches that already exist in SRCURI 1298 2. added - new patches that don't exist in SRCURI 1299 3 removed - patches that exist in SRCURI but not in exported patches 1300 In each dict the key is the 'basepath' of the URI and value is the 1301 absolute path to the existing file in recipe space (if any). 1302 """ 1303 import oe.recipeutils 1304 from oe.patch import GitApplyTree 1305 updated = OrderedDict() 1306 added = OrderedDict() 1307 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1308 1309 existing_patches = dict((os.path.basename(path), path) for path in 1310 oe.recipeutils.get_recipe_patches(rd)) 1311 logger.debug('Existing patches: %s' % existing_patches) 1312 1313 # Generate patches from Git, exclude local files directory 1314 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1315 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) 1316 1317 new_patches = sorted(os.listdir(destdir)) 1318 for new_patch in new_patches: 1319 # Strip numbering from patch names. If it's a git sequence named patch, 1320 # the numbers might not match up since we are starting from a different 1321 # revision This does assume that people are using unique shortlog 1322 # values, but they ought to be anyway... 1323 new_basename = seqpatch_re.match(new_patch).group(2) 1324 match_name = None 1325 for old_patch in existing_patches: 1326 old_basename = seqpatch_re.match(old_patch).group(2) 1327 old_basename_splitext = os.path.splitext(old_basename) 1328 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1329 old_patch_noext = os.path.splitext(old_patch)[0] 1330 match_name = old_patch_noext 1331 break 1332 elif new_basename == old_basename: 1333 match_name = old_patch 1334 break 1335 if match_name: 1336 # Rename patch files 1337 if new_patch != match_name: 1338 bb.utils.rename(os.path.join(destdir, new_patch), 1339 os.path.join(destdir, match_name)) 1340 # Need to pop it off the list now before checking changed_revs 1341 oldpath = existing_patches.pop(old_patch) 1342 if changed_revs is not None: 1343 # Avoid updating patches that have not actually changed 1344 with open(os.path.join(destdir, match_name), 'r') as f: 1345 firstlineitems = f.readline().split() 1346 # Looking for "From <hash>" line 1347 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1348 if not firstlineitems[1] in changed_revs: 1349 continue 1350 # Recompress if necessary 1351 if oldpath.endswith(('.gz', '.Z')): 1352 bb.process.run(['gzip', match_name], cwd=destdir) 1353 if oldpath.endswith('.gz'): 1354 match_name += '.gz' 1355 else: 1356 match_name += '.Z' 1357 elif oldpath.endswith('.bz2'): 1358 bb.process.run(['bzip2', match_name], cwd=destdir) 1359 match_name += '.bz2' 1360 updated[match_name] = oldpath 1361 else: 1362 added[new_patch] = None 1363 return (updated, added, existing_patches) 1364 1365 1366def _create_kconfig_diff(srctree, rd, outfile): 1367 """Create a kconfig fragment""" 1368 # Only update config fragment if both config files exist 1369 orig_config = os.path.join(srctree, '.config.baseline') 1370 new_config = os.path.join(srctree, '.config.new') 1371 if os.path.exists(orig_config) and os.path.exists(new_config): 1372 cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1373 '--unchanged-line-format=', orig_config, new_config] 1374 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1375 stderr=subprocess.PIPE) 1376 stdout, stderr = pipe.communicate() 1377 if pipe.returncode == 1: 1378 logger.info("Updating config fragment %s" % outfile) 1379 with open(outfile, 'wb') as fobj: 1380 fobj.write(stdout) 1381 elif pipe.returncode == 0: 1382 logger.info("Would remove config fragment %s" % outfile) 1383 if os.path.exists(outfile): 1384 # Remove fragment file in case of empty diff 1385 logger.info("Removing config fragment %s" % outfile) 1386 os.unlink(outfile) 1387 else: 1388 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1389 return True 1390 return False 1391 1392 1393def _export_local_files(srctree, rd, destdir, srctreebase): 1394 """Copy local files from srctree to given location. 1395 Returns three-tuple of dicts: 1396 1. updated - files that already exist in SRCURI 1397 2. added - new files files that don't exist in SRCURI 1398 3 removed - files that exist in SRCURI but not in exported files 1399 In each dict the key is the 'basepath' of the URI and value is the 1400 absolute path to the existing file in recipe space (if any). 1401 """ 1402 import oe.recipeutils 1403 1404 # Find out local files (SRC_URI files that exist in the "recipe space"). 1405 # Local files that reside in srctree are not included in patch generation. 1406 # Instead they are directly copied over the original source files (in 1407 # recipe space). 1408 existing_files = oe.recipeutils.get_recipe_local_files(rd) 1409 new_set = None 1410 updated = OrderedDict() 1411 added = OrderedDict() 1412 removed = OrderedDict() 1413 1414 # Get current branch and return early with empty lists 1415 # if on one of the override branches 1416 # (local files are provided only for the main branch and processing 1417 # them against lists from recipe overrides will result in mismatches 1418 # and broken modifications to recipes). 1419 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1420 cwd=srctree) 1421 branchname = stdout.rstrip() 1422 if branchname.startswith(override_branch_prefix): 1423 return (updated, added, removed) 1424 1425 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1426 git_files = _git_ls_tree(srctree) 1427 if 'oe-local-files' in git_files: 1428 # If tracked by Git, take the files from srctree HEAD. First get 1429 # the tree object of the directory 1430 tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1431 tree = git_files['oe-local-files'][2] 1432 bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1433 env=dict(os.environ, GIT_WORK_TREE=destdir, 1434 GIT_INDEX_FILE=tmp_index)) 1435 new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1436 elif os.path.isdir(local_files_dir): 1437 # If not tracked by Git, just copy from working copy 1438 new_set = _ls_tree(local_files_dir) 1439 bb.process.run(['cp', '-ax', 1440 os.path.join(local_files_dir, '.'), destdir]) 1441 else: 1442 new_set = [] 1443 1444 # Special handling for kernel config 1445 if bb.data.inherits_class('kernel-yocto', rd): 1446 fragment_fn = 'devtool-fragment.cfg' 1447 fragment_path = os.path.join(destdir, fragment_fn) 1448 if _create_kconfig_diff(srctree, rd, fragment_path): 1449 if os.path.exists(fragment_path): 1450 if fragment_fn not in new_set: 1451 new_set.append(fragment_fn) 1452 # Copy fragment to local-files 1453 if os.path.isdir(local_files_dir): 1454 shutil.copy2(fragment_path, local_files_dir) 1455 else: 1456 if fragment_fn in new_set: 1457 new_set.remove(fragment_fn) 1458 # Remove fragment from local-files 1459 if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1460 os.unlink(os.path.join(local_files_dir, fragment_fn)) 1461 1462 # Special handling for cml1, ccmake, etc bbclasses that generated 1463 # configuration fragment files that are consumed as source files 1464 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1465 if bb.data.inherits_class(frag_class, rd): 1466 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1467 if os.path.exists(srcpath): 1468 if frag_name not in new_set: 1469 new_set.append(frag_name) 1470 # copy fragment into destdir 1471 shutil.copy2(srcpath, destdir) 1472 # copy fragment into local files if exists 1473 if os.path.isdir(local_files_dir): 1474 shutil.copy2(srcpath, local_files_dir) 1475 1476 if new_set is not None: 1477 for fname in new_set: 1478 if fname in existing_files: 1479 origpath = existing_files.pop(fname) 1480 workpath = os.path.join(local_files_dir, fname) 1481 if not filecmp.cmp(origpath, workpath): 1482 updated[fname] = origpath 1483 elif fname != '.gitignore': 1484 added[fname] = None 1485 1486 workdir = rd.getVar('WORKDIR') 1487 s = rd.getVar('S') 1488 if not s.endswith(os.sep): 1489 s += os.sep 1490 1491 if workdir != s: 1492 # Handle files where subdir= was specified 1493 for fname in list(existing_files.keys()): 1494 # FIXME handle both subdir starting with BP and not? 1495 fworkpath = os.path.join(workdir, fname) 1496 if fworkpath.startswith(s): 1497 fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1498 if os.path.exists(fpath): 1499 origpath = existing_files.pop(fname) 1500 if not filecmp.cmp(origpath, fpath): 1501 updated[fpath] = origpath 1502 1503 removed = existing_files 1504 return (updated, added, removed) 1505 1506 1507def _determine_files_dir(rd): 1508 """Determine the appropriate files directory for a recipe""" 1509 recipedir = rd.getVar('FILE_DIRNAME') 1510 for entry in rd.getVar('FILESPATH').split(':'): 1511 relpth = os.path.relpath(entry, recipedir) 1512 if not os.sep in relpth: 1513 # One (or zero) levels below only, so we don't put anything in machine-specific directories 1514 if os.path.isdir(entry): 1515 return entry 1516 return os.path.join(recipedir, rd.getVar('BPN')) 1517 1518 1519def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1520 """Implement the 'srcrev' mode of update-recipe""" 1521 import bb 1522 import oe.recipeutils 1523 1524 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1525 1526 recipefile = rd.getVar('FILE') 1527 recipedir = os.path.basename(recipefile) 1528 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1529 1530 # Get HEAD revision 1531 try: 1532 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1533 except bb.process.ExecutionError as err: 1534 raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1535 (srctree, err)) 1536 srcrev = stdout.strip() 1537 if len(srcrev) != 40: 1538 raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1539 1540 destpath = None 1541 remove_files = [] 1542 patchfields = {} 1543 patchfields['SRCREV'] = srcrev 1544 orig_src_uri = rd.getVar('SRC_URI', False) or '' 1545 srcuri = orig_src_uri.split() 1546 tempdir = tempfile.mkdtemp(prefix='devtool') 1547 update_srcuri = False 1548 appendfile = None 1549 try: 1550 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1551 srctreebase = workspace[recipename]['srctreebase'] 1552 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1553 if not no_remove: 1554 # Find list of existing patches in recipe file 1555 patches_dir = tempfile.mkdtemp(dir=tempdir) 1556 old_srcrev = rd.getVar('SRCREV') or '' 1557 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1558 patches_dir) 1559 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1560 1561 # Remove deleted local files and "overlapping" patches 1562 remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) 1563 if remove_files: 1564 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1565 update_srcuri = True 1566 1567 if appendlayerdir: 1568 files = dict((os.path.join(local_files_dir, key), val) for 1569 key, val in list(upd_f.items()) + list(new_f.items())) 1570 removevalues = {} 1571 if update_srcuri: 1572 removevalues = {'SRC_URI': removedentries} 1573 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1574 if dry_run_outdir: 1575 logger.info('Creating bbappend (dry-run)') 1576 else: 1577 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1578 rd, appendlayerdir, files, wildcardver=wildcard_version, 1579 extralines=patchfields, removevalues=removevalues, 1580 redirect_output=dry_run_outdir) 1581 else: 1582 files_dir = _determine_files_dir(rd) 1583 for basepath, path in upd_f.items(): 1584 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1585 if os.path.isabs(basepath): 1586 # Original file (probably with subdir pointing inside source tree) 1587 # so we do not want to move it, just copy 1588 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1589 else: 1590 _move_file(os.path.join(local_files_dir, basepath), path, 1591 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1592 update_srcuri= True 1593 for basepath, path in new_f.items(): 1594 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1595 _move_file(os.path.join(local_files_dir, basepath), 1596 os.path.join(files_dir, basepath), 1597 dry_run_outdir=dry_run_outdir, 1598 base_outdir=recipedir) 1599 srcuri.append('file://%s' % basepath) 1600 update_srcuri = True 1601 if update_srcuri: 1602 patchfields['SRC_URI'] = ' '.join(srcuri) 1603 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1604 finally: 1605 shutil.rmtree(tempdir) 1606 if not 'git://' in orig_src_uri: 1607 logger.info('You will need to update SRC_URI within the recipe to ' 1608 'point to a git repository where you have pushed your ' 1609 'changes') 1610 1611 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1612 return True, appendfile, remove_files 1613 1614def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1615 """Implement the 'patch' mode of update-recipe""" 1616 import bb 1617 import oe.recipeutils 1618 1619 recipefile = rd.getVar('FILE') 1620 recipedir = os.path.dirname(recipefile) 1621 append = workspace[recipename]['bbappend'] 1622 if not os.path.exists(append): 1623 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1624 recipename) 1625 srctreebase = workspace[recipename]['srctreebase'] 1626 relpatchdir = os.path.relpath(srctreebase, srctree) 1627 if relpatchdir == '.': 1628 patchdir_params = {} 1629 else: 1630 patchdir_params = {'patchdir': relpatchdir} 1631 1632 def srcuri_entry(basepath): 1633 if patchdir_params: 1634 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1635 else: 1636 paramstr = '' 1637 return 'file://%s%s' % (basepath, paramstr) 1638 1639 initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1640 if not initial_rev: 1641 raise DevtoolError('Unable to find initial revision - please specify ' 1642 'it with --initial-rev') 1643 1644 appendfile = None 1645 dl_dir = rd.getVar('DL_DIR') 1646 if not dl_dir.endswith('/'): 1647 dl_dir += '/' 1648 1649 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1650 1651 tempdir = tempfile.mkdtemp(prefix='devtool') 1652 try: 1653 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1654 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1655 1656 # Get updated patches from source tree 1657 patches_dir = tempfile.mkdtemp(dir=tempdir) 1658 upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, 1659 patches_dir, changed_revs) 1660 # Get all patches from source tree and check if any should be removed 1661 all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1662 _, _, del_p = _export_patches(srctree, rd, initial_rev, 1663 all_patches_dir) 1664 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1665 if filter_patches: 1666 new_p = OrderedDict() 1667 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1668 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) 1669 remove_files = [] 1670 if not no_remove: 1671 # Remove deleted local files and patches 1672 remove_files = list(del_f.values()) + list(del_p.values()) 1673 updatefiles = False 1674 updaterecipe = False 1675 destpath = None 1676 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1677 if appendlayerdir: 1678 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1679 key, val in list(upd_f.items()) + list(new_f.items())) 1680 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1681 key, val in list(upd_p.items()) + list(new_p.items()))) 1682 if files or remove_files: 1683 removevalues = None 1684 if remove_files: 1685 removedentries, remaining = _remove_file_entries( 1686 srcuri, remove_files) 1687 if removedentries or remaining: 1688 remaining = [srcuri_entry(os.path.basename(item)) for 1689 item in remaining] 1690 removevalues = {'SRC_URI': removedentries + remaining} 1691 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1692 rd, appendlayerdir, files, 1693 wildcardver=wildcard_version, 1694 removevalues=removevalues, 1695 redirect_output=dry_run_outdir, 1696 params=[patchdir_params] * len(files)) 1697 else: 1698 logger.info('No patches or local source files needed updating') 1699 else: 1700 # Update existing files 1701 files_dir = _determine_files_dir(rd) 1702 for basepath, path in upd_f.items(): 1703 logger.info('Updating file %s' % basepath) 1704 if os.path.isabs(basepath): 1705 # Original file (probably with subdir pointing inside source tree) 1706 # so we do not want to move it, just copy 1707 _copy_file(basepath, path, 1708 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1709 else: 1710 _move_file(os.path.join(local_files_dir, basepath), path, 1711 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1712 updatefiles = True 1713 for basepath, path in upd_p.items(): 1714 patchfn = os.path.join(patches_dir, basepath) 1715 if os.path.dirname(path) + '/' == dl_dir: 1716 # This is a a downloaded patch file - we now need to 1717 # replace the entry in SRC_URI with our local version 1718 logger.info('Replacing remote patch %s with updated local version' % basepath) 1719 path = os.path.join(files_dir, basepath) 1720 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath)) 1721 updaterecipe = True 1722 else: 1723 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1724 _move_file(patchfn, path, 1725 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1726 updatefiles = True 1727 # Add any new files 1728 for basepath, path in new_f.items(): 1729 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1730 _move_file(os.path.join(local_files_dir, basepath), 1731 os.path.join(files_dir, basepath), 1732 dry_run_outdir=dry_run_outdir, 1733 base_outdir=recipedir) 1734 srcuri.append(srcuri_entry(basepath)) 1735 updaterecipe = True 1736 for basepath, path in new_p.items(): 1737 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1738 _move_file(os.path.join(patches_dir, basepath), 1739 os.path.join(files_dir, basepath), 1740 dry_run_outdir=dry_run_outdir, 1741 base_outdir=recipedir) 1742 srcuri.append(srcuri_entry(basepath)) 1743 updaterecipe = True 1744 # Update recipe, if needed 1745 if _remove_file_entries(srcuri, remove_files)[0]: 1746 updaterecipe = True 1747 if updaterecipe: 1748 if not dry_run_outdir: 1749 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1750 ret = oe.recipeutils.patch_recipe(rd, recipefile, 1751 {'SRC_URI': ' '.join(srcuri)}, 1752 redirect_output=dry_run_outdir) 1753 elif not updatefiles: 1754 # Neither patches nor recipe were updated 1755 logger.info('No patches or files need updating') 1756 return False, None, [] 1757 finally: 1758 shutil.rmtree(tempdir) 1759 1760 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1761 return True, appendfile, remove_files 1762 1763def _guess_recipe_update_mode(srctree, rdata): 1764 """Guess the recipe update mode to use""" 1765 src_uri = (rdata.getVar('SRC_URI') or '').split() 1766 git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1767 if not git_uris: 1768 return 'patch' 1769 # Just use the first URI for now 1770 uri = git_uris[0] 1771 # Check remote branch 1772 params = bb.fetch.decodeurl(uri)[5] 1773 upstr_branch = params['branch'] if 'branch' in params else 'master' 1774 # Check if current branch HEAD is found in upstream branch 1775 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1776 head_rev = stdout.rstrip() 1777 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1778 cwd=srctree) 1779 remote_brs = [branch.strip() for branch in stdout.splitlines()] 1780 if 'origin/' + upstr_branch in remote_brs: 1781 return 'srcrev' 1782 1783 return 'patch' 1784 1785def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1786 srctree = workspace[recipename]['srctree'] 1787 if mode == 'auto': 1788 mode = _guess_recipe_update_mode(srctree, rd) 1789 1790 override_branches = [] 1791 mainbranch = None 1792 startbranch = None 1793 if not no_overrides: 1794 stdout, _ = bb.process.run('git branch', cwd=srctree) 1795 other_branches = [] 1796 for line in stdout.splitlines(): 1797 branchname = line[2:] 1798 if line.startswith('* '): 1799 startbranch = branchname 1800 if branchname.startswith(override_branch_prefix): 1801 override_branches.append(branchname) 1802 else: 1803 other_branches.append(branchname) 1804 1805 if override_branches: 1806 logger.debug('_update_recipe: override branches: %s' % override_branches) 1807 logger.debug('_update_recipe: other branches: %s' % other_branches) 1808 if startbranch.startswith(override_branch_prefix): 1809 if len(other_branches) == 1: 1810 mainbranch = other_branches[1] 1811 else: 1812 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1813 else: 1814 mainbranch = startbranch 1815 1816 checkedout = None 1817 anyupdated = False 1818 appendfile = None 1819 allremoved = [] 1820 if override_branches: 1821 logger.info('Handling main branch (%s)...' % mainbranch) 1822 if startbranch != mainbranch: 1823 bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1824 checkedout = mainbranch 1825 try: 1826 branchlist = [mainbranch] + override_branches 1827 for branch in branchlist: 1828 crd = bb.data.createCopy(rd) 1829 if branch != mainbranch: 1830 logger.info('Handling branch %s...' % branch) 1831 override = branch[len(override_branch_prefix):] 1832 crd.appendVar('OVERRIDES', ':%s' % override) 1833 bb.process.run('git checkout %s' % branch, cwd=srctree) 1834 checkedout = branch 1835 1836 if mode == 'srcrev': 1837 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1838 elif mode == 'patch': 1839 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1840 else: 1841 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1842 if updated: 1843 anyupdated = True 1844 if appendf: 1845 appendfile = appendf 1846 allremoved.extend(removed) 1847 finally: 1848 if startbranch and checkedout != startbranch: 1849 bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1850 1851 return anyupdated, appendfile, allremoved 1852 1853def update_recipe(args, config, basepath, workspace): 1854 """Entry point for the devtool 'update-recipe' subcommand""" 1855 check_workspace_recipe(workspace, args.recipename) 1856 1857 if args.append: 1858 if not os.path.exists(args.append): 1859 raise DevtoolError('bbappend destination layer directory "%s" ' 1860 'does not exist' % args.append) 1861 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1862 raise DevtoolError('conf/layer.conf not found in bbappend ' 1863 'destination layer "%s"' % args.append) 1864 1865 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1866 try: 1867 1868 rd = parse_recipe(config, tinfoil, args.recipename, True) 1869 if not rd: 1870 return 1 1871 1872 dry_run_output = None 1873 dry_run_outdir = None 1874 if args.dry_run: 1875 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1876 dry_run_outdir = dry_run_output.name 1877 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1878 1879 if updated: 1880 rf = rd.getVar('FILE') 1881 if rf.startswith(config.workspace_path): 1882 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1883 finally: 1884 tinfoil.shutdown() 1885 1886 return 0 1887 1888 1889def status(args, config, basepath, workspace): 1890 """Entry point for the devtool 'status' subcommand""" 1891 if workspace: 1892 for recipe, value in sorted(workspace.items()): 1893 recipefile = value['recipefile'] 1894 if recipefile: 1895 recipestr = ' (%s)' % recipefile 1896 else: 1897 recipestr = '' 1898 print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1899 else: 1900 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1901 return 0 1902 1903 1904def _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1905 """Reset one or more recipes""" 1906 import oe.path 1907 1908 def clean_preferred_provider(pn, layerconf_path): 1909 """Remove PREFERRED_PROVIDER from layer.conf'""" 1910 import re 1911 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1912 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1913 pprovider_found = False 1914 with open(layerconf_file, 'r') as f: 1915 lines = f.readlines() 1916 with open(new_layerconf_file, 'a') as nf: 1917 for line in lines: 1918 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1919 if not re.match(pprovider_exp, line): 1920 nf.write(line) 1921 else: 1922 pprovider_found = True 1923 if pprovider_found: 1924 shutil.move(new_layerconf_file, layerconf_file) 1925 else: 1926 os.remove(new_layerconf_file) 1927 1928 if recipes and not no_clean: 1929 if len(recipes) == 1: 1930 logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 1931 else: 1932 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 1933 # If the recipe file itself was created in the workspace, and 1934 # it uses BBCLASSEXTEND, then we need to also clean the other 1935 # variants 1936 targets = [] 1937 for recipe in recipes: 1938 targets.append(recipe) 1939 recipefile = workspace[recipe]['recipefile'] 1940 if recipefile and os.path.exists(recipefile): 1941 targets.extend(get_bbclassextend_targets(recipefile, recipe)) 1942 try: 1943 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 1944 except bb.process.ExecutionError as e: 1945 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 1946 'wish, you may specify -n/--no-clean to ' 1947 'skip running this command when resetting' % 1948 (e.command, e.stdout)) 1949 1950 for pn in recipes: 1951 _check_preserve(config, pn) 1952 1953 appendfile = workspace[pn]['bbappend'] 1954 if os.path.exists(appendfile): 1955 # This shouldn't happen, but is possible if devtool errored out prior to 1956 # writing the md5 file. We need to delete this here or the recipe won't 1957 # actually be reset 1958 os.remove(appendfile) 1959 1960 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 1961 def preservedir(origdir): 1962 if os.path.exists(origdir): 1963 for root, dirs, files in os.walk(origdir): 1964 for fn in files: 1965 logger.warning('Preserving %s in %s' % (fn, preservepath)) 1966 _move_file(os.path.join(origdir, fn), 1967 os.path.join(preservepath, fn)) 1968 for dn in dirs: 1969 preservedir(os.path.join(root, dn)) 1970 os.rmdir(origdir) 1971 1972 recipefile = workspace[pn]['recipefile'] 1973 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 1974 # This should always be true if recipefile is set, but just in case 1975 preservedir(os.path.dirname(recipefile)) 1976 # We don't automatically create this dir next to appends, but the user can 1977 preservedir(os.path.join(config.workspace_path, 'appends', pn)) 1978 1979 srctreebase = workspace[pn]['srctreebase'] 1980 if os.path.isdir(srctreebase): 1981 if os.listdir(srctreebase): 1982 if remove_work: 1983 logger.info('-r argument used on %s, removing source tree.' 1984 ' You will lose any unsaved work' %pn) 1985 shutil.rmtree(srctreebase) 1986 else: 1987 # We don't want to risk wiping out any work in progress 1988 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): 1989 from datetime import datetime 1990 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn,datetime.now().strftime("%Y%m%d%H%M%S"))) 1991 logger.info('Preserving source tree in %s\nIf you no ' 1992 'longer need it then please delete it manually.\n' 1993 'It is also possible to reuse it via devtool source tree argument.' 1994 % preservesrc) 1995 bb.utils.mkdirhier(os.path.dirname(preservesrc)) 1996 shutil.move(srctreebase, preservesrc) 1997 else: 1998 logger.info('Leaving source tree %s as-is; if you no ' 1999 'longer need it then please delete it manually' 2000 % srctreebase) 2001 else: 2002 # This is unlikely, but if it's empty we can just remove it 2003 os.rmdir(srctreebase) 2004 2005 clean_preferred_provider(pn, config.workspace_path) 2006 2007def reset(args, config, basepath, workspace): 2008 """Entry point for the devtool 'reset' subcommand""" 2009 import bb 2010 import shutil 2011 2012 recipes = "" 2013 2014 if args.recipename: 2015 if args.all: 2016 raise DevtoolError("Recipe cannot be specified if -a/--all is used") 2017 else: 2018 for recipe in args.recipename: 2019 check_workspace_recipe(workspace, recipe, checksrc=False) 2020 elif not args.all: 2021 raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2022 "reset all recipes") 2023 if args.all: 2024 recipes = list(workspace.keys()) 2025 else: 2026 recipes = args.recipename 2027 2028 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2029 2030 return 0 2031 2032 2033def _get_layer(layername, d): 2034 """Determine the base layer path for the specified layer name/path""" 2035 layerdirs = d.getVar('BBLAYERS').split() 2036 layers = {} # {basename: layer_paths} 2037 for p in layerdirs: 2038 bn = os.path.basename(p) 2039 if bn not in layers: 2040 layers[bn] = [p] 2041 else: 2042 layers[bn].append(p) 2043 # Provide some shortcuts 2044 if layername.lower() in ['oe-core', 'openembedded-core']: 2045 layername = 'meta' 2046 layer_paths = layers.get(layername, None) 2047 if not layer_paths: 2048 return os.path.abspath(layername) 2049 elif len(layer_paths) == 1: 2050 return os.path.abspath(layer_paths[0]) 2051 else: 2052 # multiple layers having the same base name 2053 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 2054 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 2055 return os.path.abspath(layer_paths[0]) 2056 2057 2058def finish(args, config, basepath, workspace): 2059 """Entry point for the devtool 'finish' subcommand""" 2060 import bb 2061 import oe.recipeutils 2062 2063 check_workspace_recipe(workspace, args.recipename) 2064 2065 dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2066 2067 # Grab the equivalent of COREBASE without having to initialise tinfoil 2068 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2069 2070 srctree = workspace[args.recipename]['srctree'] 2071 check_git_repo_op(srctree, [corebasedir]) 2072 dirty = check_git_repo_dirty(srctree) 2073 if dirty: 2074 if args.force: 2075 logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2076 else: 2077 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2078 2079 no_clean = args.no_clean 2080 remove_work=args.remove_work 2081 tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2082 try: 2083 rd = parse_recipe(config, tinfoil, args.recipename, True) 2084 if not rd: 2085 return 1 2086 2087 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2088 recipefile = rd.getVar('FILE') 2089 recipedir = os.path.dirname(recipefile) 2090 origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2091 2092 if not os.path.isdir(destlayerdir): 2093 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2094 2095 if os.path.abspath(destlayerdir) == config.workspace_path: 2096 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2097 2098 # If it's an upgrade, grab the original path 2099 origpath = None 2100 origfilelist = None 2101 append = workspace[args.recipename]['bbappend'] 2102 with open(append, 'r') as f: 2103 for line in f: 2104 if line.startswith('# original_path:'): 2105 origpath = line.split(':')[1].strip() 2106 elif line.startswith('# original_files:'): 2107 origfilelist = line.split(':')[1].split() 2108 2109 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2110 2111 if origlayerdir == config.workspace_path: 2112 # Recipe file itself is in workspace, update it there first 2113 appendlayerdir = None 2114 origrelpath = None 2115 if origpath: 2116 origlayerpath = oe.recipeutils.find_layerdir(origpath) 2117 if origlayerpath: 2118 origrelpath = os.path.relpath(origpath, origlayerpath) 2119 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2120 if not destpath: 2121 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2122 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2123 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2124 if not os.path.abspath(destlayerbasedir) in layerdirs: 2125 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2126 2127 elif destlayerdir == origlayerdir: 2128 # Same layer, update the original recipe 2129 appendlayerdir = None 2130 destpath = None 2131 else: 2132 # Create/update a bbappend in the specified layer 2133 appendlayerdir = destlayerdir 2134 destpath = None 2135 2136 # Actually update the recipe / bbappend 2137 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2138 dry_run_output = None 2139 dry_run_outdir = None 2140 if args.dry_run: 2141 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2142 dry_run_outdir = dry_run_output.name 2143 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2144 removed = [os.path.relpath(pth, recipedir) for pth in removed] 2145 2146 # Remove any old files in the case of an upgrade 2147 if removing_original: 2148 for fn in origfilelist: 2149 fnp = os.path.join(origpath, fn) 2150 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2151 logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2152 if not args.dry_run: 2153 try: 2154 os.remove(fnp) 2155 except FileNotFoundError: 2156 pass 2157 2158 if origlayerdir == config.workspace_path and destpath: 2159 # Recipe file itself is in the workspace - need to move it and any 2160 # associated files to the specified layer 2161 no_clean = True 2162 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2163 for root, _, files in os.walk(recipedir): 2164 for fn in files: 2165 srcpath = os.path.join(root, fn) 2166 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2167 destdir = os.path.abspath(os.path.join(destpath, relpth)) 2168 destfp = os.path.join(destdir, fn) 2169 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2170 2171 if dry_run_outdir: 2172 import difflib 2173 comparelist = [] 2174 for root, _, files in os.walk(dry_run_outdir): 2175 for fn in files: 2176 outf = os.path.join(root, fn) 2177 relf = os.path.relpath(outf, dry_run_outdir) 2178 logger.debug('dry-run: output file %s' % relf) 2179 if fn.endswith('.bb'): 2180 if origfilelist and origpath and destpath: 2181 # Need to match this up with the pre-upgrade recipe file 2182 for origf in origfilelist: 2183 if origf.endswith('.bb'): 2184 comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2185 outf, 2186 os.path.abspath(os.path.join(destpath, relf)))) 2187 break 2188 else: 2189 # Compare to the existing recipe 2190 comparelist.append((recipefile, outf, recipefile)) 2191 elif fn.endswith('.bbappend'): 2192 if appendfile: 2193 if os.path.exists(appendfile): 2194 comparelist.append((appendfile, outf, appendfile)) 2195 else: 2196 comparelist.append((None, outf, appendfile)) 2197 else: 2198 if destpath: 2199 recipedest = destpath 2200 elif appendfile: 2201 recipedest = os.path.dirname(appendfile) 2202 else: 2203 recipedest = os.path.dirname(recipefile) 2204 destfp = os.path.join(recipedest, relf) 2205 if os.path.exists(destfp): 2206 comparelist.append((destfp, outf, destfp)) 2207 output = '' 2208 for oldfile, newfile, newfileshow in comparelist: 2209 if oldfile: 2210 with open(oldfile, 'r') as f: 2211 oldlines = f.readlines() 2212 else: 2213 oldfile = '/dev/null' 2214 oldlines = [] 2215 with open(newfile, 'r') as f: 2216 newlines = f.readlines() 2217 if not newfileshow: 2218 newfileshow = newfile 2219 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2220 difflines = list(diff) 2221 if difflines: 2222 output += ''.join(difflines) 2223 if output: 2224 logger.info('Diff of changed files:\n%s' % output) 2225 finally: 2226 tinfoil.shutdown() 2227 2228 # Everything else has succeeded, we can now reset 2229 if args.dry_run: 2230 logger.info('Resetting recipe (dry-run)') 2231 else: 2232 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2233 2234 return 0 2235 2236 2237def get_default_srctree(config, recipename=''): 2238 """Get the default srctree path""" 2239 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2240 if recipename: 2241 return os.path.join(srctreeparent, 'sources', recipename) 2242 else: 2243 return os.path.join(srctreeparent, 'sources') 2244 2245def register_commands(subparsers, context): 2246 """Register devtool subcommands from this plugin""" 2247 2248 defsrctree = get_default_srctree(context.config) 2249 parser_add = subparsers.add_parser('add', help='Add a new recipe', 2250 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2251 group='starting', order=100) 2252 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2253 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2254 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2255 group = parser_add.add_mutually_exclusive_group() 2256 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2257 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2258 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2259 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2260 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2261 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2262 group = parser_add.add_mutually_exclusive_group() 2263 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2264 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2265 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2266 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2267 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2268 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2269 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2270 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2271 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2272 2273 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2274 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2275 group='starting', order=90) 2276 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2277 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2278 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2279 group = parser_modify.add_mutually_exclusive_group() 2280 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2281 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2282 group = parser_modify.add_mutually_exclusive_group() 2283 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2284 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2285 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2286 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2287 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2288 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2289 2290 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2291 description='Extracts the source for an existing recipe', 2292 group='advanced') 2293 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2294 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2295 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2296 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2297 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2298 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2299 2300 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2301 description='Synchronize the previously extracted source tree for an existing recipe', 2302 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2303 group='advanced') 2304 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2305 parser_sync.add_argument('srctree', help='Path to the source tree') 2306 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2307 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2308 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2309 2310 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2311 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2312 group='working', order=10) 2313 parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2314 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2315 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2316 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2317 parser_rename.set_defaults(func=rename) 2318 2319 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2320 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2321 group='working', order=-90) 2322 parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2323 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2324 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2325 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2326 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2327 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2328 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2329 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2330 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2331 parser_update_recipe.set_defaults(func=update_recipe) 2332 2333 parser_status = subparsers.add_parser('status', help='Show workspace status', 2334 description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2335 group='info', order=100) 2336 parser_status.set_defaults(func=status) 2337 2338 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2339 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2340 group='working', order=-100) 2341 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2342 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2343 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2344 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2345 parser_reset.set_defaults(func=reset) 2346 2347 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2348 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2349 group='working', order=-100) 2350 parser_finish.add_argument('recipename', help='Recipe to finish') 2351 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2352 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2353 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2354 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2355 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 2356 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2357 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2358 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2359 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2360 parser_finish.set_defaults(func=finish) 2361