1eb8dc403SDave Cobbley# Development tool - standard commands plugin 2eb8dc403SDave Cobbley# 3eb8dc403SDave Cobbley# Copyright (C) 2014-2017 Intel Corporation 4eb8dc403SDave Cobbley# 5c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 6eb8dc403SDave Cobbley# 7eb8dc403SDave Cobbley"""Devtool standard plugins""" 8eb8dc403SDave Cobbley 9eb8dc403SDave Cobbleyimport os 10eb8dc403SDave Cobbleyimport sys 11eb8dc403SDave Cobbleyimport re 12eb8dc403SDave Cobbleyimport shutil 13eb8dc403SDave Cobbleyimport subprocess 14eb8dc403SDave Cobbleyimport tempfile 15eb8dc403SDave Cobbleyimport logging 16eb8dc403SDave Cobbleyimport argparse 17eb8dc403SDave Cobbleyimport argparse_oe 18eb8dc403SDave Cobbleyimport scriptutils 19eb8dc403SDave Cobbleyimport errno 20eb8dc403SDave Cobbleyimport glob 21eb8dc403SDave Cobbleyimport filecmp 22eb8dc403SDave Cobbleyfrom collections import OrderedDict 23eb8dc403SDave Cobbleyfrom devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 24eb8dc403SDave Cobbleyfrom devtool import parse_recipe 25eb8dc403SDave Cobbley 26eb8dc403SDave Cobbleylogger = logging.getLogger('devtool') 27eb8dc403SDave Cobbley 28eb8dc403SDave Cobbleyoverride_branch_prefix = 'devtool-override-' 29eb8dc403SDave Cobbley 30eb8dc403SDave Cobbley 31eb8dc403SDave Cobbleydef add(args, config, basepath, workspace): 32eb8dc403SDave Cobbley """Entry point for the devtool 'add' subcommand""" 33eb8dc403SDave Cobbley import bb 34eb8dc403SDave Cobbley import oe.recipeutils 35eb8dc403SDave Cobbley 36eb8dc403SDave Cobbley if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 37eb8dc403SDave Cobbley raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 38eb8dc403SDave Cobbley 39eb8dc403SDave Cobbley # These are positional arguments, but because we're nice, allow 40eb8dc403SDave Cobbley # specifying e.g. source tree without name, or fetch URI without name or 41eb8dc403SDave Cobbley # source tree (if we can detect that that is what the user meant) 42eb8dc403SDave Cobbley if scriptutils.is_src_url(args.recipename): 43eb8dc403SDave Cobbley if not args.fetchuri: 44eb8dc403SDave Cobbley if args.fetch: 45eb8dc403SDave Cobbley raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 46eb8dc403SDave Cobbley args.fetchuri = args.recipename 47eb8dc403SDave Cobbley args.recipename = '' 48eb8dc403SDave Cobbley elif scriptutils.is_src_url(args.srctree): 49eb8dc403SDave Cobbley if not args.fetchuri: 50eb8dc403SDave Cobbley if args.fetch: 51eb8dc403SDave Cobbley raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 52eb8dc403SDave Cobbley args.fetchuri = args.srctree 53eb8dc403SDave Cobbley args.srctree = '' 54eb8dc403SDave Cobbley elif args.recipename and not args.srctree: 55eb8dc403SDave Cobbley if os.sep in args.recipename: 56eb8dc403SDave Cobbley args.srctree = args.recipename 57eb8dc403SDave Cobbley args.recipename = None 58eb8dc403SDave Cobbley elif os.path.isdir(args.recipename): 591a4b7ee2SBrad Bishop logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 60eb8dc403SDave Cobbley 61eb8dc403SDave Cobbley if not args.fetchuri: 62eb8dc403SDave Cobbley if args.srcrev: 63eb8dc403SDave Cobbley raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 64eb8dc403SDave Cobbley if args.srcbranch: 65eb8dc403SDave Cobbley raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 66eb8dc403SDave Cobbley 67eb8dc403SDave Cobbley if args.srctree and os.path.isfile(args.srctree): 68eb8dc403SDave Cobbley args.fetchuri = 'file://' + os.path.abspath(args.srctree) 69eb8dc403SDave Cobbley args.srctree = '' 70eb8dc403SDave Cobbley 71eb8dc403SDave Cobbley if args.fetch: 72eb8dc403SDave Cobbley if args.fetchuri: 73eb8dc403SDave Cobbley raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 74eb8dc403SDave Cobbley else: 751a4b7ee2SBrad Bishop logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 76eb8dc403SDave Cobbley args.fetchuri = args.fetch 77eb8dc403SDave Cobbley 78eb8dc403SDave Cobbley if args.recipename: 79eb8dc403SDave Cobbley if args.recipename in workspace: 80eb8dc403SDave Cobbley raise DevtoolError("recipe %s is already in your workspace" % 81eb8dc403SDave Cobbley args.recipename) 82eb8dc403SDave Cobbley reason = oe.recipeutils.validate_pn(args.recipename) 83eb8dc403SDave Cobbley if reason: 84eb8dc403SDave Cobbley raise DevtoolError(reason) 85eb8dc403SDave Cobbley 86eb8dc403SDave Cobbley if args.srctree: 87eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 88eb8dc403SDave Cobbley srctreeparent = None 89eb8dc403SDave Cobbley tmpsrcdir = None 90eb8dc403SDave Cobbley else: 91eb8dc403SDave Cobbley srctree = None 92eb8dc403SDave Cobbley srctreeparent = get_default_srctree(config) 93eb8dc403SDave Cobbley bb.utils.mkdirhier(srctreeparent) 94eb8dc403SDave Cobbley tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 95eb8dc403SDave Cobbley 96eb8dc403SDave Cobbley if srctree and os.path.exists(srctree): 97eb8dc403SDave Cobbley if args.fetchuri: 98eb8dc403SDave Cobbley if not os.path.isdir(srctree): 99eb8dc403SDave Cobbley raise DevtoolError("Cannot fetch into source tree path %s as " 100eb8dc403SDave Cobbley "it exists and is not a directory" % 101eb8dc403SDave Cobbley srctree) 102eb8dc403SDave Cobbley elif os.listdir(srctree): 103eb8dc403SDave Cobbley raise DevtoolError("Cannot fetch into source tree path %s as " 104eb8dc403SDave Cobbley "it already exists and is non-empty" % 105eb8dc403SDave Cobbley srctree) 106eb8dc403SDave Cobbley elif not args.fetchuri: 107eb8dc403SDave Cobbley if args.srctree: 108eb8dc403SDave Cobbley raise DevtoolError("Specified source tree %s could not be found" % 109eb8dc403SDave Cobbley args.srctree) 110eb8dc403SDave Cobbley elif srctree: 111eb8dc403SDave Cobbley raise DevtoolError("No source tree exists at default path %s - " 112eb8dc403SDave Cobbley "either create and populate this directory, " 113eb8dc403SDave Cobbley "or specify a path to a source tree, or a " 114eb8dc403SDave Cobbley "URI to fetch source from" % srctree) 115eb8dc403SDave Cobbley else: 116eb8dc403SDave Cobbley raise DevtoolError("You must either specify a source tree " 117eb8dc403SDave Cobbley "or a URI to fetch source from") 118eb8dc403SDave Cobbley 119eb8dc403SDave Cobbley if args.version: 120eb8dc403SDave Cobbley if '_' in args.version or ' ' in args.version: 121eb8dc403SDave Cobbley raise DevtoolError('Invalid version string "%s"' % args.version) 122eb8dc403SDave Cobbley 123eb8dc403SDave Cobbley if args.color == 'auto' and sys.stdout.isatty(): 124eb8dc403SDave Cobbley color = 'always' 125eb8dc403SDave Cobbley else: 126eb8dc403SDave Cobbley color = args.color 127eb8dc403SDave Cobbley extracmdopts = '' 128eb8dc403SDave Cobbley if args.fetchuri: 129eb8dc403SDave Cobbley source = args.fetchuri 130eb8dc403SDave Cobbley if srctree: 131eb8dc403SDave Cobbley extracmdopts += ' -x %s' % srctree 132eb8dc403SDave Cobbley else: 133eb8dc403SDave Cobbley extracmdopts += ' -x %s' % tmpsrcdir 134eb8dc403SDave Cobbley else: 135eb8dc403SDave Cobbley source = srctree 136eb8dc403SDave Cobbley if args.recipename: 137eb8dc403SDave Cobbley extracmdopts += ' -N %s' % args.recipename 138eb8dc403SDave Cobbley if args.version: 139eb8dc403SDave Cobbley extracmdopts += ' -V %s' % args.version 140eb8dc403SDave Cobbley if args.binary: 141eb8dc403SDave Cobbley extracmdopts += ' -b' 142eb8dc403SDave Cobbley if args.also_native: 143eb8dc403SDave Cobbley extracmdopts += ' --also-native' 144eb8dc403SDave Cobbley if args.src_subdir: 145eb8dc403SDave Cobbley extracmdopts += ' --src-subdir "%s"' % args.src_subdir 146eb8dc403SDave Cobbley if args.autorev: 147eb8dc403SDave Cobbley extracmdopts += ' -a' 14882c905dcSAndrew Geissler if args.npm_dev: 14982c905dcSAndrew Geissler extracmdopts += ' --npm-dev' 150169d7bccSPatrick Williams if args.no_pypi: 151169d7bccSPatrick Williams extracmdopts += ' --no-pypi' 152eb8dc403SDave Cobbley if args.mirrors: 153eb8dc403SDave Cobbley extracmdopts += ' --mirrors' 154eb8dc403SDave Cobbley if args.srcrev: 155eb8dc403SDave Cobbley extracmdopts += ' --srcrev %s' % args.srcrev 156eb8dc403SDave Cobbley if args.srcbranch: 157eb8dc403SDave Cobbley extracmdopts += ' --srcbranch %s' % args.srcbranch 158eb8dc403SDave Cobbley if args.provides: 159eb8dc403SDave Cobbley extracmdopts += ' --provides %s' % args.provides 160eb8dc403SDave Cobbley 161eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtool') 162eb8dc403SDave Cobbley try: 163eb8dc403SDave Cobbley try: 164eb8dc403SDave Cobbley stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 165eb8dc403SDave Cobbley except bb.process.ExecutionError as e: 166eb8dc403SDave Cobbley if e.exitcode == 15: 167eb8dc403SDave Cobbley raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 168eb8dc403SDave Cobbley else: 169eb8dc403SDave Cobbley raise DevtoolError('Command \'%s\' failed' % e.command) 170eb8dc403SDave Cobbley 171eb8dc403SDave Cobbley recipes = glob.glob(os.path.join(tempdir, '*.bb')) 172eb8dc403SDave Cobbley if recipes: 173eb8dc403SDave Cobbley recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 174eb8dc403SDave Cobbley if recipename in workspace: 175eb8dc403SDave Cobbley raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 176eb8dc403SDave Cobbley recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 177eb8dc403SDave Cobbley bb.utils.mkdirhier(recipedir) 178eb8dc403SDave Cobbley recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 179eb8dc403SDave Cobbley appendfile = recipe_to_append(recipefile, config) 180eb8dc403SDave Cobbley if os.path.exists(appendfile): 181eb8dc403SDave Cobbley # This shouldn't be possible, but just in case 182eb8dc403SDave Cobbley raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 183eb8dc403SDave Cobbley if os.path.exists(recipefile): 184eb8dc403SDave Cobbley raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 185eb8dc403SDave Cobbley if tmpsrcdir: 186eb8dc403SDave Cobbley srctree = os.path.join(srctreeparent, recipename) 187eb8dc403SDave Cobbley if os.path.exists(tmpsrcdir): 188eb8dc403SDave Cobbley if os.path.exists(srctree): 189eb8dc403SDave Cobbley if os.path.isdir(srctree): 190eb8dc403SDave Cobbley try: 191eb8dc403SDave Cobbley os.rmdir(srctree) 192eb8dc403SDave Cobbley except OSError as e: 193eb8dc403SDave Cobbley if e.errno == errno.ENOTEMPTY: 194eb8dc403SDave Cobbley raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 195eb8dc403SDave Cobbley else: 196eb8dc403SDave Cobbley raise 197eb8dc403SDave Cobbley else: 198eb8dc403SDave Cobbley raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 199eb8dc403SDave Cobbley logger.info('Using default source tree path %s' % srctree) 200eb8dc403SDave Cobbley shutil.move(tmpsrcdir, srctree) 201eb8dc403SDave Cobbley else: 202eb8dc403SDave Cobbley raise DevtoolError('Couldn\'t find source tree created by recipetool') 203eb8dc403SDave Cobbley bb.utils.mkdirhier(recipedir) 204eb8dc403SDave Cobbley shutil.move(recipes[0], recipefile) 205eb8dc403SDave Cobbley # Move any additional files created by recipetool 206eb8dc403SDave Cobbley for fn in os.listdir(tempdir): 207eb8dc403SDave Cobbley shutil.move(os.path.join(tempdir, fn), recipedir) 208eb8dc403SDave Cobbley else: 209eb8dc403SDave Cobbley raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 210eb8dc403SDave Cobbley attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 211eb8dc403SDave Cobbley if os.path.exists(attic_recipe): 2121a4b7ee2SBrad Bishop logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 213eb8dc403SDave Cobbley finally: 214eb8dc403SDave Cobbley if tmpsrcdir and os.path.exists(tmpsrcdir): 215eb8dc403SDave Cobbley shutil.rmtree(tmpsrcdir) 216eb8dc403SDave Cobbley shutil.rmtree(tempdir) 217eb8dc403SDave Cobbley 218eb8dc403SDave Cobbley for fn in os.listdir(recipedir): 219eb8dc403SDave Cobbley _add_md5(config, recipename, os.path.join(recipedir, fn)) 220eb8dc403SDave Cobbley 221eb8dc403SDave Cobbley tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 222eb8dc403SDave Cobbley try: 223eb8dc403SDave Cobbley try: 224eb8dc403SDave Cobbley rd = tinfoil.parse_recipe_file(recipefile, False) 225eb8dc403SDave Cobbley except Exception as e: 226eb8dc403SDave Cobbley logger.error(str(e)) 227eb8dc403SDave Cobbley rd = None 228eb8dc403SDave Cobbley if not rd: 229eb8dc403SDave Cobbley # Parsing failed. We just created this recipe and we shouldn't 230eb8dc403SDave Cobbley # leave it in the workdir or it'll prevent bitbake from starting 231eb8dc403SDave Cobbley movefn = '%s.parsefailed' % recipefile 232eb8dc403SDave Cobbley logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 233eb8dc403SDave Cobbley shutil.move(recipefile, movefn) 234eb8dc403SDave Cobbley return 1 235eb8dc403SDave Cobbley 236eb8dc403SDave Cobbley if args.fetchuri and not args.no_git: 237eb8dc403SDave Cobbley setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 238eb8dc403SDave Cobbley 239da295319SPatrick Williams initial_rev = {} 240eb8dc403SDave Cobbley if os.path.exists(os.path.join(srctree, '.git')): 241eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 242da295319SPatrick Williams initial_rev["."] = stdout.rstrip() 243da295319SPatrick Williams (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree) 244da295319SPatrick Williams for line in stdout.splitlines(): 245da295319SPatrick Williams (rev, submodule) = line.split() 246da295319SPatrick Williams initial_rev[os.path.relpath(submodule, srctree)] = rev 247eb8dc403SDave Cobbley 248eb8dc403SDave Cobbley if args.src_subdir: 249eb8dc403SDave Cobbley srctree = os.path.join(srctree, args.src_subdir) 250eb8dc403SDave Cobbley 251eb8dc403SDave Cobbley bb.utils.mkdirhier(os.path.dirname(appendfile)) 252eb8dc403SDave Cobbley with open(appendfile, 'w') as f: 253eb8dc403SDave Cobbley f.write('inherit externalsrc\n') 254eb8dc403SDave Cobbley f.write('EXTERNALSRC = "%s"\n' % srctree) 255eb8dc403SDave Cobbley 256eb8dc403SDave Cobbley b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 257eb8dc403SDave Cobbley if b_is_s: 258eb8dc403SDave Cobbley f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 259eb8dc403SDave Cobbley if initial_rev: 260da295319SPatrick Williams for key, value in initial_rev.items(): 261da295319SPatrick Williams f.write('\n# initial_rev %s: %s\n' % (key, value)) 262eb8dc403SDave Cobbley 263eb8dc403SDave Cobbley if args.binary: 264213cb269SPatrick Williams f.write('do_install:append() {\n') 265eb8dc403SDave Cobbley f.write(' rm -rf ${D}/.git\n') 266eb8dc403SDave Cobbley f.write(' rm -f ${D}/singletask.lock\n') 267eb8dc403SDave Cobbley f.write('}\n') 268eb8dc403SDave Cobbley 269eb8dc403SDave Cobbley if bb.data.inherits_class('npm', rd): 270213cb269SPatrick Williams f.write('python do_configure:append() {\n') 27182c905dcSAndrew Geissler f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 27282c905dcSAndrew Geissler f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 27382c905dcSAndrew Geissler f.write(' bb.utils.remove(lockfile)\n') 274eb8dc403SDave Cobbley f.write('}\n') 275eb8dc403SDave Cobbley 276eb8dc403SDave Cobbley # Check if the new layer provides recipes whose priorities have been 277eb8dc403SDave Cobbley # overriden by PREFERRED_PROVIDER. 278eb8dc403SDave Cobbley recipe_name = rd.getVar('PN') 279eb8dc403SDave Cobbley provides = rd.getVar('PROVIDES') 280eb8dc403SDave Cobbley # Search every item defined in PROVIDES 281eb8dc403SDave Cobbley for recipe_provided in provides.split(): 282eb8dc403SDave Cobbley preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 283eb8dc403SDave Cobbley current_pprovider = rd.getVar(preferred_provider) 284eb8dc403SDave Cobbley if current_pprovider and current_pprovider != recipe_name: 285eb8dc403SDave Cobbley if args.fixed_setup: 286eb8dc403SDave Cobbley #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 287eb8dc403SDave Cobbley layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 288eb8dc403SDave Cobbley with open(layerconf_file, 'a') as f: 289eb8dc403SDave Cobbley f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 290eb8dc403SDave Cobbley else: 2911a4b7ee2SBrad Bishop logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 292eb8dc403SDave Cobbley break 293eb8dc403SDave Cobbley 294eb8dc403SDave Cobbley _add_md5(config, recipename, appendfile) 295eb8dc403SDave Cobbley 296eb8dc403SDave Cobbley check_prerelease_version(rd.getVar('PV'), 'devtool add') 297eb8dc403SDave Cobbley 298eb8dc403SDave Cobbley logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 299eb8dc403SDave Cobbley 300eb8dc403SDave Cobbley finally: 301eb8dc403SDave Cobbley tinfoil.shutdown() 302eb8dc403SDave Cobbley 303eb8dc403SDave Cobbley return 0 304eb8dc403SDave Cobbley 305eb8dc403SDave Cobbley 306eb8dc403SDave Cobbleydef _check_compatible_recipe(pn, d): 307eb8dc403SDave Cobbley """Check if the recipe is supported by devtool""" 308eb8dc403SDave Cobbley if pn == 'perf': 309eb8dc403SDave Cobbley raise DevtoolError("The perf recipe does not actually check out " 310eb8dc403SDave Cobbley "source and thus cannot be supported by this tool", 311eb8dc403SDave Cobbley 4) 312eb8dc403SDave Cobbley 313eb8dc403SDave Cobbley if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 314eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 315eb8dc403SDave Cobbley 316eb8dc403SDave Cobbley if bb.data.inherits_class('image', d): 317eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is an image, and therefore is not " 318eb8dc403SDave Cobbley "supported by this tool" % pn, 4) 319eb8dc403SDave Cobbley 320eb8dc403SDave Cobbley if bb.data.inherits_class('populate_sdk', d): 321eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is an SDK, and therefore is not " 322eb8dc403SDave Cobbley "supported by this tool" % pn, 4) 323eb8dc403SDave Cobbley 324eb8dc403SDave Cobbley if bb.data.inherits_class('packagegroup', d): 325eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 326eb8dc403SDave Cobbley "not supported by this tool" % pn, 4) 327eb8dc403SDave Cobbley 328eb8dc403SDave Cobbley if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 329eb8dc403SDave Cobbley # Not an incompatibility error per se, so we don't pass the error code 330eb8dc403SDave Cobbley raise DevtoolError("externalsrc is currently enabled for the %s " 331eb8dc403SDave Cobbley "recipe. This prevents the normal do_patch task " 332eb8dc403SDave Cobbley "from working. You will need to disable this " 333eb8dc403SDave Cobbley "first." % pn) 334eb8dc403SDave Cobbley 335eb8dc403SDave Cobbleydef _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 336eb8dc403SDave Cobbley """Common function for copying a file to the dry run output directory""" 337eb8dc403SDave Cobbley relpath = os.path.relpath(dst, base_outdir) 338eb8dc403SDave Cobbley if relpath.startswith('..'): 339eb8dc403SDave Cobbley raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 340eb8dc403SDave Cobbley dst = os.path.join(dry_run_outdir, relpath) 341eb8dc403SDave Cobbley dst_d = os.path.dirname(dst) 342eb8dc403SDave Cobbley if dst_d: 343eb8dc403SDave Cobbley bb.utils.mkdirhier(dst_d) 344eb8dc403SDave Cobbley # Don't overwrite existing files, otherwise in the case of an upgrade 345eb8dc403SDave Cobbley # the dry-run written out recipe will be overwritten with an unmodified 346eb8dc403SDave Cobbley # version 347eb8dc403SDave Cobbley if not os.path.exists(dst): 348eb8dc403SDave Cobbley shutil.copy(src, dst) 349eb8dc403SDave Cobbley 350eb8dc403SDave Cobbleydef _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 351eb8dc403SDave Cobbley """Move a file. Creates all the directory components of destination path.""" 352eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 353eb8dc403SDave Cobbley logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 354eb8dc403SDave Cobbley if dry_run_outdir: 355eb8dc403SDave Cobbley # We want to copy here, not move 356eb8dc403SDave Cobbley _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 357eb8dc403SDave Cobbley else: 358eb8dc403SDave Cobbley dst_d = os.path.dirname(dst) 359eb8dc403SDave Cobbley if dst_d: 360eb8dc403SDave Cobbley bb.utils.mkdirhier(dst_d) 361eb8dc403SDave Cobbley shutil.move(src, dst) 362eb8dc403SDave Cobbley 36378b72798SAndrew Geisslerdef _copy_file(src, dst, dry_run_outdir=None, base_outdir=None): 364eb8dc403SDave Cobbley """Copy a file. Creates all the directory components of destination path.""" 365eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 366eb8dc403SDave Cobbley logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 367eb8dc403SDave Cobbley if dry_run_outdir: 368eb8dc403SDave Cobbley _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 369eb8dc403SDave Cobbley else: 370eb8dc403SDave Cobbley dst_d = os.path.dirname(dst) 371eb8dc403SDave Cobbley if dst_d: 372eb8dc403SDave Cobbley bb.utils.mkdirhier(dst_d) 373eb8dc403SDave Cobbley shutil.copy(src, dst) 374eb8dc403SDave Cobbley 375eb8dc403SDave Cobbleydef _git_ls_tree(repodir, treeish='HEAD', recursive=False): 376eb8dc403SDave Cobbley """List contents of a git treeish""" 377eb8dc403SDave Cobbley import bb 378eb8dc403SDave Cobbley cmd = ['git', 'ls-tree', '-z', treeish] 379eb8dc403SDave Cobbley if recursive: 380eb8dc403SDave Cobbley cmd.append('-r') 381eb8dc403SDave Cobbley out, _ = bb.process.run(cmd, cwd=repodir) 382eb8dc403SDave Cobbley ret = {} 383eb8dc403SDave Cobbley if out: 384eb8dc403SDave Cobbley for line in out.split('\0'): 385eb8dc403SDave Cobbley if line: 386eb8dc403SDave Cobbley split = line.split(None, 4) 387eb8dc403SDave Cobbley ret[split[3]] = split[0:3] 388eb8dc403SDave Cobbley return ret 389eb8dc403SDave Cobbley 390eb8dc403SDave Cobbleydef _git_exclude_path(srctree, path): 391eb8dc403SDave Cobbley """Return pathspec (list of paths) that excludes certain path""" 392eb8dc403SDave Cobbley # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 393eb8dc403SDave Cobbley # we don't catch files that are deleted, for example. A more reliable way 394eb8dc403SDave Cobbley # to implement this would be to use "negative pathspecs" which were 395eb8dc403SDave Cobbley # introduced in Git v1.9.0. Revisit this when/if the required Git version 396eb8dc403SDave Cobbley # becomes greater than that. 397eb8dc403SDave Cobbley path = os.path.normpath(path) 398eb8dc403SDave Cobbley recurse = True if len(path.split(os.path.sep)) > 1 else False 399eb8dc403SDave Cobbley git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 400eb8dc403SDave Cobbley if path in git_files: 401eb8dc403SDave Cobbley git_files.remove(path) 402eb8dc403SDave Cobbley return git_files 403eb8dc403SDave Cobbley else: 404eb8dc403SDave Cobbley return ['.'] 405eb8dc403SDave Cobbley 406eb8dc403SDave Cobbleydef _ls_tree(directory): 407eb8dc403SDave Cobbley """Recursive listing of files in a directory""" 408eb8dc403SDave Cobbley ret = [] 409eb8dc403SDave Cobbley for root, dirs, files in os.walk(directory): 410eb8dc403SDave Cobbley ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 411eb8dc403SDave Cobbley fname in files]) 412eb8dc403SDave Cobbley return ret 413eb8dc403SDave Cobbley 414eb8dc403SDave Cobbley 415eb8dc403SDave Cobbleydef extract(args, config, basepath, workspace): 416eb8dc403SDave Cobbley """Entry point for the devtool 'extract' subcommand""" 417eb8dc403SDave Cobbley import bb 418eb8dc403SDave Cobbley 419eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 420eb8dc403SDave Cobbley if not tinfoil: 421eb8dc403SDave Cobbley # Error already shown 422eb8dc403SDave Cobbley return 1 423eb8dc403SDave Cobbley try: 424eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 425eb8dc403SDave Cobbley if not rd: 426eb8dc403SDave Cobbley return 1 427eb8dc403SDave Cobbley 428eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 429eb8dc403SDave Cobbley initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 430eb8dc403SDave Cobbley logger.info('Source tree extracted to %s' % srctree) 431eb8dc403SDave Cobbley 432eb8dc403SDave Cobbley if initial_rev: 433eb8dc403SDave Cobbley return 0 434eb8dc403SDave Cobbley else: 435eb8dc403SDave Cobbley return 1 436eb8dc403SDave Cobbley finally: 437eb8dc403SDave Cobbley tinfoil.shutdown() 438eb8dc403SDave Cobbley 439eb8dc403SDave Cobbleydef sync(args, config, basepath, workspace): 440eb8dc403SDave Cobbley """Entry point for the devtool 'sync' subcommand""" 441eb8dc403SDave Cobbley import bb 442eb8dc403SDave Cobbley 443eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 444eb8dc403SDave Cobbley if not tinfoil: 445eb8dc403SDave Cobbley # Error already shown 446eb8dc403SDave Cobbley return 1 447eb8dc403SDave Cobbley try: 448eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 449eb8dc403SDave Cobbley if not rd: 450eb8dc403SDave Cobbley return 1 451eb8dc403SDave Cobbley 452eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 453eb8dc403SDave Cobbley initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 454eb8dc403SDave Cobbley logger.info('Source tree %s synchronized' % srctree) 455eb8dc403SDave Cobbley 456eb8dc403SDave Cobbley if initial_rev: 457eb8dc403SDave Cobbley return 0 458eb8dc403SDave Cobbley else: 459eb8dc403SDave Cobbley return 1 460eb8dc403SDave Cobbley finally: 461eb8dc403SDave Cobbley tinfoil.shutdown() 462eb8dc403SDave Cobbley 46396ff1984SBrad Bishopdef symlink_oelocal_files_srctree(rd,srctree): 46496ff1984SBrad Bishop import oe.patch 46596ff1984SBrad Bishop if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 46696ff1984SBrad Bishop # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 46796ff1984SBrad Bishop # (otherwise the recipe won't build as expected) 46896ff1984SBrad Bishop local_files_dir = os.path.join(srctree, 'oe-local-files') 46996ff1984SBrad Bishop addfiles = [] 47096ff1984SBrad Bishop for root, _, files in os.walk(local_files_dir): 47196ff1984SBrad Bishop relpth = os.path.relpath(root, local_files_dir) 47296ff1984SBrad Bishop if relpth != '.': 47396ff1984SBrad Bishop bb.utils.mkdirhier(os.path.join(srctree, relpth)) 47496ff1984SBrad Bishop for fn in files: 47596ff1984SBrad Bishop if fn == '.gitignore': 47696ff1984SBrad Bishop continue 47796ff1984SBrad Bishop destpth = os.path.join(srctree, relpth, fn) 47896ff1984SBrad Bishop if os.path.exists(destpth): 47996ff1984SBrad Bishop os.unlink(destpth) 480d1e89497SAndrew Geissler if relpth != '.': 481d1e89497SAndrew Geissler back_relpth = os.path.relpath(local_files_dir, root) 482d1e89497SAndrew Geissler os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) 483d1e89497SAndrew Geissler else: 48496ff1984SBrad Bishop os.symlink('oe-local-files/%s' % fn, destpth) 48596ff1984SBrad Bishop addfiles.append(os.path.join(relpth, fn)) 48696ff1984SBrad Bishop if addfiles: 48796ff1984SBrad Bishop bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 48896ff1984SBrad Bishop useroptions = [] 48996ff1984SBrad Bishop oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) 49079641f25SBrad Bishop bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 49196ff1984SBrad Bishop 492eb8dc403SDave Cobbley 493eb8dc403SDave Cobbleydef _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 494eb8dc403SDave Cobbley """Extract sources of a recipe""" 495eb8dc403SDave Cobbley import oe.recipeutils 496eb8dc403SDave Cobbley import oe.patch 49796ff1984SBrad Bishop import oe.path 498eb8dc403SDave Cobbley 499eb8dc403SDave Cobbley pn = d.getVar('PN') 500eb8dc403SDave Cobbley 501eb8dc403SDave Cobbley _check_compatible_recipe(pn, d) 502eb8dc403SDave Cobbley 503eb8dc403SDave Cobbley if sync: 504eb8dc403SDave Cobbley if not os.path.exists(srctree): 505eb8dc403SDave Cobbley raise DevtoolError("output path %s does not exist" % srctree) 506eb8dc403SDave Cobbley else: 507eb8dc403SDave Cobbley if os.path.exists(srctree): 508eb8dc403SDave Cobbley if not os.path.isdir(srctree): 509eb8dc403SDave Cobbley raise DevtoolError("output path %s exists and is not a directory" % 510eb8dc403SDave Cobbley srctree) 511eb8dc403SDave Cobbley elif os.listdir(srctree): 512eb8dc403SDave Cobbley raise DevtoolError("output path %s already exists and is " 513eb8dc403SDave Cobbley "non-empty" % srctree) 514eb8dc403SDave Cobbley 515eb8dc403SDave Cobbley if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 516eb8dc403SDave Cobbley raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 517eb8dc403SDave Cobbley "extract source" % pn, 4) 518eb8dc403SDave Cobbley 519eb8dc403SDave Cobbley if not sync: 520eb8dc403SDave Cobbley # Prepare for shutil.move later on 521eb8dc403SDave Cobbley bb.utils.mkdirhier(srctree) 522eb8dc403SDave Cobbley os.rmdir(srctree) 523eb8dc403SDave Cobbley 524eb8dc403SDave Cobbley extra_overrides = [] 525eb8dc403SDave Cobbley if not no_overrides: 526eb8dc403SDave Cobbley history = d.varhistory.variable('SRC_URI') 527eb8dc403SDave Cobbley for event in history: 528eb8dc403SDave Cobbley if not 'flag' in event: 529213cb269SPatrick Williams if event['op'].startswith((':append[', ':prepend[')): 530615f2f11SAndrew Geissler override = event['op'].split('[')[1].split(']')[0] 531615f2f11SAndrew Geissler if not override.startswith('pn-'): 532615f2f11SAndrew Geissler extra_overrides.append(override) 53399467dabSAndrew Geissler # We want to remove duplicate overrides. If a recipe had multiple 53499467dabSAndrew Geissler # SRC_URI_override += values it would cause mulitple instances of 53599467dabSAndrew Geissler # overrides. This doesn't play nicely with things like creating a 53699467dabSAndrew Geissler # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 53799467dabSAndrew Geissler extra_overrides = list(set(extra_overrides)) 538eb8dc403SDave Cobbley if extra_overrides: 539eb8dc403SDave Cobbley logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 540eb8dc403SDave Cobbley 541eb8dc403SDave Cobbley initial_rev = None 542eb8dc403SDave Cobbley 543eb8dc403SDave Cobbley recipefile = d.getVar('FILE') 544eb8dc403SDave Cobbley appendfile = recipe_to_append(recipefile, config) 545eb8dc403SDave Cobbley is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 546eb8dc403SDave Cobbley 547eb8dc403SDave Cobbley # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 548eb8dc403SDave Cobbley # directory so that: 549eb8dc403SDave Cobbley # (a) we pick up all files that get unpacked to the WORKDIR, and 550eb8dc403SDave Cobbley # (b) we don't disturb the existing build 551eb8dc403SDave Cobbley # However, with recipe-specific sysroots the sysroots for the recipe 552eb8dc403SDave Cobbley # will be prepared under WORKDIR, and if we used the system temporary 553eb8dc403SDave Cobbley # directory (i.e. usually /tmp) as used by mkdtemp by default, then 554eb8dc403SDave Cobbley # our attempts to hardlink files into the recipe-specific sysroots 555eb8dc403SDave Cobbley # will fail on systems where /tmp is a different filesystem, and it 556eb8dc403SDave Cobbley # would have to fall back to copying the files which is a waste of 557eb8dc403SDave Cobbley # time. Put the temp directory under the WORKDIR to prevent that from 558eb8dc403SDave Cobbley # being a problem. 559eb8dc403SDave Cobbley tempbasedir = d.getVar('WORKDIR') 560eb8dc403SDave Cobbley bb.utils.mkdirhier(tempbasedir) 561eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 562eb8dc403SDave Cobbley try: 563eb8dc403SDave Cobbley tinfoil.logger.setLevel(logging.WARNING) 564eb8dc403SDave Cobbley 565eb8dc403SDave Cobbley # FIXME this results in a cache reload under control of tinfoil, which is fine 566eb8dc403SDave Cobbley # except we don't get the knotty progress bar 567eb8dc403SDave Cobbley 568eb8dc403SDave Cobbley if os.path.exists(appendfile): 569eb8dc403SDave Cobbley appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 570eb8dc403SDave Cobbley shutil.copyfile(appendfile, appendbackup) 571eb8dc403SDave Cobbley else: 572eb8dc403SDave Cobbley appendbackup = None 573eb8dc403SDave Cobbley bb.utils.mkdirhier(os.path.dirname(appendfile)) 574eb8dc403SDave Cobbley logger.debug('writing append file %s' % appendfile) 575eb8dc403SDave Cobbley with open(appendfile, 'a') as f: 576eb8dc403SDave Cobbley f.write('###--- _extract_source\n') 5772a25492cSPatrick Williams f.write('deltask do_recipe_qa\n') 5782a25492cSPatrick Williams f.write('deltask do_recipe_qa_setscene\n') 5796aa7eec5SAndrew Geissler f.write('ERROR_QA:remove = "patch-fuzz"\n') 580eb8dc403SDave Cobbley f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 581eb8dc403SDave Cobbley f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 582eb8dc403SDave Cobbley if not is_kernel_yocto: 583eb8dc403SDave Cobbley f.write('PATCHTOOL = "git"\n') 584eb8dc403SDave Cobbley f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 585eb8dc403SDave Cobbley if extra_overrides: 586eb8dc403SDave Cobbley f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 587eb8dc403SDave Cobbley f.write('inherit devtool-source\n') 588eb8dc403SDave Cobbley f.write('###--- _extract_source\n') 589eb8dc403SDave Cobbley 590eb8dc403SDave Cobbley update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 591eb8dc403SDave Cobbley 592eb8dc403SDave Cobbley sstate_manifests = d.getVar('SSTATE_MANIFESTS') 593eb8dc403SDave Cobbley bb.utils.mkdirhier(sstate_manifests) 594eb8dc403SDave Cobbley preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 595eb8dc403SDave Cobbley with open(preservestampfile, 'w') as f: 596eb8dc403SDave Cobbley f.write(d.getVar('STAMP')) 597220dafdbSAndrew Geissler tinfoil.modified_files() 598eb8dc403SDave Cobbley try: 59996ff1984SBrad Bishop if is_kernel_yocto: 600eb8dc403SDave Cobbley # We need to generate the kernel config 601eb8dc403SDave Cobbley task = 'do_configure' 602eb8dc403SDave Cobbley else: 603eb8dc403SDave Cobbley task = 'do_patch' 604eb8dc403SDave Cobbley 605d1e89497SAndrew Geissler if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []): 606d1e89497SAndrew Geissler logger.info('The %s recipe has %s disabled. Running only ' 607d1e89497SAndrew Geissler 'do_configure task dependencies' % (pn, task)) 608d1e89497SAndrew Geissler 609d1e89497SAndrew Geissler if 'depends' in d.getVarFlags('do_configure', False): 610d1e89497SAndrew Geissler pn = d.getVarFlags('do_configure', False)['depends'] 611d1e89497SAndrew Geissler pn = pn.replace('${PV}', d.getVar('PV')) 612d1e89497SAndrew Geissler pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP')) 613d1e89497SAndrew Geissler task = None 614d1e89497SAndrew Geissler 615eb8dc403SDave Cobbley # Run the fetch + unpack tasks 616eb8dc403SDave Cobbley res = tinfoil.build_targets(pn, 617eb8dc403SDave Cobbley task, 618eb8dc403SDave Cobbley handle_events=True) 619eb8dc403SDave Cobbley finally: 620eb8dc403SDave Cobbley if os.path.exists(preservestampfile): 621eb8dc403SDave Cobbley os.remove(preservestampfile) 622eb8dc403SDave Cobbley 623eb8dc403SDave Cobbley if not res: 624eb8dc403SDave Cobbley raise DevtoolError('Extracting source for %s failed' % pn) 625eb8dc403SDave Cobbley 626d1e89497SAndrew Geissler if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])): 627d1e89497SAndrew Geissler workshareddir = d.getVar('S') 628d1e89497SAndrew Geissler if os.path.islink(srctree): 629d1e89497SAndrew Geissler os.unlink(srctree) 630d1e89497SAndrew Geissler 631d1e89497SAndrew Geissler os.symlink(workshareddir, srctree) 632d1e89497SAndrew Geissler 633d1e89497SAndrew Geissler # The initial_rev file is created in devtool_post_unpack function that will not be executed if 634d1e89497SAndrew Geissler # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful 635d1e89497SAndrew Geissler return True, True 636d1e89497SAndrew Geissler 637eb8dc403SDave Cobbley try: 638eb8dc403SDave Cobbley with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 639eb8dc403SDave Cobbley initial_rev = f.read() 640eb8dc403SDave Cobbley 641eb8dc403SDave Cobbley with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 642eb8dc403SDave Cobbley srcsubdir = f.read() 643eb8dc403SDave Cobbley except FileNotFoundError as e: 644eb8dc403SDave Cobbley raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 645eb8dc403SDave Cobbley srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 646eb8dc403SDave Cobbley 64796ff1984SBrad Bishop # Check if work-shared is empty, if yes 64896ff1984SBrad Bishop # find source and copy to work-shared 64996ff1984SBrad Bishop if is_kernel_yocto: 65096ff1984SBrad Bishop workshareddir = d.getVar('STAGING_KERNEL_DIR') 65196ff1984SBrad Bishop staging_kerVer = get_staging_kver(workshareddir) 65296ff1984SBrad Bishop kernelVersion = d.getVar('LINUX_VERSION') 65396ff1984SBrad Bishop 65496ff1984SBrad Bishop # handle dangling symbolic link in work-shared: 65596ff1984SBrad Bishop if os.path.islink(workshareddir): 65696ff1984SBrad Bishop os.unlink(workshareddir) 65796ff1984SBrad Bishop 65896ff1984SBrad Bishop if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 65996ff1984SBrad Bishop shutil.rmtree(workshareddir) 66096ff1984SBrad Bishop oe.path.copyhardlinktree(srcsubdir,workshareddir) 66196ff1984SBrad Bishop elif not os.path.exists(workshareddir): 66296ff1984SBrad Bishop oe.path.copyhardlinktree(srcsubdir,workshareddir) 66396ff1984SBrad Bishop 664eb8dc403SDave Cobbley tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 665eb8dc403SDave Cobbley srctree_localdir = os.path.join(srctree, 'oe-local-files') 666eb8dc403SDave Cobbley 667eb8dc403SDave Cobbley if sync: 668eb8dc403SDave Cobbley bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 669eb8dc403SDave Cobbley 670eb8dc403SDave Cobbley # Move oe-local-files directory to srctree 671eb8dc403SDave Cobbley # As the oe-local-files is not part of the constructed git tree, 672eb8dc403SDave Cobbley # remove them directly during the synchrounizating might surprise 673eb8dc403SDave Cobbley # the users. Instead, we move it to oe-local-files.bak and remind 674eb8dc403SDave Cobbley # user in the log message. 675eb8dc403SDave Cobbley if os.path.exists(srctree_localdir + '.bak'): 676eb8dc403SDave Cobbley shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 677eb8dc403SDave Cobbley 678eb8dc403SDave Cobbley if os.path.exists(srctree_localdir): 679eb8dc403SDave Cobbley logger.info('Backing up current local file directory %s' % srctree_localdir) 680eb8dc403SDave Cobbley shutil.move(srctree_localdir, srctree_localdir + '.bak') 681eb8dc403SDave Cobbley 682eb8dc403SDave Cobbley if os.path.exists(tempdir_localdir): 683eb8dc403SDave Cobbley logger.info('Syncing local source files to srctree...') 684eb8dc403SDave Cobbley shutil.copytree(tempdir_localdir, srctree_localdir) 685eb8dc403SDave Cobbley else: 686eb8dc403SDave Cobbley # Move oe-local-files directory to srctree 687eb8dc403SDave Cobbley if os.path.exists(tempdir_localdir): 688eb8dc403SDave Cobbley logger.info('Adding local source files to srctree...') 689eb8dc403SDave Cobbley shutil.move(tempdir_localdir, srcsubdir) 690eb8dc403SDave Cobbley 691eb8dc403SDave Cobbley shutil.move(srcsubdir, srctree) 69296ff1984SBrad Bishop symlink_oelocal_files_srctree(d,srctree) 693eb8dc403SDave Cobbley 694eb8dc403SDave Cobbley if is_kernel_yocto: 695eb8dc403SDave Cobbley logger.info('Copying kernel config to srctree') 696eb8dc403SDave Cobbley shutil.copy2(os.path.join(tempdir, '.config'), srctree) 697eb8dc403SDave Cobbley 698eb8dc403SDave Cobbley finally: 699eb8dc403SDave Cobbley if appendbackup: 700eb8dc403SDave Cobbley shutil.copyfile(appendbackup, appendfile) 701eb8dc403SDave Cobbley elif os.path.exists(appendfile): 702eb8dc403SDave Cobbley os.remove(appendfile) 703eb8dc403SDave Cobbley if keep_temp: 704eb8dc403SDave Cobbley logger.info('Preserving temporary directory %s' % tempdir) 705eb8dc403SDave Cobbley else: 706eb8dc403SDave Cobbley shutil.rmtree(tempdir) 707eb8dc403SDave Cobbley return initial_rev, srcsubdir_rel 708eb8dc403SDave Cobbley 709eb8dc403SDave Cobbleydef _add_md5(config, recipename, filename): 710eb8dc403SDave Cobbley """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 711eb8dc403SDave Cobbley import bb.utils 712eb8dc403SDave Cobbley 713eb8dc403SDave Cobbley def addfile(fn): 714eb8dc403SDave Cobbley md5 = bb.utils.md5_file(fn) 715eb8dc403SDave Cobbley with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 716eb8dc403SDave Cobbley md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 717eb8dc403SDave Cobbley f.seek(0, os.SEEK_SET) 718eb8dc403SDave Cobbley if not md5_str in f.read(): 719eb8dc403SDave Cobbley f.write(md5_str) 720eb8dc403SDave Cobbley 721eb8dc403SDave Cobbley if os.path.isdir(filename): 722eb8dc403SDave Cobbley for root, _, files in os.walk(filename): 723eb8dc403SDave Cobbley for f in files: 724eb8dc403SDave Cobbley addfile(os.path.join(root, f)) 725eb8dc403SDave Cobbley else: 726eb8dc403SDave Cobbley addfile(filename) 727eb8dc403SDave Cobbley 728eb8dc403SDave Cobbleydef _check_preserve(config, recipename): 729eb8dc403SDave Cobbley """Check if a file was manually changed and needs to be saved in 'attic' 730eb8dc403SDave Cobbley directory""" 731eb8dc403SDave Cobbley import bb.utils 732eb8dc403SDave Cobbley origfile = os.path.join(config.workspace_path, '.devtool_md5') 733eb8dc403SDave Cobbley newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 734eb8dc403SDave Cobbley preservepath = os.path.join(config.workspace_path, 'attic', recipename) 735eb8dc403SDave Cobbley with open(origfile, 'r') as f: 736eb8dc403SDave Cobbley with open(newfile, 'w') as tf: 737eb8dc403SDave Cobbley for line in f.readlines(): 738eb8dc403SDave Cobbley splitline = line.rstrip().split('|') 739eb8dc403SDave Cobbley if splitline[0] == recipename: 740eb8dc403SDave Cobbley removefile = os.path.join(config.workspace_path, splitline[1]) 741eb8dc403SDave Cobbley try: 742eb8dc403SDave Cobbley md5 = bb.utils.md5_file(removefile) 743eb8dc403SDave Cobbley except IOError as err: 744eb8dc403SDave Cobbley if err.errno == 2: 745eb8dc403SDave Cobbley # File no longer exists, skip it 746eb8dc403SDave Cobbley continue 747eb8dc403SDave Cobbley else: 748eb8dc403SDave Cobbley raise 749eb8dc403SDave Cobbley if splitline[2] != md5: 750eb8dc403SDave Cobbley bb.utils.mkdirhier(preservepath) 751eb8dc403SDave Cobbley preservefile = os.path.basename(removefile) 7521a4b7ee2SBrad Bishop logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 753eb8dc403SDave Cobbley shutil.move(removefile, os.path.join(preservepath, preservefile)) 754eb8dc403SDave Cobbley else: 755eb8dc403SDave Cobbley os.remove(removefile) 756eb8dc403SDave Cobbley else: 757eb8dc403SDave Cobbley tf.write(line) 758c926e17cSAndrew Geissler bb.utils.rename(newfile, origfile) 759eb8dc403SDave Cobbley 76096ff1984SBrad Bishopdef get_staging_kver(srcdir): 76196ff1984SBrad Bishop # Kernel version from work-shared 76296ff1984SBrad Bishop kerver = [] 76396ff1984SBrad Bishop staging_kerVer="" 76496ff1984SBrad Bishop if os.path.exists(srcdir) and os.listdir(srcdir): 76596ff1984SBrad Bishop with open(os.path.join(srcdir,"Makefile")) as f: 76696ff1984SBrad Bishop version = [next(f) for x in range(5)][1:4] 76796ff1984SBrad Bishop for word in version: 76896ff1984SBrad Bishop kerver.append(word.split('= ')[1].split('\n')[0]) 76996ff1984SBrad Bishop staging_kerVer = ".".join(kerver) 77096ff1984SBrad Bishop return staging_kerVer 77196ff1984SBrad Bishop 77296ff1984SBrad Bishopdef get_staging_kbranch(srcdir): 77396ff1984SBrad Bishop staging_kbranch = "" 77496ff1984SBrad Bishop if os.path.exists(srcdir) and os.listdir(srcdir): 775705982a5SPatrick Williams (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir) 77696ff1984SBrad Bishop staging_kbranch = "".join(branch.split('\n')[0]) 77796ff1984SBrad Bishop return staging_kbranch 77896ff1984SBrad Bishop 779517393d9SAndrew Geisslerdef get_real_srctree(srctree, s, workdir): 780517393d9SAndrew Geissler # Check that recipe isn't using a shared workdir 781517393d9SAndrew Geissler s = os.path.abspath(s) 782517393d9SAndrew Geissler workdir = os.path.abspath(workdir) 783517393d9SAndrew Geissler if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 784517393d9SAndrew Geissler # Handle if S is set to a subdirectory of the source 785517393d9SAndrew Geissler srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 786517393d9SAndrew Geissler srctree = os.path.join(srctree, srcsubdir) 787517393d9SAndrew Geissler return srctree 788517393d9SAndrew Geissler 789eb8dc403SDave Cobbleydef modify(args, config, basepath, workspace): 790eb8dc403SDave Cobbley """Entry point for the devtool 'modify' subcommand""" 791eb8dc403SDave Cobbley import bb 792eb8dc403SDave Cobbley import oe.recipeutils 793eb8dc403SDave Cobbley import oe.patch 79496ff1984SBrad Bishop import oe.path 795eb8dc403SDave Cobbley 796eb8dc403SDave Cobbley if args.recipename in workspace: 797eb8dc403SDave Cobbley raise DevtoolError("recipe %s is already in your workspace" % 798eb8dc403SDave Cobbley args.recipename) 799eb8dc403SDave Cobbley 800eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 801eb8dc403SDave Cobbley try: 802eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 803eb8dc403SDave Cobbley if not rd: 804eb8dc403SDave Cobbley return 1 805eb8dc403SDave Cobbley 806eb8dc403SDave Cobbley pn = rd.getVar('PN') 807eb8dc403SDave Cobbley if pn != args.recipename: 808eb8dc403SDave Cobbley logger.info('Mapping %s to %s' % (args.recipename, pn)) 809eb8dc403SDave Cobbley if pn in workspace: 810eb8dc403SDave Cobbley raise DevtoolError("recipe %s is already in your workspace" % 811eb8dc403SDave Cobbley pn) 812eb8dc403SDave Cobbley 813eb8dc403SDave Cobbley if args.srctree: 814eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 815eb8dc403SDave Cobbley else: 816eb8dc403SDave Cobbley srctree = get_default_srctree(config, pn) 817eb8dc403SDave Cobbley 818eb8dc403SDave Cobbley if args.no_extract and not os.path.isdir(srctree): 819eb8dc403SDave Cobbley raise DevtoolError("--no-extract specified and source path %s does " 820eb8dc403SDave Cobbley "not exist or is not a directory" % 821eb8dc403SDave Cobbley srctree) 822eb8dc403SDave Cobbley 823eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 824eb8dc403SDave Cobbley appendfile = recipe_to_append(recipefile, config, args.wildcard) 825eb8dc403SDave Cobbley if os.path.exists(appendfile): 826eb8dc403SDave Cobbley raise DevtoolError("Another variant of recipe %s is already in your " 827eb8dc403SDave Cobbley "workspace (only one variant of a recipe can " 828eb8dc403SDave Cobbley "currently be worked on at once)" 829eb8dc403SDave Cobbley % pn) 830eb8dc403SDave Cobbley 831eb8dc403SDave Cobbley _check_compatible_recipe(pn, rd) 832eb8dc403SDave Cobbley 833da295319SPatrick Williams initial_revs = {} 834da295319SPatrick Williams commits = {} 835eb8dc403SDave Cobbley check_commits = False 83696ff1984SBrad Bishop 83796ff1984SBrad Bishop if bb.data.inherits_class('kernel-yocto', rd): 83896ff1984SBrad Bishop # Current set kernel version 83996ff1984SBrad Bishop kernelVersion = rd.getVar('LINUX_VERSION') 84096ff1984SBrad Bishop srcdir = rd.getVar('STAGING_KERNEL_DIR') 84196ff1984SBrad Bishop kbranch = rd.getVar('KBRANCH') 84296ff1984SBrad Bishop 84396ff1984SBrad Bishop staging_kerVer = get_staging_kver(srcdir) 84496ff1984SBrad Bishop staging_kbranch = get_staging_kbranch(srcdir) 84596ff1984SBrad Bishop if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 84696ff1984SBrad Bishop oe.path.copyhardlinktree(srcdir,srctree) 84796ff1984SBrad Bishop workdir = rd.getVar('WORKDIR') 84896ff1984SBrad Bishop srcsubdir = rd.getVar('S') 84996ff1984SBrad Bishop localfilesdir = os.path.join(srctree,'oe-local-files') 85096ff1984SBrad Bishop # Move local source files into separate subdir 85196ff1984SBrad Bishop recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 85296ff1984SBrad Bishop local_files = oe.recipeutils.get_recipe_local_files(rd) 85396ff1984SBrad Bishop 85496ff1984SBrad Bishop for key in local_files.copy(): 85596ff1984SBrad Bishop if key.endswith('scc'): 85696ff1984SBrad Bishop sccfile = open(local_files[key], 'r') 85796ff1984SBrad Bishop for l in sccfile: 85896ff1984SBrad Bishop line = l.split() 85996ff1984SBrad Bishop if line and line[0] in ('kconf', 'patch'): 86096ff1984SBrad Bishop cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) 86196ff1984SBrad Bishop if not cfg in local_files.values(): 86296ff1984SBrad Bishop local_files[line[-1]] = cfg 86396ff1984SBrad Bishop shutil.copy2(cfg, workdir) 86496ff1984SBrad Bishop sccfile.close() 86596ff1984SBrad Bishop 86696ff1984SBrad Bishop # Ignore local files with subdir={BP} 86796ff1984SBrad Bishop srcabspath = os.path.abspath(srcsubdir) 86896ff1984SBrad Bishop local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] 86996ff1984SBrad Bishop if local_files: 87096ff1984SBrad Bishop for fname in local_files: 87196ff1984SBrad Bishop _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 87296ff1984SBrad Bishop with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 87396ff1984SBrad Bishop f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 87496ff1984SBrad Bishop 87596ff1984SBrad Bishop symlink_oelocal_files_srctree(rd,srctree) 87696ff1984SBrad Bishop 87796ff1984SBrad Bishop task = 'do_configure' 87896ff1984SBrad Bishop res = tinfoil.build_targets(pn, task, handle_events=True) 87996ff1984SBrad Bishop 88096ff1984SBrad Bishop # Copy .config to workspace 88196ff1984SBrad Bishop kconfpath = rd.getVar('B') 88296ff1984SBrad Bishop logger.info('Copying kernel config to workspace') 88396ff1984SBrad Bishop shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 88496ff1984SBrad Bishop 88596ff1984SBrad Bishop # Set this to true, we still need to get initial_rev 88696ff1984SBrad Bishop # by parsing the git repo 88796ff1984SBrad Bishop args.no_extract = True 88896ff1984SBrad Bishop 889eb8dc403SDave Cobbley if not args.no_extract: 890da295319SPatrick Williams initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 891da295319SPatrick Williams if not initial_revs["."]: 892eb8dc403SDave Cobbley return 1 893eb8dc403SDave Cobbley logger.info('Source tree extracted to %s' % srctree) 894da295319SPatrick Williams 895d1e89497SAndrew Geissler if os.path.exists(os.path.join(srctree, '.git')): 896eb8dc403SDave Cobbley # Get list of commits since this revision 897da295319SPatrick Williams (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) 898da295319SPatrick Williams commits["."] = stdout.split() 899eb8dc403SDave Cobbley check_commits = True 900da295319SPatrick Williams (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) 901da295319SPatrick Williams for line in stdout.splitlines(): 902da295319SPatrick Williams (rev, submodule_path) = line.split() 903da295319SPatrick Williams submodule = os.path.relpath(submodule_path, srctree) 904da295319SPatrick Williams initial_revs[submodule] = rev 905da295319SPatrick Williams (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path) 906da295319SPatrick Williams commits[submodule] = stdout.split() 907eb8dc403SDave Cobbley else: 908eb8dc403SDave Cobbley if os.path.exists(os.path.join(srctree, '.git')): 90999467dabSAndrew Geissler # Check if it's a tree previously extracted by us. This is done 91099467dabSAndrew Geissler # by ensuring that devtool-base and args.branch (devtool) exist. 91199467dabSAndrew Geissler # The check_commits logic will cause an exception if either one 91299467dabSAndrew Geissler # of these doesn't exist 913eb8dc403SDave Cobbley try: 914eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 91599467dabSAndrew Geissler bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 916eb8dc403SDave Cobbley except bb.process.ExecutionError: 917eb8dc403SDave Cobbley stdout = '' 918eb8dc403SDave Cobbley if stdout: 919eb8dc403SDave Cobbley check_commits = True 920eb8dc403SDave Cobbley for line in stdout.splitlines(): 921eb8dc403SDave Cobbley if line.startswith('*'): 922eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 923da295319SPatrick Williams initial_revs["."] = stdout.rstrip() 924705982a5SPatrick Williams if "." not in initial_revs: 925eb8dc403SDave Cobbley # Otherwise, just grab the head revision 926eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 927da295319SPatrick Williams initial_revs["."] = stdout.rstrip() 928eb8dc403SDave Cobbley 929eb8dc403SDave Cobbley branch_patches = {} 930eb8dc403SDave Cobbley if check_commits: 931eb8dc403SDave Cobbley # Check if there are override branches 932eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git branch', cwd=srctree) 933eb8dc403SDave Cobbley branches = [] 934eb8dc403SDave Cobbley for line in stdout.rstrip().splitlines(): 935eb8dc403SDave Cobbley branchname = line[2:].rstrip() 936eb8dc403SDave Cobbley if branchname.startswith(override_branch_prefix): 937eb8dc403SDave Cobbley branches.append(branchname) 938eb8dc403SDave Cobbley if branches: 9391a4b7ee2SBrad Bishop logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 940eb8dc403SDave Cobbley branches.insert(0, args.branch) 941eb8dc403SDave Cobbley seen_patches = [] 942eb8dc403SDave Cobbley for branch in branches: 943eb8dc403SDave Cobbley branch_patches[branch] = [] 944eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 945eb8dc403SDave Cobbley for line in stdout.splitlines(): 946eb8dc403SDave Cobbley line = line.strip() 947eb8dc403SDave Cobbley if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 948eb8dc403SDave Cobbley origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 949eb8dc403SDave Cobbley if not origpatch in seen_patches: 950eb8dc403SDave Cobbley seen_patches.append(origpatch) 951eb8dc403SDave Cobbley branch_patches[branch].append(origpatch) 952eb8dc403SDave Cobbley 953eb8dc403SDave Cobbley # Need to grab this here in case the source is within a subdirectory 954eb8dc403SDave Cobbley srctreebase = srctree 955517393d9SAndrew Geissler srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR')) 956eb8dc403SDave Cobbley 957eb8dc403SDave Cobbley bb.utils.mkdirhier(os.path.dirname(appendfile)) 958eb8dc403SDave Cobbley with open(appendfile, 'w') as f: 959*f52e3ddeSPatrick Williams # if not present, add type=git-dependency to the secondary sources 960*f52e3ddeSPatrick Williams # (non local files) so they can be extracted correctly when building a recipe after 961*f52e3ddeSPatrick Williams # doing a devtool modify on it 962*f52e3ddeSPatrick Williams src_uri = rd.getVar('SRC_URI').split() 963*f52e3ddeSPatrick Williams src_uri_append = [] 964*f52e3ddeSPatrick Williams src_uri_remove = [] 965*f52e3ddeSPatrick Williams 966*f52e3ddeSPatrick Williams # Assume first entry is main source extracted in ${S} so skip it 967*f52e3ddeSPatrick Williams src_uri = src_uri[1::] 968*f52e3ddeSPatrick Williams 969*f52e3ddeSPatrick Williams #Add "type=git-dependency" to all non local sources 970*f52e3ddeSPatrick Williams for url in src_uri: 971*f52e3ddeSPatrick Williams if not url.startswith('file://') and not 'type=' in url: 972*f52e3ddeSPatrick Williams src_uri_remove.append(url) 973*f52e3ddeSPatrick Williams src_uri_append.append('%s;type=git-dependency' % url) 974*f52e3ddeSPatrick Williams 975*f52e3ddeSPatrick Williams if src_uri_remove: 976*f52e3ddeSPatrick Williams f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove)) 977*f52e3ddeSPatrick Williams f.write('SRC_URI:append = "%s"\n\n' % ' '.join(src_uri_append)) 978*f52e3ddeSPatrick Williams 979213cb269SPatrick Williams f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n') 980eb8dc403SDave Cobbley # Local files can be modified/tracked in separate subdir under srctree 981eb8dc403SDave Cobbley # Mostly useful for packages with S != WORKDIR 982213cb269SPatrick Williams f.write('FILESPATH:prepend := "%s:"\n' % 983eb8dc403SDave Cobbley os.path.join(srctreebase, 'oe-local-files')) 984eb8dc403SDave Cobbley f.write('# srctreebase: %s\n' % srctreebase) 985eb8dc403SDave Cobbley 986eb8dc403SDave Cobbley f.write('\ninherit externalsrc\n') 987eb8dc403SDave Cobbley f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 988213cb269SPatrick Williams f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree)) 989eb8dc403SDave Cobbley 990eb8dc403SDave Cobbley b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 991eb8dc403SDave Cobbley if b_is_s: 992213cb269SPatrick Williams f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree)) 993eb8dc403SDave Cobbley 994eb8dc403SDave Cobbley if bb.data.inherits_class('kernel', rd): 995eb8dc403SDave Cobbley f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 99695ac1b8dSAndrew Geissler 'do_fetch do_unpack do_kernel_configcheck"\n') 99719323693SBrad Bishop f.write('\ndo_patch[noexec] = "1"\n') 998213cb269SPatrick Williams f.write('\ndo_configure:append() {\n' 999eb8dc403SDave Cobbley ' cp ${B}/.config ${S}/.config.baseline\n' 1000eb8dc403SDave Cobbley ' ln -sfT ${B}/.config ${S}/.config.new\n' 1001eb8dc403SDave Cobbley '}\n') 1002213cb269SPatrick Williams f.write('\ndo_kernel_configme:prepend() {\n' 100395ac1b8dSAndrew Geissler ' if [ -e ${S}/.config ]; then\n' 100495ac1b8dSAndrew Geissler ' mv ${S}/.config ${S}/.config.old\n' 100595ac1b8dSAndrew Geissler ' fi\n' 100695ac1b8dSAndrew Geissler '}\n') 100796ff1984SBrad Bishop if rd.getVarFlag('do_menuconfig','task'): 1008213cb269SPatrick Williams f.write('\ndo_configure:append() {\n' 1009169d7bccSPatrick Williams ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n' 1010520786ccSPatrick Williams ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n' 1011520786ccSPatrick Williams ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n' 101282c905dcSAndrew Geissler ' fi\n' 101396ff1984SBrad Bishop '}\n') 1014da295319SPatrick Williams if initial_revs: 1015da295319SPatrick Williams for name, rev in initial_revs.items(): 1016da295319SPatrick Williams f.write('\n# initial_rev %s: %s\n' % (name, rev)) 1017169d7bccSPatrick Williams if name in commits: 1018da295319SPatrick Williams for commit in commits[name]: 1019da295319SPatrick Williams f.write('# commit %s: %s\n' % (name, commit)) 1020eb8dc403SDave Cobbley if branch_patches: 1021eb8dc403SDave Cobbley for branch in branch_patches: 1022eb8dc403SDave Cobbley if branch == args.branch: 1023eb8dc403SDave Cobbley continue 1024eb8dc403SDave Cobbley f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 1025eb8dc403SDave Cobbley 1026eb8dc403SDave Cobbley update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 1027eb8dc403SDave Cobbley 1028eb8dc403SDave Cobbley _add_md5(config, pn, appendfile) 1029eb8dc403SDave Cobbley 1030eb8dc403SDave Cobbley logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 1031eb8dc403SDave Cobbley 1032eb8dc403SDave Cobbley finally: 1033eb8dc403SDave Cobbley tinfoil.shutdown() 1034eb8dc403SDave Cobbley 1035eb8dc403SDave Cobbley return 0 1036eb8dc403SDave Cobbley 1037eb8dc403SDave Cobbley 1038eb8dc403SDave Cobbleydef rename(args, config, basepath, workspace): 1039eb8dc403SDave Cobbley """Entry point for the devtool 'rename' subcommand""" 1040eb8dc403SDave Cobbley import bb 1041eb8dc403SDave Cobbley import oe.recipeutils 1042eb8dc403SDave Cobbley 1043eb8dc403SDave Cobbley check_workspace_recipe(workspace, args.recipename) 1044eb8dc403SDave Cobbley 1045eb8dc403SDave Cobbley if not (args.newname or args.version): 1046eb8dc403SDave Cobbley raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 1047eb8dc403SDave Cobbley 1048eb8dc403SDave Cobbley recipefile = workspace[args.recipename]['recipefile'] 1049eb8dc403SDave Cobbley if not recipefile: 1050eb8dc403SDave Cobbley raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 1051eb8dc403SDave Cobbley 1052eb8dc403SDave Cobbley if args.newname and args.newname != args.recipename: 1053eb8dc403SDave Cobbley reason = oe.recipeutils.validate_pn(args.newname) 1054eb8dc403SDave Cobbley if reason: 1055eb8dc403SDave Cobbley raise DevtoolError(reason) 1056eb8dc403SDave Cobbley newname = args.newname 1057eb8dc403SDave Cobbley else: 1058eb8dc403SDave Cobbley newname = args.recipename 1059eb8dc403SDave Cobbley 1060eb8dc403SDave Cobbley append = workspace[args.recipename]['bbappend'] 1061eb8dc403SDave Cobbley appendfn = os.path.splitext(os.path.basename(append))[0] 1062eb8dc403SDave Cobbley splitfn = appendfn.split('_') 1063eb8dc403SDave Cobbley if len(splitfn) > 1: 1064eb8dc403SDave Cobbley origfnver = appendfn.split('_')[1] 1065eb8dc403SDave Cobbley else: 1066eb8dc403SDave Cobbley origfnver = '' 1067eb8dc403SDave Cobbley 1068eb8dc403SDave Cobbley recipefilemd5 = None 1069eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1070eb8dc403SDave Cobbley try: 1071eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 1072eb8dc403SDave Cobbley if not rd: 1073eb8dc403SDave Cobbley return 1 1074eb8dc403SDave Cobbley 1075eb8dc403SDave Cobbley bp = rd.getVar('BP') 1076eb8dc403SDave Cobbley bpn = rd.getVar('BPN') 1077eb8dc403SDave Cobbley if newname != args.recipename: 1078eb8dc403SDave Cobbley localdata = rd.createCopy() 1079eb8dc403SDave Cobbley localdata.setVar('PN', newname) 1080eb8dc403SDave Cobbley newbpn = localdata.getVar('BPN') 1081eb8dc403SDave Cobbley else: 1082eb8dc403SDave Cobbley newbpn = bpn 1083eb8dc403SDave Cobbley s = rd.getVar('S', False) 1084eb8dc403SDave Cobbley src_uri = rd.getVar('SRC_URI', False) 1085eb8dc403SDave Cobbley pv = rd.getVar('PV') 1086eb8dc403SDave Cobbley 1087eb8dc403SDave Cobbley # Correct variable values that refer to the upstream source - these 1088eb8dc403SDave Cobbley # values must stay the same, so if the name/version are changing then 1089eb8dc403SDave Cobbley # we need to fix them up 1090eb8dc403SDave Cobbley new_s = s 1091eb8dc403SDave Cobbley new_src_uri = src_uri 1092eb8dc403SDave Cobbley if newbpn != bpn: 1093eb8dc403SDave Cobbley # ${PN} here is technically almost always incorrect, but people do use it 1094eb8dc403SDave Cobbley new_s = new_s.replace('${BPN}', bpn) 1095eb8dc403SDave Cobbley new_s = new_s.replace('${PN}', bpn) 1096eb8dc403SDave Cobbley new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 1097eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${BPN}', bpn) 1098eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${PN}', bpn) 1099eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 1100eb8dc403SDave Cobbley if args.version and origfnver == pv: 1101eb8dc403SDave Cobbley new_s = new_s.replace('${PV}', pv) 1102eb8dc403SDave Cobbley new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 1103eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${PV}', pv) 1104eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 1105eb8dc403SDave Cobbley patchfields = {} 1106eb8dc403SDave Cobbley if new_s != s: 1107eb8dc403SDave Cobbley patchfields['S'] = new_s 1108eb8dc403SDave Cobbley if new_src_uri != src_uri: 1109eb8dc403SDave Cobbley patchfields['SRC_URI'] = new_src_uri 1110eb8dc403SDave Cobbley if patchfields: 1111eb8dc403SDave Cobbley recipefilemd5 = bb.utils.md5_file(recipefile) 1112eb8dc403SDave Cobbley oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1113eb8dc403SDave Cobbley newrecipefilemd5 = bb.utils.md5_file(recipefile) 1114eb8dc403SDave Cobbley finally: 1115eb8dc403SDave Cobbley tinfoil.shutdown() 1116eb8dc403SDave Cobbley 1117eb8dc403SDave Cobbley if args.version: 1118eb8dc403SDave Cobbley newver = args.version 1119eb8dc403SDave Cobbley else: 1120eb8dc403SDave Cobbley newver = origfnver 1121eb8dc403SDave Cobbley 1122eb8dc403SDave Cobbley if newver: 1123eb8dc403SDave Cobbley newappend = '%s_%s.bbappend' % (newname, newver) 1124eb8dc403SDave Cobbley newfile = '%s_%s.bb' % (newname, newver) 1125eb8dc403SDave Cobbley else: 1126eb8dc403SDave Cobbley newappend = '%s.bbappend' % newname 1127eb8dc403SDave Cobbley newfile = '%s.bb' % newname 1128eb8dc403SDave Cobbley 1129eb8dc403SDave Cobbley oldrecipedir = os.path.dirname(recipefile) 1130eb8dc403SDave Cobbley newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 1131eb8dc403SDave Cobbley if oldrecipedir != newrecipedir: 1132eb8dc403SDave Cobbley bb.utils.mkdirhier(newrecipedir) 1133eb8dc403SDave Cobbley 1134eb8dc403SDave Cobbley newappend = os.path.join(os.path.dirname(append), newappend) 1135eb8dc403SDave Cobbley newfile = os.path.join(newrecipedir, newfile) 1136eb8dc403SDave Cobbley 1137eb8dc403SDave Cobbley # Rename bbappend 1138eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (append, newappend)) 1139c926e17cSAndrew Geissler bb.utils.rename(append, newappend) 1140eb8dc403SDave Cobbley # Rename recipe file 1141eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (recipefile, newfile)) 1142c926e17cSAndrew Geissler bb.utils.rename(recipefile, newfile) 1143eb8dc403SDave Cobbley 1144eb8dc403SDave Cobbley # Rename source tree if it's the default path 1145eb8dc403SDave Cobbley appendmd5 = None 1146eb8dc403SDave Cobbley if not args.no_srctree: 1147eb8dc403SDave Cobbley srctree = workspace[args.recipename]['srctree'] 1148eb8dc403SDave Cobbley if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 1149eb8dc403SDave Cobbley newsrctree = os.path.join(config.workspace_path, 'sources', newname) 1150eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (srctree, newsrctree)) 1151eb8dc403SDave Cobbley shutil.move(srctree, newsrctree) 1152eb8dc403SDave Cobbley # Correct any references (basically EXTERNALSRC*) in the .bbappend 1153eb8dc403SDave Cobbley appendmd5 = bb.utils.md5_file(newappend) 1154eb8dc403SDave Cobbley appendlines = [] 1155eb8dc403SDave Cobbley with open(newappend, 'r') as f: 1156eb8dc403SDave Cobbley for line in f: 1157eb8dc403SDave Cobbley appendlines.append(line) 1158eb8dc403SDave Cobbley with open(newappend, 'w') as f: 1159eb8dc403SDave Cobbley for line in appendlines: 1160eb8dc403SDave Cobbley if srctree in line: 1161eb8dc403SDave Cobbley line = line.replace(srctree, newsrctree) 1162eb8dc403SDave Cobbley f.write(line) 1163eb8dc403SDave Cobbley newappendmd5 = bb.utils.md5_file(newappend) 1164eb8dc403SDave Cobbley 1165eb8dc403SDave Cobbley bpndir = None 1166eb8dc403SDave Cobbley newbpndir = None 1167eb8dc403SDave Cobbley if newbpn != bpn: 1168eb8dc403SDave Cobbley bpndir = os.path.join(oldrecipedir, bpn) 1169eb8dc403SDave Cobbley if os.path.exists(bpndir): 1170eb8dc403SDave Cobbley newbpndir = os.path.join(newrecipedir, newbpn) 1171eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1172eb8dc403SDave Cobbley shutil.move(bpndir, newbpndir) 1173eb8dc403SDave Cobbley 1174eb8dc403SDave Cobbley bpdir = None 1175eb8dc403SDave Cobbley newbpdir = None 1176eb8dc403SDave Cobbley if newver != origfnver or newbpn != bpn: 1177eb8dc403SDave Cobbley bpdir = os.path.join(oldrecipedir, bp) 1178eb8dc403SDave Cobbley if os.path.exists(bpdir): 1179eb8dc403SDave Cobbley newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1180eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1181eb8dc403SDave Cobbley shutil.move(bpdir, newbpdir) 1182eb8dc403SDave Cobbley 1183eb8dc403SDave Cobbley if oldrecipedir != newrecipedir: 1184eb8dc403SDave Cobbley # Move any stray files and delete the old recipe directory 1185eb8dc403SDave Cobbley for entry in os.listdir(oldrecipedir): 1186eb8dc403SDave Cobbley oldpath = os.path.join(oldrecipedir, entry) 1187eb8dc403SDave Cobbley newpath = os.path.join(newrecipedir, entry) 1188eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (oldpath, newpath)) 1189eb8dc403SDave Cobbley shutil.move(oldpath, newpath) 1190eb8dc403SDave Cobbley os.rmdir(oldrecipedir) 1191eb8dc403SDave Cobbley 1192eb8dc403SDave Cobbley # Now take care of entries in .devtool_md5 1193eb8dc403SDave Cobbley md5entries = [] 1194eb8dc403SDave Cobbley with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1195eb8dc403SDave Cobbley for line in f: 1196eb8dc403SDave Cobbley md5entries.append(line) 1197eb8dc403SDave Cobbley 1198eb8dc403SDave Cobbley if bpndir and newbpndir: 1199eb8dc403SDave Cobbley relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1200eb8dc403SDave Cobbley else: 1201eb8dc403SDave Cobbley relbpndir = None 1202eb8dc403SDave Cobbley if bpdir and newbpdir: 1203eb8dc403SDave Cobbley relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1204eb8dc403SDave Cobbley else: 1205eb8dc403SDave Cobbley relbpdir = None 1206eb8dc403SDave Cobbley 1207eb8dc403SDave Cobbley with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1208eb8dc403SDave Cobbley for entry in md5entries: 1209eb8dc403SDave Cobbley splitentry = entry.rstrip().split('|') 1210eb8dc403SDave Cobbley if len(splitentry) > 2: 1211eb8dc403SDave Cobbley if splitentry[0] == args.recipename: 1212eb8dc403SDave Cobbley splitentry[0] = newname 1213eb8dc403SDave Cobbley if splitentry[1] == os.path.relpath(append, config.workspace_path): 1214eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1215eb8dc403SDave Cobbley if appendmd5 and splitentry[2] == appendmd5: 1216eb8dc403SDave Cobbley splitentry[2] = newappendmd5 1217eb8dc403SDave Cobbley elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1218eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1219eb8dc403SDave Cobbley if recipefilemd5 and splitentry[2] == recipefilemd5: 1220eb8dc403SDave Cobbley splitentry[2] = newrecipefilemd5 1221eb8dc403SDave Cobbley elif relbpndir and splitentry[1].startswith(relbpndir): 1222eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1223eb8dc403SDave Cobbley elif relbpdir and splitentry[1].startswith(relbpdir): 1224eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1225eb8dc403SDave Cobbley entry = '|'.join(splitentry) + '\n' 1226eb8dc403SDave Cobbley f.write(entry) 1227eb8dc403SDave Cobbley return 0 1228eb8dc403SDave Cobbley 1229eb8dc403SDave Cobbley 1230eb8dc403SDave Cobbleydef _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1231eb8dc403SDave Cobbley """Get initial and update rev of a recipe. These are the start point of the 1232eb8dc403SDave Cobbley whole patchset and start point for the patches to be re-generated/updated. 1233eb8dc403SDave Cobbley """ 1234eb8dc403SDave Cobbley import bb 1235eb8dc403SDave Cobbley 1236eb8dc403SDave Cobbley # Get current branch 1237eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1238eb8dc403SDave Cobbley cwd=srctree) 1239eb8dc403SDave Cobbley branchname = stdout.rstrip() 1240eb8dc403SDave Cobbley 1241eb8dc403SDave Cobbley # Parse initial rev from recipe if not specified 1242da295319SPatrick Williams commits = {} 1243eb8dc403SDave Cobbley patches = [] 1244da295319SPatrick Williams initial_revs = {} 1245eb8dc403SDave Cobbley with open(recipe_path, 'r') as f: 1246eb8dc403SDave Cobbley for line in f: 1247da295319SPatrick Williams pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$' 1248da295319SPatrick Williams match = re.search(pattern, line) 1249da295319SPatrick Williams if match: 1250da295319SPatrick Williams name = match.group(1) 1251da295319SPatrick Williams rev = match.group(2) 1252da295319SPatrick Williams if line.startswith('# initial_rev'): 1253da295319SPatrick Williams if not (name == "." and initial_rev): 1254da295319SPatrick Williams initial_revs[name] = rev 1255da295319SPatrick Williams elif line.startswith('# commit') and not force_patch_refresh: 1256da295319SPatrick Williams if name not in commits: 1257da295319SPatrick Williams commits[name] = [rev] 1258da295319SPatrick Williams else: 1259da295319SPatrick Williams commits[name].append(rev) 1260eb8dc403SDave Cobbley elif line.startswith('# patches_%s:' % branchname): 1261eb8dc403SDave Cobbley patches = line.split(':')[-1].strip().split(',') 1262eb8dc403SDave Cobbley 1263da295319SPatrick Williams update_revs = dict(initial_revs) 1264da295319SPatrick Williams changed_revs = {} 1265da295319SPatrick Williams for name, rev in initial_revs.items(): 1266eb8dc403SDave Cobbley # Find first actually changed revision 1267eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1268da295319SPatrick Williams rev, cwd=os.path.join(srctree, name)) 1269eb8dc403SDave Cobbley newcommits = stdout.split() 1270da295319SPatrick Williams if name in commits: 1271da295319SPatrick Williams for i in range(min(len(commits[name]), len(newcommits))): 1272da295319SPatrick Williams if newcommits[i] == commits[name][i]: 1273da295319SPatrick Williams update_revs[name] = commits[name][i] 1274eb8dc403SDave Cobbley 1275eb8dc403SDave Cobbley try: 1276eb8dc403SDave Cobbley stdout, _ = bb.process.run('git cherry devtool-patched', 1277da295319SPatrick Williams cwd=os.path.join(srctree, name)) 1278eb8dc403SDave Cobbley except bb.process.ExecutionError as err: 1279eb8dc403SDave Cobbley stdout = None 1280eb8dc403SDave Cobbley 1281eb8dc403SDave Cobbley if stdout is not None and not force_patch_refresh: 1282eb8dc403SDave Cobbley for line in stdout.splitlines(): 1283eb8dc403SDave Cobbley if line.startswith('+ '): 1284eb8dc403SDave Cobbley rev = line.split()[1] 1285eb8dc403SDave Cobbley if rev in newcommits: 1286da295319SPatrick Williams if name not in changed_revs: 1287da295319SPatrick Williams changed_revs[name] = [rev] 1288da295319SPatrick Williams else: 1289da295319SPatrick Williams changed_revs[name].append(rev) 1290eb8dc403SDave Cobbley 1291da295319SPatrick Williams return initial_revs, update_revs, changed_revs, patches 1292eb8dc403SDave Cobbley 1293eb8dc403SDave Cobbleydef _remove_file_entries(srcuri, filelist): 1294eb8dc403SDave Cobbley """Remove file:// entries from SRC_URI""" 1295eb8dc403SDave Cobbley remaining = filelist[:] 1296eb8dc403SDave Cobbley entries = [] 1297eb8dc403SDave Cobbley for fname in filelist: 1298eb8dc403SDave Cobbley basename = os.path.basename(fname) 1299eb8dc403SDave Cobbley for i in range(len(srcuri)): 1300eb8dc403SDave Cobbley if (srcuri[i].startswith('file://') and 1301eb8dc403SDave Cobbley os.path.basename(srcuri[i].split(';')[0]) == basename): 1302eb8dc403SDave Cobbley entries.append(srcuri[i]) 1303eb8dc403SDave Cobbley remaining.remove(fname) 1304eb8dc403SDave Cobbley srcuri.pop(i) 1305eb8dc403SDave Cobbley break 1306eb8dc403SDave Cobbley return entries, remaining 1307eb8dc403SDave Cobbley 1308eb8dc403SDave Cobbleydef _replace_srcuri_entry(srcuri, filename, newentry): 1309eb8dc403SDave Cobbley """Replace entry corresponding to specified file with a new entry""" 1310eb8dc403SDave Cobbley basename = os.path.basename(filename) 1311eb8dc403SDave Cobbley for i in range(len(srcuri)): 1312eb8dc403SDave Cobbley if os.path.basename(srcuri[i].split(';')[0]) == basename: 1313eb8dc403SDave Cobbley srcuri.pop(i) 1314eb8dc403SDave Cobbley srcuri.insert(i, newentry) 1315eb8dc403SDave Cobbley break 1316eb8dc403SDave Cobbley 1317eb8dc403SDave Cobbleydef _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1318eb8dc403SDave Cobbley """Unlink existing patch files""" 1319eb8dc403SDave Cobbley 1320eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run else '' 1321eb8dc403SDave Cobbley 1322eb8dc403SDave Cobbley for path in files: 1323eb8dc403SDave Cobbley if append: 1324eb8dc403SDave Cobbley if not destpath: 1325eb8dc403SDave Cobbley raise Exception('destpath should be set here') 1326eb8dc403SDave Cobbley path = os.path.join(destpath, os.path.basename(path)) 1327eb8dc403SDave Cobbley 1328eb8dc403SDave Cobbley if os.path.exists(path): 1329eb8dc403SDave Cobbley if not no_report_remove: 1330eb8dc403SDave Cobbley logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1331eb8dc403SDave Cobbley if not dry_run: 1332eb8dc403SDave Cobbley # FIXME "git rm" here would be nice if the file in question is 1333eb8dc403SDave Cobbley # tracked 1334eb8dc403SDave Cobbley # FIXME there's a chance that this file is referred to by 1335eb8dc403SDave Cobbley # another recipe, in which case deleting wouldn't be the 1336eb8dc403SDave Cobbley # right thing to do 1337eb8dc403SDave Cobbley os.remove(path) 1338eb8dc403SDave Cobbley # Remove directory if empty 1339eb8dc403SDave Cobbley try: 1340eb8dc403SDave Cobbley os.rmdir(os.path.dirname(path)) 1341eb8dc403SDave Cobbley except OSError as ose: 1342eb8dc403SDave Cobbley if ose.errno != errno.ENOTEMPTY: 1343eb8dc403SDave Cobbley raise 1344eb8dc403SDave Cobbley 1345eb8dc403SDave Cobbley 1346da295319SPatrick Williamsdef _export_patches(srctree, rd, start_revs, destdir, changed_revs=None): 1347eb8dc403SDave Cobbley """Export patches from srctree to given location. 1348eb8dc403SDave Cobbley Returns three-tuple of dicts: 1349eb8dc403SDave Cobbley 1. updated - patches that already exist in SRCURI 1350eb8dc403SDave Cobbley 2. added - new patches that don't exist in SRCURI 1351eb8dc403SDave Cobbley 3 removed - patches that exist in SRCURI but not in exported patches 1352da295319SPatrick Williams In each dict the key is the 'basepath' of the URI and value is: 1353da295319SPatrick Williams - for updated and added dicts, a dict with 2 optionnal keys: 1354da295319SPatrick Williams - 'path': the absolute path to the existing file in recipe space (if any) 1355da295319SPatrick Williams - 'patchdir': the directory in wich the patch should be applied (if any) 1356da295319SPatrick Williams - for removed dict, the absolute path to the existing file in recipe space 1357eb8dc403SDave Cobbley """ 1358eb8dc403SDave Cobbley import oe.recipeutils 1359eb8dc403SDave Cobbley from oe.patch import GitApplyTree 1360eb8dc403SDave Cobbley updated = OrderedDict() 1361eb8dc403SDave Cobbley added = OrderedDict() 1362eb8dc403SDave Cobbley seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1363eb8dc403SDave Cobbley 1364eb8dc403SDave Cobbley existing_patches = dict((os.path.basename(path), path) for path in 1365eb8dc403SDave Cobbley oe.recipeutils.get_recipe_patches(rd)) 1366eb8dc403SDave Cobbley logger.debug('Existing patches: %s' % existing_patches) 1367eb8dc403SDave Cobbley 1368eb8dc403SDave Cobbley # Generate patches from Git, exclude local files directory 1369eb8dc403SDave Cobbley patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1370da295319SPatrick Williams GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec) 1371da295319SPatrick Williams for dirpath, dirnames, filenames in os.walk(destdir): 1372da295319SPatrick Williams new_patches = filenames 1373da295319SPatrick Williams reldirpath = os.path.relpath(dirpath, destdir) 1374eb8dc403SDave Cobbley for new_patch in new_patches: 1375eb8dc403SDave Cobbley # Strip numbering from patch names. If it's a git sequence named patch, 1376eb8dc403SDave Cobbley # the numbers might not match up since we are starting from a different 1377eb8dc403SDave Cobbley # revision This does assume that people are using unique shortlog 1378eb8dc403SDave Cobbley # values, but they ought to be anyway... 1379eb8dc403SDave Cobbley new_basename = seqpatch_re.match(new_patch).group(2) 1380eb8dc403SDave Cobbley match_name = None 1381eb8dc403SDave Cobbley for old_patch in existing_patches: 1382eb8dc403SDave Cobbley old_basename = seqpatch_re.match(old_patch).group(2) 1383eb8dc403SDave Cobbley old_basename_splitext = os.path.splitext(old_basename) 1384eb8dc403SDave Cobbley if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1385eb8dc403SDave Cobbley old_patch_noext = os.path.splitext(old_patch)[0] 1386eb8dc403SDave Cobbley match_name = old_patch_noext 1387eb8dc403SDave Cobbley break 1388eb8dc403SDave Cobbley elif new_basename == old_basename: 1389eb8dc403SDave Cobbley match_name = old_patch 1390eb8dc403SDave Cobbley break 1391eb8dc403SDave Cobbley if match_name: 1392eb8dc403SDave Cobbley # Rename patch files 1393eb8dc403SDave Cobbley if new_patch != match_name: 1394c926e17cSAndrew Geissler bb.utils.rename(os.path.join(destdir, new_patch), 1395eb8dc403SDave Cobbley os.path.join(destdir, match_name)) 1396eb8dc403SDave Cobbley # Need to pop it off the list now before checking changed_revs 1397eb8dc403SDave Cobbley oldpath = existing_patches.pop(old_patch) 1398da295319SPatrick Williams if changed_revs is not None and dirpath in changed_revs: 1399eb8dc403SDave Cobbley # Avoid updating patches that have not actually changed 1400da295319SPatrick Williams with open(os.path.join(dirpath, match_name), 'r') as f: 1401eb8dc403SDave Cobbley firstlineitems = f.readline().split() 1402eb8dc403SDave Cobbley # Looking for "From <hash>" line 1403eb8dc403SDave Cobbley if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1404da295319SPatrick Williams if not firstlineitems[1] in changed_revs[dirpath]: 1405eb8dc403SDave Cobbley continue 1406eb8dc403SDave Cobbley # Recompress if necessary 1407eb8dc403SDave Cobbley if oldpath.endswith(('.gz', '.Z')): 1408eb8dc403SDave Cobbley bb.process.run(['gzip', match_name], cwd=destdir) 1409eb8dc403SDave Cobbley if oldpath.endswith('.gz'): 1410eb8dc403SDave Cobbley match_name += '.gz' 1411eb8dc403SDave Cobbley else: 1412eb8dc403SDave Cobbley match_name += '.Z' 1413eb8dc403SDave Cobbley elif oldpath.endswith('.bz2'): 1414eb8dc403SDave Cobbley bb.process.run(['bzip2', match_name], cwd=destdir) 1415eb8dc403SDave Cobbley match_name += '.bz2' 1416da295319SPatrick Williams updated[match_name] = {'path' : oldpath} 1417da295319SPatrick Williams if reldirpath != ".": 1418da295319SPatrick Williams updated[match_name]['patchdir'] = reldirpath 1419eb8dc403SDave Cobbley else: 1420da295319SPatrick Williams added[new_patch] = {} 1421da295319SPatrick Williams if reldirpath != ".": 1422da295319SPatrick Williams added[new_patch]['patchdir'] = reldirpath 1423da295319SPatrick Williams 1424eb8dc403SDave Cobbley return (updated, added, existing_patches) 1425eb8dc403SDave Cobbley 1426eb8dc403SDave Cobbley 1427eb8dc403SDave Cobbleydef _create_kconfig_diff(srctree, rd, outfile): 1428eb8dc403SDave Cobbley """Create a kconfig fragment""" 1429eb8dc403SDave Cobbley # Only update config fragment if both config files exist 1430eb8dc403SDave Cobbley orig_config = os.path.join(srctree, '.config.baseline') 1431eb8dc403SDave Cobbley new_config = os.path.join(srctree, '.config.new') 1432eb8dc403SDave Cobbley if os.path.exists(orig_config) and os.path.exists(new_config): 1433eb8dc403SDave Cobbley cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1434eb8dc403SDave Cobbley '--unchanged-line-format=', orig_config, new_config] 1435eb8dc403SDave Cobbley pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1436eb8dc403SDave Cobbley stderr=subprocess.PIPE) 1437eb8dc403SDave Cobbley stdout, stderr = pipe.communicate() 1438eb8dc403SDave Cobbley if pipe.returncode == 1: 1439eb8dc403SDave Cobbley logger.info("Updating config fragment %s" % outfile) 1440eb8dc403SDave Cobbley with open(outfile, 'wb') as fobj: 1441eb8dc403SDave Cobbley fobj.write(stdout) 1442eb8dc403SDave Cobbley elif pipe.returncode == 0: 1443eb8dc403SDave Cobbley logger.info("Would remove config fragment %s" % outfile) 1444eb8dc403SDave Cobbley if os.path.exists(outfile): 1445eb8dc403SDave Cobbley # Remove fragment file in case of empty diff 1446eb8dc403SDave Cobbley logger.info("Removing config fragment %s" % outfile) 1447eb8dc403SDave Cobbley os.unlink(outfile) 1448eb8dc403SDave Cobbley else: 1449eb8dc403SDave Cobbley raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1450eb8dc403SDave Cobbley return True 1451eb8dc403SDave Cobbley return False 1452eb8dc403SDave Cobbley 1453eb8dc403SDave Cobbley 1454eb8dc403SDave Cobbleydef _export_local_files(srctree, rd, destdir, srctreebase): 1455eb8dc403SDave Cobbley """Copy local files from srctree to given location. 1456eb8dc403SDave Cobbley Returns three-tuple of dicts: 1457eb8dc403SDave Cobbley 1. updated - files that already exist in SRCURI 1458eb8dc403SDave Cobbley 2. added - new files files that don't exist in SRCURI 1459eb8dc403SDave Cobbley 3 removed - files that exist in SRCURI but not in exported files 1460eb8dc403SDave Cobbley In each dict the key is the 'basepath' of the URI and value is the 1461eb8dc403SDave Cobbley absolute path to the existing file in recipe space (if any). 1462eb8dc403SDave Cobbley """ 1463eb8dc403SDave Cobbley import oe.recipeutils 1464eb8dc403SDave Cobbley 1465eb8dc403SDave Cobbley # Find out local files (SRC_URI files that exist in the "recipe space"). 1466eb8dc403SDave Cobbley # Local files that reside in srctree are not included in patch generation. 1467eb8dc403SDave Cobbley # Instead they are directly copied over the original source files (in 1468eb8dc403SDave Cobbley # recipe space). 1469eb8dc403SDave Cobbley existing_files = oe.recipeutils.get_recipe_local_files(rd) 1470eb8dc403SDave Cobbley new_set = None 1471eb8dc403SDave Cobbley updated = OrderedDict() 1472eb8dc403SDave Cobbley added = OrderedDict() 1473eb8dc403SDave Cobbley removed = OrderedDict() 1474517393d9SAndrew Geissler 1475517393d9SAndrew Geissler # Get current branch and return early with empty lists 1476517393d9SAndrew Geissler # if on one of the override branches 1477517393d9SAndrew Geissler # (local files are provided only for the main branch and processing 1478517393d9SAndrew Geissler # them against lists from recipe overrides will result in mismatches 1479517393d9SAndrew Geissler # and broken modifications to recipes). 1480517393d9SAndrew Geissler stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1481517393d9SAndrew Geissler cwd=srctree) 1482517393d9SAndrew Geissler branchname = stdout.rstrip() 1483517393d9SAndrew Geissler if branchname.startswith(override_branch_prefix): 1484517393d9SAndrew Geissler return (updated, added, removed) 1485517393d9SAndrew Geissler 1486eb8dc403SDave Cobbley local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1487eb8dc403SDave Cobbley git_files = _git_ls_tree(srctree) 1488eb8dc403SDave Cobbley if 'oe-local-files' in git_files: 1489eb8dc403SDave Cobbley # If tracked by Git, take the files from srctree HEAD. First get 1490eb8dc403SDave Cobbley # the tree object of the directory 1491eb8dc403SDave Cobbley tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1492eb8dc403SDave Cobbley tree = git_files['oe-local-files'][2] 1493eb8dc403SDave Cobbley bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1494eb8dc403SDave Cobbley env=dict(os.environ, GIT_WORK_TREE=destdir, 1495eb8dc403SDave Cobbley GIT_INDEX_FILE=tmp_index)) 1496eb8dc403SDave Cobbley new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1497eb8dc403SDave Cobbley elif os.path.isdir(local_files_dir): 1498eb8dc403SDave Cobbley # If not tracked by Git, just copy from working copy 1499eb8dc403SDave Cobbley new_set = _ls_tree(local_files_dir) 1500eb8dc403SDave Cobbley bb.process.run(['cp', '-ax', 1501eb8dc403SDave Cobbley os.path.join(local_files_dir, '.'), destdir]) 1502eb8dc403SDave Cobbley else: 1503eb8dc403SDave Cobbley new_set = [] 1504eb8dc403SDave Cobbley 1505eb8dc403SDave Cobbley # Special handling for kernel config 1506eb8dc403SDave Cobbley if bb.data.inherits_class('kernel-yocto', rd): 1507eb8dc403SDave Cobbley fragment_fn = 'devtool-fragment.cfg' 1508eb8dc403SDave Cobbley fragment_path = os.path.join(destdir, fragment_fn) 1509eb8dc403SDave Cobbley if _create_kconfig_diff(srctree, rd, fragment_path): 1510eb8dc403SDave Cobbley if os.path.exists(fragment_path): 1511eb8dc403SDave Cobbley if fragment_fn not in new_set: 1512eb8dc403SDave Cobbley new_set.append(fragment_fn) 1513eb8dc403SDave Cobbley # Copy fragment to local-files 1514eb8dc403SDave Cobbley if os.path.isdir(local_files_dir): 1515eb8dc403SDave Cobbley shutil.copy2(fragment_path, local_files_dir) 1516eb8dc403SDave Cobbley else: 1517eb8dc403SDave Cobbley if fragment_fn in new_set: 1518eb8dc403SDave Cobbley new_set.remove(fragment_fn) 1519eb8dc403SDave Cobbley # Remove fragment from local-files 1520eb8dc403SDave Cobbley if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1521eb8dc403SDave Cobbley os.unlink(os.path.join(local_files_dir, fragment_fn)) 1522eb8dc403SDave Cobbley 1523d89cb5f0SBrad Bishop # Special handling for cml1, ccmake, etc bbclasses that generated 1524d89cb5f0SBrad Bishop # configuration fragment files that are consumed as source files 1525d89cb5f0SBrad Bishop for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]: 1526d89cb5f0SBrad Bishop if bb.data.inherits_class(frag_class, rd): 1527d89cb5f0SBrad Bishop srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) 1528d89cb5f0SBrad Bishop if os.path.exists(srcpath): 1529d89cb5f0SBrad Bishop if frag_name not in new_set: 1530d89cb5f0SBrad Bishop new_set.append(frag_name) 1531d89cb5f0SBrad Bishop # copy fragment into destdir 1532d89cb5f0SBrad Bishop shutil.copy2(srcpath, destdir) 1533d89cb5f0SBrad Bishop # copy fragment into local files if exists 1534d89cb5f0SBrad Bishop if os.path.isdir(local_files_dir): 1535d89cb5f0SBrad Bishop shutil.copy2(srcpath, local_files_dir) 1536d89cb5f0SBrad Bishop 1537eb8dc403SDave Cobbley if new_set is not None: 1538eb8dc403SDave Cobbley for fname in new_set: 1539eb8dc403SDave Cobbley if fname in existing_files: 1540eb8dc403SDave Cobbley origpath = existing_files.pop(fname) 1541eb8dc403SDave Cobbley workpath = os.path.join(local_files_dir, fname) 1542eb8dc403SDave Cobbley if not filecmp.cmp(origpath, workpath): 1543eb8dc403SDave Cobbley updated[fname] = origpath 1544eb8dc403SDave Cobbley elif fname != '.gitignore': 1545eb8dc403SDave Cobbley added[fname] = None 1546eb8dc403SDave Cobbley 1547eb8dc403SDave Cobbley workdir = rd.getVar('WORKDIR') 1548eb8dc403SDave Cobbley s = rd.getVar('S') 1549eb8dc403SDave Cobbley if not s.endswith(os.sep): 1550eb8dc403SDave Cobbley s += os.sep 1551eb8dc403SDave Cobbley 1552eb8dc403SDave Cobbley if workdir != s: 1553eb8dc403SDave Cobbley # Handle files where subdir= was specified 1554eb8dc403SDave Cobbley for fname in list(existing_files.keys()): 1555eb8dc403SDave Cobbley # FIXME handle both subdir starting with BP and not? 1556eb8dc403SDave Cobbley fworkpath = os.path.join(workdir, fname) 1557eb8dc403SDave Cobbley if fworkpath.startswith(s): 1558eb8dc403SDave Cobbley fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1559eb8dc403SDave Cobbley if os.path.exists(fpath): 1560eb8dc403SDave Cobbley origpath = existing_files.pop(fname) 1561eb8dc403SDave Cobbley if not filecmp.cmp(origpath, fpath): 1562eb8dc403SDave Cobbley updated[fpath] = origpath 1563eb8dc403SDave Cobbley 1564eb8dc403SDave Cobbley removed = existing_files 1565eb8dc403SDave Cobbley return (updated, added, removed) 1566eb8dc403SDave Cobbley 1567eb8dc403SDave Cobbley 1568eb8dc403SDave Cobbleydef _determine_files_dir(rd): 1569eb8dc403SDave Cobbley """Determine the appropriate files directory for a recipe""" 1570eb8dc403SDave Cobbley recipedir = rd.getVar('FILE_DIRNAME') 1571eb8dc403SDave Cobbley for entry in rd.getVar('FILESPATH').split(':'): 1572eb8dc403SDave Cobbley relpth = os.path.relpath(entry, recipedir) 1573eb8dc403SDave Cobbley if not os.sep in relpth: 1574eb8dc403SDave Cobbley # One (or zero) levels below only, so we don't put anything in machine-specific directories 1575eb8dc403SDave Cobbley if os.path.isdir(entry): 1576eb8dc403SDave Cobbley return entry 1577eb8dc403SDave Cobbley return os.path.join(recipedir, rd.getVar('BPN')) 1578eb8dc403SDave Cobbley 1579eb8dc403SDave Cobbley 1580eb8dc403SDave Cobbleydef _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1581eb8dc403SDave Cobbley """Implement the 'srcrev' mode of update-recipe""" 1582eb8dc403SDave Cobbley import bb 1583eb8dc403SDave Cobbley import oe.recipeutils 1584eb8dc403SDave Cobbley 1585eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1586eb8dc403SDave Cobbley 1587eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 1588eb8dc403SDave Cobbley recipedir = os.path.basename(recipefile) 1589eb8dc403SDave Cobbley logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1590eb8dc403SDave Cobbley 1591da295319SPatrick Williams # Get original SRCREV 1592da295319SPatrick Williams old_srcrev = rd.getVar('SRCREV') or '' 1593da295319SPatrick Williams if old_srcrev == "INVALID": 1594da295319SPatrick Williams raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository') 1595da295319SPatrick Williams old_srcrev = {'.': old_srcrev} 1596da295319SPatrick Williams 1597eb8dc403SDave Cobbley # Get HEAD revision 1598eb8dc403SDave Cobbley try: 1599eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1600eb8dc403SDave Cobbley except bb.process.ExecutionError as err: 1601eb8dc403SDave Cobbley raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1602eb8dc403SDave Cobbley (srctree, err)) 1603eb8dc403SDave Cobbley srcrev = stdout.strip() 1604eb8dc403SDave Cobbley if len(srcrev) != 40: 1605eb8dc403SDave Cobbley raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1606eb8dc403SDave Cobbley 1607eb8dc403SDave Cobbley destpath = None 1608eb8dc403SDave Cobbley remove_files = [] 1609eb8dc403SDave Cobbley patchfields = {} 1610eb8dc403SDave Cobbley patchfields['SRCREV'] = srcrev 1611eb8dc403SDave Cobbley orig_src_uri = rd.getVar('SRC_URI', False) or '' 1612eb8dc403SDave Cobbley srcuri = orig_src_uri.split() 1613eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtool') 1614eb8dc403SDave Cobbley update_srcuri = False 1615eb8dc403SDave Cobbley appendfile = None 1616eb8dc403SDave Cobbley try: 1617eb8dc403SDave Cobbley local_files_dir = tempfile.mkdtemp(dir=tempdir) 1618eb8dc403SDave Cobbley srctreebase = workspace[recipename]['srctreebase'] 1619eb8dc403SDave Cobbley upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1620eb8dc403SDave Cobbley if not no_remove: 1621eb8dc403SDave Cobbley # Find list of existing patches in recipe file 1622eb8dc403SDave Cobbley patches_dir = tempfile.mkdtemp(dir=tempdir) 1623eb8dc403SDave Cobbley upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1624eb8dc403SDave Cobbley patches_dir) 1625eb8dc403SDave Cobbley logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1626eb8dc403SDave Cobbley 1627eb8dc403SDave Cobbley # Remove deleted local files and "overlapping" patches 1628da295319SPatrick Williams remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value] 1629eb8dc403SDave Cobbley if remove_files: 1630eb8dc403SDave Cobbley removedentries = _remove_file_entries(srcuri, remove_files)[0] 1631eb8dc403SDave Cobbley update_srcuri = True 1632eb8dc403SDave Cobbley 1633eb8dc403SDave Cobbley if appendlayerdir: 1634eb8dc403SDave Cobbley files = dict((os.path.join(local_files_dir, key), val) for 1635eb8dc403SDave Cobbley key, val in list(upd_f.items()) + list(new_f.items())) 1636eb8dc403SDave Cobbley removevalues = {} 1637eb8dc403SDave Cobbley if update_srcuri: 1638eb8dc403SDave Cobbley removevalues = {'SRC_URI': removedentries} 1639eb8dc403SDave Cobbley patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1640eb8dc403SDave Cobbley if dry_run_outdir: 1641eb8dc403SDave Cobbley logger.info('Creating bbappend (dry-run)') 1642eb8dc403SDave Cobbley appendfile, destpath = oe.recipeutils.bbappend_recipe( 1643eb8dc403SDave Cobbley rd, appendlayerdir, files, wildcardver=wildcard_version, 1644eb8dc403SDave Cobbley extralines=patchfields, removevalues=removevalues, 1645eb8dc403SDave Cobbley redirect_output=dry_run_outdir) 1646eb8dc403SDave Cobbley else: 1647eb8dc403SDave Cobbley files_dir = _determine_files_dir(rd) 1648eb8dc403SDave Cobbley for basepath, path in upd_f.items(): 1649eb8dc403SDave Cobbley logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1650eb8dc403SDave Cobbley if os.path.isabs(basepath): 1651eb8dc403SDave Cobbley # Original file (probably with subdir pointing inside source tree) 1652eb8dc403SDave Cobbley # so we do not want to move it, just copy 1653eb8dc403SDave Cobbley _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1654eb8dc403SDave Cobbley else: 1655eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), path, 1656eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1657eb8dc403SDave Cobbley update_srcuri= True 1658eb8dc403SDave Cobbley for basepath, path in new_f.items(): 1659eb8dc403SDave Cobbley logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1660eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), 1661eb8dc403SDave Cobbley os.path.join(files_dir, basepath), 1662eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, 1663eb8dc403SDave Cobbley base_outdir=recipedir) 1664eb8dc403SDave Cobbley srcuri.append('file://%s' % basepath) 1665eb8dc403SDave Cobbley update_srcuri = True 1666eb8dc403SDave Cobbley if update_srcuri: 1667eb8dc403SDave Cobbley patchfields['SRC_URI'] = ' '.join(srcuri) 1668eb8dc403SDave Cobbley ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1669eb8dc403SDave Cobbley finally: 1670eb8dc403SDave Cobbley shutil.rmtree(tempdir) 1671eb8dc403SDave Cobbley if not 'git://' in orig_src_uri: 1672eb8dc403SDave Cobbley logger.info('You will need to update SRC_URI within the recipe to ' 1673eb8dc403SDave Cobbley 'point to a git repository where you have pushed your ' 1674eb8dc403SDave Cobbley 'changes') 1675eb8dc403SDave Cobbley 1676eb8dc403SDave Cobbley _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1677eb8dc403SDave Cobbley return True, appendfile, remove_files 1678eb8dc403SDave Cobbley 1679eb8dc403SDave Cobbleydef _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1680eb8dc403SDave Cobbley """Implement the 'patch' mode of update-recipe""" 1681eb8dc403SDave Cobbley import bb 1682eb8dc403SDave Cobbley import oe.recipeutils 1683eb8dc403SDave Cobbley 1684eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 1685eb8dc403SDave Cobbley recipedir = os.path.dirname(recipefile) 1686eb8dc403SDave Cobbley append = workspace[recipename]['bbappend'] 1687eb8dc403SDave Cobbley if not os.path.exists(append): 1688eb8dc403SDave Cobbley raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1689eb8dc403SDave Cobbley recipename) 1690615f2f11SAndrew Geissler srctreebase = workspace[recipename]['srctreebase'] 1691615f2f11SAndrew Geissler relpatchdir = os.path.relpath(srctreebase, srctree) 1692615f2f11SAndrew Geissler if relpatchdir == '.': 1693615f2f11SAndrew Geissler patchdir_params = {} 1694615f2f11SAndrew Geissler else: 1695615f2f11SAndrew Geissler patchdir_params = {'patchdir': relpatchdir} 1696615f2f11SAndrew Geissler 1697da295319SPatrick Williams def srcuri_entry(basepath, patchdir_params): 1698615f2f11SAndrew Geissler if patchdir_params: 1699615f2f11SAndrew Geissler paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items()) 1700615f2f11SAndrew Geissler else: 1701615f2f11SAndrew Geissler paramstr = '' 1702615f2f11SAndrew Geissler return 'file://%s%s' % (basepath, paramstr) 1703eb8dc403SDave Cobbley 1704da295319SPatrick Williams initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1705da295319SPatrick Williams if not initial_revs: 1706eb8dc403SDave Cobbley raise DevtoolError('Unable to find initial revision - please specify ' 1707eb8dc403SDave Cobbley 'it with --initial-rev') 1708eb8dc403SDave Cobbley 1709eb8dc403SDave Cobbley appendfile = None 1710eb8dc403SDave Cobbley dl_dir = rd.getVar('DL_DIR') 1711eb8dc403SDave Cobbley if not dl_dir.endswith('/'): 1712eb8dc403SDave Cobbley dl_dir += '/' 1713eb8dc403SDave Cobbley 1714eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1715eb8dc403SDave Cobbley 1716eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtool') 1717eb8dc403SDave Cobbley try: 1718eb8dc403SDave Cobbley local_files_dir = tempfile.mkdtemp(dir=tempdir) 1719eb8dc403SDave Cobbley upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1720eb8dc403SDave Cobbley 1721eb8dc403SDave Cobbley # Get updated patches from source tree 1722eb8dc403SDave Cobbley patches_dir = tempfile.mkdtemp(dir=tempdir) 1723da295319SPatrick Williams upd_p, new_p, _ = _export_patches(srctree, rd, update_revs, 1724eb8dc403SDave Cobbley patches_dir, changed_revs) 1725517393d9SAndrew Geissler # Get all patches from source tree and check if any should be removed 1726517393d9SAndrew Geissler all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1727da295319SPatrick Williams _, _, del_p = _export_patches(srctree, rd, initial_revs, 1728517393d9SAndrew Geissler all_patches_dir) 1729eb8dc403SDave Cobbley logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1730eb8dc403SDave Cobbley if filter_patches: 173100e122a7SBrad Bishop new_p = OrderedDict() 173200e122a7SBrad Bishop upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1733517393d9SAndrew Geissler del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches) 1734517393d9SAndrew Geissler remove_files = [] 1735517393d9SAndrew Geissler if not no_remove: 1736517393d9SAndrew Geissler # Remove deleted local files and patches 1737517393d9SAndrew Geissler remove_files = list(del_f.values()) + list(del_p.values()) 1738eb8dc403SDave Cobbley updatefiles = False 1739eb8dc403SDave Cobbley updaterecipe = False 1740eb8dc403SDave Cobbley destpath = None 1741eb8dc403SDave Cobbley srcuri = (rd.getVar('SRC_URI', False) or '').split() 1742da295319SPatrick Williams 1743eb8dc403SDave Cobbley if appendlayerdir: 174400e122a7SBrad Bishop files = OrderedDict((os.path.join(local_files_dir, key), val) for 1745eb8dc403SDave Cobbley key, val in list(upd_f.items()) + list(new_f.items())) 174600e122a7SBrad Bishop files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1747eb8dc403SDave Cobbley key, val in list(upd_p.items()) + list(new_p.items()))) 1748da295319SPatrick Williams 1749da295319SPatrick Williams params = [] 1750da295319SPatrick Williams for file, param in files.items(): 1751da295319SPatrick Williams patchdir_param = dict(patchdir_params) 1752da295319SPatrick Williams patchdir = param.get('patchdir', ".") 1753da295319SPatrick Williams if patchdir != "." : 1754da295319SPatrick Williams if patchdir_param: 1755da295319SPatrick Williams patchdir_param['patchdir'] += patchdir 1756da295319SPatrick Williams else: 1757da295319SPatrick Williams patchdir_param['patchdir'] = patchdir 1758da295319SPatrick Williams params.append(patchdir_param) 1759da295319SPatrick Williams 1760eb8dc403SDave Cobbley if files or remove_files: 1761eb8dc403SDave Cobbley removevalues = None 1762eb8dc403SDave Cobbley if remove_files: 1763eb8dc403SDave Cobbley removedentries, remaining = _remove_file_entries( 1764eb8dc403SDave Cobbley srcuri, remove_files) 1765eb8dc403SDave Cobbley if removedentries or remaining: 1766da295319SPatrick Williams remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for 1767eb8dc403SDave Cobbley item in remaining] 1768eb8dc403SDave Cobbley removevalues = {'SRC_URI': removedentries + remaining} 1769eb8dc403SDave Cobbley appendfile, destpath = oe.recipeutils.bbappend_recipe( 1770eb8dc403SDave Cobbley rd, appendlayerdir, files, 1771eb8dc403SDave Cobbley wildcardver=wildcard_version, 1772eb8dc403SDave Cobbley removevalues=removevalues, 1773615f2f11SAndrew Geissler redirect_output=dry_run_outdir, 1774da295319SPatrick Williams params=params) 1775eb8dc403SDave Cobbley else: 1776eb8dc403SDave Cobbley logger.info('No patches or local source files needed updating') 1777eb8dc403SDave Cobbley else: 1778eb8dc403SDave Cobbley # Update existing files 1779eb8dc403SDave Cobbley files_dir = _determine_files_dir(rd) 1780eb8dc403SDave Cobbley for basepath, path in upd_f.items(): 1781eb8dc403SDave Cobbley logger.info('Updating file %s' % basepath) 1782eb8dc403SDave Cobbley if os.path.isabs(basepath): 1783eb8dc403SDave Cobbley # Original file (probably with subdir pointing inside source tree) 1784eb8dc403SDave Cobbley # so we do not want to move it, just copy 1785eb8dc403SDave Cobbley _copy_file(basepath, path, 1786eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1787eb8dc403SDave Cobbley else: 1788eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), path, 1789eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1790eb8dc403SDave Cobbley updatefiles = True 1791da295319SPatrick Williams for basepath, param in upd_p.items(): 1792da295319SPatrick Williams path = param['path'] 1793da295319SPatrick Williams patchdir = param.get('patchdir', ".") 1794da295319SPatrick Williams if patchdir != "." : 1795da295319SPatrick Williams patchdir_param = dict(patchdir_params) 1796da295319SPatrick Williams if patchdir_param: 1797da295319SPatrick Williams patchdir_param['patchdir'] += patchdir 1798da295319SPatrick Williams else: 1799da295319SPatrick Williams patchdir_param['patchdir'] = patchdir 1800da295319SPatrick Williams patchfn = os.path.join(patches_dir, patchdir, basepath) 1801eb8dc403SDave Cobbley if os.path.dirname(path) + '/' == dl_dir: 1802eb8dc403SDave Cobbley # This is a a downloaded patch file - we now need to 1803eb8dc403SDave Cobbley # replace the entry in SRC_URI with our local version 1804eb8dc403SDave Cobbley logger.info('Replacing remote patch %s with updated local version' % basepath) 1805eb8dc403SDave Cobbley path = os.path.join(files_dir, basepath) 1806da295319SPatrick Williams _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param)) 1807eb8dc403SDave Cobbley updaterecipe = True 1808eb8dc403SDave Cobbley else: 1809eb8dc403SDave Cobbley logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1810eb8dc403SDave Cobbley _move_file(patchfn, path, 1811eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1812eb8dc403SDave Cobbley updatefiles = True 1813eb8dc403SDave Cobbley # Add any new files 1814eb8dc403SDave Cobbley for basepath, path in new_f.items(): 1815eb8dc403SDave Cobbley logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1816eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), 1817eb8dc403SDave Cobbley os.path.join(files_dir, basepath), 1818eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, 1819eb8dc403SDave Cobbley base_outdir=recipedir) 1820da295319SPatrick Williams srcuri.append(srcuri_entry(basepath, patchdir_params)) 1821eb8dc403SDave Cobbley updaterecipe = True 1822da295319SPatrick Williams for basepath, param in new_p.items(): 1823da295319SPatrick Williams patchdir = param.get('patchdir', ".") 1824eb8dc403SDave Cobbley logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1825da295319SPatrick Williams _move_file(os.path.join(patches_dir, patchdir, basepath), 1826eb8dc403SDave Cobbley os.path.join(files_dir, basepath), 1827eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, 1828eb8dc403SDave Cobbley base_outdir=recipedir) 1829da295319SPatrick Williams params = dict(patchdir_params) 1830da295319SPatrick Williams if patchdir != "." : 1831da295319SPatrick Williams if params: 1832da295319SPatrick Williams params['patchdir'] += patchdir 1833da295319SPatrick Williams else: 1834da295319SPatrick Williams params['patchdir'] = patchdir 1835da295319SPatrick Williams 1836da295319SPatrick Williams srcuri.append(srcuri_entry(basepath, params)) 1837eb8dc403SDave Cobbley updaterecipe = True 1838eb8dc403SDave Cobbley # Update recipe, if needed 1839eb8dc403SDave Cobbley if _remove_file_entries(srcuri, remove_files)[0]: 1840eb8dc403SDave Cobbley updaterecipe = True 1841eb8dc403SDave Cobbley if updaterecipe: 1842eb8dc403SDave Cobbley if not dry_run_outdir: 1843eb8dc403SDave Cobbley logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1844eb8dc403SDave Cobbley ret = oe.recipeutils.patch_recipe(rd, recipefile, 1845eb8dc403SDave Cobbley {'SRC_URI': ' '.join(srcuri)}, 1846eb8dc403SDave Cobbley redirect_output=dry_run_outdir) 1847eb8dc403SDave Cobbley elif not updatefiles: 1848eb8dc403SDave Cobbley # Neither patches nor recipe were updated 1849eb8dc403SDave Cobbley logger.info('No patches or files need updating') 1850eb8dc403SDave Cobbley return False, None, [] 1851eb8dc403SDave Cobbley finally: 1852eb8dc403SDave Cobbley shutil.rmtree(tempdir) 1853eb8dc403SDave Cobbley 1854eb8dc403SDave Cobbley _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1855eb8dc403SDave Cobbley return True, appendfile, remove_files 1856eb8dc403SDave Cobbley 1857eb8dc403SDave Cobbleydef _guess_recipe_update_mode(srctree, rdata): 1858eb8dc403SDave Cobbley """Guess the recipe update mode to use""" 1859c9f7865aSAndrew Geissler src_uri = (rdata.getVar('SRC_URI') or '').split() 1860eb8dc403SDave Cobbley git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1861eb8dc403SDave Cobbley if not git_uris: 1862eb8dc403SDave Cobbley return 'patch' 1863eb8dc403SDave Cobbley # Just use the first URI for now 1864eb8dc403SDave Cobbley uri = git_uris[0] 1865eb8dc403SDave Cobbley # Check remote branch 1866eb8dc403SDave Cobbley params = bb.fetch.decodeurl(uri)[5] 1867eb8dc403SDave Cobbley upstr_branch = params['branch'] if 'branch' in params else 'master' 1868eb8dc403SDave Cobbley # Check if current branch HEAD is found in upstream branch 1869eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1870eb8dc403SDave Cobbley head_rev = stdout.rstrip() 1871eb8dc403SDave Cobbley stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1872eb8dc403SDave Cobbley cwd=srctree) 1873eb8dc403SDave Cobbley remote_brs = [branch.strip() for branch in stdout.splitlines()] 1874eb8dc403SDave Cobbley if 'origin/' + upstr_branch in remote_brs: 1875eb8dc403SDave Cobbley return 'srcrev' 1876eb8dc403SDave Cobbley 1877eb8dc403SDave Cobbley return 'patch' 1878eb8dc403SDave Cobbley 1879eb8dc403SDave Cobbleydef _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1880eb8dc403SDave Cobbley srctree = workspace[recipename]['srctree'] 1881eb8dc403SDave Cobbley if mode == 'auto': 1882eb8dc403SDave Cobbley mode = _guess_recipe_update_mode(srctree, rd) 1883eb8dc403SDave Cobbley 1884eb8dc403SDave Cobbley override_branches = [] 1885eb8dc403SDave Cobbley mainbranch = None 1886eb8dc403SDave Cobbley startbranch = None 1887eb8dc403SDave Cobbley if not no_overrides: 1888eb8dc403SDave Cobbley stdout, _ = bb.process.run('git branch', cwd=srctree) 1889eb8dc403SDave Cobbley other_branches = [] 1890eb8dc403SDave Cobbley for line in stdout.splitlines(): 1891eb8dc403SDave Cobbley branchname = line[2:] 1892eb8dc403SDave Cobbley if line.startswith('* '): 1893eb8dc403SDave Cobbley startbranch = branchname 1894eb8dc403SDave Cobbley if branchname.startswith(override_branch_prefix): 1895eb8dc403SDave Cobbley override_branches.append(branchname) 1896eb8dc403SDave Cobbley else: 1897eb8dc403SDave Cobbley other_branches.append(branchname) 1898eb8dc403SDave Cobbley 1899eb8dc403SDave Cobbley if override_branches: 1900eb8dc403SDave Cobbley logger.debug('_update_recipe: override branches: %s' % override_branches) 1901eb8dc403SDave Cobbley logger.debug('_update_recipe: other branches: %s' % other_branches) 1902eb8dc403SDave Cobbley if startbranch.startswith(override_branch_prefix): 1903eb8dc403SDave Cobbley if len(other_branches) == 1: 1904eb8dc403SDave Cobbley mainbranch = other_branches[1] 1905eb8dc403SDave Cobbley else: 1906eb8dc403SDave Cobbley raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1907eb8dc403SDave Cobbley else: 1908eb8dc403SDave Cobbley mainbranch = startbranch 1909eb8dc403SDave Cobbley 1910eb8dc403SDave Cobbley checkedout = None 1911eb8dc403SDave Cobbley anyupdated = False 1912eb8dc403SDave Cobbley appendfile = None 1913eb8dc403SDave Cobbley allremoved = [] 1914eb8dc403SDave Cobbley if override_branches: 1915eb8dc403SDave Cobbley logger.info('Handling main branch (%s)...' % mainbranch) 1916eb8dc403SDave Cobbley if startbranch != mainbranch: 1917eb8dc403SDave Cobbley bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1918eb8dc403SDave Cobbley checkedout = mainbranch 1919eb8dc403SDave Cobbley try: 1920eb8dc403SDave Cobbley branchlist = [mainbranch] + override_branches 1921eb8dc403SDave Cobbley for branch in branchlist: 1922eb8dc403SDave Cobbley crd = bb.data.createCopy(rd) 1923eb8dc403SDave Cobbley if branch != mainbranch: 1924eb8dc403SDave Cobbley logger.info('Handling branch %s...' % branch) 1925eb8dc403SDave Cobbley override = branch[len(override_branch_prefix):] 1926eb8dc403SDave Cobbley crd.appendVar('OVERRIDES', ':%s' % override) 1927eb8dc403SDave Cobbley bb.process.run('git checkout %s' % branch, cwd=srctree) 1928eb8dc403SDave Cobbley checkedout = branch 1929eb8dc403SDave Cobbley 1930eb8dc403SDave Cobbley if mode == 'srcrev': 1931eb8dc403SDave Cobbley updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1932eb8dc403SDave Cobbley elif mode == 'patch': 1933eb8dc403SDave Cobbley updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1934eb8dc403SDave Cobbley else: 1935eb8dc403SDave Cobbley raise DevtoolError('update_recipe: invalid mode %s' % mode) 1936eb8dc403SDave Cobbley if updated: 1937eb8dc403SDave Cobbley anyupdated = True 1938eb8dc403SDave Cobbley if appendf: 1939eb8dc403SDave Cobbley appendfile = appendf 1940eb8dc403SDave Cobbley allremoved.extend(removed) 1941eb8dc403SDave Cobbley finally: 1942eb8dc403SDave Cobbley if startbranch and checkedout != startbranch: 1943eb8dc403SDave Cobbley bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1944eb8dc403SDave Cobbley 1945eb8dc403SDave Cobbley return anyupdated, appendfile, allremoved 1946eb8dc403SDave Cobbley 1947eb8dc403SDave Cobbleydef update_recipe(args, config, basepath, workspace): 1948eb8dc403SDave Cobbley """Entry point for the devtool 'update-recipe' subcommand""" 1949eb8dc403SDave Cobbley check_workspace_recipe(workspace, args.recipename) 1950eb8dc403SDave Cobbley 1951eb8dc403SDave Cobbley if args.append: 1952eb8dc403SDave Cobbley if not os.path.exists(args.append): 1953eb8dc403SDave Cobbley raise DevtoolError('bbappend destination layer directory "%s" ' 1954eb8dc403SDave Cobbley 'does not exist' % args.append) 1955eb8dc403SDave Cobbley if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1956eb8dc403SDave Cobbley raise DevtoolError('conf/layer.conf not found in bbappend ' 1957eb8dc403SDave Cobbley 'destination layer "%s"' % args.append) 1958eb8dc403SDave Cobbley 1959eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1960eb8dc403SDave Cobbley try: 1961eb8dc403SDave Cobbley 1962eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 1963eb8dc403SDave Cobbley if not rd: 1964eb8dc403SDave Cobbley return 1 1965eb8dc403SDave Cobbley 1966eb8dc403SDave Cobbley dry_run_output = None 1967eb8dc403SDave Cobbley dry_run_outdir = None 1968eb8dc403SDave Cobbley if args.dry_run: 1969eb8dc403SDave Cobbley dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1970eb8dc403SDave Cobbley dry_run_outdir = dry_run_output.name 1971eb8dc403SDave Cobbley updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1972eb8dc403SDave Cobbley 1973eb8dc403SDave Cobbley if updated: 1974eb8dc403SDave Cobbley rf = rd.getVar('FILE') 1975eb8dc403SDave Cobbley if rf.startswith(config.workspace_path): 19761a4b7ee2SBrad Bishop logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1977eb8dc403SDave Cobbley finally: 1978eb8dc403SDave Cobbley tinfoil.shutdown() 1979eb8dc403SDave Cobbley 1980eb8dc403SDave Cobbley return 0 1981eb8dc403SDave Cobbley 1982eb8dc403SDave Cobbley 1983eb8dc403SDave Cobbleydef status(args, config, basepath, workspace): 1984eb8dc403SDave Cobbley """Entry point for the devtool 'status' subcommand""" 1985eb8dc403SDave Cobbley if workspace: 1986eb8dc403SDave Cobbley for recipe, value in sorted(workspace.items()): 1987eb8dc403SDave Cobbley recipefile = value['recipefile'] 1988eb8dc403SDave Cobbley if recipefile: 1989eb8dc403SDave Cobbley recipestr = ' (%s)' % recipefile 1990eb8dc403SDave Cobbley else: 1991eb8dc403SDave Cobbley recipestr = '' 1992eb8dc403SDave Cobbley print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1993eb8dc403SDave Cobbley else: 1994eb8dc403SDave Cobbley logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1995eb8dc403SDave Cobbley return 0 1996eb8dc403SDave Cobbley 1997eb8dc403SDave Cobbley 199864c979e8SBrad Bishopdef _reset(recipes, no_clean, remove_work, config, basepath, workspace): 1999eb8dc403SDave Cobbley """Reset one or more recipes""" 2000eb8dc403SDave Cobbley import oe.path 2001eb8dc403SDave Cobbley 2002eb8dc403SDave Cobbley def clean_preferred_provider(pn, layerconf_path): 2003eb8dc403SDave Cobbley """Remove PREFERRED_PROVIDER from layer.conf'""" 2004eb8dc403SDave Cobbley import re 2005eb8dc403SDave Cobbley layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 2006eb8dc403SDave Cobbley new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 2007eb8dc403SDave Cobbley pprovider_found = False 2008eb8dc403SDave Cobbley with open(layerconf_file, 'r') as f: 2009eb8dc403SDave Cobbley lines = f.readlines() 2010eb8dc403SDave Cobbley with open(new_layerconf_file, 'a') as nf: 2011eb8dc403SDave Cobbley for line in lines: 2012eb8dc403SDave Cobbley pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 2013eb8dc403SDave Cobbley if not re.match(pprovider_exp, line): 2014eb8dc403SDave Cobbley nf.write(line) 2015eb8dc403SDave Cobbley else: 2016eb8dc403SDave Cobbley pprovider_found = True 2017eb8dc403SDave Cobbley if pprovider_found: 2018eb8dc403SDave Cobbley shutil.move(new_layerconf_file, layerconf_file) 2019eb8dc403SDave Cobbley else: 2020eb8dc403SDave Cobbley os.remove(new_layerconf_file) 2021eb8dc403SDave Cobbley 2022eb8dc403SDave Cobbley if recipes and not no_clean: 2023eb8dc403SDave Cobbley if len(recipes) == 1: 2024eb8dc403SDave Cobbley logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 2025eb8dc403SDave Cobbley else: 2026eb8dc403SDave Cobbley logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 2027eb8dc403SDave Cobbley # If the recipe file itself was created in the workspace, and 2028eb8dc403SDave Cobbley # it uses BBCLASSEXTEND, then we need to also clean the other 2029eb8dc403SDave Cobbley # variants 2030eb8dc403SDave Cobbley targets = [] 2031eb8dc403SDave Cobbley for recipe in recipes: 2032eb8dc403SDave Cobbley targets.append(recipe) 2033eb8dc403SDave Cobbley recipefile = workspace[recipe]['recipefile'] 2034eb8dc403SDave Cobbley if recipefile and os.path.exists(recipefile): 2035eb8dc403SDave Cobbley targets.extend(get_bbclassextend_targets(recipefile, recipe)) 2036eb8dc403SDave Cobbley try: 2037eb8dc403SDave Cobbley exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 2038eb8dc403SDave Cobbley except bb.process.ExecutionError as e: 2039eb8dc403SDave Cobbley raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 2040eb8dc403SDave Cobbley 'wish, you may specify -n/--no-clean to ' 2041eb8dc403SDave Cobbley 'skip running this command when resetting' % 2042eb8dc403SDave Cobbley (e.command, e.stdout)) 2043eb8dc403SDave Cobbley 2044eb8dc403SDave Cobbley for pn in recipes: 2045eb8dc403SDave Cobbley _check_preserve(config, pn) 2046eb8dc403SDave Cobbley 2047eb8dc403SDave Cobbley appendfile = workspace[pn]['bbappend'] 2048eb8dc403SDave Cobbley if os.path.exists(appendfile): 2049eb8dc403SDave Cobbley # This shouldn't happen, but is possible if devtool errored out prior to 2050eb8dc403SDave Cobbley # writing the md5 file. We need to delete this here or the recipe won't 2051eb8dc403SDave Cobbley # actually be reset 2052eb8dc403SDave Cobbley os.remove(appendfile) 2053eb8dc403SDave Cobbley 2054eb8dc403SDave Cobbley preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 2055eb8dc403SDave Cobbley def preservedir(origdir): 2056eb8dc403SDave Cobbley if os.path.exists(origdir): 2057eb8dc403SDave Cobbley for root, dirs, files in os.walk(origdir): 2058eb8dc403SDave Cobbley for fn in files: 20591a4b7ee2SBrad Bishop logger.warning('Preserving %s in %s' % (fn, preservepath)) 2060eb8dc403SDave Cobbley _move_file(os.path.join(origdir, fn), 2061eb8dc403SDave Cobbley os.path.join(preservepath, fn)) 2062eb8dc403SDave Cobbley for dn in dirs: 2063eb8dc403SDave Cobbley preservedir(os.path.join(root, dn)) 2064eb8dc403SDave Cobbley os.rmdir(origdir) 2065eb8dc403SDave Cobbley 2066eb8dc403SDave Cobbley recipefile = workspace[pn]['recipefile'] 2067eb8dc403SDave Cobbley if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 2068eb8dc403SDave Cobbley # This should always be true if recipefile is set, but just in case 2069eb8dc403SDave Cobbley preservedir(os.path.dirname(recipefile)) 2070eb8dc403SDave Cobbley # We don't automatically create this dir next to appends, but the user can 2071eb8dc403SDave Cobbley preservedir(os.path.join(config.workspace_path, 'appends', pn)) 2072eb8dc403SDave Cobbley 2073eb8dc403SDave Cobbley srctreebase = workspace[pn]['srctreebase'] 2074eb8dc403SDave Cobbley if os.path.isdir(srctreebase): 2075eb8dc403SDave Cobbley if os.listdir(srctreebase): 207664c979e8SBrad Bishop if remove_work: 207764c979e8SBrad Bishop logger.info('-r argument used on %s, removing source tree.' 207864c979e8SBrad Bishop ' You will lose any unsaved work' %pn) 207964c979e8SBrad Bishop shutil.rmtree(srctreebase) 208064c979e8SBrad Bishop else: 2081eb8dc403SDave Cobbley # We don't want to risk wiping out any work in progress 208292b42cb3SPatrick Williams if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')): 208392b42cb3SPatrick Williams from datetime import datetime 208492b42cb3SPatrick Williams preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn,datetime.now().strftime("%Y%m%d%H%M%S"))) 208592b42cb3SPatrick Williams logger.info('Preserving source tree in %s\nIf you no ' 208692b42cb3SPatrick Williams 'longer need it then please delete it manually.\n' 208792b42cb3SPatrick Williams 'It is also possible to reuse it via devtool source tree argument.' 208892b42cb3SPatrick Williams % preservesrc) 208992b42cb3SPatrick Williams bb.utils.mkdirhier(os.path.dirname(preservesrc)) 209092b42cb3SPatrick Williams shutil.move(srctreebase, preservesrc) 209192b42cb3SPatrick Williams else: 2092eb8dc403SDave Cobbley logger.info('Leaving source tree %s as-is; if you no ' 2093eb8dc403SDave Cobbley 'longer need it then please delete it manually' 2094eb8dc403SDave Cobbley % srctreebase) 2095eb8dc403SDave Cobbley else: 2096eb8dc403SDave Cobbley # This is unlikely, but if it's empty we can just remove it 2097eb8dc403SDave Cobbley os.rmdir(srctreebase) 2098eb8dc403SDave Cobbley 2099eb8dc403SDave Cobbley clean_preferred_provider(pn, config.workspace_path) 2100eb8dc403SDave Cobbley 2101eb8dc403SDave Cobbleydef reset(args, config, basepath, workspace): 2102eb8dc403SDave Cobbley """Entry point for the devtool 'reset' subcommand""" 2103eb8dc403SDave Cobbley import bb 210464c979e8SBrad Bishop import shutil 210564c979e8SBrad Bishop 210664c979e8SBrad Bishop recipes = "" 210764c979e8SBrad Bishop 2108eb8dc403SDave Cobbley if args.recipename: 2109eb8dc403SDave Cobbley if args.all: 2110eb8dc403SDave Cobbley raise DevtoolError("Recipe cannot be specified if -a/--all is used") 2111eb8dc403SDave Cobbley else: 2112eb8dc403SDave Cobbley for recipe in args.recipename: 2113eb8dc403SDave Cobbley check_workspace_recipe(workspace, recipe, checksrc=False) 2114eb8dc403SDave Cobbley elif not args.all: 2115eb8dc403SDave Cobbley raise DevtoolError("Recipe must be specified, or specify -a/--all to " 2116eb8dc403SDave Cobbley "reset all recipes") 2117eb8dc403SDave Cobbley if args.all: 2118eb8dc403SDave Cobbley recipes = list(workspace.keys()) 2119eb8dc403SDave Cobbley else: 2120eb8dc403SDave Cobbley recipes = args.recipename 2121eb8dc403SDave Cobbley 212264c979e8SBrad Bishop _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace) 2123eb8dc403SDave Cobbley 2124eb8dc403SDave Cobbley return 0 2125eb8dc403SDave Cobbley 2126eb8dc403SDave Cobbley 2127eb8dc403SDave Cobbleydef _get_layer(layername, d): 2128eb8dc403SDave Cobbley """Determine the base layer path for the specified layer name/path""" 2129eb8dc403SDave Cobbley layerdirs = d.getVar('BBLAYERS').split() 213096ff1984SBrad Bishop layers = {} # {basename: layer_paths} 213196ff1984SBrad Bishop for p in layerdirs: 213296ff1984SBrad Bishop bn = os.path.basename(p) 213396ff1984SBrad Bishop if bn not in layers: 213496ff1984SBrad Bishop layers[bn] = [p] 213596ff1984SBrad Bishop else: 213696ff1984SBrad Bishop layers[bn].append(p) 2137eb8dc403SDave Cobbley # Provide some shortcuts 2138eb8dc403SDave Cobbley if layername.lower() in ['oe-core', 'openembedded-core']: 213996ff1984SBrad Bishop layername = 'meta' 214096ff1984SBrad Bishop layer_paths = layers.get(layername, None) 214196ff1984SBrad Bishop if not layer_paths: 214296ff1984SBrad Bishop return os.path.abspath(layername) 214396ff1984SBrad Bishop elif len(layer_paths) == 1: 214496ff1984SBrad Bishop return os.path.abspath(layer_paths[0]) 2145eb8dc403SDave Cobbley else: 214696ff1984SBrad Bishop # multiple layers having the same base name 214796ff1984SBrad Bishop logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0])) 214896ff1984SBrad Bishop logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths)) 214996ff1984SBrad Bishop return os.path.abspath(layer_paths[0]) 215096ff1984SBrad Bishop 2151eb8dc403SDave Cobbley 2152eb8dc403SDave Cobbleydef finish(args, config, basepath, workspace): 2153eb8dc403SDave Cobbley """Entry point for the devtool 'finish' subcommand""" 2154eb8dc403SDave Cobbley import bb 2155eb8dc403SDave Cobbley import oe.recipeutils 2156eb8dc403SDave Cobbley 2157eb8dc403SDave Cobbley check_workspace_recipe(workspace, args.recipename) 2158eb8dc403SDave Cobbley 2159eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if args.dry_run else '' 2160eb8dc403SDave Cobbley 2161eb8dc403SDave Cobbley # Grab the equivalent of COREBASE without having to initialise tinfoil 2162eb8dc403SDave Cobbley corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 2163eb8dc403SDave Cobbley 2164eb8dc403SDave Cobbley srctree = workspace[args.recipename]['srctree'] 2165eb8dc403SDave Cobbley check_git_repo_op(srctree, [corebasedir]) 2166eb8dc403SDave Cobbley dirty = check_git_repo_dirty(srctree) 2167eb8dc403SDave Cobbley if dirty: 2168eb8dc403SDave Cobbley if args.force: 2169eb8dc403SDave Cobbley logger.warning('Source tree is not clean, continuing as requested by -f/--force') 2170eb8dc403SDave Cobbley else: 2171eb8dc403SDave Cobbley raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 2172eb8dc403SDave Cobbley 217300e122a7SBrad Bishop no_clean = args.no_clean 217464c979e8SBrad Bishop remove_work=args.remove_work 2175eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 2176eb8dc403SDave Cobbley try: 21776dbb316aSBrad Bishop rd = parse_recipe(config, tinfoil, args.recipename, True) 2178eb8dc403SDave Cobbley if not rd: 2179eb8dc403SDave Cobbley return 1 2180eb8dc403SDave Cobbley 2181eb8dc403SDave Cobbley destlayerdir = _get_layer(args.destination, tinfoil.config_data) 2182eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 2183eb8dc403SDave Cobbley recipedir = os.path.dirname(recipefile) 2184eb8dc403SDave Cobbley origlayerdir = oe.recipeutils.find_layerdir(recipefile) 2185eb8dc403SDave Cobbley 2186eb8dc403SDave Cobbley if not os.path.isdir(destlayerdir): 2187eb8dc403SDave Cobbley raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 2188eb8dc403SDave Cobbley 2189eb8dc403SDave Cobbley if os.path.abspath(destlayerdir) == config.workspace_path: 2190eb8dc403SDave Cobbley raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 2191eb8dc403SDave Cobbley 2192eb8dc403SDave Cobbley # If it's an upgrade, grab the original path 2193eb8dc403SDave Cobbley origpath = None 2194eb8dc403SDave Cobbley origfilelist = None 2195eb8dc403SDave Cobbley append = workspace[args.recipename]['bbappend'] 2196eb8dc403SDave Cobbley with open(append, 'r') as f: 2197eb8dc403SDave Cobbley for line in f: 2198eb8dc403SDave Cobbley if line.startswith('# original_path:'): 2199eb8dc403SDave Cobbley origpath = line.split(':')[1].strip() 2200eb8dc403SDave Cobbley elif line.startswith('# original_files:'): 2201eb8dc403SDave Cobbley origfilelist = line.split(':')[1].split() 2202eb8dc403SDave Cobbley 2203eb8dc403SDave Cobbley destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 2204eb8dc403SDave Cobbley 2205eb8dc403SDave Cobbley if origlayerdir == config.workspace_path: 2206eb8dc403SDave Cobbley # Recipe file itself is in workspace, update it there first 2207eb8dc403SDave Cobbley appendlayerdir = None 2208eb8dc403SDave Cobbley origrelpath = None 2209eb8dc403SDave Cobbley if origpath: 2210eb8dc403SDave Cobbley origlayerpath = oe.recipeutils.find_layerdir(origpath) 2211eb8dc403SDave Cobbley if origlayerpath: 2212eb8dc403SDave Cobbley origrelpath = os.path.relpath(origpath, origlayerpath) 2213eb8dc403SDave Cobbley destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 2214eb8dc403SDave Cobbley if not destpath: 2215eb8dc403SDave Cobbley raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 2216eb8dc403SDave Cobbley # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 2217eb8dc403SDave Cobbley layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 2218eb8dc403SDave Cobbley if not os.path.abspath(destlayerbasedir) in layerdirs: 2219eb8dc403SDave Cobbley bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 2220eb8dc403SDave Cobbley 2221eb8dc403SDave Cobbley elif destlayerdir == origlayerdir: 2222eb8dc403SDave Cobbley # Same layer, update the original recipe 2223eb8dc403SDave Cobbley appendlayerdir = None 2224eb8dc403SDave Cobbley destpath = None 2225eb8dc403SDave Cobbley else: 2226eb8dc403SDave Cobbley # Create/update a bbappend in the specified layer 2227eb8dc403SDave Cobbley appendlayerdir = destlayerdir 2228eb8dc403SDave Cobbley destpath = None 2229eb8dc403SDave Cobbley 2230eb8dc403SDave Cobbley # Actually update the recipe / bbappend 2231eb8dc403SDave Cobbley removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 2232eb8dc403SDave Cobbley dry_run_output = None 2233eb8dc403SDave Cobbley dry_run_outdir = None 2234eb8dc403SDave Cobbley if args.dry_run: 2235eb8dc403SDave Cobbley dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 2236eb8dc403SDave Cobbley dry_run_outdir = dry_run_output.name 2237eb8dc403SDave Cobbley updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 2238eb8dc403SDave Cobbley removed = [os.path.relpath(pth, recipedir) for pth in removed] 2239eb8dc403SDave Cobbley 2240eb8dc403SDave Cobbley # Remove any old files in the case of an upgrade 2241eb8dc403SDave Cobbley if removing_original: 2242eb8dc403SDave Cobbley for fn in origfilelist: 2243eb8dc403SDave Cobbley fnp = os.path.join(origpath, fn) 2244eb8dc403SDave Cobbley if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 2245eb8dc403SDave Cobbley logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 2246eb8dc403SDave Cobbley if not args.dry_run: 2247eb8dc403SDave Cobbley try: 2248eb8dc403SDave Cobbley os.remove(fnp) 2249eb8dc403SDave Cobbley except FileNotFoundError: 2250eb8dc403SDave Cobbley pass 2251eb8dc403SDave Cobbley 2252eb8dc403SDave Cobbley if origlayerdir == config.workspace_path and destpath: 2253eb8dc403SDave Cobbley # Recipe file itself is in the workspace - need to move it and any 2254eb8dc403SDave Cobbley # associated files to the specified layer 2255eb8dc403SDave Cobbley no_clean = True 2256eb8dc403SDave Cobbley logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 2257eb8dc403SDave Cobbley for root, _, files in os.walk(recipedir): 2258eb8dc403SDave Cobbley for fn in files: 2259eb8dc403SDave Cobbley srcpath = os.path.join(root, fn) 2260eb8dc403SDave Cobbley relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 2261eb8dc403SDave Cobbley destdir = os.path.abspath(os.path.join(destpath, relpth)) 2262eb8dc403SDave Cobbley destfp = os.path.join(destdir, fn) 2263eb8dc403SDave Cobbley _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 2264eb8dc403SDave Cobbley 2265eb8dc403SDave Cobbley if dry_run_outdir: 2266eb8dc403SDave Cobbley import difflib 2267eb8dc403SDave Cobbley comparelist = [] 2268eb8dc403SDave Cobbley for root, _, files in os.walk(dry_run_outdir): 2269eb8dc403SDave Cobbley for fn in files: 2270eb8dc403SDave Cobbley outf = os.path.join(root, fn) 2271eb8dc403SDave Cobbley relf = os.path.relpath(outf, dry_run_outdir) 2272eb8dc403SDave Cobbley logger.debug('dry-run: output file %s' % relf) 2273eb8dc403SDave Cobbley if fn.endswith('.bb'): 2274eb8dc403SDave Cobbley if origfilelist and origpath and destpath: 2275eb8dc403SDave Cobbley # Need to match this up with the pre-upgrade recipe file 2276eb8dc403SDave Cobbley for origf in origfilelist: 2277eb8dc403SDave Cobbley if origf.endswith('.bb'): 2278eb8dc403SDave Cobbley comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2279eb8dc403SDave Cobbley outf, 2280eb8dc403SDave Cobbley os.path.abspath(os.path.join(destpath, relf)))) 2281eb8dc403SDave Cobbley break 2282eb8dc403SDave Cobbley else: 2283eb8dc403SDave Cobbley # Compare to the existing recipe 2284eb8dc403SDave Cobbley comparelist.append((recipefile, outf, recipefile)) 2285eb8dc403SDave Cobbley elif fn.endswith('.bbappend'): 2286eb8dc403SDave Cobbley if appendfile: 2287eb8dc403SDave Cobbley if os.path.exists(appendfile): 2288eb8dc403SDave Cobbley comparelist.append((appendfile, outf, appendfile)) 2289eb8dc403SDave Cobbley else: 2290eb8dc403SDave Cobbley comparelist.append((None, outf, appendfile)) 2291eb8dc403SDave Cobbley else: 2292eb8dc403SDave Cobbley if destpath: 2293eb8dc403SDave Cobbley recipedest = destpath 2294eb8dc403SDave Cobbley elif appendfile: 2295eb8dc403SDave Cobbley recipedest = os.path.dirname(appendfile) 2296eb8dc403SDave Cobbley else: 2297eb8dc403SDave Cobbley recipedest = os.path.dirname(recipefile) 2298eb8dc403SDave Cobbley destfp = os.path.join(recipedest, relf) 2299eb8dc403SDave Cobbley if os.path.exists(destfp): 2300eb8dc403SDave Cobbley comparelist.append((destfp, outf, destfp)) 2301eb8dc403SDave Cobbley output = '' 2302eb8dc403SDave Cobbley for oldfile, newfile, newfileshow in comparelist: 2303eb8dc403SDave Cobbley if oldfile: 2304eb8dc403SDave Cobbley with open(oldfile, 'r') as f: 2305eb8dc403SDave Cobbley oldlines = f.readlines() 2306eb8dc403SDave Cobbley else: 2307eb8dc403SDave Cobbley oldfile = '/dev/null' 2308eb8dc403SDave Cobbley oldlines = [] 2309eb8dc403SDave Cobbley with open(newfile, 'r') as f: 2310eb8dc403SDave Cobbley newlines = f.readlines() 2311eb8dc403SDave Cobbley if not newfileshow: 2312eb8dc403SDave Cobbley newfileshow = newfile 2313eb8dc403SDave Cobbley diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2314eb8dc403SDave Cobbley difflines = list(diff) 2315eb8dc403SDave Cobbley if difflines: 2316eb8dc403SDave Cobbley output += ''.join(difflines) 2317eb8dc403SDave Cobbley if output: 2318eb8dc403SDave Cobbley logger.info('Diff of changed files:\n%s' % output) 2319eb8dc403SDave Cobbley finally: 2320eb8dc403SDave Cobbley tinfoil.shutdown() 2321eb8dc403SDave Cobbley 2322eb8dc403SDave Cobbley # Everything else has succeeded, we can now reset 2323eb8dc403SDave Cobbley if args.dry_run: 2324eb8dc403SDave Cobbley logger.info('Resetting recipe (dry-run)') 2325eb8dc403SDave Cobbley else: 232664c979e8SBrad Bishop _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace) 2327eb8dc403SDave Cobbley 2328eb8dc403SDave Cobbley return 0 2329eb8dc403SDave Cobbley 2330eb8dc403SDave Cobbley 2331eb8dc403SDave Cobbleydef get_default_srctree(config, recipename=''): 2332eb8dc403SDave Cobbley """Get the default srctree path""" 2333eb8dc403SDave Cobbley srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2334eb8dc403SDave Cobbley if recipename: 2335eb8dc403SDave Cobbley return os.path.join(srctreeparent, 'sources', recipename) 2336eb8dc403SDave Cobbley else: 2337eb8dc403SDave Cobbley return os.path.join(srctreeparent, 'sources') 2338eb8dc403SDave Cobbley 2339eb8dc403SDave Cobbleydef register_commands(subparsers, context): 2340eb8dc403SDave Cobbley """Register devtool subcommands from this plugin""" 2341eb8dc403SDave Cobbley 2342eb8dc403SDave Cobbley defsrctree = get_default_srctree(context.config) 2343eb8dc403SDave Cobbley parser_add = subparsers.add_parser('add', help='Add a new recipe', 2344eb8dc403SDave Cobbley description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2345eb8dc403SDave Cobbley group='starting', order=100) 2346eb8dc403SDave Cobbley parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2347eb8dc403SDave Cobbley parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2348eb8dc403SDave Cobbley parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2349eb8dc403SDave Cobbley group = parser_add.add_mutually_exclusive_group() 2350eb8dc403SDave Cobbley group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2351eb8dc403SDave Cobbley group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2352eb8dc403SDave Cobbley parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 235382c905dcSAndrew Geissler parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2354169d7bccSPatrick Williams parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true") 2355eb8dc403SDave Cobbley parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2356eb8dc403SDave Cobbley parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2357eb8dc403SDave Cobbley group = parser_add.add_mutually_exclusive_group() 2358eb8dc403SDave Cobbley group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2359eb8dc403SDave Cobbley group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2360eb8dc403SDave Cobbley parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2361eb8dc403SDave Cobbley parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2362eb8dc403SDave Cobbley parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2363eb8dc403SDave Cobbley parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2364eb8dc403SDave Cobbley parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2365eb8dc403SDave Cobbley parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2366eb8dc403SDave Cobbley parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2367eb8dc403SDave Cobbley 2368eb8dc403SDave Cobbley parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2369eb8dc403SDave Cobbley description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2370eb8dc403SDave Cobbley group='starting', order=90) 2371eb8dc403SDave Cobbley parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2372eb8dc403SDave Cobbley parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2373eb8dc403SDave Cobbley parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2374eb8dc403SDave Cobbley group = parser_modify.add_mutually_exclusive_group() 2375eb8dc403SDave Cobbley group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2376eb8dc403SDave Cobbley group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2377eb8dc403SDave Cobbley group = parser_modify.add_mutually_exclusive_group() 2378eb8dc403SDave Cobbley group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2379eb8dc403SDave Cobbley group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2380eb8dc403SDave Cobbley parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2381eb8dc403SDave Cobbley parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2382eb8dc403SDave Cobbley parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2383eb8dc403SDave Cobbley parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2384eb8dc403SDave Cobbley 2385eb8dc403SDave Cobbley parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2386eb8dc403SDave Cobbley description='Extracts the source for an existing recipe', 2387eb8dc403SDave Cobbley group='advanced') 2388eb8dc403SDave Cobbley parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2389eb8dc403SDave Cobbley parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2390eb8dc403SDave Cobbley parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2391eb8dc403SDave Cobbley parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2392eb8dc403SDave Cobbley parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2393eb8dc403SDave Cobbley parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2394eb8dc403SDave Cobbley 2395eb8dc403SDave Cobbley parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2396eb8dc403SDave Cobbley description='Synchronize the previously extracted source tree for an existing recipe', 2397eb8dc403SDave Cobbley formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2398eb8dc403SDave Cobbley group='advanced') 2399eb8dc403SDave Cobbley parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2400eb8dc403SDave Cobbley parser_sync.add_argument('srctree', help='Path to the source tree') 2401eb8dc403SDave Cobbley parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2402eb8dc403SDave Cobbley parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2403eb8dc403SDave Cobbley parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2404eb8dc403SDave Cobbley 2405eb8dc403SDave Cobbley parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2406eb8dc403SDave Cobbley description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2407eb8dc403SDave Cobbley group='working', order=10) 2408eb8dc403SDave Cobbley parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2409eb8dc403SDave Cobbley parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2410eb8dc403SDave Cobbley parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2411eb8dc403SDave Cobbley parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2412eb8dc403SDave Cobbley parser_rename.set_defaults(func=rename) 2413eb8dc403SDave Cobbley 2414eb8dc403SDave Cobbley parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2415eb8dc403SDave Cobbley description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2416eb8dc403SDave Cobbley group='working', order=-90) 2417eb8dc403SDave Cobbley parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2418eb8dc403SDave Cobbley parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2419eb8dc403SDave Cobbley parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2420eb8dc403SDave Cobbley parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2421eb8dc403SDave Cobbley parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2422eb8dc403SDave Cobbley parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2423eb8dc403SDave Cobbley parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2424eb8dc403SDave Cobbley parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2425eb8dc403SDave Cobbley parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2426eb8dc403SDave Cobbley parser_update_recipe.set_defaults(func=update_recipe) 2427eb8dc403SDave Cobbley 2428eb8dc403SDave Cobbley parser_status = subparsers.add_parser('status', help='Show workspace status', 2429eb8dc403SDave Cobbley description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2430eb8dc403SDave Cobbley group='info', order=100) 2431eb8dc403SDave Cobbley parser_status.set_defaults(func=status) 2432eb8dc403SDave Cobbley 2433eb8dc403SDave Cobbley parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2434eb8dc403SDave Cobbley description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2435eb8dc403SDave Cobbley group='working', order=-100) 2436eb8dc403SDave Cobbley parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2437eb8dc403SDave Cobbley parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2438eb8dc403SDave Cobbley parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 243964c979e8SBrad Bishop parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append') 2440eb8dc403SDave Cobbley parser_reset.set_defaults(func=reset) 2441eb8dc403SDave Cobbley 2442eb8dc403SDave Cobbley parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2443eb8dc403SDave Cobbley description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2444eb8dc403SDave Cobbley group='working', order=-100) 2445eb8dc403SDave Cobbley parser_finish.add_argument('recipename', help='Recipe to finish') 2446eb8dc403SDave Cobbley parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2447eb8dc403SDave Cobbley parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2448eb8dc403SDave Cobbley parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2449eb8dc403SDave Cobbley parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 245064c979e8SBrad Bishop parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace') 245100e122a7SBrad Bishop parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2452eb8dc403SDave Cobbley parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2453eb8dc403SDave Cobbley parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2454eb8dc403SDave Cobbley parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2455eb8dc403SDave Cobbley parser_finish.set_defaults(func=finish) 2456