1eb8dc403SDave Cobbley# Development tool - standard commands plugin 2eb8dc403SDave Cobbley# 3eb8dc403SDave Cobbley# Copyright (C) 2014-2017 Intel Corporation 4eb8dc403SDave Cobbley# 5eb8dc403SDave Cobbley# This program is free software; you can redistribute it and/or modify 6eb8dc403SDave Cobbley# it under the terms of the GNU General Public License version 2 as 7eb8dc403SDave Cobbley# published by the Free Software Foundation. 8eb8dc403SDave Cobbley# 9eb8dc403SDave Cobbley# This program is distributed in the hope that it will be useful, 10eb8dc403SDave Cobbley# but WITHOUT ANY WARRANTY; without even the implied warranty of 11eb8dc403SDave Cobbley# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12eb8dc403SDave Cobbley# GNU General Public License for more details. 13eb8dc403SDave Cobbley# 14eb8dc403SDave Cobbley# You should have received a copy of the GNU General Public License along 15eb8dc403SDave Cobbley# with this program; if not, write to the Free Software Foundation, Inc., 16eb8dc403SDave Cobbley# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17eb8dc403SDave Cobbley"""Devtool standard plugins""" 18eb8dc403SDave Cobbley 19eb8dc403SDave Cobbleyimport os 20eb8dc403SDave Cobbleyimport sys 21eb8dc403SDave Cobbleyimport re 22eb8dc403SDave Cobbleyimport shutil 23eb8dc403SDave Cobbleyimport subprocess 24eb8dc403SDave Cobbleyimport tempfile 25eb8dc403SDave Cobbleyimport logging 26eb8dc403SDave Cobbleyimport argparse 27eb8dc403SDave Cobbleyimport argparse_oe 28eb8dc403SDave Cobbleyimport scriptutils 29eb8dc403SDave Cobbleyimport errno 30eb8dc403SDave Cobbleyimport glob 31eb8dc403SDave Cobbleyimport filecmp 32eb8dc403SDave Cobbleyfrom collections import OrderedDict 33eb8dc403SDave Cobbleyfrom devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError 34eb8dc403SDave Cobbleyfrom devtool import parse_recipe 35eb8dc403SDave Cobbley 36eb8dc403SDave Cobbleylogger = logging.getLogger('devtool') 37eb8dc403SDave Cobbley 38eb8dc403SDave Cobbleyoverride_branch_prefix = 'devtool-override-' 39eb8dc403SDave Cobbley 40eb8dc403SDave Cobbley 41eb8dc403SDave Cobbleydef add(args, config, basepath, workspace): 42eb8dc403SDave Cobbley """Entry point for the devtool 'add' subcommand""" 43eb8dc403SDave Cobbley import bb 44eb8dc403SDave Cobbley import oe.recipeutils 45eb8dc403SDave Cobbley 46eb8dc403SDave Cobbley if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri: 47eb8dc403SDave Cobbley raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add') 48eb8dc403SDave Cobbley 49eb8dc403SDave Cobbley # These are positional arguments, but because we're nice, allow 50eb8dc403SDave Cobbley # specifying e.g. source tree without name, or fetch URI without name or 51eb8dc403SDave Cobbley # source tree (if we can detect that that is what the user meant) 52eb8dc403SDave Cobbley if scriptutils.is_src_url(args.recipename): 53eb8dc403SDave Cobbley if not args.fetchuri: 54eb8dc403SDave Cobbley if args.fetch: 55eb8dc403SDave Cobbley raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 56eb8dc403SDave Cobbley args.fetchuri = args.recipename 57eb8dc403SDave Cobbley args.recipename = '' 58eb8dc403SDave Cobbley elif scriptutils.is_src_url(args.srctree): 59eb8dc403SDave Cobbley if not args.fetchuri: 60eb8dc403SDave Cobbley if args.fetch: 61eb8dc403SDave Cobbley raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 62eb8dc403SDave Cobbley args.fetchuri = args.srctree 63eb8dc403SDave Cobbley args.srctree = '' 64eb8dc403SDave Cobbley elif args.recipename and not args.srctree: 65eb8dc403SDave Cobbley if os.sep in args.recipename: 66eb8dc403SDave Cobbley args.srctree = args.recipename 67eb8dc403SDave Cobbley args.recipename = None 68eb8dc403SDave Cobbley elif os.path.isdir(args.recipename): 691a4b7ee2SBrad Bishop logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename) 70eb8dc403SDave Cobbley 71eb8dc403SDave Cobbley if not args.fetchuri: 72eb8dc403SDave Cobbley if args.srcrev: 73eb8dc403SDave Cobbley raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository') 74eb8dc403SDave Cobbley if args.srcbranch: 75eb8dc403SDave Cobbley raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository') 76eb8dc403SDave Cobbley 77eb8dc403SDave Cobbley if args.srctree and os.path.isfile(args.srctree): 78eb8dc403SDave Cobbley args.fetchuri = 'file://' + os.path.abspath(args.srctree) 79eb8dc403SDave Cobbley args.srctree = '' 80eb8dc403SDave Cobbley 81eb8dc403SDave Cobbley if args.fetch: 82eb8dc403SDave Cobbley if args.fetchuri: 83eb8dc403SDave Cobbley raise DevtoolError('URI specified as positional argument as well as -f/--fetch') 84eb8dc403SDave Cobbley else: 851a4b7ee2SBrad Bishop logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead') 86eb8dc403SDave Cobbley args.fetchuri = args.fetch 87eb8dc403SDave Cobbley 88eb8dc403SDave Cobbley if args.recipename: 89eb8dc403SDave Cobbley if args.recipename in workspace: 90eb8dc403SDave Cobbley raise DevtoolError("recipe %s is already in your workspace" % 91eb8dc403SDave Cobbley args.recipename) 92eb8dc403SDave Cobbley reason = oe.recipeutils.validate_pn(args.recipename) 93eb8dc403SDave Cobbley if reason: 94eb8dc403SDave Cobbley raise DevtoolError(reason) 95eb8dc403SDave Cobbley 96eb8dc403SDave Cobbley if args.srctree: 97eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 98eb8dc403SDave Cobbley srctreeparent = None 99eb8dc403SDave Cobbley tmpsrcdir = None 100eb8dc403SDave Cobbley else: 101eb8dc403SDave Cobbley srctree = None 102eb8dc403SDave Cobbley srctreeparent = get_default_srctree(config) 103eb8dc403SDave Cobbley bb.utils.mkdirhier(srctreeparent) 104eb8dc403SDave Cobbley tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent) 105eb8dc403SDave Cobbley 106eb8dc403SDave Cobbley if srctree and os.path.exists(srctree): 107eb8dc403SDave Cobbley if args.fetchuri: 108eb8dc403SDave Cobbley if not os.path.isdir(srctree): 109eb8dc403SDave Cobbley raise DevtoolError("Cannot fetch into source tree path %s as " 110eb8dc403SDave Cobbley "it exists and is not a directory" % 111eb8dc403SDave Cobbley srctree) 112eb8dc403SDave Cobbley elif os.listdir(srctree): 113eb8dc403SDave Cobbley raise DevtoolError("Cannot fetch into source tree path %s as " 114eb8dc403SDave Cobbley "it already exists and is non-empty" % 115eb8dc403SDave Cobbley srctree) 116eb8dc403SDave Cobbley elif not args.fetchuri: 117eb8dc403SDave Cobbley if args.srctree: 118eb8dc403SDave Cobbley raise DevtoolError("Specified source tree %s could not be found" % 119eb8dc403SDave Cobbley args.srctree) 120eb8dc403SDave Cobbley elif srctree: 121eb8dc403SDave Cobbley raise DevtoolError("No source tree exists at default path %s - " 122eb8dc403SDave Cobbley "either create and populate this directory, " 123eb8dc403SDave Cobbley "or specify a path to a source tree, or a " 124eb8dc403SDave Cobbley "URI to fetch source from" % srctree) 125eb8dc403SDave Cobbley else: 126eb8dc403SDave Cobbley raise DevtoolError("You must either specify a source tree " 127eb8dc403SDave Cobbley "or a URI to fetch source from") 128eb8dc403SDave Cobbley 129eb8dc403SDave Cobbley if args.version: 130eb8dc403SDave Cobbley if '_' in args.version or ' ' in args.version: 131eb8dc403SDave Cobbley raise DevtoolError('Invalid version string "%s"' % args.version) 132eb8dc403SDave Cobbley 133eb8dc403SDave Cobbley if args.color == 'auto' and sys.stdout.isatty(): 134eb8dc403SDave Cobbley color = 'always' 135eb8dc403SDave Cobbley else: 136eb8dc403SDave Cobbley color = args.color 137eb8dc403SDave Cobbley extracmdopts = '' 138eb8dc403SDave Cobbley if args.fetchuri: 139eb8dc403SDave Cobbley source = args.fetchuri 140eb8dc403SDave Cobbley if srctree: 141eb8dc403SDave Cobbley extracmdopts += ' -x %s' % srctree 142eb8dc403SDave Cobbley else: 143eb8dc403SDave Cobbley extracmdopts += ' -x %s' % tmpsrcdir 144eb8dc403SDave Cobbley else: 145eb8dc403SDave Cobbley source = srctree 146eb8dc403SDave Cobbley if args.recipename: 147eb8dc403SDave Cobbley extracmdopts += ' -N %s' % args.recipename 148eb8dc403SDave Cobbley if args.version: 149eb8dc403SDave Cobbley extracmdopts += ' -V %s' % args.version 150eb8dc403SDave Cobbley if args.binary: 151eb8dc403SDave Cobbley extracmdopts += ' -b' 152eb8dc403SDave Cobbley if args.also_native: 153eb8dc403SDave Cobbley extracmdopts += ' --also-native' 154eb8dc403SDave Cobbley if args.src_subdir: 155eb8dc403SDave Cobbley extracmdopts += ' --src-subdir "%s"' % args.src_subdir 156eb8dc403SDave Cobbley if args.autorev: 157eb8dc403SDave Cobbley extracmdopts += ' -a' 158eb8dc403SDave Cobbley if args.fetch_dev: 159eb8dc403SDave Cobbley extracmdopts += ' --fetch-dev' 160eb8dc403SDave Cobbley if args.mirrors: 161eb8dc403SDave Cobbley extracmdopts += ' --mirrors' 162eb8dc403SDave Cobbley if args.srcrev: 163eb8dc403SDave Cobbley extracmdopts += ' --srcrev %s' % args.srcrev 164eb8dc403SDave Cobbley if args.srcbranch: 165eb8dc403SDave Cobbley extracmdopts += ' --srcbranch %s' % args.srcbranch 166eb8dc403SDave Cobbley if args.provides: 167eb8dc403SDave Cobbley extracmdopts += ' --provides %s' % args.provides 168eb8dc403SDave Cobbley 169eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtool') 170eb8dc403SDave Cobbley try: 171eb8dc403SDave Cobbley try: 172eb8dc403SDave Cobbley stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) 173eb8dc403SDave Cobbley except bb.process.ExecutionError as e: 174eb8dc403SDave Cobbley if e.exitcode == 15: 175eb8dc403SDave Cobbley raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') 176eb8dc403SDave Cobbley else: 177eb8dc403SDave Cobbley raise DevtoolError('Command \'%s\' failed' % e.command) 178eb8dc403SDave Cobbley 179eb8dc403SDave Cobbley recipes = glob.glob(os.path.join(tempdir, '*.bb')) 180eb8dc403SDave Cobbley if recipes: 181eb8dc403SDave Cobbley recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0] 182eb8dc403SDave Cobbley if recipename in workspace: 183eb8dc403SDave Cobbley raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename) 184eb8dc403SDave Cobbley recipedir = os.path.join(config.workspace_path, 'recipes', recipename) 185eb8dc403SDave Cobbley bb.utils.mkdirhier(recipedir) 186eb8dc403SDave Cobbley recipefile = os.path.join(recipedir, os.path.basename(recipes[0])) 187eb8dc403SDave Cobbley appendfile = recipe_to_append(recipefile, config) 188eb8dc403SDave Cobbley if os.path.exists(appendfile): 189eb8dc403SDave Cobbley # This shouldn't be possible, but just in case 190eb8dc403SDave Cobbley raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace') 191eb8dc403SDave Cobbley if os.path.exists(recipefile): 192eb8dc403SDave Cobbley raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile) 193eb8dc403SDave Cobbley if tmpsrcdir: 194eb8dc403SDave Cobbley srctree = os.path.join(srctreeparent, recipename) 195eb8dc403SDave Cobbley if os.path.exists(tmpsrcdir): 196eb8dc403SDave Cobbley if os.path.exists(srctree): 197eb8dc403SDave Cobbley if os.path.isdir(srctree): 198eb8dc403SDave Cobbley try: 199eb8dc403SDave Cobbley os.rmdir(srctree) 200eb8dc403SDave Cobbley except OSError as e: 201eb8dc403SDave Cobbley if e.errno == errno.ENOTEMPTY: 202eb8dc403SDave Cobbley raise DevtoolError('Source tree path %s already exists and is not empty' % srctree) 203eb8dc403SDave Cobbley else: 204eb8dc403SDave Cobbley raise 205eb8dc403SDave Cobbley else: 206eb8dc403SDave Cobbley raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree) 207eb8dc403SDave Cobbley logger.info('Using default source tree path %s' % srctree) 208eb8dc403SDave Cobbley shutil.move(tmpsrcdir, srctree) 209eb8dc403SDave Cobbley else: 210eb8dc403SDave Cobbley raise DevtoolError('Couldn\'t find source tree created by recipetool') 211eb8dc403SDave Cobbley bb.utils.mkdirhier(recipedir) 212eb8dc403SDave Cobbley shutil.move(recipes[0], recipefile) 213eb8dc403SDave Cobbley # Move any additional files created by recipetool 214eb8dc403SDave Cobbley for fn in os.listdir(tempdir): 215eb8dc403SDave Cobbley shutil.move(os.path.join(tempdir, fn), recipedir) 216eb8dc403SDave Cobbley else: 217eb8dc403SDave Cobbley raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout)) 218eb8dc403SDave Cobbley attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile)) 219eb8dc403SDave Cobbley if os.path.exists(attic_recipe): 2201a4b7ee2SBrad Bishop logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe) 221eb8dc403SDave Cobbley finally: 222eb8dc403SDave Cobbley if tmpsrcdir and os.path.exists(tmpsrcdir): 223eb8dc403SDave Cobbley shutil.rmtree(tmpsrcdir) 224eb8dc403SDave Cobbley shutil.rmtree(tempdir) 225eb8dc403SDave Cobbley 226eb8dc403SDave Cobbley for fn in os.listdir(recipedir): 227eb8dc403SDave Cobbley _add_md5(config, recipename, os.path.join(recipedir, fn)) 228eb8dc403SDave Cobbley 229eb8dc403SDave Cobbley tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 230eb8dc403SDave Cobbley try: 231eb8dc403SDave Cobbley try: 232eb8dc403SDave Cobbley rd = tinfoil.parse_recipe_file(recipefile, False) 233eb8dc403SDave Cobbley except Exception as e: 234eb8dc403SDave Cobbley logger.error(str(e)) 235eb8dc403SDave Cobbley rd = None 236eb8dc403SDave Cobbley if not rd: 237eb8dc403SDave Cobbley # Parsing failed. We just created this recipe and we shouldn't 238eb8dc403SDave Cobbley # leave it in the workdir or it'll prevent bitbake from starting 239eb8dc403SDave Cobbley movefn = '%s.parsefailed' % recipefile 240eb8dc403SDave Cobbley logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn) 241eb8dc403SDave Cobbley shutil.move(recipefile, movefn) 242eb8dc403SDave Cobbley return 1 243eb8dc403SDave Cobbley 244eb8dc403SDave Cobbley if args.fetchuri and not args.no_git: 245eb8dc403SDave Cobbley setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 246eb8dc403SDave Cobbley 247eb8dc403SDave Cobbley initial_rev = None 248eb8dc403SDave Cobbley if os.path.exists(os.path.join(srctree, '.git')): 249eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 250eb8dc403SDave Cobbley initial_rev = stdout.rstrip() 251eb8dc403SDave Cobbley 252eb8dc403SDave Cobbley if args.src_subdir: 253eb8dc403SDave Cobbley srctree = os.path.join(srctree, args.src_subdir) 254eb8dc403SDave Cobbley 255eb8dc403SDave Cobbley bb.utils.mkdirhier(os.path.dirname(appendfile)) 256eb8dc403SDave Cobbley with open(appendfile, 'w') as f: 257eb8dc403SDave Cobbley f.write('inherit externalsrc\n') 258eb8dc403SDave Cobbley f.write('EXTERNALSRC = "%s"\n' % srctree) 259eb8dc403SDave Cobbley 260eb8dc403SDave Cobbley b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 261eb8dc403SDave Cobbley if b_is_s: 262eb8dc403SDave Cobbley f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 263eb8dc403SDave Cobbley if initial_rev: 264eb8dc403SDave Cobbley f.write('\n# initial_rev: %s\n' % initial_rev) 265eb8dc403SDave Cobbley 266eb8dc403SDave Cobbley if args.binary: 267eb8dc403SDave Cobbley f.write('do_install_append() {\n') 268eb8dc403SDave Cobbley f.write(' rm -rf ${D}/.git\n') 269eb8dc403SDave Cobbley f.write(' rm -f ${D}/singletask.lock\n') 270eb8dc403SDave Cobbley f.write('}\n') 271eb8dc403SDave Cobbley 272eb8dc403SDave Cobbley if bb.data.inherits_class('npm', rd): 273eb8dc403SDave Cobbley f.write('do_install_append() {\n') 274eb8dc403SDave Cobbley f.write(' # Remove files added to source dir by devtool/externalsrc\n') 275eb8dc403SDave Cobbley f.write(' rm -f ${NPM_INSTALLDIR}/singletask.lock\n') 276eb8dc403SDave Cobbley f.write(' rm -rf ${NPM_INSTALLDIR}/.git\n') 277eb8dc403SDave Cobbley f.write(' rm -rf ${NPM_INSTALLDIR}/oe-local-files\n') 278eb8dc403SDave Cobbley f.write(' for symlink in ${EXTERNALSRC_SYMLINKS} ; do\n') 279eb8dc403SDave Cobbley f.write(' rm -f ${NPM_INSTALLDIR}/${symlink%%:*}\n') 280eb8dc403SDave Cobbley f.write(' done\n') 281eb8dc403SDave Cobbley f.write('}\n') 282eb8dc403SDave Cobbley 283eb8dc403SDave Cobbley # Check if the new layer provides recipes whose priorities have been 284eb8dc403SDave Cobbley # overriden by PREFERRED_PROVIDER. 285eb8dc403SDave Cobbley recipe_name = rd.getVar('PN') 286eb8dc403SDave Cobbley provides = rd.getVar('PROVIDES') 287eb8dc403SDave Cobbley # Search every item defined in PROVIDES 288eb8dc403SDave Cobbley for recipe_provided in provides.split(): 289eb8dc403SDave Cobbley preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided 290eb8dc403SDave Cobbley current_pprovider = rd.getVar(preferred_provider) 291eb8dc403SDave Cobbley if current_pprovider and current_pprovider != recipe_name: 292eb8dc403SDave Cobbley if args.fixed_setup: 293eb8dc403SDave Cobbley #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf 294eb8dc403SDave Cobbley layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf") 295eb8dc403SDave Cobbley with open(layerconf_file, 'a') as f: 296eb8dc403SDave Cobbley f.write('%s = "%s"\n' % (preferred_provider, recipe_name)) 297eb8dc403SDave Cobbley else: 2981a4b7ee2SBrad Bishop logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider) 299eb8dc403SDave Cobbley break 300eb8dc403SDave Cobbley 301eb8dc403SDave Cobbley _add_md5(config, recipename, appendfile) 302eb8dc403SDave Cobbley 303eb8dc403SDave Cobbley check_prerelease_version(rd.getVar('PV'), 'devtool add') 304eb8dc403SDave Cobbley 305eb8dc403SDave Cobbley logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile) 306eb8dc403SDave Cobbley 307eb8dc403SDave Cobbley finally: 308eb8dc403SDave Cobbley tinfoil.shutdown() 309eb8dc403SDave Cobbley 310eb8dc403SDave Cobbley return 0 311eb8dc403SDave Cobbley 312eb8dc403SDave Cobbley 313eb8dc403SDave Cobbleydef _check_compatible_recipe(pn, d): 314eb8dc403SDave Cobbley """Check if the recipe is supported by devtool""" 315eb8dc403SDave Cobbley if pn == 'perf': 316eb8dc403SDave Cobbley raise DevtoolError("The perf recipe does not actually check out " 317eb8dc403SDave Cobbley "source and thus cannot be supported by this tool", 318eb8dc403SDave Cobbley 4) 319eb8dc403SDave Cobbley 320eb8dc403SDave Cobbley if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'): 321eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4) 322eb8dc403SDave Cobbley 323eb8dc403SDave Cobbley if bb.data.inherits_class('image', d): 324eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is an image, and therefore is not " 325eb8dc403SDave Cobbley "supported by this tool" % pn, 4) 326eb8dc403SDave Cobbley 327eb8dc403SDave Cobbley if bb.data.inherits_class('populate_sdk', d): 328eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is an SDK, and therefore is not " 329eb8dc403SDave Cobbley "supported by this tool" % pn, 4) 330eb8dc403SDave Cobbley 331eb8dc403SDave Cobbley if bb.data.inherits_class('packagegroup', d): 332eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 333eb8dc403SDave Cobbley "not supported by this tool" % pn, 4) 334eb8dc403SDave Cobbley 335eb8dc403SDave Cobbley if bb.data.inherits_class('meta', d): 336eb8dc403SDave Cobbley raise DevtoolError("The %s recipe is a meta-recipe, and therefore is " 337eb8dc403SDave Cobbley "not supported by this tool" % pn, 4) 338eb8dc403SDave Cobbley 339eb8dc403SDave Cobbley if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 340eb8dc403SDave Cobbley # Not an incompatibility error per se, so we don't pass the error code 341eb8dc403SDave Cobbley raise DevtoolError("externalsrc is currently enabled for the %s " 342eb8dc403SDave Cobbley "recipe. This prevents the normal do_patch task " 343eb8dc403SDave Cobbley "from working. You will need to disable this " 344eb8dc403SDave Cobbley "first." % pn) 345eb8dc403SDave Cobbley 346eb8dc403SDave Cobbleydef _dry_run_copy(src, dst, dry_run_outdir, base_outdir): 347eb8dc403SDave Cobbley """Common function for copying a file to the dry run output directory""" 348eb8dc403SDave Cobbley relpath = os.path.relpath(dst, base_outdir) 349eb8dc403SDave Cobbley if relpath.startswith('..'): 350eb8dc403SDave Cobbley raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst)) 351eb8dc403SDave Cobbley dst = os.path.join(dry_run_outdir, relpath) 352eb8dc403SDave Cobbley dst_d = os.path.dirname(dst) 353eb8dc403SDave Cobbley if dst_d: 354eb8dc403SDave Cobbley bb.utils.mkdirhier(dst_d) 355eb8dc403SDave Cobbley # Don't overwrite existing files, otherwise in the case of an upgrade 356eb8dc403SDave Cobbley # the dry-run written out recipe will be overwritten with an unmodified 357eb8dc403SDave Cobbley # version 358eb8dc403SDave Cobbley if not os.path.exists(dst): 359eb8dc403SDave Cobbley shutil.copy(src, dst) 360eb8dc403SDave Cobbley 361eb8dc403SDave Cobbleydef _move_file(src, dst, dry_run_outdir=None, base_outdir=None): 362eb8dc403SDave Cobbley """Move a file. Creates all the directory components of destination path.""" 363eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 364eb8dc403SDave Cobbley logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix)) 365eb8dc403SDave Cobbley if dry_run_outdir: 366eb8dc403SDave Cobbley # We want to copy here, not move 367eb8dc403SDave Cobbley _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 368eb8dc403SDave Cobbley else: 369eb8dc403SDave Cobbley dst_d = os.path.dirname(dst) 370eb8dc403SDave Cobbley if dst_d: 371eb8dc403SDave Cobbley bb.utils.mkdirhier(dst_d) 372eb8dc403SDave Cobbley shutil.move(src, dst) 373eb8dc403SDave Cobbley 374eb8dc403SDave Cobbleydef _copy_file(src, dst, dry_run_outdir=None): 375eb8dc403SDave Cobbley """Copy a file. Creates all the directory components of destination path.""" 376eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 377eb8dc403SDave Cobbley logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 378eb8dc403SDave Cobbley if dry_run_outdir: 379eb8dc403SDave Cobbley _dry_run_copy(src, dst, dry_run_outdir, base_outdir) 380eb8dc403SDave Cobbley else: 381eb8dc403SDave Cobbley dst_d = os.path.dirname(dst) 382eb8dc403SDave Cobbley if dst_d: 383eb8dc403SDave Cobbley bb.utils.mkdirhier(dst_d) 384eb8dc403SDave Cobbley shutil.copy(src, dst) 385eb8dc403SDave Cobbley 386eb8dc403SDave Cobbleydef _git_ls_tree(repodir, treeish='HEAD', recursive=False): 387eb8dc403SDave Cobbley """List contents of a git treeish""" 388eb8dc403SDave Cobbley import bb 389eb8dc403SDave Cobbley cmd = ['git', 'ls-tree', '-z', treeish] 390eb8dc403SDave Cobbley if recursive: 391eb8dc403SDave Cobbley cmd.append('-r') 392eb8dc403SDave Cobbley out, _ = bb.process.run(cmd, cwd=repodir) 393eb8dc403SDave Cobbley ret = {} 394eb8dc403SDave Cobbley if out: 395eb8dc403SDave Cobbley for line in out.split('\0'): 396eb8dc403SDave Cobbley if line: 397eb8dc403SDave Cobbley split = line.split(None, 4) 398eb8dc403SDave Cobbley ret[split[3]] = split[0:3] 399eb8dc403SDave Cobbley return ret 400eb8dc403SDave Cobbley 401eb8dc403SDave Cobbleydef _git_exclude_path(srctree, path): 402eb8dc403SDave Cobbley """Return pathspec (list of paths) that excludes certain path""" 403eb8dc403SDave Cobbley # NOTE: "Filtering out" files/paths in this way is not entirely reliable - 404eb8dc403SDave Cobbley # we don't catch files that are deleted, for example. A more reliable way 405eb8dc403SDave Cobbley # to implement this would be to use "negative pathspecs" which were 406eb8dc403SDave Cobbley # introduced in Git v1.9.0. Revisit this when/if the required Git version 407eb8dc403SDave Cobbley # becomes greater than that. 408eb8dc403SDave Cobbley path = os.path.normpath(path) 409eb8dc403SDave Cobbley recurse = True if len(path.split(os.path.sep)) > 1 else False 410eb8dc403SDave Cobbley git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys()) 411eb8dc403SDave Cobbley if path in git_files: 412eb8dc403SDave Cobbley git_files.remove(path) 413eb8dc403SDave Cobbley return git_files 414eb8dc403SDave Cobbley else: 415eb8dc403SDave Cobbley return ['.'] 416eb8dc403SDave Cobbley 417eb8dc403SDave Cobbleydef _ls_tree(directory): 418eb8dc403SDave Cobbley """Recursive listing of files in a directory""" 419eb8dc403SDave Cobbley ret = [] 420eb8dc403SDave Cobbley for root, dirs, files in os.walk(directory): 421eb8dc403SDave Cobbley ret.extend([os.path.relpath(os.path.join(root, fname), directory) for 422eb8dc403SDave Cobbley fname in files]) 423eb8dc403SDave Cobbley return ret 424eb8dc403SDave Cobbley 425eb8dc403SDave Cobbley 426eb8dc403SDave Cobbleydef extract(args, config, basepath, workspace): 427eb8dc403SDave Cobbley """Entry point for the devtool 'extract' subcommand""" 428eb8dc403SDave Cobbley import bb 429eb8dc403SDave Cobbley 430eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 431eb8dc403SDave Cobbley if not tinfoil: 432eb8dc403SDave Cobbley # Error already shown 433eb8dc403SDave Cobbley return 1 434eb8dc403SDave Cobbley try: 435eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 436eb8dc403SDave Cobbley if not rd: 437eb8dc403SDave Cobbley return 1 438eb8dc403SDave Cobbley 439eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 440eb8dc403SDave Cobbley initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 441eb8dc403SDave Cobbley logger.info('Source tree extracted to %s' % srctree) 442eb8dc403SDave Cobbley 443eb8dc403SDave Cobbley if initial_rev: 444eb8dc403SDave Cobbley return 0 445eb8dc403SDave Cobbley else: 446eb8dc403SDave Cobbley return 1 447eb8dc403SDave Cobbley finally: 448eb8dc403SDave Cobbley tinfoil.shutdown() 449eb8dc403SDave Cobbley 450eb8dc403SDave Cobbleydef sync(args, config, basepath, workspace): 451eb8dc403SDave Cobbley """Entry point for the devtool 'sync' subcommand""" 452eb8dc403SDave Cobbley import bb 453eb8dc403SDave Cobbley 454eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 455eb8dc403SDave Cobbley if not tinfoil: 456eb8dc403SDave Cobbley # Error already shown 457eb8dc403SDave Cobbley return 1 458eb8dc403SDave Cobbley try: 459eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 460eb8dc403SDave Cobbley if not rd: 461eb8dc403SDave Cobbley return 1 462eb8dc403SDave Cobbley 463eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 464eb8dc403SDave Cobbley initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True) 465eb8dc403SDave Cobbley logger.info('Source tree %s synchronized' % srctree) 466eb8dc403SDave Cobbley 467eb8dc403SDave Cobbley if initial_rev: 468eb8dc403SDave Cobbley return 0 469eb8dc403SDave Cobbley else: 470eb8dc403SDave Cobbley return 1 471eb8dc403SDave Cobbley finally: 472eb8dc403SDave Cobbley tinfoil.shutdown() 473eb8dc403SDave Cobbley 474eb8dc403SDave Cobbley 475eb8dc403SDave Cobbleydef _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 476eb8dc403SDave Cobbley """Extract sources of a recipe""" 477eb8dc403SDave Cobbley import oe.recipeutils 478eb8dc403SDave Cobbley import oe.patch 479eb8dc403SDave Cobbley 480eb8dc403SDave Cobbley pn = d.getVar('PN') 481eb8dc403SDave Cobbley 482eb8dc403SDave Cobbley _check_compatible_recipe(pn, d) 483eb8dc403SDave Cobbley 484eb8dc403SDave Cobbley if sync: 485eb8dc403SDave Cobbley if not os.path.exists(srctree): 486eb8dc403SDave Cobbley raise DevtoolError("output path %s does not exist" % srctree) 487eb8dc403SDave Cobbley else: 488eb8dc403SDave Cobbley if os.path.exists(srctree): 489eb8dc403SDave Cobbley if not os.path.isdir(srctree): 490eb8dc403SDave Cobbley raise DevtoolError("output path %s exists and is not a directory" % 491eb8dc403SDave Cobbley srctree) 492eb8dc403SDave Cobbley elif os.listdir(srctree): 493eb8dc403SDave Cobbley raise DevtoolError("output path %s already exists and is " 494eb8dc403SDave Cobbley "non-empty" % srctree) 495eb8dc403SDave Cobbley 496eb8dc403SDave Cobbley if 'noexec' in (d.getVarFlags('do_unpack', False) or []): 497eb8dc403SDave Cobbley raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 498eb8dc403SDave Cobbley "extract source" % pn, 4) 499eb8dc403SDave Cobbley 500eb8dc403SDave Cobbley if not sync: 501eb8dc403SDave Cobbley # Prepare for shutil.move later on 502eb8dc403SDave Cobbley bb.utils.mkdirhier(srctree) 503eb8dc403SDave Cobbley os.rmdir(srctree) 504eb8dc403SDave Cobbley 505eb8dc403SDave Cobbley extra_overrides = [] 506eb8dc403SDave Cobbley if not no_overrides: 507eb8dc403SDave Cobbley history = d.varhistory.variable('SRC_URI') 508eb8dc403SDave Cobbley for event in history: 509eb8dc403SDave Cobbley if not 'flag' in event: 510eb8dc403SDave Cobbley if event['op'].startswith(('_append[', '_prepend[')): 511eb8dc403SDave Cobbley extra_overrides.append(event['op'].split('[')[1].split(']')[0]) 512*99467dabSAndrew Geissler # We want to remove duplicate overrides. If a recipe had multiple 513*99467dabSAndrew Geissler # SRC_URI_override += values it would cause mulitple instances of 514*99467dabSAndrew Geissler # overrides. This doesn't play nicely with things like creating a 515*99467dabSAndrew Geissler # branch for every instance of DEVTOOL_EXTRA_OVERRIDES. 516*99467dabSAndrew Geissler extra_overrides = list(set(extra_overrides)) 517eb8dc403SDave Cobbley if extra_overrides: 518eb8dc403SDave Cobbley logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these') 519eb8dc403SDave Cobbley 520eb8dc403SDave Cobbley initial_rev = None 521eb8dc403SDave Cobbley 522eb8dc403SDave Cobbley appendexisted = False 523eb8dc403SDave Cobbley recipefile = d.getVar('FILE') 524eb8dc403SDave Cobbley appendfile = recipe_to_append(recipefile, config) 525eb8dc403SDave Cobbley is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) 526eb8dc403SDave Cobbley 527eb8dc403SDave Cobbley # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 528eb8dc403SDave Cobbley # directory so that: 529eb8dc403SDave Cobbley # (a) we pick up all files that get unpacked to the WORKDIR, and 530eb8dc403SDave Cobbley # (b) we don't disturb the existing build 531eb8dc403SDave Cobbley # However, with recipe-specific sysroots the sysroots for the recipe 532eb8dc403SDave Cobbley # will be prepared under WORKDIR, and if we used the system temporary 533eb8dc403SDave Cobbley # directory (i.e. usually /tmp) as used by mkdtemp by default, then 534eb8dc403SDave Cobbley # our attempts to hardlink files into the recipe-specific sysroots 535eb8dc403SDave Cobbley # will fail on systems where /tmp is a different filesystem, and it 536eb8dc403SDave Cobbley # would have to fall back to copying the files which is a waste of 537eb8dc403SDave Cobbley # time. Put the temp directory under the WORKDIR to prevent that from 538eb8dc403SDave Cobbley # being a problem. 539eb8dc403SDave Cobbley tempbasedir = d.getVar('WORKDIR') 540eb8dc403SDave Cobbley bb.utils.mkdirhier(tempbasedir) 541eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir) 542eb8dc403SDave Cobbley try: 543eb8dc403SDave Cobbley tinfoil.logger.setLevel(logging.WARNING) 544eb8dc403SDave Cobbley 545eb8dc403SDave Cobbley # FIXME this results in a cache reload under control of tinfoil, which is fine 546eb8dc403SDave Cobbley # except we don't get the knotty progress bar 547eb8dc403SDave Cobbley 548eb8dc403SDave Cobbley if os.path.exists(appendfile): 549eb8dc403SDave Cobbley appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') 550eb8dc403SDave Cobbley shutil.copyfile(appendfile, appendbackup) 551eb8dc403SDave Cobbley else: 552eb8dc403SDave Cobbley appendbackup = None 553eb8dc403SDave Cobbley bb.utils.mkdirhier(os.path.dirname(appendfile)) 554eb8dc403SDave Cobbley logger.debug('writing append file %s' % appendfile) 555eb8dc403SDave Cobbley with open(appendfile, 'a') as f: 556eb8dc403SDave Cobbley f.write('###--- _extract_source\n') 557eb8dc403SDave Cobbley f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 558eb8dc403SDave Cobbley f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 559eb8dc403SDave Cobbley if not is_kernel_yocto: 560eb8dc403SDave Cobbley f.write('PATCHTOOL = "git"\n') 561eb8dc403SDave Cobbley f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') 562eb8dc403SDave Cobbley if extra_overrides: 563eb8dc403SDave Cobbley f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides)) 564eb8dc403SDave Cobbley f.write('inherit devtool-source\n') 565eb8dc403SDave Cobbley f.write('###--- _extract_source\n') 566eb8dc403SDave Cobbley 567eb8dc403SDave Cobbley update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) 568eb8dc403SDave Cobbley 569eb8dc403SDave Cobbley sstate_manifests = d.getVar('SSTATE_MANIFESTS') 570eb8dc403SDave Cobbley bb.utils.mkdirhier(sstate_manifests) 571eb8dc403SDave Cobbley preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 572eb8dc403SDave Cobbley with open(preservestampfile, 'w') as f: 573eb8dc403SDave Cobbley f.write(d.getVar('STAMP')) 574eb8dc403SDave Cobbley try: 575eb8dc403SDave Cobbley if bb.data.inherits_class('kernel-yocto', d): 576eb8dc403SDave Cobbley # We need to generate the kernel config 577eb8dc403SDave Cobbley task = 'do_configure' 578eb8dc403SDave Cobbley else: 579eb8dc403SDave Cobbley task = 'do_patch' 580eb8dc403SDave Cobbley 581eb8dc403SDave Cobbley # Run the fetch + unpack tasks 582eb8dc403SDave Cobbley res = tinfoil.build_targets(pn, 583eb8dc403SDave Cobbley task, 584eb8dc403SDave Cobbley handle_events=True) 585eb8dc403SDave Cobbley finally: 586eb8dc403SDave Cobbley if os.path.exists(preservestampfile): 587eb8dc403SDave Cobbley os.remove(preservestampfile) 588eb8dc403SDave Cobbley 589eb8dc403SDave Cobbley if not res: 590eb8dc403SDave Cobbley raise DevtoolError('Extracting source for %s failed' % pn) 591eb8dc403SDave Cobbley 592eb8dc403SDave Cobbley try: 593eb8dc403SDave Cobbley with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: 594eb8dc403SDave Cobbley initial_rev = f.read() 595eb8dc403SDave Cobbley 596eb8dc403SDave Cobbley with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: 597eb8dc403SDave Cobbley srcsubdir = f.read() 598eb8dc403SDave Cobbley except FileNotFoundError as e: 599eb8dc403SDave Cobbley raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e)) 600eb8dc403SDave Cobbley srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir')) 601eb8dc403SDave Cobbley 602eb8dc403SDave Cobbley tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 603eb8dc403SDave Cobbley srctree_localdir = os.path.join(srctree, 'oe-local-files') 604eb8dc403SDave Cobbley 605eb8dc403SDave Cobbley if sync: 606eb8dc403SDave Cobbley bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 607eb8dc403SDave Cobbley 608eb8dc403SDave Cobbley # Move oe-local-files directory to srctree 609eb8dc403SDave Cobbley # As the oe-local-files is not part of the constructed git tree, 610eb8dc403SDave Cobbley # remove them directly during the synchrounizating might surprise 611eb8dc403SDave Cobbley # the users. Instead, we move it to oe-local-files.bak and remind 612eb8dc403SDave Cobbley # user in the log message. 613eb8dc403SDave Cobbley if os.path.exists(srctree_localdir + '.bak'): 614eb8dc403SDave Cobbley shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 615eb8dc403SDave Cobbley 616eb8dc403SDave Cobbley if os.path.exists(srctree_localdir): 617eb8dc403SDave Cobbley logger.info('Backing up current local file directory %s' % srctree_localdir) 618eb8dc403SDave Cobbley shutil.move(srctree_localdir, srctree_localdir + '.bak') 619eb8dc403SDave Cobbley 620eb8dc403SDave Cobbley if os.path.exists(tempdir_localdir): 621eb8dc403SDave Cobbley logger.info('Syncing local source files to srctree...') 622eb8dc403SDave Cobbley shutil.copytree(tempdir_localdir, srctree_localdir) 623eb8dc403SDave Cobbley else: 624eb8dc403SDave Cobbley # Move oe-local-files directory to srctree 625eb8dc403SDave Cobbley if os.path.exists(tempdir_localdir): 626eb8dc403SDave Cobbley logger.info('Adding local source files to srctree...') 627eb8dc403SDave Cobbley shutil.move(tempdir_localdir, srcsubdir) 628eb8dc403SDave Cobbley 629eb8dc403SDave Cobbley shutil.move(srcsubdir, srctree) 630eb8dc403SDave Cobbley 631eb8dc403SDave Cobbley if os.path.abspath(d.getVar('S')) == os.path.abspath(d.getVar('WORKDIR')): 632eb8dc403SDave Cobbley # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 633eb8dc403SDave Cobbley # (otherwise the recipe won't build as expected) 634eb8dc403SDave Cobbley local_files_dir = os.path.join(srctree, 'oe-local-files') 635eb8dc403SDave Cobbley addfiles = [] 636eb8dc403SDave Cobbley for root, _, files in os.walk(local_files_dir): 637eb8dc403SDave Cobbley relpth = os.path.relpath(root, local_files_dir) 638eb8dc403SDave Cobbley if relpth != '.': 639eb8dc403SDave Cobbley bb.utils.mkdirhier(os.path.join(srctree, relpth)) 640eb8dc403SDave Cobbley for fn in files: 641eb8dc403SDave Cobbley if fn == '.gitignore': 642eb8dc403SDave Cobbley continue 643eb8dc403SDave Cobbley destpth = os.path.join(srctree, relpth, fn) 644eb8dc403SDave Cobbley if os.path.exists(destpth): 645eb8dc403SDave Cobbley os.unlink(destpth) 646eb8dc403SDave Cobbley os.symlink('oe-local-files/%s' % fn, destpth) 647eb8dc403SDave Cobbley addfiles.append(os.path.join(relpth, fn)) 648eb8dc403SDave Cobbley if addfiles: 649eb8dc403SDave Cobbley bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 650eb8dc403SDave Cobbley useroptions = [] 651eb8dc403SDave Cobbley oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) 652eb8dc403SDave Cobbley bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 653eb8dc403SDave Cobbley 654eb8dc403SDave Cobbley if is_kernel_yocto: 655eb8dc403SDave Cobbley logger.info('Copying kernel config to srctree') 656eb8dc403SDave Cobbley shutil.copy2(os.path.join(tempdir, '.config'), srctree) 657eb8dc403SDave Cobbley 658eb8dc403SDave Cobbley finally: 659eb8dc403SDave Cobbley if appendbackup: 660eb8dc403SDave Cobbley shutil.copyfile(appendbackup, appendfile) 661eb8dc403SDave Cobbley elif os.path.exists(appendfile): 662eb8dc403SDave Cobbley os.remove(appendfile) 663eb8dc403SDave Cobbley if keep_temp: 664eb8dc403SDave Cobbley logger.info('Preserving temporary directory %s' % tempdir) 665eb8dc403SDave Cobbley else: 666eb8dc403SDave Cobbley shutil.rmtree(tempdir) 667eb8dc403SDave Cobbley return initial_rev, srcsubdir_rel 668eb8dc403SDave Cobbley 669eb8dc403SDave Cobbleydef _add_md5(config, recipename, filename): 670eb8dc403SDave Cobbley """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace""" 671eb8dc403SDave Cobbley import bb.utils 672eb8dc403SDave Cobbley 673eb8dc403SDave Cobbley def addfile(fn): 674eb8dc403SDave Cobbley md5 = bb.utils.md5_file(fn) 675eb8dc403SDave Cobbley with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f: 676eb8dc403SDave Cobbley md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5) 677eb8dc403SDave Cobbley f.seek(0, os.SEEK_SET) 678eb8dc403SDave Cobbley if not md5_str in f.read(): 679eb8dc403SDave Cobbley f.write(md5_str) 680eb8dc403SDave Cobbley 681eb8dc403SDave Cobbley if os.path.isdir(filename): 682eb8dc403SDave Cobbley for root, _, files in os.walk(filename): 683eb8dc403SDave Cobbley for f in files: 684eb8dc403SDave Cobbley addfile(os.path.join(root, f)) 685eb8dc403SDave Cobbley else: 686eb8dc403SDave Cobbley addfile(filename) 687eb8dc403SDave Cobbley 688eb8dc403SDave Cobbleydef _check_preserve(config, recipename): 689eb8dc403SDave Cobbley """Check if a file was manually changed and needs to be saved in 'attic' 690eb8dc403SDave Cobbley directory""" 691eb8dc403SDave Cobbley import bb.utils 692eb8dc403SDave Cobbley origfile = os.path.join(config.workspace_path, '.devtool_md5') 693eb8dc403SDave Cobbley newfile = os.path.join(config.workspace_path, '.devtool_md5_new') 694eb8dc403SDave Cobbley preservepath = os.path.join(config.workspace_path, 'attic', recipename) 695eb8dc403SDave Cobbley with open(origfile, 'r') as f: 696eb8dc403SDave Cobbley with open(newfile, 'w') as tf: 697eb8dc403SDave Cobbley for line in f.readlines(): 698eb8dc403SDave Cobbley splitline = line.rstrip().split('|') 699eb8dc403SDave Cobbley if splitline[0] == recipename: 700eb8dc403SDave Cobbley removefile = os.path.join(config.workspace_path, splitline[1]) 701eb8dc403SDave Cobbley try: 702eb8dc403SDave Cobbley md5 = bb.utils.md5_file(removefile) 703eb8dc403SDave Cobbley except IOError as err: 704eb8dc403SDave Cobbley if err.errno == 2: 705eb8dc403SDave Cobbley # File no longer exists, skip it 706eb8dc403SDave Cobbley continue 707eb8dc403SDave Cobbley else: 708eb8dc403SDave Cobbley raise 709eb8dc403SDave Cobbley if splitline[2] != md5: 710eb8dc403SDave Cobbley bb.utils.mkdirhier(preservepath) 711eb8dc403SDave Cobbley preservefile = os.path.basename(removefile) 7121a4b7ee2SBrad Bishop logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath)) 713eb8dc403SDave Cobbley shutil.move(removefile, os.path.join(preservepath, preservefile)) 714eb8dc403SDave Cobbley else: 715eb8dc403SDave Cobbley os.remove(removefile) 716eb8dc403SDave Cobbley else: 717eb8dc403SDave Cobbley tf.write(line) 718eb8dc403SDave Cobbley os.rename(newfile, origfile) 719eb8dc403SDave Cobbley 720eb8dc403SDave Cobbleydef modify(args, config, basepath, workspace): 721eb8dc403SDave Cobbley """Entry point for the devtool 'modify' subcommand""" 722eb8dc403SDave Cobbley import bb 723eb8dc403SDave Cobbley import oe.recipeutils 724eb8dc403SDave Cobbley import oe.patch 725eb8dc403SDave Cobbley 726eb8dc403SDave Cobbley if args.recipename in workspace: 727eb8dc403SDave Cobbley raise DevtoolError("recipe %s is already in your workspace" % 728eb8dc403SDave Cobbley args.recipename) 729eb8dc403SDave Cobbley 730eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 731eb8dc403SDave Cobbley try: 732eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 733eb8dc403SDave Cobbley if not rd: 734eb8dc403SDave Cobbley return 1 735eb8dc403SDave Cobbley 736eb8dc403SDave Cobbley pn = rd.getVar('PN') 737eb8dc403SDave Cobbley if pn != args.recipename: 738eb8dc403SDave Cobbley logger.info('Mapping %s to %s' % (args.recipename, pn)) 739eb8dc403SDave Cobbley if pn in workspace: 740eb8dc403SDave Cobbley raise DevtoolError("recipe %s is already in your workspace" % 741eb8dc403SDave Cobbley pn) 742eb8dc403SDave Cobbley 743eb8dc403SDave Cobbley if args.srctree: 744eb8dc403SDave Cobbley srctree = os.path.abspath(args.srctree) 745eb8dc403SDave Cobbley else: 746eb8dc403SDave Cobbley srctree = get_default_srctree(config, pn) 747eb8dc403SDave Cobbley 748eb8dc403SDave Cobbley if args.no_extract and not os.path.isdir(srctree): 749eb8dc403SDave Cobbley raise DevtoolError("--no-extract specified and source path %s does " 750eb8dc403SDave Cobbley "not exist or is not a directory" % 751eb8dc403SDave Cobbley srctree) 752eb8dc403SDave Cobbley 753eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 754eb8dc403SDave Cobbley appendfile = recipe_to_append(recipefile, config, args.wildcard) 755eb8dc403SDave Cobbley if os.path.exists(appendfile): 756eb8dc403SDave Cobbley raise DevtoolError("Another variant of recipe %s is already in your " 757eb8dc403SDave Cobbley "workspace (only one variant of a recipe can " 758eb8dc403SDave Cobbley "currently be worked on at once)" 759eb8dc403SDave Cobbley % pn) 760eb8dc403SDave Cobbley 761eb8dc403SDave Cobbley _check_compatible_recipe(pn, rd) 762eb8dc403SDave Cobbley 763eb8dc403SDave Cobbley initial_rev = None 764eb8dc403SDave Cobbley commits = [] 765eb8dc403SDave Cobbley check_commits = False 766eb8dc403SDave Cobbley if not args.no_extract: 767eb8dc403SDave Cobbley initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 768eb8dc403SDave Cobbley if not initial_rev: 769eb8dc403SDave Cobbley return 1 770eb8dc403SDave Cobbley logger.info('Source tree extracted to %s' % srctree) 771eb8dc403SDave Cobbley # Get list of commits since this revision 772eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) 773eb8dc403SDave Cobbley commits = stdout.split() 774eb8dc403SDave Cobbley check_commits = True 775eb8dc403SDave Cobbley else: 776eb8dc403SDave Cobbley if os.path.exists(os.path.join(srctree, '.git')): 777*99467dabSAndrew Geissler # Check if it's a tree previously extracted by us. This is done 778*99467dabSAndrew Geissler # by ensuring that devtool-base and args.branch (devtool) exist. 779*99467dabSAndrew Geissler # The check_commits logic will cause an exception if either one 780*99467dabSAndrew Geissler # of these doesn't exist 781eb8dc403SDave Cobbley try: 782eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree) 783*99467dabSAndrew Geissler bb.process.run('git rev-parse %s' % args.branch, cwd=srctree) 784eb8dc403SDave Cobbley except bb.process.ExecutionError: 785eb8dc403SDave Cobbley stdout = '' 786eb8dc403SDave Cobbley if stdout: 787eb8dc403SDave Cobbley check_commits = True 788eb8dc403SDave Cobbley for line in stdout.splitlines(): 789eb8dc403SDave Cobbley if line.startswith('*'): 790eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 791eb8dc403SDave Cobbley initial_rev = stdout.rstrip() 792eb8dc403SDave Cobbley if not initial_rev: 793eb8dc403SDave Cobbley # Otherwise, just grab the head revision 794eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 795eb8dc403SDave Cobbley initial_rev = stdout.rstrip() 796eb8dc403SDave Cobbley 797eb8dc403SDave Cobbley branch_patches = {} 798eb8dc403SDave Cobbley if check_commits: 799eb8dc403SDave Cobbley # Check if there are override branches 800eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git branch', cwd=srctree) 801eb8dc403SDave Cobbley branches = [] 802eb8dc403SDave Cobbley for line in stdout.rstrip().splitlines(): 803eb8dc403SDave Cobbley branchname = line[2:].rstrip() 804eb8dc403SDave Cobbley if branchname.startswith(override_branch_prefix): 805eb8dc403SDave Cobbley branches.append(branchname) 806eb8dc403SDave Cobbley if branches: 8071a4b7ee2SBrad Bishop logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix)) 808eb8dc403SDave Cobbley branches.insert(0, args.branch) 809eb8dc403SDave Cobbley seen_patches = [] 810eb8dc403SDave Cobbley for branch in branches: 811eb8dc403SDave Cobbley branch_patches[branch] = [] 812eb8dc403SDave Cobbley (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 813eb8dc403SDave Cobbley for line in stdout.splitlines(): 814eb8dc403SDave Cobbley line = line.strip() 815eb8dc403SDave Cobbley if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 816eb8dc403SDave Cobbley origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 817eb8dc403SDave Cobbley if not origpatch in seen_patches: 818eb8dc403SDave Cobbley seen_patches.append(origpatch) 819eb8dc403SDave Cobbley branch_patches[branch].append(origpatch) 820eb8dc403SDave Cobbley 821eb8dc403SDave Cobbley # Need to grab this here in case the source is within a subdirectory 822eb8dc403SDave Cobbley srctreebase = srctree 823eb8dc403SDave Cobbley 824eb8dc403SDave Cobbley # Check that recipe isn't using a shared workdir 825eb8dc403SDave Cobbley s = os.path.abspath(rd.getVar('S')) 826eb8dc403SDave Cobbley workdir = os.path.abspath(rd.getVar('WORKDIR')) 827eb8dc403SDave Cobbley if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir: 828eb8dc403SDave Cobbley # Handle if S is set to a subdirectory of the source 829eb8dc403SDave Cobbley srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1] 830eb8dc403SDave Cobbley srctree = os.path.join(srctree, srcsubdir) 831eb8dc403SDave Cobbley 832eb8dc403SDave Cobbley bb.utils.mkdirhier(os.path.dirname(appendfile)) 833eb8dc403SDave Cobbley with open(appendfile, 'w') as f: 834eb8dc403SDave Cobbley f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n') 835eb8dc403SDave Cobbley # Local files can be modified/tracked in separate subdir under srctree 836eb8dc403SDave Cobbley # Mostly useful for packages with S != WORKDIR 837eb8dc403SDave Cobbley f.write('FILESPATH_prepend := "%s:"\n' % 838eb8dc403SDave Cobbley os.path.join(srctreebase, 'oe-local-files')) 839eb8dc403SDave Cobbley f.write('# srctreebase: %s\n' % srctreebase) 840eb8dc403SDave Cobbley 841eb8dc403SDave Cobbley f.write('\ninherit externalsrc\n') 842eb8dc403SDave Cobbley f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 843eb8dc403SDave Cobbley f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) 844eb8dc403SDave Cobbley 845eb8dc403SDave Cobbley b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 846eb8dc403SDave Cobbley if b_is_s: 847eb8dc403SDave Cobbley f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) 848eb8dc403SDave Cobbley 849eb8dc403SDave Cobbley if bb.data.inherits_class('kernel', rd): 850eb8dc403SDave Cobbley f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 851eb8dc403SDave Cobbley 'do_fetch do_unpack do_kernel_configme do_kernel_configcheck"\n') 852eb8dc403SDave Cobbley f.write('\ndo_patch() {\n' 853eb8dc403SDave Cobbley ' :\n' 854eb8dc403SDave Cobbley '}\n') 855eb8dc403SDave Cobbley f.write('\ndo_configure_append() {\n' 856eb8dc403SDave Cobbley ' cp ${B}/.config ${S}/.config.baseline\n' 857eb8dc403SDave Cobbley ' ln -sfT ${B}/.config ${S}/.config.new\n' 858eb8dc403SDave Cobbley '}\n') 859eb8dc403SDave Cobbley if initial_rev: 860eb8dc403SDave Cobbley f.write('\n# initial_rev: %s\n' % initial_rev) 861eb8dc403SDave Cobbley for commit in commits: 862eb8dc403SDave Cobbley f.write('# commit: %s\n' % commit) 863eb8dc403SDave Cobbley if branch_patches: 864eb8dc403SDave Cobbley for branch in branch_patches: 865eb8dc403SDave Cobbley if branch == args.branch: 866eb8dc403SDave Cobbley continue 867eb8dc403SDave Cobbley f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch]))) 868eb8dc403SDave Cobbley 869eb8dc403SDave Cobbley update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn]) 870eb8dc403SDave Cobbley 871eb8dc403SDave Cobbley _add_md5(config, pn, appendfile) 872eb8dc403SDave Cobbley 873eb8dc403SDave Cobbley logger.info('Recipe %s now set up to build from %s' % (pn, srctree)) 874eb8dc403SDave Cobbley 875eb8dc403SDave Cobbley finally: 876eb8dc403SDave Cobbley tinfoil.shutdown() 877eb8dc403SDave Cobbley 878eb8dc403SDave Cobbley return 0 879eb8dc403SDave Cobbley 880eb8dc403SDave Cobbley 881eb8dc403SDave Cobbleydef rename(args, config, basepath, workspace): 882eb8dc403SDave Cobbley """Entry point for the devtool 'rename' subcommand""" 883eb8dc403SDave Cobbley import bb 884eb8dc403SDave Cobbley import oe.recipeutils 885eb8dc403SDave Cobbley 886eb8dc403SDave Cobbley check_workspace_recipe(workspace, args.recipename) 887eb8dc403SDave Cobbley 888eb8dc403SDave Cobbley if not (args.newname or args.version): 889eb8dc403SDave Cobbley raise DevtoolError('You must specify a new name, a version with -V/--version, or both') 890eb8dc403SDave Cobbley 891eb8dc403SDave Cobbley recipefile = workspace[args.recipename]['recipefile'] 892eb8dc403SDave Cobbley if not recipefile: 893eb8dc403SDave Cobbley raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)') 894eb8dc403SDave Cobbley 895eb8dc403SDave Cobbley if args.newname and args.newname != args.recipename: 896eb8dc403SDave Cobbley reason = oe.recipeutils.validate_pn(args.newname) 897eb8dc403SDave Cobbley if reason: 898eb8dc403SDave Cobbley raise DevtoolError(reason) 899eb8dc403SDave Cobbley newname = args.newname 900eb8dc403SDave Cobbley else: 901eb8dc403SDave Cobbley newname = args.recipename 902eb8dc403SDave Cobbley 903eb8dc403SDave Cobbley append = workspace[args.recipename]['bbappend'] 904eb8dc403SDave Cobbley appendfn = os.path.splitext(os.path.basename(append))[0] 905eb8dc403SDave Cobbley splitfn = appendfn.split('_') 906eb8dc403SDave Cobbley if len(splitfn) > 1: 907eb8dc403SDave Cobbley origfnver = appendfn.split('_')[1] 908eb8dc403SDave Cobbley else: 909eb8dc403SDave Cobbley origfnver = '' 910eb8dc403SDave Cobbley 911eb8dc403SDave Cobbley recipefilemd5 = None 912eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 913eb8dc403SDave Cobbley try: 914eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 915eb8dc403SDave Cobbley if not rd: 916eb8dc403SDave Cobbley return 1 917eb8dc403SDave Cobbley 918eb8dc403SDave Cobbley bp = rd.getVar('BP') 919eb8dc403SDave Cobbley bpn = rd.getVar('BPN') 920eb8dc403SDave Cobbley if newname != args.recipename: 921eb8dc403SDave Cobbley localdata = rd.createCopy() 922eb8dc403SDave Cobbley localdata.setVar('PN', newname) 923eb8dc403SDave Cobbley newbpn = localdata.getVar('BPN') 924eb8dc403SDave Cobbley else: 925eb8dc403SDave Cobbley newbpn = bpn 926eb8dc403SDave Cobbley s = rd.getVar('S', False) 927eb8dc403SDave Cobbley src_uri = rd.getVar('SRC_URI', False) 928eb8dc403SDave Cobbley pv = rd.getVar('PV') 929eb8dc403SDave Cobbley 930eb8dc403SDave Cobbley # Correct variable values that refer to the upstream source - these 931eb8dc403SDave Cobbley # values must stay the same, so if the name/version are changing then 932eb8dc403SDave Cobbley # we need to fix them up 933eb8dc403SDave Cobbley new_s = s 934eb8dc403SDave Cobbley new_src_uri = src_uri 935eb8dc403SDave Cobbley if newbpn != bpn: 936eb8dc403SDave Cobbley # ${PN} here is technically almost always incorrect, but people do use it 937eb8dc403SDave Cobbley new_s = new_s.replace('${BPN}', bpn) 938eb8dc403SDave Cobbley new_s = new_s.replace('${PN}', bpn) 939eb8dc403SDave Cobbley new_s = new_s.replace('${BP}', '%s-${PV}' % bpn) 940eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${BPN}', bpn) 941eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${PN}', bpn) 942eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn) 943eb8dc403SDave Cobbley if args.version and origfnver == pv: 944eb8dc403SDave Cobbley new_s = new_s.replace('${PV}', pv) 945eb8dc403SDave Cobbley new_s = new_s.replace('${BP}', '${BPN}-%s' % pv) 946eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${PV}', pv) 947eb8dc403SDave Cobbley new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv) 948eb8dc403SDave Cobbley patchfields = {} 949eb8dc403SDave Cobbley if new_s != s: 950eb8dc403SDave Cobbley patchfields['S'] = new_s 951eb8dc403SDave Cobbley if new_src_uri != src_uri: 952eb8dc403SDave Cobbley patchfields['SRC_URI'] = new_src_uri 953eb8dc403SDave Cobbley if patchfields: 954eb8dc403SDave Cobbley recipefilemd5 = bb.utils.md5_file(recipefile) 955eb8dc403SDave Cobbley oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 956eb8dc403SDave Cobbley newrecipefilemd5 = bb.utils.md5_file(recipefile) 957eb8dc403SDave Cobbley finally: 958eb8dc403SDave Cobbley tinfoil.shutdown() 959eb8dc403SDave Cobbley 960eb8dc403SDave Cobbley if args.version: 961eb8dc403SDave Cobbley newver = args.version 962eb8dc403SDave Cobbley else: 963eb8dc403SDave Cobbley newver = origfnver 964eb8dc403SDave Cobbley 965eb8dc403SDave Cobbley if newver: 966eb8dc403SDave Cobbley newappend = '%s_%s.bbappend' % (newname, newver) 967eb8dc403SDave Cobbley newfile = '%s_%s.bb' % (newname, newver) 968eb8dc403SDave Cobbley else: 969eb8dc403SDave Cobbley newappend = '%s.bbappend' % newname 970eb8dc403SDave Cobbley newfile = '%s.bb' % newname 971eb8dc403SDave Cobbley 972eb8dc403SDave Cobbley oldrecipedir = os.path.dirname(recipefile) 973eb8dc403SDave Cobbley newrecipedir = os.path.join(config.workspace_path, 'recipes', newname) 974eb8dc403SDave Cobbley if oldrecipedir != newrecipedir: 975eb8dc403SDave Cobbley bb.utils.mkdirhier(newrecipedir) 976eb8dc403SDave Cobbley 977eb8dc403SDave Cobbley newappend = os.path.join(os.path.dirname(append), newappend) 978eb8dc403SDave Cobbley newfile = os.path.join(newrecipedir, newfile) 979eb8dc403SDave Cobbley 980eb8dc403SDave Cobbley # Rename bbappend 981eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (append, newappend)) 982eb8dc403SDave Cobbley os.rename(append, newappend) 983eb8dc403SDave Cobbley # Rename recipe file 984eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (recipefile, newfile)) 985eb8dc403SDave Cobbley os.rename(recipefile, newfile) 986eb8dc403SDave Cobbley 987eb8dc403SDave Cobbley # Rename source tree if it's the default path 988eb8dc403SDave Cobbley appendmd5 = None 989eb8dc403SDave Cobbley if not args.no_srctree: 990eb8dc403SDave Cobbley srctree = workspace[args.recipename]['srctree'] 991eb8dc403SDave Cobbley if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename): 992eb8dc403SDave Cobbley newsrctree = os.path.join(config.workspace_path, 'sources', newname) 993eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (srctree, newsrctree)) 994eb8dc403SDave Cobbley shutil.move(srctree, newsrctree) 995eb8dc403SDave Cobbley # Correct any references (basically EXTERNALSRC*) in the .bbappend 996eb8dc403SDave Cobbley appendmd5 = bb.utils.md5_file(newappend) 997eb8dc403SDave Cobbley appendlines = [] 998eb8dc403SDave Cobbley with open(newappend, 'r') as f: 999eb8dc403SDave Cobbley for line in f: 1000eb8dc403SDave Cobbley appendlines.append(line) 1001eb8dc403SDave Cobbley with open(newappend, 'w') as f: 1002eb8dc403SDave Cobbley for line in appendlines: 1003eb8dc403SDave Cobbley if srctree in line: 1004eb8dc403SDave Cobbley line = line.replace(srctree, newsrctree) 1005eb8dc403SDave Cobbley f.write(line) 1006eb8dc403SDave Cobbley newappendmd5 = bb.utils.md5_file(newappend) 1007eb8dc403SDave Cobbley 1008eb8dc403SDave Cobbley bpndir = None 1009eb8dc403SDave Cobbley newbpndir = None 1010eb8dc403SDave Cobbley if newbpn != bpn: 1011eb8dc403SDave Cobbley bpndir = os.path.join(oldrecipedir, bpn) 1012eb8dc403SDave Cobbley if os.path.exists(bpndir): 1013eb8dc403SDave Cobbley newbpndir = os.path.join(newrecipedir, newbpn) 1014eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (bpndir, newbpndir)) 1015eb8dc403SDave Cobbley shutil.move(bpndir, newbpndir) 1016eb8dc403SDave Cobbley 1017eb8dc403SDave Cobbley bpdir = None 1018eb8dc403SDave Cobbley newbpdir = None 1019eb8dc403SDave Cobbley if newver != origfnver or newbpn != bpn: 1020eb8dc403SDave Cobbley bpdir = os.path.join(oldrecipedir, bp) 1021eb8dc403SDave Cobbley if os.path.exists(bpdir): 1022eb8dc403SDave Cobbley newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver)) 1023eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (bpdir, newbpdir)) 1024eb8dc403SDave Cobbley shutil.move(bpdir, newbpdir) 1025eb8dc403SDave Cobbley 1026eb8dc403SDave Cobbley if oldrecipedir != newrecipedir: 1027eb8dc403SDave Cobbley # Move any stray files and delete the old recipe directory 1028eb8dc403SDave Cobbley for entry in os.listdir(oldrecipedir): 1029eb8dc403SDave Cobbley oldpath = os.path.join(oldrecipedir, entry) 1030eb8dc403SDave Cobbley newpath = os.path.join(newrecipedir, entry) 1031eb8dc403SDave Cobbley logger.info('Renaming %s to %s' % (oldpath, newpath)) 1032eb8dc403SDave Cobbley shutil.move(oldpath, newpath) 1033eb8dc403SDave Cobbley os.rmdir(oldrecipedir) 1034eb8dc403SDave Cobbley 1035eb8dc403SDave Cobbley # Now take care of entries in .devtool_md5 1036eb8dc403SDave Cobbley md5entries = [] 1037eb8dc403SDave Cobbley with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f: 1038eb8dc403SDave Cobbley for line in f: 1039eb8dc403SDave Cobbley md5entries.append(line) 1040eb8dc403SDave Cobbley 1041eb8dc403SDave Cobbley if bpndir and newbpndir: 1042eb8dc403SDave Cobbley relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/' 1043eb8dc403SDave Cobbley else: 1044eb8dc403SDave Cobbley relbpndir = None 1045eb8dc403SDave Cobbley if bpdir and newbpdir: 1046eb8dc403SDave Cobbley relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/' 1047eb8dc403SDave Cobbley else: 1048eb8dc403SDave Cobbley relbpdir = None 1049eb8dc403SDave Cobbley 1050eb8dc403SDave Cobbley with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f: 1051eb8dc403SDave Cobbley for entry in md5entries: 1052eb8dc403SDave Cobbley splitentry = entry.rstrip().split('|') 1053eb8dc403SDave Cobbley if len(splitentry) > 2: 1054eb8dc403SDave Cobbley if splitentry[0] == args.recipename: 1055eb8dc403SDave Cobbley splitentry[0] = newname 1056eb8dc403SDave Cobbley if splitentry[1] == os.path.relpath(append, config.workspace_path): 1057eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(newappend, config.workspace_path) 1058eb8dc403SDave Cobbley if appendmd5 and splitentry[2] == appendmd5: 1059eb8dc403SDave Cobbley splitentry[2] = newappendmd5 1060eb8dc403SDave Cobbley elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path): 1061eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(newfile, config.workspace_path) 1062eb8dc403SDave Cobbley if recipefilemd5 and splitentry[2] == recipefilemd5: 1063eb8dc403SDave Cobbley splitentry[2] = newrecipefilemd5 1064eb8dc403SDave Cobbley elif relbpndir and splitentry[1].startswith(relbpndir): 1065eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path) 1066eb8dc403SDave Cobbley elif relbpdir and splitentry[1].startswith(relbpdir): 1067eb8dc403SDave Cobbley splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path) 1068eb8dc403SDave Cobbley entry = '|'.join(splitentry) + '\n' 1069eb8dc403SDave Cobbley f.write(entry) 1070eb8dc403SDave Cobbley return 0 1071eb8dc403SDave Cobbley 1072eb8dc403SDave Cobbley 1073eb8dc403SDave Cobbleydef _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False): 1074eb8dc403SDave Cobbley """Get initial and update rev of a recipe. These are the start point of the 1075eb8dc403SDave Cobbley whole patchset and start point for the patches to be re-generated/updated. 1076eb8dc403SDave Cobbley """ 1077eb8dc403SDave Cobbley import bb 1078eb8dc403SDave Cobbley 1079eb8dc403SDave Cobbley # Get current branch 1080eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD', 1081eb8dc403SDave Cobbley cwd=srctree) 1082eb8dc403SDave Cobbley branchname = stdout.rstrip() 1083eb8dc403SDave Cobbley 1084eb8dc403SDave Cobbley # Parse initial rev from recipe if not specified 1085eb8dc403SDave Cobbley commits = [] 1086eb8dc403SDave Cobbley patches = [] 1087eb8dc403SDave Cobbley with open(recipe_path, 'r') as f: 1088eb8dc403SDave Cobbley for line in f: 1089eb8dc403SDave Cobbley if line.startswith('# initial_rev:'): 1090eb8dc403SDave Cobbley if not initial_rev: 1091eb8dc403SDave Cobbley initial_rev = line.split(':')[-1].strip() 1092eb8dc403SDave Cobbley elif line.startswith('# commit:') and not force_patch_refresh: 1093eb8dc403SDave Cobbley commits.append(line.split(':')[-1].strip()) 1094eb8dc403SDave Cobbley elif line.startswith('# patches_%s:' % branchname): 1095eb8dc403SDave Cobbley patches = line.split(':')[-1].strip().split(',') 1096eb8dc403SDave Cobbley 1097eb8dc403SDave Cobbley update_rev = initial_rev 1098eb8dc403SDave Cobbley changed_revs = None 1099eb8dc403SDave Cobbley if initial_rev: 1100eb8dc403SDave Cobbley # Find first actually changed revision 1101eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1102eb8dc403SDave Cobbley initial_rev, cwd=srctree) 1103eb8dc403SDave Cobbley newcommits = stdout.split() 1104eb8dc403SDave Cobbley for i in range(min(len(commits), len(newcommits))): 1105eb8dc403SDave Cobbley if newcommits[i] == commits[i]: 1106eb8dc403SDave Cobbley update_rev = commits[i] 1107eb8dc403SDave Cobbley 1108eb8dc403SDave Cobbley try: 1109eb8dc403SDave Cobbley stdout, _ = bb.process.run('git cherry devtool-patched', 1110eb8dc403SDave Cobbley cwd=srctree) 1111eb8dc403SDave Cobbley except bb.process.ExecutionError as err: 1112eb8dc403SDave Cobbley stdout = None 1113eb8dc403SDave Cobbley 1114eb8dc403SDave Cobbley if stdout is not None and not force_patch_refresh: 1115eb8dc403SDave Cobbley changed_revs = [] 1116eb8dc403SDave Cobbley for line in stdout.splitlines(): 1117eb8dc403SDave Cobbley if line.startswith('+ '): 1118eb8dc403SDave Cobbley rev = line.split()[1] 1119eb8dc403SDave Cobbley if rev in newcommits: 1120eb8dc403SDave Cobbley changed_revs.append(rev) 1121eb8dc403SDave Cobbley 1122eb8dc403SDave Cobbley return initial_rev, update_rev, changed_revs, patches 1123eb8dc403SDave Cobbley 1124eb8dc403SDave Cobbleydef _remove_file_entries(srcuri, filelist): 1125eb8dc403SDave Cobbley """Remove file:// entries from SRC_URI""" 1126eb8dc403SDave Cobbley remaining = filelist[:] 1127eb8dc403SDave Cobbley entries = [] 1128eb8dc403SDave Cobbley for fname in filelist: 1129eb8dc403SDave Cobbley basename = os.path.basename(fname) 1130eb8dc403SDave Cobbley for i in range(len(srcuri)): 1131eb8dc403SDave Cobbley if (srcuri[i].startswith('file://') and 1132eb8dc403SDave Cobbley os.path.basename(srcuri[i].split(';')[0]) == basename): 1133eb8dc403SDave Cobbley entries.append(srcuri[i]) 1134eb8dc403SDave Cobbley remaining.remove(fname) 1135eb8dc403SDave Cobbley srcuri.pop(i) 1136eb8dc403SDave Cobbley break 1137eb8dc403SDave Cobbley return entries, remaining 1138eb8dc403SDave Cobbley 1139eb8dc403SDave Cobbleydef _replace_srcuri_entry(srcuri, filename, newentry): 1140eb8dc403SDave Cobbley """Replace entry corresponding to specified file with a new entry""" 1141eb8dc403SDave Cobbley basename = os.path.basename(filename) 1142eb8dc403SDave Cobbley for i in range(len(srcuri)): 1143eb8dc403SDave Cobbley if os.path.basename(srcuri[i].split(';')[0]) == basename: 1144eb8dc403SDave Cobbley srcuri.pop(i) 1145eb8dc403SDave Cobbley srcuri.insert(i, newentry) 1146eb8dc403SDave Cobbley break 1147eb8dc403SDave Cobbley 1148eb8dc403SDave Cobbleydef _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False): 1149eb8dc403SDave Cobbley """Unlink existing patch files""" 1150eb8dc403SDave Cobbley 1151eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run else '' 1152eb8dc403SDave Cobbley 1153eb8dc403SDave Cobbley for path in files: 1154eb8dc403SDave Cobbley if append: 1155eb8dc403SDave Cobbley if not destpath: 1156eb8dc403SDave Cobbley raise Exception('destpath should be set here') 1157eb8dc403SDave Cobbley path = os.path.join(destpath, os.path.basename(path)) 1158eb8dc403SDave Cobbley 1159eb8dc403SDave Cobbley if os.path.exists(path): 1160eb8dc403SDave Cobbley if not no_report_remove: 1161eb8dc403SDave Cobbley logger.info('Removing file %s%s' % (path, dry_run_suffix)) 1162eb8dc403SDave Cobbley if not dry_run: 1163eb8dc403SDave Cobbley # FIXME "git rm" here would be nice if the file in question is 1164eb8dc403SDave Cobbley # tracked 1165eb8dc403SDave Cobbley # FIXME there's a chance that this file is referred to by 1166eb8dc403SDave Cobbley # another recipe, in which case deleting wouldn't be the 1167eb8dc403SDave Cobbley # right thing to do 1168eb8dc403SDave Cobbley os.remove(path) 1169eb8dc403SDave Cobbley # Remove directory if empty 1170eb8dc403SDave Cobbley try: 1171eb8dc403SDave Cobbley os.rmdir(os.path.dirname(path)) 1172eb8dc403SDave Cobbley except OSError as ose: 1173eb8dc403SDave Cobbley if ose.errno != errno.ENOTEMPTY: 1174eb8dc403SDave Cobbley raise 1175eb8dc403SDave Cobbley 1176eb8dc403SDave Cobbley 1177eb8dc403SDave Cobbleydef _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1178eb8dc403SDave Cobbley """Export patches from srctree to given location. 1179eb8dc403SDave Cobbley Returns three-tuple of dicts: 1180eb8dc403SDave Cobbley 1. updated - patches that already exist in SRCURI 1181eb8dc403SDave Cobbley 2. added - new patches that don't exist in SRCURI 1182eb8dc403SDave Cobbley 3 removed - patches that exist in SRCURI but not in exported patches 1183eb8dc403SDave Cobbley In each dict the key is the 'basepath' of the URI and value is the 1184eb8dc403SDave Cobbley absolute path to the existing file in recipe space (if any). 1185eb8dc403SDave Cobbley """ 1186eb8dc403SDave Cobbley import oe.recipeutils 1187eb8dc403SDave Cobbley from oe.patch import GitApplyTree 1188eb8dc403SDave Cobbley updated = OrderedDict() 1189eb8dc403SDave Cobbley added = OrderedDict() 1190eb8dc403SDave Cobbley seqpatch_re = re.compile('^([0-9]{4}-)?(.+)') 1191eb8dc403SDave Cobbley 1192eb8dc403SDave Cobbley existing_patches = dict((os.path.basename(path), path) for path in 1193eb8dc403SDave Cobbley oe.recipeutils.get_recipe_patches(rd)) 1194eb8dc403SDave Cobbley logger.debug('Existing patches: %s' % existing_patches) 1195eb8dc403SDave Cobbley 1196eb8dc403SDave Cobbley # Generate patches from Git, exclude local files directory 1197eb8dc403SDave Cobbley patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1198eb8dc403SDave Cobbley GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) 1199eb8dc403SDave Cobbley 1200eb8dc403SDave Cobbley new_patches = sorted(os.listdir(destdir)) 1201eb8dc403SDave Cobbley for new_patch in new_patches: 1202eb8dc403SDave Cobbley # Strip numbering from patch names. If it's a git sequence named patch, 1203eb8dc403SDave Cobbley # the numbers might not match up since we are starting from a different 1204eb8dc403SDave Cobbley # revision This does assume that people are using unique shortlog 1205eb8dc403SDave Cobbley # values, but they ought to be anyway... 1206eb8dc403SDave Cobbley new_basename = seqpatch_re.match(new_patch).group(2) 1207eb8dc403SDave Cobbley match_name = None 1208eb8dc403SDave Cobbley for old_patch in existing_patches: 1209eb8dc403SDave Cobbley old_basename = seqpatch_re.match(old_patch).group(2) 1210eb8dc403SDave Cobbley old_basename_splitext = os.path.splitext(old_basename) 1211eb8dc403SDave Cobbley if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1212eb8dc403SDave Cobbley old_patch_noext = os.path.splitext(old_patch)[0] 1213eb8dc403SDave Cobbley match_name = old_patch_noext 1214eb8dc403SDave Cobbley break 1215eb8dc403SDave Cobbley elif new_basename == old_basename: 1216eb8dc403SDave Cobbley match_name = old_patch 1217eb8dc403SDave Cobbley break 1218eb8dc403SDave Cobbley if match_name: 1219eb8dc403SDave Cobbley # Rename patch files 1220eb8dc403SDave Cobbley if new_patch != match_name: 1221eb8dc403SDave Cobbley os.rename(os.path.join(destdir, new_patch), 1222eb8dc403SDave Cobbley os.path.join(destdir, match_name)) 1223eb8dc403SDave Cobbley # Need to pop it off the list now before checking changed_revs 1224eb8dc403SDave Cobbley oldpath = existing_patches.pop(old_patch) 1225eb8dc403SDave Cobbley if changed_revs is not None: 1226eb8dc403SDave Cobbley # Avoid updating patches that have not actually changed 1227eb8dc403SDave Cobbley with open(os.path.join(destdir, match_name), 'r') as f: 1228eb8dc403SDave Cobbley firstlineitems = f.readline().split() 1229eb8dc403SDave Cobbley # Looking for "From <hash>" line 1230eb8dc403SDave Cobbley if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1231eb8dc403SDave Cobbley if not firstlineitems[1] in changed_revs: 1232eb8dc403SDave Cobbley continue 1233eb8dc403SDave Cobbley # Recompress if necessary 1234eb8dc403SDave Cobbley if oldpath.endswith(('.gz', '.Z')): 1235eb8dc403SDave Cobbley bb.process.run(['gzip', match_name], cwd=destdir) 1236eb8dc403SDave Cobbley if oldpath.endswith('.gz'): 1237eb8dc403SDave Cobbley match_name += '.gz' 1238eb8dc403SDave Cobbley else: 1239eb8dc403SDave Cobbley match_name += '.Z' 1240eb8dc403SDave Cobbley elif oldpath.endswith('.bz2'): 1241eb8dc403SDave Cobbley bb.process.run(['bzip2', match_name], cwd=destdir) 1242eb8dc403SDave Cobbley match_name += '.bz2' 1243eb8dc403SDave Cobbley updated[match_name] = oldpath 1244eb8dc403SDave Cobbley else: 1245eb8dc403SDave Cobbley added[new_patch] = None 1246eb8dc403SDave Cobbley return (updated, added, existing_patches) 1247eb8dc403SDave Cobbley 1248eb8dc403SDave Cobbley 1249eb8dc403SDave Cobbleydef _create_kconfig_diff(srctree, rd, outfile): 1250eb8dc403SDave Cobbley """Create a kconfig fragment""" 1251eb8dc403SDave Cobbley # Only update config fragment if both config files exist 1252eb8dc403SDave Cobbley orig_config = os.path.join(srctree, '.config.baseline') 1253eb8dc403SDave Cobbley new_config = os.path.join(srctree, '.config.new') 1254eb8dc403SDave Cobbley if os.path.exists(orig_config) and os.path.exists(new_config): 1255eb8dc403SDave Cobbley cmd = ['diff', '--new-line-format=%L', '--old-line-format=', 1256eb8dc403SDave Cobbley '--unchanged-line-format=', orig_config, new_config] 1257eb8dc403SDave Cobbley pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 1258eb8dc403SDave Cobbley stderr=subprocess.PIPE) 1259eb8dc403SDave Cobbley stdout, stderr = pipe.communicate() 1260eb8dc403SDave Cobbley if pipe.returncode == 1: 1261eb8dc403SDave Cobbley logger.info("Updating config fragment %s" % outfile) 1262eb8dc403SDave Cobbley with open(outfile, 'wb') as fobj: 1263eb8dc403SDave Cobbley fobj.write(stdout) 1264eb8dc403SDave Cobbley elif pipe.returncode == 0: 1265eb8dc403SDave Cobbley logger.info("Would remove config fragment %s" % outfile) 1266eb8dc403SDave Cobbley if os.path.exists(outfile): 1267eb8dc403SDave Cobbley # Remove fragment file in case of empty diff 1268eb8dc403SDave Cobbley logger.info("Removing config fragment %s" % outfile) 1269eb8dc403SDave Cobbley os.unlink(outfile) 1270eb8dc403SDave Cobbley else: 1271eb8dc403SDave Cobbley raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr) 1272eb8dc403SDave Cobbley return True 1273eb8dc403SDave Cobbley return False 1274eb8dc403SDave Cobbley 1275eb8dc403SDave Cobbley 1276eb8dc403SDave Cobbleydef _export_local_files(srctree, rd, destdir, srctreebase): 1277eb8dc403SDave Cobbley """Copy local files from srctree to given location. 1278eb8dc403SDave Cobbley Returns three-tuple of dicts: 1279eb8dc403SDave Cobbley 1. updated - files that already exist in SRCURI 1280eb8dc403SDave Cobbley 2. added - new files files that don't exist in SRCURI 1281eb8dc403SDave Cobbley 3 removed - files that exist in SRCURI but not in exported files 1282eb8dc403SDave Cobbley In each dict the key is the 'basepath' of the URI and value is the 1283eb8dc403SDave Cobbley absolute path to the existing file in recipe space (if any). 1284eb8dc403SDave Cobbley """ 1285eb8dc403SDave Cobbley import oe.recipeutils 1286eb8dc403SDave Cobbley 1287eb8dc403SDave Cobbley # Find out local files (SRC_URI files that exist in the "recipe space"). 1288eb8dc403SDave Cobbley # Local files that reside in srctree are not included in patch generation. 1289eb8dc403SDave Cobbley # Instead they are directly copied over the original source files (in 1290eb8dc403SDave Cobbley # recipe space). 1291eb8dc403SDave Cobbley existing_files = oe.recipeutils.get_recipe_local_files(rd) 1292eb8dc403SDave Cobbley new_set = None 1293eb8dc403SDave Cobbley updated = OrderedDict() 1294eb8dc403SDave Cobbley added = OrderedDict() 1295eb8dc403SDave Cobbley removed = OrderedDict() 1296eb8dc403SDave Cobbley local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1297eb8dc403SDave Cobbley git_files = _git_ls_tree(srctree) 1298eb8dc403SDave Cobbley if 'oe-local-files' in git_files: 1299eb8dc403SDave Cobbley # If tracked by Git, take the files from srctree HEAD. First get 1300eb8dc403SDave Cobbley # the tree object of the directory 1301eb8dc403SDave Cobbley tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') 1302eb8dc403SDave Cobbley tree = git_files['oe-local-files'][2] 1303eb8dc403SDave Cobbley bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, 1304eb8dc403SDave Cobbley env=dict(os.environ, GIT_WORK_TREE=destdir, 1305eb8dc403SDave Cobbley GIT_INDEX_FILE=tmp_index)) 1306eb8dc403SDave Cobbley new_set = list(_git_ls_tree(srctree, tree, True).keys()) 1307eb8dc403SDave Cobbley elif os.path.isdir(local_files_dir): 1308eb8dc403SDave Cobbley # If not tracked by Git, just copy from working copy 1309eb8dc403SDave Cobbley new_set = _ls_tree(local_files_dir) 1310eb8dc403SDave Cobbley bb.process.run(['cp', '-ax', 1311eb8dc403SDave Cobbley os.path.join(local_files_dir, '.'), destdir]) 1312eb8dc403SDave Cobbley else: 1313eb8dc403SDave Cobbley new_set = [] 1314eb8dc403SDave Cobbley 1315eb8dc403SDave Cobbley # Special handling for kernel config 1316eb8dc403SDave Cobbley if bb.data.inherits_class('kernel-yocto', rd): 1317eb8dc403SDave Cobbley fragment_fn = 'devtool-fragment.cfg' 1318eb8dc403SDave Cobbley fragment_path = os.path.join(destdir, fragment_fn) 1319eb8dc403SDave Cobbley if _create_kconfig_diff(srctree, rd, fragment_path): 1320eb8dc403SDave Cobbley if os.path.exists(fragment_path): 1321eb8dc403SDave Cobbley if fragment_fn not in new_set: 1322eb8dc403SDave Cobbley new_set.append(fragment_fn) 1323eb8dc403SDave Cobbley # Copy fragment to local-files 1324eb8dc403SDave Cobbley if os.path.isdir(local_files_dir): 1325eb8dc403SDave Cobbley shutil.copy2(fragment_path, local_files_dir) 1326eb8dc403SDave Cobbley else: 1327eb8dc403SDave Cobbley if fragment_fn in new_set: 1328eb8dc403SDave Cobbley new_set.remove(fragment_fn) 1329eb8dc403SDave Cobbley # Remove fragment from local-files 1330eb8dc403SDave Cobbley if os.path.exists(os.path.join(local_files_dir, fragment_fn)): 1331eb8dc403SDave Cobbley os.unlink(os.path.join(local_files_dir, fragment_fn)) 1332eb8dc403SDave Cobbley 1333eb8dc403SDave Cobbley if new_set is not None: 1334eb8dc403SDave Cobbley for fname in new_set: 1335eb8dc403SDave Cobbley if fname in existing_files: 1336eb8dc403SDave Cobbley origpath = existing_files.pop(fname) 1337eb8dc403SDave Cobbley workpath = os.path.join(local_files_dir, fname) 1338eb8dc403SDave Cobbley if not filecmp.cmp(origpath, workpath): 1339eb8dc403SDave Cobbley updated[fname] = origpath 1340eb8dc403SDave Cobbley elif fname != '.gitignore': 1341eb8dc403SDave Cobbley added[fname] = None 1342eb8dc403SDave Cobbley 1343eb8dc403SDave Cobbley workdir = rd.getVar('WORKDIR') 1344eb8dc403SDave Cobbley s = rd.getVar('S') 1345eb8dc403SDave Cobbley if not s.endswith(os.sep): 1346eb8dc403SDave Cobbley s += os.sep 1347eb8dc403SDave Cobbley 1348eb8dc403SDave Cobbley if workdir != s: 1349eb8dc403SDave Cobbley # Handle files where subdir= was specified 1350eb8dc403SDave Cobbley for fname in list(existing_files.keys()): 1351eb8dc403SDave Cobbley # FIXME handle both subdir starting with BP and not? 1352eb8dc403SDave Cobbley fworkpath = os.path.join(workdir, fname) 1353eb8dc403SDave Cobbley if fworkpath.startswith(s): 1354eb8dc403SDave Cobbley fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) 1355eb8dc403SDave Cobbley if os.path.exists(fpath): 1356eb8dc403SDave Cobbley origpath = existing_files.pop(fname) 1357eb8dc403SDave Cobbley if not filecmp.cmp(origpath, fpath): 1358eb8dc403SDave Cobbley updated[fpath] = origpath 1359eb8dc403SDave Cobbley 1360eb8dc403SDave Cobbley removed = existing_files 1361eb8dc403SDave Cobbley return (updated, added, removed) 1362eb8dc403SDave Cobbley 1363eb8dc403SDave Cobbley 1364eb8dc403SDave Cobbleydef _determine_files_dir(rd): 1365eb8dc403SDave Cobbley """Determine the appropriate files directory for a recipe""" 1366eb8dc403SDave Cobbley recipedir = rd.getVar('FILE_DIRNAME') 1367eb8dc403SDave Cobbley for entry in rd.getVar('FILESPATH').split(':'): 1368eb8dc403SDave Cobbley relpth = os.path.relpath(entry, recipedir) 1369eb8dc403SDave Cobbley if not os.sep in relpth: 1370eb8dc403SDave Cobbley # One (or zero) levels below only, so we don't put anything in machine-specific directories 1371eb8dc403SDave Cobbley if os.path.isdir(entry): 1372eb8dc403SDave Cobbley return entry 1373eb8dc403SDave Cobbley return os.path.join(recipedir, rd.getVar('BPN')) 1374eb8dc403SDave Cobbley 1375eb8dc403SDave Cobbley 1376eb8dc403SDave Cobbleydef _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None): 1377eb8dc403SDave Cobbley """Implement the 'srcrev' mode of update-recipe""" 1378eb8dc403SDave Cobbley import bb 1379eb8dc403SDave Cobbley import oe.recipeutils 1380eb8dc403SDave Cobbley 1381eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1382eb8dc403SDave Cobbley 1383eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 1384eb8dc403SDave Cobbley recipedir = os.path.basename(recipefile) 1385eb8dc403SDave Cobbley logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1386eb8dc403SDave Cobbley 1387eb8dc403SDave Cobbley # Get HEAD revision 1388eb8dc403SDave Cobbley try: 1389eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1390eb8dc403SDave Cobbley except bb.process.ExecutionError as err: 1391eb8dc403SDave Cobbley raise DevtoolError('Failed to get HEAD revision in %s: %s' % 1392eb8dc403SDave Cobbley (srctree, err)) 1393eb8dc403SDave Cobbley srcrev = stdout.strip() 1394eb8dc403SDave Cobbley if len(srcrev) != 40: 1395eb8dc403SDave Cobbley raise DevtoolError('Invalid hash returned by git: %s' % stdout) 1396eb8dc403SDave Cobbley 1397eb8dc403SDave Cobbley destpath = None 1398eb8dc403SDave Cobbley remove_files = [] 1399eb8dc403SDave Cobbley patchfields = {} 1400eb8dc403SDave Cobbley patchfields['SRCREV'] = srcrev 1401eb8dc403SDave Cobbley orig_src_uri = rd.getVar('SRC_URI', False) or '' 1402eb8dc403SDave Cobbley srcuri = orig_src_uri.split() 1403eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtool') 1404eb8dc403SDave Cobbley update_srcuri = False 1405eb8dc403SDave Cobbley appendfile = None 1406eb8dc403SDave Cobbley try: 1407eb8dc403SDave Cobbley local_files_dir = tempfile.mkdtemp(dir=tempdir) 1408eb8dc403SDave Cobbley srctreebase = workspace[recipename]['srctreebase'] 1409eb8dc403SDave Cobbley upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1410eb8dc403SDave Cobbley if not no_remove: 1411eb8dc403SDave Cobbley # Find list of existing patches in recipe file 1412eb8dc403SDave Cobbley patches_dir = tempfile.mkdtemp(dir=tempdir) 1413eb8dc403SDave Cobbley old_srcrev = rd.getVar('SRCREV') or '' 1414eb8dc403SDave Cobbley upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1415eb8dc403SDave Cobbley patches_dir) 1416eb8dc403SDave Cobbley logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1417eb8dc403SDave Cobbley 1418eb8dc403SDave Cobbley # Remove deleted local files and "overlapping" patches 1419eb8dc403SDave Cobbley remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) 1420eb8dc403SDave Cobbley if remove_files: 1421eb8dc403SDave Cobbley removedentries = _remove_file_entries(srcuri, remove_files)[0] 1422eb8dc403SDave Cobbley update_srcuri = True 1423eb8dc403SDave Cobbley 1424eb8dc403SDave Cobbley if appendlayerdir: 1425eb8dc403SDave Cobbley files = dict((os.path.join(local_files_dir, key), val) for 1426eb8dc403SDave Cobbley key, val in list(upd_f.items()) + list(new_f.items())) 1427eb8dc403SDave Cobbley removevalues = {} 1428eb8dc403SDave Cobbley if update_srcuri: 1429eb8dc403SDave Cobbley removevalues = {'SRC_URI': removedentries} 1430eb8dc403SDave Cobbley patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1431eb8dc403SDave Cobbley if dry_run_outdir: 1432eb8dc403SDave Cobbley logger.info('Creating bbappend (dry-run)') 1433eb8dc403SDave Cobbley else: 1434eb8dc403SDave Cobbley appendfile, destpath = oe.recipeutils.bbappend_recipe( 1435eb8dc403SDave Cobbley rd, appendlayerdir, files, wildcardver=wildcard_version, 1436eb8dc403SDave Cobbley extralines=patchfields, removevalues=removevalues, 1437eb8dc403SDave Cobbley redirect_output=dry_run_outdir) 1438eb8dc403SDave Cobbley else: 1439eb8dc403SDave Cobbley files_dir = _determine_files_dir(rd) 1440eb8dc403SDave Cobbley for basepath, path in upd_f.items(): 1441eb8dc403SDave Cobbley logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1442eb8dc403SDave Cobbley if os.path.isabs(basepath): 1443eb8dc403SDave Cobbley # Original file (probably with subdir pointing inside source tree) 1444eb8dc403SDave Cobbley # so we do not want to move it, just copy 1445eb8dc403SDave Cobbley _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1446eb8dc403SDave Cobbley else: 1447eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), path, 1448eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1449eb8dc403SDave Cobbley update_srcuri= True 1450eb8dc403SDave Cobbley for basepath, path in new_f.items(): 1451eb8dc403SDave Cobbley logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1452eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), 1453eb8dc403SDave Cobbley os.path.join(files_dir, basepath), 1454eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, 1455eb8dc403SDave Cobbley base_outdir=recipedir) 1456eb8dc403SDave Cobbley srcuri.append('file://%s' % basepath) 1457eb8dc403SDave Cobbley update_srcuri = True 1458eb8dc403SDave Cobbley if update_srcuri: 1459eb8dc403SDave Cobbley patchfields['SRC_URI'] = ' '.join(srcuri) 1460eb8dc403SDave Cobbley ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir) 1461eb8dc403SDave Cobbley finally: 1462eb8dc403SDave Cobbley shutil.rmtree(tempdir) 1463eb8dc403SDave Cobbley if not 'git://' in orig_src_uri: 1464eb8dc403SDave Cobbley logger.info('You will need to update SRC_URI within the recipe to ' 1465eb8dc403SDave Cobbley 'point to a git repository where you have pushed your ' 1466eb8dc403SDave Cobbley 'changes') 1467eb8dc403SDave Cobbley 1468eb8dc403SDave Cobbley _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1469eb8dc403SDave Cobbley return True, appendfile, remove_files 1470eb8dc403SDave Cobbley 1471eb8dc403SDave Cobbleydef _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False): 1472eb8dc403SDave Cobbley """Implement the 'patch' mode of update-recipe""" 1473eb8dc403SDave Cobbley import bb 1474eb8dc403SDave Cobbley import oe.recipeutils 1475eb8dc403SDave Cobbley 1476eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 1477eb8dc403SDave Cobbley recipedir = os.path.dirname(recipefile) 1478eb8dc403SDave Cobbley append = workspace[recipename]['bbappend'] 1479eb8dc403SDave Cobbley if not os.path.exists(append): 1480eb8dc403SDave Cobbley raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1481eb8dc403SDave Cobbley recipename) 1482eb8dc403SDave Cobbley 1483eb8dc403SDave Cobbley initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1484eb8dc403SDave Cobbley if not initial_rev: 1485eb8dc403SDave Cobbley raise DevtoolError('Unable to find initial revision - please specify ' 1486eb8dc403SDave Cobbley 'it with --initial-rev') 1487eb8dc403SDave Cobbley 1488eb8dc403SDave Cobbley appendfile = None 1489eb8dc403SDave Cobbley dl_dir = rd.getVar('DL_DIR') 1490eb8dc403SDave Cobbley if not dl_dir.endswith('/'): 1491eb8dc403SDave Cobbley dl_dir += '/' 1492eb8dc403SDave Cobbley 1493eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 1494eb8dc403SDave Cobbley 1495eb8dc403SDave Cobbley tempdir = tempfile.mkdtemp(prefix='devtool') 1496eb8dc403SDave Cobbley try: 1497eb8dc403SDave Cobbley local_files_dir = tempfile.mkdtemp(dir=tempdir) 1498eb8dc403SDave Cobbley if filter_patches: 1499eb8dc403SDave Cobbley upd_f = {} 1500eb8dc403SDave Cobbley new_f = {} 1501eb8dc403SDave Cobbley del_f = {} 1502eb8dc403SDave Cobbley else: 1503eb8dc403SDave Cobbley srctreebase = workspace[recipename]['srctreebase'] 1504eb8dc403SDave Cobbley upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase) 1505eb8dc403SDave Cobbley 1506eb8dc403SDave Cobbley remove_files = [] 1507eb8dc403SDave Cobbley if not no_remove: 1508eb8dc403SDave Cobbley # Get all patches from source tree and check if any should be removed 1509eb8dc403SDave Cobbley all_patches_dir = tempfile.mkdtemp(dir=tempdir) 1510eb8dc403SDave Cobbley _, _, del_p = _export_patches(srctree, rd, initial_rev, 1511eb8dc403SDave Cobbley all_patches_dir) 1512eb8dc403SDave Cobbley # Remove deleted local files and patches 1513eb8dc403SDave Cobbley remove_files = list(del_f.values()) + list(del_p.values()) 1514eb8dc403SDave Cobbley 1515eb8dc403SDave Cobbley # Get updated patches from source tree 1516eb8dc403SDave Cobbley patches_dir = tempfile.mkdtemp(dir=tempdir) 1517eb8dc403SDave Cobbley upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, 1518eb8dc403SDave Cobbley patches_dir, changed_revs) 1519eb8dc403SDave Cobbley logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1520eb8dc403SDave Cobbley if filter_patches: 1521eb8dc403SDave Cobbley new_p = {} 1522eb8dc403SDave Cobbley upd_p = {k:v for k,v in upd_p.items() if k in filter_patches} 1523eb8dc403SDave Cobbley remove_files = [f for f in remove_files if f in filter_patches] 1524eb8dc403SDave Cobbley updatefiles = False 1525eb8dc403SDave Cobbley updaterecipe = False 1526eb8dc403SDave Cobbley destpath = None 1527eb8dc403SDave Cobbley srcuri = (rd.getVar('SRC_URI', False) or '').split() 1528eb8dc403SDave Cobbley if appendlayerdir: 1529eb8dc403SDave Cobbley files = dict((os.path.join(local_files_dir, key), val) for 1530eb8dc403SDave Cobbley key, val in list(upd_f.items()) + list(new_f.items())) 1531eb8dc403SDave Cobbley files.update(dict((os.path.join(patches_dir, key), val) for 1532eb8dc403SDave Cobbley key, val in list(upd_p.items()) + list(new_p.items()))) 1533eb8dc403SDave Cobbley if files or remove_files: 1534eb8dc403SDave Cobbley removevalues = None 1535eb8dc403SDave Cobbley if remove_files: 1536eb8dc403SDave Cobbley removedentries, remaining = _remove_file_entries( 1537eb8dc403SDave Cobbley srcuri, remove_files) 1538eb8dc403SDave Cobbley if removedentries or remaining: 1539eb8dc403SDave Cobbley remaining = ['file://' + os.path.basename(item) for 1540eb8dc403SDave Cobbley item in remaining] 1541eb8dc403SDave Cobbley removevalues = {'SRC_URI': removedentries + remaining} 1542eb8dc403SDave Cobbley appendfile, destpath = oe.recipeutils.bbappend_recipe( 1543eb8dc403SDave Cobbley rd, appendlayerdir, files, 1544eb8dc403SDave Cobbley wildcardver=wildcard_version, 1545eb8dc403SDave Cobbley removevalues=removevalues, 1546eb8dc403SDave Cobbley redirect_output=dry_run_outdir) 1547eb8dc403SDave Cobbley else: 1548eb8dc403SDave Cobbley logger.info('No patches or local source files needed updating') 1549eb8dc403SDave Cobbley else: 1550eb8dc403SDave Cobbley # Update existing files 1551eb8dc403SDave Cobbley files_dir = _determine_files_dir(rd) 1552eb8dc403SDave Cobbley for basepath, path in upd_f.items(): 1553eb8dc403SDave Cobbley logger.info('Updating file %s' % basepath) 1554eb8dc403SDave Cobbley if os.path.isabs(basepath): 1555eb8dc403SDave Cobbley # Original file (probably with subdir pointing inside source tree) 1556eb8dc403SDave Cobbley # so we do not want to move it, just copy 1557eb8dc403SDave Cobbley _copy_file(basepath, path, 1558eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1559eb8dc403SDave Cobbley else: 1560eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), path, 1561eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1562eb8dc403SDave Cobbley updatefiles = True 1563eb8dc403SDave Cobbley for basepath, path in upd_p.items(): 1564eb8dc403SDave Cobbley patchfn = os.path.join(patches_dir, basepath) 1565eb8dc403SDave Cobbley if os.path.dirname(path) + '/' == dl_dir: 1566eb8dc403SDave Cobbley # This is a a downloaded patch file - we now need to 1567eb8dc403SDave Cobbley # replace the entry in SRC_URI with our local version 1568eb8dc403SDave Cobbley logger.info('Replacing remote patch %s with updated local version' % basepath) 1569eb8dc403SDave Cobbley path = os.path.join(files_dir, basepath) 1570eb8dc403SDave Cobbley _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) 1571eb8dc403SDave Cobbley updaterecipe = True 1572eb8dc403SDave Cobbley else: 1573eb8dc403SDave Cobbley logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1574eb8dc403SDave Cobbley _move_file(patchfn, path, 1575eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1576eb8dc403SDave Cobbley updatefiles = True 1577eb8dc403SDave Cobbley # Add any new files 1578eb8dc403SDave Cobbley for basepath, path in new_f.items(): 1579eb8dc403SDave Cobbley logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1580eb8dc403SDave Cobbley _move_file(os.path.join(local_files_dir, basepath), 1581eb8dc403SDave Cobbley os.path.join(files_dir, basepath), 1582eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, 1583eb8dc403SDave Cobbley base_outdir=recipedir) 1584eb8dc403SDave Cobbley srcuri.append('file://%s' % basepath) 1585eb8dc403SDave Cobbley updaterecipe = True 1586eb8dc403SDave Cobbley for basepath, path in new_p.items(): 1587eb8dc403SDave Cobbley logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1588eb8dc403SDave Cobbley _move_file(os.path.join(patches_dir, basepath), 1589eb8dc403SDave Cobbley os.path.join(files_dir, basepath), 1590eb8dc403SDave Cobbley dry_run_outdir=dry_run_outdir, 1591eb8dc403SDave Cobbley base_outdir=recipedir) 1592eb8dc403SDave Cobbley srcuri.append('file://%s' % basepath) 1593eb8dc403SDave Cobbley updaterecipe = True 1594eb8dc403SDave Cobbley # Update recipe, if needed 1595eb8dc403SDave Cobbley if _remove_file_entries(srcuri, remove_files)[0]: 1596eb8dc403SDave Cobbley updaterecipe = True 1597eb8dc403SDave Cobbley if updaterecipe: 1598eb8dc403SDave Cobbley if not dry_run_outdir: 1599eb8dc403SDave Cobbley logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1600eb8dc403SDave Cobbley ret = oe.recipeutils.patch_recipe(rd, recipefile, 1601eb8dc403SDave Cobbley {'SRC_URI': ' '.join(srcuri)}, 1602eb8dc403SDave Cobbley redirect_output=dry_run_outdir) 1603eb8dc403SDave Cobbley elif not updatefiles: 1604eb8dc403SDave Cobbley # Neither patches nor recipe were updated 1605eb8dc403SDave Cobbley logger.info('No patches or files need updating') 1606eb8dc403SDave Cobbley return False, None, [] 1607eb8dc403SDave Cobbley finally: 1608eb8dc403SDave Cobbley shutil.rmtree(tempdir) 1609eb8dc403SDave Cobbley 1610eb8dc403SDave Cobbley _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir) 1611eb8dc403SDave Cobbley return True, appendfile, remove_files 1612eb8dc403SDave Cobbley 1613eb8dc403SDave Cobbleydef _guess_recipe_update_mode(srctree, rdata): 1614eb8dc403SDave Cobbley """Guess the recipe update mode to use""" 1615eb8dc403SDave Cobbley src_uri = (rdata.getVar('SRC_URI', False) or '').split() 1616eb8dc403SDave Cobbley git_uris = [uri for uri in src_uri if uri.startswith('git://')] 1617eb8dc403SDave Cobbley if not git_uris: 1618eb8dc403SDave Cobbley return 'patch' 1619eb8dc403SDave Cobbley # Just use the first URI for now 1620eb8dc403SDave Cobbley uri = git_uris[0] 1621eb8dc403SDave Cobbley # Check remote branch 1622eb8dc403SDave Cobbley params = bb.fetch.decodeurl(uri)[5] 1623eb8dc403SDave Cobbley upstr_branch = params['branch'] if 'branch' in params else 'master' 1624eb8dc403SDave Cobbley # Check if current branch HEAD is found in upstream branch 1625eb8dc403SDave Cobbley stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1626eb8dc403SDave Cobbley head_rev = stdout.rstrip() 1627eb8dc403SDave Cobbley stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev, 1628eb8dc403SDave Cobbley cwd=srctree) 1629eb8dc403SDave Cobbley remote_brs = [branch.strip() for branch in stdout.splitlines()] 1630eb8dc403SDave Cobbley if 'origin/' + upstr_branch in remote_brs: 1631eb8dc403SDave Cobbley return 'srcrev' 1632eb8dc403SDave Cobbley 1633eb8dc403SDave Cobbley return 'patch' 1634eb8dc403SDave Cobbley 1635eb8dc403SDave Cobbleydef _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False): 1636eb8dc403SDave Cobbley srctree = workspace[recipename]['srctree'] 1637eb8dc403SDave Cobbley if mode == 'auto': 1638eb8dc403SDave Cobbley mode = _guess_recipe_update_mode(srctree, rd) 1639eb8dc403SDave Cobbley 1640eb8dc403SDave Cobbley override_branches = [] 1641eb8dc403SDave Cobbley mainbranch = None 1642eb8dc403SDave Cobbley startbranch = None 1643eb8dc403SDave Cobbley if not no_overrides: 1644eb8dc403SDave Cobbley stdout, _ = bb.process.run('git branch', cwd=srctree) 1645eb8dc403SDave Cobbley other_branches = [] 1646eb8dc403SDave Cobbley for line in stdout.splitlines(): 1647eb8dc403SDave Cobbley branchname = line[2:] 1648eb8dc403SDave Cobbley if line.startswith('* '): 1649eb8dc403SDave Cobbley startbranch = branchname 1650eb8dc403SDave Cobbley if branchname.startswith(override_branch_prefix): 1651eb8dc403SDave Cobbley override_branches.append(branchname) 1652eb8dc403SDave Cobbley else: 1653eb8dc403SDave Cobbley other_branches.append(branchname) 1654eb8dc403SDave Cobbley 1655eb8dc403SDave Cobbley if override_branches: 1656eb8dc403SDave Cobbley logger.debug('_update_recipe: override branches: %s' % override_branches) 1657eb8dc403SDave Cobbley logger.debug('_update_recipe: other branches: %s' % other_branches) 1658eb8dc403SDave Cobbley if startbranch.startswith(override_branch_prefix): 1659eb8dc403SDave Cobbley if len(other_branches) == 1: 1660eb8dc403SDave Cobbley mainbranch = other_branches[1] 1661eb8dc403SDave Cobbley else: 1662eb8dc403SDave Cobbley raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first') 1663eb8dc403SDave Cobbley else: 1664eb8dc403SDave Cobbley mainbranch = startbranch 1665eb8dc403SDave Cobbley 1666eb8dc403SDave Cobbley checkedout = None 1667eb8dc403SDave Cobbley anyupdated = False 1668eb8dc403SDave Cobbley appendfile = None 1669eb8dc403SDave Cobbley allremoved = [] 1670eb8dc403SDave Cobbley if override_branches: 1671eb8dc403SDave Cobbley logger.info('Handling main branch (%s)...' % mainbranch) 1672eb8dc403SDave Cobbley if startbranch != mainbranch: 1673eb8dc403SDave Cobbley bb.process.run('git checkout %s' % mainbranch, cwd=srctree) 1674eb8dc403SDave Cobbley checkedout = mainbranch 1675eb8dc403SDave Cobbley try: 1676eb8dc403SDave Cobbley branchlist = [mainbranch] + override_branches 1677eb8dc403SDave Cobbley for branch in branchlist: 1678eb8dc403SDave Cobbley crd = bb.data.createCopy(rd) 1679eb8dc403SDave Cobbley if branch != mainbranch: 1680eb8dc403SDave Cobbley logger.info('Handling branch %s...' % branch) 1681eb8dc403SDave Cobbley override = branch[len(override_branch_prefix):] 1682eb8dc403SDave Cobbley crd.appendVar('OVERRIDES', ':%s' % override) 1683eb8dc403SDave Cobbley bb.process.run('git checkout %s' % branch, cwd=srctree) 1684eb8dc403SDave Cobbley checkedout = branch 1685eb8dc403SDave Cobbley 1686eb8dc403SDave Cobbley if mode == 'srcrev': 1687eb8dc403SDave Cobbley updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir) 1688eb8dc403SDave Cobbley elif mode == 'patch': 1689eb8dc403SDave Cobbley updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh) 1690eb8dc403SDave Cobbley else: 1691eb8dc403SDave Cobbley raise DevtoolError('update_recipe: invalid mode %s' % mode) 1692eb8dc403SDave Cobbley if updated: 1693eb8dc403SDave Cobbley anyupdated = True 1694eb8dc403SDave Cobbley if appendf: 1695eb8dc403SDave Cobbley appendfile = appendf 1696eb8dc403SDave Cobbley allremoved.extend(removed) 1697eb8dc403SDave Cobbley finally: 1698eb8dc403SDave Cobbley if startbranch and checkedout != startbranch: 1699eb8dc403SDave Cobbley bb.process.run('git checkout %s' % startbranch, cwd=srctree) 1700eb8dc403SDave Cobbley 1701eb8dc403SDave Cobbley return anyupdated, appendfile, allremoved 1702eb8dc403SDave Cobbley 1703eb8dc403SDave Cobbleydef update_recipe(args, config, basepath, workspace): 1704eb8dc403SDave Cobbley """Entry point for the devtool 'update-recipe' subcommand""" 1705eb8dc403SDave Cobbley check_workspace_recipe(workspace, args.recipename) 1706eb8dc403SDave Cobbley 1707eb8dc403SDave Cobbley if args.append: 1708eb8dc403SDave Cobbley if not os.path.exists(args.append): 1709eb8dc403SDave Cobbley raise DevtoolError('bbappend destination layer directory "%s" ' 1710eb8dc403SDave Cobbley 'does not exist' % args.append) 1711eb8dc403SDave Cobbley if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')): 1712eb8dc403SDave Cobbley raise DevtoolError('conf/layer.conf not found in bbappend ' 1713eb8dc403SDave Cobbley 'destination layer "%s"' % args.append) 1714eb8dc403SDave Cobbley 1715eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1716eb8dc403SDave Cobbley try: 1717eb8dc403SDave Cobbley 1718eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 1719eb8dc403SDave Cobbley if not rd: 1720eb8dc403SDave Cobbley return 1 1721eb8dc403SDave Cobbley 1722eb8dc403SDave Cobbley dry_run_output = None 1723eb8dc403SDave Cobbley dry_run_outdir = None 1724eb8dc403SDave Cobbley if args.dry_run: 1725eb8dc403SDave Cobbley dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1726eb8dc403SDave Cobbley dry_run_outdir = dry_run_output.name 1727eb8dc403SDave Cobbley updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1728eb8dc403SDave Cobbley 1729eb8dc403SDave Cobbley if updated: 1730eb8dc403SDave Cobbley rf = rd.getVar('FILE') 1731eb8dc403SDave Cobbley if rf.startswith(config.workspace_path): 17321a4b7ee2SBrad Bishop logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf) 1733eb8dc403SDave Cobbley finally: 1734eb8dc403SDave Cobbley tinfoil.shutdown() 1735eb8dc403SDave Cobbley 1736eb8dc403SDave Cobbley return 0 1737eb8dc403SDave Cobbley 1738eb8dc403SDave Cobbley 1739eb8dc403SDave Cobbleydef status(args, config, basepath, workspace): 1740eb8dc403SDave Cobbley """Entry point for the devtool 'status' subcommand""" 1741eb8dc403SDave Cobbley if workspace: 1742eb8dc403SDave Cobbley for recipe, value in sorted(workspace.items()): 1743eb8dc403SDave Cobbley recipefile = value['recipefile'] 1744eb8dc403SDave Cobbley if recipefile: 1745eb8dc403SDave Cobbley recipestr = ' (%s)' % recipefile 1746eb8dc403SDave Cobbley else: 1747eb8dc403SDave Cobbley recipestr = '' 1748eb8dc403SDave Cobbley print("%s: %s%s" % (recipe, value['srctree'], recipestr)) 1749eb8dc403SDave Cobbley else: 1750eb8dc403SDave Cobbley logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one') 1751eb8dc403SDave Cobbley return 0 1752eb8dc403SDave Cobbley 1753eb8dc403SDave Cobbley 1754eb8dc403SDave Cobbleydef _reset(recipes, no_clean, config, basepath, workspace): 1755eb8dc403SDave Cobbley """Reset one or more recipes""" 1756eb8dc403SDave Cobbley import oe.path 1757eb8dc403SDave Cobbley 1758eb8dc403SDave Cobbley def clean_preferred_provider(pn, layerconf_path): 1759eb8dc403SDave Cobbley """Remove PREFERRED_PROVIDER from layer.conf'""" 1760eb8dc403SDave Cobbley import re 1761eb8dc403SDave Cobbley layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf') 1762eb8dc403SDave Cobbley new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf') 1763eb8dc403SDave Cobbley pprovider_found = False 1764eb8dc403SDave Cobbley with open(layerconf_file, 'r') as f: 1765eb8dc403SDave Cobbley lines = f.readlines() 1766eb8dc403SDave Cobbley with open(new_layerconf_file, 'a') as nf: 1767eb8dc403SDave Cobbley for line in lines: 1768eb8dc403SDave Cobbley pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$' 1769eb8dc403SDave Cobbley if not re.match(pprovider_exp, line): 1770eb8dc403SDave Cobbley nf.write(line) 1771eb8dc403SDave Cobbley else: 1772eb8dc403SDave Cobbley pprovider_found = True 1773eb8dc403SDave Cobbley if pprovider_found: 1774eb8dc403SDave Cobbley shutil.move(new_layerconf_file, layerconf_file) 1775eb8dc403SDave Cobbley else: 1776eb8dc403SDave Cobbley os.remove(new_layerconf_file) 1777eb8dc403SDave Cobbley 1778eb8dc403SDave Cobbley if recipes and not no_clean: 1779eb8dc403SDave Cobbley if len(recipes) == 1: 1780eb8dc403SDave Cobbley logger.info('Cleaning sysroot for recipe %s...' % recipes[0]) 1781eb8dc403SDave Cobbley else: 1782eb8dc403SDave Cobbley logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes)) 1783eb8dc403SDave Cobbley # If the recipe file itself was created in the workspace, and 1784eb8dc403SDave Cobbley # it uses BBCLASSEXTEND, then we need to also clean the other 1785eb8dc403SDave Cobbley # variants 1786eb8dc403SDave Cobbley targets = [] 1787eb8dc403SDave Cobbley for recipe in recipes: 1788eb8dc403SDave Cobbley targets.append(recipe) 1789eb8dc403SDave Cobbley recipefile = workspace[recipe]['recipefile'] 1790eb8dc403SDave Cobbley if recipefile and os.path.exists(recipefile): 1791eb8dc403SDave Cobbley targets.extend(get_bbclassextend_targets(recipefile, recipe)) 1792eb8dc403SDave Cobbley try: 1793eb8dc403SDave Cobbley exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets)) 1794eb8dc403SDave Cobbley except bb.process.ExecutionError as e: 1795eb8dc403SDave Cobbley raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you ' 1796eb8dc403SDave Cobbley 'wish, you may specify -n/--no-clean to ' 1797eb8dc403SDave Cobbley 'skip running this command when resetting' % 1798eb8dc403SDave Cobbley (e.command, e.stdout)) 1799eb8dc403SDave Cobbley 1800eb8dc403SDave Cobbley for pn in recipes: 1801eb8dc403SDave Cobbley _check_preserve(config, pn) 1802eb8dc403SDave Cobbley 1803eb8dc403SDave Cobbley appendfile = workspace[pn]['bbappend'] 1804eb8dc403SDave Cobbley if os.path.exists(appendfile): 1805eb8dc403SDave Cobbley # This shouldn't happen, but is possible if devtool errored out prior to 1806eb8dc403SDave Cobbley # writing the md5 file. We need to delete this here or the recipe won't 1807eb8dc403SDave Cobbley # actually be reset 1808eb8dc403SDave Cobbley os.remove(appendfile) 1809eb8dc403SDave Cobbley 1810eb8dc403SDave Cobbley preservepath = os.path.join(config.workspace_path, 'attic', pn, pn) 1811eb8dc403SDave Cobbley def preservedir(origdir): 1812eb8dc403SDave Cobbley if os.path.exists(origdir): 1813eb8dc403SDave Cobbley for root, dirs, files in os.walk(origdir): 1814eb8dc403SDave Cobbley for fn in files: 18151a4b7ee2SBrad Bishop logger.warning('Preserving %s in %s' % (fn, preservepath)) 1816eb8dc403SDave Cobbley _move_file(os.path.join(origdir, fn), 1817eb8dc403SDave Cobbley os.path.join(preservepath, fn)) 1818eb8dc403SDave Cobbley for dn in dirs: 1819eb8dc403SDave Cobbley preservedir(os.path.join(root, dn)) 1820eb8dc403SDave Cobbley os.rmdir(origdir) 1821eb8dc403SDave Cobbley 1822eb8dc403SDave Cobbley recipefile = workspace[pn]['recipefile'] 1823eb8dc403SDave Cobbley if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile): 1824eb8dc403SDave Cobbley # This should always be true if recipefile is set, but just in case 1825eb8dc403SDave Cobbley preservedir(os.path.dirname(recipefile)) 1826eb8dc403SDave Cobbley # We don't automatically create this dir next to appends, but the user can 1827eb8dc403SDave Cobbley preservedir(os.path.join(config.workspace_path, 'appends', pn)) 1828eb8dc403SDave Cobbley 1829eb8dc403SDave Cobbley srctreebase = workspace[pn]['srctreebase'] 1830eb8dc403SDave Cobbley if os.path.isdir(srctreebase): 1831eb8dc403SDave Cobbley if os.listdir(srctreebase): 1832eb8dc403SDave Cobbley # We don't want to risk wiping out any work in progress 1833eb8dc403SDave Cobbley logger.info('Leaving source tree %s as-is; if you no ' 1834eb8dc403SDave Cobbley 'longer need it then please delete it manually' 1835eb8dc403SDave Cobbley % srctreebase) 1836eb8dc403SDave Cobbley else: 1837eb8dc403SDave Cobbley # This is unlikely, but if it's empty we can just remove it 1838eb8dc403SDave Cobbley os.rmdir(srctreebase) 1839eb8dc403SDave Cobbley 1840eb8dc403SDave Cobbley clean_preferred_provider(pn, config.workspace_path) 1841eb8dc403SDave Cobbley 1842eb8dc403SDave Cobbleydef reset(args, config, basepath, workspace): 1843eb8dc403SDave Cobbley """Entry point for the devtool 'reset' subcommand""" 1844eb8dc403SDave Cobbley import bb 1845eb8dc403SDave Cobbley if args.recipename: 1846eb8dc403SDave Cobbley if args.all: 1847eb8dc403SDave Cobbley raise DevtoolError("Recipe cannot be specified if -a/--all is used") 1848eb8dc403SDave Cobbley else: 1849eb8dc403SDave Cobbley for recipe in args.recipename: 1850eb8dc403SDave Cobbley check_workspace_recipe(workspace, recipe, checksrc=False) 1851eb8dc403SDave Cobbley elif not args.all: 1852eb8dc403SDave Cobbley raise DevtoolError("Recipe must be specified, or specify -a/--all to " 1853eb8dc403SDave Cobbley "reset all recipes") 1854eb8dc403SDave Cobbley if args.all: 1855eb8dc403SDave Cobbley recipes = list(workspace.keys()) 1856eb8dc403SDave Cobbley else: 1857eb8dc403SDave Cobbley recipes = args.recipename 1858eb8dc403SDave Cobbley 1859eb8dc403SDave Cobbley _reset(recipes, args.no_clean, config, basepath, workspace) 1860eb8dc403SDave Cobbley 1861eb8dc403SDave Cobbley return 0 1862eb8dc403SDave Cobbley 1863eb8dc403SDave Cobbley 1864eb8dc403SDave Cobbleydef _get_layer(layername, d): 1865eb8dc403SDave Cobbley """Determine the base layer path for the specified layer name/path""" 1866eb8dc403SDave Cobbley layerdirs = d.getVar('BBLAYERS').split() 1867eb8dc403SDave Cobbley layers = {os.path.basename(p): p for p in layerdirs} 1868eb8dc403SDave Cobbley # Provide some shortcuts 1869eb8dc403SDave Cobbley if layername.lower() in ['oe-core', 'openembedded-core']: 1870eb8dc403SDave Cobbley layerdir = layers.get('meta', None) 1871eb8dc403SDave Cobbley else: 1872eb8dc403SDave Cobbley layerdir = layers.get(layername, None) 1873eb8dc403SDave Cobbley return os.path.abspath(layerdir or layername) 1874eb8dc403SDave Cobbley 1875eb8dc403SDave Cobbleydef finish(args, config, basepath, workspace): 1876eb8dc403SDave Cobbley """Entry point for the devtool 'finish' subcommand""" 1877eb8dc403SDave Cobbley import bb 1878eb8dc403SDave Cobbley import oe.recipeutils 1879eb8dc403SDave Cobbley 1880eb8dc403SDave Cobbley check_workspace_recipe(workspace, args.recipename) 1881eb8dc403SDave Cobbley 1882eb8dc403SDave Cobbley dry_run_suffix = ' (dry-run)' if args.dry_run else '' 1883eb8dc403SDave Cobbley 1884eb8dc403SDave Cobbley # Grab the equivalent of COREBASE without having to initialise tinfoil 1885eb8dc403SDave Cobbley corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 1886eb8dc403SDave Cobbley 1887eb8dc403SDave Cobbley srctree = workspace[args.recipename]['srctree'] 1888eb8dc403SDave Cobbley check_git_repo_op(srctree, [corebasedir]) 1889eb8dc403SDave Cobbley dirty = check_git_repo_dirty(srctree) 1890eb8dc403SDave Cobbley if dirty: 1891eb8dc403SDave Cobbley if args.force: 1892eb8dc403SDave Cobbley logger.warning('Source tree is not clean, continuing as requested by -f/--force') 1893eb8dc403SDave Cobbley else: 1894eb8dc403SDave Cobbley raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty) 1895eb8dc403SDave Cobbley 1896eb8dc403SDave Cobbley no_clean = False 1897eb8dc403SDave Cobbley tinfoil = setup_tinfoil(basepath=basepath, tracking=True) 1898eb8dc403SDave Cobbley try: 1899eb8dc403SDave Cobbley rd = parse_recipe(config, tinfoil, args.recipename, True) 1900eb8dc403SDave Cobbley if not rd: 1901eb8dc403SDave Cobbley return 1 1902eb8dc403SDave Cobbley 1903eb8dc403SDave Cobbley destlayerdir = _get_layer(args.destination, tinfoil.config_data) 1904eb8dc403SDave Cobbley recipefile = rd.getVar('FILE') 1905eb8dc403SDave Cobbley recipedir = os.path.dirname(recipefile) 1906eb8dc403SDave Cobbley origlayerdir = oe.recipeutils.find_layerdir(recipefile) 1907eb8dc403SDave Cobbley 1908eb8dc403SDave Cobbley if not os.path.isdir(destlayerdir): 1909eb8dc403SDave Cobbley raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 1910eb8dc403SDave Cobbley 1911eb8dc403SDave Cobbley if os.path.abspath(destlayerdir) == config.workspace_path: 1912eb8dc403SDave Cobbley raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination) 1913eb8dc403SDave Cobbley 1914eb8dc403SDave Cobbley # If it's an upgrade, grab the original path 1915eb8dc403SDave Cobbley origpath = None 1916eb8dc403SDave Cobbley origfilelist = None 1917eb8dc403SDave Cobbley append = workspace[args.recipename]['bbappend'] 1918eb8dc403SDave Cobbley with open(append, 'r') as f: 1919eb8dc403SDave Cobbley for line in f: 1920eb8dc403SDave Cobbley if line.startswith('# original_path:'): 1921eb8dc403SDave Cobbley origpath = line.split(':')[1].strip() 1922eb8dc403SDave Cobbley elif line.startswith('# original_files:'): 1923eb8dc403SDave Cobbley origfilelist = line.split(':')[1].split() 1924eb8dc403SDave Cobbley 1925eb8dc403SDave Cobbley destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir) 1926eb8dc403SDave Cobbley 1927eb8dc403SDave Cobbley if origlayerdir == config.workspace_path: 1928eb8dc403SDave Cobbley # Recipe file itself is in workspace, update it there first 1929eb8dc403SDave Cobbley appendlayerdir = None 1930eb8dc403SDave Cobbley origrelpath = None 1931eb8dc403SDave Cobbley if origpath: 1932eb8dc403SDave Cobbley origlayerpath = oe.recipeutils.find_layerdir(origpath) 1933eb8dc403SDave Cobbley if origlayerpath: 1934eb8dc403SDave Cobbley origrelpath = os.path.relpath(origpath, origlayerpath) 1935eb8dc403SDave Cobbley destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath) 1936eb8dc403SDave Cobbley if not destpath: 1937eb8dc403SDave Cobbley raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir)) 1938eb8dc403SDave Cobbley # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases) 1939eb8dc403SDave Cobbley layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()] 1940eb8dc403SDave Cobbley if not os.path.abspath(destlayerbasedir) in layerdirs: 1941eb8dc403SDave Cobbley bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename) 1942eb8dc403SDave Cobbley 1943eb8dc403SDave Cobbley elif destlayerdir == origlayerdir: 1944eb8dc403SDave Cobbley # Same layer, update the original recipe 1945eb8dc403SDave Cobbley appendlayerdir = None 1946eb8dc403SDave Cobbley destpath = None 1947eb8dc403SDave Cobbley else: 1948eb8dc403SDave Cobbley # Create/update a bbappend in the specified layer 1949eb8dc403SDave Cobbley appendlayerdir = destlayerdir 1950eb8dc403SDave Cobbley destpath = None 1951eb8dc403SDave Cobbley 1952eb8dc403SDave Cobbley # Actually update the recipe / bbappend 1953eb8dc403SDave Cobbley removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 1954eb8dc403SDave Cobbley dry_run_output = None 1955eb8dc403SDave Cobbley dry_run_outdir = None 1956eb8dc403SDave Cobbley if args.dry_run: 1957eb8dc403SDave Cobbley dry_run_output = tempfile.TemporaryDirectory(prefix='devtool') 1958eb8dc403SDave Cobbley dry_run_outdir = dry_run_output.name 1959eb8dc403SDave Cobbley updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh) 1960eb8dc403SDave Cobbley removed = [os.path.relpath(pth, recipedir) for pth in removed] 1961eb8dc403SDave Cobbley 1962eb8dc403SDave Cobbley # Remove any old files in the case of an upgrade 1963eb8dc403SDave Cobbley if removing_original: 1964eb8dc403SDave Cobbley for fn in origfilelist: 1965eb8dc403SDave Cobbley fnp = os.path.join(origpath, fn) 1966eb8dc403SDave Cobbley if fn in removed or not os.path.exists(os.path.join(recipedir, fn)): 1967eb8dc403SDave Cobbley logger.info('Removing file %s%s' % (fnp, dry_run_suffix)) 1968eb8dc403SDave Cobbley if not args.dry_run: 1969eb8dc403SDave Cobbley try: 1970eb8dc403SDave Cobbley os.remove(fnp) 1971eb8dc403SDave Cobbley except FileNotFoundError: 1972eb8dc403SDave Cobbley pass 1973eb8dc403SDave Cobbley 1974eb8dc403SDave Cobbley if origlayerdir == config.workspace_path and destpath: 1975eb8dc403SDave Cobbley # Recipe file itself is in the workspace - need to move it and any 1976eb8dc403SDave Cobbley # associated files to the specified layer 1977eb8dc403SDave Cobbley no_clean = True 1978eb8dc403SDave Cobbley logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix)) 1979eb8dc403SDave Cobbley for root, _, files in os.walk(recipedir): 1980eb8dc403SDave Cobbley for fn in files: 1981eb8dc403SDave Cobbley srcpath = os.path.join(root, fn) 1982eb8dc403SDave Cobbley relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 1983eb8dc403SDave Cobbley destdir = os.path.abspath(os.path.join(destpath, relpth)) 1984eb8dc403SDave Cobbley destfp = os.path.join(destdir, fn) 1985eb8dc403SDave Cobbley _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath) 1986eb8dc403SDave Cobbley 1987eb8dc403SDave Cobbley if dry_run_outdir: 1988eb8dc403SDave Cobbley import difflib 1989eb8dc403SDave Cobbley comparelist = [] 1990eb8dc403SDave Cobbley for root, _, files in os.walk(dry_run_outdir): 1991eb8dc403SDave Cobbley for fn in files: 1992eb8dc403SDave Cobbley outf = os.path.join(root, fn) 1993eb8dc403SDave Cobbley relf = os.path.relpath(outf, dry_run_outdir) 1994eb8dc403SDave Cobbley logger.debug('dry-run: output file %s' % relf) 1995eb8dc403SDave Cobbley if fn.endswith('.bb'): 1996eb8dc403SDave Cobbley if origfilelist and origpath and destpath: 1997eb8dc403SDave Cobbley # Need to match this up with the pre-upgrade recipe file 1998eb8dc403SDave Cobbley for origf in origfilelist: 1999eb8dc403SDave Cobbley if origf.endswith('.bb'): 2000eb8dc403SDave Cobbley comparelist.append((os.path.abspath(os.path.join(origpath, origf)), 2001eb8dc403SDave Cobbley outf, 2002eb8dc403SDave Cobbley os.path.abspath(os.path.join(destpath, relf)))) 2003eb8dc403SDave Cobbley break 2004eb8dc403SDave Cobbley else: 2005eb8dc403SDave Cobbley # Compare to the existing recipe 2006eb8dc403SDave Cobbley comparelist.append((recipefile, outf, recipefile)) 2007eb8dc403SDave Cobbley elif fn.endswith('.bbappend'): 2008eb8dc403SDave Cobbley if appendfile: 2009eb8dc403SDave Cobbley if os.path.exists(appendfile): 2010eb8dc403SDave Cobbley comparelist.append((appendfile, outf, appendfile)) 2011eb8dc403SDave Cobbley else: 2012eb8dc403SDave Cobbley comparelist.append((None, outf, appendfile)) 2013eb8dc403SDave Cobbley else: 2014eb8dc403SDave Cobbley if destpath: 2015eb8dc403SDave Cobbley recipedest = destpath 2016eb8dc403SDave Cobbley elif appendfile: 2017eb8dc403SDave Cobbley recipedest = os.path.dirname(appendfile) 2018eb8dc403SDave Cobbley else: 2019eb8dc403SDave Cobbley recipedest = os.path.dirname(recipefile) 2020eb8dc403SDave Cobbley destfp = os.path.join(recipedest, relf) 2021eb8dc403SDave Cobbley if os.path.exists(destfp): 2022eb8dc403SDave Cobbley comparelist.append((destfp, outf, destfp)) 2023eb8dc403SDave Cobbley output = '' 2024eb8dc403SDave Cobbley for oldfile, newfile, newfileshow in comparelist: 2025eb8dc403SDave Cobbley if oldfile: 2026eb8dc403SDave Cobbley with open(oldfile, 'r') as f: 2027eb8dc403SDave Cobbley oldlines = f.readlines() 2028eb8dc403SDave Cobbley else: 2029eb8dc403SDave Cobbley oldfile = '/dev/null' 2030eb8dc403SDave Cobbley oldlines = [] 2031eb8dc403SDave Cobbley with open(newfile, 'r') as f: 2032eb8dc403SDave Cobbley newlines = f.readlines() 2033eb8dc403SDave Cobbley if not newfileshow: 2034eb8dc403SDave Cobbley newfileshow = newfile 2035eb8dc403SDave Cobbley diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow) 2036eb8dc403SDave Cobbley difflines = list(diff) 2037eb8dc403SDave Cobbley if difflines: 2038eb8dc403SDave Cobbley output += ''.join(difflines) 2039eb8dc403SDave Cobbley if output: 2040eb8dc403SDave Cobbley logger.info('Diff of changed files:\n%s' % output) 2041eb8dc403SDave Cobbley finally: 2042eb8dc403SDave Cobbley tinfoil.shutdown() 2043eb8dc403SDave Cobbley 2044eb8dc403SDave Cobbley # Everything else has succeeded, we can now reset 2045eb8dc403SDave Cobbley if args.dry_run: 2046eb8dc403SDave Cobbley logger.info('Resetting recipe (dry-run)') 2047eb8dc403SDave Cobbley else: 2048eb8dc403SDave Cobbley _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace) 2049eb8dc403SDave Cobbley 2050eb8dc403SDave Cobbley return 0 2051eb8dc403SDave Cobbley 2052eb8dc403SDave Cobbley 2053eb8dc403SDave Cobbleydef get_default_srctree(config, recipename=''): 2054eb8dc403SDave Cobbley """Get the default srctree path""" 2055eb8dc403SDave Cobbley srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path) 2056eb8dc403SDave Cobbley if recipename: 2057eb8dc403SDave Cobbley return os.path.join(srctreeparent, 'sources', recipename) 2058eb8dc403SDave Cobbley else: 2059eb8dc403SDave Cobbley return os.path.join(srctreeparent, 'sources') 2060eb8dc403SDave Cobbley 2061eb8dc403SDave Cobbleydef register_commands(subparsers, context): 2062eb8dc403SDave Cobbley """Register devtool subcommands from this plugin""" 2063eb8dc403SDave Cobbley 2064eb8dc403SDave Cobbley defsrctree = get_default_srctree(context.config) 2065eb8dc403SDave Cobbley parser_add = subparsers.add_parser('add', help='Add a new recipe', 2066eb8dc403SDave Cobbley description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.', 2067eb8dc403SDave Cobbley group='starting', order=100) 2068eb8dc403SDave Cobbley parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.') 2069eb8dc403SDave Cobbley parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2070eb8dc403SDave Cobbley parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree') 2071eb8dc403SDave Cobbley group = parser_add.add_mutually_exclusive_group() 2072eb8dc403SDave Cobbley group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2073eb8dc403SDave Cobbley group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2074eb8dc403SDave Cobbley parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2075eb8dc403SDave Cobbley parser_add.add_argument('--fetch-dev', help='For npm, also fetch devDependencies', action="store_true") 2076eb8dc403SDave Cobbley parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2077eb8dc403SDave Cobbley parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2078eb8dc403SDave Cobbley group = parser_add.add_mutually_exclusive_group() 2079eb8dc403SDave Cobbley group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)') 2080eb8dc403SDave Cobbley group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true") 2081eb8dc403SDave Cobbley parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)') 2082eb8dc403SDave Cobbley parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true') 2083eb8dc403SDave Cobbley parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true') 2084eb8dc403SDave Cobbley parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR') 2085eb8dc403SDave Cobbley parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true") 2086eb8dc403SDave Cobbley parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl') 2087eb8dc403SDave Cobbley parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup) 2088eb8dc403SDave Cobbley 2089eb8dc403SDave Cobbley parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe', 2090eb8dc403SDave Cobbley description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.', 2091eb8dc403SDave Cobbley group='starting', order=90) 2092eb8dc403SDave Cobbley parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)') 2093eb8dc403SDave Cobbley parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree) 2094eb8dc403SDave Cobbley parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend') 2095eb8dc403SDave Cobbley group = parser_modify.add_mutually_exclusive_group() 2096eb8dc403SDave Cobbley group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)') 2097eb8dc403SDave Cobbley group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist') 2098eb8dc403SDave Cobbley group = parser_modify.add_mutually_exclusive_group() 2099eb8dc403SDave Cobbley group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true") 2100eb8dc403SDave Cobbley group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2101eb8dc403SDave Cobbley parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")') 2102eb8dc403SDave Cobbley parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2103eb8dc403SDave Cobbley parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true") 2104eb8dc403SDave Cobbley parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup) 2105eb8dc403SDave Cobbley 2106eb8dc403SDave Cobbley parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe', 2107eb8dc403SDave Cobbley description='Extracts the source for an existing recipe', 2108eb8dc403SDave Cobbley group='advanced') 2109eb8dc403SDave Cobbley parser_extract.add_argument('recipename', help='Name of recipe to extract the source for') 2110eb8dc403SDave Cobbley parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 2111eb8dc403SDave Cobbley parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 2112eb8dc403SDave Cobbley parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations') 2113eb8dc403SDave Cobbley parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2114eb8dc403SDave Cobbley parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup) 2115eb8dc403SDave Cobbley 2116eb8dc403SDave Cobbley parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 2117eb8dc403SDave Cobbley description='Synchronize the previously extracted source tree for an existing recipe', 2118eb8dc403SDave Cobbley formatter_class=argparse.ArgumentDefaultsHelpFormatter, 2119eb8dc403SDave Cobbley group='advanced') 2120eb8dc403SDave Cobbley parser_sync.add_argument('recipename', help='Name of recipe to sync the source for') 2121eb8dc403SDave Cobbley parser_sync.add_argument('srctree', help='Path to the source tree') 2122eb8dc403SDave Cobbley parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout') 2123eb8dc403SDave Cobbley parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 2124eb8dc403SDave Cobbley parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup) 2125eb8dc403SDave Cobbley 2126eb8dc403SDave Cobbley parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace', 2127eb8dc403SDave Cobbley description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.', 2128eb8dc403SDave Cobbley group='working', order=10) 2129eb8dc403SDave Cobbley parser_rename.add_argument('recipename', help='Current name of recipe to rename') 2130eb8dc403SDave Cobbley parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)') 2131eb8dc403SDave Cobbley parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)') 2132eb8dc403SDave Cobbley parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path') 2133eb8dc403SDave Cobbley parser_rename.set_defaults(func=rename) 2134eb8dc403SDave Cobbley 2135eb8dc403SDave Cobbley parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe', 2136eb8dc403SDave Cobbley description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.', 2137eb8dc403SDave Cobbley group='working', order=-90) 2138eb8dc403SDave Cobbley parser_update_recipe.add_argument('recipename', help='Name of recipe to update') 2139eb8dc403SDave Cobbley parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2140eb8dc403SDave Cobbley parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches') 2141eb8dc403SDave Cobbley parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 2142eb8dc403SDave Cobbley parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2143eb8dc403SDave Cobbley parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2144eb8dc403SDave Cobbley parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2145eb8dc403SDave Cobbley parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2146eb8dc403SDave Cobbley parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2147eb8dc403SDave Cobbley parser_update_recipe.set_defaults(func=update_recipe) 2148eb8dc403SDave Cobbley 2149eb8dc403SDave Cobbley parser_status = subparsers.add_parser('status', help='Show workspace status', 2150eb8dc403SDave Cobbley description='Lists recipes currently in your workspace and the paths to their respective external source trees', 2151eb8dc403SDave Cobbley group='info', order=100) 2152eb8dc403SDave Cobbley parser_status.set_defaults(func=status) 2153eb8dc403SDave Cobbley 2154eb8dc403SDave Cobbley parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace', 2155eb8dc403SDave Cobbley description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).', 2156eb8dc403SDave Cobbley group='working', order=-100) 2157eb8dc403SDave Cobbley parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset') 2158eb8dc403SDave Cobbley parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)') 2159eb8dc403SDave Cobbley parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output') 2160eb8dc403SDave Cobbley parser_reset.set_defaults(func=reset) 2161eb8dc403SDave Cobbley 2162eb8dc403SDave Cobbley parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace', 2163eb8dc403SDave Cobbley description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.', 2164eb8dc403SDave Cobbley group='working', order=-100) 2165eb8dc403SDave Cobbley parser_finish.add_argument('recipename', help='Recipe to finish') 2166eb8dc403SDave Cobbley parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.') 2167eb8dc403SDave Cobbley parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2168eb8dc403SDave Cobbley parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2169eb8dc403SDave Cobbley parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2170eb8dc403SDave Cobbley parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)') 2171eb8dc403SDave Cobbley parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)') 2172eb8dc403SDave Cobbley parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)') 2173eb8dc403SDave Cobbley parser_finish.set_defaults(func=finish) 2174