1eb8dc403SDave Cobbley#!/usr/bin/env python3 2eb8dc403SDave Cobbley# ex:ts=4:sw=4:sts=4:et 3eb8dc403SDave Cobbley# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 4eb8dc403SDave Cobbley# 5eb8dc403SDave Cobbley# Copyright 2011 Intel Corporation 6eb8dc403SDave Cobbley# Authored-by: Yu Ke <ke.yu@intel.com> 7eb8dc403SDave Cobbley# Paul Eggleton <paul.eggleton@intel.com> 8eb8dc403SDave Cobbley# Richard Purdie <richard.purdie@intel.com> 9eb8dc403SDave Cobbley# 10c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 11eb8dc403SDave Cobbley# 12eb8dc403SDave Cobbley 13eb8dc403SDave Cobbleyimport fnmatch 14eb8dc403SDave Cobbleyimport os, sys 15eb8dc403SDave Cobbleyimport optparse 16eb8dc403SDave Cobbleyimport logging 17eb8dc403SDave Cobbleyimport subprocess 18eb8dc403SDave Cobbleyimport tempfile 19eb8dc403SDave Cobbleyimport configparser 20eb8dc403SDave Cobbleyimport re 21eb8dc403SDave Cobbleyimport copy 22fc113eadSAndrew Geisslerimport shlex 23eb8dc403SDave Cobbleyimport shutil 24eb8dc403SDave Cobbleyfrom string import Template 25eb8dc403SDave Cobbleyfrom functools import reduce 26eb8dc403SDave Cobbley 27eb8dc403SDave Cobbley__version__ = "0.2.1" 28eb8dc403SDave Cobbley 29eb8dc403SDave Cobbleydef logger_create(): 30eb8dc403SDave Cobbley logger = logging.getLogger("") 31eb8dc403SDave Cobbley loggerhandler = logging.StreamHandler() 32eb8dc403SDave Cobbley loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S")) 33eb8dc403SDave Cobbley logger.addHandler(loggerhandler) 34eb8dc403SDave Cobbley logger.setLevel(logging.INFO) 35eb8dc403SDave Cobbley return logger 36eb8dc403SDave Cobbley 37eb8dc403SDave Cobbleylogger = logger_create() 38eb8dc403SDave Cobbley 39eb8dc403SDave Cobbleydef get_current_branch(repodir=None): 40eb8dc403SDave Cobbley try: 41eb8dc403SDave Cobbley if not os.path.exists(os.path.join(repodir if repodir else '', ".git")): 42eb8dc403SDave Cobbley # Repo not created yet (i.e. during init) so just assume master 43eb8dc403SDave Cobbley return "master" 44eb8dc403SDave Cobbley branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip() 45eb8dc403SDave Cobbley if branchname.startswith("refs/heads/"): 46eb8dc403SDave Cobbley branchname = branchname[11:] 47eb8dc403SDave Cobbley return branchname 48eb8dc403SDave Cobbley except subprocess.CalledProcessError: 49eb8dc403SDave Cobbley return "" 50eb8dc403SDave Cobbley 51eb8dc403SDave Cobbleyclass Configuration(object): 52eb8dc403SDave Cobbley """ 53eb8dc403SDave Cobbley Manages the configuration 54eb8dc403SDave Cobbley 55eb8dc403SDave Cobbley For an example config file, see combo-layer.conf.example 56eb8dc403SDave Cobbley 57eb8dc403SDave Cobbley """ 58eb8dc403SDave Cobbley def __init__(self, options): 59eb8dc403SDave Cobbley for key, val in options.__dict__.items(): 60eb8dc403SDave Cobbley setattr(self, key, val) 61eb8dc403SDave Cobbley 62eb8dc403SDave Cobbley def readsection(parser, section, repo): 63eb8dc403SDave Cobbley for (name, value) in parser.items(section): 64eb8dc403SDave Cobbley if value.startswith("@"): 65eb8dc403SDave Cobbley self.repos[repo][name] = eval(value.strip("@")) 66eb8dc403SDave Cobbley else: 67eb8dc403SDave Cobbley # Apply special type transformations for some properties. 68eb8dc403SDave Cobbley # Type matches the RawConfigParser.get*() methods. 69eb8dc403SDave Cobbley types = {'signoff': 'boolean', 'update': 'boolean', 'history': 'boolean'} 70eb8dc403SDave Cobbley if name in types: 71eb8dc403SDave Cobbley value = getattr(parser, 'get' + types[name])(section, name) 72eb8dc403SDave Cobbley self.repos[repo][name] = value 73eb8dc403SDave Cobbley 74eb8dc403SDave Cobbley def readglobalsection(parser, section): 75eb8dc403SDave Cobbley for (name, value) in parser.items(section): 76eb8dc403SDave Cobbley if name == "commit_msg": 77eb8dc403SDave Cobbley self.commit_msg_template = value 78eb8dc403SDave Cobbley 79eb8dc403SDave Cobbley logger.debug("Loading config file %s" % self.conffile) 80eb8dc403SDave Cobbley self.parser = configparser.ConfigParser() 81eb8dc403SDave Cobbley with open(self.conffile) as f: 8282c905dcSAndrew Geissler self.parser.read_file(f) 83eb8dc403SDave Cobbley 84eb8dc403SDave Cobbley # initialize default values 85eb8dc403SDave Cobbley self.commit_msg_template = "Automatic commit to update last_revision" 86eb8dc403SDave Cobbley 87eb8dc403SDave Cobbley self.repos = {} 88eb8dc403SDave Cobbley for repo in self.parser.sections(): 89eb8dc403SDave Cobbley if repo == "combo-layer-settings": 90eb8dc403SDave Cobbley # special handling for global settings 91eb8dc403SDave Cobbley readglobalsection(self.parser, repo) 92eb8dc403SDave Cobbley else: 93eb8dc403SDave Cobbley self.repos[repo] = {} 94eb8dc403SDave Cobbley readsection(self.parser, repo, repo) 95eb8dc403SDave Cobbley 96eb8dc403SDave Cobbley # Load local configuration, if available 97eb8dc403SDave Cobbley self.localconffile = None 98eb8dc403SDave Cobbley self.localparser = None 99eb8dc403SDave Cobbley self.combobranch = None 100eb8dc403SDave Cobbley if self.conffile.endswith('.conf'): 101eb8dc403SDave Cobbley lcfile = self.conffile.replace('.conf', '-local.conf') 102eb8dc403SDave Cobbley if os.path.exists(lcfile): 103eb8dc403SDave Cobbley # Read combo layer branch 104eb8dc403SDave Cobbley self.combobranch = get_current_branch() 105eb8dc403SDave Cobbley logger.debug("Combo layer branch is %s" % self.combobranch) 106eb8dc403SDave Cobbley 107eb8dc403SDave Cobbley self.localconffile = lcfile 108eb8dc403SDave Cobbley logger.debug("Loading local config file %s" % self.localconffile) 109eb8dc403SDave Cobbley self.localparser = configparser.ConfigParser() 110eb8dc403SDave Cobbley with open(self.localconffile) as f: 111eb8dc403SDave Cobbley self.localparser.readfp(f) 112eb8dc403SDave Cobbley 113eb8dc403SDave Cobbley for section in self.localparser.sections(): 114eb8dc403SDave Cobbley if '|' in section: 115eb8dc403SDave Cobbley sectionvals = section.split('|') 116eb8dc403SDave Cobbley repo = sectionvals[0] 117eb8dc403SDave Cobbley if sectionvals[1] != self.combobranch: 118eb8dc403SDave Cobbley continue 119eb8dc403SDave Cobbley else: 120eb8dc403SDave Cobbley repo = section 121eb8dc403SDave Cobbley if repo in self.repos: 122eb8dc403SDave Cobbley readsection(self.localparser, section, repo) 123eb8dc403SDave Cobbley 124eb8dc403SDave Cobbley def update(self, repo, option, value, initmode=False): 125eb8dc403SDave Cobbley # If the main config has the option already, that is what we 126eb8dc403SDave Cobbley # are expected to modify. 127eb8dc403SDave Cobbley if self.localparser and not self.parser.has_option(repo, option): 128eb8dc403SDave Cobbley parser = self.localparser 129eb8dc403SDave Cobbley section = "%s|%s" % (repo, self.combobranch) 130eb8dc403SDave Cobbley conffile = self.localconffile 131eb8dc403SDave Cobbley if initmode and not parser.has_section(section): 132eb8dc403SDave Cobbley parser.add_section(section) 133eb8dc403SDave Cobbley else: 134eb8dc403SDave Cobbley parser = self.parser 135eb8dc403SDave Cobbley section = repo 136eb8dc403SDave Cobbley conffile = self.conffile 137eb8dc403SDave Cobbley parser.set(section, option, value) 138eb8dc403SDave Cobbley with open(conffile, "w") as f: 139eb8dc403SDave Cobbley parser.write(f) 140eb8dc403SDave Cobbley self.repos[repo][option] = value 141eb8dc403SDave Cobbley 142eb8dc403SDave Cobbley def sanity_check(self, initmode=False): 143eb8dc403SDave Cobbley required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"] 144eb8dc403SDave Cobbley if initmode: 145eb8dc403SDave Cobbley required_options.remove("last_revision") 146eb8dc403SDave Cobbley msg = "" 147eb8dc403SDave Cobbley missing_options = [] 148eb8dc403SDave Cobbley for name in self.repos: 149eb8dc403SDave Cobbley for option in required_options: 150eb8dc403SDave Cobbley if option not in self.repos[name]: 151eb8dc403SDave Cobbley msg = "%s\nOption %s is not defined for component %s" %(msg, option, name) 152eb8dc403SDave Cobbley missing_options.append(option) 153eb8dc403SDave Cobbley # Sanitize dest_dir so that we do not have to deal with edge cases 154eb8dc403SDave Cobbley # (unset, empty string, double slashes) in the rest of the code. 155eb8dc403SDave Cobbley # It not being set will still be flagged as error because it is 156eb8dc403SDave Cobbley # listed as required option above; that could be changed now. 157eb8dc403SDave Cobbley dest_dir = os.path.normpath(self.repos[name].get("dest_dir", ".")) 158eb8dc403SDave Cobbley self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir 159eb8dc403SDave Cobbley if msg != "": 160eb8dc403SDave Cobbley logger.error("configuration file %s has the following error: %s" % (self.conffile,msg)) 161eb8dc403SDave Cobbley if self.localconffile and 'last_revision' in missing_options: 162eb8dc403SDave Cobbley logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch)) 163eb8dc403SDave Cobbley sys.exit(1) 164eb8dc403SDave Cobbley 165eb8dc403SDave Cobbley # filterdiff is required by action_splitpatch, so check its availability 166eb8dc403SDave Cobbley if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0: 167eb8dc403SDave Cobbley logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)") 168eb8dc403SDave Cobbley sys.exit(1) 169eb8dc403SDave Cobbley 170eb8dc403SDave Cobbleydef runcmd(cmd,destdir=None,printerr=True,out=None,env=None): 171eb8dc403SDave Cobbley """ 172eb8dc403SDave Cobbley execute command, raise CalledProcessError if fail 173eb8dc403SDave Cobbley return output if succeed 174eb8dc403SDave Cobbley """ 175eb8dc403SDave Cobbley logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir)) 176eb8dc403SDave Cobbley if not out: 177eb8dc403SDave Cobbley out = tempfile.TemporaryFile() 178eb8dc403SDave Cobbley err = out 179eb8dc403SDave Cobbley else: 180eb8dc403SDave Cobbley err = tempfile.TemporaryFile() 181eb8dc403SDave Cobbley try: 182eb8dc403SDave Cobbley subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str), env=env or os.environ) 183eb8dc403SDave Cobbley except subprocess.CalledProcessError as e: 184eb8dc403SDave Cobbley err.seek(0) 185eb8dc403SDave Cobbley if printerr: 186eb8dc403SDave Cobbley logger.error("%s" % err.read()) 187eb8dc403SDave Cobbley raise e 188eb8dc403SDave Cobbley 189eb8dc403SDave Cobbley err.seek(0) 190eb8dc403SDave Cobbley output = err.read().decode('utf-8') 191eb8dc403SDave Cobbley logger.debug("output: %s" % output.replace(chr(0), '\\0')) 192eb8dc403SDave Cobbley return output 193eb8dc403SDave Cobbley 194517393d9SAndrew Geisslerdef action_sync_revs(conf, args): 195517393d9SAndrew Geissler """ 196517393d9SAndrew Geissler Update the last_revision config option for each repo with the latest 197517393d9SAndrew Geissler revision in the remote's branch. Useful if multiple people are using 198517393d9SAndrew Geissler combo-layer. 199517393d9SAndrew Geissler """ 200517393d9SAndrew Geissler repos = get_repos(conf, args[1:]) 201517393d9SAndrew Geissler 202517393d9SAndrew Geissler for name in repos: 203517393d9SAndrew Geissler repo = conf.repos[name] 204517393d9SAndrew Geissler ldir = repo['local_repo_dir'] 205517393d9SAndrew Geissler branch = repo.get('branch', "master") 206517393d9SAndrew Geissler runcmd("git fetch", ldir) 207517393d9SAndrew Geissler lastrev = runcmd('git rev-parse origin/%s' % branch, ldir).strip() 208517393d9SAndrew Geissler print("Updating %s to %s" % (name, lastrev)) 209517393d9SAndrew Geissler conf.update(name, "last_revision", lastrev) 210517393d9SAndrew Geissler 211eb8dc403SDave Cobbleydef action_init(conf, args): 212eb8dc403SDave Cobbley """ 213eb8dc403SDave Cobbley Clone component repositories 214eb8dc403SDave Cobbley Check git is initialised; if not, copy initial data from component repos 215eb8dc403SDave Cobbley """ 216eb8dc403SDave Cobbley for name in conf.repos: 217eb8dc403SDave Cobbley ldir = conf.repos[name]['local_repo_dir'] 218eb8dc403SDave Cobbley if not os.path.exists(ldir): 219eb8dc403SDave Cobbley logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir)) 220eb8dc403SDave Cobbley subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True) 221eb8dc403SDave Cobbley if not os.path.exists(".git"): 222eb8dc403SDave Cobbley runcmd("git init") 223eb8dc403SDave Cobbley if conf.history: 224eb8dc403SDave Cobbley # Need a common ref for all trees. 225eb8dc403SDave Cobbley runcmd('git commit -m "initial empty commit" --allow-empty') 226eb8dc403SDave Cobbley startrev = runcmd('git rev-parse master').strip() 227eb8dc403SDave Cobbley 228eb8dc403SDave Cobbley for name in conf.repos: 229eb8dc403SDave Cobbley repo = conf.repos[name] 230eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 231eb8dc403SDave Cobbley branch = repo.get('branch', "master") 232eb8dc403SDave Cobbley lastrev = repo.get('last_revision', None) 233eb8dc403SDave Cobbley if lastrev and lastrev != "HEAD": 234eb8dc403SDave Cobbley initialrev = lastrev 235eb8dc403SDave Cobbley if branch: 236eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, lastrev, branch): 237eb8dc403SDave Cobbley sys.exit(1) 238eb8dc403SDave Cobbley logger.info("Copying data from %s at specified revision %s..." % (name, lastrev)) 239eb8dc403SDave Cobbley else: 240eb8dc403SDave Cobbley lastrev = None 241eb8dc403SDave Cobbley initialrev = branch 242eb8dc403SDave Cobbley logger.info("Copying data from %s..." % name) 243eb8dc403SDave Cobbley # Sanity check initialrev and turn it into hash (required for copying history, 244eb8dc403SDave Cobbley # because resolving a name ref only works in the component repo). 245eb8dc403SDave Cobbley rev = runcmd('git rev-parse %s' % initialrev, ldir).strip() 246eb8dc403SDave Cobbley if rev != initialrev: 247eb8dc403SDave Cobbley try: 248eb8dc403SDave Cobbley refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n') 249eb8dc403SDave Cobbley if len(set(refs)) > 1: 250eb8dc403SDave Cobbley # Happens for example when configured to track 251eb8dc403SDave Cobbley # "master" and there is a refs/heads/master. The 252eb8dc403SDave Cobbley # traditional behavior from "git archive" (preserved 253eb8dc403SDave Cobbley # here) it to choose the first one. This might not be 254eb8dc403SDave Cobbley # intended, so at least warn about it. 2551a4b7ee2SBrad Bishop logger.warning("%s: initial revision '%s' not unique, picking result of rev-parse = %s" % 256eb8dc403SDave Cobbley (name, initialrev, refs[0])) 257eb8dc403SDave Cobbley initialrev = rev 258eb8dc403SDave Cobbley except: 259eb8dc403SDave Cobbley # show-ref fails for hashes. Skip the sanity warning in that case. 260eb8dc403SDave Cobbley pass 261eb8dc403SDave Cobbley initialrev = rev 262eb8dc403SDave Cobbley dest_dir = repo['dest_dir'] 263eb8dc403SDave Cobbley if dest_dir != ".": 264eb8dc403SDave Cobbley extract_dir = os.path.join(os.getcwd(), dest_dir) 265eb8dc403SDave Cobbley if not os.path.exists(extract_dir): 266eb8dc403SDave Cobbley os.makedirs(extract_dir) 267eb8dc403SDave Cobbley else: 268eb8dc403SDave Cobbley extract_dir = os.getcwd() 269eb8dc403SDave Cobbley file_filter = repo.get('file_filter', "") 270eb8dc403SDave Cobbley exclude_patterns = repo.get('file_exclude', '').split() 271eb8dc403SDave Cobbley def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir, 272eb8dc403SDave Cobbley subdir=""): 273eb8dc403SDave Cobbley # When working inside a filtered branch which had the 274eb8dc403SDave Cobbley # files already moved, we need to prepend the 275eb8dc403SDave Cobbley # subdirectory to all filters, otherwise they would 276eb8dc403SDave Cobbley # not match. 277eb8dc403SDave Cobbley if subdir == '.': 278eb8dc403SDave Cobbley subdir = '' 279eb8dc403SDave Cobbley elif subdir: 280eb8dc403SDave Cobbley subdir = os.path.normpath(subdir) 281eb8dc403SDave Cobbley file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()]) 282eb8dc403SDave Cobbley exclude_patterns = [subdir + '/' + x for x in exclude_patterns] 283eb8dc403SDave Cobbley # To handle both cases, we cd into the target 284eb8dc403SDave Cobbley # directory and optionally tell tar to strip the path 285eb8dc403SDave Cobbley # prefix when the files were already moved. 286eb8dc403SDave Cobbley subdir_components = len(subdir.split(os.path.sep)) if subdir else 0 287eb8dc403SDave Cobbley strip=('--strip-components=%d' % subdir_components) if subdir else '' 288eb8dc403SDave Cobbley # TODO: file_filter wild cards do not work (and haven't worked before either), because 289eb8dc403SDave Cobbley # a) GNU tar requires a --wildcards parameter before turning on wild card matching. 290eb8dc403SDave Cobbley # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c, 291eb8dc403SDave Cobbley # in contrast to the other use of file_filter as parameter of "git archive" 292eb8dc403SDave Cobbley # where it only matches .c files directly in src). 293eb8dc403SDave Cobbley files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" % 294eb8dc403SDave Cobbley (initialrev, subdir, 295eb8dc403SDave Cobbley strip, extract_dir, file_filter), 296eb8dc403SDave Cobbley ldir) 297eb8dc403SDave Cobbley if exclude_patterns: 298eb8dc403SDave Cobbley # Implement file removal by letting tar create the 299eb8dc403SDave Cobbley # file and then deleting it in the file system 300eb8dc403SDave Cobbley # again. Uses the list of files created by tar (easier 301eb8dc403SDave Cobbley # than walking the tree). 302eb8dc403SDave Cobbley for file in files.split('\n'): 303eb8dc403SDave Cobbley if file.endswith(os.path.sep): 304eb8dc403SDave Cobbley continue 305eb8dc403SDave Cobbley for pattern in exclude_patterns: 306eb8dc403SDave Cobbley if fnmatch.fnmatch(file, pattern): 307eb8dc403SDave Cobbley os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file]))) 308eb8dc403SDave Cobbley break 309eb8dc403SDave Cobbley 310eb8dc403SDave Cobbley if not conf.history: 311eb8dc403SDave Cobbley copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir) 312eb8dc403SDave Cobbley else: 313eb8dc403SDave Cobbley # First fetch remote history into local repository. 314eb8dc403SDave Cobbley # We need a ref for that, so ensure that there is one. 315eb8dc403SDave Cobbley refname = "combo-layer-init-%s" % name 316eb8dc403SDave Cobbley runcmd("git branch -f %s %s" % (refname, initialrev), ldir) 317eb8dc403SDave Cobbley runcmd("git fetch %s %s" % (ldir, refname)) 318eb8dc403SDave Cobbley runcmd("git branch -D %s" % refname, ldir) 319eb8dc403SDave Cobbley # Make that the head revision. 320eb8dc403SDave Cobbley runcmd("git checkout -b %s %s" % (name, initialrev)) 321eb8dc403SDave Cobbley # Optional: cut the history by replacing the given 322eb8dc403SDave Cobbley # start point(s) with commits providing the same 323eb8dc403SDave Cobbley # content (aka tree), but with commit information that 324eb8dc403SDave Cobbley # makes it clear that this is an artifically created 325eb8dc403SDave Cobbley # commit and nothing the original authors had anything 326eb8dc403SDave Cobbley # to do with. 327eb8dc403SDave Cobbley since_rev = repo.get('since_revision', '') 328eb8dc403SDave Cobbley if since_rev: 329eb8dc403SDave Cobbley committer = runcmd('git var GIT_AUTHOR_IDENT').strip() 330eb8dc403SDave Cobbley # Same time stamp, no name. 331eb8dc403SDave Cobbley author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer) 332eb8dc403SDave Cobbley logger.info('author %s' % author) 333eb8dc403SDave Cobbley for rev in since_rev.split(): 334eb8dc403SDave Cobbley # Resolve in component repo... 335eb8dc403SDave Cobbley rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0] 336eb8dc403SDave Cobbley # ... and then get the tree in current 337eb8dc403SDave Cobbley # one. The commit should be in both repos with 338eb8dc403SDave Cobbley # the same tree, but better check here. 339eb8dc403SDave Cobbley tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip() 340eb8dc403SDave Cobbley with tempfile.NamedTemporaryFile(mode='wt') as editor: 341eb8dc403SDave Cobbley editor.write('''cat >$1 <<EOF 342eb8dc403SDave Cobbleytree %s 343eb8dc403SDave Cobbleyauthor %s 344eb8dc403SDave Cobbleycommitter %s 345eb8dc403SDave Cobbley 346eb8dc403SDave Cobbley%s: squashed import of component 347eb8dc403SDave Cobbley 348eb8dc403SDave CobbleyThis commit copies the entire set of files as found in 349eb8dc403SDave Cobbley%s %s 350eb8dc403SDave Cobbley 351eb8dc403SDave CobbleyFor more information about previous commits, see the 352eb8dc403SDave Cobbleyupstream repository. 353eb8dc403SDave Cobbley 354eb8dc403SDave CobbleyCommit created by combo-layer. 355eb8dc403SDave CobbleyEOF 356eb8dc403SDave Cobbley''' % (tree, author, committer, name, name, since_rev)) 357eb8dc403SDave Cobbley editor.flush() 358eb8dc403SDave Cobbley os.environ['GIT_EDITOR'] = 'sh %s' % editor.name 359eb8dc403SDave Cobbley runcmd('git replace --edit %s' % rev) 360eb8dc403SDave Cobbley 361eb8dc403SDave Cobbley # Optional: rewrite history to change commit messages or to move files. 362eb8dc403SDave Cobbley if 'hook' in repo or dest_dir != ".": 363eb8dc403SDave Cobbley filter_branch = ['git', 'filter-branch', '--force'] 364eb8dc403SDave Cobbley with tempfile.NamedTemporaryFile(mode='wt') as hookwrapper: 365eb8dc403SDave Cobbley if 'hook' in repo: 366eb8dc403SDave Cobbley # Create a shell script wrapper around the original hook that 367eb8dc403SDave Cobbley # can be used by git filter-branch. Hook may or may not have 368eb8dc403SDave Cobbley # an absolute path. 369eb8dc403SDave Cobbley hook = repo['hook'] 370eb8dc403SDave Cobbley hook = os.path.join(os.path.dirname(conf.conffile), '..', hook) 371eb8dc403SDave Cobbley # The wrappers turns the commit message 372eb8dc403SDave Cobbley # from stdin into a fake patch header. 373eb8dc403SDave Cobbley # This is good enough for changing Subject 374eb8dc403SDave Cobbley # and commit msg body with normal 375eb8dc403SDave Cobbley # combo-layer hooks. 376eb8dc403SDave Cobbley hookwrapper.write('''set -e 377eb8dc403SDave Cobbleytmpname=$(mktemp) 378eb8dc403SDave Cobbleytrap "rm $tmpname" EXIT 379eb8dc403SDave Cobbleyecho -n 'Subject: [PATCH] ' >>$tmpname 380eb8dc403SDave Cobbleycat >>$tmpname 381eb8dc403SDave Cobbleyif ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then 382eb8dc403SDave Cobbley echo >>$tmpname 383eb8dc403SDave Cobbleyfi 384eb8dc403SDave Cobbleyecho '---' >>$tmpname 385eb8dc403SDave Cobbley%s $tmpname $GIT_COMMIT %s 386eb8dc403SDave Cobbleytail -c +18 $tmpname | head -c -4 387eb8dc403SDave Cobbley''' % (hook, name)) 388eb8dc403SDave Cobbley hookwrapper.flush() 389eb8dc403SDave Cobbley filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name]) 390eb8dc403SDave Cobbley if dest_dir != ".": 391eb8dc403SDave Cobbley parent = os.path.dirname(dest_dir) 392eb8dc403SDave Cobbley if not parent: 393eb8dc403SDave Cobbley parent = '.' 394eb8dc403SDave Cobbley # May run outside of the current directory, so do not assume that .git exists. 395eb8dc403SDave Cobbley filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)]) 396eb8dc403SDave Cobbley filter_branch.append('HEAD') 397eb8dc403SDave Cobbley runcmd(filter_branch) 398eb8dc403SDave Cobbley runcmd('git update-ref -d refs/original/refs/heads/%s' % name) 399eb8dc403SDave Cobbley repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip() 400eb8dc403SDave Cobbley repo['stripped_revision'] = repo['rewritten_revision'] 401eb8dc403SDave Cobbley # Optional filter files: remove everything and re-populate using the normal filtering code. 402eb8dc403SDave Cobbley # Override any potential .gitignore. 403eb8dc403SDave Cobbley if file_filter or exclude_patterns: 404eb8dc403SDave Cobbley runcmd('git rm -rf .') 405eb8dc403SDave Cobbley if not os.path.exists(extract_dir): 406eb8dc403SDave Cobbley os.makedirs(extract_dir) 407eb8dc403SDave Cobbley copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.', 408eb8dc403SDave Cobbley subdir=dest_dir) 409eb8dc403SDave Cobbley runcmd('git add --all --force .') 410eb8dc403SDave Cobbley if runcmd('git status --porcelain'): 411eb8dc403SDave Cobbley # Something to commit. 412eb8dc403SDave Cobbley runcmd(['git', 'commit', '-m', 413eb8dc403SDave Cobbley '''%s: select file subset 414eb8dc403SDave Cobbley 415eb8dc403SDave CobbleyFiles from the component repository were chosen based on 416eb8dc403SDave Cobbleythe following filters: 417eb8dc403SDave Cobbleyfile_filter = %s 418eb8dc403SDave Cobbleyfile_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))]) 419eb8dc403SDave Cobbley repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip() 420eb8dc403SDave Cobbley 421eb8dc403SDave Cobbley if not lastrev: 422eb8dc403SDave Cobbley lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip() 423eb8dc403SDave Cobbley conf.update(name, "last_revision", lastrev, initmode=True) 424eb8dc403SDave Cobbley 425eb8dc403SDave Cobbley if not conf.history: 426eb8dc403SDave Cobbley runcmd("git add .") 427eb8dc403SDave Cobbley else: 428eb8dc403SDave Cobbley # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies 429eb8dc403SDave Cobbley runcmd('git checkout master') 430eb8dc403SDave Cobbley merge = ['git', 'merge', '--no-commit'] 431eb8dc403SDave Cobbley for name in conf.repos: 432eb8dc403SDave Cobbley repo = conf.repos[name] 433eb8dc403SDave Cobbley # Use branch created earlier. 434eb8dc403SDave Cobbley merge.append(name) 435eb8dc403SDave Cobbley # Root all commits which have no parent in the common 436eb8dc403SDave Cobbley # ancestor in the new repository. 437eb8dc403SDave Cobbley for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s --' % name).split('\n'): 438eb8dc403SDave Cobbley runcmd('git replace --graft %s %s' % (start, startrev)) 439eb8dc403SDave Cobbley try: 440eb8dc403SDave Cobbley runcmd(merge) 441eb8dc403SDave Cobbley except Exception as error: 442eb8dc403SDave Cobbley logger.info('''Merging component repository history failed, perhaps because of merge conflicts. 443eb8dc403SDave CobbleyIt may be possible to commit anyway after resolving these conflicts. 444eb8dc403SDave Cobbley 445eb8dc403SDave Cobbley%s''' % error) 446eb8dc403SDave Cobbley # Create MERGE_HEAD and MERGE_MSG. "git merge" itself 447eb8dc403SDave Cobbley # does not create MERGE_HEAD in case of a (harmless) failure, 448eb8dc403SDave Cobbley # and we want certain auto-generated information in the 449eb8dc403SDave Cobbley # commit message for future reference and/or automation. 450eb8dc403SDave Cobbley with open('.git/MERGE_HEAD', 'w') as head: 451eb8dc403SDave Cobbley with open('.git/MERGE_MSG', 'w') as msg: 452eb8dc403SDave Cobbley msg.write('repo: initial import of components\n\n') 453eb8dc403SDave Cobbley # head.write('%s\n' % startrev) 454eb8dc403SDave Cobbley for name in conf.repos: 455eb8dc403SDave Cobbley repo = conf.repos[name] 456eb8dc403SDave Cobbley # <upstream ref> <rewritten ref> <rewritten + files removed> 457eb8dc403SDave Cobbley msg.write('combo-layer-%s: %s %s %s\n' % (name, 458eb8dc403SDave Cobbley repo['last_revision'], 459eb8dc403SDave Cobbley repo['rewritten_revision'], 460eb8dc403SDave Cobbley repo['stripped_revision'])) 461eb8dc403SDave Cobbley rev = runcmd('git rev-parse %s' % name).strip() 462eb8dc403SDave Cobbley head.write('%s\n' % rev) 463eb8dc403SDave Cobbley 464eb8dc403SDave Cobbley if conf.localconffile: 465eb8dc403SDave Cobbley localadded = True 466eb8dc403SDave Cobbley try: 467eb8dc403SDave Cobbley runcmd("git rm --cached %s" % conf.localconffile, printerr=False) 468eb8dc403SDave Cobbley except subprocess.CalledProcessError: 469eb8dc403SDave Cobbley localadded = False 470eb8dc403SDave Cobbley if localadded: 471eb8dc403SDave Cobbley localrelpath = os.path.relpath(conf.localconffile) 472eb8dc403SDave Cobbley runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath)) 473eb8dc403SDave Cobbley runcmd("git add .gitignore") 474eb8dc403SDave Cobbley logger.info("Added local configuration file %s to .gitignore", localrelpath) 475eb8dc403SDave Cobbley logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.") 476eb8dc403SDave Cobbley else: 477eb8dc403SDave Cobbley logger.info("Repository already initialised, nothing to do.") 478eb8dc403SDave Cobbley 479eb8dc403SDave Cobbley 480eb8dc403SDave Cobbleydef check_repo_clean(repodir): 481eb8dc403SDave Cobbley """ 482eb8dc403SDave Cobbley check if the repo is clean 483eb8dc403SDave Cobbley exit if repo is dirty 484eb8dc403SDave Cobbley """ 485eb8dc403SDave Cobbley output=runcmd("git status --porcelain", repodir) 486*73bd93f1SPatrick Williams r = re.compile(r'\?\? patch-.*/') 487eb8dc403SDave Cobbley dirtyout = [item for item in output.splitlines() if not r.match(item)] 488eb8dc403SDave Cobbley if dirtyout: 489eb8dc403SDave Cobbley logger.error("git repo %s is dirty, please fix it first", repodir) 490eb8dc403SDave Cobbley sys.exit(1) 491eb8dc403SDave Cobbley 492eb8dc403SDave Cobbleydef check_patch(patchfile): 493eb8dc403SDave Cobbley f = open(patchfile, 'rb') 494eb8dc403SDave Cobbley ln = f.readline() 495eb8dc403SDave Cobbley of = None 496eb8dc403SDave Cobbley in_patch = False 497eb8dc403SDave Cobbley beyond_msg = False 498eb8dc403SDave Cobbley pre_buf = b'' 499eb8dc403SDave Cobbley while ln: 500eb8dc403SDave Cobbley if not beyond_msg: 501eb8dc403SDave Cobbley if ln == b'---\n': 502eb8dc403SDave Cobbley if not of: 503eb8dc403SDave Cobbley break 504eb8dc403SDave Cobbley in_patch = False 505eb8dc403SDave Cobbley beyond_msg = True 506eb8dc403SDave Cobbley elif ln.startswith(b'--- '): 507eb8dc403SDave Cobbley # We have a diff in the commit message 508eb8dc403SDave Cobbley in_patch = True 509eb8dc403SDave Cobbley if not of: 510eb8dc403SDave Cobbley print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile) 511eb8dc403SDave Cobbley of = open(patchfile + '.tmp', 'wb') 512eb8dc403SDave Cobbley of.write(pre_buf) 513eb8dc403SDave Cobbley pre_buf = b'' 514eb8dc403SDave Cobbley elif in_patch and not ln[0] in b'+-@ \n\r': 515eb8dc403SDave Cobbley in_patch = False 516eb8dc403SDave Cobbley if of: 517eb8dc403SDave Cobbley if in_patch: 518eb8dc403SDave Cobbley of.write(b' ' + ln) 519eb8dc403SDave Cobbley else: 520eb8dc403SDave Cobbley of.write(ln) 521eb8dc403SDave Cobbley else: 522eb8dc403SDave Cobbley pre_buf += ln 523eb8dc403SDave Cobbley ln = f.readline() 524eb8dc403SDave Cobbley f.close() 525eb8dc403SDave Cobbley if of: 526eb8dc403SDave Cobbley of.close() 527517393d9SAndrew Geissler os.rename(of.name, patchfile) 528eb8dc403SDave Cobbley 529eb8dc403SDave Cobbleydef drop_to_shell(workdir=None): 530eb8dc403SDave Cobbley if not sys.stdin.isatty(): 531eb8dc403SDave Cobbley print("Not a TTY so can't drop to shell for resolution, exiting.") 532eb8dc403SDave Cobbley return False 533eb8dc403SDave Cobbley 534eb8dc403SDave Cobbley shell = os.environ.get('SHELL', 'bash') 535eb8dc403SDave Cobbley print('Dropping to shell "%s"\n' \ 536eb8dc403SDave Cobbley 'When you are finished, run the following to continue:\n' \ 537eb8dc403SDave Cobbley ' exit -- continue to apply the patches\n' \ 538eb8dc403SDave Cobbley ' exit 1 -- abort\n' % shell); 539eb8dc403SDave Cobbley ret = subprocess.call([shell], cwd=workdir) 540eb8dc403SDave Cobbley if ret != 0: 541eb8dc403SDave Cobbley print("Aborting") 542eb8dc403SDave Cobbley return False 543eb8dc403SDave Cobbley else: 544eb8dc403SDave Cobbley return True 545eb8dc403SDave Cobbley 546eb8dc403SDave Cobbleydef check_rev_branch(component, repodir, rev, branch): 547eb8dc403SDave Cobbley try: 548eb8dc403SDave Cobbley actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False) 549eb8dc403SDave Cobbley except subprocess.CalledProcessError as e: 550eb8dc403SDave Cobbley if e.returncode == 129: 551eb8dc403SDave Cobbley actualbranch = "" 552eb8dc403SDave Cobbley else: 553eb8dc403SDave Cobbley raise 554eb8dc403SDave Cobbley 555eb8dc403SDave Cobbley if not actualbranch: 556eb8dc403SDave Cobbley logger.error("%s: specified revision %s is invalid!" % (component, rev)) 557eb8dc403SDave Cobbley return False 558eb8dc403SDave Cobbley 559eb8dc403SDave Cobbley branches = [] 560eb8dc403SDave Cobbley branchlist = actualbranch.split("\n") 561eb8dc403SDave Cobbley for b in branchlist: 562eb8dc403SDave Cobbley branches.append(b.strip().split(' ')[-1]) 563eb8dc403SDave Cobbley 564eb8dc403SDave Cobbley if branch not in branches: 565eb8dc403SDave Cobbley logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch)) 566eb8dc403SDave Cobbley return False 567eb8dc403SDave Cobbley return True 568eb8dc403SDave Cobbley 569eb8dc403SDave Cobbleydef get_repos(conf, repo_names): 570eb8dc403SDave Cobbley repos = [] 571eb8dc403SDave Cobbley for name in repo_names: 572eb8dc403SDave Cobbley if name.startswith('-'): 573eb8dc403SDave Cobbley break 574eb8dc403SDave Cobbley else: 575eb8dc403SDave Cobbley repos.append(name) 576eb8dc403SDave Cobbley for repo in repos: 577eb8dc403SDave Cobbley if not repo in conf.repos: 578eb8dc403SDave Cobbley logger.error("Specified component '%s' not found in configuration" % repo) 579eb8dc403SDave Cobbley sys.exit(1) 580eb8dc403SDave Cobbley 581eb8dc403SDave Cobbley if not repos: 582eb8dc403SDave Cobbley repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ] 583eb8dc403SDave Cobbley 584eb8dc403SDave Cobbley return repos 585eb8dc403SDave Cobbley 586eb8dc403SDave Cobbleydef action_pull(conf, args): 587eb8dc403SDave Cobbley """ 588eb8dc403SDave Cobbley update the component repos only 589eb8dc403SDave Cobbley """ 590eb8dc403SDave Cobbley repos = get_repos(conf, args[1:]) 591eb8dc403SDave Cobbley 592eb8dc403SDave Cobbley # make sure all repos are clean 593eb8dc403SDave Cobbley for name in repos: 594eb8dc403SDave Cobbley check_repo_clean(conf.repos[name]['local_repo_dir']) 595eb8dc403SDave Cobbley 596eb8dc403SDave Cobbley for name in repos: 597eb8dc403SDave Cobbley repo = conf.repos[name] 598eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 599eb8dc403SDave Cobbley branch = repo.get('branch', "master") 600eb8dc403SDave Cobbley logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir)) 601eb8dc403SDave Cobbley if not conf.hard_reset: 602eb8dc403SDave Cobbley # Try to pull only the configured branch. Beware that this may fail 603eb8dc403SDave Cobbley # when the branch is currently unknown (for example, after reconfiguring 604eb8dc403SDave Cobbley # combo-layer). In that case we need to fetch everything and try the check out 605eb8dc403SDave Cobbley # and pull again. 606eb8dc403SDave Cobbley try: 607eb8dc403SDave Cobbley runcmd("git checkout %s" % branch, ldir, printerr=False) 608eb8dc403SDave Cobbley except subprocess.CalledProcessError: 609eb8dc403SDave Cobbley output=runcmd("git fetch", ldir) 610eb8dc403SDave Cobbley logger.info(output) 611eb8dc403SDave Cobbley runcmd("git checkout %s" % branch, ldir) 612eb8dc403SDave Cobbley runcmd("git pull --ff-only", ldir) 613eb8dc403SDave Cobbley else: 614eb8dc403SDave Cobbley output=runcmd("git pull --ff-only", ldir) 615eb8dc403SDave Cobbley logger.info(output) 616eb8dc403SDave Cobbley else: 617eb8dc403SDave Cobbley output=runcmd("git fetch", ldir) 618eb8dc403SDave Cobbley logger.info(output) 619eb8dc403SDave Cobbley runcmd("git checkout %s" % branch, ldir) 620eb8dc403SDave Cobbley runcmd("git reset --hard FETCH_HEAD", ldir) 621eb8dc403SDave Cobbley 622eb8dc403SDave Cobbleydef action_update(conf, args): 623eb8dc403SDave Cobbley """ 624eb8dc403SDave Cobbley update the component repos 625eb8dc403SDave Cobbley either: 626eb8dc403SDave Cobbley generate the patch list 627eb8dc403SDave Cobbley apply the generated patches 628eb8dc403SDave Cobbley or: 629eb8dc403SDave Cobbley re-creates the entire component history and merges them 630eb8dc403SDave Cobbley into the current branch with a merge commit 631eb8dc403SDave Cobbley """ 632eb8dc403SDave Cobbley components = [arg.split(':')[0] for arg in args[1:]] 633eb8dc403SDave Cobbley revisions = {} 634eb8dc403SDave Cobbley for arg in args[1:]: 635eb8dc403SDave Cobbley if ':' in arg: 636eb8dc403SDave Cobbley a = arg.split(':', 1) 637eb8dc403SDave Cobbley revisions[a[0]] = a[1] 638eb8dc403SDave Cobbley repos = get_repos(conf, components) 639eb8dc403SDave Cobbley 640eb8dc403SDave Cobbley # make sure combo repo is clean 641eb8dc403SDave Cobbley check_repo_clean(os.getcwd()) 642eb8dc403SDave Cobbley 643eb8dc403SDave Cobbley # Check whether we keep the component histories. Must be 644eb8dc403SDave Cobbley # set either via --history command line parameter or consistently 645eb8dc403SDave Cobbley # in combo-layer.conf. Mixing modes is (currently, and probably 646eb8dc403SDave Cobbley # permanently because it would be complicated) not supported. 647eb8dc403SDave Cobbley if conf.history: 648eb8dc403SDave Cobbley history = True 649eb8dc403SDave Cobbley else: 650eb8dc403SDave Cobbley history = None 651eb8dc403SDave Cobbley for name in repos: 652eb8dc403SDave Cobbley repo = conf.repos[name] 653eb8dc403SDave Cobbley repo_history = repo.get('history', False) 654eb8dc403SDave Cobbley if history is None: 655eb8dc403SDave Cobbley history = repo_history 656eb8dc403SDave Cobbley elif history != repo_history: 657eb8dc403SDave Cobbley logger.error("'history' property is set inconsistently") 658eb8dc403SDave Cobbley sys.exit(1) 659eb8dc403SDave Cobbley 660eb8dc403SDave Cobbley # Step 1: update the component repos 661eb8dc403SDave Cobbley if conf.nopull: 662eb8dc403SDave Cobbley logger.info("Skipping pull (-n)") 663eb8dc403SDave Cobbley else: 664eb8dc403SDave Cobbley action_pull(conf, ['arg0'] + components) 665eb8dc403SDave Cobbley 666eb8dc403SDave Cobbley if history: 667eb8dc403SDave Cobbley update_with_history(conf, components, revisions, repos) 668eb8dc403SDave Cobbley else: 669eb8dc403SDave Cobbley update_with_patches(conf, components, revisions, repos) 670eb8dc403SDave Cobbley 671eb8dc403SDave Cobbleydef update_with_patches(conf, components, revisions, repos): 672eb8dc403SDave Cobbley import uuid 673eb8dc403SDave Cobbley patch_dir = "patch-%s" % uuid.uuid4() 674eb8dc403SDave Cobbley if not os.path.exists(patch_dir): 675eb8dc403SDave Cobbley os.mkdir(patch_dir) 676eb8dc403SDave Cobbley 677eb8dc403SDave Cobbley for name in repos: 678eb8dc403SDave Cobbley revision = revisions.get(name, None) 679eb8dc403SDave Cobbley repo = conf.repos[name] 680eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 681eb8dc403SDave Cobbley dest_dir = repo['dest_dir'] 682eb8dc403SDave Cobbley branch = repo.get('branch', "master") 683eb8dc403SDave Cobbley repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name) 684eb8dc403SDave Cobbley 685eb8dc403SDave Cobbley # Step 2: generate the patch list and store to patch dir 686eb8dc403SDave Cobbley logger.info("Generating patches from %s..." % name) 687eb8dc403SDave Cobbley top_revision = revision or branch 688eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, top_revision, branch): 689eb8dc403SDave Cobbley sys.exit(1) 690eb8dc403SDave Cobbley if dest_dir != ".": 691eb8dc403SDave Cobbley prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir) 692eb8dc403SDave Cobbley else: 693eb8dc403SDave Cobbley prefix = "" 694eb8dc403SDave Cobbley if repo['last_revision'] == "": 695eb8dc403SDave Cobbley logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name) 696eb8dc403SDave Cobbley patch_cmd_range = "--root %s" % top_revision 697eb8dc403SDave Cobbley rev_cmd_range = top_revision 698eb8dc403SDave Cobbley else: 699eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, repo['last_revision'], branch): 700eb8dc403SDave Cobbley sys.exit(1) 701eb8dc403SDave Cobbley patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision) 702eb8dc403SDave Cobbley rev_cmd_range = patch_cmd_range 703eb8dc403SDave Cobbley 704eb8dc403SDave Cobbley file_filter = repo.get('file_filter',".") 705eb8dc403SDave Cobbley 706eb8dc403SDave Cobbley # Filter out unwanted files 707eb8dc403SDave Cobbley exclude = repo.get('file_exclude', '') 708eb8dc403SDave Cobbley if exclude: 709eb8dc403SDave Cobbley for path in exclude.split(): 710eb8dc403SDave Cobbley p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path 711eb8dc403SDave Cobbley file_filter += " ':!%s'" % p 712eb8dc403SDave Cobbley 713eb8dc403SDave Cobbley patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \ 714eb8dc403SDave Cobbley (prefix,repo_patch_dir, patch_cmd_range, file_filter) 715eb8dc403SDave Cobbley output = runcmd(patch_cmd, ldir) 716eb8dc403SDave Cobbley logger.debug("generated patch set:\n%s" % output) 717eb8dc403SDave Cobbley patchlist = output.splitlines() 718eb8dc403SDave Cobbley 719eb8dc403SDave Cobbley rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter) 720eb8dc403SDave Cobbley revlist = runcmd(rev_cmd, ldir).splitlines() 721eb8dc403SDave Cobbley 722eb8dc403SDave Cobbley # Step 3: Call repo specific hook to adjust patch 723eb8dc403SDave Cobbley if 'hook' in repo: 724eb8dc403SDave Cobbley # hook parameter is: ./hook patchpath revision reponame 725eb8dc403SDave Cobbley count=len(revlist)-1 726eb8dc403SDave Cobbley for patch in patchlist: 727eb8dc403SDave Cobbley runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name)) 728eb8dc403SDave Cobbley count=count-1 729eb8dc403SDave Cobbley 730eb8dc403SDave Cobbley # Step 4: write patch list and revision list to file, for user to edit later 731eb8dc403SDave Cobbley patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name) 732eb8dc403SDave Cobbley repo['patchlist'] = patchlist_file 733eb8dc403SDave Cobbley f = open(patchlist_file, 'w') 734eb8dc403SDave Cobbley count=len(revlist)-1 735eb8dc403SDave Cobbley for patch in patchlist: 736eb8dc403SDave Cobbley f.write("%s %s\n" % (patch, revlist[count])) 737eb8dc403SDave Cobbley check_patch(os.path.join(patch_dir, patch)) 738eb8dc403SDave Cobbley count=count-1 739eb8dc403SDave Cobbley f.close() 740eb8dc403SDave Cobbley 741eb8dc403SDave Cobbley # Step 5: invoke bash for user to edit patch and patch list 742eb8dc403SDave Cobbley if conf.interactive: 743eb8dc403SDave Cobbley print('You may now edit the patch and patch list in %s\n' \ 744eb8dc403SDave Cobbley 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir); 745eb8dc403SDave Cobbley if not drop_to_shell(patch_dir): 746eb8dc403SDave Cobbley sys.exit(1) 747eb8dc403SDave Cobbley 748eb8dc403SDave Cobbley # Step 6: apply the generated and revised patch 749eb8dc403SDave Cobbley apply_patchlist(conf, repos) 750eb8dc403SDave Cobbley runcmd("rm -rf %s" % patch_dir) 751eb8dc403SDave Cobbley 752eb8dc403SDave Cobbley # Step 7: commit the updated config file if it's being tracked 753eb8dc403SDave Cobbley commit_conf_file(conf, components) 754eb8dc403SDave Cobbley 755eb8dc403SDave Cobbleydef conf_commit_msg(conf, components): 756eb8dc403SDave Cobbley # create the "components" string 757eb8dc403SDave Cobbley component_str = "all components" 758eb8dc403SDave Cobbley if len(components) > 0: 759eb8dc403SDave Cobbley # otherwise tell which components were actually changed 760eb8dc403SDave Cobbley component_str = ", ".join(components) 761eb8dc403SDave Cobbley 762eb8dc403SDave Cobbley # expand the template with known values 763eb8dc403SDave Cobbley template = Template(conf.commit_msg_template) 764eb8dc403SDave Cobbley msg = template.substitute(components = component_str) 765eb8dc403SDave Cobbley return msg 766eb8dc403SDave Cobbley 767eb8dc403SDave Cobbleydef commit_conf_file(conf, components, commit=True): 768eb8dc403SDave Cobbley relpath = os.path.relpath(conf.conffile) 769eb8dc403SDave Cobbley try: 770eb8dc403SDave Cobbley output = runcmd("git status --porcelain %s" % relpath, printerr=False) 771eb8dc403SDave Cobbley except: 772eb8dc403SDave Cobbley # Outside the repository 773eb8dc403SDave Cobbley output = None 774eb8dc403SDave Cobbley if output: 775eb8dc403SDave Cobbley if output.lstrip().startswith("M"): 776eb8dc403SDave Cobbley logger.info("Committing updated configuration file") 777eb8dc403SDave Cobbley if commit: 778eb8dc403SDave Cobbley msg = conf_commit_msg(conf, components) 779eb8dc403SDave Cobbley runcmd('git commit -m'.split() + [msg, relpath]) 780eb8dc403SDave Cobbley else: 781eb8dc403SDave Cobbley runcmd('git add %s' % relpath) 782eb8dc403SDave Cobbley return True 783eb8dc403SDave Cobbley return False 784eb8dc403SDave Cobbley 785eb8dc403SDave Cobbleydef apply_patchlist(conf, repos): 786eb8dc403SDave Cobbley """ 787eb8dc403SDave Cobbley apply the generated patch list to combo repo 788eb8dc403SDave Cobbley """ 789eb8dc403SDave Cobbley for name in repos: 790eb8dc403SDave Cobbley repo = conf.repos[name] 791eb8dc403SDave Cobbley lastrev = repo["last_revision"] 792eb8dc403SDave Cobbley prevrev = lastrev 793eb8dc403SDave Cobbley 794eb8dc403SDave Cobbley # Get non-blank lines from patch list file 795eb8dc403SDave Cobbley patchlist = [] 796eb8dc403SDave Cobbley if os.path.exists(repo['patchlist']) or not conf.interactive: 797eb8dc403SDave Cobbley # Note: we want this to fail here if the file doesn't exist and we're not in 798eb8dc403SDave Cobbley # interactive mode since the file should exist in this case 799eb8dc403SDave Cobbley with open(repo['patchlist']) as f: 800eb8dc403SDave Cobbley for line in f: 801eb8dc403SDave Cobbley line = line.rstrip() 802eb8dc403SDave Cobbley if line: 803eb8dc403SDave Cobbley patchlist.append(line) 804eb8dc403SDave Cobbley 805eb8dc403SDave Cobbley ldir = conf.repos[name]['local_repo_dir'] 806eb8dc403SDave Cobbley branch = conf.repos[name].get('branch', "master") 807eb8dc403SDave Cobbley branchrev = runcmd("git rev-parse %s" % branch, ldir).strip() 808eb8dc403SDave Cobbley 809eb8dc403SDave Cobbley if patchlist: 810eb8dc403SDave Cobbley logger.info("Applying patches from %s..." % name) 811eb8dc403SDave Cobbley linecount = len(patchlist) 812eb8dc403SDave Cobbley i = 1 813eb8dc403SDave Cobbley for line in patchlist: 814eb8dc403SDave Cobbley patchfile = line.split()[0] 815eb8dc403SDave Cobbley lastrev = line.split()[1] 816eb8dc403SDave Cobbley patchdisp = os.path.relpath(patchfile) 817eb8dc403SDave Cobbley if os.path.getsize(patchfile) == 0: 818eb8dc403SDave Cobbley logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp)) 819eb8dc403SDave Cobbley else: 820eb8dc403SDave Cobbley cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile) 821eb8dc403SDave Cobbley logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp)) 822eb8dc403SDave Cobbley try: 823eb8dc403SDave Cobbley runcmd(cmd) 824eb8dc403SDave Cobbley except subprocess.CalledProcessError: 825eb8dc403SDave Cobbley logger.info('Running "git am --abort" to cleanup repo') 826eb8dc403SDave Cobbley runcmd("git am --abort") 827eb8dc403SDave Cobbley logger.error('"%s" failed' % cmd) 828eb8dc403SDave Cobbley logger.info("Please manually apply patch %s" % patchdisp) 829eb8dc403SDave Cobbley logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped") 830eb8dc403SDave Cobbley if not drop_to_shell(): 831eb8dc403SDave Cobbley if prevrev != repo['last_revision']: 832eb8dc403SDave Cobbley conf.update(name, "last_revision", prevrev) 833eb8dc403SDave Cobbley sys.exit(1) 834eb8dc403SDave Cobbley prevrev = lastrev 835eb8dc403SDave Cobbley i += 1 836eb8dc403SDave Cobbley # Once all patches are applied, we should update 837eb8dc403SDave Cobbley # last_revision to the branch head instead of the last 838eb8dc403SDave Cobbley # applied patch. The two are not necessarily the same when 839eb8dc403SDave Cobbley # the last commit is a merge commit or when the patches at 840eb8dc403SDave Cobbley # the branch head were intentionally excluded. 841eb8dc403SDave Cobbley # 842eb8dc403SDave Cobbley # If we do not do that for a merge commit, the next 843eb8dc403SDave Cobbley # combo-layer run will only exclude patches reachable from 844eb8dc403SDave Cobbley # one of the merged branches and try to re-apply patches 845eb8dc403SDave Cobbley # from other branches even though they were already 846eb8dc403SDave Cobbley # copied. 847eb8dc403SDave Cobbley # 848eb8dc403SDave Cobbley # If patches were intentionally excluded, the next run will 849eb8dc403SDave Cobbley # present them again instead of skipping over them. This 850eb8dc403SDave Cobbley # may or may not be intended, so the code here is conservative 851eb8dc403SDave Cobbley # and only addresses the "head is merge commit" case. 852eb8dc403SDave Cobbley if lastrev != branchrev and \ 853eb8dc403SDave Cobbley len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1: 854eb8dc403SDave Cobbley lastrev = branchrev 855eb8dc403SDave Cobbley else: 856eb8dc403SDave Cobbley logger.info("No patches to apply from %s" % name) 857eb8dc403SDave Cobbley lastrev = branchrev 858eb8dc403SDave Cobbley 859eb8dc403SDave Cobbley if lastrev != repo['last_revision']: 860eb8dc403SDave Cobbley conf.update(name, "last_revision", lastrev) 861eb8dc403SDave Cobbley 862eb8dc403SDave Cobbleydef action_splitpatch(conf, args): 863eb8dc403SDave Cobbley """ 864eb8dc403SDave Cobbley generate the commit patch and 865eb8dc403SDave Cobbley split the patch per repo 866eb8dc403SDave Cobbley """ 867eb8dc403SDave Cobbley logger.debug("action_splitpatch") 868eb8dc403SDave Cobbley if len(args) > 1: 869eb8dc403SDave Cobbley commit = args[1] 870eb8dc403SDave Cobbley else: 871eb8dc403SDave Cobbley commit = "HEAD" 872eb8dc403SDave Cobbley patchdir = "splitpatch-%s" % commit 873eb8dc403SDave Cobbley if not os.path.exists(patchdir): 874eb8dc403SDave Cobbley os.mkdir(patchdir) 875eb8dc403SDave Cobbley 876eb8dc403SDave Cobbley # filerange_root is for the repo whose dest_dir is root "." 877eb8dc403SDave Cobbley # and it should be specified by excluding all other repo dest dir 878eb8dc403SDave Cobbley # like "-x repo1 -x repo2 -x repo3 ..." 879eb8dc403SDave Cobbley filerange_root = "" 880eb8dc403SDave Cobbley for name in conf.repos: 881eb8dc403SDave Cobbley dest_dir = conf.repos[name]['dest_dir'] 882eb8dc403SDave Cobbley if dest_dir != ".": 883eb8dc403SDave Cobbley filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir) 884eb8dc403SDave Cobbley 885eb8dc403SDave Cobbley for name in conf.repos: 886eb8dc403SDave Cobbley dest_dir = conf.repos[name]['dest_dir'] 887eb8dc403SDave Cobbley patch_filename = "%s/%s.patch" % (patchdir, name) 888eb8dc403SDave Cobbley if dest_dir == ".": 889eb8dc403SDave Cobbley cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename) 890eb8dc403SDave Cobbley else: 891eb8dc403SDave Cobbley cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename) 892eb8dc403SDave Cobbley runcmd(cmd) 893eb8dc403SDave Cobbley # Detect empty patches (including those produced by filterdiff above 894eb8dc403SDave Cobbley # that contain only preamble text) 895eb8dc403SDave Cobbley if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "": 896eb8dc403SDave Cobbley os.remove(patch_filename) 897eb8dc403SDave Cobbley logger.info("(skipping %s - no changes)", name) 898eb8dc403SDave Cobbley else: 899eb8dc403SDave Cobbley logger.info(patch_filename) 900eb8dc403SDave Cobbley 901eb8dc403SDave Cobbleydef update_with_history(conf, components, revisions, repos): 902eb8dc403SDave Cobbley '''Update all components with full history. 903eb8dc403SDave Cobbley 904eb8dc403SDave Cobbley Works by importing all commits reachable from a component's 905eb8dc403SDave Cobbley current head revision. If those commits are rooted in an already 906eb8dc403SDave Cobbley imported commit, their content gets mixed with the content of the 907eb8dc403SDave Cobbley combined repo of that commit (new or modified files overwritten, 908eb8dc403SDave Cobbley removed files removed). 909eb8dc403SDave Cobbley 910eb8dc403SDave Cobbley The last commit is an artificial merge commit that merges all the 911eb8dc403SDave Cobbley updated components into the combined repository. 912eb8dc403SDave Cobbley 913eb8dc403SDave Cobbley The HEAD ref only gets updated at the very end. All intermediate work 914eb8dc403SDave Cobbley happens in a worktree which will get garbage collected by git eventually 915eb8dc403SDave Cobbley after a failure. 916eb8dc403SDave Cobbley ''' 917eb8dc403SDave Cobbley # Remember current HEAD and what we need to add to it. 918eb8dc403SDave Cobbley head = runcmd("git rev-parse HEAD").strip() 919eb8dc403SDave Cobbley additional_heads = {} 920eb8dc403SDave Cobbley 921eb8dc403SDave Cobbley # Track the mapping between original commit and commit in the 922eb8dc403SDave Cobbley # combined repo. We do not have to distinguish between components, 923eb8dc403SDave Cobbley # because commit hashes are different anyway. Often we can 924eb8dc403SDave Cobbley # skip find_revs() entirely (for example, when all new commits 925eb8dc403SDave Cobbley # are derived from the last imported revision). 926eb8dc403SDave Cobbley # 927eb8dc403SDave Cobbley # Using "head" (typically the merge commit) instead of the actual 928eb8dc403SDave Cobbley # commit for the component leads to a nicer history in the combined 929eb8dc403SDave Cobbley # repo. 930eb8dc403SDave Cobbley old2new_revs = {} 931eb8dc403SDave Cobbley for name in repos: 932eb8dc403SDave Cobbley repo = conf.repos[name] 933eb8dc403SDave Cobbley revision = repo['last_revision'] 934eb8dc403SDave Cobbley if revision: 935eb8dc403SDave Cobbley old2new_revs[revision] = head 936eb8dc403SDave Cobbley 937eb8dc403SDave Cobbley def add_p(parents): 938eb8dc403SDave Cobbley '''Insert -p before each entry.''' 939eb8dc403SDave Cobbley parameters = [] 940eb8dc403SDave Cobbley for p in parents: 941eb8dc403SDave Cobbley parameters.append('-p') 942eb8dc403SDave Cobbley parameters.append(p) 943eb8dc403SDave Cobbley return parameters 944eb8dc403SDave Cobbley 945eb8dc403SDave Cobbley # Do all intermediate work with a separate work dir and index, 946eb8dc403SDave Cobbley # chosen via env variables (can't use "git worktree", it is too 947eb8dc403SDave Cobbley # new). This is useful (no changes to current work tree unless the 948eb8dc403SDave Cobbley # update succeeds) and required (otherwise we end up temporarily 949eb8dc403SDave Cobbley # removing the combo-layer hooks that we currently use when 950eb8dc403SDave Cobbley # importing a new component). 951eb8dc403SDave Cobbley # 952eb8dc403SDave Cobbley # Not cleaned up after a failure at the moment. 953eb8dc403SDave Cobbley wdir = os.path.join(os.getcwd(), ".git", "combo-layer") 954eb8dc403SDave Cobbley windex = wdir + ".index" 955eb8dc403SDave Cobbley if os.path.isdir(wdir): 956eb8dc403SDave Cobbley shutil.rmtree(wdir) 957eb8dc403SDave Cobbley os.mkdir(wdir) 958eb8dc403SDave Cobbley wenv = copy.deepcopy(os.environ) 959eb8dc403SDave Cobbley wenv["GIT_WORK_TREE"] = wdir 960eb8dc403SDave Cobbley wenv["GIT_INDEX_FILE"] = windex 961eb8dc403SDave Cobbley # This one turned out to be needed in practice. 962eb8dc403SDave Cobbley wenv["GIT_OBJECT_DIRECTORY"] = os.path.join(os.getcwd(), ".git", "objects") 963eb8dc403SDave Cobbley wargs = {"destdir": wdir, "env": wenv} 964eb8dc403SDave Cobbley 965eb8dc403SDave Cobbley for name in repos: 966eb8dc403SDave Cobbley revision = revisions.get(name, None) 967eb8dc403SDave Cobbley repo = conf.repos[name] 968eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 969eb8dc403SDave Cobbley dest_dir = repo['dest_dir'] 970eb8dc403SDave Cobbley branch = repo.get('branch', "master") 971eb8dc403SDave Cobbley hook = repo.get('hook', None) 972eb8dc403SDave Cobbley largs = {"destdir": ldir, "env": None} 973eb8dc403SDave Cobbley file_include = repo.get('file_filter', '').split() 974eb8dc403SDave Cobbley file_include.sort() # make sure that short entries like '.' come first. 975eb8dc403SDave Cobbley file_exclude = repo.get('file_exclude', '').split() 976eb8dc403SDave Cobbley 977eb8dc403SDave Cobbley def include_file(file): 978eb8dc403SDave Cobbley if not file_include: 979eb8dc403SDave Cobbley # No explicit filter set, include file. 980eb8dc403SDave Cobbley return True 981eb8dc403SDave Cobbley for filter in file_include: 982eb8dc403SDave Cobbley if filter == '.': 983eb8dc403SDave Cobbley # Another special case: include current directory and thus all files. 984eb8dc403SDave Cobbley return True 985eb8dc403SDave Cobbley if os.path.commonprefix((filter, file)) == filter: 986eb8dc403SDave Cobbley # Included in directory or direct file match. 987eb8dc403SDave Cobbley return True 988eb8dc403SDave Cobbley # Check for wildcard match *with* allowing * to match /, i.e. 989eb8dc403SDave Cobbley # src/*.c does match src/foobar/*.c. That's not how it is done elsewhere 990eb8dc403SDave Cobbley # when passing the filtering to "git archive", but it is unclear what 991eb8dc403SDave Cobbley # the intended semantic is (the comment on file_exclude that "append a * wildcard 992eb8dc403SDave Cobbley # at the end" to match the full content of a directories implies that 993eb8dc403SDave Cobbley # slashes are indeed not special), so here we simply do what's easy to 994eb8dc403SDave Cobbley # implement in Python. 995eb8dc403SDave Cobbley logger.debug('fnmatch(%s, %s)' % (file, filter)) 996eb8dc403SDave Cobbley if fnmatch.fnmatchcase(file, filter): 997eb8dc403SDave Cobbley return True 998eb8dc403SDave Cobbley return False 999eb8dc403SDave Cobbley 1000eb8dc403SDave Cobbley def exclude_file(file): 1001eb8dc403SDave Cobbley for filter in file_exclude: 1002eb8dc403SDave Cobbley if fnmatch.fnmatchcase(file, filter): 1003eb8dc403SDave Cobbley return True 1004eb8dc403SDave Cobbley return False 1005eb8dc403SDave Cobbley 1006eb8dc403SDave Cobbley def file_filter(files): 1007eb8dc403SDave Cobbley '''Clean up file list so that only included files remain.''' 1008eb8dc403SDave Cobbley index = 0 1009eb8dc403SDave Cobbley while index < len(files): 1010eb8dc403SDave Cobbley file = files[index] 1011eb8dc403SDave Cobbley if not include_file(file) or exclude_file(file): 1012eb8dc403SDave Cobbley del files[index] 1013eb8dc403SDave Cobbley else: 1014eb8dc403SDave Cobbley index += 1 1015eb8dc403SDave Cobbley 1016eb8dc403SDave Cobbley 1017eb8dc403SDave Cobbley # Generate the revision list. 1018eb8dc403SDave Cobbley logger.info("Analyzing commits from %s..." % name) 1019eb8dc403SDave Cobbley top_revision = revision or branch 1020eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, top_revision, branch): 1021eb8dc403SDave Cobbley sys.exit(1) 1022eb8dc403SDave Cobbley 1023eb8dc403SDave Cobbley last_revision = repo['last_revision'] 1024eb8dc403SDave Cobbley rev_list_args = "--full-history --sparse --topo-order --reverse" 1025eb8dc403SDave Cobbley if not last_revision: 1026eb8dc403SDave Cobbley logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name) 1027eb8dc403SDave Cobbley rev_list_args = rev_list_args + ' ' + top_revision 1028eb8dc403SDave Cobbley else: 1029eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, last_revision, branch): 1030eb8dc403SDave Cobbley sys.exit(1) 1031eb8dc403SDave Cobbley rev_list_args = "%s %s..%s" % (rev_list_args, last_revision, top_revision) 1032eb8dc403SDave Cobbley 1033eb8dc403SDave Cobbley # By definition, the current HEAD contains the latest imported 1034eb8dc403SDave Cobbley # commit of each component. We use that as initial mapping even 1035eb8dc403SDave Cobbley # though the commits do not match exactly because 1036eb8dc403SDave Cobbley # a) it always works (in contrast to find_revs, which relies on special 1037eb8dc403SDave Cobbley # commit messages) 1038eb8dc403SDave Cobbley # b) it is faster than find_revs, which will only be called on demand 1039eb8dc403SDave Cobbley # and can be skipped entirely in most cases 1040eb8dc403SDave Cobbley # c) last but not least, the combined history looks nicer when all 1041eb8dc403SDave Cobbley # new commits are rooted in the same merge commit 1042eb8dc403SDave Cobbley old2new_revs[last_revision] = head 1043eb8dc403SDave Cobbley 1044eb8dc403SDave Cobbley # We care about all commits (--full-history and --sparse) and 1045eb8dc403SDave Cobbley # we want reconstruct the topology and thus do not care 1046eb8dc403SDave Cobbley # about ordering by time (--topo-order). We ask for the ones 1047eb8dc403SDave Cobbley # we need to import first to be listed first (--reverse). 1048eb8dc403SDave Cobbley revs = runcmd("git rev-list %s" % rev_list_args, **largs).split() 1049eb8dc403SDave Cobbley logger.debug("To be imported: %s" % revs) 1050eb8dc403SDave Cobbley # Now 'revs' contains all revisions reachable from the top revision. 1051eb8dc403SDave Cobbley # All revisions derived from the 'last_revision' definitely are new, 1052eb8dc403SDave Cobbley # whereas the others may or may not have been imported before. For 1053eb8dc403SDave Cobbley # a linear history in the component, that second set will be empty. 1054eb8dc403SDave Cobbley # To distinguish between them, we also get the shorter list 1055eb8dc403SDave Cobbley # of revisions starting at the ancestor. 1056eb8dc403SDave Cobbley if last_revision: 1057eb8dc403SDave Cobbley ancestor_revs = runcmd("git rev-list --ancestry-path %s" % rev_list_args, **largs).split() 1058eb8dc403SDave Cobbley else: 1059eb8dc403SDave Cobbley ancestor_revs = [] 1060eb8dc403SDave Cobbley logger.debug("Ancestors: %s" % ancestor_revs) 1061eb8dc403SDave Cobbley 1062eb8dc403SDave Cobbley # Now import each revision. 1063eb8dc403SDave Cobbley logger.info("Importing commits from %s..." % name) 1064eb8dc403SDave Cobbley def import_rev(rev): 1065eb8dc403SDave Cobbley global scanned_revs 1066eb8dc403SDave Cobbley 1067eb8dc403SDave Cobbley # If it is part of the new commits, we definitely need 1068eb8dc403SDave Cobbley # to import it. Otherwise we need to check, we might have 1069eb8dc403SDave Cobbley # imported it before. If it was imported and we merely 1070eb8dc403SDave Cobbley # fail to find it because commit messages did not track 1071eb8dc403SDave Cobbley # the mapping, then we end up importing it again. So 1072eb8dc403SDave Cobbley # combined repos using "updating with history" really should 1073eb8dc403SDave Cobbley # enable the "From ... rev:" commit header modifications. 1074eb8dc403SDave Cobbley if rev not in ancestor_revs and rev not in old2new_revs and not scanned_revs: 1075eb8dc403SDave Cobbley logger.debug("Revision %s triggers log analysis." % rev) 1076eb8dc403SDave Cobbley find_revs(old2new_revs, head) 1077eb8dc403SDave Cobbley scanned_revs = True 1078eb8dc403SDave Cobbley new_rev = old2new_revs.get(rev, None) 1079eb8dc403SDave Cobbley if new_rev: 1080eb8dc403SDave Cobbley return new_rev 1081eb8dc403SDave Cobbley 1082eb8dc403SDave Cobbley # If the commit is not in the original list of revisions 1083eb8dc403SDave Cobbley # to be imported, then it must be a parent of one of those 1084eb8dc403SDave Cobbley # commits and it was skipped during earlier imports or not 1085eb8dc403SDave Cobbley # found. Importing such merge commits leads to very ugly 1086eb8dc403SDave Cobbley # history (long cascade of merge commits which all point 1087eb8dc403SDave Cobbley # to to older commits) when switching from "update via 1088eb8dc403SDave Cobbley # patches" to "update with history". 1089eb8dc403SDave Cobbley # 1090eb8dc403SDave Cobbley # We can avoid importing merge commits if all non-merge commits 1091eb8dc403SDave Cobbley # reachable from it were already imported. In that case we 1092eb8dc403SDave Cobbley # can root the new commits in the current head revision. 1093eb8dc403SDave Cobbley def is_imported(prev): 1094eb8dc403SDave Cobbley parents = runcmd("git show --no-patch --pretty=format:%P " + prev, **largs).split() 1095eb8dc403SDave Cobbley if len(parents) > 1: 1096eb8dc403SDave Cobbley for p in parents: 1097eb8dc403SDave Cobbley if not is_imported(p): 1098eb8dc403SDave Cobbley logger.debug("Must import %s because %s is not imported." % (rev, p)) 1099eb8dc403SDave Cobbley return False 1100eb8dc403SDave Cobbley return True 1101eb8dc403SDave Cobbley elif prev in old2new_revs: 1102eb8dc403SDave Cobbley return True 1103eb8dc403SDave Cobbley else: 1104eb8dc403SDave Cobbley logger.debug("Must import %s because %s is not imported." % (rev, prev)) 1105eb8dc403SDave Cobbley return False 1106eb8dc403SDave Cobbley if rev not in revs and is_imported(rev): 1107eb8dc403SDave Cobbley old2new_revs[rev] = head 1108eb8dc403SDave Cobbley return head 1109eb8dc403SDave Cobbley 1110eb8dc403SDave Cobbley # Need to import rev. Collect some information about it. 1111eb8dc403SDave Cobbley logger.debug("Importing %s" % rev) 1112eb8dc403SDave Cobbley (parents, author_name, author_email, author_timestamp, body) = \ 1113eb8dc403SDave Cobbley runcmd("git show --no-patch --pretty=format:%P%x00%an%x00%ae%x00%at%x00%B " + rev, **largs).split(chr(0)) 1114eb8dc403SDave Cobbley parents = parents.split() 1115eb8dc403SDave Cobbley if parents: 1116eb8dc403SDave Cobbley # Arbitrarily pick the first parent as base. It may or may not have 1117eb8dc403SDave Cobbley # been imported before. For example, if the parent is a merge commit 1118eb8dc403SDave Cobbley # and previously the combined repository used patching as update 1119eb8dc403SDave Cobbley # method, then the actual merge commit parent never was imported. 1120eb8dc403SDave Cobbley # To cover this, We recursively import parents. 1121eb8dc403SDave Cobbley parent = parents[0] 1122eb8dc403SDave Cobbley new_parent = import_rev(parent) 1123eb8dc403SDave Cobbley # Clean index and working tree. TODO: can we combine this and the 1124eb8dc403SDave Cobbley # next into one command with less file IO? 1125eb8dc403SDave Cobbley # "git reset --hard" does not work, it changes HEAD of the parent 1126eb8dc403SDave Cobbley # repo, which we wanted to avoid. Probably need to keep 1127eb8dc403SDave Cobbley # track of the rev that corresponds to the index and use apply_commit(). 1128eb8dc403SDave Cobbley runcmd("git rm -q --ignore-unmatch -rf .", **wargs) 1129eb8dc403SDave Cobbley # Update index and working tree to match the parent. 1130eb8dc403SDave Cobbley runcmd("git checkout -q -f %s ." % new_parent, **wargs) 1131eb8dc403SDave Cobbley else: 1132eb8dc403SDave Cobbley parent = None 1133eb8dc403SDave Cobbley # Clean index and working tree. 1134eb8dc403SDave Cobbley runcmd("git rm -q --ignore-unmatch -rf .", **wargs) 1135eb8dc403SDave Cobbley 1136eb8dc403SDave Cobbley # Modify index and working tree such that it mirrors the commit. 1137eb8dc403SDave Cobbley apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=file_filter) 1138eb8dc403SDave Cobbley 1139eb8dc403SDave Cobbley # Now commit. 1140eb8dc403SDave Cobbley new_tree = runcmd("git write-tree", **wargs).strip() 1141eb8dc403SDave Cobbley env = copy.deepcopy(wenv) 1142eb8dc403SDave Cobbley env['GIT_AUTHOR_NAME'] = author_name 1143eb8dc403SDave Cobbley env['GIT_AUTHOR_EMAIL'] = author_email 1144eb8dc403SDave Cobbley env['GIT_AUTHOR_DATE'] = author_timestamp 1145eb8dc403SDave Cobbley if hook: 1146eb8dc403SDave Cobbley # Need to turn the verbatim commit message into something resembling a patch header 1147eb8dc403SDave Cobbley # for the hook. 1148eb8dc403SDave Cobbley with tempfile.NamedTemporaryFile(mode='wt', delete=False) as patch: 1149eb8dc403SDave Cobbley patch.write('Subject: [PATCH] ') 1150eb8dc403SDave Cobbley patch.write(body) 1151eb8dc403SDave Cobbley patch.write('\n---\n') 1152eb8dc403SDave Cobbley patch.close() 1153eb8dc403SDave Cobbley runcmd([hook, patch.name, rev, name]) 1154eb8dc403SDave Cobbley with open(patch.name) as f: 1155eb8dc403SDave Cobbley body = f.read()[len('Subject: [PATCH] '):][:-len('\n---\n')] 1156eb8dc403SDave Cobbley 1157eb8dc403SDave Cobbley # We can skip non-merge commits that did not change any files. Those are typically 1158eb8dc403SDave Cobbley # the result of file filtering, although they could also have been introduced 1159eb8dc403SDave Cobbley # intentionally upstream, in which case we drop some information here. 1160eb8dc403SDave Cobbley if len(parents) == 1: 1161eb8dc403SDave Cobbley parent_rev = import_rev(parents[0]) 1162eb8dc403SDave Cobbley old_tree = runcmd("git show -s --pretty=format:%T " + parent_rev, **wargs).strip() 1163eb8dc403SDave Cobbley commit = old_tree != new_tree 1164eb8dc403SDave Cobbley if not commit: 1165eb8dc403SDave Cobbley new_rev = parent_rev 1166eb8dc403SDave Cobbley else: 1167eb8dc403SDave Cobbley commit = True 1168eb8dc403SDave Cobbley if commit: 1169eb8dc403SDave Cobbley new_rev = runcmd("git commit-tree".split() + add_p([import_rev(p) for p in parents]) + 1170eb8dc403SDave Cobbley ["-m", body, new_tree], 1171eb8dc403SDave Cobbley env=env).strip() 1172eb8dc403SDave Cobbley old2new_revs[rev] = new_rev 1173eb8dc403SDave Cobbley 1174eb8dc403SDave Cobbley return new_rev 1175eb8dc403SDave Cobbley 1176eb8dc403SDave Cobbley if revs: 1177eb8dc403SDave Cobbley for rev in revs: 1178eb8dc403SDave Cobbley import_rev(rev) 1179eb8dc403SDave Cobbley # Remember how to update our current head. New components get added, 1180eb8dc403SDave Cobbley # updated components get the delta between current head and the updated component 1181eb8dc403SDave Cobbley # applied. 1182eb8dc403SDave Cobbley additional_heads[old2new_revs[revs[-1]]] = head if repo['last_revision'] else None 1183eb8dc403SDave Cobbley repo['last_revision'] = revs[-1] 1184eb8dc403SDave Cobbley 1185eb8dc403SDave Cobbley # Now construct the final merge commit. We create the tree by 1186eb8dc403SDave Cobbley # starting with the head and applying the changes from each 1187eb8dc403SDave Cobbley # components imported head revision. 1188eb8dc403SDave Cobbley if additional_heads: 1189eb8dc403SDave Cobbley runcmd("git reset --hard", **wargs) 1190eb8dc403SDave Cobbley for rev, base in additional_heads.items(): 1191eb8dc403SDave Cobbley apply_commit(base, rev, wargs, wargs, None) 1192eb8dc403SDave Cobbley 1193eb8dc403SDave Cobbley # Commit with all component branches as parents as well as the previous head. 1194eb8dc403SDave Cobbley logger.info("Writing final merge commit...") 1195eb8dc403SDave Cobbley msg = conf_commit_msg(conf, components) 1196eb8dc403SDave Cobbley new_tree = runcmd("git write-tree", **wargs).strip() 1197eb8dc403SDave Cobbley new_rev = runcmd("git commit-tree".split() + 1198eb8dc403SDave Cobbley add_p([head] + list(additional_heads.keys())) + 1199eb8dc403SDave Cobbley ["-m", msg, new_tree], 1200eb8dc403SDave Cobbley **wargs).strip() 1201eb8dc403SDave Cobbley # And done! This is the first time we change the HEAD in the actual work tree. 1202eb8dc403SDave Cobbley runcmd("git reset --hard %s" % new_rev) 1203eb8dc403SDave Cobbley 1204eb8dc403SDave Cobbley # Update and stage the (potentially modified) 1205eb8dc403SDave Cobbley # combo-layer.conf, but do not commit separately. 1206eb8dc403SDave Cobbley for name in repos: 1207eb8dc403SDave Cobbley repo = conf.repos[name] 1208eb8dc403SDave Cobbley rev = repo['last_revision'] 1209eb8dc403SDave Cobbley conf.update(name, "last_revision", rev) 1210eb8dc403SDave Cobbley if commit_conf_file(conf, components, False): 1211eb8dc403SDave Cobbley # Must augment the previous commit. 1212eb8dc403SDave Cobbley runcmd("git commit --amend -C HEAD") 1213eb8dc403SDave Cobbley 1214eb8dc403SDave Cobbley 1215eb8dc403SDave Cobbleyscanned_revs = False 1216eb8dc403SDave Cobbleydef find_revs(old2new, head): 1217eb8dc403SDave Cobbley '''Construct mapping from original commit hash to commit hash in 1218eb8dc403SDave Cobbley combined repo by looking at the commit messages. Depends on the 1219eb8dc403SDave Cobbley "From ... rev: ..." convention.''' 1220eb8dc403SDave Cobbley logger.info("Analyzing log messages to find previously imported commits...") 1221eb8dc403SDave Cobbley num_known = len(old2new) 1222eb8dc403SDave Cobbley log = runcmd("git log --grep='From .* rev: [a-fA-F0-9][a-fA-F0-9]*' --pretty=format:%H%x00%B%x00 " + head).split(chr(0)) 1223eb8dc403SDave Cobbley regex = re.compile(r'From .* rev: ([a-fA-F0-9]+)') 1224eb8dc403SDave Cobbley for new_rev, body in zip(*[iter(log)]* 2): 1225eb8dc403SDave Cobbley # Use the last one, in the unlikely case there are more than one. 1226eb8dc403SDave Cobbley rev = regex.findall(body)[-1] 1227eb8dc403SDave Cobbley if rev not in old2new: 1228eb8dc403SDave Cobbley old2new[rev] = new_rev.strip() 1229eb8dc403SDave Cobbley logger.info("Found %d additional commits, leading to: %s" % (len(old2new) - num_known, old2new)) 1230eb8dc403SDave Cobbley 1231eb8dc403SDave Cobbley 1232eb8dc403SDave Cobbleydef apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=None): 1233eb8dc403SDave Cobbley '''Compare revision against parent, remove files deleted in the 1234eb8dc403SDave Cobbley commit, re-write new or modified ones. Moves them into dest_dir. 1235eb8dc403SDave Cobbley Optionally filters files. 1236eb8dc403SDave Cobbley ''' 1237eb8dc403SDave Cobbley if not dest_dir: 1238eb8dc403SDave Cobbley dest_dir = "." 1239eb8dc403SDave Cobbley # -r recurses into sub-directories, given is the full overview of 1240eb8dc403SDave Cobbley # what changed. We do not care about copy/edits or renames, so we 1241eb8dc403SDave Cobbley # can disable those with --no-renames (but we still parse them, 1242eb8dc403SDave Cobbley # because it was not clear from git documentation whether C and M 1243eb8dc403SDave Cobbley # lines can still occur). 1244eb8dc403SDave Cobbley logger.debug("Applying changes between %s and %s in %s" % (parent, rev, largs["destdir"])) 1245eb8dc403SDave Cobbley delete = [] 1246eb8dc403SDave Cobbley update = [] 1247eb8dc403SDave Cobbley if parent: 1248eb8dc403SDave Cobbley # Apply delta. 1249eb8dc403SDave Cobbley changes = runcmd("git diff-tree --no-commit-id --no-renames --name-status -r --raw -z %s %s" % (parent, rev), **largs).split(chr(0)) 1250eb8dc403SDave Cobbley for status, name in zip(*[iter(changes)]*2): 1251eb8dc403SDave Cobbley if status[0] in "ACMRT": 1252eb8dc403SDave Cobbley update.append(name) 1253eb8dc403SDave Cobbley elif status[0] in "D": 1254eb8dc403SDave Cobbley delete.append(name) 1255eb8dc403SDave Cobbley else: 1256eb8dc403SDave Cobbley logger.error("Unknown status %s of file %s in revision %s" % (status, name, rev)) 1257eb8dc403SDave Cobbley sys.exit(1) 1258eb8dc403SDave Cobbley else: 1259eb8dc403SDave Cobbley # Copy all files. 1260eb8dc403SDave Cobbley update.extend(runcmd("git ls-tree -r --name-only -z %s" % rev, **largs).split(chr(0))) 1261eb8dc403SDave Cobbley 1262eb8dc403SDave Cobbley # Include/exclude files as define in the component config. 1263eb8dc403SDave Cobbley # Both updated and deleted file lists get filtered, because it might happen 1264eb8dc403SDave Cobbley # that a file gets excluded, pulled from a different component, and then the 1265eb8dc403SDave Cobbley # excluded file gets deleted. In that case we must keep the copy. 1266eb8dc403SDave Cobbley if file_filter: 1267eb8dc403SDave Cobbley file_filter(update) 1268eb8dc403SDave Cobbley file_filter(delete) 1269eb8dc403SDave Cobbley 1270eb8dc403SDave Cobbley # We export into a tar archive here and extract with tar because it is simple (no 1271eb8dc403SDave Cobbley # need to implement file and symlink writing ourselves) and gives us some degree 1272eb8dc403SDave Cobbley # of parallel IO. The downside is that we have to pass the list of files via 1273eb8dc403SDave Cobbley # command line parameters - hopefully there will never be too many at once. 1274eb8dc403SDave Cobbley if update: 1275eb8dc403SDave Cobbley target = os.path.join(wargs["destdir"], dest_dir) 1276eb8dc403SDave Cobbley if not os.path.isdir(target): 1277eb8dc403SDave Cobbley os.makedirs(target) 1278fc113eadSAndrew Geissler quoted_target = shlex.quote(target) 1279eb8dc403SDave Cobbley # os.sysconf('SC_ARG_MAX') is lying: running a command with 1280eb8dc403SDave Cobbley # string length 629343 already failed with "Argument list too 1281eb8dc403SDave Cobbley # long" although SC_ARG_MAX = 2097152. "man execve" explains 1282eb8dc403SDave Cobbley # the limitations, but those are pretty complicated. So here 1283eb8dc403SDave Cobbley # we just hard-code a fixed value which is more likely to work. 1284eb8dc403SDave Cobbley max_cmdsize = 64 * 1024 1285eb8dc403SDave Cobbley while update: 1286eb8dc403SDave Cobbley quoted_args = [] 1287eb8dc403SDave Cobbley unquoted_args = [] 1288eb8dc403SDave Cobbley cmdsize = 100 + len(quoted_target) 1289eb8dc403SDave Cobbley while update: 1290fc113eadSAndrew Geissler quoted_next = shlex.quote(update[0]) 1291eb8dc403SDave Cobbley size_next = len(quoted_next) + len(dest_dir) + 1 1292eb8dc403SDave Cobbley logger.debug('cmdline length %d + %d < %d?' % (cmdsize, size_next, os.sysconf('SC_ARG_MAX'))) 1293eb8dc403SDave Cobbley if cmdsize + size_next < max_cmdsize: 1294eb8dc403SDave Cobbley quoted_args.append(quoted_next) 1295eb8dc403SDave Cobbley unquoted_args.append(update.pop(0)) 1296eb8dc403SDave Cobbley cmdsize += size_next 1297eb8dc403SDave Cobbley else: 1298eb8dc403SDave Cobbley logger.debug('Breaking the cmdline at length %d' % cmdsize) 1299eb8dc403SDave Cobbley break 1300eb8dc403SDave Cobbley logger.debug('Final cmdline length %d / %d' % (cmdsize, os.sysconf('SC_ARG_MAX'))) 1301eb8dc403SDave Cobbley cmd = "git archive %s %s | tar -C %s -xf -" % (rev, ' '.join(quoted_args), quoted_target) 1302eb8dc403SDave Cobbley logger.debug('First cmdline length %d' % len(cmd)) 1303eb8dc403SDave Cobbley runcmd(cmd, **largs) 1304eb8dc403SDave Cobbley cmd = "git add -f".split() + [os.path.join(dest_dir, x) for x in unquoted_args] 1305eb8dc403SDave Cobbley logger.debug('Second cmdline length %d' % reduce(lambda x, y: x + len(y), cmd, 0)) 1306eb8dc403SDave Cobbley runcmd(cmd, **wargs) 1307eb8dc403SDave Cobbley if delete: 1308eb8dc403SDave Cobbley for path in delete: 1309eb8dc403SDave Cobbley if dest_dir: 1310eb8dc403SDave Cobbley path = os.path.join(dest_dir, path) 1311eb8dc403SDave Cobbley runcmd("git rm -f --ignore-unmatch".split() + [os.path.join(dest_dir, x) for x in delete], **wargs) 1312eb8dc403SDave Cobbley 1313eb8dc403SDave Cobbleydef action_error(conf, args): 1314eb8dc403SDave Cobbley logger.info("invalid action %s" % args[0]) 1315eb8dc403SDave Cobbley 1316eb8dc403SDave Cobbleyactions = { 1317eb8dc403SDave Cobbley "init": action_init, 1318eb8dc403SDave Cobbley "update": action_update, 1319eb8dc403SDave Cobbley "pull": action_pull, 1320eb8dc403SDave Cobbley "splitpatch": action_splitpatch, 1321517393d9SAndrew Geissler "sync-revs": action_sync_revs, 1322eb8dc403SDave Cobbley} 1323eb8dc403SDave Cobbley 1324eb8dc403SDave Cobbleydef main(): 1325eb8dc403SDave Cobbley parser = optparse.OptionParser( 1326eb8dc403SDave Cobbley version = "Combo Layer Repo Tool version %s" % __version__, 1327eb8dc403SDave Cobbley usage = """%prog [options] action 1328eb8dc403SDave Cobbley 1329eb8dc403SDave CobbleyCreate and update a combination layer repository from multiple component repositories. 1330eb8dc403SDave Cobbley 1331eb8dc403SDave CobbleyAction: 1332eb8dc403SDave Cobbley init initialise the combo layer repo 1333eb8dc403SDave Cobbley update [components] get patches from component repos and apply them to the combo repo 1334eb8dc403SDave Cobbley pull [components] just pull component repos only 1335517393d9SAndrew Geissler sync-revs [components] update the config file's last_revision for each repository 1336eb8dc403SDave Cobbley splitpatch [commit] generate commit patch and split per component, default commit is HEAD""") 1337eb8dc403SDave Cobbley 1338eb8dc403SDave Cobbley parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).", 1339eb8dc403SDave Cobbley action = "store", dest = "conffile", default = "conf/combo-layer.conf") 1340eb8dc403SDave Cobbley 1341eb8dc403SDave Cobbley parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches", 1342eb8dc403SDave Cobbley action = "store_true", dest = "interactive", default = False) 1343eb8dc403SDave Cobbley 1344eb8dc403SDave Cobbley parser.add_option("-D", "--debug", help = "output debug information", 1345eb8dc403SDave Cobbley action = "store_true", dest = "debug", default = False) 1346eb8dc403SDave Cobbley 1347eb8dc403SDave Cobbley parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update", 1348eb8dc403SDave Cobbley action = "store_true", dest = "nopull", default = False) 1349eb8dc403SDave Cobbley 1350eb8dc403SDave Cobbley parser.add_option("--hard-reset", 1351eb8dc403SDave Cobbley help = "instead of pull do fetch and hard-reset in component repos", 1352eb8dc403SDave Cobbley action = "store_true", dest = "hard_reset", default = False) 1353eb8dc403SDave Cobbley 1354eb8dc403SDave Cobbley parser.add_option("-H", "--history", help = "import full history of components during init", 1355eb8dc403SDave Cobbley action = "store_true", default = False) 1356eb8dc403SDave Cobbley 1357eb8dc403SDave Cobbley options, args = parser.parse_args(sys.argv) 1358eb8dc403SDave Cobbley 1359eb8dc403SDave Cobbley # Dispatch to action handler 1360eb8dc403SDave Cobbley if len(args) == 1: 1361eb8dc403SDave Cobbley logger.error("No action specified, exiting") 1362eb8dc403SDave Cobbley parser.print_help() 1363eb8dc403SDave Cobbley elif args[1] not in actions: 1364eb8dc403SDave Cobbley logger.error("Unsupported action %s, exiting\n" % (args[1])) 1365eb8dc403SDave Cobbley parser.print_help() 1366eb8dc403SDave Cobbley elif not os.path.exists(options.conffile): 1367eb8dc403SDave Cobbley logger.error("No valid config file, exiting\n") 1368eb8dc403SDave Cobbley parser.print_help() 1369eb8dc403SDave Cobbley else: 1370eb8dc403SDave Cobbley if options.debug: 1371eb8dc403SDave Cobbley logger.setLevel(logging.DEBUG) 1372eb8dc403SDave Cobbley confdata = Configuration(options) 1373eb8dc403SDave Cobbley initmode = (args[1] == 'init') 1374eb8dc403SDave Cobbley confdata.sanity_check(initmode) 1375eb8dc403SDave Cobbley actions.get(args[1], action_error)(confdata, args[1:]) 1376eb8dc403SDave Cobbley 1377eb8dc403SDave Cobbleyif __name__ == "__main__": 1378eb8dc403SDave Cobbley try: 1379eb8dc403SDave Cobbley ret = main() 1380eb8dc403SDave Cobbley except Exception: 1381eb8dc403SDave Cobbley ret = 1 1382eb8dc403SDave Cobbley import traceback 1383eb8dc403SDave Cobbley traceback.print_exc() 1384eb8dc403SDave Cobbley sys.exit(ret) 1385