1eb8dc403SDave Cobbley#!/usr/bin/env python3 2eb8dc403SDave Cobbley# ex:ts=4:sw=4:sts=4:et 3eb8dc403SDave Cobbley# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 4eb8dc403SDave Cobbley# 5eb8dc403SDave Cobbley# Copyright 2011 Intel Corporation 6eb8dc403SDave Cobbley# Authored-by: Yu Ke <ke.yu@intel.com> 7eb8dc403SDave Cobbley# Paul Eggleton <paul.eggleton@intel.com> 8eb8dc403SDave Cobbley# Richard Purdie <richard.purdie@intel.com> 9eb8dc403SDave Cobbley# 10c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 11eb8dc403SDave Cobbley# 12eb8dc403SDave Cobbley 13eb8dc403SDave Cobbleyimport fnmatch 14eb8dc403SDave Cobbleyimport os, sys 15eb8dc403SDave Cobbleyimport optparse 16eb8dc403SDave Cobbleyimport logging 17eb8dc403SDave Cobbleyimport subprocess 18eb8dc403SDave Cobbleyimport tempfile 19eb8dc403SDave Cobbleyimport configparser 20eb8dc403SDave Cobbleyimport re 21eb8dc403SDave Cobbleyimport copy 22eb8dc403SDave Cobbleyimport pipes 23eb8dc403SDave Cobbleyimport shutil 24eb8dc403SDave Cobbleyfrom collections import OrderedDict 25eb8dc403SDave Cobbleyfrom string import Template 26eb8dc403SDave Cobbleyfrom functools import reduce 27eb8dc403SDave Cobbley 28eb8dc403SDave Cobbley__version__ = "0.2.1" 29eb8dc403SDave Cobbley 30eb8dc403SDave Cobbleydef logger_create(): 31eb8dc403SDave Cobbley logger = logging.getLogger("") 32eb8dc403SDave Cobbley loggerhandler = logging.StreamHandler() 33eb8dc403SDave Cobbley loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S")) 34eb8dc403SDave Cobbley logger.addHandler(loggerhandler) 35eb8dc403SDave Cobbley logger.setLevel(logging.INFO) 36eb8dc403SDave Cobbley return logger 37eb8dc403SDave Cobbley 38eb8dc403SDave Cobbleylogger = logger_create() 39eb8dc403SDave Cobbley 40eb8dc403SDave Cobbleydef get_current_branch(repodir=None): 41eb8dc403SDave Cobbley try: 42eb8dc403SDave Cobbley if not os.path.exists(os.path.join(repodir if repodir else '', ".git")): 43eb8dc403SDave Cobbley # Repo not created yet (i.e. during init) so just assume master 44eb8dc403SDave Cobbley return "master" 45eb8dc403SDave Cobbley branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip() 46eb8dc403SDave Cobbley if branchname.startswith("refs/heads/"): 47eb8dc403SDave Cobbley branchname = branchname[11:] 48eb8dc403SDave Cobbley return branchname 49eb8dc403SDave Cobbley except subprocess.CalledProcessError: 50eb8dc403SDave Cobbley return "" 51eb8dc403SDave Cobbley 52eb8dc403SDave Cobbleyclass Configuration(object): 53eb8dc403SDave Cobbley """ 54eb8dc403SDave Cobbley Manages the configuration 55eb8dc403SDave Cobbley 56eb8dc403SDave Cobbley For an example config file, see combo-layer.conf.example 57eb8dc403SDave Cobbley 58eb8dc403SDave Cobbley """ 59eb8dc403SDave Cobbley def __init__(self, options): 60eb8dc403SDave Cobbley for key, val in options.__dict__.items(): 61eb8dc403SDave Cobbley setattr(self, key, val) 62eb8dc403SDave Cobbley 63eb8dc403SDave Cobbley def readsection(parser, section, repo): 64eb8dc403SDave Cobbley for (name, value) in parser.items(section): 65eb8dc403SDave Cobbley if value.startswith("@"): 66eb8dc403SDave Cobbley self.repos[repo][name] = eval(value.strip("@")) 67eb8dc403SDave Cobbley else: 68eb8dc403SDave Cobbley # Apply special type transformations for some properties. 69eb8dc403SDave Cobbley # Type matches the RawConfigParser.get*() methods. 70eb8dc403SDave Cobbley types = {'signoff': 'boolean', 'update': 'boolean', 'history': 'boolean'} 71eb8dc403SDave Cobbley if name in types: 72eb8dc403SDave Cobbley value = getattr(parser, 'get' + types[name])(section, name) 73eb8dc403SDave Cobbley self.repos[repo][name] = value 74eb8dc403SDave Cobbley 75eb8dc403SDave Cobbley def readglobalsection(parser, section): 76eb8dc403SDave Cobbley for (name, value) in parser.items(section): 77eb8dc403SDave Cobbley if name == "commit_msg": 78eb8dc403SDave Cobbley self.commit_msg_template = value 79eb8dc403SDave Cobbley 80eb8dc403SDave Cobbley logger.debug("Loading config file %s" % self.conffile) 81eb8dc403SDave Cobbley self.parser = configparser.ConfigParser() 82eb8dc403SDave Cobbley with open(self.conffile) as f: 83*82c905dcSAndrew Geissler self.parser.read_file(f) 84eb8dc403SDave Cobbley 85eb8dc403SDave Cobbley # initialize default values 86eb8dc403SDave Cobbley self.commit_msg_template = "Automatic commit to update last_revision" 87eb8dc403SDave Cobbley 88eb8dc403SDave Cobbley self.repos = {} 89eb8dc403SDave Cobbley for repo in self.parser.sections(): 90eb8dc403SDave Cobbley if repo == "combo-layer-settings": 91eb8dc403SDave Cobbley # special handling for global settings 92eb8dc403SDave Cobbley readglobalsection(self.parser, repo) 93eb8dc403SDave Cobbley else: 94eb8dc403SDave Cobbley self.repos[repo] = {} 95eb8dc403SDave Cobbley readsection(self.parser, repo, repo) 96eb8dc403SDave Cobbley 97eb8dc403SDave Cobbley # Load local configuration, if available 98eb8dc403SDave Cobbley self.localconffile = None 99eb8dc403SDave Cobbley self.localparser = None 100eb8dc403SDave Cobbley self.combobranch = None 101eb8dc403SDave Cobbley if self.conffile.endswith('.conf'): 102eb8dc403SDave Cobbley lcfile = self.conffile.replace('.conf', '-local.conf') 103eb8dc403SDave Cobbley if os.path.exists(lcfile): 104eb8dc403SDave Cobbley # Read combo layer branch 105eb8dc403SDave Cobbley self.combobranch = get_current_branch() 106eb8dc403SDave Cobbley logger.debug("Combo layer branch is %s" % self.combobranch) 107eb8dc403SDave Cobbley 108eb8dc403SDave Cobbley self.localconffile = lcfile 109eb8dc403SDave Cobbley logger.debug("Loading local config file %s" % self.localconffile) 110eb8dc403SDave Cobbley self.localparser = configparser.ConfigParser() 111eb8dc403SDave Cobbley with open(self.localconffile) as f: 112eb8dc403SDave Cobbley self.localparser.readfp(f) 113eb8dc403SDave Cobbley 114eb8dc403SDave Cobbley for section in self.localparser.sections(): 115eb8dc403SDave Cobbley if '|' in section: 116eb8dc403SDave Cobbley sectionvals = section.split('|') 117eb8dc403SDave Cobbley repo = sectionvals[0] 118eb8dc403SDave Cobbley if sectionvals[1] != self.combobranch: 119eb8dc403SDave Cobbley continue 120eb8dc403SDave Cobbley else: 121eb8dc403SDave Cobbley repo = section 122eb8dc403SDave Cobbley if repo in self.repos: 123eb8dc403SDave Cobbley readsection(self.localparser, section, repo) 124eb8dc403SDave Cobbley 125eb8dc403SDave Cobbley def update(self, repo, option, value, initmode=False): 126eb8dc403SDave Cobbley # If the main config has the option already, that is what we 127eb8dc403SDave Cobbley # are expected to modify. 128eb8dc403SDave Cobbley if self.localparser and not self.parser.has_option(repo, option): 129eb8dc403SDave Cobbley parser = self.localparser 130eb8dc403SDave Cobbley section = "%s|%s" % (repo, self.combobranch) 131eb8dc403SDave Cobbley conffile = self.localconffile 132eb8dc403SDave Cobbley if initmode and not parser.has_section(section): 133eb8dc403SDave Cobbley parser.add_section(section) 134eb8dc403SDave Cobbley else: 135eb8dc403SDave Cobbley parser = self.parser 136eb8dc403SDave Cobbley section = repo 137eb8dc403SDave Cobbley conffile = self.conffile 138eb8dc403SDave Cobbley parser.set(section, option, value) 139eb8dc403SDave Cobbley with open(conffile, "w") as f: 140eb8dc403SDave Cobbley parser.write(f) 141eb8dc403SDave Cobbley self.repos[repo][option] = value 142eb8dc403SDave Cobbley 143eb8dc403SDave Cobbley def sanity_check(self, initmode=False): 144eb8dc403SDave Cobbley required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"] 145eb8dc403SDave Cobbley if initmode: 146eb8dc403SDave Cobbley required_options.remove("last_revision") 147eb8dc403SDave Cobbley msg = "" 148eb8dc403SDave Cobbley missing_options = [] 149eb8dc403SDave Cobbley for name in self.repos: 150eb8dc403SDave Cobbley for option in required_options: 151eb8dc403SDave Cobbley if option not in self.repos[name]: 152eb8dc403SDave Cobbley msg = "%s\nOption %s is not defined for component %s" %(msg, option, name) 153eb8dc403SDave Cobbley missing_options.append(option) 154eb8dc403SDave Cobbley # Sanitize dest_dir so that we do not have to deal with edge cases 155eb8dc403SDave Cobbley # (unset, empty string, double slashes) in the rest of the code. 156eb8dc403SDave Cobbley # It not being set will still be flagged as error because it is 157eb8dc403SDave Cobbley # listed as required option above; that could be changed now. 158eb8dc403SDave Cobbley dest_dir = os.path.normpath(self.repos[name].get("dest_dir", ".")) 159eb8dc403SDave Cobbley self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir 160eb8dc403SDave Cobbley if msg != "": 161eb8dc403SDave Cobbley logger.error("configuration file %s has the following error: %s" % (self.conffile,msg)) 162eb8dc403SDave Cobbley if self.localconffile and 'last_revision' in missing_options: 163eb8dc403SDave Cobbley logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch)) 164eb8dc403SDave Cobbley sys.exit(1) 165eb8dc403SDave Cobbley 166eb8dc403SDave Cobbley # filterdiff is required by action_splitpatch, so check its availability 167eb8dc403SDave Cobbley if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0: 168eb8dc403SDave Cobbley logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)") 169eb8dc403SDave Cobbley sys.exit(1) 170eb8dc403SDave Cobbley 171eb8dc403SDave Cobbleydef runcmd(cmd,destdir=None,printerr=True,out=None,env=None): 172eb8dc403SDave Cobbley """ 173eb8dc403SDave Cobbley execute command, raise CalledProcessError if fail 174eb8dc403SDave Cobbley return output if succeed 175eb8dc403SDave Cobbley """ 176eb8dc403SDave Cobbley logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir)) 177eb8dc403SDave Cobbley if not out: 178eb8dc403SDave Cobbley out = tempfile.TemporaryFile() 179eb8dc403SDave Cobbley err = out 180eb8dc403SDave Cobbley else: 181eb8dc403SDave Cobbley err = tempfile.TemporaryFile() 182eb8dc403SDave Cobbley try: 183eb8dc403SDave Cobbley subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str), env=env or os.environ) 184eb8dc403SDave Cobbley except subprocess.CalledProcessError as e: 185eb8dc403SDave Cobbley err.seek(0) 186eb8dc403SDave Cobbley if printerr: 187eb8dc403SDave Cobbley logger.error("%s" % err.read()) 188eb8dc403SDave Cobbley raise e 189eb8dc403SDave Cobbley 190eb8dc403SDave Cobbley err.seek(0) 191eb8dc403SDave Cobbley output = err.read().decode('utf-8') 192eb8dc403SDave Cobbley logger.debug("output: %s" % output.replace(chr(0), '\\0')) 193eb8dc403SDave Cobbley return output 194eb8dc403SDave Cobbley 195eb8dc403SDave Cobbleydef action_init(conf, args): 196eb8dc403SDave Cobbley """ 197eb8dc403SDave Cobbley Clone component repositories 198eb8dc403SDave Cobbley Check git is initialised; if not, copy initial data from component repos 199eb8dc403SDave Cobbley """ 200eb8dc403SDave Cobbley for name in conf.repos: 201eb8dc403SDave Cobbley ldir = conf.repos[name]['local_repo_dir'] 202eb8dc403SDave Cobbley if not os.path.exists(ldir): 203eb8dc403SDave Cobbley logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir)) 204eb8dc403SDave Cobbley subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True) 205eb8dc403SDave Cobbley if not os.path.exists(".git"): 206eb8dc403SDave Cobbley runcmd("git init") 207eb8dc403SDave Cobbley if conf.history: 208eb8dc403SDave Cobbley # Need a common ref for all trees. 209eb8dc403SDave Cobbley runcmd('git commit -m "initial empty commit" --allow-empty') 210eb8dc403SDave Cobbley startrev = runcmd('git rev-parse master').strip() 211eb8dc403SDave Cobbley 212eb8dc403SDave Cobbley for name in conf.repos: 213eb8dc403SDave Cobbley repo = conf.repos[name] 214eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 215eb8dc403SDave Cobbley branch = repo.get('branch', "master") 216eb8dc403SDave Cobbley lastrev = repo.get('last_revision', None) 217eb8dc403SDave Cobbley if lastrev and lastrev != "HEAD": 218eb8dc403SDave Cobbley initialrev = lastrev 219eb8dc403SDave Cobbley if branch: 220eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, lastrev, branch): 221eb8dc403SDave Cobbley sys.exit(1) 222eb8dc403SDave Cobbley logger.info("Copying data from %s at specified revision %s..." % (name, lastrev)) 223eb8dc403SDave Cobbley else: 224eb8dc403SDave Cobbley lastrev = None 225eb8dc403SDave Cobbley initialrev = branch 226eb8dc403SDave Cobbley logger.info("Copying data from %s..." % name) 227eb8dc403SDave Cobbley # Sanity check initialrev and turn it into hash (required for copying history, 228eb8dc403SDave Cobbley # because resolving a name ref only works in the component repo). 229eb8dc403SDave Cobbley rev = runcmd('git rev-parse %s' % initialrev, ldir).strip() 230eb8dc403SDave Cobbley if rev != initialrev: 231eb8dc403SDave Cobbley try: 232eb8dc403SDave Cobbley refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n') 233eb8dc403SDave Cobbley if len(set(refs)) > 1: 234eb8dc403SDave Cobbley # Happens for example when configured to track 235eb8dc403SDave Cobbley # "master" and there is a refs/heads/master. The 236eb8dc403SDave Cobbley # traditional behavior from "git archive" (preserved 237eb8dc403SDave Cobbley # here) it to choose the first one. This might not be 238eb8dc403SDave Cobbley # intended, so at least warn about it. 2391a4b7ee2SBrad Bishop logger.warning("%s: initial revision '%s' not unique, picking result of rev-parse = %s" % 240eb8dc403SDave Cobbley (name, initialrev, refs[0])) 241eb8dc403SDave Cobbley initialrev = rev 242eb8dc403SDave Cobbley except: 243eb8dc403SDave Cobbley # show-ref fails for hashes. Skip the sanity warning in that case. 244eb8dc403SDave Cobbley pass 245eb8dc403SDave Cobbley initialrev = rev 246eb8dc403SDave Cobbley dest_dir = repo['dest_dir'] 247eb8dc403SDave Cobbley if dest_dir != ".": 248eb8dc403SDave Cobbley extract_dir = os.path.join(os.getcwd(), dest_dir) 249eb8dc403SDave Cobbley if not os.path.exists(extract_dir): 250eb8dc403SDave Cobbley os.makedirs(extract_dir) 251eb8dc403SDave Cobbley else: 252eb8dc403SDave Cobbley extract_dir = os.getcwd() 253eb8dc403SDave Cobbley file_filter = repo.get('file_filter', "") 254eb8dc403SDave Cobbley exclude_patterns = repo.get('file_exclude', '').split() 255eb8dc403SDave Cobbley def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir, 256eb8dc403SDave Cobbley subdir=""): 257eb8dc403SDave Cobbley # When working inside a filtered branch which had the 258eb8dc403SDave Cobbley # files already moved, we need to prepend the 259eb8dc403SDave Cobbley # subdirectory to all filters, otherwise they would 260eb8dc403SDave Cobbley # not match. 261eb8dc403SDave Cobbley if subdir == '.': 262eb8dc403SDave Cobbley subdir = '' 263eb8dc403SDave Cobbley elif subdir: 264eb8dc403SDave Cobbley subdir = os.path.normpath(subdir) 265eb8dc403SDave Cobbley file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()]) 266eb8dc403SDave Cobbley exclude_patterns = [subdir + '/' + x for x in exclude_patterns] 267eb8dc403SDave Cobbley # To handle both cases, we cd into the target 268eb8dc403SDave Cobbley # directory and optionally tell tar to strip the path 269eb8dc403SDave Cobbley # prefix when the files were already moved. 270eb8dc403SDave Cobbley subdir_components = len(subdir.split(os.path.sep)) if subdir else 0 271eb8dc403SDave Cobbley strip=('--strip-components=%d' % subdir_components) if subdir else '' 272eb8dc403SDave Cobbley # TODO: file_filter wild cards do not work (and haven't worked before either), because 273eb8dc403SDave Cobbley # a) GNU tar requires a --wildcards parameter before turning on wild card matching. 274eb8dc403SDave Cobbley # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c, 275eb8dc403SDave Cobbley # in contrast to the other use of file_filter as parameter of "git archive" 276eb8dc403SDave Cobbley # where it only matches .c files directly in src). 277eb8dc403SDave Cobbley files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" % 278eb8dc403SDave Cobbley (initialrev, subdir, 279eb8dc403SDave Cobbley strip, extract_dir, file_filter), 280eb8dc403SDave Cobbley ldir) 281eb8dc403SDave Cobbley if exclude_patterns: 282eb8dc403SDave Cobbley # Implement file removal by letting tar create the 283eb8dc403SDave Cobbley # file and then deleting it in the file system 284eb8dc403SDave Cobbley # again. Uses the list of files created by tar (easier 285eb8dc403SDave Cobbley # than walking the tree). 286eb8dc403SDave Cobbley for file in files.split('\n'): 287eb8dc403SDave Cobbley if file.endswith(os.path.sep): 288eb8dc403SDave Cobbley continue 289eb8dc403SDave Cobbley for pattern in exclude_patterns: 290eb8dc403SDave Cobbley if fnmatch.fnmatch(file, pattern): 291eb8dc403SDave Cobbley os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file]))) 292eb8dc403SDave Cobbley break 293eb8dc403SDave Cobbley 294eb8dc403SDave Cobbley if not conf.history: 295eb8dc403SDave Cobbley copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir) 296eb8dc403SDave Cobbley else: 297eb8dc403SDave Cobbley # First fetch remote history into local repository. 298eb8dc403SDave Cobbley # We need a ref for that, so ensure that there is one. 299eb8dc403SDave Cobbley refname = "combo-layer-init-%s" % name 300eb8dc403SDave Cobbley runcmd("git branch -f %s %s" % (refname, initialrev), ldir) 301eb8dc403SDave Cobbley runcmd("git fetch %s %s" % (ldir, refname)) 302eb8dc403SDave Cobbley runcmd("git branch -D %s" % refname, ldir) 303eb8dc403SDave Cobbley # Make that the head revision. 304eb8dc403SDave Cobbley runcmd("git checkout -b %s %s" % (name, initialrev)) 305eb8dc403SDave Cobbley # Optional: cut the history by replacing the given 306eb8dc403SDave Cobbley # start point(s) with commits providing the same 307eb8dc403SDave Cobbley # content (aka tree), but with commit information that 308eb8dc403SDave Cobbley # makes it clear that this is an artifically created 309eb8dc403SDave Cobbley # commit and nothing the original authors had anything 310eb8dc403SDave Cobbley # to do with. 311eb8dc403SDave Cobbley since_rev = repo.get('since_revision', '') 312eb8dc403SDave Cobbley if since_rev: 313eb8dc403SDave Cobbley committer = runcmd('git var GIT_AUTHOR_IDENT').strip() 314eb8dc403SDave Cobbley # Same time stamp, no name. 315eb8dc403SDave Cobbley author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer) 316eb8dc403SDave Cobbley logger.info('author %s' % author) 317eb8dc403SDave Cobbley for rev in since_rev.split(): 318eb8dc403SDave Cobbley # Resolve in component repo... 319eb8dc403SDave Cobbley rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0] 320eb8dc403SDave Cobbley # ... and then get the tree in current 321eb8dc403SDave Cobbley # one. The commit should be in both repos with 322eb8dc403SDave Cobbley # the same tree, but better check here. 323eb8dc403SDave Cobbley tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip() 324eb8dc403SDave Cobbley with tempfile.NamedTemporaryFile(mode='wt') as editor: 325eb8dc403SDave Cobbley editor.write('''cat >$1 <<EOF 326eb8dc403SDave Cobbleytree %s 327eb8dc403SDave Cobbleyauthor %s 328eb8dc403SDave Cobbleycommitter %s 329eb8dc403SDave Cobbley 330eb8dc403SDave Cobbley%s: squashed import of component 331eb8dc403SDave Cobbley 332eb8dc403SDave CobbleyThis commit copies the entire set of files as found in 333eb8dc403SDave Cobbley%s %s 334eb8dc403SDave Cobbley 335eb8dc403SDave CobbleyFor more information about previous commits, see the 336eb8dc403SDave Cobbleyupstream repository. 337eb8dc403SDave Cobbley 338eb8dc403SDave CobbleyCommit created by combo-layer. 339eb8dc403SDave CobbleyEOF 340eb8dc403SDave Cobbley''' % (tree, author, committer, name, name, since_rev)) 341eb8dc403SDave Cobbley editor.flush() 342eb8dc403SDave Cobbley os.environ['GIT_EDITOR'] = 'sh %s' % editor.name 343eb8dc403SDave Cobbley runcmd('git replace --edit %s' % rev) 344eb8dc403SDave Cobbley 345eb8dc403SDave Cobbley # Optional: rewrite history to change commit messages or to move files. 346eb8dc403SDave Cobbley if 'hook' in repo or dest_dir != ".": 347eb8dc403SDave Cobbley filter_branch = ['git', 'filter-branch', '--force'] 348eb8dc403SDave Cobbley with tempfile.NamedTemporaryFile(mode='wt') as hookwrapper: 349eb8dc403SDave Cobbley if 'hook' in repo: 350eb8dc403SDave Cobbley # Create a shell script wrapper around the original hook that 351eb8dc403SDave Cobbley # can be used by git filter-branch. Hook may or may not have 352eb8dc403SDave Cobbley # an absolute path. 353eb8dc403SDave Cobbley hook = repo['hook'] 354eb8dc403SDave Cobbley hook = os.path.join(os.path.dirname(conf.conffile), '..', hook) 355eb8dc403SDave Cobbley # The wrappers turns the commit message 356eb8dc403SDave Cobbley # from stdin into a fake patch header. 357eb8dc403SDave Cobbley # This is good enough for changing Subject 358eb8dc403SDave Cobbley # and commit msg body with normal 359eb8dc403SDave Cobbley # combo-layer hooks. 360eb8dc403SDave Cobbley hookwrapper.write('''set -e 361eb8dc403SDave Cobbleytmpname=$(mktemp) 362eb8dc403SDave Cobbleytrap "rm $tmpname" EXIT 363eb8dc403SDave Cobbleyecho -n 'Subject: [PATCH] ' >>$tmpname 364eb8dc403SDave Cobbleycat >>$tmpname 365eb8dc403SDave Cobbleyif ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then 366eb8dc403SDave Cobbley echo >>$tmpname 367eb8dc403SDave Cobbleyfi 368eb8dc403SDave Cobbleyecho '---' >>$tmpname 369eb8dc403SDave Cobbley%s $tmpname $GIT_COMMIT %s 370eb8dc403SDave Cobbleytail -c +18 $tmpname | head -c -4 371eb8dc403SDave Cobbley''' % (hook, name)) 372eb8dc403SDave Cobbley hookwrapper.flush() 373eb8dc403SDave Cobbley filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name]) 374eb8dc403SDave Cobbley if dest_dir != ".": 375eb8dc403SDave Cobbley parent = os.path.dirname(dest_dir) 376eb8dc403SDave Cobbley if not parent: 377eb8dc403SDave Cobbley parent = '.' 378eb8dc403SDave Cobbley # May run outside of the current directory, so do not assume that .git exists. 379eb8dc403SDave Cobbley filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)]) 380eb8dc403SDave Cobbley filter_branch.append('HEAD') 381eb8dc403SDave Cobbley runcmd(filter_branch) 382eb8dc403SDave Cobbley runcmd('git update-ref -d refs/original/refs/heads/%s' % name) 383eb8dc403SDave Cobbley repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip() 384eb8dc403SDave Cobbley repo['stripped_revision'] = repo['rewritten_revision'] 385eb8dc403SDave Cobbley # Optional filter files: remove everything and re-populate using the normal filtering code. 386eb8dc403SDave Cobbley # Override any potential .gitignore. 387eb8dc403SDave Cobbley if file_filter or exclude_patterns: 388eb8dc403SDave Cobbley runcmd('git rm -rf .') 389eb8dc403SDave Cobbley if not os.path.exists(extract_dir): 390eb8dc403SDave Cobbley os.makedirs(extract_dir) 391eb8dc403SDave Cobbley copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.', 392eb8dc403SDave Cobbley subdir=dest_dir) 393eb8dc403SDave Cobbley runcmd('git add --all --force .') 394eb8dc403SDave Cobbley if runcmd('git status --porcelain'): 395eb8dc403SDave Cobbley # Something to commit. 396eb8dc403SDave Cobbley runcmd(['git', 'commit', '-m', 397eb8dc403SDave Cobbley '''%s: select file subset 398eb8dc403SDave Cobbley 399eb8dc403SDave CobbleyFiles from the component repository were chosen based on 400eb8dc403SDave Cobbleythe following filters: 401eb8dc403SDave Cobbleyfile_filter = %s 402eb8dc403SDave Cobbleyfile_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))]) 403eb8dc403SDave Cobbley repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip() 404eb8dc403SDave Cobbley 405eb8dc403SDave Cobbley if not lastrev: 406eb8dc403SDave Cobbley lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip() 407eb8dc403SDave Cobbley conf.update(name, "last_revision", lastrev, initmode=True) 408eb8dc403SDave Cobbley 409eb8dc403SDave Cobbley if not conf.history: 410eb8dc403SDave Cobbley runcmd("git add .") 411eb8dc403SDave Cobbley else: 412eb8dc403SDave Cobbley # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies 413eb8dc403SDave Cobbley runcmd('git checkout master') 414eb8dc403SDave Cobbley merge = ['git', 'merge', '--no-commit'] 415eb8dc403SDave Cobbley for name in conf.repos: 416eb8dc403SDave Cobbley repo = conf.repos[name] 417eb8dc403SDave Cobbley # Use branch created earlier. 418eb8dc403SDave Cobbley merge.append(name) 419eb8dc403SDave Cobbley # Root all commits which have no parent in the common 420eb8dc403SDave Cobbley # ancestor in the new repository. 421eb8dc403SDave Cobbley for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s --' % name).split('\n'): 422eb8dc403SDave Cobbley runcmd('git replace --graft %s %s' % (start, startrev)) 423eb8dc403SDave Cobbley try: 424eb8dc403SDave Cobbley runcmd(merge) 425eb8dc403SDave Cobbley except Exception as error: 426eb8dc403SDave Cobbley logger.info('''Merging component repository history failed, perhaps because of merge conflicts. 427eb8dc403SDave CobbleyIt may be possible to commit anyway after resolving these conflicts. 428eb8dc403SDave Cobbley 429eb8dc403SDave Cobbley%s''' % error) 430eb8dc403SDave Cobbley # Create MERGE_HEAD and MERGE_MSG. "git merge" itself 431eb8dc403SDave Cobbley # does not create MERGE_HEAD in case of a (harmless) failure, 432eb8dc403SDave Cobbley # and we want certain auto-generated information in the 433eb8dc403SDave Cobbley # commit message for future reference and/or automation. 434eb8dc403SDave Cobbley with open('.git/MERGE_HEAD', 'w') as head: 435eb8dc403SDave Cobbley with open('.git/MERGE_MSG', 'w') as msg: 436eb8dc403SDave Cobbley msg.write('repo: initial import of components\n\n') 437eb8dc403SDave Cobbley # head.write('%s\n' % startrev) 438eb8dc403SDave Cobbley for name in conf.repos: 439eb8dc403SDave Cobbley repo = conf.repos[name] 440eb8dc403SDave Cobbley # <upstream ref> <rewritten ref> <rewritten + files removed> 441eb8dc403SDave Cobbley msg.write('combo-layer-%s: %s %s %s\n' % (name, 442eb8dc403SDave Cobbley repo['last_revision'], 443eb8dc403SDave Cobbley repo['rewritten_revision'], 444eb8dc403SDave Cobbley repo['stripped_revision'])) 445eb8dc403SDave Cobbley rev = runcmd('git rev-parse %s' % name).strip() 446eb8dc403SDave Cobbley head.write('%s\n' % rev) 447eb8dc403SDave Cobbley 448eb8dc403SDave Cobbley if conf.localconffile: 449eb8dc403SDave Cobbley localadded = True 450eb8dc403SDave Cobbley try: 451eb8dc403SDave Cobbley runcmd("git rm --cached %s" % conf.localconffile, printerr=False) 452eb8dc403SDave Cobbley except subprocess.CalledProcessError: 453eb8dc403SDave Cobbley localadded = False 454eb8dc403SDave Cobbley if localadded: 455eb8dc403SDave Cobbley localrelpath = os.path.relpath(conf.localconffile) 456eb8dc403SDave Cobbley runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath)) 457eb8dc403SDave Cobbley runcmd("git add .gitignore") 458eb8dc403SDave Cobbley logger.info("Added local configuration file %s to .gitignore", localrelpath) 459eb8dc403SDave Cobbley logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.") 460eb8dc403SDave Cobbley else: 461eb8dc403SDave Cobbley logger.info("Repository already initialised, nothing to do.") 462eb8dc403SDave Cobbley 463eb8dc403SDave Cobbley 464eb8dc403SDave Cobbleydef check_repo_clean(repodir): 465eb8dc403SDave Cobbley """ 466eb8dc403SDave Cobbley check if the repo is clean 467eb8dc403SDave Cobbley exit if repo is dirty 468eb8dc403SDave Cobbley """ 469eb8dc403SDave Cobbley output=runcmd("git status --porcelain", repodir) 470eb8dc403SDave Cobbley r = re.compile('\?\? patch-.*/') 471eb8dc403SDave Cobbley dirtyout = [item for item in output.splitlines() if not r.match(item)] 472eb8dc403SDave Cobbley if dirtyout: 473eb8dc403SDave Cobbley logger.error("git repo %s is dirty, please fix it first", repodir) 474eb8dc403SDave Cobbley sys.exit(1) 475eb8dc403SDave Cobbley 476eb8dc403SDave Cobbleydef check_patch(patchfile): 477eb8dc403SDave Cobbley f = open(patchfile, 'rb') 478eb8dc403SDave Cobbley ln = f.readline() 479eb8dc403SDave Cobbley of = None 480eb8dc403SDave Cobbley in_patch = False 481eb8dc403SDave Cobbley beyond_msg = False 482eb8dc403SDave Cobbley pre_buf = b'' 483eb8dc403SDave Cobbley while ln: 484eb8dc403SDave Cobbley if not beyond_msg: 485eb8dc403SDave Cobbley if ln == b'---\n': 486eb8dc403SDave Cobbley if not of: 487eb8dc403SDave Cobbley break 488eb8dc403SDave Cobbley in_patch = False 489eb8dc403SDave Cobbley beyond_msg = True 490eb8dc403SDave Cobbley elif ln.startswith(b'--- '): 491eb8dc403SDave Cobbley # We have a diff in the commit message 492eb8dc403SDave Cobbley in_patch = True 493eb8dc403SDave Cobbley if not of: 494eb8dc403SDave Cobbley print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile) 495eb8dc403SDave Cobbley of = open(patchfile + '.tmp', 'wb') 496eb8dc403SDave Cobbley of.write(pre_buf) 497eb8dc403SDave Cobbley pre_buf = b'' 498eb8dc403SDave Cobbley elif in_patch and not ln[0] in b'+-@ \n\r': 499eb8dc403SDave Cobbley in_patch = False 500eb8dc403SDave Cobbley if of: 501eb8dc403SDave Cobbley if in_patch: 502eb8dc403SDave Cobbley of.write(b' ' + ln) 503eb8dc403SDave Cobbley else: 504eb8dc403SDave Cobbley of.write(ln) 505eb8dc403SDave Cobbley else: 506eb8dc403SDave Cobbley pre_buf += ln 507eb8dc403SDave Cobbley ln = f.readline() 508eb8dc403SDave Cobbley f.close() 509eb8dc403SDave Cobbley if of: 510eb8dc403SDave Cobbley of.close() 511eb8dc403SDave Cobbley os.rename(patchfile + '.tmp', patchfile) 512eb8dc403SDave Cobbley 513eb8dc403SDave Cobbleydef drop_to_shell(workdir=None): 514eb8dc403SDave Cobbley if not sys.stdin.isatty(): 515eb8dc403SDave Cobbley print("Not a TTY so can't drop to shell for resolution, exiting.") 516eb8dc403SDave Cobbley return False 517eb8dc403SDave Cobbley 518eb8dc403SDave Cobbley shell = os.environ.get('SHELL', 'bash') 519eb8dc403SDave Cobbley print('Dropping to shell "%s"\n' \ 520eb8dc403SDave Cobbley 'When you are finished, run the following to continue:\n' \ 521eb8dc403SDave Cobbley ' exit -- continue to apply the patches\n' \ 522eb8dc403SDave Cobbley ' exit 1 -- abort\n' % shell); 523eb8dc403SDave Cobbley ret = subprocess.call([shell], cwd=workdir) 524eb8dc403SDave Cobbley if ret != 0: 525eb8dc403SDave Cobbley print("Aborting") 526eb8dc403SDave Cobbley return False 527eb8dc403SDave Cobbley else: 528eb8dc403SDave Cobbley return True 529eb8dc403SDave Cobbley 530eb8dc403SDave Cobbleydef check_rev_branch(component, repodir, rev, branch): 531eb8dc403SDave Cobbley try: 532eb8dc403SDave Cobbley actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False) 533eb8dc403SDave Cobbley except subprocess.CalledProcessError as e: 534eb8dc403SDave Cobbley if e.returncode == 129: 535eb8dc403SDave Cobbley actualbranch = "" 536eb8dc403SDave Cobbley else: 537eb8dc403SDave Cobbley raise 538eb8dc403SDave Cobbley 539eb8dc403SDave Cobbley if not actualbranch: 540eb8dc403SDave Cobbley logger.error("%s: specified revision %s is invalid!" % (component, rev)) 541eb8dc403SDave Cobbley return False 542eb8dc403SDave Cobbley 543eb8dc403SDave Cobbley branches = [] 544eb8dc403SDave Cobbley branchlist = actualbranch.split("\n") 545eb8dc403SDave Cobbley for b in branchlist: 546eb8dc403SDave Cobbley branches.append(b.strip().split(' ')[-1]) 547eb8dc403SDave Cobbley 548eb8dc403SDave Cobbley if branch not in branches: 549eb8dc403SDave Cobbley logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch)) 550eb8dc403SDave Cobbley return False 551eb8dc403SDave Cobbley return True 552eb8dc403SDave Cobbley 553eb8dc403SDave Cobbleydef get_repos(conf, repo_names): 554eb8dc403SDave Cobbley repos = [] 555eb8dc403SDave Cobbley for name in repo_names: 556eb8dc403SDave Cobbley if name.startswith('-'): 557eb8dc403SDave Cobbley break 558eb8dc403SDave Cobbley else: 559eb8dc403SDave Cobbley repos.append(name) 560eb8dc403SDave Cobbley for repo in repos: 561eb8dc403SDave Cobbley if not repo in conf.repos: 562eb8dc403SDave Cobbley logger.error("Specified component '%s' not found in configuration" % repo) 563eb8dc403SDave Cobbley sys.exit(1) 564eb8dc403SDave Cobbley 565eb8dc403SDave Cobbley if not repos: 566eb8dc403SDave Cobbley repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ] 567eb8dc403SDave Cobbley 568eb8dc403SDave Cobbley return repos 569eb8dc403SDave Cobbley 570eb8dc403SDave Cobbleydef action_pull(conf, args): 571eb8dc403SDave Cobbley """ 572eb8dc403SDave Cobbley update the component repos only 573eb8dc403SDave Cobbley """ 574eb8dc403SDave Cobbley repos = get_repos(conf, args[1:]) 575eb8dc403SDave Cobbley 576eb8dc403SDave Cobbley # make sure all repos are clean 577eb8dc403SDave Cobbley for name in repos: 578eb8dc403SDave Cobbley check_repo_clean(conf.repos[name]['local_repo_dir']) 579eb8dc403SDave Cobbley 580eb8dc403SDave Cobbley for name in repos: 581eb8dc403SDave Cobbley repo = conf.repos[name] 582eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 583eb8dc403SDave Cobbley branch = repo.get('branch', "master") 584eb8dc403SDave Cobbley logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir)) 585eb8dc403SDave Cobbley if not conf.hard_reset: 586eb8dc403SDave Cobbley # Try to pull only the configured branch. Beware that this may fail 587eb8dc403SDave Cobbley # when the branch is currently unknown (for example, after reconfiguring 588eb8dc403SDave Cobbley # combo-layer). In that case we need to fetch everything and try the check out 589eb8dc403SDave Cobbley # and pull again. 590eb8dc403SDave Cobbley try: 591eb8dc403SDave Cobbley runcmd("git checkout %s" % branch, ldir, printerr=False) 592eb8dc403SDave Cobbley except subprocess.CalledProcessError: 593eb8dc403SDave Cobbley output=runcmd("git fetch", ldir) 594eb8dc403SDave Cobbley logger.info(output) 595eb8dc403SDave Cobbley runcmd("git checkout %s" % branch, ldir) 596eb8dc403SDave Cobbley runcmd("git pull --ff-only", ldir) 597eb8dc403SDave Cobbley else: 598eb8dc403SDave Cobbley output=runcmd("git pull --ff-only", ldir) 599eb8dc403SDave Cobbley logger.info(output) 600eb8dc403SDave Cobbley else: 601eb8dc403SDave Cobbley output=runcmd("git fetch", ldir) 602eb8dc403SDave Cobbley logger.info(output) 603eb8dc403SDave Cobbley runcmd("git checkout %s" % branch, ldir) 604eb8dc403SDave Cobbley runcmd("git reset --hard FETCH_HEAD", ldir) 605eb8dc403SDave Cobbley 606eb8dc403SDave Cobbleydef action_update(conf, args): 607eb8dc403SDave Cobbley """ 608eb8dc403SDave Cobbley update the component repos 609eb8dc403SDave Cobbley either: 610eb8dc403SDave Cobbley generate the patch list 611eb8dc403SDave Cobbley apply the generated patches 612eb8dc403SDave Cobbley or: 613eb8dc403SDave Cobbley re-creates the entire component history and merges them 614eb8dc403SDave Cobbley into the current branch with a merge commit 615eb8dc403SDave Cobbley """ 616eb8dc403SDave Cobbley components = [arg.split(':')[0] for arg in args[1:]] 617eb8dc403SDave Cobbley revisions = {} 618eb8dc403SDave Cobbley for arg in args[1:]: 619eb8dc403SDave Cobbley if ':' in arg: 620eb8dc403SDave Cobbley a = arg.split(':', 1) 621eb8dc403SDave Cobbley revisions[a[0]] = a[1] 622eb8dc403SDave Cobbley repos = get_repos(conf, components) 623eb8dc403SDave Cobbley 624eb8dc403SDave Cobbley # make sure combo repo is clean 625eb8dc403SDave Cobbley check_repo_clean(os.getcwd()) 626eb8dc403SDave Cobbley 627eb8dc403SDave Cobbley # Check whether we keep the component histories. Must be 628eb8dc403SDave Cobbley # set either via --history command line parameter or consistently 629eb8dc403SDave Cobbley # in combo-layer.conf. Mixing modes is (currently, and probably 630eb8dc403SDave Cobbley # permanently because it would be complicated) not supported. 631eb8dc403SDave Cobbley if conf.history: 632eb8dc403SDave Cobbley history = True 633eb8dc403SDave Cobbley else: 634eb8dc403SDave Cobbley history = None 635eb8dc403SDave Cobbley for name in repos: 636eb8dc403SDave Cobbley repo = conf.repos[name] 637eb8dc403SDave Cobbley repo_history = repo.get('history', False) 638eb8dc403SDave Cobbley if history is None: 639eb8dc403SDave Cobbley history = repo_history 640eb8dc403SDave Cobbley elif history != repo_history: 641eb8dc403SDave Cobbley logger.error("'history' property is set inconsistently") 642eb8dc403SDave Cobbley sys.exit(1) 643eb8dc403SDave Cobbley 644eb8dc403SDave Cobbley # Step 1: update the component repos 645eb8dc403SDave Cobbley if conf.nopull: 646eb8dc403SDave Cobbley logger.info("Skipping pull (-n)") 647eb8dc403SDave Cobbley else: 648eb8dc403SDave Cobbley action_pull(conf, ['arg0'] + components) 649eb8dc403SDave Cobbley 650eb8dc403SDave Cobbley if history: 651eb8dc403SDave Cobbley update_with_history(conf, components, revisions, repos) 652eb8dc403SDave Cobbley else: 653eb8dc403SDave Cobbley update_with_patches(conf, components, revisions, repos) 654eb8dc403SDave Cobbley 655eb8dc403SDave Cobbleydef update_with_patches(conf, components, revisions, repos): 656eb8dc403SDave Cobbley import uuid 657eb8dc403SDave Cobbley patch_dir = "patch-%s" % uuid.uuid4() 658eb8dc403SDave Cobbley if not os.path.exists(patch_dir): 659eb8dc403SDave Cobbley os.mkdir(patch_dir) 660eb8dc403SDave Cobbley 661eb8dc403SDave Cobbley for name in repos: 662eb8dc403SDave Cobbley revision = revisions.get(name, None) 663eb8dc403SDave Cobbley repo = conf.repos[name] 664eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 665eb8dc403SDave Cobbley dest_dir = repo['dest_dir'] 666eb8dc403SDave Cobbley branch = repo.get('branch', "master") 667eb8dc403SDave Cobbley repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name) 668eb8dc403SDave Cobbley 669eb8dc403SDave Cobbley # Step 2: generate the patch list and store to patch dir 670eb8dc403SDave Cobbley logger.info("Generating patches from %s..." % name) 671eb8dc403SDave Cobbley top_revision = revision or branch 672eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, top_revision, branch): 673eb8dc403SDave Cobbley sys.exit(1) 674eb8dc403SDave Cobbley if dest_dir != ".": 675eb8dc403SDave Cobbley prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir) 676eb8dc403SDave Cobbley else: 677eb8dc403SDave Cobbley prefix = "" 678eb8dc403SDave Cobbley if repo['last_revision'] == "": 679eb8dc403SDave Cobbley logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name) 680eb8dc403SDave Cobbley patch_cmd_range = "--root %s" % top_revision 681eb8dc403SDave Cobbley rev_cmd_range = top_revision 682eb8dc403SDave Cobbley else: 683eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, repo['last_revision'], branch): 684eb8dc403SDave Cobbley sys.exit(1) 685eb8dc403SDave Cobbley patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision) 686eb8dc403SDave Cobbley rev_cmd_range = patch_cmd_range 687eb8dc403SDave Cobbley 688eb8dc403SDave Cobbley file_filter = repo.get('file_filter',".") 689eb8dc403SDave Cobbley 690eb8dc403SDave Cobbley # Filter out unwanted files 691eb8dc403SDave Cobbley exclude = repo.get('file_exclude', '') 692eb8dc403SDave Cobbley if exclude: 693eb8dc403SDave Cobbley for path in exclude.split(): 694eb8dc403SDave Cobbley p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path 695eb8dc403SDave Cobbley file_filter += " ':!%s'" % p 696eb8dc403SDave Cobbley 697eb8dc403SDave Cobbley patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \ 698eb8dc403SDave Cobbley (prefix,repo_patch_dir, patch_cmd_range, file_filter) 699eb8dc403SDave Cobbley output = runcmd(patch_cmd, ldir) 700eb8dc403SDave Cobbley logger.debug("generated patch set:\n%s" % output) 701eb8dc403SDave Cobbley patchlist = output.splitlines() 702eb8dc403SDave Cobbley 703eb8dc403SDave Cobbley rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter) 704eb8dc403SDave Cobbley revlist = runcmd(rev_cmd, ldir).splitlines() 705eb8dc403SDave Cobbley 706eb8dc403SDave Cobbley # Step 3: Call repo specific hook to adjust patch 707eb8dc403SDave Cobbley if 'hook' in repo: 708eb8dc403SDave Cobbley # hook parameter is: ./hook patchpath revision reponame 709eb8dc403SDave Cobbley count=len(revlist)-1 710eb8dc403SDave Cobbley for patch in patchlist: 711eb8dc403SDave Cobbley runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name)) 712eb8dc403SDave Cobbley count=count-1 713eb8dc403SDave Cobbley 714eb8dc403SDave Cobbley # Step 4: write patch list and revision list to file, for user to edit later 715eb8dc403SDave Cobbley patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name) 716eb8dc403SDave Cobbley repo['patchlist'] = patchlist_file 717eb8dc403SDave Cobbley f = open(patchlist_file, 'w') 718eb8dc403SDave Cobbley count=len(revlist)-1 719eb8dc403SDave Cobbley for patch in patchlist: 720eb8dc403SDave Cobbley f.write("%s %s\n" % (patch, revlist[count])) 721eb8dc403SDave Cobbley check_patch(os.path.join(patch_dir, patch)) 722eb8dc403SDave Cobbley count=count-1 723eb8dc403SDave Cobbley f.close() 724eb8dc403SDave Cobbley 725eb8dc403SDave Cobbley # Step 5: invoke bash for user to edit patch and patch list 726eb8dc403SDave Cobbley if conf.interactive: 727eb8dc403SDave Cobbley print('You may now edit the patch and patch list in %s\n' \ 728eb8dc403SDave Cobbley 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir); 729eb8dc403SDave Cobbley if not drop_to_shell(patch_dir): 730eb8dc403SDave Cobbley sys.exit(1) 731eb8dc403SDave Cobbley 732eb8dc403SDave Cobbley # Step 6: apply the generated and revised patch 733eb8dc403SDave Cobbley apply_patchlist(conf, repos) 734eb8dc403SDave Cobbley runcmd("rm -rf %s" % patch_dir) 735eb8dc403SDave Cobbley 736eb8dc403SDave Cobbley # Step 7: commit the updated config file if it's being tracked 737eb8dc403SDave Cobbley commit_conf_file(conf, components) 738eb8dc403SDave Cobbley 739eb8dc403SDave Cobbleydef conf_commit_msg(conf, components): 740eb8dc403SDave Cobbley # create the "components" string 741eb8dc403SDave Cobbley component_str = "all components" 742eb8dc403SDave Cobbley if len(components) > 0: 743eb8dc403SDave Cobbley # otherwise tell which components were actually changed 744eb8dc403SDave Cobbley component_str = ", ".join(components) 745eb8dc403SDave Cobbley 746eb8dc403SDave Cobbley # expand the template with known values 747eb8dc403SDave Cobbley template = Template(conf.commit_msg_template) 748eb8dc403SDave Cobbley msg = template.substitute(components = component_str) 749eb8dc403SDave Cobbley return msg 750eb8dc403SDave Cobbley 751eb8dc403SDave Cobbleydef commit_conf_file(conf, components, commit=True): 752eb8dc403SDave Cobbley relpath = os.path.relpath(conf.conffile) 753eb8dc403SDave Cobbley try: 754eb8dc403SDave Cobbley output = runcmd("git status --porcelain %s" % relpath, printerr=False) 755eb8dc403SDave Cobbley except: 756eb8dc403SDave Cobbley # Outside the repository 757eb8dc403SDave Cobbley output = None 758eb8dc403SDave Cobbley if output: 759eb8dc403SDave Cobbley if output.lstrip().startswith("M"): 760eb8dc403SDave Cobbley logger.info("Committing updated configuration file") 761eb8dc403SDave Cobbley if commit: 762eb8dc403SDave Cobbley msg = conf_commit_msg(conf, components) 763eb8dc403SDave Cobbley runcmd('git commit -m'.split() + [msg, relpath]) 764eb8dc403SDave Cobbley else: 765eb8dc403SDave Cobbley runcmd('git add %s' % relpath) 766eb8dc403SDave Cobbley return True 767eb8dc403SDave Cobbley return False 768eb8dc403SDave Cobbley 769eb8dc403SDave Cobbleydef apply_patchlist(conf, repos): 770eb8dc403SDave Cobbley """ 771eb8dc403SDave Cobbley apply the generated patch list to combo repo 772eb8dc403SDave Cobbley """ 773eb8dc403SDave Cobbley for name in repos: 774eb8dc403SDave Cobbley repo = conf.repos[name] 775eb8dc403SDave Cobbley lastrev = repo["last_revision"] 776eb8dc403SDave Cobbley prevrev = lastrev 777eb8dc403SDave Cobbley 778eb8dc403SDave Cobbley # Get non-blank lines from patch list file 779eb8dc403SDave Cobbley patchlist = [] 780eb8dc403SDave Cobbley if os.path.exists(repo['patchlist']) or not conf.interactive: 781eb8dc403SDave Cobbley # Note: we want this to fail here if the file doesn't exist and we're not in 782eb8dc403SDave Cobbley # interactive mode since the file should exist in this case 783eb8dc403SDave Cobbley with open(repo['patchlist']) as f: 784eb8dc403SDave Cobbley for line in f: 785eb8dc403SDave Cobbley line = line.rstrip() 786eb8dc403SDave Cobbley if line: 787eb8dc403SDave Cobbley patchlist.append(line) 788eb8dc403SDave Cobbley 789eb8dc403SDave Cobbley ldir = conf.repos[name]['local_repo_dir'] 790eb8dc403SDave Cobbley branch = conf.repos[name].get('branch', "master") 791eb8dc403SDave Cobbley branchrev = runcmd("git rev-parse %s" % branch, ldir).strip() 792eb8dc403SDave Cobbley 793eb8dc403SDave Cobbley if patchlist: 794eb8dc403SDave Cobbley logger.info("Applying patches from %s..." % name) 795eb8dc403SDave Cobbley linecount = len(patchlist) 796eb8dc403SDave Cobbley i = 1 797eb8dc403SDave Cobbley for line in patchlist: 798eb8dc403SDave Cobbley patchfile = line.split()[0] 799eb8dc403SDave Cobbley lastrev = line.split()[1] 800eb8dc403SDave Cobbley patchdisp = os.path.relpath(patchfile) 801eb8dc403SDave Cobbley if os.path.getsize(patchfile) == 0: 802eb8dc403SDave Cobbley logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp)) 803eb8dc403SDave Cobbley else: 804eb8dc403SDave Cobbley cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile) 805eb8dc403SDave Cobbley logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp)) 806eb8dc403SDave Cobbley try: 807eb8dc403SDave Cobbley runcmd(cmd) 808eb8dc403SDave Cobbley except subprocess.CalledProcessError: 809eb8dc403SDave Cobbley logger.info('Running "git am --abort" to cleanup repo') 810eb8dc403SDave Cobbley runcmd("git am --abort") 811eb8dc403SDave Cobbley logger.error('"%s" failed' % cmd) 812eb8dc403SDave Cobbley logger.info("Please manually apply patch %s" % patchdisp) 813eb8dc403SDave Cobbley logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped") 814eb8dc403SDave Cobbley if not drop_to_shell(): 815eb8dc403SDave Cobbley if prevrev != repo['last_revision']: 816eb8dc403SDave Cobbley conf.update(name, "last_revision", prevrev) 817eb8dc403SDave Cobbley sys.exit(1) 818eb8dc403SDave Cobbley prevrev = lastrev 819eb8dc403SDave Cobbley i += 1 820eb8dc403SDave Cobbley # Once all patches are applied, we should update 821eb8dc403SDave Cobbley # last_revision to the branch head instead of the last 822eb8dc403SDave Cobbley # applied patch. The two are not necessarily the same when 823eb8dc403SDave Cobbley # the last commit is a merge commit or when the patches at 824eb8dc403SDave Cobbley # the branch head were intentionally excluded. 825eb8dc403SDave Cobbley # 826eb8dc403SDave Cobbley # If we do not do that for a merge commit, the next 827eb8dc403SDave Cobbley # combo-layer run will only exclude patches reachable from 828eb8dc403SDave Cobbley # one of the merged branches and try to re-apply patches 829eb8dc403SDave Cobbley # from other branches even though they were already 830eb8dc403SDave Cobbley # copied. 831eb8dc403SDave Cobbley # 832eb8dc403SDave Cobbley # If patches were intentionally excluded, the next run will 833eb8dc403SDave Cobbley # present them again instead of skipping over them. This 834eb8dc403SDave Cobbley # may or may not be intended, so the code here is conservative 835eb8dc403SDave Cobbley # and only addresses the "head is merge commit" case. 836eb8dc403SDave Cobbley if lastrev != branchrev and \ 837eb8dc403SDave Cobbley len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1: 838eb8dc403SDave Cobbley lastrev = branchrev 839eb8dc403SDave Cobbley else: 840eb8dc403SDave Cobbley logger.info("No patches to apply from %s" % name) 841eb8dc403SDave Cobbley lastrev = branchrev 842eb8dc403SDave Cobbley 843eb8dc403SDave Cobbley if lastrev != repo['last_revision']: 844eb8dc403SDave Cobbley conf.update(name, "last_revision", lastrev) 845eb8dc403SDave Cobbley 846eb8dc403SDave Cobbleydef action_splitpatch(conf, args): 847eb8dc403SDave Cobbley """ 848eb8dc403SDave Cobbley generate the commit patch and 849eb8dc403SDave Cobbley split the patch per repo 850eb8dc403SDave Cobbley """ 851eb8dc403SDave Cobbley logger.debug("action_splitpatch") 852eb8dc403SDave Cobbley if len(args) > 1: 853eb8dc403SDave Cobbley commit = args[1] 854eb8dc403SDave Cobbley else: 855eb8dc403SDave Cobbley commit = "HEAD" 856eb8dc403SDave Cobbley patchdir = "splitpatch-%s" % commit 857eb8dc403SDave Cobbley if not os.path.exists(patchdir): 858eb8dc403SDave Cobbley os.mkdir(patchdir) 859eb8dc403SDave Cobbley 860eb8dc403SDave Cobbley # filerange_root is for the repo whose dest_dir is root "." 861eb8dc403SDave Cobbley # and it should be specified by excluding all other repo dest dir 862eb8dc403SDave Cobbley # like "-x repo1 -x repo2 -x repo3 ..." 863eb8dc403SDave Cobbley filerange_root = "" 864eb8dc403SDave Cobbley for name in conf.repos: 865eb8dc403SDave Cobbley dest_dir = conf.repos[name]['dest_dir'] 866eb8dc403SDave Cobbley if dest_dir != ".": 867eb8dc403SDave Cobbley filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir) 868eb8dc403SDave Cobbley 869eb8dc403SDave Cobbley for name in conf.repos: 870eb8dc403SDave Cobbley dest_dir = conf.repos[name]['dest_dir'] 871eb8dc403SDave Cobbley patch_filename = "%s/%s.patch" % (patchdir, name) 872eb8dc403SDave Cobbley if dest_dir == ".": 873eb8dc403SDave Cobbley cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename) 874eb8dc403SDave Cobbley else: 875eb8dc403SDave Cobbley cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename) 876eb8dc403SDave Cobbley runcmd(cmd) 877eb8dc403SDave Cobbley # Detect empty patches (including those produced by filterdiff above 878eb8dc403SDave Cobbley # that contain only preamble text) 879eb8dc403SDave Cobbley if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "": 880eb8dc403SDave Cobbley os.remove(patch_filename) 881eb8dc403SDave Cobbley logger.info("(skipping %s - no changes)", name) 882eb8dc403SDave Cobbley else: 883eb8dc403SDave Cobbley logger.info(patch_filename) 884eb8dc403SDave Cobbley 885eb8dc403SDave Cobbleydef update_with_history(conf, components, revisions, repos): 886eb8dc403SDave Cobbley '''Update all components with full history. 887eb8dc403SDave Cobbley 888eb8dc403SDave Cobbley Works by importing all commits reachable from a component's 889eb8dc403SDave Cobbley current head revision. If those commits are rooted in an already 890eb8dc403SDave Cobbley imported commit, their content gets mixed with the content of the 891eb8dc403SDave Cobbley combined repo of that commit (new or modified files overwritten, 892eb8dc403SDave Cobbley removed files removed). 893eb8dc403SDave Cobbley 894eb8dc403SDave Cobbley The last commit is an artificial merge commit that merges all the 895eb8dc403SDave Cobbley updated components into the combined repository. 896eb8dc403SDave Cobbley 897eb8dc403SDave Cobbley The HEAD ref only gets updated at the very end. All intermediate work 898eb8dc403SDave Cobbley happens in a worktree which will get garbage collected by git eventually 899eb8dc403SDave Cobbley after a failure. 900eb8dc403SDave Cobbley ''' 901eb8dc403SDave Cobbley # Remember current HEAD and what we need to add to it. 902eb8dc403SDave Cobbley head = runcmd("git rev-parse HEAD").strip() 903eb8dc403SDave Cobbley additional_heads = {} 904eb8dc403SDave Cobbley 905eb8dc403SDave Cobbley # Track the mapping between original commit and commit in the 906eb8dc403SDave Cobbley # combined repo. We do not have to distinguish between components, 907eb8dc403SDave Cobbley # because commit hashes are different anyway. Often we can 908eb8dc403SDave Cobbley # skip find_revs() entirely (for example, when all new commits 909eb8dc403SDave Cobbley # are derived from the last imported revision). 910eb8dc403SDave Cobbley # 911eb8dc403SDave Cobbley # Using "head" (typically the merge commit) instead of the actual 912eb8dc403SDave Cobbley # commit for the component leads to a nicer history in the combined 913eb8dc403SDave Cobbley # repo. 914eb8dc403SDave Cobbley old2new_revs = {} 915eb8dc403SDave Cobbley for name in repos: 916eb8dc403SDave Cobbley repo = conf.repos[name] 917eb8dc403SDave Cobbley revision = repo['last_revision'] 918eb8dc403SDave Cobbley if revision: 919eb8dc403SDave Cobbley old2new_revs[revision] = head 920eb8dc403SDave Cobbley 921eb8dc403SDave Cobbley def add_p(parents): 922eb8dc403SDave Cobbley '''Insert -p before each entry.''' 923eb8dc403SDave Cobbley parameters = [] 924eb8dc403SDave Cobbley for p in parents: 925eb8dc403SDave Cobbley parameters.append('-p') 926eb8dc403SDave Cobbley parameters.append(p) 927eb8dc403SDave Cobbley return parameters 928eb8dc403SDave Cobbley 929eb8dc403SDave Cobbley # Do all intermediate work with a separate work dir and index, 930eb8dc403SDave Cobbley # chosen via env variables (can't use "git worktree", it is too 931eb8dc403SDave Cobbley # new). This is useful (no changes to current work tree unless the 932eb8dc403SDave Cobbley # update succeeds) and required (otherwise we end up temporarily 933eb8dc403SDave Cobbley # removing the combo-layer hooks that we currently use when 934eb8dc403SDave Cobbley # importing a new component). 935eb8dc403SDave Cobbley # 936eb8dc403SDave Cobbley # Not cleaned up after a failure at the moment. 937eb8dc403SDave Cobbley wdir = os.path.join(os.getcwd(), ".git", "combo-layer") 938eb8dc403SDave Cobbley windex = wdir + ".index" 939eb8dc403SDave Cobbley if os.path.isdir(wdir): 940eb8dc403SDave Cobbley shutil.rmtree(wdir) 941eb8dc403SDave Cobbley os.mkdir(wdir) 942eb8dc403SDave Cobbley wenv = copy.deepcopy(os.environ) 943eb8dc403SDave Cobbley wenv["GIT_WORK_TREE"] = wdir 944eb8dc403SDave Cobbley wenv["GIT_INDEX_FILE"] = windex 945eb8dc403SDave Cobbley # This one turned out to be needed in practice. 946eb8dc403SDave Cobbley wenv["GIT_OBJECT_DIRECTORY"] = os.path.join(os.getcwd(), ".git", "objects") 947eb8dc403SDave Cobbley wargs = {"destdir": wdir, "env": wenv} 948eb8dc403SDave Cobbley 949eb8dc403SDave Cobbley for name in repos: 950eb8dc403SDave Cobbley revision = revisions.get(name, None) 951eb8dc403SDave Cobbley repo = conf.repos[name] 952eb8dc403SDave Cobbley ldir = repo['local_repo_dir'] 953eb8dc403SDave Cobbley dest_dir = repo['dest_dir'] 954eb8dc403SDave Cobbley branch = repo.get('branch', "master") 955eb8dc403SDave Cobbley hook = repo.get('hook', None) 956eb8dc403SDave Cobbley largs = {"destdir": ldir, "env": None} 957eb8dc403SDave Cobbley file_include = repo.get('file_filter', '').split() 958eb8dc403SDave Cobbley file_include.sort() # make sure that short entries like '.' come first. 959eb8dc403SDave Cobbley file_exclude = repo.get('file_exclude', '').split() 960eb8dc403SDave Cobbley 961eb8dc403SDave Cobbley def include_file(file): 962eb8dc403SDave Cobbley if not file_include: 963eb8dc403SDave Cobbley # No explicit filter set, include file. 964eb8dc403SDave Cobbley return True 965eb8dc403SDave Cobbley for filter in file_include: 966eb8dc403SDave Cobbley if filter == '.': 967eb8dc403SDave Cobbley # Another special case: include current directory and thus all files. 968eb8dc403SDave Cobbley return True 969eb8dc403SDave Cobbley if os.path.commonprefix((filter, file)) == filter: 970eb8dc403SDave Cobbley # Included in directory or direct file match. 971eb8dc403SDave Cobbley return True 972eb8dc403SDave Cobbley # Check for wildcard match *with* allowing * to match /, i.e. 973eb8dc403SDave Cobbley # src/*.c does match src/foobar/*.c. That's not how it is done elsewhere 974eb8dc403SDave Cobbley # when passing the filtering to "git archive", but it is unclear what 975eb8dc403SDave Cobbley # the intended semantic is (the comment on file_exclude that "append a * wildcard 976eb8dc403SDave Cobbley # at the end" to match the full content of a directories implies that 977eb8dc403SDave Cobbley # slashes are indeed not special), so here we simply do what's easy to 978eb8dc403SDave Cobbley # implement in Python. 979eb8dc403SDave Cobbley logger.debug('fnmatch(%s, %s)' % (file, filter)) 980eb8dc403SDave Cobbley if fnmatch.fnmatchcase(file, filter): 981eb8dc403SDave Cobbley return True 982eb8dc403SDave Cobbley return False 983eb8dc403SDave Cobbley 984eb8dc403SDave Cobbley def exclude_file(file): 985eb8dc403SDave Cobbley for filter in file_exclude: 986eb8dc403SDave Cobbley if fnmatch.fnmatchcase(file, filter): 987eb8dc403SDave Cobbley return True 988eb8dc403SDave Cobbley return False 989eb8dc403SDave Cobbley 990eb8dc403SDave Cobbley def file_filter(files): 991eb8dc403SDave Cobbley '''Clean up file list so that only included files remain.''' 992eb8dc403SDave Cobbley index = 0 993eb8dc403SDave Cobbley while index < len(files): 994eb8dc403SDave Cobbley file = files[index] 995eb8dc403SDave Cobbley if not include_file(file) or exclude_file(file): 996eb8dc403SDave Cobbley del files[index] 997eb8dc403SDave Cobbley else: 998eb8dc403SDave Cobbley index += 1 999eb8dc403SDave Cobbley 1000eb8dc403SDave Cobbley 1001eb8dc403SDave Cobbley # Generate the revision list. 1002eb8dc403SDave Cobbley logger.info("Analyzing commits from %s..." % name) 1003eb8dc403SDave Cobbley top_revision = revision or branch 1004eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, top_revision, branch): 1005eb8dc403SDave Cobbley sys.exit(1) 1006eb8dc403SDave Cobbley 1007eb8dc403SDave Cobbley last_revision = repo['last_revision'] 1008eb8dc403SDave Cobbley rev_list_args = "--full-history --sparse --topo-order --reverse" 1009eb8dc403SDave Cobbley if not last_revision: 1010eb8dc403SDave Cobbley logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name) 1011eb8dc403SDave Cobbley rev_list_args = rev_list_args + ' ' + top_revision 1012eb8dc403SDave Cobbley else: 1013eb8dc403SDave Cobbley if not check_rev_branch(name, ldir, last_revision, branch): 1014eb8dc403SDave Cobbley sys.exit(1) 1015eb8dc403SDave Cobbley rev_list_args = "%s %s..%s" % (rev_list_args, last_revision, top_revision) 1016eb8dc403SDave Cobbley 1017eb8dc403SDave Cobbley # By definition, the current HEAD contains the latest imported 1018eb8dc403SDave Cobbley # commit of each component. We use that as initial mapping even 1019eb8dc403SDave Cobbley # though the commits do not match exactly because 1020eb8dc403SDave Cobbley # a) it always works (in contrast to find_revs, which relies on special 1021eb8dc403SDave Cobbley # commit messages) 1022eb8dc403SDave Cobbley # b) it is faster than find_revs, which will only be called on demand 1023eb8dc403SDave Cobbley # and can be skipped entirely in most cases 1024eb8dc403SDave Cobbley # c) last but not least, the combined history looks nicer when all 1025eb8dc403SDave Cobbley # new commits are rooted in the same merge commit 1026eb8dc403SDave Cobbley old2new_revs[last_revision] = head 1027eb8dc403SDave Cobbley 1028eb8dc403SDave Cobbley # We care about all commits (--full-history and --sparse) and 1029eb8dc403SDave Cobbley # we want reconstruct the topology and thus do not care 1030eb8dc403SDave Cobbley # about ordering by time (--topo-order). We ask for the ones 1031eb8dc403SDave Cobbley # we need to import first to be listed first (--reverse). 1032eb8dc403SDave Cobbley revs = runcmd("git rev-list %s" % rev_list_args, **largs).split() 1033eb8dc403SDave Cobbley logger.debug("To be imported: %s" % revs) 1034eb8dc403SDave Cobbley # Now 'revs' contains all revisions reachable from the top revision. 1035eb8dc403SDave Cobbley # All revisions derived from the 'last_revision' definitely are new, 1036eb8dc403SDave Cobbley # whereas the others may or may not have been imported before. For 1037eb8dc403SDave Cobbley # a linear history in the component, that second set will be empty. 1038eb8dc403SDave Cobbley # To distinguish between them, we also get the shorter list 1039eb8dc403SDave Cobbley # of revisions starting at the ancestor. 1040eb8dc403SDave Cobbley if last_revision: 1041eb8dc403SDave Cobbley ancestor_revs = runcmd("git rev-list --ancestry-path %s" % rev_list_args, **largs).split() 1042eb8dc403SDave Cobbley else: 1043eb8dc403SDave Cobbley ancestor_revs = [] 1044eb8dc403SDave Cobbley logger.debug("Ancestors: %s" % ancestor_revs) 1045eb8dc403SDave Cobbley 1046eb8dc403SDave Cobbley # Now import each revision. 1047eb8dc403SDave Cobbley logger.info("Importing commits from %s..." % name) 1048eb8dc403SDave Cobbley def import_rev(rev): 1049eb8dc403SDave Cobbley global scanned_revs 1050eb8dc403SDave Cobbley 1051eb8dc403SDave Cobbley # If it is part of the new commits, we definitely need 1052eb8dc403SDave Cobbley # to import it. Otherwise we need to check, we might have 1053eb8dc403SDave Cobbley # imported it before. If it was imported and we merely 1054eb8dc403SDave Cobbley # fail to find it because commit messages did not track 1055eb8dc403SDave Cobbley # the mapping, then we end up importing it again. So 1056eb8dc403SDave Cobbley # combined repos using "updating with history" really should 1057eb8dc403SDave Cobbley # enable the "From ... rev:" commit header modifications. 1058eb8dc403SDave Cobbley if rev not in ancestor_revs and rev not in old2new_revs and not scanned_revs: 1059eb8dc403SDave Cobbley logger.debug("Revision %s triggers log analysis." % rev) 1060eb8dc403SDave Cobbley find_revs(old2new_revs, head) 1061eb8dc403SDave Cobbley scanned_revs = True 1062eb8dc403SDave Cobbley new_rev = old2new_revs.get(rev, None) 1063eb8dc403SDave Cobbley if new_rev: 1064eb8dc403SDave Cobbley return new_rev 1065eb8dc403SDave Cobbley 1066eb8dc403SDave Cobbley # If the commit is not in the original list of revisions 1067eb8dc403SDave Cobbley # to be imported, then it must be a parent of one of those 1068eb8dc403SDave Cobbley # commits and it was skipped during earlier imports or not 1069eb8dc403SDave Cobbley # found. Importing such merge commits leads to very ugly 1070eb8dc403SDave Cobbley # history (long cascade of merge commits which all point 1071eb8dc403SDave Cobbley # to to older commits) when switching from "update via 1072eb8dc403SDave Cobbley # patches" to "update with history". 1073eb8dc403SDave Cobbley # 1074eb8dc403SDave Cobbley # We can avoid importing merge commits if all non-merge commits 1075eb8dc403SDave Cobbley # reachable from it were already imported. In that case we 1076eb8dc403SDave Cobbley # can root the new commits in the current head revision. 1077eb8dc403SDave Cobbley def is_imported(prev): 1078eb8dc403SDave Cobbley parents = runcmd("git show --no-patch --pretty=format:%P " + prev, **largs).split() 1079eb8dc403SDave Cobbley if len(parents) > 1: 1080eb8dc403SDave Cobbley for p in parents: 1081eb8dc403SDave Cobbley if not is_imported(p): 1082eb8dc403SDave Cobbley logger.debug("Must import %s because %s is not imported." % (rev, p)) 1083eb8dc403SDave Cobbley return False 1084eb8dc403SDave Cobbley return True 1085eb8dc403SDave Cobbley elif prev in old2new_revs: 1086eb8dc403SDave Cobbley return True 1087eb8dc403SDave Cobbley else: 1088eb8dc403SDave Cobbley logger.debug("Must import %s because %s is not imported." % (rev, prev)) 1089eb8dc403SDave Cobbley return False 1090eb8dc403SDave Cobbley if rev not in revs and is_imported(rev): 1091eb8dc403SDave Cobbley old2new_revs[rev] = head 1092eb8dc403SDave Cobbley return head 1093eb8dc403SDave Cobbley 1094eb8dc403SDave Cobbley # Need to import rev. Collect some information about it. 1095eb8dc403SDave Cobbley logger.debug("Importing %s" % rev) 1096eb8dc403SDave Cobbley (parents, author_name, author_email, author_timestamp, body) = \ 1097eb8dc403SDave Cobbley runcmd("git show --no-patch --pretty=format:%P%x00%an%x00%ae%x00%at%x00%B " + rev, **largs).split(chr(0)) 1098eb8dc403SDave Cobbley parents = parents.split() 1099eb8dc403SDave Cobbley if parents: 1100eb8dc403SDave Cobbley # Arbitrarily pick the first parent as base. It may or may not have 1101eb8dc403SDave Cobbley # been imported before. For example, if the parent is a merge commit 1102eb8dc403SDave Cobbley # and previously the combined repository used patching as update 1103eb8dc403SDave Cobbley # method, then the actual merge commit parent never was imported. 1104eb8dc403SDave Cobbley # To cover this, We recursively import parents. 1105eb8dc403SDave Cobbley parent = parents[0] 1106eb8dc403SDave Cobbley new_parent = import_rev(parent) 1107eb8dc403SDave Cobbley # Clean index and working tree. TODO: can we combine this and the 1108eb8dc403SDave Cobbley # next into one command with less file IO? 1109eb8dc403SDave Cobbley # "git reset --hard" does not work, it changes HEAD of the parent 1110eb8dc403SDave Cobbley # repo, which we wanted to avoid. Probably need to keep 1111eb8dc403SDave Cobbley # track of the rev that corresponds to the index and use apply_commit(). 1112eb8dc403SDave Cobbley runcmd("git rm -q --ignore-unmatch -rf .", **wargs) 1113eb8dc403SDave Cobbley # Update index and working tree to match the parent. 1114eb8dc403SDave Cobbley runcmd("git checkout -q -f %s ." % new_parent, **wargs) 1115eb8dc403SDave Cobbley else: 1116eb8dc403SDave Cobbley parent = None 1117eb8dc403SDave Cobbley # Clean index and working tree. 1118eb8dc403SDave Cobbley runcmd("git rm -q --ignore-unmatch -rf .", **wargs) 1119eb8dc403SDave Cobbley 1120eb8dc403SDave Cobbley # Modify index and working tree such that it mirrors the commit. 1121eb8dc403SDave Cobbley apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=file_filter) 1122eb8dc403SDave Cobbley 1123eb8dc403SDave Cobbley # Now commit. 1124eb8dc403SDave Cobbley new_tree = runcmd("git write-tree", **wargs).strip() 1125eb8dc403SDave Cobbley env = copy.deepcopy(wenv) 1126eb8dc403SDave Cobbley env['GIT_AUTHOR_NAME'] = author_name 1127eb8dc403SDave Cobbley env['GIT_AUTHOR_EMAIL'] = author_email 1128eb8dc403SDave Cobbley env['GIT_AUTHOR_DATE'] = author_timestamp 1129eb8dc403SDave Cobbley if hook: 1130eb8dc403SDave Cobbley # Need to turn the verbatim commit message into something resembling a patch header 1131eb8dc403SDave Cobbley # for the hook. 1132eb8dc403SDave Cobbley with tempfile.NamedTemporaryFile(mode='wt', delete=False) as patch: 1133eb8dc403SDave Cobbley patch.write('Subject: [PATCH] ') 1134eb8dc403SDave Cobbley patch.write(body) 1135eb8dc403SDave Cobbley patch.write('\n---\n') 1136eb8dc403SDave Cobbley patch.close() 1137eb8dc403SDave Cobbley runcmd([hook, patch.name, rev, name]) 1138eb8dc403SDave Cobbley with open(patch.name) as f: 1139eb8dc403SDave Cobbley body = f.read()[len('Subject: [PATCH] '):][:-len('\n---\n')] 1140eb8dc403SDave Cobbley 1141eb8dc403SDave Cobbley # We can skip non-merge commits that did not change any files. Those are typically 1142eb8dc403SDave Cobbley # the result of file filtering, although they could also have been introduced 1143eb8dc403SDave Cobbley # intentionally upstream, in which case we drop some information here. 1144eb8dc403SDave Cobbley if len(parents) == 1: 1145eb8dc403SDave Cobbley parent_rev = import_rev(parents[0]) 1146eb8dc403SDave Cobbley old_tree = runcmd("git show -s --pretty=format:%T " + parent_rev, **wargs).strip() 1147eb8dc403SDave Cobbley commit = old_tree != new_tree 1148eb8dc403SDave Cobbley if not commit: 1149eb8dc403SDave Cobbley new_rev = parent_rev 1150eb8dc403SDave Cobbley else: 1151eb8dc403SDave Cobbley commit = True 1152eb8dc403SDave Cobbley if commit: 1153eb8dc403SDave Cobbley new_rev = runcmd("git commit-tree".split() + add_p([import_rev(p) for p in parents]) + 1154eb8dc403SDave Cobbley ["-m", body, new_tree], 1155eb8dc403SDave Cobbley env=env).strip() 1156eb8dc403SDave Cobbley old2new_revs[rev] = new_rev 1157eb8dc403SDave Cobbley 1158eb8dc403SDave Cobbley return new_rev 1159eb8dc403SDave Cobbley 1160eb8dc403SDave Cobbley if revs: 1161eb8dc403SDave Cobbley for rev in revs: 1162eb8dc403SDave Cobbley import_rev(rev) 1163eb8dc403SDave Cobbley # Remember how to update our current head. New components get added, 1164eb8dc403SDave Cobbley # updated components get the delta between current head and the updated component 1165eb8dc403SDave Cobbley # applied. 1166eb8dc403SDave Cobbley additional_heads[old2new_revs[revs[-1]]] = head if repo['last_revision'] else None 1167eb8dc403SDave Cobbley repo['last_revision'] = revs[-1] 1168eb8dc403SDave Cobbley 1169eb8dc403SDave Cobbley # Now construct the final merge commit. We create the tree by 1170eb8dc403SDave Cobbley # starting with the head and applying the changes from each 1171eb8dc403SDave Cobbley # components imported head revision. 1172eb8dc403SDave Cobbley if additional_heads: 1173eb8dc403SDave Cobbley runcmd("git reset --hard", **wargs) 1174eb8dc403SDave Cobbley for rev, base in additional_heads.items(): 1175eb8dc403SDave Cobbley apply_commit(base, rev, wargs, wargs, None) 1176eb8dc403SDave Cobbley 1177eb8dc403SDave Cobbley # Commit with all component branches as parents as well as the previous head. 1178eb8dc403SDave Cobbley logger.info("Writing final merge commit...") 1179eb8dc403SDave Cobbley msg = conf_commit_msg(conf, components) 1180eb8dc403SDave Cobbley new_tree = runcmd("git write-tree", **wargs).strip() 1181eb8dc403SDave Cobbley new_rev = runcmd("git commit-tree".split() + 1182eb8dc403SDave Cobbley add_p([head] + list(additional_heads.keys())) + 1183eb8dc403SDave Cobbley ["-m", msg, new_tree], 1184eb8dc403SDave Cobbley **wargs).strip() 1185eb8dc403SDave Cobbley # And done! This is the first time we change the HEAD in the actual work tree. 1186eb8dc403SDave Cobbley runcmd("git reset --hard %s" % new_rev) 1187eb8dc403SDave Cobbley 1188eb8dc403SDave Cobbley # Update and stage the (potentially modified) 1189eb8dc403SDave Cobbley # combo-layer.conf, but do not commit separately. 1190eb8dc403SDave Cobbley for name in repos: 1191eb8dc403SDave Cobbley repo = conf.repos[name] 1192eb8dc403SDave Cobbley rev = repo['last_revision'] 1193eb8dc403SDave Cobbley conf.update(name, "last_revision", rev) 1194eb8dc403SDave Cobbley if commit_conf_file(conf, components, False): 1195eb8dc403SDave Cobbley # Must augment the previous commit. 1196eb8dc403SDave Cobbley runcmd("git commit --amend -C HEAD") 1197eb8dc403SDave Cobbley 1198eb8dc403SDave Cobbley 1199eb8dc403SDave Cobbleyscanned_revs = False 1200eb8dc403SDave Cobbleydef find_revs(old2new, head): 1201eb8dc403SDave Cobbley '''Construct mapping from original commit hash to commit hash in 1202eb8dc403SDave Cobbley combined repo by looking at the commit messages. Depends on the 1203eb8dc403SDave Cobbley "From ... rev: ..." convention.''' 1204eb8dc403SDave Cobbley logger.info("Analyzing log messages to find previously imported commits...") 1205eb8dc403SDave Cobbley num_known = len(old2new) 1206eb8dc403SDave Cobbley log = runcmd("git log --grep='From .* rev: [a-fA-F0-9][a-fA-F0-9]*' --pretty=format:%H%x00%B%x00 " + head).split(chr(0)) 1207eb8dc403SDave Cobbley regex = re.compile(r'From .* rev: ([a-fA-F0-9]+)') 1208eb8dc403SDave Cobbley for new_rev, body in zip(*[iter(log)]* 2): 1209eb8dc403SDave Cobbley # Use the last one, in the unlikely case there are more than one. 1210eb8dc403SDave Cobbley rev = regex.findall(body)[-1] 1211eb8dc403SDave Cobbley if rev not in old2new: 1212eb8dc403SDave Cobbley old2new[rev] = new_rev.strip() 1213eb8dc403SDave Cobbley logger.info("Found %d additional commits, leading to: %s" % (len(old2new) - num_known, old2new)) 1214eb8dc403SDave Cobbley 1215eb8dc403SDave Cobbley 1216eb8dc403SDave Cobbleydef apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=None): 1217eb8dc403SDave Cobbley '''Compare revision against parent, remove files deleted in the 1218eb8dc403SDave Cobbley commit, re-write new or modified ones. Moves them into dest_dir. 1219eb8dc403SDave Cobbley Optionally filters files. 1220eb8dc403SDave Cobbley ''' 1221eb8dc403SDave Cobbley if not dest_dir: 1222eb8dc403SDave Cobbley dest_dir = "." 1223eb8dc403SDave Cobbley # -r recurses into sub-directories, given is the full overview of 1224eb8dc403SDave Cobbley # what changed. We do not care about copy/edits or renames, so we 1225eb8dc403SDave Cobbley # can disable those with --no-renames (but we still parse them, 1226eb8dc403SDave Cobbley # because it was not clear from git documentation whether C and M 1227eb8dc403SDave Cobbley # lines can still occur). 1228eb8dc403SDave Cobbley logger.debug("Applying changes between %s and %s in %s" % (parent, rev, largs["destdir"])) 1229eb8dc403SDave Cobbley delete = [] 1230eb8dc403SDave Cobbley update = [] 1231eb8dc403SDave Cobbley if parent: 1232eb8dc403SDave Cobbley # Apply delta. 1233eb8dc403SDave Cobbley changes = runcmd("git diff-tree --no-commit-id --no-renames --name-status -r --raw -z %s %s" % (parent, rev), **largs).split(chr(0)) 1234eb8dc403SDave Cobbley for status, name in zip(*[iter(changes)]*2): 1235eb8dc403SDave Cobbley if status[0] in "ACMRT": 1236eb8dc403SDave Cobbley update.append(name) 1237eb8dc403SDave Cobbley elif status[0] in "D": 1238eb8dc403SDave Cobbley delete.append(name) 1239eb8dc403SDave Cobbley else: 1240eb8dc403SDave Cobbley logger.error("Unknown status %s of file %s in revision %s" % (status, name, rev)) 1241eb8dc403SDave Cobbley sys.exit(1) 1242eb8dc403SDave Cobbley else: 1243eb8dc403SDave Cobbley # Copy all files. 1244eb8dc403SDave Cobbley update.extend(runcmd("git ls-tree -r --name-only -z %s" % rev, **largs).split(chr(0))) 1245eb8dc403SDave Cobbley 1246eb8dc403SDave Cobbley # Include/exclude files as define in the component config. 1247eb8dc403SDave Cobbley # Both updated and deleted file lists get filtered, because it might happen 1248eb8dc403SDave Cobbley # that a file gets excluded, pulled from a different component, and then the 1249eb8dc403SDave Cobbley # excluded file gets deleted. In that case we must keep the copy. 1250eb8dc403SDave Cobbley if file_filter: 1251eb8dc403SDave Cobbley file_filter(update) 1252eb8dc403SDave Cobbley file_filter(delete) 1253eb8dc403SDave Cobbley 1254eb8dc403SDave Cobbley # We export into a tar archive here and extract with tar because it is simple (no 1255eb8dc403SDave Cobbley # need to implement file and symlink writing ourselves) and gives us some degree 1256eb8dc403SDave Cobbley # of parallel IO. The downside is that we have to pass the list of files via 1257eb8dc403SDave Cobbley # command line parameters - hopefully there will never be too many at once. 1258eb8dc403SDave Cobbley if update: 1259eb8dc403SDave Cobbley target = os.path.join(wargs["destdir"], dest_dir) 1260eb8dc403SDave Cobbley if not os.path.isdir(target): 1261eb8dc403SDave Cobbley os.makedirs(target) 1262eb8dc403SDave Cobbley quoted_target = pipes.quote(target) 1263eb8dc403SDave Cobbley # os.sysconf('SC_ARG_MAX') is lying: running a command with 1264eb8dc403SDave Cobbley # string length 629343 already failed with "Argument list too 1265eb8dc403SDave Cobbley # long" although SC_ARG_MAX = 2097152. "man execve" explains 1266eb8dc403SDave Cobbley # the limitations, but those are pretty complicated. So here 1267eb8dc403SDave Cobbley # we just hard-code a fixed value which is more likely to work. 1268eb8dc403SDave Cobbley max_cmdsize = 64 * 1024 1269eb8dc403SDave Cobbley while update: 1270eb8dc403SDave Cobbley quoted_args = [] 1271eb8dc403SDave Cobbley unquoted_args = [] 1272eb8dc403SDave Cobbley cmdsize = 100 + len(quoted_target) 1273eb8dc403SDave Cobbley while update: 1274eb8dc403SDave Cobbley quoted_next = pipes.quote(update[0]) 1275eb8dc403SDave Cobbley size_next = len(quoted_next) + len(dest_dir) + 1 1276eb8dc403SDave Cobbley logger.debug('cmdline length %d + %d < %d?' % (cmdsize, size_next, os.sysconf('SC_ARG_MAX'))) 1277eb8dc403SDave Cobbley if cmdsize + size_next < max_cmdsize: 1278eb8dc403SDave Cobbley quoted_args.append(quoted_next) 1279eb8dc403SDave Cobbley unquoted_args.append(update.pop(0)) 1280eb8dc403SDave Cobbley cmdsize += size_next 1281eb8dc403SDave Cobbley else: 1282eb8dc403SDave Cobbley logger.debug('Breaking the cmdline at length %d' % cmdsize) 1283eb8dc403SDave Cobbley break 1284eb8dc403SDave Cobbley logger.debug('Final cmdline length %d / %d' % (cmdsize, os.sysconf('SC_ARG_MAX'))) 1285eb8dc403SDave Cobbley cmd = "git archive %s %s | tar -C %s -xf -" % (rev, ' '.join(quoted_args), quoted_target) 1286eb8dc403SDave Cobbley logger.debug('First cmdline length %d' % len(cmd)) 1287eb8dc403SDave Cobbley runcmd(cmd, **largs) 1288eb8dc403SDave Cobbley cmd = "git add -f".split() + [os.path.join(dest_dir, x) for x in unquoted_args] 1289eb8dc403SDave Cobbley logger.debug('Second cmdline length %d' % reduce(lambda x, y: x + len(y), cmd, 0)) 1290eb8dc403SDave Cobbley runcmd(cmd, **wargs) 1291eb8dc403SDave Cobbley if delete: 1292eb8dc403SDave Cobbley for path in delete: 1293eb8dc403SDave Cobbley if dest_dir: 1294eb8dc403SDave Cobbley path = os.path.join(dest_dir, path) 1295eb8dc403SDave Cobbley runcmd("git rm -f --ignore-unmatch".split() + [os.path.join(dest_dir, x) for x in delete], **wargs) 1296eb8dc403SDave Cobbley 1297eb8dc403SDave Cobbleydef action_error(conf, args): 1298eb8dc403SDave Cobbley logger.info("invalid action %s" % args[0]) 1299eb8dc403SDave Cobbley 1300eb8dc403SDave Cobbleyactions = { 1301eb8dc403SDave Cobbley "init": action_init, 1302eb8dc403SDave Cobbley "update": action_update, 1303eb8dc403SDave Cobbley "pull": action_pull, 1304eb8dc403SDave Cobbley "splitpatch": action_splitpatch, 1305eb8dc403SDave Cobbley} 1306eb8dc403SDave Cobbley 1307eb8dc403SDave Cobbleydef main(): 1308eb8dc403SDave Cobbley parser = optparse.OptionParser( 1309eb8dc403SDave Cobbley version = "Combo Layer Repo Tool version %s" % __version__, 1310eb8dc403SDave Cobbley usage = """%prog [options] action 1311eb8dc403SDave Cobbley 1312eb8dc403SDave CobbleyCreate and update a combination layer repository from multiple component repositories. 1313eb8dc403SDave Cobbley 1314eb8dc403SDave CobbleyAction: 1315eb8dc403SDave Cobbley init initialise the combo layer repo 1316eb8dc403SDave Cobbley update [components] get patches from component repos and apply them to the combo repo 1317eb8dc403SDave Cobbley pull [components] just pull component repos only 1318eb8dc403SDave Cobbley splitpatch [commit] generate commit patch and split per component, default commit is HEAD""") 1319eb8dc403SDave Cobbley 1320eb8dc403SDave Cobbley parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).", 1321eb8dc403SDave Cobbley action = "store", dest = "conffile", default = "conf/combo-layer.conf") 1322eb8dc403SDave Cobbley 1323eb8dc403SDave Cobbley parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches", 1324eb8dc403SDave Cobbley action = "store_true", dest = "interactive", default = False) 1325eb8dc403SDave Cobbley 1326eb8dc403SDave Cobbley parser.add_option("-D", "--debug", help = "output debug information", 1327eb8dc403SDave Cobbley action = "store_true", dest = "debug", default = False) 1328eb8dc403SDave Cobbley 1329eb8dc403SDave Cobbley parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update", 1330eb8dc403SDave Cobbley action = "store_true", dest = "nopull", default = False) 1331eb8dc403SDave Cobbley 1332eb8dc403SDave Cobbley parser.add_option("--hard-reset", 1333eb8dc403SDave Cobbley help = "instead of pull do fetch and hard-reset in component repos", 1334eb8dc403SDave Cobbley action = "store_true", dest = "hard_reset", default = False) 1335eb8dc403SDave Cobbley 1336eb8dc403SDave Cobbley parser.add_option("-H", "--history", help = "import full history of components during init", 1337eb8dc403SDave Cobbley action = "store_true", default = False) 1338eb8dc403SDave Cobbley 1339eb8dc403SDave Cobbley options, args = parser.parse_args(sys.argv) 1340eb8dc403SDave Cobbley 1341eb8dc403SDave Cobbley # Dispatch to action handler 1342eb8dc403SDave Cobbley if len(args) == 1: 1343eb8dc403SDave Cobbley logger.error("No action specified, exiting") 1344eb8dc403SDave Cobbley parser.print_help() 1345eb8dc403SDave Cobbley elif args[1] not in actions: 1346eb8dc403SDave Cobbley logger.error("Unsupported action %s, exiting\n" % (args[1])) 1347eb8dc403SDave Cobbley parser.print_help() 1348eb8dc403SDave Cobbley elif not os.path.exists(options.conffile): 1349eb8dc403SDave Cobbley logger.error("No valid config file, exiting\n") 1350eb8dc403SDave Cobbley parser.print_help() 1351eb8dc403SDave Cobbley else: 1352eb8dc403SDave Cobbley if options.debug: 1353eb8dc403SDave Cobbley logger.setLevel(logging.DEBUG) 1354eb8dc403SDave Cobbley confdata = Configuration(options) 1355eb8dc403SDave Cobbley initmode = (args[1] == 'init') 1356eb8dc403SDave Cobbley confdata.sanity_check(initmode) 1357eb8dc403SDave Cobbley actions.get(args[1], action_error)(confdata, args[1:]) 1358eb8dc403SDave Cobbley 1359eb8dc403SDave Cobbleyif __name__ == "__main__": 1360eb8dc403SDave Cobbley try: 1361eb8dc403SDave Cobbley ret = main() 1362eb8dc403SDave Cobbley except Exception: 1363eb8dc403SDave Cobbley ret = 1 1364eb8dc403SDave Cobbley import traceback 1365eb8dc403SDave Cobbley traceback.print_exc() 1366eb8dc403SDave Cobbley sys.exit(ret) 1367