1#!/usr/bin/env python 2# ex:ts=4:sw=4:sts=4:et 3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 4# 5# Copyright 2011 Intel Corporation 6# Authored-by: Yu Ke <ke.yu@intel.com> 7# Paul Eggleton <paul.eggleton@intel.com> 8# Richard Purdie <richard.purdie@intel.com> 9# 10# This program is free software; you can redistribute it and/or modify 11# it under the terms of the GNU General Public License version 2 as 12# published by the Free Software Foundation. 13# 14# This program is distributed in the hope that it will be useful, 15# but WITHOUT ANY WARRANTY; without even the implied warranty of 16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17# GNU General Public License for more details. 18# 19# You should have received a copy of the GNU General Public License along 20# with this program; if not, write to the Free Software Foundation, Inc., 21# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 22 23import fnmatch 24import os, sys 25import optparse 26import logging 27import subprocess 28import tempfile 29import ConfigParser 30import re 31from collections import OrderedDict 32from string import Template 33 34__version__ = "0.2.1" 35 36def logger_create(): 37 logger = logging.getLogger("") 38 loggerhandler = logging.StreamHandler() 39 loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S")) 40 logger.addHandler(loggerhandler) 41 logger.setLevel(logging.INFO) 42 return logger 43 44logger = logger_create() 45 46def get_current_branch(repodir=None): 47 try: 48 if not os.path.exists(os.path.join(repodir if repodir else '', ".git")): 49 # Repo not created yet (i.e. during init) so just assume master 50 return "master" 51 branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip() 52 if branchname.startswith("refs/heads/"): 53 branchname = branchname[11:] 54 return branchname 55 except subprocess.CalledProcessError: 56 return "" 57 58class Configuration(object): 59 """ 60 Manages the configuration 61 62 For an example config file, see combo-layer.conf.example 63 64 """ 65 def __init__(self, options): 66 for key, val in options.__dict__.items(): 67 setattr(self, key, val) 68 69 def readsection(parser, section, repo): 70 for (name, value) in parser.items(section): 71 if value.startswith("@"): 72 self.repos[repo][name] = eval(value.strip("@")) 73 else: 74 # Apply special type transformations for some properties. 75 # Type matches the RawConfigParser.get*() methods. 76 types = {'signoff': 'boolean'} 77 if name in types: 78 value = getattr(parser, 'get' + types[name])(section, name) 79 self.repos[repo][name] = value 80 81 def readglobalsection(parser, section): 82 for (name, value) in parser.items(section): 83 if name == "commit_msg": 84 self.commit_msg_template = value 85 86 logger.debug("Loading config file %s" % self.conffile) 87 self.parser = ConfigParser.ConfigParser() 88 with open(self.conffile) as f: 89 self.parser.readfp(f) 90 91 # initialize default values 92 self.commit_msg_template = "Automatic commit to update last_revision" 93 94 self.repos = {} 95 for repo in self.parser.sections(): 96 if repo == "combo-layer-settings": 97 # special handling for global settings 98 readglobalsection(self.parser, repo) 99 else: 100 self.repos[repo] = {} 101 readsection(self.parser, repo, repo) 102 103 # Load local configuration, if available 104 self.localconffile = None 105 self.localparser = None 106 self.combobranch = None 107 if self.conffile.endswith('.conf'): 108 lcfile = self.conffile.replace('.conf', '-local.conf') 109 if os.path.exists(lcfile): 110 # Read combo layer branch 111 self.combobranch = get_current_branch() 112 logger.debug("Combo layer branch is %s" % self.combobranch) 113 114 self.localconffile = lcfile 115 logger.debug("Loading local config file %s" % self.localconffile) 116 self.localparser = ConfigParser.ConfigParser() 117 with open(self.localconffile) as f: 118 self.localparser.readfp(f) 119 120 for section in self.localparser.sections(): 121 if '|' in section: 122 sectionvals = section.split('|') 123 repo = sectionvals[0] 124 if sectionvals[1] != self.combobranch: 125 continue 126 else: 127 repo = section 128 if repo in self.repos: 129 readsection(self.localparser, section, repo) 130 131 def update(self, repo, option, value, initmode=False): 132 # If the main config has the option already, that is what we 133 # are expected to modify. 134 if self.localparser and not self.parser.has_option(repo, option): 135 parser = self.localparser 136 section = "%s|%s" % (repo, self.combobranch) 137 conffile = self.localconffile 138 if initmode and not parser.has_section(section): 139 parser.add_section(section) 140 else: 141 parser = self.parser 142 section = repo 143 conffile = self.conffile 144 parser.set(section, option, value) 145 with open(conffile, "w") as f: 146 parser.write(f) 147 self.repos[repo][option] = value 148 149 def sanity_check(self, initmode=False): 150 required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"] 151 if initmode: 152 required_options.remove("last_revision") 153 msg = "" 154 missing_options = [] 155 for name in self.repos: 156 for option in required_options: 157 if option not in self.repos[name]: 158 msg = "%s\nOption %s is not defined for component %s" %(msg, option, name) 159 missing_options.append(option) 160 # Sanitize dest_dir so that we do not have to deal with edge cases 161 # (unset, empty string, double slashes) in the rest of the code. 162 # It not being set will still be flagged as error because it is 163 # listed as required option above; that could be changed now. 164 dest_dir = os.path.normpath(self.repos[name].get("dest_dir", ".")) 165 self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir 166 if msg != "": 167 logger.error("configuration file %s has the following error: %s" % (self.conffile,msg)) 168 if self.localconffile and 'last_revision' in missing_options: 169 logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch)) 170 sys.exit(1) 171 172 # filterdiff is required by action_splitpatch, so check its availability 173 if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0: 174 logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)") 175 sys.exit(1) 176 177def runcmd(cmd,destdir=None,printerr=True,out=None): 178 """ 179 execute command, raise CalledProcessError if fail 180 return output if succeed 181 """ 182 logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir)) 183 if not out: 184 out = os.tmpfile() 185 err = out 186 else: 187 err = os.tmpfile() 188 try: 189 subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str)) 190 except subprocess.CalledProcessError,e: 191 err.seek(0) 192 if printerr: 193 logger.error("%s" % err.read()) 194 raise e 195 196 err.seek(0) 197 output = err.read() 198 logger.debug("output: %s" % output ) 199 return output 200 201def action_init(conf, args): 202 """ 203 Clone component repositories 204 Check git is initialised; if not, copy initial data from component repos 205 """ 206 for name in conf.repos: 207 ldir = conf.repos[name]['local_repo_dir'] 208 if not os.path.exists(ldir): 209 logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir)) 210 subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True) 211 if not os.path.exists(".git"): 212 runcmd("git init") 213 if conf.history: 214 # Need a common ref for all trees. 215 runcmd('git commit -m "initial empty commit" --allow-empty') 216 startrev = runcmd('git rev-parse master').strip() 217 218 for name in conf.repos: 219 repo = conf.repos[name] 220 ldir = repo['local_repo_dir'] 221 branch = repo.get('branch', "master") 222 lastrev = repo.get('last_revision', None) 223 if lastrev and lastrev != "HEAD": 224 initialrev = lastrev 225 if branch: 226 if not check_rev_branch(name, ldir, lastrev, branch): 227 sys.exit(1) 228 logger.info("Copying data from %s at specified revision %s..." % (name, lastrev)) 229 else: 230 lastrev = None 231 initialrev = branch 232 logger.info("Copying data from %s..." % name) 233 # Sanity check initialrev and turn it into hash (required for copying history, 234 # because resolving a name ref only works in the component repo). 235 rev = runcmd('git rev-parse %s' % initialrev, ldir).strip() 236 if rev != initialrev: 237 try: 238 refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n') 239 if len(set(refs)) > 1: 240 # Happens for example when configured to track 241 # "master" and there is a refs/heads/master. The 242 # traditional behavior from "git archive" (preserved 243 # here) it to choose the first one. This might not be 244 # intended, so at least warn about it. 245 logger.warn("%s: initial revision '%s' not unique, picking result of rev-parse = %s" % 246 (name, initialrev, refs[0])) 247 initialrev = rev 248 except: 249 # show-ref fails for hashes. Skip the sanity warning in that case. 250 pass 251 initialrev = rev 252 dest_dir = repo['dest_dir'] 253 if dest_dir != ".": 254 extract_dir = os.path.join(os.getcwd(), dest_dir) 255 if not os.path.exists(extract_dir): 256 os.makedirs(extract_dir) 257 else: 258 extract_dir = os.getcwd() 259 file_filter = repo.get('file_filter', "") 260 exclude_patterns = repo.get('file_exclude', '').split() 261 def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir, 262 subdir=""): 263 # When working inside a filtered branch which had the 264 # files already moved, we need to prepend the 265 # subdirectory to all filters, otherwise they would 266 # not match. 267 if subdir == '.': 268 subdir = '' 269 elif subdir: 270 subdir = os.path.normpath(subdir) 271 file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()]) 272 exclude_patterns = [subdir + '/' + x for x in exclude_patterns] 273 # To handle both cases, we cd into the target 274 # directory and optionally tell tar to strip the path 275 # prefix when the files were already moved. 276 subdir_components = len(subdir.split(os.path.sep)) if subdir else 0 277 strip=('--strip-components=%d' % subdir_components) if subdir else '' 278 # TODO: file_filter wild cards do not work (and haven't worked before either), because 279 # a) GNU tar requires a --wildcards parameter before turning on wild card matching. 280 # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c, 281 # in contrast to the other use of file_filter as parameter of "git archive" 282 # where it only matches .c files directly in src). 283 files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" % 284 (initialrev, subdir, 285 strip, extract_dir, file_filter), 286 ldir) 287 if exclude_patterns: 288 # Implement file removal by letting tar create the 289 # file and then deleting it in the file system 290 # again. Uses the list of files created by tar (easier 291 # than walking the tree). 292 for file in files.split('\n'): 293 for pattern in exclude_patterns: 294 if fnmatch.fnmatch(file, pattern): 295 os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file]))) 296 break 297 298 if not conf.history: 299 copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir) 300 else: 301 # First fetch remote history into local repository. 302 # We need a ref for that, so ensure that there is one. 303 refname = "combo-layer-init-%s" % name 304 runcmd("git branch -f %s %s" % (refname, initialrev), ldir) 305 runcmd("git fetch %s %s" % (ldir, refname)) 306 runcmd("git branch -D %s" % refname, ldir) 307 # Make that the head revision. 308 runcmd("git checkout -b %s %s" % (name, initialrev)) 309 # Optional: cut the history by replacing the given 310 # start point(s) with commits providing the same 311 # content (aka tree), but with commit information that 312 # makes it clear that this is an artifically created 313 # commit and nothing the original authors had anything 314 # to do with. 315 since_rev = repo.get('since_revision', '') 316 if since_rev: 317 committer = runcmd('git var GIT_AUTHOR_IDENT').strip() 318 # Same time stamp, no name. 319 author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer) 320 logger.info('author %s' % author) 321 for rev in since_rev.split(): 322 # Resolve in component repo... 323 rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0] 324 # ... and then get the tree in current 325 # one. The commit should be in both repos with 326 # the same tree, but better check here. 327 tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip() 328 with tempfile.NamedTemporaryFile() as editor: 329 editor.write('''cat >$1 <<EOF 330tree %s 331author %s 332committer %s 333 334%s: squashed import of component 335 336This commit copies the entire set of files as found in 337%s %s 338 339For more information about previous commits, see the 340upstream repository. 341 342Commit created by combo-layer. 343EOF 344''' % (tree, author, committer, name, name, since_rev)) 345 editor.flush() 346 os.environ['GIT_EDITOR'] = 'sh %s' % editor.name 347 runcmd('git replace --edit %s' % rev) 348 349 # Optional: rewrite history to change commit messages or to move files. 350 if 'hook' in repo or dest_dir != ".": 351 filter_branch = ['git', 'filter-branch', '--force'] 352 with tempfile.NamedTemporaryFile() as hookwrapper: 353 if 'hook' in repo: 354 # Create a shell script wrapper around the original hook that 355 # can be used by git filter-branch. Hook may or may not have 356 # an absolute path. 357 hook = repo['hook'] 358 hook = os.path.join(os.path.dirname(conf.conffile), '..', hook) 359 # The wrappers turns the commit message 360 # from stdin into a fake patch header. 361 # This is good enough for changing Subject 362 # and commit msg body with normal 363 # combo-layer hooks. 364 hookwrapper.write('''set -e 365tmpname=$(mktemp) 366trap "rm $tmpname" EXIT 367echo -n 'Subject: [PATCH] ' >>$tmpname 368cat >>$tmpname 369if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then 370 echo >>$tmpname 371fi 372echo '---' >>$tmpname 373%s $tmpname $GIT_COMMIT %s 374tail -c +18 $tmpname | head -c -4 375''' % (hook, name)) 376 hookwrapper.flush() 377 filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name]) 378 if dest_dir != ".": 379 parent = os.path.dirname(dest_dir) 380 if not parent: 381 parent = '.' 382 # May run outside of the current directory, so do not assume that .git exists. 383 filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && mv $(ls -1 -a | grep -v -e ^.git$ -e ^.$ -e ^..$) .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)]) 384 filter_branch.append('HEAD') 385 runcmd(filter_branch) 386 runcmd('git update-ref -d refs/original/refs/heads/%s' % name) 387 repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip() 388 repo['stripped_revision'] = repo['rewritten_revision'] 389 # Optional filter files: remove everything and re-populate using the normal filtering code. 390 # Override any potential .gitignore. 391 if file_filter or exclude_patterns: 392 runcmd('git rm -rf .') 393 if not os.path.exists(extract_dir): 394 os.makedirs(extract_dir) 395 copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.', 396 subdir=dest_dir) 397 runcmd('git add --all --force .') 398 if runcmd('git status --porcelain'): 399 # Something to commit. 400 runcmd(['git', 'commit', '-m', 401 '''%s: select file subset 402 403Files from the component repository were chosen based on 404the following filters: 405file_filter = %s 406file_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))]) 407 repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip() 408 409 if not lastrev: 410 lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip() 411 conf.update(name, "last_revision", lastrev, initmode=True) 412 413 if not conf.history: 414 runcmd("git add .") 415 else: 416 # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies 417 runcmd('git checkout master') 418 merge = ['git', 'merge', '--no-commit'] 419 for name in conf.repos: 420 repo = conf.repos[name] 421 # Use branch created earlier. 422 merge.append(name) 423 # Root all commits which have no parent in the common 424 # ancestor in the new repository. 425 for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s' % name).split('\n'): 426 runcmd('git replace --graft %s %s' % (start, startrev)) 427 try: 428 runcmd(merge) 429 except Exception, error: 430 logger.info('''Merging component repository history failed, perhaps because of merge conflicts. 431It may be possible to commit anyway after resolving these conflicts. 432 433%s''' % error) 434 # Create MERGE_HEAD and MERGE_MSG. "git merge" itself 435 # does not create MERGE_HEAD in case of a (harmless) failure, 436 # and we want certain auto-generated information in the 437 # commit message for future reference and/or automation. 438 with open('.git/MERGE_HEAD', 'w') as head: 439 with open('.git/MERGE_MSG', 'w') as msg: 440 msg.write('repo: initial import of components\n\n') 441 # head.write('%s\n' % startrev) 442 for name in conf.repos: 443 repo = conf.repos[name] 444 # <upstream ref> <rewritten ref> <rewritten + files removed> 445 msg.write('combo-layer-%s: %s %s %s\n' % (name, 446 repo['last_revision'], 447 repo['rewritten_revision'], 448 repo['stripped_revision'])) 449 rev = runcmd('git rev-parse %s' % name).strip() 450 head.write('%s\n' % rev) 451 452 if conf.localconffile: 453 localadded = True 454 try: 455 runcmd("git rm --cached %s" % conf.localconffile, printerr=False) 456 except subprocess.CalledProcessError: 457 localadded = False 458 if localadded: 459 localrelpath = os.path.relpath(conf.localconffile) 460 runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath)) 461 runcmd("git add .gitignore") 462 logger.info("Added local configuration file %s to .gitignore", localrelpath) 463 logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.") 464 else: 465 logger.info("Repository already initialised, nothing to do.") 466 467 468def check_repo_clean(repodir): 469 """ 470 check if the repo is clean 471 exit if repo is dirty 472 """ 473 output=runcmd("git status --porcelain", repodir) 474 r = re.compile('\?\? patch-.*/') 475 dirtyout = [item for item in output.splitlines() if not r.match(item)] 476 if dirtyout: 477 logger.error("git repo %s is dirty, please fix it first", repodir) 478 sys.exit(1) 479 480def check_patch(patchfile): 481 f = open(patchfile) 482 ln = f.readline() 483 of = None 484 in_patch = False 485 beyond_msg = False 486 pre_buf = '' 487 while ln: 488 if not beyond_msg: 489 if ln == '---\n': 490 if not of: 491 break 492 in_patch = False 493 beyond_msg = True 494 elif ln.startswith('--- '): 495 # We have a diff in the commit message 496 in_patch = True 497 if not of: 498 print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile) 499 of = open(patchfile + '.tmp', 'w') 500 of.write(pre_buf) 501 pre_buf = '' 502 elif in_patch and not ln[0] in '+-@ \n\r': 503 in_patch = False 504 if of: 505 if in_patch: 506 of.write(' ' + ln) 507 else: 508 of.write(ln) 509 else: 510 pre_buf += ln 511 ln = f.readline() 512 f.close() 513 if of: 514 of.close() 515 os.rename(patchfile + '.tmp', patchfile) 516 517def drop_to_shell(workdir=None): 518 if not sys.stdin.isatty(): 519 print "Not a TTY so can't drop to shell for resolution, exiting." 520 return False 521 522 shell = os.environ.get('SHELL', 'bash') 523 print('Dropping to shell "%s"\n' \ 524 'When you are finished, run the following to continue:\n' \ 525 ' exit -- continue to apply the patches\n' \ 526 ' exit 1 -- abort\n' % shell); 527 ret = subprocess.call([shell], cwd=workdir) 528 if ret != 0: 529 print "Aborting" 530 return False 531 else: 532 return True 533 534def check_rev_branch(component, repodir, rev, branch): 535 try: 536 actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False) 537 except subprocess.CalledProcessError as e: 538 if e.returncode == 129: 539 actualbranch = "" 540 else: 541 raise 542 543 if not actualbranch: 544 logger.error("%s: specified revision %s is invalid!" % (component, rev)) 545 return False 546 547 branches = [] 548 branchlist = actualbranch.split("\n") 549 for b in branchlist: 550 branches.append(b.strip().split(' ')[-1]) 551 552 if branch not in branches: 553 logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch)) 554 return False 555 return True 556 557def get_repos(conf, repo_names): 558 repos = [] 559 for name in repo_names: 560 if name.startswith('-'): 561 break 562 else: 563 repos.append(name) 564 for repo in repos: 565 if not repo in conf.repos: 566 logger.error("Specified component '%s' not found in configuration" % repo) 567 sys.exit(1) 568 569 if not repos: 570 repos = conf.repos 571 572 return repos 573 574def action_pull(conf, args): 575 """ 576 update the component repos only 577 """ 578 repos = get_repos(conf, args[1:]) 579 580 # make sure all repos are clean 581 for name in repos: 582 check_repo_clean(conf.repos[name]['local_repo_dir']) 583 584 for name in repos: 585 repo = conf.repos[name] 586 ldir = repo['local_repo_dir'] 587 branch = repo.get('branch', "master") 588 logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir)) 589 if not conf.hard_reset: 590 # Try to pull only the configured branch. Beware that this may fail 591 # when the branch is currently unknown (for example, after reconfiguring 592 # combo-layer). In that case we need to fetch everything and try the check out 593 # and pull again. 594 try: 595 runcmd("git checkout %s" % branch, ldir, printerr=False) 596 except subprocess.CalledProcessError: 597 output=runcmd("git fetch", ldir) 598 logger.info(output) 599 runcmd("git checkout %s" % branch, ldir) 600 runcmd("git pull --ff-only", ldir) 601 else: 602 output=runcmd("git pull --ff-only", ldir) 603 logger.info(output) 604 else: 605 output=runcmd("git fetch", ldir) 606 logger.info(output) 607 runcmd("git checkout %s" % branch, ldir) 608 runcmd("git reset --hard FETCH_HEAD", ldir) 609 610def action_update(conf, args): 611 """ 612 update the component repos 613 generate the patch list 614 apply the generated patches 615 """ 616 components = [arg.split(':')[0] for arg in args[1:]] 617 revisions = {} 618 for arg in args[1:]: 619 if ':' in arg: 620 a = arg.split(':', 1) 621 revisions[a[0]] = a[1] 622 repos = get_repos(conf, components) 623 624 # make sure combo repo is clean 625 check_repo_clean(os.getcwd()) 626 627 import uuid 628 patch_dir = "patch-%s" % uuid.uuid4() 629 if not os.path.exists(patch_dir): 630 os.mkdir(patch_dir) 631 632 # Step 1: update the component repos 633 if conf.nopull: 634 logger.info("Skipping pull (-n)") 635 else: 636 action_pull(conf, ['arg0'] + components) 637 638 for name in repos: 639 revision = revisions.get(name, None) 640 repo = conf.repos[name] 641 ldir = repo['local_repo_dir'] 642 dest_dir = repo['dest_dir'] 643 branch = repo.get('branch', "master") 644 repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name) 645 646 # Step 2: generate the patch list and store to patch dir 647 logger.info("Generating patches from %s..." % name) 648 top_revision = revision or branch 649 if not check_rev_branch(name, ldir, top_revision, branch): 650 sys.exit(1) 651 if dest_dir != ".": 652 prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir) 653 else: 654 prefix = "" 655 if repo['last_revision'] == "": 656 logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name) 657 patch_cmd_range = "--root %s" % top_revision 658 rev_cmd_range = top_revision 659 else: 660 if not check_rev_branch(name, ldir, repo['last_revision'], branch): 661 sys.exit(1) 662 patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision) 663 rev_cmd_range = patch_cmd_range 664 665 file_filter = repo.get('file_filter',"") 666 667 patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \ 668 (prefix,repo_patch_dir, patch_cmd_range, file_filter) 669 output = runcmd(patch_cmd, ldir) 670 logger.debug("generated patch set:\n%s" % output) 671 patchlist = output.splitlines() 672 673 rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter) 674 revlist = runcmd(rev_cmd, ldir).splitlines() 675 676 # Step 3: Call repo specific hook to adjust patch 677 if 'hook' in repo: 678 # hook parameter is: ./hook patchpath revision reponame 679 count=len(revlist)-1 680 for patch in patchlist: 681 runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name)) 682 count=count-1 683 684 # Step 3a: Filter out unwanted files and patches. 685 exclude = repo.get('file_exclude', '') 686 if exclude: 687 filter = ['filterdiff', '-p1'] 688 for path in exclude.split(): 689 filter.append('-x') 690 filter.append('%s/%s' % (dest_dir, path) if dest_dir != '.' else path) 691 for patch in patchlist[:]: 692 filtered = patch + '.tmp' 693 with open(filtered, 'w') as f: 694 runcmd(filter + [patch], out=f) 695 # Now check for empty patches. 696 if runcmd(['filterdiff', '--list', filtered]): 697 # Possibly modified. 698 os.unlink(patch) 699 os.rename(filtered, patch) 700 else: 701 # Empty, ignore it. Must also remove from revlist. 702 with open(patch, 'r') as f: 703 fromline = f.readline() 704 if not fromline: 705 # Patch must have been empty to start with. No need 706 # to remove it. 707 continue 708 m = re.match(r'''^From ([0-9a-fA-F]+) .*\n''', fromline) 709 rev = m.group(1) 710 logger.debug('skipping empty patch %s = %s' % (patch, rev)) 711 os.unlink(patch) 712 os.unlink(filtered) 713 patchlist.remove(patch) 714 revlist.remove(rev) 715 716 # Step 4: write patch list and revision list to file, for user to edit later 717 patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name) 718 repo['patchlist'] = patchlist_file 719 f = open(patchlist_file, 'w') 720 count=len(revlist)-1 721 for patch in patchlist: 722 f.write("%s %s\n" % (patch, revlist[count])) 723 check_patch(os.path.join(patch_dir, patch)) 724 count=count-1 725 f.close() 726 727 # Step 5: invoke bash for user to edit patch and patch list 728 if conf.interactive: 729 print('You may now edit the patch and patch list in %s\n' \ 730 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir); 731 if not drop_to_shell(patch_dir): 732 sys.exit(1) 733 734 # Step 6: apply the generated and revised patch 735 apply_patchlist(conf, repos) 736 runcmd("rm -rf %s" % patch_dir) 737 738 # Step 7: commit the updated config file if it's being tracked 739 relpath = os.path.relpath(conf.conffile) 740 try: 741 output = runcmd("git status --porcelain %s" % relpath, printerr=False) 742 except: 743 # Outside the repository 744 output = None 745 if output: 746 logger.info("Committing updated configuration file") 747 if output.lstrip().startswith("M"): 748 749 # create the "components" string 750 component_str = "all components" 751 if len(components) > 0: 752 # otherwise tell which components were actually changed 753 component_str = ", ".join(components) 754 755 # expand the template with known values 756 template = Template(conf.commit_msg_template) 757 raw_msg = template.substitute(components = component_str) 758 759 # sanitize the string before using it in command line 760 msg = raw_msg.replace('"', '\\"') 761 762 runcmd('git commit -m "%s" %s' % (msg, relpath)) 763 764def apply_patchlist(conf, repos): 765 """ 766 apply the generated patch list to combo repo 767 """ 768 for name in repos: 769 repo = conf.repos[name] 770 lastrev = repo["last_revision"] 771 prevrev = lastrev 772 773 # Get non-blank lines from patch list file 774 patchlist = [] 775 if os.path.exists(repo['patchlist']) or not conf.interactive: 776 # Note: we want this to fail here if the file doesn't exist and we're not in 777 # interactive mode since the file should exist in this case 778 with open(repo['patchlist']) as f: 779 for line in f: 780 line = line.rstrip() 781 if line: 782 patchlist.append(line) 783 784 ldir = conf.repos[name]['local_repo_dir'] 785 branch = conf.repos[name].get('branch', "master") 786 branchrev = runcmd("git rev-parse %s" % branch, ldir).strip() 787 788 if patchlist: 789 logger.info("Applying patches from %s..." % name) 790 linecount = len(patchlist) 791 i = 1 792 for line in patchlist: 793 patchfile = line.split()[0] 794 lastrev = line.split()[1] 795 patchdisp = os.path.relpath(patchfile) 796 if os.path.getsize(patchfile) == 0: 797 logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp)) 798 else: 799 cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile) 800 logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp)) 801 try: 802 runcmd(cmd) 803 except subprocess.CalledProcessError: 804 logger.info('Running "git am --abort" to cleanup repo') 805 runcmd("git am --abort") 806 logger.error('"%s" failed' % cmd) 807 logger.info("Please manually apply patch %s" % patchdisp) 808 logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped") 809 if not drop_to_shell(): 810 if prevrev != repo['last_revision']: 811 conf.update(name, "last_revision", prevrev) 812 sys.exit(1) 813 prevrev = lastrev 814 i += 1 815 # Once all patches are applied, we should update 816 # last_revision to the branch head instead of the last 817 # applied patch. The two are not necessarily the same when 818 # the last commit is a merge commit or when the patches at 819 # the branch head were intentionally excluded. 820 # 821 # If we do not do that for a merge commit, the next 822 # combo-layer run will only exclude patches reachable from 823 # one of the merged branches and try to re-apply patches 824 # from other branches even though they were already 825 # copied. 826 # 827 # If patches were intentionally excluded, the next run will 828 # present them again instead of skipping over them. This 829 # may or may not be intended, so the code here is conservative 830 # and only addresses the "head is merge commit" case. 831 if lastrev != branchrev and \ 832 len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1: 833 lastrev = branchrev 834 else: 835 logger.info("No patches to apply from %s" % name) 836 lastrev = branchrev 837 838 if lastrev != repo['last_revision']: 839 conf.update(name, "last_revision", lastrev) 840 841def action_splitpatch(conf, args): 842 """ 843 generate the commit patch and 844 split the patch per repo 845 """ 846 logger.debug("action_splitpatch") 847 if len(args) > 1: 848 commit = args[1] 849 else: 850 commit = "HEAD" 851 patchdir = "splitpatch-%s" % commit 852 if not os.path.exists(patchdir): 853 os.mkdir(patchdir) 854 855 # filerange_root is for the repo whose dest_dir is root "." 856 # and it should be specified by excluding all other repo dest dir 857 # like "-x repo1 -x repo2 -x repo3 ..." 858 filerange_root = "" 859 for name in conf.repos: 860 dest_dir = conf.repos[name]['dest_dir'] 861 if dest_dir != ".": 862 filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir) 863 864 for name in conf.repos: 865 dest_dir = conf.repos[name]['dest_dir'] 866 patch_filename = "%s/%s.patch" % (patchdir, name) 867 if dest_dir == ".": 868 cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename) 869 else: 870 cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename) 871 runcmd(cmd) 872 # Detect empty patches (including those produced by filterdiff above 873 # that contain only preamble text) 874 if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "": 875 os.remove(patch_filename) 876 logger.info("(skipping %s - no changes)", name) 877 else: 878 logger.info(patch_filename) 879 880def action_error(conf, args): 881 logger.info("invalid action %s" % args[0]) 882 883actions = { 884 "init": action_init, 885 "update": action_update, 886 "pull": action_pull, 887 "splitpatch": action_splitpatch, 888} 889 890def main(): 891 parser = optparse.OptionParser( 892 version = "Combo Layer Repo Tool version %s" % __version__, 893 usage = """%prog [options] action 894 895Create and update a combination layer repository from multiple component repositories. 896 897Action: 898 init initialise the combo layer repo 899 update [components] get patches from component repos and apply them to the combo repo 900 pull [components] just pull component repos only 901 splitpatch [commit] generate commit patch and split per component, default commit is HEAD""") 902 903 parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).", 904 action = "store", dest = "conffile", default = "conf/combo-layer.conf") 905 906 parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches", 907 action = "store_true", dest = "interactive", default = False) 908 909 parser.add_option("-D", "--debug", help = "output debug information", 910 action = "store_true", dest = "debug", default = False) 911 912 parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update", 913 action = "store_true", dest = "nopull", default = False) 914 915 parser.add_option("--hard-reset", 916 help = "instead of pull do fetch and hard-reset in component repos", 917 action = "store_true", dest = "hard_reset", default = False) 918 919 parser.add_option("-H", "--history", help = "import full history of components during init", 920 action = "store_true", default = False) 921 922 options, args = parser.parse_args(sys.argv) 923 924 # Dispatch to action handler 925 if len(args) == 1: 926 logger.error("No action specified, exiting") 927 parser.print_help() 928 elif args[1] not in actions: 929 logger.error("Unsupported action %s, exiting\n" % (args[1])) 930 parser.print_help() 931 elif not os.path.exists(options.conffile): 932 logger.error("No valid config file, exiting\n") 933 parser.print_help() 934 else: 935 if options.debug: 936 logger.setLevel(logging.DEBUG) 937 confdata = Configuration(options) 938 initmode = (args[1] == 'init') 939 confdata.sanity_check(initmode) 940 actions.get(args[1], action_error)(confdata, args[1:]) 941 942if __name__ == "__main__": 943 try: 944 ret = main() 945 except Exception: 946 ret = 1 947 import traceback 948 traceback.print_exc(5) 949 sys.exit(ret) 950