1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import command 7import re 8import os 9import series 10import subprocess 11import sys 12import terminal 13 14import checkpatch 15import settings 16 17# True to use --no-decorate - we check this in Setup() 18use_no_decorate = True 19 20def LogCmd(commit_range, git_dir=None, oneline=False, reverse=False, 21 count=None): 22 """Create a command to perform a 'git log' 23 24 Args: 25 commit_range: Range expression to use for log, None for none 26 git_dir: Path to git repositiory (None to use default) 27 oneline: True to use --oneline, else False 28 reverse: True to reverse the log (--reverse) 29 count: Number of commits to list, or None for no limit 30 Return: 31 List containing command and arguments to run 32 """ 33 cmd = ['git'] 34 if git_dir: 35 cmd += ['--git-dir', git_dir] 36 cmd += ['log', '--no-color'] 37 if oneline: 38 cmd.append('--oneline') 39 if use_no_decorate: 40 cmd.append('--no-decorate') 41 if reverse: 42 cmd.append('--reverse') 43 if count is not None: 44 cmd.append('-n%d' % count) 45 if commit_range: 46 cmd.append(commit_range) 47 return cmd 48 49def CountCommitsToBranch(): 50 """Returns number of commits between HEAD and the tracking branch. 51 52 This looks back to the tracking branch and works out the number of commits 53 since then. 54 55 Return: 56 Number of patches that exist on top of the branch 57 """ 58 pipe = [LogCmd('@{upstream}..', oneline=True), 59 ['wc', '-l']] 60 stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout 61 patch_count = int(stdout) 62 return patch_count 63 64def GetUpstream(git_dir, branch): 65 """Returns the name of the upstream for a branch 66 67 Args: 68 git_dir: Git directory containing repo 69 branch: Name of branch 70 71 Returns: 72 Name of upstream branch (e.g. 'upstream/master') or None if none 73 """ 74 try: 75 remote = command.OutputOneLine('git', '--git-dir', git_dir, 'config', 76 'branch.%s.remote' % branch) 77 merge = command.OutputOneLine('git', '--git-dir', git_dir, 'config', 78 'branch.%s.merge' % branch) 79 except: 80 return None 81 82 if remote == '.': 83 return merge 84 elif remote and merge: 85 leaf = merge.split('/')[-1] 86 return '%s/%s' % (remote, leaf) 87 else: 88 raise ValueError, ("Cannot determine upstream branch for branch " 89 "'%s' remote='%s', merge='%s'" % (branch, remote, merge)) 90 91 92def GetRangeInBranch(git_dir, branch, include_upstream=False): 93 """Returns an expression for the commits in the given branch. 94 95 Args: 96 git_dir: Directory containing git repo 97 branch: Name of branch 98 Return: 99 Expression in the form 'upstream..branch' which can be used to 100 access the commits. If the branch does not exist, returns None. 101 """ 102 upstream = GetUpstream(git_dir, branch) 103 if not upstream: 104 return None 105 return '%s%s..%s' % (upstream, '~' if include_upstream else '', branch) 106 107def CountCommitsInBranch(git_dir, branch, include_upstream=False): 108 """Returns the number of commits in the given branch. 109 110 Args: 111 git_dir: Directory containing git repo 112 branch: Name of branch 113 Return: 114 Number of patches that exist on top of the branch, or None if the 115 branch does not exist. 116 """ 117 range_expr = GetRangeInBranch(git_dir, branch, include_upstream) 118 if not range_expr: 119 return None 120 pipe = [LogCmd(range_expr, git_dir=git_dir, oneline=True), 121 ['wc', '-l']] 122 result = command.RunPipe(pipe, capture=True, oneline=True) 123 patch_count = int(result.stdout) 124 return patch_count 125 126def CountCommits(commit_range): 127 """Returns the number of commits in the given range. 128 129 Args: 130 commit_range: Range of commits to count (e.g. 'HEAD..base') 131 Return: 132 Number of patches that exist on top of the branch 133 """ 134 pipe = [LogCmd(commit_range, oneline=True), 135 ['wc', '-l']] 136 stdout = command.RunPipe(pipe, capture=True, oneline=True).stdout 137 patch_count = int(stdout) 138 return patch_count 139 140def Checkout(commit_hash, git_dir=None, work_tree=None, force=False): 141 """Checkout the selected commit for this build 142 143 Args: 144 commit_hash: Commit hash to check out 145 """ 146 pipe = ['git'] 147 if git_dir: 148 pipe.extend(['--git-dir', git_dir]) 149 if work_tree: 150 pipe.extend(['--work-tree', work_tree]) 151 pipe.append('checkout') 152 if force: 153 pipe.append('-f') 154 pipe.append(commit_hash) 155 result = command.RunPipe([pipe], capture=True, raise_on_error=False) 156 if result.return_code != 0: 157 raise OSError, 'git checkout (%s): %s' % (pipe, result.stderr) 158 159def Clone(git_dir, output_dir): 160 """Checkout the selected commit for this build 161 162 Args: 163 commit_hash: Commit hash to check out 164 """ 165 pipe = ['git', 'clone', git_dir, '.'] 166 result = command.RunPipe([pipe], capture=True, cwd=output_dir) 167 if result.return_code != 0: 168 raise OSError, 'git clone: %s' % result.stderr 169 170def Fetch(git_dir=None, work_tree=None): 171 """Fetch from the origin repo 172 173 Args: 174 commit_hash: Commit hash to check out 175 """ 176 pipe = ['git'] 177 if git_dir: 178 pipe.extend(['--git-dir', git_dir]) 179 if work_tree: 180 pipe.extend(['--work-tree', work_tree]) 181 pipe.append('fetch') 182 result = command.RunPipe([pipe], capture=True) 183 if result.return_code != 0: 184 raise OSError, 'git fetch: %s' % result.stderr 185 186def CreatePatches(start, count, series): 187 """Create a series of patches from the top of the current branch. 188 189 The patch files are written to the current directory using 190 git format-patch. 191 192 Args: 193 start: Commit to start from: 0=HEAD, 1=next one, etc. 194 count: number of commits to include 195 Return: 196 Filename of cover letter 197 List of filenames of patch files 198 """ 199 if series.get('version'): 200 version = '%s ' % series['version'] 201 cmd = ['git', 'format-patch', '-M', '--signoff'] 202 if series.get('cover'): 203 cmd.append('--cover-letter') 204 prefix = series.GetPatchPrefix() 205 if prefix: 206 cmd += ['--subject-prefix=%s' % prefix] 207 cmd += ['HEAD~%d..HEAD~%d' % (start + count, start)] 208 209 stdout = command.RunList(cmd) 210 files = stdout.splitlines() 211 212 # We have an extra file if there is a cover letter 213 if series.get('cover'): 214 return files[0], files[1:] 215 else: 216 return None, files 217 218def ApplyPatch(verbose, fname): 219 """Apply a patch with git am to test it 220 221 TODO: Convert these to use command, with stderr option 222 223 Args: 224 fname: filename of patch file to apply 225 """ 226 col = terminal.Color() 227 cmd = ['git', 'am', fname] 228 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 229 stderr=subprocess.PIPE) 230 stdout, stderr = pipe.communicate() 231 re_error = re.compile('^error: patch failed: (.+):(\d+)') 232 for line in stderr.splitlines(): 233 if verbose: 234 print line 235 match = re_error.match(line) 236 if match: 237 print checkpatch.GetWarningMsg(col, 'warning', match.group(1), 238 int(match.group(2)), 'Patch failed') 239 return pipe.returncode == 0, stdout 240 241def ApplyPatches(verbose, args, start_point): 242 """Apply the patches with git am to make sure all is well 243 244 Args: 245 verbose: Print out 'git am' output verbatim 246 args: List of patch files to apply 247 start_point: Number of commits back from HEAD to start applying. 248 Normally this is len(args), but it can be larger if a start 249 offset was given. 250 """ 251 error_count = 0 252 col = terminal.Color() 253 254 # Figure out our current position 255 cmd = ['git', 'name-rev', 'HEAD', '--name-only'] 256 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) 257 stdout, stderr = pipe.communicate() 258 if pipe.returncode: 259 str = 'Could not find current commit name' 260 print col.Color(col.RED, str) 261 print stdout 262 return False 263 old_head = stdout.splitlines()[0] 264 if old_head == 'undefined': 265 str = "Invalid HEAD '%s'" % stdout.strip() 266 print col.Color(col.RED, str) 267 return False 268 269 # Checkout the required start point 270 cmd = ['git', 'checkout', 'HEAD~%d' % start_point] 271 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, 272 stderr=subprocess.PIPE) 273 stdout, stderr = pipe.communicate() 274 if pipe.returncode: 275 str = 'Could not move to commit before patch series' 276 print col.Color(col.RED, str) 277 print stdout, stderr 278 return False 279 280 # Apply all the patches 281 for fname in args: 282 ok, stdout = ApplyPatch(verbose, fname) 283 if not ok: 284 print col.Color(col.RED, 'git am returned errors for %s: will ' 285 'skip this patch' % fname) 286 if verbose: 287 print stdout 288 error_count += 1 289 cmd = ['git', 'am', '--skip'] 290 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) 291 stdout, stderr = pipe.communicate() 292 if pipe.returncode != 0: 293 print col.Color(col.RED, 'Unable to skip patch! Aborting...') 294 print stdout 295 break 296 297 # Return to our previous position 298 cmd = ['git', 'checkout', old_head] 299 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 300 stdout, stderr = pipe.communicate() 301 if pipe.returncode: 302 print col.Color(col.RED, 'Could not move back to head commit') 303 print stdout, stderr 304 return error_count == 0 305 306def BuildEmailList(in_list, tag=None, alias=None, raise_on_error=True): 307 """Build a list of email addresses based on an input list. 308 309 Takes a list of email addresses and aliases, and turns this into a list 310 of only email address, by resolving any aliases that are present. 311 312 If the tag is given, then each email address is prepended with this 313 tag and a space. If the tag starts with a minus sign (indicating a 314 command line parameter) then the email address is quoted. 315 316 Args: 317 in_list: List of aliases/email addresses 318 tag: Text to put before each address 319 alias: Alias dictionary 320 raise_on_error: True to raise an error when an alias fails to match, 321 False to just print a message. 322 323 Returns: 324 List of email addresses 325 326 >>> alias = {} 327 >>> alias['fred'] = ['f.bloggs@napier.co.nz'] 328 >>> alias['john'] = ['j.bloggs@napier.co.nz'] 329 >>> alias['mary'] = ['Mary Poppins <m.poppins@cloud.net>'] 330 >>> alias['boys'] = ['fred', ' john'] 331 >>> alias['all'] = ['fred ', 'john', ' mary '] 332 >>> BuildEmailList(['john', 'mary'], None, alias) 333 ['j.bloggs@napier.co.nz', 'Mary Poppins <m.poppins@cloud.net>'] 334 >>> BuildEmailList(['john', 'mary'], '--to', alias) 335 ['--to "j.bloggs@napier.co.nz"', \ 336'--to "Mary Poppins <m.poppins@cloud.net>"'] 337 >>> BuildEmailList(['john', 'mary'], 'Cc', alias) 338 ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins <m.poppins@cloud.net>'] 339 """ 340 quote = '"' if tag and tag[0] == '-' else '' 341 raw = [] 342 for item in in_list: 343 raw += LookupEmail(item, alias, raise_on_error=raise_on_error) 344 result = [] 345 for item in raw: 346 if not item in result: 347 result.append(item) 348 if tag: 349 return ['%s %s%s%s' % (tag, quote, email, quote) for email in result] 350 return result 351 352def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, 353 self_only=False, alias=None, in_reply_to=None): 354 """Email a patch series. 355 356 Args: 357 series: Series object containing destination info 358 cover_fname: filename of cover letter 359 args: list of filenames of patch files 360 dry_run: Just return the command that would be run 361 raise_on_error: True to raise an error when an alias fails to match, 362 False to just print a message. 363 cc_fname: Filename of Cc file for per-commit Cc 364 self_only: True to just email to yourself as a test 365 in_reply_to: If set we'll pass this to git as --in-reply-to. 366 Should be a message ID that this is in reply to. 367 368 Returns: 369 Git command that was/would be run 370 371 # For the duration of this doctest pretend that we ran patman with ./patman 372 >>> _old_argv0 = sys.argv[0] 373 >>> sys.argv[0] = './patman' 374 375 >>> alias = {} 376 >>> alias['fred'] = ['f.bloggs@napier.co.nz'] 377 >>> alias['john'] = ['j.bloggs@napier.co.nz'] 378 >>> alias['mary'] = ['m.poppins@cloud.net'] 379 >>> alias['boys'] = ['fred', ' john'] 380 >>> alias['all'] = ['fred ', 'john', ' mary '] 381 >>> alias[os.getenv('USER')] = ['this-is-me@me.com'] 382 >>> series = series.Series() 383 >>> series.to = ['fred'] 384 >>> series.cc = ['mary'] 385 >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ 386 False, alias) 387 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ 388"m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2' 389 >>> EmailPatches(series, None, ['p1'], True, True, 'cc-fname', False, \ 390 alias) 391 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ 392"m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" p1' 393 >>> series.cc = ['all'] 394 >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ 395 True, alias) 396 'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \ 397--cc-cmd cc-fname" cover p1 p2' 398 >>> EmailPatches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ 399 False, alias) 400 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ 401"f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \ 402"m.poppins@cloud.net" --cc-cmd "./patman --cc-cmd cc-fname" cover p1 p2' 403 404 # Restore argv[0] since we clobbered it. 405 >>> sys.argv[0] = _old_argv0 406 """ 407 to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error) 408 if not to: 409 git_config_to = command.Output('git', 'config', 'sendemail.to') 410 if not git_config_to: 411 print ("No recipient.\n" 412 "Please add something like this to a commit\n" 413 "Series-to: Fred Bloggs <f.blogs@napier.co.nz>\n" 414 "Or do something like this\n" 415 "git config sendemail.to u-boot@lists.denx.de") 416 return 417 cc = BuildEmailList(series.get('cc'), '--cc', alias, raise_on_error) 418 if self_only: 419 to = BuildEmailList([os.getenv('USER')], '--to', alias, raise_on_error) 420 cc = [] 421 cmd = ['git', 'send-email', '--annotate'] 422 if in_reply_to: 423 cmd.append('--in-reply-to="%s"' % in_reply_to) 424 425 cmd += to 426 cmd += cc 427 cmd += ['--cc-cmd', '"%s --cc-cmd %s"' % (sys.argv[0], cc_fname)] 428 if cover_fname: 429 cmd.append(cover_fname) 430 cmd += args 431 str = ' '.join(cmd) 432 if not dry_run: 433 os.system(str) 434 return str 435 436 437def LookupEmail(lookup_name, alias=None, raise_on_error=True, level=0): 438 """If an email address is an alias, look it up and return the full name 439 440 TODO: Why not just use git's own alias feature? 441 442 Args: 443 lookup_name: Alias or email address to look up 444 alias: Dictionary containing aliases (None to use settings default) 445 raise_on_error: True to raise an error when an alias fails to match, 446 False to just print a message. 447 448 Returns: 449 tuple: 450 list containing a list of email addresses 451 452 Raises: 453 OSError if a recursive alias reference was found 454 ValueError if an alias was not found 455 456 >>> alias = {} 457 >>> alias['fred'] = ['f.bloggs@napier.co.nz'] 458 >>> alias['john'] = ['j.bloggs@napier.co.nz'] 459 >>> alias['mary'] = ['m.poppins@cloud.net'] 460 >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz'] 461 >>> alias['all'] = ['fred ', 'john', ' mary '] 462 >>> alias['loop'] = ['other', 'john', ' mary '] 463 >>> alias['other'] = ['loop', 'john', ' mary '] 464 >>> LookupEmail('mary', alias) 465 ['m.poppins@cloud.net'] 466 >>> LookupEmail('arthur.wellesley@howe.ro.uk', alias) 467 ['arthur.wellesley@howe.ro.uk'] 468 >>> LookupEmail('boys', alias) 469 ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz'] 470 >>> LookupEmail('all', alias) 471 ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] 472 >>> LookupEmail('odd', alias) 473 Traceback (most recent call last): 474 ... 475 ValueError: Alias 'odd' not found 476 >>> LookupEmail('loop', alias) 477 Traceback (most recent call last): 478 ... 479 OSError: Recursive email alias at 'other' 480 >>> LookupEmail('odd', alias, raise_on_error=False) 481 \033[1;31mAlias 'odd' not found\033[0m 482 [] 483 >>> # In this case the loop part will effectively be ignored. 484 >>> LookupEmail('loop', alias, raise_on_error=False) 485 \033[1;31mRecursive email alias at 'other'\033[0m 486 \033[1;31mRecursive email alias at 'john'\033[0m 487 \033[1;31mRecursive email alias at 'mary'\033[0m 488 ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] 489 """ 490 if not alias: 491 alias = settings.alias 492 lookup_name = lookup_name.strip() 493 if '@' in lookup_name: # Perhaps a real email address 494 return [lookup_name] 495 496 lookup_name = lookup_name.lower() 497 col = terminal.Color() 498 499 out_list = [] 500 if level > 10: 501 msg = "Recursive email alias at '%s'" % lookup_name 502 if raise_on_error: 503 raise OSError, msg 504 else: 505 print col.Color(col.RED, msg) 506 return out_list 507 508 if lookup_name: 509 if not lookup_name in alias: 510 msg = "Alias '%s' not found" % lookup_name 511 if raise_on_error: 512 raise ValueError, msg 513 else: 514 print col.Color(col.RED, msg) 515 return out_list 516 for item in alias[lookup_name]: 517 todo = LookupEmail(item, alias, raise_on_error, level + 1) 518 for new_item in todo: 519 if not new_item in out_list: 520 out_list.append(new_item) 521 522 #print "No match for alias '%s'" % lookup_name 523 return out_list 524 525def GetTopLevel(): 526 """Return name of top-level directory for this git repo. 527 528 Returns: 529 Full path to git top-level directory 530 531 This test makes sure that we are running tests in the right subdir 532 533 >>> os.path.realpath(os.path.dirname(__file__)) == \ 534 os.path.join(GetTopLevel(), 'tools', 'patman') 535 True 536 """ 537 return command.OutputOneLine('git', 'rev-parse', '--show-toplevel') 538 539def GetAliasFile(): 540 """Gets the name of the git alias file. 541 542 Returns: 543 Filename of git alias file, or None if none 544 """ 545 fname = command.OutputOneLine('git', 'config', 'sendemail.aliasesfile', 546 raise_on_error=False) 547 if fname: 548 fname = os.path.join(GetTopLevel(), fname.strip()) 549 return fname 550 551def GetDefaultUserName(): 552 """Gets the user.name from .gitconfig file. 553 554 Returns: 555 User name found in .gitconfig file, or None if none 556 """ 557 uname = command.OutputOneLine('git', 'config', '--global', 'user.name') 558 return uname 559 560def GetDefaultUserEmail(): 561 """Gets the user.email from the global .gitconfig file. 562 563 Returns: 564 User's email found in .gitconfig file, or None if none 565 """ 566 uemail = command.OutputOneLine('git', 'config', '--global', 'user.email') 567 return uemail 568 569def Setup(): 570 """Set up git utils, by reading the alias files.""" 571 # Check for a git alias file also 572 alias_fname = GetAliasFile() 573 if alias_fname: 574 settings.ReadGitAliases(alias_fname) 575 cmd = LogCmd(None, count=0) 576 use_no_decorate = (command.RunPipe([cmd], raise_on_error=False) 577 .return_code == 0) 578 579def GetHead(): 580 """Get the hash of the current HEAD 581 582 Returns: 583 Hash of HEAD 584 """ 585 return command.OutputOneLine('git', 'show', '-s', '--pretty=format:%H') 586 587if __name__ == "__main__": 588 import doctest 589 590 doctest.testmod() 591