1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import os 7import re 8import shutil 9import tempfile 10 11import command 12import commit 13import gitutil 14from series import Series 15 16# Tags that we detect and remove 17re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' 18 '|Reviewed-on:|Commit-\w*:') 19 20# Lines which are allowed after a TEST= line 21re_allowed_after_test = re.compile('^Signed-off-by:') 22 23# Signoffs 24re_signoff = re.compile('^Signed-off-by: *(.*)') 25 26# The start of the cover letter 27re_cover = re.compile('^Cover-letter:') 28 29# A cover letter Cc 30re_cover_cc = re.compile('^Cover-letter-cc: *(.*)') 31 32# Patch series tag 33re_series_tag = re.compile('^Series-([a-z-]*): *(.*)') 34 35# Commit series tag 36re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)') 37 38# Commit tags that we want to collect and keep 39re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)') 40 41# The start of a new commit in the git log 42re_commit = re.compile('^commit ([0-9a-f]*)$') 43 44# We detect these since checkpatch doesn't always do it 45re_space_before_tab = re.compile('^[+].* \t') 46 47# States we can be in - can we use range() and still have comments? 48STATE_MSG_HEADER = 0 # Still in the message header 49STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 50STATE_PATCH_HEADER = 2 # In patch header (after the subject) 51STATE_DIFFS = 3 # In the diff part (past --- line) 52 53class PatchStream: 54 """Class for detecting/injecting tags in a patch or series of patches 55 56 We support processing the output of 'git log' to read out the tags we 57 are interested in. We can also process a patch file in order to remove 58 unwanted tags or inject additional ones. These correspond to the two 59 phases of processing. 60 """ 61 def __init__(self, series, name=None, is_log=False): 62 self.skip_blank = False # True to skip a single blank line 63 self.found_test = False # Found a TEST= line 64 self.lines_after_test = 0 # MNumber of lines found after TEST= 65 self.warn = [] # List of warnings we have collected 66 self.linenum = 1 # Output line number we are up to 67 self.in_section = None # Name of start...END section we are in 68 self.notes = [] # Series notes 69 self.section = [] # The current section...END section 70 self.series = series # Info about the patch series 71 self.is_log = is_log # True if indent like git log 72 self.in_change = 0 # Non-zero if we are in a change list 73 self.blank_count = 0 # Number of blank lines stored up 74 self.state = STATE_MSG_HEADER # What state are we in? 75 self.tags = [] # Tags collected, like Tested-by... 76 self.signoff = [] # Contents of signoff line 77 self.commit = None # Current commit 78 79 def AddToSeries(self, line, name, value): 80 """Add a new Series-xxx tag. 81 82 When a Series-xxx tag is detected, we come here to record it, if we 83 are scanning a 'git log'. 84 85 Args: 86 line: Source line containing tag (useful for debug/error messages) 87 name: Tag name (part after 'Series-') 88 value: Tag value (part after 'Series-xxx: ') 89 """ 90 if name == 'notes': 91 self.in_section = name 92 self.skip_blank = False 93 if self.is_log: 94 self.series.AddTag(self.commit, line, name, value) 95 96 def AddToCommit(self, line, name, value): 97 """Add a new Commit-xxx tag. 98 99 When a Commit-xxx tag is detected, we come here to record it. 100 101 Args: 102 line: Source line containing tag (useful for debug/error messages) 103 name: Tag name (part after 'Commit-') 104 value: Tag value (part after 'Commit-xxx: ') 105 """ 106 if name == 'notes': 107 self.in_section = 'commit-' + name 108 self.skip_blank = False 109 110 def CloseCommit(self): 111 """Save the current commit into our commit list, and reset our state""" 112 if self.commit and self.is_log: 113 self.series.AddCommit(self.commit) 114 self.commit = None 115 116 def FormatTags(self, tags): 117 out_list = [] 118 for tag in sorted(tags): 119 if tag.startswith('Cc:'): 120 tag_list = tag[4:].split(',') 121 out_list += gitutil.BuildEmailList(tag_list, 'Cc:') 122 else: 123 out_list.append(tag) 124 return out_list 125 126 def ProcessLine(self, line): 127 """Process a single line of a patch file or commit log 128 129 This process a line and returns a list of lines to output. The list 130 may be empty or may contain multiple output lines. 131 132 This is where all the complicated logic is located. The class's 133 state is used to move between different states and detect things 134 properly. 135 136 We can be in one of two modes: 137 self.is_log == True: This is 'git log' mode, where most output is 138 indented by 4 characters and we are scanning for tags 139 140 self.is_log == False: This is 'patch' mode, where we already have 141 all the tags, and are processing patches to remove junk we 142 don't want, and add things we think are required. 143 144 Args: 145 line: text line to process 146 147 Returns: 148 list of output lines, or [] if nothing should be output 149 """ 150 # Initially we have no output. Prepare the input line string 151 out = [] 152 line = line.rstrip('\n') 153 if self.is_log: 154 if line[:4] == ' ': 155 line = line[4:] 156 157 # Handle state transition and skipping blank lines 158 series_tag_match = re_series_tag.match(line) 159 commit_tag_match = re_commit_tag.match(line) 160 commit_match = re_commit.match(line) if self.is_log else None 161 cover_cc_match = re_cover_cc.match(line) 162 signoff_match = re_signoff.match(line) 163 tag_match = None 164 if self.state == STATE_PATCH_HEADER: 165 tag_match = re_tag.match(line) 166 is_blank = not line.strip() 167 if is_blank: 168 if (self.state == STATE_MSG_HEADER 169 or self.state == STATE_PATCH_SUBJECT): 170 self.state += 1 171 172 # We don't have a subject in the text stream of patch files 173 # It has its own line with a Subject: tag 174 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 175 self.state += 1 176 elif commit_match: 177 self.state = STATE_MSG_HEADER 178 179 # If we are in a section, keep collecting lines until we see END 180 if self.in_section: 181 if line == 'END': 182 if self.in_section == 'cover': 183 self.series.cover = self.section 184 elif self.in_section == 'notes': 185 if self.is_log: 186 self.series.notes += self.section 187 elif self.in_section == 'commit-notes': 188 if self.is_log: 189 self.commit.notes += self.section 190 else: 191 self.warn.append("Unknown section '%s'" % self.in_section) 192 self.in_section = None 193 self.skip_blank = True 194 self.section = [] 195 else: 196 self.section.append(line) 197 198 # Detect the commit subject 199 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 200 self.commit.subject = line 201 202 # Detect the tags we want to remove, and skip blank lines 203 elif re_remove.match(line) and not commit_tag_match: 204 self.skip_blank = True 205 206 # TEST= should be the last thing in the commit, so remove 207 # everything after it 208 if line.startswith('TEST='): 209 self.found_test = True 210 elif self.skip_blank and is_blank: 211 self.skip_blank = False 212 213 # Detect the start of a cover letter section 214 elif re_cover.match(line): 215 self.in_section = 'cover' 216 self.skip_blank = False 217 218 elif cover_cc_match: 219 value = cover_cc_match.group(1) 220 self.AddToSeries(line, 'cover-cc', value) 221 222 # If we are in a change list, key collected lines until a blank one 223 elif self.in_change: 224 if is_blank: 225 # Blank line ends this change list 226 self.in_change = 0 227 elif line == '---': 228 self.in_change = 0 229 out = self.ProcessLine(line) 230 else: 231 if self.is_log: 232 self.series.AddChange(self.in_change, self.commit, line) 233 self.skip_blank = False 234 235 # Detect Series-xxx tags 236 elif series_tag_match: 237 name = series_tag_match.group(1) 238 value = series_tag_match.group(2) 239 if name == 'changes': 240 # value is the version number: e.g. 1, or 2 241 try: 242 value = int(value) 243 except ValueError as str: 244 raise ValueError("%s: Cannot decode version info '%s'" % 245 (self.commit.hash, line)) 246 self.in_change = int(value) 247 else: 248 self.AddToSeries(line, name, value) 249 self.skip_blank = True 250 251 # Detect Commit-xxx tags 252 elif commit_tag_match: 253 name = commit_tag_match.group(1) 254 value = commit_tag_match.group(2) 255 if name == 'notes': 256 self.AddToCommit(line, name, value) 257 self.skip_blank = True 258 259 # Detect the start of a new commit 260 elif commit_match: 261 self.CloseCommit() 262 # TODO: We should store the whole hash, and just display a subset 263 self.commit = commit.Commit(commit_match.group(1)[:8]) 264 265 # Detect tags in the commit message 266 elif tag_match: 267 # Remove Tested-by self, since few will take much notice 268 if (tag_match.group(1) == 'Tested-by' and 269 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 270 self.warn.append("Ignoring %s" % line) 271 elif tag_match.group(1) == 'Patch-cc': 272 self.commit.AddCc(tag_match.group(2).split(',')) 273 else: 274 self.tags.append(line); 275 276 # Suppress duplicate signoffs 277 elif signoff_match: 278 if self.commit.CheckDuplicateSignoff(signoff_match.group(1)): 279 out = [line] 280 281 # Well that means this is an ordinary line 282 else: 283 pos = 1 284 # Look for ugly ASCII characters 285 for ch in line: 286 # TODO: Would be nicer to report source filename and line 287 if ord(ch) > 0x80: 288 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 289 (self.linenum, pos, line)) 290 pos += 1 291 292 # Look for space before tab 293 m = re_space_before_tab.match(line) 294 if m: 295 self.warn.append('Line %d/%d has space before tab' % 296 (self.linenum, m.start())) 297 298 # OK, we have a valid non-blank line 299 out = [line] 300 self.linenum += 1 301 self.skip_blank = False 302 if self.state == STATE_DIFFS: 303 pass 304 305 # If this is the start of the diffs section, emit our tags and 306 # change log 307 elif line == '---': 308 self.state = STATE_DIFFS 309 310 # Output the tags (signeoff first), then change list 311 out = [] 312 log = self.series.MakeChangeLog(self.commit) 313 out += self.FormatTags(self.tags) 314 out += [line] + self.commit.notes + [''] + log 315 elif self.found_test: 316 if not re_allowed_after_test.match(line): 317 self.lines_after_test += 1 318 319 return out 320 321 def Finalize(self): 322 """Close out processing of this patch stream""" 323 self.CloseCommit() 324 if self.lines_after_test: 325 self.warn.append('Found %d lines after TEST=' % 326 self.lines_after_test) 327 328 def ProcessStream(self, infd, outfd): 329 """Copy a stream from infd to outfd, filtering out unwanting things. 330 331 This is used to process patch files one at a time. 332 333 Args: 334 infd: Input stream file object 335 outfd: Output stream file object 336 """ 337 # Extract the filename from each diff, for nice warnings 338 fname = None 339 last_fname = None 340 re_fname = re.compile('diff --git a/(.*) b/.*') 341 while True: 342 line = infd.readline() 343 if not line: 344 break 345 out = self.ProcessLine(line) 346 347 # Try to detect blank lines at EOF 348 for line in out: 349 match = re_fname.match(line) 350 if match: 351 last_fname = fname 352 fname = match.group(1) 353 if line == '+': 354 self.blank_count += 1 355 else: 356 if self.blank_count and (line == '-- ' or match): 357 self.warn.append("Found possible blank line(s) at " 358 "end of file '%s'" % last_fname) 359 outfd.write('+\n' * self.blank_count) 360 outfd.write(line + '\n') 361 self.blank_count = 0 362 self.Finalize() 363 364 365def GetMetaDataForList(commit_range, git_dir=None, count=None, 366 series = Series()): 367 """Reads out patch series metadata from the commits 368 369 This does a 'git log' on the relevant commits and pulls out the tags we 370 are interested in. 371 372 Args: 373 commit_range: Range of commits to count (e.g. 'HEAD..base') 374 git_dir: Path to git repositiory (None to use default) 375 count: Number of commits to list, or None for no limit 376 series: Series object to add information into. By default a new series 377 is started. 378 Returns: 379 A Series object containing information about the commits. 380 """ 381 params = ['git', 'log', '--no-color', '--reverse', '--no-decorate', 382 commit_range] 383 if count is not None: 384 params[2:2] = ['-n%d' % count] 385 if git_dir: 386 params[1:1] = ['--git-dir', git_dir] 387 pipe = [params] 388 stdout = command.RunPipe(pipe, capture=True).stdout 389 ps = PatchStream(series, is_log=True) 390 for line in stdout.splitlines(): 391 ps.ProcessLine(line) 392 ps.Finalize() 393 return series 394 395def GetMetaData(start, count): 396 """Reads out patch series metadata from the commits 397 398 This does a 'git log' on the relevant commits and pulls out the tags we 399 are interested in. 400 401 Args: 402 start: Commit to start from: 0=HEAD, 1=next one, etc. 403 count: Number of commits to list 404 """ 405 return GetMetaDataForList('HEAD~%d' % start, None, count) 406 407def FixPatch(backup_dir, fname, series, commit): 408 """Fix up a patch file, by adding/removing as required. 409 410 We remove our tags from the patch file, insert changes lists, etc. 411 The patch file is processed in place, and overwritten. 412 413 A backup file is put into backup_dir (if not None). 414 415 Args: 416 fname: Filename to patch file to process 417 series: Series information about this patch set 418 commit: Commit object for this patch file 419 Return: 420 A list of errors, or [] if all ok. 421 """ 422 handle, tmpname = tempfile.mkstemp() 423 outfd = os.fdopen(handle, 'w') 424 infd = open(fname, 'r') 425 ps = PatchStream(series) 426 ps.commit = commit 427 ps.ProcessStream(infd, outfd) 428 infd.close() 429 outfd.close() 430 431 # Create a backup file if required 432 if backup_dir: 433 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 434 shutil.move(tmpname, fname) 435 return ps.warn 436 437def FixPatches(series, fnames): 438 """Fix up a list of patches identified by filenames 439 440 The patch files are processed in place, and overwritten. 441 442 Args: 443 series: The series object 444 fnames: List of patch files to process 445 """ 446 # Current workflow creates patches, so we shouldn't need a backup 447 backup_dir = None #tempfile.mkdtemp('clean-patch') 448 count = 0 449 for fname in fnames: 450 commit = series.commits[count] 451 commit.patch = fname 452 result = FixPatch(backup_dir, fname, series, commit) 453 if result: 454 print '%d warnings for %s:' % (len(result), fname) 455 for warn in result: 456 print '\t', warn 457 print 458 count += 1 459 print 'Cleaned %d patches' % count 460 return series 461 462def InsertCoverLetter(fname, series, count): 463 """Inserts a cover letter with the required info into patch 0 464 465 Args: 466 fname: Input / output filename of the cover letter file 467 series: Series object 468 count: Number of patches in the series 469 """ 470 fd = open(fname, 'r') 471 lines = fd.readlines() 472 fd.close() 473 474 fd = open(fname, 'w') 475 text = series.cover 476 prefix = series.GetPatchPrefix() 477 for line in lines: 478 if line.startswith('Subject:'): 479 # TODO: if more than 10 patches this should save 00/xx, not 0/xx 480 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0]) 481 482 # Insert our cover letter 483 elif line.startswith('*** BLURB HERE ***'): 484 # First the blurb test 485 line = '\n'.join(text[1:]) + '\n' 486 if series.get('notes'): 487 line += '\n'.join(series.notes) + '\n' 488 489 # Now the change list 490 out = series.MakeChangeLog(None) 491 line += '\n' + '\n'.join(out) 492 fd.write(line) 493 fd.close() 494