1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import os 7import re 8import shutil 9import tempfile 10 11import command 12import commit 13import gitutil 14from series import Series 15 16# Tags that we detect and remove 17re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' 18 '|Reviewed-on:|Commit-\w*:') 19 20# Lines which are allowed after a TEST= line 21re_allowed_after_test = re.compile('^Signed-off-by:') 22 23# Signoffs 24re_signoff = re.compile('^Signed-off-by: *(.*)') 25 26# The start of the cover letter 27re_cover = re.compile('^Cover-letter:') 28 29# A cover letter Cc 30re_cover_cc = re.compile('^Cover-letter-cc: *(.*)') 31 32# Patch series tag 33re_series_tag = re.compile('^Series-([a-z-]*): *(.*)') 34 35# Commit series tag 36re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)') 37 38# Commit tags that we want to collect and keep 39re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)') 40 41# The start of a new commit in the git log 42re_commit = re.compile('^commit ([0-9a-f]*)$') 43 44# We detect these since checkpatch doesn't always do it 45re_space_before_tab = re.compile('^[+].* \t') 46 47# States we can be in - can we use range() and still have comments? 48STATE_MSG_HEADER = 0 # Still in the message header 49STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 50STATE_PATCH_HEADER = 2 # In patch header (after the subject) 51STATE_DIFFS = 3 # In the diff part (past --- line) 52 53class PatchStream: 54 """Class for detecting/injecting tags in a patch or series of patches 55 56 We support processing the output of 'git log' to read out the tags we 57 are interested in. We can also process a patch file in order to remove 58 unwanted tags or inject additional ones. These correspond to the two 59 phases of processing. 60 """ 61 def __init__(self, series, name=None, is_log=False): 62 self.skip_blank = False # True to skip a single blank line 63 self.found_test = False # Found a TEST= line 64 self.lines_after_test = 0 # MNumber of lines found after TEST= 65 self.warn = [] # List of warnings we have collected 66 self.linenum = 1 # Output line number we are up to 67 self.in_section = None # Name of start...END section we are in 68 self.notes = [] # Series notes 69 self.section = [] # The current section...END section 70 self.series = series # Info about the patch series 71 self.is_log = is_log # True if indent like git log 72 self.in_change = 0 # Non-zero if we are in a change list 73 self.blank_count = 0 # Number of blank lines stored up 74 self.state = STATE_MSG_HEADER # What state are we in? 75 self.tags = [] # Tags collected, like Tested-by... 76 self.signoff = [] # Contents of signoff line 77 self.commit = None # Current commit 78 79 def AddToSeries(self, line, name, value): 80 """Add a new Series-xxx tag. 81 82 When a Series-xxx tag is detected, we come here to record it, if we 83 are scanning a 'git log'. 84 85 Args: 86 line: Source line containing tag (useful for debug/error messages) 87 name: Tag name (part after 'Series-') 88 value: Tag value (part after 'Series-xxx: ') 89 """ 90 if name == 'notes': 91 self.in_section = name 92 self.skip_blank = False 93 if self.is_log: 94 self.series.AddTag(self.commit, line, name, value) 95 96 def AddToCommit(self, line, name, value): 97 """Add a new Commit-xxx tag. 98 99 When a Commit-xxx tag is detected, we come here to record it. 100 101 Args: 102 line: Source line containing tag (useful for debug/error messages) 103 name: Tag name (part after 'Commit-') 104 value: Tag value (part after 'Commit-xxx: ') 105 """ 106 if name == 'notes': 107 self.in_section = 'commit-' + name 108 self.skip_blank = False 109 110 def CloseCommit(self): 111 """Save the current commit into our commit list, and reset our state""" 112 if self.commit and self.is_log: 113 self.series.AddCommit(self.commit) 114 self.commit = None 115 116 def FormatTags(self, tags): 117 out_list = [] 118 for tag in sorted(tags): 119 if tag.startswith('Cc:'): 120 tag_list = tag[4:].split(',') 121 out_list += gitutil.BuildEmailList(tag_list, 'Cc:') 122 else: 123 out_list.append(tag) 124 return out_list 125 126 def ProcessLine(self, line): 127 """Process a single line of a patch file or commit log 128 129 This process a line and returns a list of lines to output. The list 130 may be empty or may contain multiple output lines. 131 132 This is where all the complicated logic is located. The class's 133 state is used to move between different states and detect things 134 properly. 135 136 We can be in one of two modes: 137 self.is_log == True: This is 'git log' mode, where most output is 138 indented by 4 characters and we are scanning for tags 139 140 self.is_log == False: This is 'patch' mode, where we already have 141 all the tags, and are processing patches to remove junk we 142 don't want, and add things we think are required. 143 144 Args: 145 line: text line to process 146 147 Returns: 148 list of output lines, or [] if nothing should be output 149 """ 150 # Initially we have no output. Prepare the input line string 151 out = [] 152 line = line.rstrip('\n') 153 if self.is_log: 154 if line[:4] == ' ': 155 line = line[4:] 156 157 # Handle state transition and skipping blank lines 158 series_tag_match = re_series_tag.match(line) 159 commit_tag_match = re_commit_tag.match(line) 160 commit_match = re_commit.match(line) if self.is_log else None 161 cover_cc_match = re_cover_cc.match(line) 162 signoff_match = re_signoff.match(line) 163 tag_match = None 164 if self.state == STATE_PATCH_HEADER: 165 tag_match = re_tag.match(line) 166 is_blank = not line.strip() 167 if is_blank: 168 if (self.state == STATE_MSG_HEADER 169 or self.state == STATE_PATCH_SUBJECT): 170 self.state += 1 171 172 # We don't have a subject in the text stream of patch files 173 # It has its own line with a Subject: tag 174 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 175 self.state += 1 176 elif commit_match: 177 self.state = STATE_MSG_HEADER 178 179 # If we are in a section, keep collecting lines until we see END 180 if self.in_section: 181 if line == 'END': 182 if self.in_section == 'cover': 183 self.series.cover = self.section 184 elif self.in_section == 'notes': 185 if self.is_log: 186 self.series.notes += self.section 187 elif self.in_section == 'commit-notes': 188 if self.is_log: 189 self.commit.notes += self.section 190 else: 191 self.warn.append("Unknown section '%s'" % self.in_section) 192 self.in_section = None 193 self.skip_blank = True 194 self.section = [] 195 else: 196 self.section.append(line) 197 198 # Detect the commit subject 199 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 200 self.commit.subject = line 201 202 # Detect the tags we want to remove, and skip blank lines 203 elif re_remove.match(line) and not commit_tag_match: 204 self.skip_blank = True 205 206 # TEST= should be the last thing in the commit, so remove 207 # everything after it 208 if line.startswith('TEST='): 209 self.found_test = True 210 elif self.skip_blank and is_blank: 211 self.skip_blank = False 212 213 # Detect the start of a cover letter section 214 elif re_cover.match(line): 215 self.in_section = 'cover' 216 self.skip_blank = False 217 218 elif cover_cc_match: 219 value = cover_cc_match.group(1) 220 self.AddToSeries(line, 'cover-cc', value) 221 222 # If we are in a change list, key collected lines until a blank one 223 elif self.in_change: 224 if is_blank: 225 # Blank line ends this change list 226 self.in_change = 0 227 elif line == '---': 228 self.in_change = 0 229 out = self.ProcessLine(line) 230 else: 231 if self.is_log: 232 self.series.AddChange(self.in_change, self.commit, line) 233 self.skip_blank = False 234 235 # Detect Series-xxx tags 236 elif series_tag_match: 237 name = series_tag_match.group(1) 238 value = series_tag_match.group(2) 239 if name == 'changes': 240 # value is the version number: e.g. 1, or 2 241 try: 242 value = int(value) 243 except ValueError as str: 244 raise ValueError("%s: Cannot decode version info '%s'" % 245 (self.commit.hash, line)) 246 self.in_change = int(value) 247 else: 248 self.AddToSeries(line, name, value) 249 self.skip_blank = True 250 251 # Detect Commit-xxx tags 252 elif commit_tag_match: 253 name = commit_tag_match.group(1) 254 value = commit_tag_match.group(2) 255 if name == 'notes': 256 self.AddToCommit(line, name, value) 257 self.skip_blank = True 258 259 # Detect the start of a new commit 260 elif commit_match: 261 self.CloseCommit() 262 # TODO: We should store the whole hash, and just display a subset 263 self.commit = commit.Commit(commit_match.group(1)[:8]) 264 265 # Detect tags in the commit message 266 elif tag_match: 267 # Remove Tested-by self, since few will take much notice 268 if (tag_match.group(1) == 'Tested-by' and 269 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 270 self.warn.append("Ignoring %s" % line) 271 elif tag_match.group(1) == 'Patch-cc': 272 self.commit.AddCc(tag_match.group(2).split(',')) 273 else: 274 self.tags.append(line); 275 276 # Suppress duplicate signoffs 277 elif signoff_match: 278 if (self.is_log or 279 self.commit.CheckDuplicateSignoff(signoff_match.group(1))): 280 out = [line] 281 282 # Well that means this is an ordinary line 283 else: 284 pos = 1 285 # Look for ugly ASCII characters 286 for ch in line: 287 # TODO: Would be nicer to report source filename and line 288 if ord(ch) > 0x80: 289 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 290 (self.linenum, pos, line)) 291 pos += 1 292 293 # Look for space before tab 294 m = re_space_before_tab.match(line) 295 if m: 296 self.warn.append('Line %d/%d has space before tab' % 297 (self.linenum, m.start())) 298 299 # OK, we have a valid non-blank line 300 out = [line] 301 self.linenum += 1 302 self.skip_blank = False 303 if self.state == STATE_DIFFS: 304 pass 305 306 # If this is the start of the diffs section, emit our tags and 307 # change log 308 elif line == '---': 309 self.state = STATE_DIFFS 310 311 # Output the tags (signeoff first), then change list 312 out = [] 313 log = self.series.MakeChangeLog(self.commit) 314 out += self.FormatTags(self.tags) 315 out += [line] + self.commit.notes + [''] + log 316 elif self.found_test: 317 if not re_allowed_after_test.match(line): 318 self.lines_after_test += 1 319 320 return out 321 322 def Finalize(self): 323 """Close out processing of this patch stream""" 324 self.CloseCommit() 325 if self.lines_after_test: 326 self.warn.append('Found %d lines after TEST=' % 327 self.lines_after_test) 328 329 def ProcessStream(self, infd, outfd): 330 """Copy a stream from infd to outfd, filtering out unwanting things. 331 332 This is used to process patch files one at a time. 333 334 Args: 335 infd: Input stream file object 336 outfd: Output stream file object 337 """ 338 # Extract the filename from each diff, for nice warnings 339 fname = None 340 last_fname = None 341 re_fname = re.compile('diff --git a/(.*) b/.*') 342 while True: 343 line = infd.readline() 344 if not line: 345 break 346 out = self.ProcessLine(line) 347 348 # Try to detect blank lines at EOF 349 for line in out: 350 match = re_fname.match(line) 351 if match: 352 last_fname = fname 353 fname = match.group(1) 354 if line == '+': 355 self.blank_count += 1 356 else: 357 if self.blank_count and (line == '-- ' or match): 358 self.warn.append("Found possible blank line(s) at " 359 "end of file '%s'" % last_fname) 360 outfd.write('+\n' * self.blank_count) 361 outfd.write(line + '\n') 362 self.blank_count = 0 363 self.Finalize() 364 365 366def GetMetaDataForList(commit_range, git_dir=None, count=None, 367 series = Series()): 368 """Reads out patch series metadata from the commits 369 370 This does a 'git log' on the relevant commits and pulls out the tags we 371 are interested in. 372 373 Args: 374 commit_range: Range of commits to count (e.g. 'HEAD..base') 375 git_dir: Path to git repositiory (None to use default) 376 count: Number of commits to list, or None for no limit 377 series: Series object to add information into. By default a new series 378 is started. 379 Returns: 380 A Series object containing information about the commits. 381 """ 382 params = gitutil.LogCmd(commit_range,reverse=True, count=count, 383 git_dir=git_dir) 384 stdout = command.RunPipe([params], capture=True).stdout 385 ps = PatchStream(series, is_log=True) 386 for line in stdout.splitlines(): 387 ps.ProcessLine(line) 388 ps.Finalize() 389 return series 390 391def GetMetaData(start, count): 392 """Reads out patch series metadata from the commits 393 394 This does a 'git log' on the relevant commits and pulls out the tags we 395 are interested in. 396 397 Args: 398 start: Commit to start from: 0=HEAD, 1=next one, etc. 399 count: Number of commits to list 400 """ 401 return GetMetaDataForList('HEAD~%d' % start, None, count) 402 403def FixPatch(backup_dir, fname, series, commit): 404 """Fix up a patch file, by adding/removing as required. 405 406 We remove our tags from the patch file, insert changes lists, etc. 407 The patch file is processed in place, and overwritten. 408 409 A backup file is put into backup_dir (if not None). 410 411 Args: 412 fname: Filename to patch file to process 413 series: Series information about this patch set 414 commit: Commit object for this patch file 415 Return: 416 A list of errors, or [] if all ok. 417 """ 418 handle, tmpname = tempfile.mkstemp() 419 outfd = os.fdopen(handle, 'w') 420 infd = open(fname, 'r') 421 ps = PatchStream(series) 422 ps.commit = commit 423 ps.ProcessStream(infd, outfd) 424 infd.close() 425 outfd.close() 426 427 # Create a backup file if required 428 if backup_dir: 429 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 430 shutil.move(tmpname, fname) 431 return ps.warn 432 433def FixPatches(series, fnames): 434 """Fix up a list of patches identified by filenames 435 436 The patch files are processed in place, and overwritten. 437 438 Args: 439 series: The series object 440 fnames: List of patch files to process 441 """ 442 # Current workflow creates patches, so we shouldn't need a backup 443 backup_dir = None #tempfile.mkdtemp('clean-patch') 444 count = 0 445 for fname in fnames: 446 commit = series.commits[count] 447 commit.patch = fname 448 result = FixPatch(backup_dir, fname, series, commit) 449 if result: 450 print '%d warnings for %s:' % (len(result), fname) 451 for warn in result: 452 print '\t', warn 453 print 454 count += 1 455 print 'Cleaned %d patches' % count 456 return series 457 458def InsertCoverLetter(fname, series, count): 459 """Inserts a cover letter with the required info into patch 0 460 461 Args: 462 fname: Input / output filename of the cover letter file 463 series: Series object 464 count: Number of patches in the series 465 """ 466 fd = open(fname, 'r') 467 lines = fd.readlines() 468 fd.close() 469 470 fd = open(fname, 'w') 471 text = series.cover 472 prefix = series.GetPatchPrefix() 473 for line in lines: 474 if line.startswith('Subject:'): 475 # TODO: if more than 10 patches this should save 00/xx, not 0/xx 476 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0]) 477 478 # Insert our cover letter 479 elif line.startswith('*** BLURB HERE ***'): 480 # First the blurb test 481 line = '\n'.join(text[1:]) + '\n' 482 if series.get('notes'): 483 line += '\n'.join(series.notes) + '\n' 484 485 # Now the change list 486 out = series.MakeChangeLog(None) 487 line += '\n' + '\n'.join(out) 488 fd.write(line) 489 fd.close() 490