1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# See file CREDITS for list of people who contributed to this 4# project. 5# 6# This program is free software; you can redistribute it and/or 7# modify it under the terms of the GNU General Public License as 8# published by the Free Software Foundation; either version 2 of 9# the License, or (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program; if not, write to the Free Software 18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, 19# MA 02111-1307 USA 20# 21 22import os 23import re 24import shutil 25import tempfile 26 27import command 28import commit 29import gitutil 30from series import Series 31 32# Tags that we detect and remove 33re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:' 34 '|Reviewed-on:|Commit-\w*:') 35 36# Lines which are allowed after a TEST= line 37re_allowed_after_test = re.compile('^Signed-off-by:') 38 39# Signoffs 40re_signoff = re.compile('^Signed-off-by:') 41 42# The start of the cover letter 43re_cover = re.compile('^Cover-letter:') 44 45# A cover letter Cc 46re_cover_cc = re.compile('^Cover-letter-cc: *(.*)') 47 48# Patch series tag 49re_series = re.compile('^Series-([a-z-]*): *(.*)') 50 51# Commit tags that we want to collect and keep 52re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Cc): (.*)') 53 54# The start of a new commit in the git log 55re_commit = re.compile('^commit ([0-9a-f]*)$') 56 57# We detect these since checkpatch doesn't always do it 58re_space_before_tab = re.compile('^[+].* \t') 59 60# States we can be in - can we use range() and still have comments? 61STATE_MSG_HEADER = 0 # Still in the message header 62STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 63STATE_PATCH_HEADER = 2 # In patch header (after the subject) 64STATE_DIFFS = 3 # In the diff part (past --- line) 65 66class PatchStream: 67 """Class for detecting/injecting tags in a patch or series of patches 68 69 We support processing the output of 'git log' to read out the tags we 70 are interested in. We can also process a patch file in order to remove 71 unwanted tags or inject additional ones. These correspond to the two 72 phases of processing. 73 """ 74 def __init__(self, series, name=None, is_log=False): 75 self.skip_blank = False # True to skip a single blank line 76 self.found_test = False # Found a TEST= line 77 self.lines_after_test = 0 # MNumber of lines found after TEST= 78 self.warn = [] # List of warnings we have collected 79 self.linenum = 1 # Output line number we are up to 80 self.in_section = None # Name of start...END section we are in 81 self.notes = [] # Series notes 82 self.section = [] # The current section...END section 83 self.series = series # Info about the patch series 84 self.is_log = is_log # True if indent like git log 85 self.in_change = 0 # Non-zero if we are in a change list 86 self.blank_count = 0 # Number of blank lines stored up 87 self.state = STATE_MSG_HEADER # What state are we in? 88 self.tags = [] # Tags collected, like Tested-by... 89 self.signoff = [] # Contents of signoff line 90 self.commit = None # Current commit 91 92 def AddToSeries(self, line, name, value): 93 """Add a new Series-xxx tag. 94 95 When a Series-xxx tag is detected, we come here to record it, if we 96 are scanning a 'git log'. 97 98 Args: 99 line: Source line containing tag (useful for debug/error messages) 100 name: Tag name (part after 'Series-') 101 value: Tag value (part after 'Series-xxx: ') 102 """ 103 if name == 'notes': 104 self.in_section = name 105 self.skip_blank = False 106 if self.is_log: 107 self.series.AddTag(self.commit, line, name, value) 108 109 def CloseCommit(self): 110 """Save the current commit into our commit list, and reset our state""" 111 if self.commit and self.is_log: 112 self.series.AddCommit(self.commit) 113 self.commit = None 114 115 def FormatTags(self, tags): 116 out_list = [] 117 for tag in sorted(tags): 118 if tag.startswith('Cc:'): 119 tag_list = tag[4:].split(',') 120 out_list += gitutil.BuildEmailList(tag_list, 'Cc:') 121 else: 122 out_list.append(tag) 123 return out_list 124 125 def ProcessLine(self, line): 126 """Process a single line of a patch file or commit log 127 128 This process a line and returns a list of lines to output. The list 129 may be empty or may contain multiple output lines. 130 131 This is where all the complicated logic is located. The class's 132 state is used to move between different states and detect things 133 properly. 134 135 We can be in one of two modes: 136 self.is_log == True: This is 'git log' mode, where most output is 137 indented by 4 characters and we are scanning for tags 138 139 self.is_log == False: This is 'patch' mode, where we already have 140 all the tags, and are processing patches to remove junk we 141 don't want, and add things we think are required. 142 143 Args: 144 line: text line to process 145 146 Returns: 147 list of output lines, or [] if nothing should be output 148 """ 149 # Initially we have no output. Prepare the input line string 150 out = [] 151 line = line.rstrip('\n') 152 if self.is_log: 153 if line[:4] == ' ': 154 line = line[4:] 155 156 # Handle state transition and skipping blank lines 157 series_match = re_series.match(line) 158 commit_match = re_commit.match(line) if self.is_log else None 159 cover_cc_match = re_cover_cc.match(line) 160 tag_match = None 161 if self.state == STATE_PATCH_HEADER: 162 tag_match = re_tag.match(line) 163 is_blank = not line.strip() 164 if is_blank: 165 if (self.state == STATE_MSG_HEADER 166 or self.state == STATE_PATCH_SUBJECT): 167 self.state += 1 168 169 # We don't have a subject in the text stream of patch files 170 # It has its own line with a Subject: tag 171 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 172 self.state += 1 173 elif commit_match: 174 self.state = STATE_MSG_HEADER 175 176 # If we are in a section, keep collecting lines until we see END 177 if self.in_section: 178 if line == 'END': 179 if self.in_section == 'cover': 180 self.series.cover = self.section 181 elif self.in_section == 'notes': 182 if self.is_log: 183 self.series.notes += self.section 184 else: 185 self.warn.append("Unknown section '%s'" % self.in_section) 186 self.in_section = None 187 self.skip_blank = True 188 self.section = [] 189 else: 190 self.section.append(line) 191 192 # Detect the commit subject 193 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 194 self.commit.subject = line 195 196 # Detect the tags we want to remove, and skip blank lines 197 elif re_remove.match(line): 198 self.skip_blank = True 199 200 # TEST= should be the last thing in the commit, so remove 201 # everything after it 202 if line.startswith('TEST='): 203 self.found_test = True 204 elif self.skip_blank and is_blank: 205 self.skip_blank = False 206 207 # Detect the start of a cover letter section 208 elif re_cover.match(line): 209 self.in_section = 'cover' 210 self.skip_blank = False 211 212 elif cover_cc_match: 213 value = cover_cc_match.group(1) 214 self.AddToSeries(line, 'cover-cc', value) 215 216 # If we are in a change list, key collected lines until a blank one 217 elif self.in_change: 218 if is_blank: 219 # Blank line ends this change list 220 self.in_change = 0 221 elif line == '---' or re_signoff.match(line): 222 self.in_change = 0 223 out = self.ProcessLine(line) 224 else: 225 if self.is_log: 226 self.series.AddChange(self.in_change, self.commit, line) 227 self.skip_blank = False 228 229 # Detect Series-xxx tags 230 elif series_match: 231 name = series_match.group(1) 232 value = series_match.group(2) 233 if name == 'changes': 234 # value is the version number: e.g. 1, or 2 235 try: 236 value = int(value) 237 except ValueError as str: 238 raise ValueError("%s: Cannot decode version info '%s'" % 239 (self.commit.hash, line)) 240 self.in_change = int(value) 241 else: 242 self.AddToSeries(line, name, value) 243 self.skip_blank = True 244 245 # Detect the start of a new commit 246 elif commit_match: 247 self.CloseCommit() 248 # TODO: We should store the whole hash, and just display a subset 249 self.commit = commit.Commit(commit_match.group(1)[:8]) 250 251 # Detect tags in the commit message 252 elif tag_match: 253 # Remove Tested-by self, since few will take much notice 254 if (tag_match.group(1) == 'Tested-by' and 255 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 256 self.warn.append("Ignoring %s" % line) 257 elif tag_match.group(1) == 'Cc': 258 self.commit.AddCc(tag_match.group(2).split(',')) 259 else: 260 self.tags.append(line); 261 262 # Well that means this is an ordinary line 263 else: 264 pos = 1 265 # Look for ugly ASCII characters 266 for ch in line: 267 # TODO: Would be nicer to report source filename and line 268 if ord(ch) > 0x80: 269 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 270 (self.linenum, pos, line)) 271 pos += 1 272 273 # Look for space before tab 274 m = re_space_before_tab.match(line) 275 if m: 276 self.warn.append('Line %d/%d has space before tab' % 277 (self.linenum, m.start())) 278 279 # OK, we have a valid non-blank line 280 out = [line] 281 self.linenum += 1 282 self.skip_blank = False 283 if self.state == STATE_DIFFS: 284 pass 285 286 # If this is the start of the diffs section, emit our tags and 287 # change log 288 elif line == '---': 289 self.state = STATE_DIFFS 290 291 # Output the tags (signeoff first), then change list 292 out = [] 293 log = self.series.MakeChangeLog(self.commit) 294 out += self.FormatTags(self.tags) 295 out += [line] + log 296 elif self.found_test: 297 if not re_allowed_after_test.match(line): 298 self.lines_after_test += 1 299 300 return out 301 302 def Finalize(self): 303 """Close out processing of this patch stream""" 304 self.CloseCommit() 305 if self.lines_after_test: 306 self.warn.append('Found %d lines after TEST=' % 307 self.lines_after_test) 308 309 def ProcessStream(self, infd, outfd): 310 """Copy a stream from infd to outfd, filtering out unwanting things. 311 312 This is used to process patch files one at a time. 313 314 Args: 315 infd: Input stream file object 316 outfd: Output stream file object 317 """ 318 # Extract the filename from each diff, for nice warnings 319 fname = None 320 last_fname = None 321 re_fname = re.compile('diff --git a/(.*) b/.*') 322 while True: 323 line = infd.readline() 324 if not line: 325 break 326 out = self.ProcessLine(line) 327 328 # Try to detect blank lines at EOF 329 for line in out: 330 match = re_fname.match(line) 331 if match: 332 last_fname = fname 333 fname = match.group(1) 334 if line == '+': 335 self.blank_count += 1 336 else: 337 if self.blank_count and (line == '-- ' or match): 338 self.warn.append("Found possible blank line(s) at " 339 "end of file '%s'" % last_fname) 340 outfd.write('+\n' * self.blank_count) 341 outfd.write(line + '\n') 342 self.blank_count = 0 343 self.Finalize() 344 345 346def GetMetaDataForList(commit_range, git_dir=None, count=None, 347 series = Series()): 348 """Reads out patch series metadata from the commits 349 350 This does a 'git log' on the relevant commits and pulls out the tags we 351 are interested in. 352 353 Args: 354 commit_range: Range of commits to count (e.g. 'HEAD..base') 355 git_dir: Path to git repositiory (None to use default) 356 count: Number of commits to list, or None for no limit 357 series: Series object to add information into. By default a new series 358 is started. 359 Returns: 360 A Series object containing information about the commits. 361 """ 362 params = ['git', 'log', '--no-color', '--reverse', commit_range] 363 if count is not None: 364 params[2:2] = ['-n%d' % count] 365 if git_dir: 366 params[1:1] = ['--git-dir', git_dir] 367 pipe = [params] 368 stdout = command.RunPipe(pipe, capture=True).stdout 369 ps = PatchStream(series, is_log=True) 370 for line in stdout.splitlines(): 371 ps.ProcessLine(line) 372 ps.Finalize() 373 return series 374 375def GetMetaData(start, count): 376 """Reads out patch series metadata from the commits 377 378 This does a 'git log' on the relevant commits and pulls out the tags we 379 are interested in. 380 381 Args: 382 start: Commit to start from: 0=HEAD, 1=next one, etc. 383 count: Number of commits to list 384 """ 385 return GetMetaDataForList('HEAD~%d' % start, None, count) 386 387def FixPatch(backup_dir, fname, series, commit): 388 """Fix up a patch file, by adding/removing as required. 389 390 We remove our tags from the patch file, insert changes lists, etc. 391 The patch file is processed in place, and overwritten. 392 393 A backup file is put into backup_dir (if not None). 394 395 Args: 396 fname: Filename to patch file to process 397 series: Series information about this patch set 398 commit: Commit object for this patch file 399 Return: 400 A list of errors, or [] if all ok. 401 """ 402 handle, tmpname = tempfile.mkstemp() 403 outfd = os.fdopen(handle, 'w') 404 infd = open(fname, 'r') 405 ps = PatchStream(series) 406 ps.commit = commit 407 ps.ProcessStream(infd, outfd) 408 infd.close() 409 outfd.close() 410 411 # Create a backup file if required 412 if backup_dir: 413 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 414 shutil.move(tmpname, fname) 415 return ps.warn 416 417def FixPatches(series, fnames): 418 """Fix up a list of patches identified by filenames 419 420 The patch files are processed in place, and overwritten. 421 422 Args: 423 series: The series object 424 fnames: List of patch files to process 425 """ 426 # Current workflow creates patches, so we shouldn't need a backup 427 backup_dir = None #tempfile.mkdtemp('clean-patch') 428 count = 0 429 for fname in fnames: 430 commit = series.commits[count] 431 commit.patch = fname 432 result = FixPatch(backup_dir, fname, series, commit) 433 if result: 434 print '%d warnings for %s:' % (len(result), fname) 435 for warn in result: 436 print '\t', warn 437 print 438 count += 1 439 print 'Cleaned %d patches' % count 440 return series 441 442def InsertCoverLetter(fname, series, count): 443 """Inserts a cover letter with the required info into patch 0 444 445 Args: 446 fname: Input / output filename of the cover letter file 447 series: Series object 448 count: Number of patches in the series 449 """ 450 fd = open(fname, 'r') 451 lines = fd.readlines() 452 fd.close() 453 454 fd = open(fname, 'w') 455 text = series.cover 456 prefix = series.GetPatchPrefix() 457 for line in lines: 458 if line.startswith('Subject:'): 459 # TODO: if more than 10 patches this should save 00/xx, not 0/xx 460 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0]) 461 462 # Insert our cover letter 463 elif line.startswith('*** BLURB HERE ***'): 464 # First the blurb test 465 line = '\n'.join(text[1:]) + '\n' 466 if series.get('notes'): 467 line += '\n'.join(series.notes) + '\n' 468 469 # Now the change list 470 out = series.MakeChangeLog(None) 471 line += '\n' + '\n'.join(out) 472 fd.write(line) 473 fd.close() 474