1# Copyright (c) 2011 The Chromium OS Authors. 2# 3# See file CREDITS for list of people who contributed to this 4# project. 5# 6# This program is free software; you can redistribute it and/or 7# modify it under the terms of the GNU General Public License as 8# published by the Free Software Foundation; either version 2 of 9# the License, or (at your option) any later version. 10# 11# This program is distributed in the hope that it will be useful, 12# but WITHOUT ANY WARRANTY; without even the implied warranty of 13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14# GNU General Public License for more details. 15# 16# You should have received a copy of the GNU General Public License 17# along with this program; if not, write to the Free Software 18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, 19# MA 02111-1307 USA 20# 21 22import os 23import re 24import shutil 25import tempfile 26 27import command 28import commit 29import gitutil 30from series import Series 31 32# Tags that we detect and remove 33re_remove = re.compile('^BUG=|^TEST=|^Change-Id:|^Review URL:' 34 '|Reviewed-on:|Reviewed-by:') 35 36# Lines which are allowed after a TEST= line 37re_allowed_after_test = re.compile('^Signed-off-by:') 38 39# The start of the cover letter 40re_cover = re.compile('^Cover-letter:') 41 42# Patch series tag 43re_series = re.compile('^Series-(\w*): *(.*)') 44 45# Commit tags that we want to collect and keep 46re_tag = re.compile('^(Tested-by|Acked-by|Signed-off-by|Cc): (.*)') 47 48# The start of a new commit in the git log 49re_commit = re.compile('^commit (.*)') 50 51# We detect these since checkpatch doesn't always do it 52re_space_before_tab = re.compile('^[+].* \t') 53 54# States we can be in - can we use range() and still have comments? 55STATE_MSG_HEADER = 0 # Still in the message header 56STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit) 57STATE_PATCH_HEADER = 2 # In patch header (after the subject) 58STATE_DIFFS = 3 # In the diff part (past --- line) 59 60class PatchStream: 61 """Class for detecting/injecting tags in a patch or series of patches 62 63 We support processing the output of 'git log' to read out the tags we 64 are interested in. We can also process a patch file in order to remove 65 unwanted tags or inject additional ones. These correspond to the two 66 phases of processing. 67 """ 68 def __init__(self, series, name=None, is_log=False): 69 self.skip_blank = False # True to skip a single blank line 70 self.found_test = False # Found a TEST= line 71 self.lines_after_test = 0 # MNumber of lines found after TEST= 72 self.warn = [] # List of warnings we have collected 73 self.linenum = 1 # Output line number we are up to 74 self.in_section = None # Name of start...END section we are in 75 self.notes = [] # Series notes 76 self.section = [] # The current section...END section 77 self.series = series # Info about the patch series 78 self.is_log = is_log # True if indent like git log 79 self.in_change = 0 # Non-zero if we are in a change list 80 self.blank_count = 0 # Number of blank lines stored up 81 self.state = STATE_MSG_HEADER # What state are we in? 82 self.tags = [] # Tags collected, like Tested-by... 83 self.signoff = [] # Contents of signoff line 84 self.commit = None # Current commit 85 86 def AddToSeries(self, line, name, value): 87 """Add a new Series-xxx tag. 88 89 When a Series-xxx tag is detected, we come here to record it, if we 90 are scanning a 'git log'. 91 92 Args: 93 line: Source line containing tag (useful for debug/error messages) 94 name: Tag name (part after 'Series-') 95 value: Tag value (part after 'Series-xxx: ') 96 """ 97 if name == 'notes': 98 self.in_section = name 99 self.skip_blank = False 100 if self.is_log: 101 self.series.AddTag(self.commit, line, name, value) 102 103 def CloseCommit(self): 104 """Save the current commit into our commit list, and reset our state""" 105 if self.commit and self.is_log: 106 self.series.AddCommit(self.commit) 107 self.commit = None 108 109 def FormatTags(self, tags): 110 out_list = [] 111 for tag in sorted(tags): 112 if tag.startswith('Cc:'): 113 tag_list = tag[4:].split(',') 114 out_list += gitutil.BuildEmailList(tag_list, 'Cc:') 115 else: 116 out_list.append(tag) 117 return out_list 118 119 def ProcessLine(self, line): 120 """Process a single line of a patch file or commit log 121 122 This process a line and returns a list of lines to output. The list 123 may be empty or may contain multiple output lines. 124 125 This is where all the complicated logic is located. The class's 126 state is used to move between different states and detect things 127 properly. 128 129 We can be in one of two modes: 130 self.is_log == True: This is 'git log' mode, where most output is 131 indented by 4 characters and we are scanning for tags 132 133 self.is_log == False: This is 'patch' mode, where we already have 134 all the tags, and are processing patches to remove junk we 135 don't want, and add things we think are required. 136 137 Args: 138 line: text line to process 139 140 Returns: 141 list of output lines, or [] if nothing should be output 142 """ 143 # Initially we have no output. Prepare the input line string 144 out = [] 145 line = line.rstrip('\n') 146 if self.is_log: 147 if line[:4] == ' ': 148 line = line[4:] 149 150 # Handle state transition and skipping blank lines 151 series_match = re_series.match(line) 152 commit_match = re_commit.match(line) if self.is_log else None 153 tag_match = None 154 if self.state == STATE_PATCH_HEADER: 155 tag_match = re_tag.match(line) 156 is_blank = not line.strip() 157 if is_blank: 158 if (self.state == STATE_MSG_HEADER 159 or self.state == STATE_PATCH_SUBJECT): 160 self.state += 1 161 162 # We don't have a subject in the text stream of patch files 163 # It has its own line with a Subject: tag 164 if not self.is_log and self.state == STATE_PATCH_SUBJECT: 165 self.state += 1 166 elif commit_match: 167 self.state = STATE_MSG_HEADER 168 169 # If we are in a section, keep collecting lines until we see END 170 if self.in_section: 171 if line == 'END': 172 if self.in_section == 'cover': 173 self.series.cover = self.section 174 elif self.in_section == 'notes': 175 if self.is_log: 176 self.series.notes += self.section 177 else: 178 self.warn.append("Unknown section '%s'" % self.in_section) 179 self.in_section = None 180 self.skip_blank = True 181 self.section = [] 182 else: 183 self.section.append(line) 184 185 # Detect the commit subject 186 elif not is_blank and self.state == STATE_PATCH_SUBJECT: 187 self.commit.subject = line 188 189 # Detect the tags we want to remove, and skip blank lines 190 elif re_remove.match(line): 191 self.skip_blank = True 192 193 # TEST= should be the last thing in the commit, so remove 194 # everything after it 195 if line.startswith('TEST='): 196 self.found_test = True 197 elif self.skip_blank and is_blank: 198 self.skip_blank = False 199 200 # Detect the start of a cover letter section 201 elif re_cover.match(line): 202 self.in_section = 'cover' 203 self.skip_blank = False 204 205 # If we are in a change list, key collected lines until a blank one 206 elif self.in_change: 207 if is_blank: 208 # Blank line ends this change list 209 self.in_change = 0 210 else: 211 self.series.AddChange(self.in_change, self.commit, line) 212 self.skip_blank = False 213 214 # Detect Series-xxx tags 215 elif series_match: 216 name = series_match.group(1) 217 value = series_match.group(2) 218 if name == 'changes': 219 # value is the version number: e.g. 1, or 2 220 try: 221 value = int(value) 222 except ValueError as str: 223 raise ValueError("%s: Cannot decode version info '%s'" % 224 (self.commit.hash, line)) 225 self.in_change = int(value) 226 else: 227 self.AddToSeries(line, name, value) 228 self.skip_blank = True 229 230 # Detect the start of a new commit 231 elif commit_match: 232 self.CloseCommit() 233 self.commit = commit.Commit(commit_match.group(1)[:7]) 234 235 # Detect tags in the commit message 236 elif tag_match: 237 # Onlly allow a single signoff tag 238 if tag_match.group(1) == 'Signed-off-by': 239 if self.signoff: 240 self.warn.append('Patch has more than one Signed-off-by ' 241 'tag') 242 self.signoff += [line] 243 244 # Remove Tested-by self, since few will take much notice 245 elif (tag_match.group(1) == 'Tested-by' and 246 tag_match.group(2).find(os.getenv('USER') + '@') != -1): 247 self.warn.append("Ignoring %s" % line) 248 elif tag_match.group(1) == 'Cc': 249 self.commit.AddCc(tag_match.group(2).split(',')) 250 else: 251 self.tags.append(line); 252 253 # Well that means this is an ordinary line 254 else: 255 pos = 1 256 # Look for ugly ASCII characters 257 for ch in line: 258 # TODO: Would be nicer to report source filename and line 259 if ord(ch) > 0x80: 260 self.warn.append("Line %d/%d ('%s') has funny ascii char" % 261 (self.linenum, pos, line)) 262 pos += 1 263 264 # Look for space before tab 265 m = re_space_before_tab.match(line) 266 if m: 267 self.warn.append('Line %d/%d has space before tab' % 268 (self.linenum, m.start())) 269 270 # OK, we have a valid non-blank line 271 out = [line] 272 self.linenum += 1 273 self.skip_blank = False 274 if self.state == STATE_DIFFS: 275 pass 276 277 # If this is the start of the diffs section, emit our tags and 278 # change log 279 elif line == '---': 280 self.state = STATE_DIFFS 281 282 # Output the tags (signeoff first), then change list 283 out = [] 284 if self.signoff: 285 out += self.signoff 286 log = self.series.MakeChangeLog(self.commit) 287 out += self.FormatTags(self.tags) 288 out += [line] + log 289 elif self.found_test: 290 if not re_allowed_after_test.match(line): 291 self.lines_after_test += 1 292 293 return out 294 295 def Finalize(self): 296 """Close out processing of this patch stream""" 297 self.CloseCommit() 298 if self.lines_after_test: 299 self.warn.append('Found %d lines after TEST=' % 300 self.lines_after_test) 301 302 def ProcessStream(self, infd, outfd): 303 """Copy a stream from infd to outfd, filtering out unwanting things. 304 305 This is used to process patch files one at a time. 306 307 Args: 308 infd: Input stream file object 309 outfd: Output stream file object 310 """ 311 # Extract the filename from each diff, for nice warnings 312 fname = None 313 last_fname = None 314 re_fname = re.compile('diff --git a/(.*) b/.*') 315 while True: 316 line = infd.readline() 317 if not line: 318 break 319 out = self.ProcessLine(line) 320 321 # Try to detect blank lines at EOF 322 for line in out: 323 match = re_fname.match(line) 324 if match: 325 last_fname = fname 326 fname = match.group(1) 327 if line == '+': 328 self.blank_count += 1 329 else: 330 if self.blank_count and (line == '-- ' or match): 331 self.warn.append("Found possible blank line(s) at " 332 "end of file '%s'" % last_fname) 333 outfd.write('+\n' * self.blank_count) 334 outfd.write(line + '\n') 335 self.blank_count = 0 336 self.Finalize() 337 338 339def GetMetaData(start, count): 340 """Reads out patch series metadata from the commits 341 342 This does a 'git log' on the relevant commits and pulls out the tags we 343 are interested in. 344 345 Args: 346 start: Commit to start from: 0=HEAD, 1=next one, etc. 347 count: Number of commits to list 348 """ 349 pipe = [['git', 'log', '--reverse', 'HEAD~%d' % start, '-n%d' % count]] 350 stdout = command.RunPipe(pipe, capture=True) 351 series = Series() 352 ps = PatchStream(series, is_log=True) 353 for line in stdout.splitlines(): 354 ps.ProcessLine(line) 355 ps.Finalize() 356 return series 357 358def FixPatch(backup_dir, fname, series, commit): 359 """Fix up a patch file, by adding/removing as required. 360 361 We remove our tags from the patch file, insert changes lists, etc. 362 The patch file is processed in place, and overwritten. 363 364 A backup file is put into backup_dir (if not None). 365 366 Args: 367 fname: Filename to patch file to process 368 series: Series information about this patch set 369 commit: Commit object for this patch file 370 Return: 371 A list of errors, or [] if all ok. 372 """ 373 handle, tmpname = tempfile.mkstemp() 374 outfd = os.fdopen(handle, 'w') 375 infd = open(fname, 'r') 376 ps = PatchStream(series) 377 ps.commit = commit 378 ps.ProcessStream(infd, outfd) 379 infd.close() 380 outfd.close() 381 382 # Create a backup file if required 383 if backup_dir: 384 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname))) 385 shutil.move(tmpname, fname) 386 return ps.warn 387 388def FixPatches(series, fnames): 389 """Fix up a list of patches identified by filenames 390 391 The patch files are processed in place, and overwritten. 392 393 Args: 394 series: The series object 395 fnames: List of patch files to process 396 """ 397 # Current workflow creates patches, so we shouldn't need a backup 398 backup_dir = None #tempfile.mkdtemp('clean-patch') 399 count = 0 400 for fname in fnames: 401 commit = series.commits[count] 402 commit.patch = fname 403 result = FixPatch(backup_dir, fname, series, commit) 404 if result: 405 print '%d warnings for %s:' % (len(result), fname) 406 for warn in result: 407 print '\t', warn 408 print 409 count += 1 410 print 'Cleaned %d patches' % count 411 return series 412 413def InsertCoverLetter(fname, series, count): 414 """Inserts a cover letter with the required info into patch 0 415 416 Args: 417 fname: Input / output filename of the cover letter file 418 series: Series object 419 count: Number of patches in the series 420 """ 421 fd = open(fname, 'r') 422 lines = fd.readlines() 423 fd.close() 424 425 fd = open(fname, 'w') 426 text = series.cover 427 prefix = series.GetPatchPrefix() 428 for line in lines: 429 if line.startswith('Subject:'): 430 # TODO: if more than 10 patches this should save 00/xx, not 0/xx 431 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0]) 432 433 # Insert our cover letter 434 elif line.startswith('*** BLURB HERE ***'): 435 # First the blurb test 436 line = '\n'.join(text[1:]) + '\n' 437 if series.get('notes'): 438 line += '\n'.join(series.notes) + '\n' 439 440 # Now the change list 441 out = series.MakeChangeLog(None) 442 line += '\n' + '\n'.join(out) 443 fd.write(line) 444 fd.close() 445