xref: /openbmc/u-boot/tools/patman/patchstream.py (revision e8f80a5a)
1# SPDX-License-Identifier: GPL-2.0+
2# Copyright (c) 2011 The Chromium OS Authors.
3#
4
5import math
6import os
7import re
8import shutil
9import tempfile
10
11import command
12import commit
13import gitutil
14from series import Series
15
16# Tags that we detect and remove
17re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
18    '|Reviewed-on:|Commit-\w*:')
19
20# Lines which are allowed after a TEST= line
21re_allowed_after_test = re.compile('^Signed-off-by:')
22
23# Signoffs
24re_signoff = re.compile('^Signed-off-by: *(.*)')
25
26# The start of the cover letter
27re_cover = re.compile('^Cover-letter:')
28
29# A cover letter Cc
30re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
31
32# Patch series tag
33re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
34
35# Commit series tag
36re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
37
38# Commit tags that we want to collect and keep
39re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
40
41# The start of a new commit in the git log
42re_commit = re.compile('^commit ([0-9a-f]*)$')
43
44# We detect these since checkpatch doesn't always do it
45re_space_before_tab = re.compile('^[+].* \t')
46
47# States we can be in - can we use range() and still have comments?
48STATE_MSG_HEADER = 0        # Still in the message header
49STATE_PATCH_SUBJECT = 1     # In patch subject (first line of log for a commit)
50STATE_PATCH_HEADER = 2      # In patch header (after the subject)
51STATE_DIFFS = 3             # In the diff part (past --- line)
52
53class PatchStream:
54    """Class for detecting/injecting tags in a patch or series of patches
55
56    We support processing the output of 'git log' to read out the tags we
57    are interested in. We can also process a patch file in order to remove
58    unwanted tags or inject additional ones. These correspond to the two
59    phases of processing.
60    """
61    def __init__(self, series, name=None, is_log=False):
62        self.skip_blank = False          # True to skip a single blank line
63        self.found_test = False          # Found a TEST= line
64        self.lines_after_test = 0        # MNumber of lines found after TEST=
65        self.warn = []                   # List of warnings we have collected
66        self.linenum = 1                 # Output line number we are up to
67        self.in_section = None           # Name of start...END section we are in
68        self.notes = []                  # Series notes
69        self.section = []                # The current section...END section
70        self.series = series             # Info about the patch series
71        self.is_log = is_log             # True if indent like git log
72        self.in_change = 0               # Non-zero if we are in a change list
73        self.blank_count = 0             # Number of blank lines stored up
74        self.state = STATE_MSG_HEADER    # What state are we in?
75        self.signoff = []                # Contents of signoff line
76        self.commit = None               # Current commit
77
78    def AddToSeries(self, line, name, value):
79        """Add a new Series-xxx tag.
80
81        When a Series-xxx tag is detected, we come here to record it, if we
82        are scanning a 'git log'.
83
84        Args:
85            line: Source line containing tag (useful for debug/error messages)
86            name: Tag name (part after 'Series-')
87            value: Tag value (part after 'Series-xxx: ')
88        """
89        if name == 'notes':
90            self.in_section = name
91            self.skip_blank = False
92        if self.is_log:
93            self.series.AddTag(self.commit, line, name, value)
94
95    def AddToCommit(self, line, name, value):
96        """Add a new Commit-xxx tag.
97
98        When a Commit-xxx tag is detected, we come here to record it.
99
100        Args:
101            line: Source line containing tag (useful for debug/error messages)
102            name: Tag name (part after 'Commit-')
103            value: Tag value (part after 'Commit-xxx: ')
104        """
105        if name == 'notes':
106            self.in_section = 'commit-' + name
107            self.skip_blank = False
108
109    def CloseCommit(self):
110        """Save the current commit into our commit list, and reset our state"""
111        if self.commit and self.is_log:
112            self.series.AddCommit(self.commit)
113            self.commit = None
114        # If 'END' is missing in a 'Cover-letter' section, and that section
115        # happens to show up at the very end of the commit message, this is
116        # the chance for us to fix it up.
117        if self.in_section == 'cover' and self.is_log:
118            self.series.cover = self.section
119            self.in_section = None
120            self.skip_blank = True
121            self.section = []
122
123    def ProcessLine(self, line):
124        """Process a single line of a patch file or commit log
125
126        This process a line and returns a list of lines to output. The list
127        may be empty or may contain multiple output lines.
128
129        This is where all the complicated logic is located. The class's
130        state is used to move between different states and detect things
131        properly.
132
133        We can be in one of two modes:
134            self.is_log == True: This is 'git log' mode, where most output is
135                indented by 4 characters and we are scanning for tags
136
137            self.is_log == False: This is 'patch' mode, where we already have
138                all the tags, and are processing patches to remove junk we
139                don't want, and add things we think are required.
140
141        Args:
142            line: text line to process
143
144        Returns:
145            list of output lines, or [] if nothing should be output
146        """
147        # Initially we have no output. Prepare the input line string
148        out = []
149        line = line.rstrip('\n')
150
151        commit_match = re_commit.match(line) if self.is_log else None
152
153        if self.is_log:
154            if line[:4] == '    ':
155                line = line[4:]
156
157        # Handle state transition and skipping blank lines
158        series_tag_match = re_series_tag.match(line)
159        commit_tag_match = re_commit_tag.match(line)
160        cover_match = re_cover.match(line)
161        cover_cc_match = re_cover_cc.match(line)
162        signoff_match = re_signoff.match(line)
163        tag_match = None
164        if self.state == STATE_PATCH_HEADER:
165            tag_match = re_tag.match(line)
166        is_blank = not line.strip()
167        if is_blank:
168            if (self.state == STATE_MSG_HEADER
169                    or self.state == STATE_PATCH_SUBJECT):
170                self.state += 1
171
172            # We don't have a subject in the text stream of patch files
173            # It has its own line with a Subject: tag
174            if not self.is_log and self.state == STATE_PATCH_SUBJECT:
175                self.state += 1
176        elif commit_match:
177            self.state = STATE_MSG_HEADER
178
179        # If a tag is detected, or a new commit starts
180        if series_tag_match or commit_tag_match or \
181           cover_match or cover_cc_match or signoff_match or \
182           self.state == STATE_MSG_HEADER:
183            # but we are already in a section, this means 'END' is missing
184            # for that section, fix it up.
185            if self.in_section:
186                self.warn.append("Missing 'END' in section '%s'" % self.in_section)
187                if self.in_section == 'cover':
188                    self.series.cover = self.section
189                elif self.in_section == 'notes':
190                    if self.is_log:
191                        self.series.notes += self.section
192                elif self.in_section == 'commit-notes':
193                    if self.is_log:
194                        self.commit.notes += self.section
195                else:
196                    self.warn.append("Unknown section '%s'" % self.in_section)
197                self.in_section = None
198                self.skip_blank = True
199                self.section = []
200            # but we are already in a change list, that means a blank line
201            # is missing, fix it up.
202            if self.in_change:
203                self.warn.append("Missing 'blank line' in section 'Series-changes'")
204                self.in_change = 0
205
206        # If we are in a section, keep collecting lines until we see END
207        if self.in_section:
208            if line == 'END':
209                if self.in_section == 'cover':
210                    self.series.cover = self.section
211                elif self.in_section == 'notes':
212                    if self.is_log:
213                        self.series.notes += self.section
214                elif self.in_section == 'commit-notes':
215                    if self.is_log:
216                        self.commit.notes += self.section
217                else:
218                    self.warn.append("Unknown section '%s'" % self.in_section)
219                self.in_section = None
220                self.skip_blank = True
221                self.section = []
222            else:
223                self.section.append(line)
224
225        # Detect the commit subject
226        elif not is_blank and self.state == STATE_PATCH_SUBJECT:
227            self.commit.subject = line
228
229        # Detect the tags we want to remove, and skip blank lines
230        elif re_remove.match(line) and not commit_tag_match:
231            self.skip_blank = True
232
233            # TEST= should be the last thing in the commit, so remove
234            # everything after it
235            if line.startswith('TEST='):
236                self.found_test = True
237        elif self.skip_blank and is_blank:
238            self.skip_blank = False
239
240        # Detect the start of a cover letter section
241        elif cover_match:
242            self.in_section = 'cover'
243            self.skip_blank = False
244
245        elif cover_cc_match:
246            value = cover_cc_match.group(1)
247            self.AddToSeries(line, 'cover-cc', value)
248
249        # If we are in a change list, key collected lines until a blank one
250        elif self.in_change:
251            if is_blank:
252                # Blank line ends this change list
253                self.in_change = 0
254            elif line == '---':
255                self.in_change = 0
256                out = self.ProcessLine(line)
257            else:
258                if self.is_log:
259                    self.series.AddChange(self.in_change, self.commit, line)
260            self.skip_blank = False
261
262        # Detect Series-xxx tags
263        elif series_tag_match:
264            name = series_tag_match.group(1)
265            value = series_tag_match.group(2)
266            if name == 'changes':
267                # value is the version number: e.g. 1, or 2
268                try:
269                    value = int(value)
270                except ValueError as str:
271                    raise ValueError("%s: Cannot decode version info '%s'" %
272                        (self.commit.hash, line))
273                self.in_change = int(value)
274            else:
275                self.AddToSeries(line, name, value)
276                self.skip_blank = True
277
278        # Detect Commit-xxx tags
279        elif commit_tag_match:
280            name = commit_tag_match.group(1)
281            value = commit_tag_match.group(2)
282            if name == 'notes':
283                self.AddToCommit(line, name, value)
284                self.skip_blank = True
285
286        # Detect the start of a new commit
287        elif commit_match:
288            self.CloseCommit()
289            self.commit = commit.Commit(commit_match.group(1))
290
291        # Detect tags in the commit message
292        elif tag_match:
293            # Remove Tested-by self, since few will take much notice
294            if (tag_match.group(1) == 'Tested-by' and
295                    tag_match.group(2).find(os.getenv('USER') + '@') != -1):
296                self.warn.append("Ignoring %s" % line)
297            elif tag_match.group(1) == 'Patch-cc':
298                self.commit.AddCc(tag_match.group(2).split(','))
299            else:
300                out = [line]
301
302        # Suppress duplicate signoffs
303        elif signoff_match:
304            if (self.is_log or not self.commit or
305                self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
306                out = [line]
307
308        # Well that means this is an ordinary line
309        else:
310            # Look for space before tab
311            m = re_space_before_tab.match(line)
312            if m:
313                self.warn.append('Line %d/%d has space before tab' %
314                    (self.linenum, m.start()))
315
316            # OK, we have a valid non-blank line
317            out = [line]
318            self.linenum += 1
319            self.skip_blank = False
320            if self.state == STATE_DIFFS:
321                pass
322
323            # If this is the start of the diffs section, emit our tags and
324            # change log
325            elif line == '---':
326                self.state = STATE_DIFFS
327
328                # Output the tags (signeoff first), then change list
329                out = []
330                log = self.series.MakeChangeLog(self.commit)
331                out += [line]
332                if self.commit:
333                    out += self.commit.notes
334                out += [''] + log
335            elif self.found_test:
336                if not re_allowed_after_test.match(line):
337                    self.lines_after_test += 1
338
339        return out
340
341    def Finalize(self):
342        """Close out processing of this patch stream"""
343        self.CloseCommit()
344        if self.lines_after_test:
345            self.warn.append('Found %d lines after TEST=' %
346                    self.lines_after_test)
347
348    def ProcessStream(self, infd, outfd):
349        """Copy a stream from infd to outfd, filtering out unwanting things.
350
351        This is used to process patch files one at a time.
352
353        Args:
354            infd: Input stream file object
355            outfd: Output stream file object
356        """
357        # Extract the filename from each diff, for nice warnings
358        fname = None
359        last_fname = None
360        re_fname = re.compile('diff --git a/(.*) b/.*')
361        while True:
362            line = infd.readline()
363            if not line:
364                break
365            out = self.ProcessLine(line)
366
367            # Try to detect blank lines at EOF
368            for line in out:
369                match = re_fname.match(line)
370                if match:
371                    last_fname = fname
372                    fname = match.group(1)
373                if line == '+':
374                    self.blank_count += 1
375                else:
376                    if self.blank_count and (line == '-- ' or match):
377                        self.warn.append("Found possible blank line(s) at "
378                                "end of file '%s'" % last_fname)
379                    outfd.write('+\n' * self.blank_count)
380                    outfd.write(line + '\n')
381                    self.blank_count = 0
382        self.Finalize()
383
384
385def GetMetaDataForList(commit_range, git_dir=None, count=None,
386                       series = None, allow_overwrite=False):
387    """Reads out patch series metadata from the commits
388
389    This does a 'git log' on the relevant commits and pulls out the tags we
390    are interested in.
391
392    Args:
393        commit_range: Range of commits to count (e.g. 'HEAD..base')
394        git_dir: Path to git repositiory (None to use default)
395        count: Number of commits to list, or None for no limit
396        series: Series object to add information into. By default a new series
397            is started.
398        allow_overwrite: Allow tags to overwrite an existing tag
399    Returns:
400        A Series object containing information about the commits.
401    """
402    if not series:
403        series = Series()
404    series.allow_overwrite = allow_overwrite
405    params = gitutil.LogCmd(commit_range, reverse=True, count=count,
406                            git_dir=git_dir)
407    stdout = command.RunPipe([params], capture=True).stdout
408    ps = PatchStream(series, is_log=True)
409    for line in stdout.splitlines():
410        ps.ProcessLine(line)
411    ps.Finalize()
412    return series
413
414def GetMetaData(start, count):
415    """Reads out patch series metadata from the commits
416
417    This does a 'git log' on the relevant commits and pulls out the tags we
418    are interested in.
419
420    Args:
421        start: Commit to start from: 0=HEAD, 1=next one, etc.
422        count: Number of commits to list
423    """
424    return GetMetaDataForList('HEAD~%d' % start, None, count)
425
426def GetMetaDataForTest(text):
427    """Process metadata from a file containing a git log. Used for tests
428
429    Args:
430        text:
431    """
432    series = Series()
433    ps = PatchStream(series, is_log=True)
434    for line in text.splitlines():
435        ps.ProcessLine(line)
436    ps.Finalize()
437    return series
438
439def FixPatch(backup_dir, fname, series, commit):
440    """Fix up a patch file, by adding/removing as required.
441
442    We remove our tags from the patch file, insert changes lists, etc.
443    The patch file is processed in place, and overwritten.
444
445    A backup file is put into backup_dir (if not None).
446
447    Args:
448        fname: Filename to patch file to process
449        series: Series information about this patch set
450        commit: Commit object for this patch file
451    Return:
452        A list of errors, or [] if all ok.
453    """
454    handle, tmpname = tempfile.mkstemp()
455    outfd = os.fdopen(handle, 'w')
456    infd = open(fname, 'r')
457    ps = PatchStream(series)
458    ps.commit = commit
459    ps.ProcessStream(infd, outfd)
460    infd.close()
461    outfd.close()
462
463    # Create a backup file if required
464    if backup_dir:
465        shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
466    shutil.move(tmpname, fname)
467    return ps.warn
468
469def FixPatches(series, fnames):
470    """Fix up a list of patches identified by filenames
471
472    The patch files are processed in place, and overwritten.
473
474    Args:
475        series: The series object
476        fnames: List of patch files to process
477    """
478    # Current workflow creates patches, so we shouldn't need a backup
479    backup_dir = None  #tempfile.mkdtemp('clean-patch')
480    count = 0
481    for fname in fnames:
482        commit = series.commits[count]
483        commit.patch = fname
484        result = FixPatch(backup_dir, fname, series, commit)
485        if result:
486            print('%d warnings for %s:' % (len(result), fname))
487            for warn in result:
488                print('\t', warn)
489            print
490        count += 1
491    print('Cleaned %d patches' % count)
492
493def InsertCoverLetter(fname, series, count):
494    """Inserts a cover letter with the required info into patch 0
495
496    Args:
497        fname: Input / output filename of the cover letter file
498        series: Series object
499        count: Number of patches in the series
500    """
501    fd = open(fname, 'r')
502    lines = fd.readlines()
503    fd.close()
504
505    fd = open(fname, 'w')
506    text = series.cover
507    prefix = series.GetPatchPrefix()
508    for line in lines:
509        if line.startswith('Subject:'):
510            # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
511            zero_repeat = int(math.log10(count)) + 1
512            zero = '0' * zero_repeat
513            line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
514
515        # Insert our cover letter
516        elif line.startswith('*** BLURB HERE ***'):
517            # First the blurb test
518            line = '\n'.join(text[1:]) + '\n'
519            if series.get('notes'):
520                line += '\n'.join(series.notes) + '\n'
521
522            # Now the change list
523            out = series.MakeChangeLog(None)
524            line += '\n' + '\n'.join(out)
525        fd.write(line)
526    fd.close()
527