1#!/usr/bin/env python2
2
3"""Find Kconfig symbols that are referenced but not defined."""
4
5# (c) 2014-2015 Valentin Rothberg <valentinrothberg@gmail.com>
6# (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
7#
8# Licensed under the terms of the GNU GPL License version 2
9
10
11import os
12import re
13import sys
14from subprocess import Popen, PIPE, STDOUT
15from optparse import OptionParser
16
17
18# regex expressions
19OPERATORS = r"&|\(|\)|\||\!"
20FEATURE = r"(?:\w*[A-Z0-9]\w*){2,}"
21DEF = r"^\s*(?:menu){,1}config\s+(" + FEATURE + r")\s*"
22EXPR = r"(?:" + OPERATORS + r"|\s|" + FEATURE + r")+"
23DEFAULT = r"default\s+.*?(?:if\s.+){,1}"
24STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR
25SOURCE_FEATURE = r"(?:\W|\b)+[D]{,1}CONFIG_(" + FEATURE + r")"
26
27# regex objects
28REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$")
29REGEX_FEATURE = re.compile(r'(?!\B"[^"]*)' + FEATURE + r'(?![^"]*"\B)')
30REGEX_SOURCE_FEATURE = re.compile(SOURCE_FEATURE)
31REGEX_KCONFIG_DEF = re.compile(DEF)
32REGEX_KCONFIG_EXPR = re.compile(EXPR)
33REGEX_KCONFIG_STMT = re.compile(STMT)
34REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$")
35REGEX_FILTER_FEATURES = re.compile(r"[A-Za-z0-9]$")
36REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+")
37
38
39def parse_options():
40    """The user interface of this module."""
41    usage = "%prog [options]\n\n"                                              \
42            "Run this tool to detect Kconfig symbols that are referenced but " \
43            "not defined in\nKconfig.  The output of this tool has the "       \
44            "format \'Undefined symbol\\tFile list\'\n\n"                      \
45            "If no option is specified, %prog will default to check your\n"    \
46            "current tree.  Please note that specifying commits will "         \
47            "\'git reset --hard\'\nyour current tree!  You may save "          \
48            "uncommitted changes to avoid losing data."
49
50    parser = OptionParser(usage=usage)
51
52    parser.add_option('-c', '--commit', dest='commit', action='store',
53                      default="",
54                      help="Check if the specified commit (hash) introduces "
55                           "undefined Kconfig symbols.")
56
57    parser.add_option('-d', '--diff', dest='diff', action='store',
58                      default="",
59                      help="Diff undefined symbols between two commits.  The "
60                           "input format bases on Git log's "
61                           "\'commmit1..commit2\'.")
62
63    parser.add_option('-f', '--find', dest='find', action='store_true',
64                      default=False,
65                      help="Find and show commits that may cause symbols to be "
66                           "missing.  Required to run with --diff.")
67
68    parser.add_option('-i', '--ignore', dest='ignore', action='store',
69                      default="",
70                      help="Ignore files matching this pattern.  Note that "
71                           "the pattern needs to be a Python regex.  To "
72                           "ignore defconfigs, specify -i '.*defconfig'.")
73
74    parser.add_option('', '--force', dest='force', action='store_true',
75                      default=False,
76                      help="Reset current Git tree even when it's dirty.")
77
78    (opts, _) = parser.parse_args()
79
80    if opts.commit and opts.diff:
81        sys.exit("Please specify only one option at once.")
82
83    if opts.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", opts.diff):
84        sys.exit("Please specify valid input in the following format: "
85                 "\'commmit1..commit2\'")
86
87    if opts.commit or opts.diff:
88        if not opts.force and tree_is_dirty():
89            sys.exit("The current Git tree is dirty (see 'git status').  "
90                     "Running this script may\ndelete important data since it "
91                     "calls 'git reset --hard' for some performance\nreasons. "
92                     " Please run this script in a clean Git tree or pass "
93                     "'--force' if you\nwant to ignore this warning and "
94                     "continue.")
95
96    if opts.commit:
97        opts.find = False
98
99    if opts.ignore:
100        try:
101            re.match(opts.ignore, "this/is/just/a/test.c")
102        except:
103            sys.exit("Please specify a valid Python regex.")
104
105    return opts
106
107
108def main():
109    """Main function of this module."""
110    opts = parse_options()
111
112    if opts.commit or opts.diff:
113        head = get_head()
114
115        # get commit range
116        commit_a = None
117        commit_b = None
118        if opts.commit:
119            commit_a = opts.commit + "~"
120            commit_b = opts.commit
121        elif opts.diff:
122            split = opts.diff.split("..")
123            commit_a = split[0]
124            commit_b = split[1]
125            undefined_a = {}
126            undefined_b = {}
127
128        # get undefined items before the commit
129        execute("git reset --hard %s" % commit_a)
130        undefined_a = check_symbols(opts.ignore)
131
132        # get undefined items for the commit
133        execute("git reset --hard %s" % commit_b)
134        undefined_b = check_symbols(opts.ignore)
135
136        # report cases that are present for the commit but not before
137        for feature in sorted(undefined_b):
138            # feature has not been undefined before
139            if not feature in undefined_a:
140                files = sorted(undefined_b.get(feature))
141                print "%s\t%s" % (yel(feature), ", ".join(files))
142                if opts.find:
143                    commits = find_commits(feature, opts.diff)
144                    print red(commits)
145            # check if there are new files that reference the undefined feature
146            else:
147                files = sorted(undefined_b.get(feature) -
148                               undefined_a.get(feature))
149                if files:
150                    print "%s\t%s" % (yel(feature), ", ".join(files))
151                    if opts.find:
152                        commits = find_commits(feature, opts.diff)
153                        print red(commits)
154
155        # reset to head
156        execute("git reset --hard %s" % head)
157
158    # default to check the entire tree
159    else:
160        undefined = check_symbols(opts.ignore)
161        for feature in sorted(undefined):
162            files = sorted(undefined.get(feature))
163            print "%s\t%s" % (yel(feature), ", ".join(files))
164
165
166def yel(string):
167    """
168    Color %string yellow.
169    """
170    return "\033[33m%s\033[0m" % string
171
172
173def red(string):
174    """
175    Color %string red.
176    """
177    return "\033[31m%s\033[0m" % string
178
179
180def execute(cmd):
181    """Execute %cmd and return stdout.  Exit in case of error."""
182    pop = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True)
183    (stdout, _) = pop.communicate()  # wait until finished
184    if pop.returncode != 0:
185        sys.exit(stdout)
186    return stdout
187
188
189def find_commits(symbol, diff):
190    """Find commits changing %symbol in the given range of %diff."""
191    commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s"
192                      % (symbol, diff))
193    return commits
194
195
196def tree_is_dirty():
197    """Return true if the current working tree is dirty (i.e., if any file has
198    been added, deleted, modified, renamed or copied but not committed)."""
199    stdout = execute("git status --porcelain")
200    for line in stdout:
201        if re.findall(r"[URMADC]{1}", line[:2]):
202            return True
203    return False
204
205
206def get_head():
207    """Return commit hash of current HEAD."""
208    stdout = execute("git rev-parse HEAD")
209    return stdout.strip('\n')
210
211
212def check_symbols(ignore):
213    """Find undefined Kconfig symbols and return a dict with the symbol as key
214    and a list of referencing files as value.  Files matching %ignore are not
215    checked for undefined symbols."""
216    source_files = []
217    kconfig_files = []
218    defined_features = set()
219    referenced_features = dict()  # {feature: [files]}
220
221    # use 'git ls-files' to get the worklist
222    stdout = execute("git ls-files")
223    if len(stdout) > 0 and stdout[-1] == "\n":
224        stdout = stdout[:-1]
225
226    for gitfile in stdout.rsplit("\n"):
227        if ".git" in gitfile or "ChangeLog" in gitfile or      \
228                ".log" in gitfile or os.path.isdir(gitfile) or \
229                gitfile.startswith("tools/"):
230            continue
231        if REGEX_FILE_KCONFIG.match(gitfile):
232            kconfig_files.append(gitfile)
233        else:
234            # all non-Kconfig files are checked for consistency
235            source_files.append(gitfile)
236
237    for sfile in source_files:
238        if ignore and re.match(ignore, sfile):
239            # do not check files matching %ignore
240            continue
241        parse_source_file(sfile, referenced_features)
242
243    for kfile in kconfig_files:
244        if ignore and re.match(ignore, kfile):
245            # do not collect references for files matching %ignore
246            parse_kconfig_file(kfile, defined_features, dict())
247        else:
248            parse_kconfig_file(kfile, defined_features, referenced_features)
249
250    undefined = {}  # {feature: [files]}
251    for feature in sorted(referenced_features):
252        # filter some false positives
253        if feature == "FOO" or feature == "BAR" or \
254                feature == "FOO_BAR" or feature == "XXX":
255            continue
256        if feature not in defined_features:
257            if feature.endswith("_MODULE"):
258                # avoid false positives for kernel modules
259                if feature[:-len("_MODULE")] in defined_features:
260                    continue
261            undefined[feature] = referenced_features.get(feature)
262    return undefined
263
264
265def parse_source_file(sfile, referenced_features):
266    """Parse @sfile for referenced Kconfig features."""
267    lines = []
268    with open(sfile, "r") as stream:
269        lines = stream.readlines()
270
271    for line in lines:
272        if not "CONFIG_" in line:
273            continue
274        features = REGEX_SOURCE_FEATURE.findall(line)
275        for feature in features:
276            if not REGEX_FILTER_FEATURES.search(feature):
277                continue
278            sfiles = referenced_features.get(feature, set())
279            sfiles.add(sfile)
280            referenced_features[feature] = sfiles
281
282
283def get_features_in_line(line):
284    """Return mentioned Kconfig features in @line."""
285    return REGEX_FEATURE.findall(line)
286
287
288def parse_kconfig_file(kfile, defined_features, referenced_features):
289    """Parse @kfile and update feature definitions and references."""
290    lines = []
291    skip = False
292
293    with open(kfile, "r") as stream:
294        lines = stream.readlines()
295
296    for i in range(len(lines)):
297        line = lines[i]
298        line = line.strip('\n')
299        line = line.split("#")[0]  # ignore comments
300
301        if REGEX_KCONFIG_DEF.match(line):
302            feature_def = REGEX_KCONFIG_DEF.findall(line)
303            defined_features.add(feature_def[0])
304            skip = False
305        elif REGEX_KCONFIG_HELP.match(line):
306            skip = True
307        elif skip:
308            # ignore content of help messages
309            pass
310        elif REGEX_KCONFIG_STMT.match(line):
311            features = get_features_in_line(line)
312            # multi-line statements
313            while line.endswith("\\"):
314                i += 1
315                line = lines[i]
316                line = line.strip('\n')
317                features.extend(get_features_in_line(line))
318            for feature in set(features):
319                if REGEX_NUMERIC.match(feature):
320                    # ignore numeric values
321                    continue
322                paths = referenced_features.get(feature, set())
323                paths.add(kfile)
324                referenced_features[feature] = paths
325
326
327if __name__ == "__main__":
328    main()
329