1#!/usr/bin/env python3
2# SPDX-License-Identifier: GPL-2.0
3#
4# Copyright (C) Google LLC, 2018
5#
6# Author: Tom Roeder <tmroeder@google.com>
7#
8"""A tool for generating compile_commands.json in the Linux kernel."""
9
10import argparse
11import json
12import logging
13import os
14import re
15import subprocess
16import sys
17
18_DEFAULT_OUTPUT = 'compile_commands.json'
19_DEFAULT_LOG_LEVEL = 'WARNING'
20
21_FILENAME_PATTERN = r'^\..*\.cmd$'
22_LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)'
23_VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
24# The tools/ directory adopts a different build system, and produces .cmd
25# files in a different format. Do not support it.
26_EXCLUDE_DIRS = ['.git', 'Documentation', 'include', 'tools']
27
28def parse_arguments():
29    """Sets up and parses command-line arguments.
30
31    Returns:
32        log_level: A logging level to filter log output.
33        directory: The work directory where the objects were built.
34        ar: Command used for parsing .a archives.
35        output: Where to write the compile-commands JSON file.
36        paths: The list of files/directories to handle to find .cmd files.
37    """
38    usage = 'Creates a compile_commands.json database from kernel .cmd files'
39    parser = argparse.ArgumentParser(description=usage)
40
41    directory_help = ('specify the output directory used for the kernel build '
42                      '(defaults to the working directory)')
43    parser.add_argument('-d', '--directory', type=str, default='.',
44                        help=directory_help)
45
46    output_help = ('path to the output command database (defaults to ' +
47                   _DEFAULT_OUTPUT + ')')
48    parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
49                        help=output_help)
50
51    log_level_help = ('the level of log messages to produce (defaults to ' +
52                      _DEFAULT_LOG_LEVEL + ')')
53    parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
54                        default=_DEFAULT_LOG_LEVEL, help=log_level_help)
55
56    ar_help = 'command used for parsing .a archives'
57    parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help)
58
59    paths_help = ('directories to search or files to parse '
60                  '(files should be *.o, *.a, or modules.order). '
61                  'If nothing is specified, the current directory is searched')
62    parser.add_argument('paths', type=str, nargs='*', help=paths_help)
63
64    args = parser.parse_args()
65
66    return (args.log_level,
67            os.path.abspath(args.directory),
68            args.output,
69            args.ar,
70            args.paths if len(args.paths) > 0 else [args.directory])
71
72
73def cmdfiles_in_dir(directory):
74    """Generate the iterator of .cmd files found under the directory.
75
76    Walk under the given directory, and yield every .cmd file found.
77
78    Args:
79        directory: The directory to search for .cmd files.
80
81    Yields:
82        The path to a .cmd file.
83    """
84
85    filename_matcher = re.compile(_FILENAME_PATTERN)
86    exclude_dirs = [ os.path.join(directory, d) for d in _EXCLUDE_DIRS ]
87
88    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
89        # Prune unwanted directories.
90        if dirpath in exclude_dirs:
91            dirnames[:] = []
92            continue
93
94        for filename in filenames:
95            if filename_matcher.match(filename):
96                yield os.path.join(dirpath, filename)
97
98
99def to_cmdfile(path):
100    """Return the path of .cmd file used for the given build artifact
101
102    Args:
103        Path: file path
104
105    Returns:
106        The path to .cmd file
107    """
108    dir, base = os.path.split(path)
109    return os.path.join(dir, '.' + base + '.cmd')
110
111
112def cmdfiles_for_o(obj):
113    """Generate the iterator of .cmd files associated with the object
114
115    Yield the .cmd file used to build the given object
116
117    Args:
118        obj: The object path
119
120    Yields:
121        The path to .cmd file
122    """
123    yield to_cmdfile(obj)
124
125
126def cmdfiles_for_a(archive, ar):
127    """Generate the iterator of .cmd files associated with the archive.
128
129    Parse the given archive, and yield every .cmd file used to build it.
130
131    Args:
132        archive: The archive to parse
133
134    Yields:
135        The path to every .cmd file found
136    """
137    for obj in subprocess.check_output([ar, '-t', archive]).decode().split():
138        yield to_cmdfile(obj)
139
140
141def cmdfiles_for_modorder(modorder):
142    """Generate the iterator of .cmd files associated with the modules.order.
143
144    Parse the given modules.order, and yield every .cmd file used to build the
145    contained modules.
146
147    Args:
148        modorder: The modules.order file to parse
149
150    Yields:
151        The path to every .cmd file found
152    """
153    with open(modorder) as f:
154        for line in f:
155            ko = line.rstrip()
156            base, ext = os.path.splitext(ko)
157            if ext != '.ko':
158                sys.exit('{}: module path must end with .ko'.format(ko))
159            mod = base + '.mod'
160            # Read from *.mod, to get a list of objects that compose the module.
161            with open(mod) as m:
162                for mod_line in m:
163                    yield to_cmdfile(mod_line.rstrip())
164
165
166def process_line(root_directory, command_prefix, file_path):
167    """Extracts information from a .cmd line and creates an entry from it.
168
169    Args:
170        root_directory: The directory that was searched for .cmd files. Usually
171            used directly in the "directory" entry in compile_commands.json.
172        command_prefix: The extracted command line, up to the last element.
173        file_path: The .c file from the end of the extracted command.
174            Usually relative to root_directory, but sometimes absolute.
175
176    Returns:
177        An entry to append to compile_commands.
178
179    Raises:
180        ValueError: Could not find the extracted file based on file_path and
181            root_directory or file_directory.
182    """
183    # The .cmd files are intended to be included directly by Make, so they
184    # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
185    # kernel version). The compile_commands.json file is not interepreted
186    # by Make, so this code replaces the escaped version with '#'.
187    prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
188
189    # Use os.path.abspath() to normalize the path resolving '.' and '..' .
190    abs_path = os.path.abspath(os.path.join(root_directory, file_path))
191    if not os.path.exists(abs_path):
192        raise ValueError('File %s not found' % abs_path)
193    return {
194        'directory': root_directory,
195        'file': abs_path,
196        'command': prefix + file_path,
197    }
198
199
200def main():
201    """Walks through the directory and finds and parses .cmd files."""
202    log_level, directory, output, ar, paths = parse_arguments()
203
204    level = getattr(logging, log_level)
205    logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
206
207    line_matcher = re.compile(_LINE_PATTERN)
208
209    compile_commands = []
210
211    for path in paths:
212        # If 'path' is a directory, handle all .cmd files under it.
213        # Otherwise, handle .cmd files associated with the file.
214        # Most of built-in objects are linked via archives (built-in.a or lib.a)
215        # but some objects are linked to vmlinux directly.
216        # Modules are listed in modules.order.
217        if os.path.isdir(path):
218            cmdfiles = cmdfiles_in_dir(path)
219        elif path.endswith('.o'):
220            cmdfiles = cmdfiles_for_o(path)
221        elif path.endswith('.a'):
222            cmdfiles = cmdfiles_for_a(path, ar)
223        elif path.endswith('modules.order'):
224            cmdfiles = cmdfiles_for_modorder(path)
225        else:
226            sys.exit('{}: unknown file type'.format(path))
227
228        for cmdfile in cmdfiles:
229            with open(cmdfile, 'rt') as f:
230                result = line_matcher.match(f.readline())
231                if result:
232                    try:
233                        entry = process_line(directory, result.group(1),
234                                             result.group(2))
235                        compile_commands.append(entry)
236                    except ValueError as err:
237                        logging.info('Could not add line from %s: %s',
238                                     cmdfile, err)
239
240    with open(output, 'wt') as f:
241        json.dump(compile_commands, f, indent=2, sort_keys=True)
242
243
244if __name__ == '__main__':
245    main()
246