xref: /openbmc/u-boot/tools/genboardscfg.py (revision 5c8fd32b) 
1#!/usr/bin/env python2
2# SPDX-License-Identifier: GPL-2.0+
3#
4# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5#
6
7"""
8Converter from Kconfig and MAINTAINERS to a board database.
9
10Run 'tools/genboardscfg.py' to create a board database.
11
12Run 'tools/genboardscfg.py -h' for available options.
13
14Python 2.6 or later, but not Python 3.x is necessary to run this script.
15"""
16
17import errno
18import fnmatch
19import glob
20import multiprocessing
21import optparse
22import os
23import sys
24import tempfile
25import time
26
27sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'buildman'))
28import kconfiglib
29
30### constant variables ###
31OUTPUT_FILE = 'boards.cfg'
32CONFIG_DIR = 'configs'
33SLEEP_TIME = 0.03
34COMMENT_BLOCK = '''#
35# List of boards
36#   Automatically generated by %s: don't edit
37#
38# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
39
40''' % __file__
41
42### helper functions ###
43def try_remove(f):
44    """Remove a file ignoring 'No such file or directory' error."""
45    try:
46        os.remove(f)
47    except OSError as exception:
48        # Ignore 'No such file or directory' error
49        if exception.errno != errno.ENOENT:
50            raise
51
52def check_top_directory():
53    """Exit if we are not at the top of source directory."""
54    for f in ('README', 'Licenses'):
55        if not os.path.exists(f):
56            sys.exit('Please run at the top of source directory.')
57
58def output_is_new(output):
59    """Check if the output file is up to date.
60
61    Returns:
62      True if the given output file exists and is newer than any of
63      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
64    """
65    try:
66        ctime = os.path.getctime(output)
67    except OSError as exception:
68        if exception.errno == errno.ENOENT:
69            # return False on 'No such file or directory' error
70            return False
71        else:
72            raise
73
74    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
75        for filename in fnmatch.filter(filenames, '*_defconfig'):
76            if fnmatch.fnmatch(filename, '.*'):
77                continue
78            filepath = os.path.join(dirpath, filename)
79            if ctime < os.path.getctime(filepath):
80                return False
81
82    for (dirpath, dirnames, filenames) in os.walk('.'):
83        for filename in filenames:
84            if (fnmatch.fnmatch(filename, '*~') or
85                not fnmatch.fnmatch(filename, 'Kconfig*') and
86                not filename == 'MAINTAINERS'):
87                continue
88            filepath = os.path.join(dirpath, filename)
89            if ctime < os.path.getctime(filepath):
90                return False
91
92    # Detect a board that has been removed since the current board database
93    # was generated
94    with open(output) as f:
95        for line in f:
96            if line[0] == '#' or line == '\n':
97                continue
98            defconfig = line.split()[6] + '_defconfig'
99            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
100                return False
101
102    return True
103
104### classes ###
105class KconfigScanner:
106
107    """Kconfig scanner."""
108
109    ### constant variable only used in this class ###
110    _SYMBOL_TABLE = {
111        'arch' : 'SYS_ARCH',
112        'cpu' : 'SYS_CPU',
113        'soc' : 'SYS_SOC',
114        'vendor' : 'SYS_VENDOR',
115        'board' : 'SYS_BOARD',
116        'config' : 'SYS_CONFIG_NAME',
117        'options' : 'SYS_EXTRA_OPTIONS'
118    }
119
120    def __init__(self):
121        """Scan all the Kconfig files and create a Config object."""
122        # Define environment variables referenced from Kconfig
123        os.environ['srctree'] = os.getcwd()
124        os.environ['UBOOTVERSION'] = 'dummy'
125        os.environ['KCONFIG_OBJDIR'] = ''
126        self._conf = kconfiglib.Config(print_warnings=False)
127
128    def __del__(self):
129        """Delete a leftover temporary file before exit.
130
131        The scan() method of this class creates a temporay file and deletes
132        it on success.  If scan() method throws an exception on the way,
133        the temporary file might be left over.  In that case, it should be
134        deleted in this destructor.
135        """
136        if hasattr(self, '_tmpfile') and self._tmpfile:
137            try_remove(self._tmpfile)
138
139    def scan(self, defconfig):
140        """Load a defconfig file to obtain board parameters.
141
142        Arguments:
143          defconfig: path to the defconfig file to be processed
144
145        Returns:
146          A dictionary of board parameters.  It has a form of:
147          {
148              'arch': <arch_name>,
149              'cpu': <cpu_name>,
150              'soc': <soc_name>,
151              'vendor': <vendor_name>,
152              'board': <board_name>,
153              'target': <target_name>,
154              'config': <config_header_name>,
155              'options': <extra_options>
156          }
157        """
158        # strip special prefixes and save it in a temporary file
159        fd, self._tmpfile = tempfile.mkstemp()
160        with os.fdopen(fd, 'w') as f:
161            for line in open(defconfig):
162                colon = line.find(':CONFIG_')
163                if colon == -1:
164                    f.write(line)
165                else:
166                    f.write(line[colon + 1:])
167
168        warnings = self._conf.load_config(self._tmpfile)
169        if warnings:
170            for warning in warnings:
171                print '%s: %s' % (defconfig, warning)
172
173        try_remove(self._tmpfile)
174        self._tmpfile = None
175
176        params = {}
177
178        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
179        # Set '-' if the value is empty.
180        for key, symbol in self._SYMBOL_TABLE.items():
181            value = self._conf.get_symbol(symbol).get_value()
182            if value:
183                params[key] = value
184            else:
185                params[key] = '-'
186
187        defconfig = os.path.basename(defconfig)
188        params['target'], match, rear = defconfig.partition('_defconfig')
189        assert match and not rear, '%s : invalid defconfig' % defconfig
190
191        # fix-up for aarch64
192        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
193            params['arch'] = 'aarch64'
194
195        # fix-up options field. It should have the form:
196        # <config name>[:comma separated config options]
197        if params['options'] != '-':
198            params['options'] = params['config'] + ':' + \
199                                params['options'].replace(r'\"', '"')
200        elif params['config'] != params['target']:
201            params['options'] = params['config']
202
203        return params
204
205def scan_defconfigs_for_multiprocess(queue, defconfigs):
206    """Scan defconfig files and queue their board parameters
207
208    This function is intended to be passed to
209    multiprocessing.Process() constructor.
210
211    Arguments:
212      queue: An instance of multiprocessing.Queue().
213             The resulting board parameters are written into it.
214      defconfigs: A sequence of defconfig files to be scanned.
215    """
216    kconf_scanner = KconfigScanner()
217    for defconfig in defconfigs:
218        queue.put(kconf_scanner.scan(defconfig))
219
220def read_queues(queues, params_list):
221    """Read the queues and append the data to the paramers list"""
222    for q in queues:
223        while not q.empty():
224            params_list.append(q.get())
225
226def scan_defconfigs(jobs=1):
227    """Collect board parameters for all defconfig files.
228
229    This function invokes multiple processes for faster processing.
230
231    Arguments:
232      jobs: The number of jobs to run simultaneously
233    """
234    all_defconfigs = []
235    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
236        for filename in fnmatch.filter(filenames, '*_defconfig'):
237            if fnmatch.fnmatch(filename, '.*'):
238                continue
239            all_defconfigs.append(os.path.join(dirpath, filename))
240
241    total_boards = len(all_defconfigs)
242    processes = []
243    queues = []
244    for i in range(jobs):
245        defconfigs = all_defconfigs[total_boards * i / jobs :
246                                    total_boards * (i + 1) / jobs]
247        q = multiprocessing.Queue(maxsize=-1)
248        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
249                                    args=(q, defconfigs))
250        p.start()
251        processes.append(p)
252        queues.append(q)
253
254    # The resulting data should be accumulated to this list
255    params_list = []
256
257    # Data in the queues should be retrieved preriodically.
258    # Otherwise, the queues would become full and subprocesses would get stuck.
259    while any([p.is_alive() for p in processes]):
260        read_queues(queues, params_list)
261        # sleep for a while until the queues are filled
262        time.sleep(SLEEP_TIME)
263
264    # Joining subprocesses just in case
265    # (All subprocesses should already have been finished)
266    for p in processes:
267        p.join()
268
269    # retrieve leftover data
270    read_queues(queues, params_list)
271
272    return params_list
273
274class MaintainersDatabase:
275
276    """The database of board status and maintainers."""
277
278    def __init__(self):
279        """Create an empty database."""
280        self.database = {}
281
282    def get_status(self, target):
283        """Return the status of the given board.
284
285        The board status is generally either 'Active' or 'Orphan'.
286        Display a warning message and return '-' if status information
287        is not found.
288
289        Returns:
290          'Active', 'Orphan' or '-'.
291        """
292        if not target in self.database:
293            print >> sys.stderr, "WARNING: no status info for '%s'" % target
294            return '-'
295
296        tmp = self.database[target][0]
297        if tmp.startswith('Maintained'):
298            return 'Active'
299        elif tmp.startswith('Supported'):
300            return 'Active'
301        elif tmp.startswith('Orphan'):
302            return 'Orphan'
303        else:
304            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
305                                  (tmp, target))
306            return '-'
307
308    def get_maintainers(self, target):
309        """Return the maintainers of the given board.
310
311        Returns:
312          Maintainers of the board.  If the board has two or more maintainers,
313          they are separated with colons.
314        """
315        if not target in self.database:
316            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
317            return ''
318
319        return ':'.join(self.database[target][1])
320
321    def parse_file(self, file):
322        """Parse a MAINTAINERS file.
323
324        Parse a MAINTAINERS file and accumulates board status and
325        maintainers information.
326
327        Arguments:
328          file: MAINTAINERS file to be parsed
329        """
330        targets = []
331        maintainers = []
332        status = '-'
333        for line in open(file):
334            # Check also commented maintainers
335            if line[:3] == '#M:':
336                line = line[1:]
337            tag, rest = line[:2], line[2:].strip()
338            if tag == 'M:':
339                maintainers.append(rest)
340            elif tag == 'F:':
341                # expand wildcard and filter by 'configs/*_defconfig'
342                for f in glob.glob(rest):
343                    front, match, rear = f.partition('configs/')
344                    if not front and match:
345                        front, match, rear = rear.rpartition('_defconfig')
346                        if match and not rear:
347                            targets.append(front)
348            elif tag == 'S:':
349                status = rest
350            elif line == '\n':
351                for target in targets:
352                    self.database[target] = (status, maintainers)
353                targets = []
354                maintainers = []
355                status = '-'
356        if targets:
357            for target in targets:
358                self.database[target] = (status, maintainers)
359
360def insert_maintainers_info(params_list):
361    """Add Status and Maintainers information to the board parameters list.
362
363    Arguments:
364      params_list: A list of the board parameters
365    """
366    database = MaintainersDatabase()
367    for (dirpath, dirnames, filenames) in os.walk('.'):
368        if 'MAINTAINERS' in filenames:
369            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
370
371    for i, params in enumerate(params_list):
372        target = params['target']
373        params['status'] = database.get_status(target)
374        params['maintainers'] = database.get_maintainers(target)
375        params_list[i] = params
376
377def format_and_output(params_list, output):
378    """Write board parameters into a file.
379
380    Columnate the board parameters, sort lines alphabetically,
381    and then write them to a file.
382
383    Arguments:
384      params_list: The list of board parameters
385      output: The path to the output file
386    """
387    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
388              'options', 'maintainers')
389
390    # First, decide the width of each column
391    max_length = dict([ (f, 0) for f in FIELDS])
392    for params in params_list:
393        for f in FIELDS:
394            max_length[f] = max(max_length[f], len(params[f]))
395
396    output_lines = []
397    for params in params_list:
398        line = ''
399        for f in FIELDS:
400            # insert two spaces between fields like column -t would
401            line += '  ' + params[f].ljust(max_length[f])
402        output_lines.append(line.strip())
403
404    # ignore case when sorting
405    output_lines.sort(key=str.lower)
406
407    with open(output, 'w') as f:
408        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
409
410def gen_boards_cfg(output, jobs=1, force=False):
411    """Generate a board database file.
412
413    Arguments:
414      output: The name of the output file
415      jobs: The number of jobs to run simultaneously
416      force: Force to generate the output even if it is new
417    """
418    check_top_directory()
419
420    if not force and output_is_new(output):
421        print "%s is up to date. Nothing to do." % output
422        sys.exit(0)
423
424    params_list = scan_defconfigs(jobs)
425    insert_maintainers_info(params_list)
426    format_and_output(params_list, output)
427
428def main():
429    try:
430        cpu_count = multiprocessing.cpu_count()
431    except NotImplementedError:
432        cpu_count = 1
433
434    parser = optparse.OptionParser()
435    # Add options here
436    parser.add_option('-f', '--force', action="store_true", default=False,
437                      help='regenerate the output even if it is new')
438    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
439                      help='the number of jobs to run simultaneously')
440    parser.add_option('-o', '--output', default=OUTPUT_FILE,
441                      help='output file [default=%s]' % OUTPUT_FILE)
442    (options, args) = parser.parse_args()
443
444    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
445
446if __name__ == '__main__':
447    main()
448