xref: /openbmc/u-boot/tools/genboardscfg.py (revision 20c700f8) 
1#!/usr/bin/env python2
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to a board database.
10
11Run 'tools/genboardscfg.py' to create a board database.
12
13Run 'tools/genboardscfg.py -h' for available options.
14
15Python 2.6 or later, but not Python 3.x is necessary to run this script.
16"""
17
18import errno
19import fnmatch
20import glob
21import multiprocessing
22import optparse
23import os
24import sys
25import tempfile
26import time
27
28sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
29import kconfiglib
30
31### constant variables ###
32OUTPUT_FILE = 'boards.cfg'
33CONFIG_DIR = 'configs'
34SLEEP_TIME = 0.03
35COMMENT_BLOCK = '''#
36# List of boards
37#   Automatically generated by %s: don't edit
38#
39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
40
41''' % __file__
42
43### helper functions ###
44def try_remove(f):
45    """Remove a file ignoring 'No such file or directory' error."""
46    try:
47        os.remove(f)
48    except OSError as exception:
49        # Ignore 'No such file or directory' error
50        if exception.errno != errno.ENOENT:
51            raise
52
53def check_top_directory():
54    """Exit if we are not at the top of source directory."""
55    for f in ('README', 'Licenses'):
56        if not os.path.exists(f):
57            sys.exit('Please run at the top of source directory.')
58
59def output_is_new(output):
60    """Check if the output file is up to date.
61
62    Returns:
63      True if the given output file exists and is newer than any of
64      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
65    """
66    try:
67        ctime = os.path.getctime(output)
68    except OSError as exception:
69        if exception.errno == errno.ENOENT:
70            # return False on 'No such file or directory' error
71            return False
72        else:
73            raise
74
75    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
76        for filename in fnmatch.filter(filenames, '*_defconfig'):
77            if fnmatch.fnmatch(filename, '.*'):
78                continue
79            filepath = os.path.join(dirpath, filename)
80            if ctime < os.path.getctime(filepath):
81                return False
82
83    for (dirpath, dirnames, filenames) in os.walk('.'):
84        for filename in filenames:
85            if (fnmatch.fnmatch(filename, '*~') or
86                not fnmatch.fnmatch(filename, 'Kconfig*') and
87                not filename == 'MAINTAINERS'):
88                continue
89            filepath = os.path.join(dirpath, filename)
90            if ctime < os.path.getctime(filepath):
91                return False
92
93    # Detect a board that has been removed since the current board database
94    # was generated
95    with open(output) as f:
96        for line in f:
97            if line[0] == '#' or line == '\n':
98                continue
99            defconfig = line.split()[6] + '_defconfig'
100            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
101                return False
102
103    return True
104
105### classes ###
106class KconfigScanner:
107
108    """Kconfig scanner."""
109
110    ### constant variable only used in this class ###
111    _SYMBOL_TABLE = {
112        'arch' : 'SYS_ARCH',
113        'cpu' : 'SYS_CPU',
114        'soc' : 'SYS_SOC',
115        'vendor' : 'SYS_VENDOR',
116        'board' : 'SYS_BOARD',
117        'config' : 'SYS_CONFIG_NAME',
118        'options' : 'SYS_EXTRA_OPTIONS'
119    }
120
121    def __init__(self):
122        """Scan all the Kconfig files and create a Config object."""
123        # Define environment variables referenced from Kconfig
124        os.environ['srctree'] = os.getcwd()
125        os.environ['UBOOTVERSION'] = 'dummy'
126        os.environ['KCONFIG_OBJDIR'] = ''
127        self._conf = kconfiglib.Config()
128
129    def __del__(self):
130        """Delete a leftover temporary file before exit.
131
132        The scan() method of this class creates a temporay file and deletes
133        it on success.  If scan() method throws an exception on the way,
134        the temporary file might be left over.  In that case, it should be
135        deleted in this destructor.
136        """
137        if hasattr(self, '_tmpfile') and self._tmpfile:
138            try_remove(self._tmpfile)
139
140    def scan(self, defconfig):
141        """Load a defconfig file to obtain board parameters.
142
143        Arguments:
144          defconfig: path to the defconfig file to be processed
145
146        Returns:
147          A dictionary of board parameters.  It has a form of:
148          {
149              'arch': <arch_name>,
150              'cpu': <cpu_name>,
151              'soc': <soc_name>,
152              'vendor': <vendor_name>,
153              'board': <board_name>,
154              'target': <target_name>,
155              'config': <config_header_name>,
156              'options': <extra_options>
157          }
158        """
159        # strip special prefixes and save it in a temporary file
160        fd, self._tmpfile = tempfile.mkstemp()
161        with os.fdopen(fd, 'w') as f:
162            for line in open(defconfig):
163                colon = line.find(':CONFIG_')
164                if colon == -1:
165                    f.write(line)
166                else:
167                    f.write(line[colon + 1:])
168
169        self._conf.load_config(self._tmpfile)
170
171        try_remove(self._tmpfile)
172        self._tmpfile = None
173
174        params = {}
175
176        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
177        # Set '-' if the value is empty.
178        for key, symbol in self._SYMBOL_TABLE.items():
179            value = self._conf.get_symbol(symbol).get_value()
180            if value:
181                params[key] = value
182            else:
183                params[key] = '-'
184
185        defconfig = os.path.basename(defconfig)
186        params['target'], match, rear = defconfig.partition('_defconfig')
187        assert match and not rear, '%s : invalid defconfig' % defconfig
188
189        # fix-up for aarch64
190        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
191            params['arch'] = 'aarch64'
192
193        # fix-up options field. It should have the form:
194        # <config name>[:comma separated config options]
195        if params['options'] != '-':
196            params['options'] = params['config'] + ':' + \
197                                params['options'].replace(r'\"', '"')
198        elif params['config'] != params['target']:
199            params['options'] = params['config']
200
201        return params
202
203def scan_defconfigs_for_multiprocess(queue, defconfigs):
204    """Scan defconfig files and queue their board parameters
205
206    This function is intended to be passed to
207    multiprocessing.Process() constructor.
208
209    Arguments:
210      queue: An instance of multiprocessing.Queue().
211             The resulting board parameters are written into it.
212      defconfigs: A sequence of defconfig files to be scanned.
213    """
214    kconf_scanner = KconfigScanner()
215    for defconfig in defconfigs:
216        queue.put(kconf_scanner.scan(defconfig))
217
218def read_queues(queues, params_list):
219    """Read the queues and append the data to the paramers list"""
220    for q in queues:
221        while not q.empty():
222            params_list.append(q.get())
223
224def scan_defconfigs(jobs=1):
225    """Collect board parameters for all defconfig files.
226
227    This function invokes multiple processes for faster processing.
228
229    Arguments:
230      jobs: The number of jobs to run simultaneously
231    """
232    all_defconfigs = []
233    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
234        for filename in fnmatch.filter(filenames, '*_defconfig'):
235            if fnmatch.fnmatch(filename, '.*'):
236                continue
237            all_defconfigs.append(os.path.join(dirpath, filename))
238
239    total_boards = len(all_defconfigs)
240    processes = []
241    queues = []
242    for i in range(jobs):
243        defconfigs = all_defconfigs[total_boards * i / jobs :
244                                    total_boards * (i + 1) / jobs]
245        q = multiprocessing.Queue(maxsize=-1)
246        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
247                                    args=(q, defconfigs))
248        p.start()
249        processes.append(p)
250        queues.append(q)
251
252    # The resulting data should be accumulated to this list
253    params_list = []
254
255    # Data in the queues should be retrieved preriodically.
256    # Otherwise, the queues would become full and subprocesses would get stuck.
257    while any([p.is_alive() for p in processes]):
258        read_queues(queues, params_list)
259        # sleep for a while until the queues are filled
260        time.sleep(SLEEP_TIME)
261
262    # Joining subprocesses just in case
263    # (All subprocesses should already have been finished)
264    for p in processes:
265        p.join()
266
267    # retrieve leftover data
268    read_queues(queues, params_list)
269
270    return params_list
271
272class MaintainersDatabase:
273
274    """The database of board status and maintainers."""
275
276    def __init__(self):
277        """Create an empty database."""
278        self.database = {}
279
280    def get_status(self, target):
281        """Return the status of the given board.
282
283        The board status is generally either 'Active' or 'Orphan'.
284        Display a warning message and return '-' if status information
285        is not found.
286
287        Returns:
288          'Active', 'Orphan' or '-'.
289        """
290        if not target in self.database:
291            print >> sys.stderr, "WARNING: no status info for '%s'" % target
292            return '-'
293
294        tmp = self.database[target][0]
295        if tmp.startswith('Maintained'):
296            return 'Active'
297        elif tmp.startswith('Orphan'):
298            return 'Orphan'
299        else:
300            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
301                                  (tmp, target))
302            return '-'
303
304    def get_maintainers(self, target):
305        """Return the maintainers of the given board.
306
307        Returns:
308          Maintainers of the board.  If the board has two or more maintainers,
309          they are separated with colons.
310        """
311        if not target in self.database:
312            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
313            return ''
314
315        return ':'.join(self.database[target][1])
316
317    def parse_file(self, file):
318        """Parse a MAINTAINERS file.
319
320        Parse a MAINTAINERS file and accumulates board status and
321        maintainers information.
322
323        Arguments:
324          file: MAINTAINERS file to be parsed
325        """
326        targets = []
327        maintainers = []
328        status = '-'
329        for line in open(file):
330            # Check also commented maintainers
331            if line[:3] == '#M:':
332                line = line[1:]
333            tag, rest = line[:2], line[2:].strip()
334            if tag == 'M:':
335                maintainers.append(rest)
336            elif tag == 'F:':
337                # expand wildcard and filter by 'configs/*_defconfig'
338                for f in glob.glob(rest):
339                    front, match, rear = f.partition('configs/')
340                    if not front and match:
341                        front, match, rear = rear.rpartition('_defconfig')
342                        if match and not rear:
343                            targets.append(front)
344            elif tag == 'S:':
345                status = rest
346            elif line == '\n':
347                for target in targets:
348                    self.database[target] = (status, maintainers)
349                targets = []
350                maintainers = []
351                status = '-'
352        if targets:
353            for target in targets:
354                self.database[target] = (status, maintainers)
355
356def insert_maintainers_info(params_list):
357    """Add Status and Maintainers information to the board parameters list.
358
359    Arguments:
360      params_list: A list of the board parameters
361    """
362    database = MaintainersDatabase()
363    for (dirpath, dirnames, filenames) in os.walk('.'):
364        if 'MAINTAINERS' in filenames:
365            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
366
367    for i, params in enumerate(params_list):
368        target = params['target']
369        params['status'] = database.get_status(target)
370        params['maintainers'] = database.get_maintainers(target)
371        params_list[i] = params
372
373def format_and_output(params_list, output):
374    """Write board parameters into a file.
375
376    Columnate the board parameters, sort lines alphabetically,
377    and then write them to a file.
378
379    Arguments:
380      params_list: The list of board parameters
381      output: The path to the output file
382    """
383    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
384              'options', 'maintainers')
385
386    # First, decide the width of each column
387    max_length = dict([ (f, 0) for f in FIELDS])
388    for params in params_list:
389        for f in FIELDS:
390            max_length[f] = max(max_length[f], len(params[f]))
391
392    output_lines = []
393    for params in params_list:
394        line = ''
395        for f in FIELDS:
396            # insert two spaces between fields like column -t would
397            line += '  ' + params[f].ljust(max_length[f])
398        output_lines.append(line.strip())
399
400    # ignore case when sorting
401    output_lines.sort(key=str.lower)
402
403    with open(output, 'w') as f:
404        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
405
406def gen_boards_cfg(output, jobs=1, force=False):
407    """Generate a board database file.
408
409    Arguments:
410      output: The name of the output file
411      jobs: The number of jobs to run simultaneously
412      force: Force to generate the output even if it is new
413    """
414    check_top_directory()
415
416    if not force and output_is_new(output):
417        print "%s is up to date. Nothing to do." % output
418        sys.exit(0)
419
420    params_list = scan_defconfigs(jobs)
421    insert_maintainers_info(params_list)
422    format_and_output(params_list, output)
423
424def main():
425    try:
426        cpu_count = multiprocessing.cpu_count()
427    except NotImplementedError:
428        cpu_count = 1
429
430    parser = optparse.OptionParser()
431    # Add options here
432    parser.add_option('-f', '--force', action="store_true", default=False,
433                      help='regenerate the output even if it is new')
434    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
435                      help='the number of jobs to run simultaneously')
436    parser.add_option('-o', '--output', default=OUTPUT_FILE,
437                      help='output file [default=%s]' % OUTPUT_FILE)
438    (options, args) = parser.parse_args()
439
440    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
441
442if __name__ == '__main__':
443    main()
444