xref: /openbmc/u-boot/tools/genboardscfg.py (revision 203e94f6) 
1#!/usr/bin/env python2
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to a board database.
10
11Run 'tools/genboardscfg.py' to create a board database.
12
13Run 'tools/genboardscfg.py -h' for available options.
14
15Python 2.6 or later, but not Python 3.x is necessary to run this script.
16"""
17
18import errno
19import fnmatch
20import glob
21import multiprocessing
22import optparse
23import os
24import sys
25import tempfile
26import time
27
28sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
29import kconfiglib
30
31### constant variables ###
32OUTPUT_FILE = 'boards.cfg'
33CONFIG_DIR = 'configs'
34SLEEP_TIME = 0.03
35COMMENT_BLOCK = '''#
36# List of boards
37#   Automatically generated by %s: don't edit
38#
39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
40
41''' % __file__
42
43### helper functions ###
44def try_remove(f):
45    """Remove a file ignoring 'No such file or directory' error."""
46    try:
47        os.remove(f)
48    except OSError as exception:
49        # Ignore 'No such file or directory' error
50        if exception.errno != errno.ENOENT:
51            raise
52
53def check_top_directory():
54    """Exit if we are not at the top of source directory."""
55    for f in ('README', 'Licenses'):
56        if not os.path.exists(f):
57            sys.exit('Please run at the top of source directory.')
58
59def output_is_new(output):
60    """Check if the output file is up to date.
61
62    Returns:
63      True if the given output file exists and is newer than any of
64      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
65    """
66    try:
67        ctime = os.path.getctime(output)
68    except OSError as exception:
69        if exception.errno == errno.ENOENT:
70            # return False on 'No such file or directory' error
71            return False
72        else:
73            raise
74
75    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
76        for filename in fnmatch.filter(filenames, '*_defconfig'):
77            if fnmatch.fnmatch(filename, '.*'):
78                continue
79            filepath = os.path.join(dirpath, filename)
80            if ctime < os.path.getctime(filepath):
81                return False
82
83    for (dirpath, dirnames, filenames) in os.walk('.'):
84        for filename in filenames:
85            if (fnmatch.fnmatch(filename, '*~') or
86                not fnmatch.fnmatch(filename, 'Kconfig*') and
87                not filename == 'MAINTAINERS'):
88                continue
89            filepath = os.path.join(dirpath, filename)
90            if ctime < os.path.getctime(filepath):
91                return False
92
93    # Detect a board that has been removed since the current board database
94    # was generated
95    with open(output) as f:
96        for line in f:
97            if line[0] == '#' or line == '\n':
98                continue
99            defconfig = line.split()[6] + '_defconfig'
100            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
101                return False
102
103    return True
104
105### classes ###
106class KconfigScanner:
107
108    """Kconfig scanner."""
109
110    ### constant variable only used in this class ###
111    _SYMBOL_TABLE = {
112        'arch' : 'SYS_ARCH',
113        'cpu' : 'SYS_CPU',
114        'soc' : 'SYS_SOC',
115        'vendor' : 'SYS_VENDOR',
116        'board' : 'SYS_BOARD',
117        'config' : 'SYS_CONFIG_NAME',
118        'options' : 'SYS_EXTRA_OPTIONS'
119    }
120
121    def __init__(self):
122        """Scan all the Kconfig files and create a Config object."""
123        # Define environment variables referenced from Kconfig
124        os.environ['srctree'] = os.getcwd()
125        os.environ['UBOOTVERSION'] = 'dummy'
126        os.environ['KCONFIG_OBJDIR'] = ''
127        self._conf = kconfiglib.Config()
128
129    def __del__(self):
130        """Delete a leftover temporary file before exit.
131
132        The scan() method of this class creates a temporay file and deletes
133        it on success.  If scan() method throws an exception on the way,
134        the temporary file might be left over.  In that case, it should be
135        deleted in this destructor.
136        """
137        if hasattr(self, '_tmpfile') and self._tmpfile:
138            try_remove(self._tmpfile)
139
140    def scan(self, defconfig):
141        """Load a defconfig file to obtain board parameters.
142
143        Arguments:
144          defconfig: path to the defconfig file to be processed
145
146        Returns:
147          A dictionary of board parameters.  It has a form of:
148          {
149              'arch': <arch_name>,
150              'cpu': <cpu_name>,
151              'soc': <soc_name>,
152              'vendor': <vendor_name>,
153              'board': <board_name>,
154              'target': <target_name>,
155              'config': <config_header_name>,
156              'options': <extra_options>
157          }
158        """
159        # strip special prefixes and save it in a temporary file
160        fd, self._tmpfile = tempfile.mkstemp()
161        with os.fdopen(fd, 'w') as f:
162            for line in open(defconfig):
163                colon = line.find(':CONFIG_')
164                if colon == -1:
165                    f.write(line)
166                else:
167                    f.write(line[colon + 1:])
168
169        self._conf.load_config(self._tmpfile)
170
171        try_remove(self._tmpfile)
172        self._tmpfile = None
173
174        params = {}
175
176        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
177        # Set '-' if the value is empty.
178        for key, symbol in self._SYMBOL_TABLE.items():
179            value = self._conf.get_symbol(symbol).get_value()
180            if value:
181                params[key] = value
182            else:
183                params[key] = '-'
184
185        defconfig = os.path.basename(defconfig)
186        params['target'], match, rear = defconfig.partition('_defconfig')
187        assert match and not rear, '%s : invalid defconfig' % defconfig
188
189        # fix-up for aarch64
190        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
191            params['arch'] = 'aarch64'
192
193        # fix-up options field. It should have the form:
194        # <config name>[:comma separated config options]
195        if params['options'] != '-':
196            params['options'] = params['config'] + ':' + \
197                                params['options'].replace(r'\"', '"')
198        elif params['config'] != params['target']:
199            params['options'] = params['config']
200
201        return params
202
203def scan_defconfigs_for_multiprocess(queue, defconfigs):
204    """Scan defconfig files and queue their board parameters
205
206    This function is intended to be passed to
207    multiprocessing.Process() constructor.
208
209    Arguments:
210      queue: An instance of multiprocessing.Queue().
211             The resulting board parameters are written into it.
212      defconfigs: A sequence of defconfig files to be scanned.
213    """
214    kconf_scanner = KconfigScanner()
215    for defconfig in defconfigs:
216        queue.put(kconf_scanner.scan(defconfig))
217
218def read_queues(queues, params_list):
219    """Read the queues and append the data to the paramers list"""
220    for q in queues:
221        while not q.empty():
222            params_list.append(q.get())
223
224def scan_defconfigs(jobs=1):
225    """Collect board parameters for all defconfig files.
226
227    This function invokes multiple processes for faster processing.
228
229    Arguments:
230      jobs: The number of jobs to run simultaneously
231    """
232    all_defconfigs = []
233    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
234        for filename in fnmatch.filter(filenames, '*_defconfig'):
235            if fnmatch.fnmatch(filename, '.*'):
236                continue
237            all_defconfigs.append(os.path.join(dirpath, filename))
238
239    total_boards = len(all_defconfigs)
240    processes = []
241    queues = []
242    for i in range(jobs):
243        defconfigs = all_defconfigs[total_boards * i / jobs :
244                                    total_boards * (i + 1) / jobs]
245        q = multiprocessing.Queue(maxsize=-1)
246        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
247                                    args=(q, defconfigs))
248        p.start()
249        processes.append(p)
250        queues.append(q)
251
252    # The resulting data should be accumulated to this list
253    params_list = []
254
255    # Data in the queues should be retrieved preriodically.
256    # Otherwise, the queues would become full and subprocesses would get stuck.
257    while any([p.is_alive() for p in processes]):
258        read_queues(queues, params_list)
259        # sleep for a while until the queues are filled
260        time.sleep(SLEEP_TIME)
261
262    # Joining subprocesses just in case
263    # (All subprocesses should already have been finished)
264    for p in processes:
265        p.join()
266
267    # retrieve leftover data
268    read_queues(queues, params_list)
269
270    return params_list
271
272class MaintainersDatabase:
273
274    """The database of board status and maintainers."""
275
276    def __init__(self):
277        """Create an empty database."""
278        self.database = {}
279
280    def get_status(self, target):
281        """Return the status of the given board.
282
283        The board status is generally either 'Active' or 'Orphan'.
284        Display a warning message and return '-' if status information
285        is not found.
286
287        Returns:
288          'Active', 'Orphan' or '-'.
289        """
290        if not target in self.database:
291            print >> sys.stderr, "WARNING: no status info for '%s'" % target
292            return '-'
293
294        tmp = self.database[target][0]
295        if tmp.startswith('Maintained'):
296            return 'Active'
297        elif tmp.startswith('Supported'):
298            return 'Active'
299        elif tmp.startswith('Orphan'):
300            return 'Orphan'
301        else:
302            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
303                                  (tmp, target))
304            return '-'
305
306    def get_maintainers(self, target):
307        """Return the maintainers of the given board.
308
309        Returns:
310          Maintainers of the board.  If the board has two or more maintainers,
311          they are separated with colons.
312        """
313        if not target in self.database:
314            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
315            return ''
316
317        return ':'.join(self.database[target][1])
318
319    def parse_file(self, file):
320        """Parse a MAINTAINERS file.
321
322        Parse a MAINTAINERS file and accumulates board status and
323        maintainers information.
324
325        Arguments:
326          file: MAINTAINERS file to be parsed
327        """
328        targets = []
329        maintainers = []
330        status = '-'
331        for line in open(file):
332            # Check also commented maintainers
333            if line[:3] == '#M:':
334                line = line[1:]
335            tag, rest = line[:2], line[2:].strip()
336            if tag == 'M:':
337                maintainers.append(rest)
338            elif tag == 'F:':
339                # expand wildcard and filter by 'configs/*_defconfig'
340                for f in glob.glob(rest):
341                    front, match, rear = f.partition('configs/')
342                    if not front and match:
343                        front, match, rear = rear.rpartition('_defconfig')
344                        if match and not rear:
345                            targets.append(front)
346            elif tag == 'S:':
347                status = rest
348            elif line == '\n':
349                for target in targets:
350                    self.database[target] = (status, maintainers)
351                targets = []
352                maintainers = []
353                status = '-'
354        if targets:
355            for target in targets:
356                self.database[target] = (status, maintainers)
357
358def insert_maintainers_info(params_list):
359    """Add Status and Maintainers information to the board parameters list.
360
361    Arguments:
362      params_list: A list of the board parameters
363    """
364    database = MaintainersDatabase()
365    for (dirpath, dirnames, filenames) in os.walk('.'):
366        if 'MAINTAINERS' in filenames:
367            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
368
369    for i, params in enumerate(params_list):
370        target = params['target']
371        params['status'] = database.get_status(target)
372        params['maintainers'] = database.get_maintainers(target)
373        params_list[i] = params
374
375def format_and_output(params_list, output):
376    """Write board parameters into a file.
377
378    Columnate the board parameters, sort lines alphabetically,
379    and then write them to a file.
380
381    Arguments:
382      params_list: The list of board parameters
383      output: The path to the output file
384    """
385    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
386              'options', 'maintainers')
387
388    # First, decide the width of each column
389    max_length = dict([ (f, 0) for f in FIELDS])
390    for params in params_list:
391        for f in FIELDS:
392            max_length[f] = max(max_length[f], len(params[f]))
393
394    output_lines = []
395    for params in params_list:
396        line = ''
397        for f in FIELDS:
398            # insert two spaces between fields like column -t would
399            line += '  ' + params[f].ljust(max_length[f])
400        output_lines.append(line.strip())
401
402    # ignore case when sorting
403    output_lines.sort(key=str.lower)
404
405    with open(output, 'w') as f:
406        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
407
408def gen_boards_cfg(output, jobs=1, force=False):
409    """Generate a board database file.
410
411    Arguments:
412      output: The name of the output file
413      jobs: The number of jobs to run simultaneously
414      force: Force to generate the output even if it is new
415    """
416    check_top_directory()
417
418    if not force and output_is_new(output):
419        print "%s is up to date. Nothing to do." % output
420        sys.exit(0)
421
422    params_list = scan_defconfigs(jobs)
423    insert_maintainers_info(params_list)
424    format_and_output(params_list, output)
425
426def main():
427    try:
428        cpu_count = multiprocessing.cpu_count()
429    except NotImplementedError:
430        cpu_count = 1
431
432    parser = optparse.OptionParser()
433    # Add options here
434    parser.add_option('-f', '--force', action="store_true", default=False,
435                      help='regenerate the output even if it is new')
436    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
437                      help='the number of jobs to run simultaneously')
438    parser.add_option('-o', '--output', default=OUTPUT_FILE,
439                      help='output file [default=%s]' % OUTPUT_FILE)
440    (options, args) = parser.parse_args()
441
442    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
443
444if __name__ == '__main__':
445    main()
446