xref: /openbmc/u-boot/tools/genboardscfg.py (revision ca11db26) 
1#!/usr/bin/env python2
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to a board database.
10
11Run 'tools/genboardscfg.py' to create a board database.
12
13Run 'tools/genboardscfg.py -h' for available options.
14
15Python 2.6 or later, but not Python 3.x is necessary to run this script.
16"""
17
18import errno
19import fnmatch
20import glob
21import multiprocessing
22import optparse
23import os
24import subprocess
25import sys
26import tempfile
27import time
28
29sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
30import kconfiglib
31
32### constant variables ###
33OUTPUT_FILE = 'boards.cfg'
34CONFIG_DIR = 'configs'
35SLEEP_TIME = 0.03
36COMMENT_BLOCK = '''#
37# List of boards
38#   Automatically generated by %s: don't edit
39#
40# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
41
42''' % __file__
43
44### helper functions ###
45def try_remove(f):
46    """Remove a file ignoring 'No such file or directory' error."""
47    try:
48        os.remove(f)
49    except OSError as exception:
50        # Ignore 'No such file or directory' error
51        if exception.errno != errno.ENOENT:
52            raise
53
54def check_top_directory():
55    """Exit if we are not at the top of source directory."""
56    for f in ('README', 'Licenses'):
57        if not os.path.exists(f):
58            sys.exit('Please run at the top of source directory.')
59
60def output_is_new(output):
61    """Check if the output file is up to date.
62
63    Returns:
64      True if the given output file exists and is newer than any of
65      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
66    """
67    try:
68        ctime = os.path.getctime(output)
69    except OSError as exception:
70        if exception.errno == errno.ENOENT:
71            # return False on 'No such file or directory' error
72            return False
73        else:
74            raise
75
76    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
77        for filename in fnmatch.filter(filenames, '*_defconfig'):
78            if fnmatch.fnmatch(filename, '.*'):
79                continue
80            filepath = os.path.join(dirpath, filename)
81            if ctime < os.path.getctime(filepath):
82                return False
83
84    for (dirpath, dirnames, filenames) in os.walk('.'):
85        for filename in filenames:
86            if (fnmatch.fnmatch(filename, '*~') or
87                not fnmatch.fnmatch(filename, 'Kconfig*') and
88                not filename == 'MAINTAINERS'):
89                continue
90            filepath = os.path.join(dirpath, filename)
91            if ctime < os.path.getctime(filepath):
92                return False
93
94    # Detect a board that has been removed since the current board database
95    # was generated
96    with open(output) as f:
97        for line in f:
98            if line[0] == '#' or line == '\n':
99                continue
100            defconfig = line.split()[6] + '_defconfig'
101            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
102                return False
103
104    return True
105
106### classes ###
107class KconfigScanner:
108
109    """Kconfig scanner."""
110
111    ### constant variable only used in this class ###
112    _SYMBOL_TABLE = {
113        'arch' : 'SYS_ARCH',
114        'cpu' : 'SYS_CPU',
115        'soc' : 'SYS_SOC',
116        'vendor' : 'SYS_VENDOR',
117        'board' : 'SYS_BOARD',
118        'config' : 'SYS_CONFIG_NAME',
119        'options' : 'SYS_EXTRA_OPTIONS'
120    }
121
122    def __init__(self):
123        """Scan all the Kconfig files and create a Config object."""
124        # Define environment variables referenced from Kconfig
125        os.environ['srctree'] = os.getcwd()
126        os.environ['UBOOTVERSION'] = 'dummy'
127        os.environ['KCONFIG_OBJDIR'] = ''
128        self._conf = kconfiglib.Config()
129
130    def __del__(self):
131        """Delete a leftover temporary file before exit.
132
133        The scan() method of this class creates a temporay file and deletes
134        it on success.  If scan() method throws an exception on the way,
135        the temporary file might be left over.  In that case, it should be
136        deleted in this destructor.
137        """
138        if hasattr(self, '_tmpfile') and self._tmpfile:
139            try_remove(self._tmpfile)
140
141    def scan(self, defconfig):
142        """Load a defconfig file to obtain board parameters.
143
144        Arguments:
145          defconfig: path to the defconfig file to be processed
146
147        Returns:
148          A dictionary of board parameters.  It has a form of:
149          {
150              'arch': <arch_name>,
151              'cpu': <cpu_name>,
152              'soc': <soc_name>,
153              'vendor': <vendor_name>,
154              'board': <board_name>,
155              'target': <target_name>,
156              'config': <config_header_name>,
157              'options': <extra_options>
158          }
159        """
160        # strip special prefixes and save it in a temporary file
161        fd, self._tmpfile = tempfile.mkstemp()
162        with os.fdopen(fd, 'w') as f:
163            for line in open(defconfig):
164                colon = line.find(':CONFIG_')
165                if colon == -1:
166                    f.write(line)
167                else:
168                    f.write(line[colon + 1:])
169
170        self._conf.load_config(self._tmpfile)
171
172        try_remove(self._tmpfile)
173        self._tmpfile = None
174
175        params = {}
176
177        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
178        # Set '-' if the value is empty.
179        for key, symbol in self._SYMBOL_TABLE.items():
180            value = self._conf.get_symbol(symbol).get_value()
181            if value:
182                params[key] = value
183            else:
184                params[key] = '-'
185
186        defconfig = os.path.basename(defconfig)
187        params['target'], match, rear = defconfig.partition('_defconfig')
188        assert match and not rear, '%s : invalid defconfig' % defconfig
189
190        # fix-up for aarch64
191        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
192            params['arch'] = 'aarch64'
193
194        # fix-up options field. It should have the form:
195        # <config name>[:comma separated config options]
196        if params['options'] != '-':
197            params['options'] = params['config'] + ':' + \
198                                params['options'].replace(r'\"', '"')
199        elif params['config'] != params['target']:
200            params['options'] = params['config']
201
202        return params
203
204def scan_defconfigs_for_multiprocess(queue, defconfigs):
205    """Scan defconfig files and queue their board parameters
206
207    This function is intended to be passed to
208    multiprocessing.Process() constructor.
209
210    Arguments:
211      queue: An instance of multiprocessing.Queue().
212             The resulting board parameters are written into it.
213      defconfigs: A sequence of defconfig files to be scanned.
214    """
215    kconf_scanner = KconfigScanner()
216    for defconfig in defconfigs:
217        queue.put(kconf_scanner.scan(defconfig))
218
219def read_queues(queues, params_list):
220    """Read the queues and append the data to the paramers list"""
221    for q in queues:
222        while not q.empty():
223            params_list.append(q.get())
224
225def scan_defconfigs(jobs=1):
226    """Collect board parameters for all defconfig files.
227
228    This function invokes multiple processes for faster processing.
229
230    Arguments:
231      jobs: The number of jobs to run simultaneously
232    """
233    all_defconfigs = []
234    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
235        for filename in fnmatch.filter(filenames, '*_defconfig'):
236            if fnmatch.fnmatch(filename, '.*'):
237                continue
238            all_defconfigs.append(os.path.join(dirpath, filename))
239
240    total_boards = len(all_defconfigs)
241    processes = []
242    queues = []
243    for i in range(jobs):
244        defconfigs = all_defconfigs[total_boards * i / jobs :
245                                    total_boards * (i + 1) / jobs]
246        q = multiprocessing.Queue(maxsize=-1)
247        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
248                                    args=(q, defconfigs))
249        p.start()
250        processes.append(p)
251        queues.append(q)
252
253    # The resulting data should be accumulated to this list
254    params_list = []
255
256    # Data in the queues should be retrieved preriodically.
257    # Otherwise, the queues would become full and subprocesses would get stuck.
258    while any([p.is_alive() for p in processes]):
259        read_queues(queues, params_list)
260        # sleep for a while until the queues are filled
261        time.sleep(SLEEP_TIME)
262
263    # Joining subprocesses just in case
264    # (All subprocesses should already have been finished)
265    for p in processes:
266        p.join()
267
268    # retrieve leftover data
269    read_queues(queues, params_list)
270
271    return params_list
272
273class MaintainersDatabase:
274
275    """The database of board status and maintainers."""
276
277    def __init__(self):
278        """Create an empty database."""
279        self.database = {}
280
281    def get_status(self, target):
282        """Return the status of the given board.
283
284        The board status is generally either 'Active' or 'Orphan'.
285        Display a warning message and return '-' if status information
286        is not found.
287
288        Returns:
289          'Active', 'Orphan' or '-'.
290        """
291        if not target in self.database:
292            print >> sys.stderr, "WARNING: no status info for '%s'" % target
293            return '-'
294
295        tmp = self.database[target][0]
296        if tmp.startswith('Maintained'):
297            return 'Active'
298        elif tmp.startswith('Orphan'):
299            return 'Orphan'
300        else:
301            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
302                                  (tmp, target))
303            return '-'
304
305    def get_maintainers(self, target):
306        """Return the maintainers of the given board.
307
308        Returns:
309          Maintainers of the board.  If the board has two or more maintainers,
310          they are separated with colons.
311        """
312        if not target in self.database:
313            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
314            return ''
315
316        return ':'.join(self.database[target][1])
317
318    def parse_file(self, file):
319        """Parse a MAINTAINERS file.
320
321        Parse a MAINTAINERS file and accumulates board status and
322        maintainers information.
323
324        Arguments:
325          file: MAINTAINERS file to be parsed
326        """
327        targets = []
328        maintainers = []
329        status = '-'
330        for line in open(file):
331            tag, rest = line[:2], line[2:].strip()
332            if tag == 'M:':
333                maintainers.append(rest)
334            elif tag == 'F:':
335                # expand wildcard and filter by 'configs/*_defconfig'
336                for f in glob.glob(rest):
337                    front, match, rear = f.partition('configs/')
338                    if not front and match:
339                        front, match, rear = rear.rpartition('_defconfig')
340                        if match and not rear:
341                            targets.append(front)
342            elif tag == 'S:':
343                status = rest
344            elif line == '\n':
345                for target in targets:
346                    self.database[target] = (status, maintainers)
347                targets = []
348                maintainers = []
349                status = '-'
350        if targets:
351            for target in targets:
352                self.database[target] = (status, maintainers)
353
354def insert_maintainers_info(params_list):
355    """Add Status and Maintainers information to the board parameters list.
356
357    Arguments:
358      params_list: A list of the board parameters
359    """
360    database = MaintainersDatabase()
361    for (dirpath, dirnames, filenames) in os.walk('.'):
362        if 'MAINTAINERS' in filenames:
363            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
364
365    for i, params in enumerate(params_list):
366        target = params['target']
367        params['status'] = database.get_status(target)
368        params['maintainers'] = database.get_maintainers(target)
369        params_list[i] = params
370
371def format_and_output(params_list, output):
372    """Write board parameters into a file.
373
374    Columnate the board parameters, sort lines alphabetically,
375    and then write them to a file.
376
377    Arguments:
378      params_list: The list of board parameters
379      output: The path to the output file
380    """
381    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
382              'options', 'maintainers')
383
384    # First, decide the width of each column
385    max_length = dict([ (f, 0) for f in FIELDS])
386    for params in params_list:
387        for f in FIELDS:
388            max_length[f] = max(max_length[f], len(params[f]))
389
390    output_lines = []
391    for params in params_list:
392        line = ''
393        for f in FIELDS:
394            # insert two spaces between fields like column -t would
395            line += '  ' + params[f].ljust(max_length[f])
396        output_lines.append(line.strip())
397
398    # ignore case when sorting
399    output_lines.sort(key=str.lower)
400
401    with open(output, 'w') as f:
402        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
403
404def gen_boards_cfg(output, jobs=1, force=False):
405    """Generate a board database file.
406
407    Arguments:
408      output: The name of the output file
409      jobs: The number of jobs to run simultaneously
410      force: Force to generate the output even if it is new
411    """
412    check_top_directory()
413
414    if not force and output_is_new(output):
415        print "%s is up to date. Nothing to do." % output
416        sys.exit(0)
417
418    params_list = scan_defconfigs(jobs)
419    insert_maintainers_info(params_list)
420    format_and_output(params_list, output)
421
422def main():
423    try:
424        cpu_count = multiprocessing.cpu_count()
425    except NotImplementedError:
426        cpu_count = 1
427
428    parser = optparse.OptionParser()
429    # Add options here
430    parser.add_option('-f', '--force', action="store_true", default=False,
431                      help='regenerate the output even if it is new')
432    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
433                      help='the number of jobs to run simultaneously')
434    parser.add_option('-o', '--output', default=OUTPUT_FILE,
435                      help='output file [default=%s]' % OUTPUT_FILE)
436    (options, args) = parser.parse_args()
437
438    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
439
440if __name__ == '__main__':
441    main()
442