xref: /openbmc/u-boot/tools/genboardscfg.py (revision 3bf801a2) 
1#!/usr/bin/env python2
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to a board database.
10
11Run 'tools/genboardscfg.py' to create a board database.
12
13Run 'tools/genboardscfg.py -h' for available options.
14
15Python 2.6 or later, but not Python 3.x is necessary to run this script.
16"""
17
18import errno
19import fnmatch
20import glob
21import multiprocessing
22import optparse
23import os
24import subprocess
25import sys
26import tempfile
27import time
28
29sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
30import kconfiglib
31
32### constant variables ###
33OUTPUT_FILE = 'boards.cfg'
34CONFIG_DIR = 'configs'
35SLEEP_TIME = 0.03
36COMMENT_BLOCK = '''#
37# List of boards
38#   Automatically generated by %s: don't edit
39#
40# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
41
42''' % __file__
43
44### helper functions ###
45def try_remove(f):
46    """Remove a file ignoring 'No such file or directory' error."""
47    try:
48        os.remove(f)
49    except OSError as exception:
50        # Ignore 'No such file or directory' error
51        if exception.errno != errno.ENOENT:
52            raise
53
54def check_top_directory():
55    """Exit if we are not at the top of source directory."""
56    for f in ('README', 'Licenses'):
57        if not os.path.exists(f):
58            sys.exit('Please run at the top of source directory.')
59
60def output_is_new(output):
61    """Check if the output file is up to date.
62
63    Returns:
64      True if the given output file exists and is newer than any of
65      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
66    """
67    try:
68        ctime = os.path.getctime(output)
69    except OSError as exception:
70        if exception.errno == errno.ENOENT:
71            # return False on 'No such file or directory' error
72            return False
73        else:
74            raise
75
76    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
77        for filename in fnmatch.filter(filenames, '*_defconfig'):
78            if fnmatch.fnmatch(filename, '.*'):
79                continue
80            filepath = os.path.join(dirpath, filename)
81            if ctime < os.path.getctime(filepath):
82                return False
83
84    for (dirpath, dirnames, filenames) in os.walk('.'):
85        for filename in filenames:
86            if (fnmatch.fnmatch(filename, '*~') or
87                not fnmatch.fnmatch(filename, 'Kconfig*') and
88                not filename == 'MAINTAINERS'):
89                continue
90            filepath = os.path.join(dirpath, filename)
91            if ctime < os.path.getctime(filepath):
92                return False
93
94    # Detect a board that has been removed since the current board database
95    # was generated
96    with open(output) as f:
97        for line in f:
98            if line[0] == '#' or line == '\n':
99                continue
100            defconfig = line.split()[6] + '_defconfig'
101            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
102                return False
103
104    return True
105
106### classes ###
107class KconfigScanner:
108
109    """Kconfig scanner."""
110
111    ### constant variable only used in this class ###
112    _SYMBOL_TABLE = {
113        'arch' : 'SYS_ARCH',
114        'cpu' : 'SYS_CPU',
115        'soc' : 'SYS_SOC',
116        'vendor' : 'SYS_VENDOR',
117        'board' : 'SYS_BOARD',
118        'config' : 'SYS_CONFIG_NAME',
119        'options' : 'SYS_EXTRA_OPTIONS'
120    }
121
122    def __init__(self):
123        """Scan all the Kconfig files and create a Config object."""
124        # Define environment variables referenced from Kconfig
125        os.environ['srctree'] = os.getcwd()
126        os.environ['UBOOTVERSION'] = 'dummy'
127        os.environ['KCONFIG_OBJDIR'] = ''
128        self._conf = kconfiglib.Config()
129
130    def __del__(self):
131        """Delete a leftover temporary file before exit.
132
133        The scan() method of this class creates a temporay file and deletes
134        it on success.  If scan() method throws an exception on the way,
135        the temporary file might be left over.  In that case, it should be
136        deleted in this destructor.
137        """
138        if hasattr(self, '_tmpfile') and self._tmpfile:
139            try_remove(self._tmpfile)
140
141    def scan(self, defconfig):
142        """Load a defconfig file to obtain board parameters.
143
144        Arguments:
145          defconfig: path to the defconfig file to be processed
146
147        Returns:
148          A dictionary of board parameters.  It has a form of:
149          {
150              'arch': <arch_name>,
151              'cpu': <cpu_name>,
152              'soc': <soc_name>,
153              'vendor': <vendor_name>,
154              'board': <board_name>,
155              'target': <target_name>,
156              'config': <config_header_name>,
157              'options': <extra_options>
158          }
159        """
160        # strip special prefixes and save it in a temporary file
161        fd, self._tmpfile = tempfile.mkstemp()
162        with os.fdopen(fd, 'w') as f:
163            for line in open(defconfig):
164                colon = line.find(':CONFIG_')
165                if colon == -1:
166                    f.write(line)
167                else:
168                    f.write(line[colon + 1:])
169
170        self._conf.load_config(self._tmpfile)
171
172        try_remove(self._tmpfile)
173        self._tmpfile = None
174
175        params = {}
176
177        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
178        # Set '-' if the value is empty.
179        for key, symbol in self._SYMBOL_TABLE.items():
180            value = self._conf.get_symbol(symbol).get_value()
181            if value:
182                params[key] = value
183            else:
184                params[key] = '-'
185
186        defconfig = os.path.basename(defconfig)
187        params['target'], match, rear = defconfig.partition('_defconfig')
188        assert match and not rear, '%s : invalid defconfig' % defconfig
189
190        # fix-up for aarch64
191        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
192            params['arch'] = 'aarch64'
193
194        # fix-up options field. It should have the form:
195        # <config name>[:comma separated config options]
196        if params['options'] != '-':
197            params['options'] = params['config'] + ':' + \
198                                params['options'].replace(r'\"', '"')
199        elif params['config'] != params['target']:
200            params['options'] = params['config']
201
202        return params
203
204def scan_defconfigs_for_multiprocess(queue, defconfigs):
205    """Scan defconfig files and queue their board parameters
206
207    This function is intended to be passed to
208    multiprocessing.Process() constructor.
209
210    Arguments:
211      queue: An instance of multiprocessing.Queue().
212             The resulting board parameters are written into it.
213      defconfigs: A sequence of defconfig files to be scanned.
214    """
215    kconf_scanner = KconfigScanner()
216    for defconfig in defconfigs:
217        queue.put(kconf_scanner.scan(defconfig))
218
219def read_queues(queues, params_list):
220    """Read the queues and append the data to the paramers list"""
221    for q in queues:
222        while not q.empty():
223            params_list.append(q.get())
224
225def scan_defconfigs(jobs=1):
226    """Collect board parameters for all defconfig files.
227
228    This function invokes multiple processes for faster processing.
229
230    Arguments:
231      jobs: The number of jobs to run simultaneously
232    """
233    all_defconfigs = []
234    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
235        for filename in fnmatch.filter(filenames, '*_defconfig'):
236            if fnmatch.fnmatch(filename, '.*'):
237                continue
238            all_defconfigs.append(os.path.join(dirpath, filename))
239
240    total_boards = len(all_defconfigs)
241    processes = []
242    queues = []
243    for i in range(jobs):
244        defconfigs = all_defconfigs[total_boards * i / jobs :
245                                    total_boards * (i + 1) / jobs]
246        q = multiprocessing.Queue(maxsize=-1)
247        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
248                                    args=(q, defconfigs))
249        p.start()
250        processes.append(p)
251        queues.append(q)
252
253    # The resulting data should be accumulated to this list
254    params_list = []
255
256    # Data in the queues should be retrieved preriodically.
257    # Otherwise, the queues would become full and subprocesses would get stuck.
258    while any([p.is_alive() for p in processes]):
259        read_queues(queues, params_list)
260        # sleep for a while until the queues are filled
261        time.sleep(SLEEP_TIME)
262
263    # Joining subprocesses just in case
264    # (All subprocesses should already have been finished)
265    for p in processes:
266        p.join()
267
268    # retrieve leftover data
269    read_queues(queues, params_list)
270
271    return params_list
272
273class MaintainersDatabase:
274
275    """The database of board status and maintainers."""
276
277    def __init__(self):
278        """Create an empty database."""
279        self.database = {}
280
281    def get_status(self, target):
282        """Return the status of the given board.
283
284        The board status is generally either 'Active' or 'Orphan'.
285        Display a warning message and return '-' if status information
286        is not found.
287
288        Returns:
289          'Active', 'Orphan' or '-'.
290        """
291        if not target in self.database:
292            print >> sys.stderr, "WARNING: no status info for '%s'" % target
293            return '-'
294
295        tmp = self.database[target][0]
296        if tmp.startswith('Maintained'):
297            return 'Active'
298        elif tmp.startswith('Orphan'):
299            return 'Orphan'
300        else:
301            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
302                                  (tmp, target))
303            return '-'
304
305    def get_maintainers(self, target):
306        """Return the maintainers of the given board.
307
308        Returns:
309          Maintainers of the board.  If the board has two or more maintainers,
310          they are separated with colons.
311        """
312        if not target in self.database:
313            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
314            return ''
315
316        return ':'.join(self.database[target][1])
317
318    def parse_file(self, file):
319        """Parse a MAINTAINERS file.
320
321        Parse a MAINTAINERS file and accumulates board status and
322        maintainers information.
323
324        Arguments:
325          file: MAINTAINERS file to be parsed
326        """
327        targets = []
328        maintainers = []
329        status = '-'
330        for line in open(file):
331            # Check also commented maintainers
332            if line[:3] == '#M:':
333                line = line[1:]
334            tag, rest = line[:2], line[2:].strip()
335            if tag == 'M:':
336                maintainers.append(rest)
337            elif tag == 'F:':
338                # expand wildcard and filter by 'configs/*_defconfig'
339                for f in glob.glob(rest):
340                    front, match, rear = f.partition('configs/')
341                    if not front and match:
342                        front, match, rear = rear.rpartition('_defconfig')
343                        if match and not rear:
344                            targets.append(front)
345            elif tag == 'S:':
346                status = rest
347            elif line == '\n':
348                for target in targets:
349                    self.database[target] = (status, maintainers)
350                targets = []
351                maintainers = []
352                status = '-'
353        if targets:
354            for target in targets:
355                self.database[target] = (status, maintainers)
356
357def insert_maintainers_info(params_list):
358    """Add Status and Maintainers information to the board parameters list.
359
360    Arguments:
361      params_list: A list of the board parameters
362    """
363    database = MaintainersDatabase()
364    for (dirpath, dirnames, filenames) in os.walk('.'):
365        if 'MAINTAINERS' in filenames:
366            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
367
368    for i, params in enumerate(params_list):
369        target = params['target']
370        params['status'] = database.get_status(target)
371        params['maintainers'] = database.get_maintainers(target)
372        params_list[i] = params
373
374def format_and_output(params_list, output):
375    """Write board parameters into a file.
376
377    Columnate the board parameters, sort lines alphabetically,
378    and then write them to a file.
379
380    Arguments:
381      params_list: The list of board parameters
382      output: The path to the output file
383    """
384    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
385              'options', 'maintainers')
386
387    # First, decide the width of each column
388    max_length = dict([ (f, 0) for f in FIELDS])
389    for params in params_list:
390        for f in FIELDS:
391            max_length[f] = max(max_length[f], len(params[f]))
392
393    output_lines = []
394    for params in params_list:
395        line = ''
396        for f in FIELDS:
397            # insert two spaces between fields like column -t would
398            line += '  ' + params[f].ljust(max_length[f])
399        output_lines.append(line.strip())
400
401    # ignore case when sorting
402    output_lines.sort(key=str.lower)
403
404    with open(output, 'w') as f:
405        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
406
407def gen_boards_cfg(output, jobs=1, force=False):
408    """Generate a board database file.
409
410    Arguments:
411      output: The name of the output file
412      jobs: The number of jobs to run simultaneously
413      force: Force to generate the output even if it is new
414    """
415    check_top_directory()
416
417    if not force and output_is_new(output):
418        print "%s is up to date. Nothing to do." % output
419        sys.exit(0)
420
421    params_list = scan_defconfigs(jobs)
422    insert_maintainers_info(params_list)
423    format_and_output(params_list, output)
424
425def main():
426    try:
427        cpu_count = multiprocessing.cpu_count()
428    except NotImplementedError:
429        cpu_count = 1
430
431    parser = optparse.OptionParser()
432    # Add options here
433    parser.add_option('-f', '--force', action="store_true", default=False,
434                      help='regenerate the output even if it is new')
435    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
436                      help='the number of jobs to run simultaneously')
437    parser.add_option('-o', '--output', default=OUTPUT_FILE,
438                      help='output file [default=%s]' % OUTPUT_FILE)
439    (options, args) = parser.parse_args()
440
441    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
442
443if __name__ == '__main__':
444    main()
445