xref: /openbmc/u-boot/tools/genboardscfg.py (revision 014a953c) 
1#!/usr/bin/env python2
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to a board database.
10
11Run 'tools/genboardscfg.py' to create a board database.
12
13Run 'tools/genboardscfg.py -h' for available options.
14
15Python 2.6 or later, but not Python 3.x is necessary to run this script.
16"""
17
18import errno
19import fnmatch
20import glob
21import multiprocessing
22import optparse
23import os
24import sys
25import tempfile
26import time
27
28sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
29import kconfiglib
30
31### constant variables ###
32OUTPUT_FILE = 'boards.cfg'
33CONFIG_DIR = 'configs'
34SLEEP_TIME = 0.03
35COMMENT_BLOCK = '''#
36# List of boards
37#   Automatically generated by %s: don't edit
38#
39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
40
41''' % __file__
42
43### helper functions ###
44def try_remove(f):
45    """Remove a file ignoring 'No such file or directory' error."""
46    try:
47        os.remove(f)
48    except OSError as exception:
49        # Ignore 'No such file or directory' error
50        if exception.errno != errno.ENOENT:
51            raise
52
53def check_top_directory():
54    """Exit if we are not at the top of source directory."""
55    for f in ('README', 'Licenses'):
56        if not os.path.exists(f):
57            sys.exit('Please run at the top of source directory.')
58
59def output_is_new(output):
60    """Check if the output file is up to date.
61
62    Returns:
63      True if the given output file exists and is newer than any of
64      *_defconfig, MAINTAINERS and Kconfig*.  False otherwise.
65    """
66    try:
67        ctime = os.path.getctime(output)
68    except OSError as exception:
69        if exception.errno == errno.ENOENT:
70            # return False on 'No such file or directory' error
71            return False
72        else:
73            raise
74
75    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
76        for filename in fnmatch.filter(filenames, '*_defconfig'):
77            if fnmatch.fnmatch(filename, '.*'):
78                continue
79            filepath = os.path.join(dirpath, filename)
80            if ctime < os.path.getctime(filepath):
81                return False
82
83    for (dirpath, dirnames, filenames) in os.walk('.'):
84        for filename in filenames:
85            if (fnmatch.fnmatch(filename, '*~') or
86                not fnmatch.fnmatch(filename, 'Kconfig*') and
87                not filename == 'MAINTAINERS'):
88                continue
89            filepath = os.path.join(dirpath, filename)
90            if ctime < os.path.getctime(filepath):
91                return False
92
93    # Detect a board that has been removed since the current board database
94    # was generated
95    with open(output) as f:
96        for line in f:
97            if line[0] == '#' or line == '\n':
98                continue
99            defconfig = line.split()[6] + '_defconfig'
100            if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
101                return False
102
103    return True
104
105### classes ###
106class KconfigScanner:
107
108    """Kconfig scanner."""
109
110    ### constant variable only used in this class ###
111    _SYMBOL_TABLE = {
112        'arch' : 'SYS_ARCH',
113        'cpu' : 'SYS_CPU',
114        'soc' : 'SYS_SOC',
115        'vendor' : 'SYS_VENDOR',
116        'board' : 'SYS_BOARD',
117        'config' : 'SYS_CONFIG_NAME',
118        'options' : 'SYS_EXTRA_OPTIONS'
119    }
120
121    def __init__(self):
122        """Scan all the Kconfig files and create a Config object."""
123        # Define environment variables referenced from Kconfig
124        os.environ['srctree'] = os.getcwd()
125        os.environ['UBOOTVERSION'] = 'dummy'
126        os.environ['KCONFIG_OBJDIR'] = ''
127        self._conf = kconfiglib.Config(print_warnings=False)
128
129    def __del__(self):
130        """Delete a leftover temporary file before exit.
131
132        The scan() method of this class creates a temporay file and deletes
133        it on success.  If scan() method throws an exception on the way,
134        the temporary file might be left over.  In that case, it should be
135        deleted in this destructor.
136        """
137        if hasattr(self, '_tmpfile') and self._tmpfile:
138            try_remove(self._tmpfile)
139
140    def scan(self, defconfig):
141        """Load a defconfig file to obtain board parameters.
142
143        Arguments:
144          defconfig: path to the defconfig file to be processed
145
146        Returns:
147          A dictionary of board parameters.  It has a form of:
148          {
149              'arch': <arch_name>,
150              'cpu': <cpu_name>,
151              'soc': <soc_name>,
152              'vendor': <vendor_name>,
153              'board': <board_name>,
154              'target': <target_name>,
155              'config': <config_header_name>,
156              'options': <extra_options>
157          }
158        """
159        # strip special prefixes and save it in a temporary file
160        fd, self._tmpfile = tempfile.mkstemp()
161        with os.fdopen(fd, 'w') as f:
162            for line in open(defconfig):
163                colon = line.find(':CONFIG_')
164                if colon == -1:
165                    f.write(line)
166                else:
167                    f.write(line[colon + 1:])
168
169        warnings = self._conf.load_config(self._tmpfile)
170        if warnings:
171            for warning in warnings:
172                print '%s: %s' % (defconfig, warning)
173
174        try_remove(self._tmpfile)
175        self._tmpfile = None
176
177        params = {}
178
179        # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
180        # Set '-' if the value is empty.
181        for key, symbol in self._SYMBOL_TABLE.items():
182            value = self._conf.get_symbol(symbol).get_value()
183            if value:
184                params[key] = value
185            else:
186                params[key] = '-'
187
188        defconfig = os.path.basename(defconfig)
189        params['target'], match, rear = defconfig.partition('_defconfig')
190        assert match and not rear, '%s : invalid defconfig' % defconfig
191
192        # fix-up for aarch64
193        if params['arch'] == 'arm' and params['cpu'] == 'armv8':
194            params['arch'] = 'aarch64'
195
196        # fix-up options field. It should have the form:
197        # <config name>[:comma separated config options]
198        if params['options'] != '-':
199            params['options'] = params['config'] + ':' + \
200                                params['options'].replace(r'\"', '"')
201        elif params['config'] != params['target']:
202            params['options'] = params['config']
203
204        return params
205
206def scan_defconfigs_for_multiprocess(queue, defconfigs):
207    """Scan defconfig files and queue their board parameters
208
209    This function is intended to be passed to
210    multiprocessing.Process() constructor.
211
212    Arguments:
213      queue: An instance of multiprocessing.Queue().
214             The resulting board parameters are written into it.
215      defconfigs: A sequence of defconfig files to be scanned.
216    """
217    kconf_scanner = KconfigScanner()
218    for defconfig in defconfigs:
219        queue.put(kconf_scanner.scan(defconfig))
220
221def read_queues(queues, params_list):
222    """Read the queues and append the data to the paramers list"""
223    for q in queues:
224        while not q.empty():
225            params_list.append(q.get())
226
227def scan_defconfigs(jobs=1):
228    """Collect board parameters for all defconfig files.
229
230    This function invokes multiple processes for faster processing.
231
232    Arguments:
233      jobs: The number of jobs to run simultaneously
234    """
235    all_defconfigs = []
236    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
237        for filename in fnmatch.filter(filenames, '*_defconfig'):
238            if fnmatch.fnmatch(filename, '.*'):
239                continue
240            all_defconfigs.append(os.path.join(dirpath, filename))
241
242    total_boards = len(all_defconfigs)
243    processes = []
244    queues = []
245    for i in range(jobs):
246        defconfigs = all_defconfigs[total_boards * i / jobs :
247                                    total_boards * (i + 1) / jobs]
248        q = multiprocessing.Queue(maxsize=-1)
249        p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
250                                    args=(q, defconfigs))
251        p.start()
252        processes.append(p)
253        queues.append(q)
254
255    # The resulting data should be accumulated to this list
256    params_list = []
257
258    # Data in the queues should be retrieved preriodically.
259    # Otherwise, the queues would become full and subprocesses would get stuck.
260    while any([p.is_alive() for p in processes]):
261        read_queues(queues, params_list)
262        # sleep for a while until the queues are filled
263        time.sleep(SLEEP_TIME)
264
265    # Joining subprocesses just in case
266    # (All subprocesses should already have been finished)
267    for p in processes:
268        p.join()
269
270    # retrieve leftover data
271    read_queues(queues, params_list)
272
273    return params_list
274
275class MaintainersDatabase:
276
277    """The database of board status and maintainers."""
278
279    def __init__(self):
280        """Create an empty database."""
281        self.database = {}
282
283    def get_status(self, target):
284        """Return the status of the given board.
285
286        The board status is generally either 'Active' or 'Orphan'.
287        Display a warning message and return '-' if status information
288        is not found.
289
290        Returns:
291          'Active', 'Orphan' or '-'.
292        """
293        if not target in self.database:
294            print >> sys.stderr, "WARNING: no status info for '%s'" % target
295            return '-'
296
297        tmp = self.database[target][0]
298        if tmp.startswith('Maintained'):
299            return 'Active'
300        elif tmp.startswith('Supported'):
301            return 'Active'
302        elif tmp.startswith('Orphan'):
303            return 'Orphan'
304        else:
305            print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
306                                  (tmp, target))
307            return '-'
308
309    def get_maintainers(self, target):
310        """Return the maintainers of the given board.
311
312        Returns:
313          Maintainers of the board.  If the board has two or more maintainers,
314          they are separated with colons.
315        """
316        if not target in self.database:
317            print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
318            return ''
319
320        return ':'.join(self.database[target][1])
321
322    def parse_file(self, file):
323        """Parse a MAINTAINERS file.
324
325        Parse a MAINTAINERS file and accumulates board status and
326        maintainers information.
327
328        Arguments:
329          file: MAINTAINERS file to be parsed
330        """
331        targets = []
332        maintainers = []
333        status = '-'
334        for line in open(file):
335            # Check also commented maintainers
336            if line[:3] == '#M:':
337                line = line[1:]
338            tag, rest = line[:2], line[2:].strip()
339            if tag == 'M:':
340                maintainers.append(rest)
341            elif tag == 'F:':
342                # expand wildcard and filter by 'configs/*_defconfig'
343                for f in glob.glob(rest):
344                    front, match, rear = f.partition('configs/')
345                    if not front and match:
346                        front, match, rear = rear.rpartition('_defconfig')
347                        if match and not rear:
348                            targets.append(front)
349            elif tag == 'S:':
350                status = rest
351            elif line == '\n':
352                for target in targets:
353                    self.database[target] = (status, maintainers)
354                targets = []
355                maintainers = []
356                status = '-'
357        if targets:
358            for target in targets:
359                self.database[target] = (status, maintainers)
360
361def insert_maintainers_info(params_list):
362    """Add Status and Maintainers information to the board parameters list.
363
364    Arguments:
365      params_list: A list of the board parameters
366    """
367    database = MaintainersDatabase()
368    for (dirpath, dirnames, filenames) in os.walk('.'):
369        if 'MAINTAINERS' in filenames:
370            database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
371
372    for i, params in enumerate(params_list):
373        target = params['target']
374        params['status'] = database.get_status(target)
375        params['maintainers'] = database.get_maintainers(target)
376        params_list[i] = params
377
378def format_and_output(params_list, output):
379    """Write board parameters into a file.
380
381    Columnate the board parameters, sort lines alphabetically,
382    and then write them to a file.
383
384    Arguments:
385      params_list: The list of board parameters
386      output: The path to the output file
387    """
388    FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
389              'options', 'maintainers')
390
391    # First, decide the width of each column
392    max_length = dict([ (f, 0) for f in FIELDS])
393    for params in params_list:
394        for f in FIELDS:
395            max_length[f] = max(max_length[f], len(params[f]))
396
397    output_lines = []
398    for params in params_list:
399        line = ''
400        for f in FIELDS:
401            # insert two spaces between fields like column -t would
402            line += '  ' + params[f].ljust(max_length[f])
403        output_lines.append(line.strip())
404
405    # ignore case when sorting
406    output_lines.sort(key=str.lower)
407
408    with open(output, 'w') as f:
409        f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
410
411def gen_boards_cfg(output, jobs=1, force=False):
412    """Generate a board database file.
413
414    Arguments:
415      output: The name of the output file
416      jobs: The number of jobs to run simultaneously
417      force: Force to generate the output even if it is new
418    """
419    check_top_directory()
420
421    if not force and output_is_new(output):
422        print "%s is up to date. Nothing to do." % output
423        sys.exit(0)
424
425    params_list = scan_defconfigs(jobs)
426    insert_maintainers_info(params_list)
427    format_and_output(params_list, output)
428
429def main():
430    try:
431        cpu_count = multiprocessing.cpu_count()
432    except NotImplementedError:
433        cpu_count = 1
434
435    parser = optparse.OptionParser()
436    # Add options here
437    parser.add_option('-f', '--force', action="store_true", default=False,
438                      help='regenerate the output even if it is new')
439    parser.add_option('-j', '--jobs', type='int', default=cpu_count,
440                      help='the number of jobs to run simultaneously')
441    parser.add_option('-o', '--output', default=OUTPUT_FILE,
442                      help='output file [default=%s]' % OUTPUT_FILE)
443    (options, args) = parser.parse_args()
444
445    gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
446
447if __name__ == '__main__':
448    main()
449