xref: /openbmc/u-boot/tools/genboardscfg.py (revision 5ff10aa7) 
1#!/usr/bin/env python
2#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier:	GPL-2.0+
6#
7
8"""
9Converter from Kconfig and MAINTAINERS to boards.cfg
10
11Run 'tools/genboardscfg.py' to create boards.cfg file.
12
13Run 'tools/genboardscfg.py -h' for available options.
14"""
15
16import errno
17import fnmatch
18import glob
19import optparse
20import os
21import re
22import shutil
23import subprocess
24import sys
25import tempfile
26import time
27
28BOARD_FILE = 'boards.cfg'
29CONFIG_DIR = 'configs'
30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'),
31                '-i', '-d', '-', '-s', '8']
32SHOW_GNU_MAKE = 'scripts/show-gnu-make'
33SLEEP_TIME=0.03
34
35COMMENT_BLOCK = '''#
36# List of boards
37#   Automatically generated by %s: don't edit
38#
39# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
40
41''' % __file__
42
43### helper functions ###
44def get_terminal_columns():
45    """Get the width of the terminal.
46
47    Returns:
48      The width of the terminal, or zero if the stdout is not
49      associated with tty.
50    """
51    try:
52        return shutil.get_terminal_size().columns # Python 3.3~
53    except AttributeError:
54        import fcntl
55        import termios
56        import struct
57        arg = struct.pack('hhhh', 0, 0, 0, 0)
58        try:
59            ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg)
60        except IOError as exception:
61            # If 'Inappropriate ioctl for device' error occurs,
62            # stdout is probably redirected. Return 0.
63            return 0
64        return struct.unpack('hhhh', ret)[1]
65
66def get_devnull():
67    """Get the file object of '/dev/null' device."""
68    try:
69        devnull = subprocess.DEVNULL # py3k
70    except AttributeError:
71        devnull = open(os.devnull, 'wb')
72    return devnull
73
74def check_top_directory():
75    """Exit if we are not at the top of source directory."""
76    for f in ('README', 'Licenses'):
77        if not os.path.exists(f):
78            sys.exit('Please run at the top of source directory.')
79
80def get_make_cmd():
81    """Get the command name of GNU Make."""
82    process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE)
83    ret = process.communicate()
84    if process.returncode:
85        sys.exit('GNU Make not found')
86    return ret[0].rstrip()
87
88### classes ###
89class MaintainersDatabase:
90
91    """The database of board status and maintainers."""
92
93    def __init__(self):
94        """Create an empty database."""
95        self.database = {}
96
97    def get_status(self, target):
98        """Return the status of the given board.
99
100        Returns:
101          Either 'Active' or 'Orphan'
102        """
103        tmp = self.database[target][0]
104        if tmp.startswith('Maintained'):
105            return 'Active'
106        elif tmp.startswith('Orphan'):
107            return 'Orphan'
108        else:
109            print >> sys.stderr, 'Error: %s: unknown status' % tmp
110
111    def get_maintainers(self, target):
112        """Return the maintainers of the given board.
113
114        If the board has two or more maintainers, they are separated
115        with colons.
116        """
117        return ':'.join(self.database[target][1])
118
119    def parse_file(self, file):
120        """Parse the given MAINTAINERS file.
121
122        This method parses MAINTAINERS and add board status and
123        maintainers information to the database.
124
125        Arguments:
126          file: MAINTAINERS file to be parsed
127        """
128        targets = []
129        maintainers = []
130        status = '-'
131        for line in open(file):
132            tag, rest = line[:2], line[2:].strip()
133            if tag == 'M:':
134                maintainers.append(rest)
135            elif tag == 'F:':
136                # expand wildcard and filter by 'configs/*_defconfig'
137                for f in glob.glob(rest):
138                    front, match, rear = f.partition('configs/')
139                    if not front and match:
140                        front, match, rear = rear.rpartition('_defconfig')
141                        if match and not rear:
142                            targets.append(front)
143            elif tag == 'S:':
144                status = rest
145            elif line == '\n' and targets:
146                for target in targets:
147                    self.database[target] = (status, maintainers)
148                targets = []
149                maintainers = []
150                status = '-'
151        if targets:
152            for target in targets:
153                self.database[target] = (status, maintainers)
154
155class DotConfigParser:
156
157    """A parser of .config file.
158
159    Each line of the output should have the form of:
160    Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
161    Most of them are extracted from .config file.
162    MAINTAINERS files are also consulted for Status and Maintainers fields.
163    """
164
165    re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
166    re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
167    re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"')
168    re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"')
169    re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"')
170    re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"')
171    re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"')
172    re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc),
173               ('vendor', re_vendor), ('board', re_board),
174               ('config', re_config), ('options', re_options))
175    must_fields = ('arch', 'config')
176
177    def __init__(self, build_dir, output, maintainers_database):
178        """Create a new .config perser.
179
180        Arguments:
181          build_dir: Build directory where .config is located
182          output: File object which the result is written to
183          maintainers_database: An instance of class MaintainersDatabase
184        """
185        self.dotconfig = os.path.join(build_dir, '.config')
186        self.output = output
187        self.database = maintainers_database
188
189    def parse(self, defconfig):
190        """Parse .config file and output one-line database for the given board.
191
192        Arguments:
193          defconfig: Board (defconfig) name
194        """
195        fields = {}
196        for line in open(self.dotconfig):
197            if not line.startswith('CONFIG_SYS_'):
198                continue
199            for (key, pattern) in self.re_list:
200                m = pattern.match(line)
201                if m and m.group(1):
202                    fields[key] = m.group(1)
203                    break
204
205        # sanity check of '.config' file
206        for field in self.must_fields:
207            if not field in fields:
208                sys.exit('Error: %s is not defined in %s' % (field, defconfig))
209
210        # fix-up for aarch64
211        if fields['arch'] == 'arm' and 'cpu' in fields:
212            if fields['cpu'] == 'armv8':
213                fields['arch'] = 'aarch64'
214
215        target, match, rear = defconfig.partition('_defconfig')
216        assert match and not rear, \
217                                '%s : invalid defconfig file name' % defconfig
218
219        fields['status'] = self.database.get_status(target)
220        fields['maintainers'] = self.database.get_maintainers(target)
221
222        if 'options' in fields:
223            options = fields['config'] + ':' + \
224                      fields['options'].replace(r'\"', '"')
225        elif fields['config'] != target:
226            options = fields['config']
227        else:
228            options = '-'
229
230        self.output.write((' '.join(['%s'] * 9) + '\n')  %
231                          (fields['status'],
232                           fields['arch'],
233                           fields.get('cpu', '-'),
234                           fields.get('soc', '-'),
235                           fields.get('vendor', '-'),
236                           fields.get('board', '-'),
237                           target,
238                           options,
239                           fields['maintainers']))
240
241class Slot:
242
243    """A slot to store a subprocess.
244
245    Each instance of this class handles one subprocess.
246    This class is useful to control multiple processes
247    for faster processing.
248    """
249
250    def __init__(self, output, maintainers_database, devnull, make_cmd):
251        """Create a new slot.
252
253        Arguments:
254          output: File object which the result is written to
255          maintainers_database: An instance of class MaintainersDatabase
256        """
257        self.occupied = False
258        self.build_dir = tempfile.mkdtemp()
259        self.devnull = devnull
260        self.make_cmd = make_cmd
261        self.parser = DotConfigParser(self.build_dir, output,
262                                      maintainers_database)
263
264    def __del__(self):
265        """Delete the working directory"""
266        shutil.rmtree(self.build_dir)
267
268    def add(self, defconfig):
269        """Add a new subprocess to the slot.
270
271        Fails if the slot is occupied, that is, the current subprocess
272        is still running.
273
274        Arguments:
275          defconfig: Board (defconfig) name
276
277        Returns:
278          Return True on success or False on fail
279        """
280        if self.occupied:
281            return False
282        o = 'O=' + self.build_dir
283        self.ps = subprocess.Popen([self.make_cmd, o, defconfig],
284                                   stdout=self.devnull)
285        self.defconfig = defconfig
286        self.occupied = True
287        return True
288
289    def poll(self):
290        """Check if the subprocess is running and invoke the .config
291        parser if the subprocess is terminated.
292
293        Returns:
294          Return True if the subprocess is terminated, False otherwise
295        """
296        if not self.occupied:
297            return True
298        if self.ps.poll() == None:
299            return False
300        self.parser.parse(self.defconfig)
301        self.occupied = False
302        return True
303
304class Slots:
305
306    """Controller of the array of subprocess slots."""
307
308    def __init__(self, jobs, output, maintainers_database):
309        """Create a new slots controller.
310
311        Arguments:
312          jobs: A number of slots to instantiate
313          output: File object which the result is written to
314          maintainers_database: An instance of class MaintainersDatabase
315        """
316        self.slots = []
317        devnull = get_devnull()
318        make_cmd = get_make_cmd()
319        for i in range(jobs):
320            self.slots.append(Slot(output, maintainers_database,
321                                   devnull, make_cmd))
322
323    def add(self, defconfig):
324        """Add a new subprocess if a vacant slot is available.
325
326        Arguments:
327          defconfig: Board (defconfig) name
328
329        Returns:
330          Return True on success or False on fail
331        """
332        for slot in self.slots:
333            if slot.add(defconfig):
334                return True
335        return False
336
337    def available(self):
338        """Check if there is a vacant slot.
339
340        Returns:
341          Return True if a vacant slot is found, False if all slots are full
342        """
343        for slot in self.slots:
344            if slot.poll():
345                return True
346        return False
347
348    def empty(self):
349        """Check if all slots are vacant.
350
351        Returns:
352          Return True if all slots are vacant, False if at least one slot
353          is running
354        """
355        ret = True
356        for slot in self.slots:
357            if not slot.poll():
358                ret = False
359        return ret
360
361class Indicator:
362
363    """A class to control the progress indicator."""
364
365    MIN_WIDTH = 15
366    MAX_WIDTH = 70
367
368    def __init__(self, total):
369        """Create an instance.
370
371        Arguments:
372          total: A number of boards
373        """
374        self.total = total
375        self.cur = 0
376        width = get_terminal_columns()
377        width = min(width, self.MAX_WIDTH)
378        width -= self.MIN_WIDTH
379        if width > 0:
380            self.enabled = True
381        else:
382            self.enabled = False
383        self.width = width
384
385    def inc(self):
386        """Increment the counter and show the progress bar."""
387        if not self.enabled:
388            return
389        self.cur += 1
390        arrow_len = self.width * self.cur // self.total
391        msg = '%4d/%d [' % (self.cur, self.total)
392        msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']'
393        sys.stdout.write('\r' + msg)
394        sys.stdout.flush()
395
396def __gen_boards_cfg(jobs):
397    """Generate boards.cfg file.
398
399    Arguments:
400      jobs: The number of jobs to run simultaneously
401
402    Note:
403      The incomplete boards.cfg is left over when an error (including
404      the termination by the keyboard interrupt) occurs on the halfway.
405    """
406    check_top_directory()
407    print 'Generating %s ...  (jobs: %d)' % (BOARD_FILE, jobs)
408
409    # All the defconfig files to be processed
410    defconfigs = []
411    for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
412        dirpath = dirpath[len(CONFIG_DIR) + 1:]
413        for filename in fnmatch.filter(filenames, '*_defconfig'):
414            defconfigs.append(os.path.join(dirpath, filename))
415
416    # Parse all the MAINTAINERS files
417    maintainers_database = MaintainersDatabase()
418    for (dirpath, dirnames, filenames) in os.walk('.'):
419        if 'MAINTAINERS' in filenames:
420            maintainers_database.parse_file(os.path.join(dirpath,
421                                                         'MAINTAINERS'))
422
423    # Output lines should be piped into the reformat tool
424    reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE,
425                                        stdout=open(BOARD_FILE, 'w'))
426    pipe = reformat_process.stdin
427    pipe.write(COMMENT_BLOCK)
428
429    indicator = Indicator(len(defconfigs))
430    slots = Slots(jobs, pipe, maintainers_database)
431
432    # Main loop to process defconfig files:
433    #  Add a new subprocess into a vacant slot.
434    #  Sleep if there is no available slot.
435    for defconfig in defconfigs:
436        while not slots.add(defconfig):
437            while not slots.available():
438                # No available slot: sleep for a while
439                time.sleep(SLEEP_TIME)
440        indicator.inc()
441
442    # wait until all the subprocesses finish
443    while not slots.empty():
444        time.sleep(SLEEP_TIME)
445    print ''
446
447    # wait until the reformat tool finishes
448    reformat_process.communicate()
449    if reformat_process.returncode != 0:
450        sys.exit('"%s" failed' % REFORMAT_CMD[0])
451
452def gen_boards_cfg(jobs):
453    """Generate boards.cfg file.
454
455    The incomplete boards.cfg is deleted if an error (including
456    the termination by the keyboard interrupt) occurs on the halfway.
457
458    Arguments:
459      jobs: The number of jobs to run simultaneously
460    """
461    try:
462        __gen_boards_cfg(jobs)
463    except:
464        # We should remove incomplete boards.cfg
465        try:
466            os.remove(BOARD_FILE)
467        except OSError as exception:
468            # Ignore 'No such file or directory' error
469            if exception.errno != errno.ENOENT:
470                raise
471        raise
472
473def main():
474    parser = optparse.OptionParser()
475    # Add options here
476    parser.add_option('-j', '--jobs',
477                      help='the number of jobs to run simultaneously')
478    (options, args) = parser.parse_args()
479    if options.jobs:
480        try:
481            jobs = int(options.jobs)
482        except ValueError:
483            sys.exit('Option -j (--jobs) takes a number')
484    else:
485        try:
486            jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'],
487                                     stdout=subprocess.PIPE).communicate()[0])
488        except (OSError, ValueError):
489            print 'info: failed to get the number of CPUs. Set jobs to 1'
490            jobs = 1
491    gen_boards_cfg(jobs)
492
493if __name__ == '__main__':
494    main()
495