1#!/usr/bin/env python 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14""" 15 16import errno 17import fnmatch 18import glob 19import optparse 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import time 27 28BOARD_FILE = 'boards.cfg' 29CONFIG_DIR = 'configs' 30REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 31 '-i', '-d', '-', '-s', '8'] 32SHOW_GNU_MAKE = 'scripts/show-gnu-make' 33SLEEP_TIME=0.03 34 35COMMENT_BLOCK = '''# 36# List of boards 37# Automatically generated by %s: don't edit 38# 39# Status, Arch, CPU(:SPLCPU), SoC, Vendor, Board, Target, Options, Maintainers 40 41''' % __file__ 42 43### helper functions ### 44def get_terminal_columns(): 45 """Get the width of the terminal. 46 47 Returns: 48 The width of the terminal, or zero if the stdout is not 49 associated with tty. 50 """ 51 try: 52 return shutil.get_terminal_size().columns # Python 3.3~ 53 except AttributeError: 54 import fcntl 55 import termios 56 import struct 57 arg = struct.pack('hhhh', 0, 0, 0, 0) 58 try: 59 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 60 except IOError as exception: 61 if exception.errno != errno.ENOTTY: 62 raise 63 # If 'Inappropriate ioctl for device' error occurs, 64 # stdout is probably redirected. Return 0. 65 return 0 66 return struct.unpack('hhhh', ret)[1] 67 68def get_devnull(): 69 """Get the file object of '/dev/null' device.""" 70 try: 71 devnull = subprocess.DEVNULL # py3k 72 except AttributeError: 73 devnull = open(os.devnull, 'wb') 74 return devnull 75 76def check_top_directory(): 77 """Exit if we are not at the top of source directory.""" 78 for f in ('README', 'Licenses'): 79 if not os.path.exists(f): 80 print >> sys.stderr, 'Please run at the top of source directory.' 81 sys.exit(1) 82 83def get_make_cmd(): 84 """Get the command name of GNU Make.""" 85 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 86 ret = process.communicate() 87 if process.returncode: 88 print >> sys.stderr, 'GNU Make not found' 89 sys.exit(1) 90 return ret[0].rstrip() 91 92### classes ### 93class MaintainersDatabase: 94 95 """The database of board status and maintainers.""" 96 97 def __init__(self): 98 """Create an empty database.""" 99 self.database = {} 100 101 def get_status(self, target): 102 """Return the status of the given board. 103 104 Returns: 105 Either 'Active' or 'Orphan' 106 """ 107 tmp = self.database[target][0] 108 if tmp.startswith('Maintained'): 109 return 'Active' 110 elif tmp.startswith('Orphan'): 111 return 'Orphan' 112 else: 113 print >> sys.stderr, 'Error: %s: unknown status' % tmp 114 115 def get_maintainers(self, target): 116 """Return the maintainers of the given board. 117 118 If the board has two or more maintainers, they are separated 119 with colons. 120 """ 121 return ':'.join(self.database[target][1]) 122 123 def parse_file(self, file): 124 """Parse the given MAINTAINERS file. 125 126 This method parses MAINTAINERS and add board status and 127 maintainers information to the database. 128 129 Arguments: 130 file: MAINTAINERS file to be parsed 131 """ 132 targets = [] 133 maintainers = [] 134 status = '-' 135 for line in open(file): 136 tag, rest = line[:2], line[2:].strip() 137 if tag == 'M:': 138 maintainers.append(rest) 139 elif tag == 'F:': 140 # expand wildcard and filter by 'configs/*_defconfig' 141 for f in glob.glob(rest): 142 front, match, rear = f.partition('configs/') 143 if not front and match: 144 front, match, rear = rear.rpartition('_defconfig') 145 if match and not rear: 146 targets.append(front) 147 elif tag == 'S:': 148 status = rest 149 elif line == '\n' and targets: 150 for target in targets: 151 self.database[target] = (status, maintainers) 152 targets = [] 153 maintainers = [] 154 status = '-' 155 if targets: 156 for target in targets: 157 self.database[target] = (status, maintainers) 158 159class DotConfigParser: 160 161 """A parser of .config file. 162 163 Each line of the output should have the form of: 164 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 165 Most of them are extracted from .config file. 166 MAINTAINERS files are also consulted for Status and Maintainers fields. 167 """ 168 169 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 170 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 171 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 172 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 173 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 174 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 175 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 176 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 177 ('vendor', re_vendor), ('board', re_board), 178 ('config', re_config), ('options', re_options)) 179 must_fields = ('arch', 'config') 180 181 def __init__(self, build_dir, output, maintainers_database): 182 """Create a new .config perser. 183 184 Arguments: 185 build_dir: Build directory where .config is located 186 output: File object which the result is written to 187 maintainers_database: An instance of class MaintainersDatabase 188 """ 189 self.dotconfig = os.path.join(build_dir, '.config') 190 self.output = output 191 self.database = maintainers_database 192 193 def parse(self, defconfig): 194 """Parse .config file and output one-line database for the given board. 195 196 Arguments: 197 defconfig: Board (defconfig) name 198 """ 199 fields = {} 200 for line in open(self.dotconfig): 201 if not line.startswith('CONFIG_SYS_'): 202 continue 203 for (key, pattern) in self.re_list: 204 m = pattern.match(line) 205 if m and m.group(1): 206 fields[key] = m.group(1) 207 break 208 209 # sanity check of '.config' file 210 for field in self.must_fields: 211 if not field in fields: 212 print >> sys.stderr, 'Error: %s is not defined in %s' % \ 213 (field, defconfig) 214 sys.exit(1) 215 216 # fix-up for aarch64 and tegra 217 if fields['arch'] == 'arm' and 'cpu' in fields: 218 if fields['cpu'] == 'armv8': 219 fields['arch'] = 'aarch64' 220 if 'soc' in fields and re.match('tegra[0-9]*$', fields['soc']): 221 fields['cpu'] += ':arm720t' 222 223 target, match, rear = defconfig.partition('_defconfig') 224 assert match and not rear, \ 225 '%s : invalid defconfig file name' % defconfig 226 227 fields['status'] = self.database.get_status(target) 228 fields['maintainers'] = self.database.get_maintainers(target) 229 230 if 'options' in fields: 231 options = fields['config'] + ':' + \ 232 fields['options'].replace(r'\"', '"') 233 elif fields['config'] != target: 234 options = fields['config'] 235 else: 236 options = '-' 237 238 self.output.write((' '.join(['%s'] * 9) + '\n') % 239 (fields['status'], 240 fields['arch'], 241 fields.get('cpu', '-'), 242 fields.get('soc', '-'), 243 fields.get('vendor', '-'), 244 fields.get('board', '-'), 245 target, 246 options, 247 fields['maintainers'])) 248 249class Slot: 250 251 """A slot to store a subprocess. 252 253 Each instance of this class handles one subprocess. 254 This class is useful to control multiple processes 255 for faster processing. 256 """ 257 258 def __init__(self, output, maintainers_database, devnull, make_cmd): 259 """Create a new slot. 260 261 Arguments: 262 output: File object which the result is written to 263 maintainers_database: An instance of class MaintainersDatabase 264 """ 265 self.occupied = False 266 self.build_dir = tempfile.mkdtemp() 267 self.devnull = devnull 268 self.make_cmd = make_cmd 269 self.parser = DotConfigParser(self.build_dir, output, 270 maintainers_database) 271 272 def __del__(self): 273 """Delete the working directory""" 274 shutil.rmtree(self.build_dir) 275 276 def add(self, defconfig): 277 """Add a new subprocess to the slot. 278 279 Fails if the slot is occupied, that is, the current subprocess 280 is still running. 281 282 Arguments: 283 defconfig: Board (defconfig) name 284 285 Returns: 286 Return True on success or False on fail 287 """ 288 if self.occupied: 289 return False 290 o = 'O=' + self.build_dir 291 self.ps = subprocess.Popen([self.make_cmd, o, defconfig], 292 stdout=self.devnull) 293 self.defconfig = defconfig 294 self.occupied = True 295 return True 296 297 def poll(self): 298 """Check if the subprocess is running and invoke the .config 299 parser if the subprocess is terminated. 300 301 Returns: 302 Return True if the subprocess is terminated, False otherwise 303 """ 304 if not self.occupied: 305 return True 306 if self.ps.poll() == None: 307 return False 308 self.parser.parse(self.defconfig) 309 self.occupied = False 310 return True 311 312class Slots: 313 314 """Controller of the array of subprocess slots.""" 315 316 def __init__(self, jobs, output, maintainers_database): 317 """Create a new slots controller. 318 319 Arguments: 320 jobs: A number of slots to instantiate 321 output: File object which the result is written to 322 maintainers_database: An instance of class MaintainersDatabase 323 """ 324 self.slots = [] 325 devnull = get_devnull() 326 make_cmd = get_make_cmd() 327 for i in range(jobs): 328 self.slots.append(Slot(output, maintainers_database, 329 devnull, make_cmd)) 330 331 def add(self, defconfig): 332 """Add a new subprocess if a vacant slot is available. 333 334 Arguments: 335 defconfig: Board (defconfig) name 336 337 Returns: 338 Return True on success or False on fail 339 """ 340 for slot in self.slots: 341 if slot.add(defconfig): 342 return True 343 return False 344 345 def available(self): 346 """Check if there is a vacant slot. 347 348 Returns: 349 Return True if a vacant slot is found, False if all slots are full 350 """ 351 for slot in self.slots: 352 if slot.poll(): 353 return True 354 return False 355 356 def empty(self): 357 """Check if all slots are vacant. 358 359 Returns: 360 Return True if all slots are vacant, False if at least one slot 361 is running 362 """ 363 ret = True 364 for slot in self.slots: 365 if not slot.poll(): 366 ret = False 367 return ret 368 369class Indicator: 370 371 """A class to control the progress indicator.""" 372 373 MIN_WIDTH = 15 374 MAX_WIDTH = 70 375 376 def __init__(self, total): 377 """Create an instance. 378 379 Arguments: 380 total: A number of boards 381 """ 382 self.total = total 383 self.cur = 0 384 width = get_terminal_columns() 385 width = min(width, self.MAX_WIDTH) 386 width -= self.MIN_WIDTH 387 if width > 0: 388 self.enabled = True 389 else: 390 self.enabled = False 391 self.width = width 392 393 def inc(self): 394 """Increment the counter and show the progress bar.""" 395 if not self.enabled: 396 return 397 self.cur += 1 398 arrow_len = self.width * self.cur // self.total 399 msg = '%4d/%d [' % (self.cur, self.total) 400 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 401 sys.stdout.write('\r' + msg) 402 sys.stdout.flush() 403 404def __gen_boards_cfg(jobs): 405 """Generate boards.cfg file. 406 407 Arguments: 408 jobs: The number of jobs to run simultaneously 409 410 Note: 411 The incomplete boards.cfg is left over when an error (including 412 the termination by the keyboard interrupt) occurs on the halfway. 413 """ 414 check_top_directory() 415 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 416 417 # All the defconfig files to be processed 418 defconfigs = [] 419 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 420 dirpath = dirpath[len(CONFIG_DIR) + 1:] 421 for filename in fnmatch.filter(filenames, '*_defconfig'): 422 defconfigs.append(os.path.join(dirpath, filename)) 423 424 # Parse all the MAINTAINERS files 425 maintainers_database = MaintainersDatabase() 426 for (dirpath, dirnames, filenames) in os.walk('.'): 427 if 'MAINTAINERS' in filenames: 428 maintainers_database.parse_file(os.path.join(dirpath, 429 'MAINTAINERS')) 430 431 # Output lines should be piped into the reformat tool 432 reformat_process = subprocess.Popen(REFORMAT_CMD, stdin=subprocess.PIPE, 433 stdout=open(BOARD_FILE, 'w')) 434 pipe = reformat_process.stdin 435 pipe.write(COMMENT_BLOCK) 436 437 indicator = Indicator(len(defconfigs)) 438 slots = Slots(jobs, pipe, maintainers_database) 439 440 # Main loop to process defconfig files: 441 # Add a new subprocess into a vacant slot. 442 # Sleep if there is no available slot. 443 for defconfig in defconfigs: 444 while not slots.add(defconfig): 445 while not slots.available(): 446 # No available slot: sleep for a while 447 time.sleep(SLEEP_TIME) 448 indicator.inc() 449 450 # wait until all the subprocesses finish 451 while not slots.empty(): 452 time.sleep(SLEEP_TIME) 453 print '' 454 455 # wait until the reformat tool finishes 456 reformat_process.communicate() 457 if reformat_process.returncode != 0: 458 print >> sys.stderr, '"%s" failed' % REFORMAT_CMD[0] 459 sys.exit(1) 460 461def gen_boards_cfg(jobs): 462 """Generate boards.cfg file. 463 464 The incomplete boards.cfg is deleted if an error (including 465 the termination by the keyboard interrupt) occurs on the halfway. 466 467 Arguments: 468 jobs: The number of jobs to run simultaneously 469 """ 470 try: 471 __gen_boards_cfg(jobs) 472 except: 473 # We should remove incomplete boards.cfg 474 try: 475 os.remove(BOARD_FILE) 476 except OSError as exception: 477 # Ignore 'No such file or directory' error 478 if exception.errno != errno.ENOENT: 479 raise 480 raise 481 482def main(): 483 parser = optparse.OptionParser() 484 # Add options here 485 parser.add_option('-j', '--jobs', 486 help='the number of jobs to run simultaneously') 487 (options, args) = parser.parse_args() 488 if options.jobs: 489 try: 490 jobs = int(options.jobs) 491 except ValueError: 492 print >> sys.stderr, 'Option -j (--jobs) takes a number' 493 sys.exit(1) 494 else: 495 try: 496 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 497 stdout=subprocess.PIPE).communicate()[0]) 498 except (OSError, ValueError): 499 print 'info: failed to get the number of CPUs. Set jobs to 1' 500 jobs = 1 501 gen_boards_cfg(jobs) 502 503if __name__ == '__main__': 504 main() 505