1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Converter from Kconfig and MAINTAINERS to boards.cfg 10 11Run 'tools/genboardscfg.py' to create boards.cfg file. 12 13Run 'tools/genboardscfg.py -h' for available options. 14 15This script only works on python 2.6 or later, but not python 3.x. 16""" 17 18import errno 19import fnmatch 20import glob 21import optparse 22import os 23import re 24import shutil 25import subprocess 26import sys 27import tempfile 28import time 29 30BOARD_FILE = 'boards.cfg' 31CONFIG_DIR = 'configs' 32REFORMAT_CMD = [os.path.join('tools', 'reformat.py'), 33 '-i', '-d', '-', '-s', '8'] 34SHOW_GNU_MAKE = 'scripts/show-gnu-make' 35SLEEP_TIME=0.003 36 37COMMENT_BLOCK = '''# 38# List of boards 39# Automatically generated by %s: don't edit 40# 41# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 42 43''' % __file__ 44 45### helper functions ### 46def get_terminal_columns(): 47 """Get the width of the terminal. 48 49 Returns: 50 The width of the terminal, or zero if the stdout is not 51 associated with tty. 52 """ 53 try: 54 return shutil.get_terminal_size().columns # Python 3.3~ 55 except AttributeError: 56 import fcntl 57 import termios 58 import struct 59 arg = struct.pack('hhhh', 0, 0, 0, 0) 60 try: 61 ret = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, arg) 62 except IOError as exception: 63 # If 'Inappropriate ioctl for device' error occurs, 64 # stdout is probably redirected. Return 0. 65 return 0 66 return struct.unpack('hhhh', ret)[1] 67 68def get_devnull(): 69 """Get the file object of '/dev/null' device.""" 70 try: 71 devnull = subprocess.DEVNULL # py3k 72 except AttributeError: 73 devnull = open(os.devnull, 'wb') 74 return devnull 75 76def check_top_directory(): 77 """Exit if we are not at the top of source directory.""" 78 for f in ('README', 'Licenses'): 79 if not os.path.exists(f): 80 sys.exit('Please run at the top of source directory.') 81 82def get_make_cmd(): 83 """Get the command name of GNU Make.""" 84 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 85 ret = process.communicate() 86 if process.returncode: 87 sys.exit('GNU Make not found') 88 return ret[0].rstrip() 89 90def output_is_new(): 91 """Check if the boards.cfg file is up to date. 92 93 Returns: 94 True if the boards.cfg file exists and is newer than any of 95 *_defconfig, MAINTAINERS and Kconfig*. False otherwise. 96 """ 97 try: 98 ctime = os.path.getctime(BOARD_FILE) 99 except OSError as exception: 100 if exception.errno == errno.ENOENT: 101 # return False on 'No such file or directory' error 102 return False 103 else: 104 raise 105 106 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 107 for filename in fnmatch.filter(filenames, '*_defconfig'): 108 if fnmatch.fnmatch(filename, '.*'): 109 continue 110 filepath = os.path.join(dirpath, filename) 111 if ctime < os.path.getctime(filepath): 112 return False 113 114 for (dirpath, dirnames, filenames) in os.walk('.'): 115 for filename in filenames: 116 if (fnmatch.fnmatch(filename, '*~') or 117 not fnmatch.fnmatch(filename, 'Kconfig*') and 118 not filename == 'MAINTAINERS'): 119 continue 120 filepath = os.path.join(dirpath, filename) 121 if ctime < os.path.getctime(filepath): 122 return False 123 124 # Detect a board that has been removed since the current boards.cfg 125 # was generated 126 with open(BOARD_FILE) as f: 127 for line in f: 128 if line[0] == '#' or line == '\n': 129 continue 130 defconfig = line.split()[6] + '_defconfig' 131 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): 132 return False 133 134 return True 135 136### classes ### 137class MaintainersDatabase: 138 139 """The database of board status and maintainers.""" 140 141 def __init__(self): 142 """Create an empty database.""" 143 self.database = {} 144 145 def get_status(self, target): 146 """Return the status of the given board. 147 148 Returns: 149 Either 'Active' or 'Orphan' 150 """ 151 if not target in self.database: 152 print >> sys.stderr, "WARNING: no status info for '%s'" % target 153 return '-' 154 155 tmp = self.database[target][0] 156 if tmp.startswith('Maintained'): 157 return 'Active' 158 elif tmp.startswith('Orphan'): 159 return 'Orphan' 160 else: 161 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" % 162 (tmp, target)) 163 return '-' 164 165 def get_maintainers(self, target): 166 """Return the maintainers of the given board. 167 168 If the board has two or more maintainers, they are separated 169 with colons. 170 """ 171 if not target in self.database: 172 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target 173 return '' 174 175 return ':'.join(self.database[target][1]) 176 177 def parse_file(self, file): 178 """Parse the given MAINTAINERS file. 179 180 This method parses MAINTAINERS and add board status and 181 maintainers information to the database. 182 183 Arguments: 184 file: MAINTAINERS file to be parsed 185 """ 186 targets = [] 187 maintainers = [] 188 status = '-' 189 for line in open(file): 190 tag, rest = line[:2], line[2:].strip() 191 if tag == 'M:': 192 maintainers.append(rest) 193 elif tag == 'F:': 194 # expand wildcard and filter by 'configs/*_defconfig' 195 for f in glob.glob(rest): 196 front, match, rear = f.partition('configs/') 197 if not front and match: 198 front, match, rear = rear.rpartition('_defconfig') 199 if match and not rear: 200 targets.append(front) 201 elif tag == 'S:': 202 status = rest 203 elif line == '\n': 204 for target in targets: 205 self.database[target] = (status, maintainers) 206 targets = [] 207 maintainers = [] 208 status = '-' 209 if targets: 210 for target in targets: 211 self.database[target] = (status, maintainers) 212 213class DotConfigParser: 214 215 """A parser of .config file. 216 217 Each line of the output should have the form of: 218 Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers 219 Most of them are extracted from .config file. 220 MAINTAINERS files are also consulted for Status and Maintainers fields. 221 """ 222 223 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 224 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 225 re_soc = re.compile(r'CONFIG_SYS_SOC="(.*)"') 226 re_vendor = re.compile(r'CONFIG_SYS_VENDOR="(.*)"') 227 re_board = re.compile(r'CONFIG_SYS_BOARD="(.*)"') 228 re_config = re.compile(r'CONFIG_SYS_CONFIG_NAME="(.*)"') 229 re_options = re.compile(r'CONFIG_SYS_EXTRA_OPTIONS="(.*)"') 230 re_list = (('arch', re_arch), ('cpu', re_cpu), ('soc', re_soc), 231 ('vendor', re_vendor), ('board', re_board), 232 ('config', re_config), ('options', re_options)) 233 must_fields = ('arch', 'config') 234 235 def __init__(self, build_dir, output, maintainers_database): 236 """Create a new .config perser. 237 238 Arguments: 239 build_dir: Build directory where .config is located 240 output: File object which the result is written to 241 maintainers_database: An instance of class MaintainersDatabase 242 """ 243 self.dotconfig = os.path.join(build_dir, '.config') 244 self.output = output 245 self.database = maintainers_database 246 247 def parse(self, defconfig): 248 """Parse .config file and output one-line database for the given board. 249 250 Arguments: 251 defconfig: Board (defconfig) name 252 """ 253 fields = {} 254 for line in open(self.dotconfig): 255 if not line.startswith('CONFIG_SYS_'): 256 continue 257 for (key, pattern) in self.re_list: 258 m = pattern.match(line) 259 if m and m.group(1): 260 fields[key] = m.group(1) 261 break 262 263 # sanity check of '.config' file 264 for field in self.must_fields: 265 if not field in fields: 266 print >> sys.stderr, ( 267 "WARNING: '%s' is not defined in '%s'. Skip." % 268 (field, defconfig)) 269 return 270 271 # fix-up for aarch64 272 if fields['arch'] == 'arm' and 'cpu' in fields: 273 if fields['cpu'] == 'armv8': 274 fields['arch'] = 'aarch64' 275 276 target, match, rear = defconfig.partition('_defconfig') 277 assert match and not rear, \ 278 '%s : invalid defconfig file name' % defconfig 279 280 fields['status'] = self.database.get_status(target) 281 fields['maintainers'] = self.database.get_maintainers(target) 282 283 if 'options' in fields: 284 options = fields['config'] + ':' + \ 285 fields['options'].replace(r'\"', '"') 286 elif fields['config'] != target: 287 options = fields['config'] 288 else: 289 options = '-' 290 291 self.output.write((' '.join(['%s'] * 9) + '\n') % 292 (fields['status'], 293 fields['arch'], 294 fields.get('cpu', '-'), 295 fields.get('soc', '-'), 296 fields.get('vendor', '-'), 297 fields.get('board', '-'), 298 target, 299 options, 300 fields['maintainers'])) 301 302class Slot: 303 304 """A slot to store a subprocess. 305 306 Each instance of this class handles one subprocess. 307 This class is useful to control multiple processes 308 for faster processing. 309 """ 310 311 def __init__(self, output, maintainers_database, devnull, make_cmd): 312 """Create a new slot. 313 314 Arguments: 315 output: File object which the result is written to 316 maintainers_database: An instance of class MaintainersDatabase 317 devnull: file object of 'dev/null' 318 make_cmd: the command name of Make 319 """ 320 self.build_dir = tempfile.mkdtemp() 321 self.devnull = devnull 322 self.ps = subprocess.Popen([make_cmd, 'O=' + self.build_dir, 323 'allnoconfig'], stdout=devnull) 324 self.occupied = True 325 self.parser = DotConfigParser(self.build_dir, output, 326 maintainers_database) 327 self.env = os.environ.copy() 328 self.env['srctree'] = os.getcwd() 329 self.env['UBOOTVERSION'] = 'dummy' 330 self.env['KCONFIG_OBJDIR'] = '' 331 332 def __del__(self): 333 """Delete the working directory""" 334 if not self.occupied: 335 while self.ps.poll() == None: 336 pass 337 shutil.rmtree(self.build_dir) 338 339 def add(self, defconfig): 340 """Add a new subprocess to the slot. 341 342 Fails if the slot is occupied, that is, the current subprocess 343 is still running. 344 345 Arguments: 346 defconfig: Board (defconfig) name 347 348 Returns: 349 Return True on success or False on fail 350 """ 351 if self.occupied: 352 return False 353 354 with open(os.path.join(self.build_dir, '.tmp_defconfig'), 'w') as f: 355 for line in open(os.path.join(CONFIG_DIR, defconfig)): 356 colon = line.find(':CONFIG_') 357 if colon == -1: 358 f.write(line) 359 else: 360 f.write(line[colon + 1:]) 361 362 self.ps = subprocess.Popen([os.path.join('scripts', 'kconfig', 'conf'), 363 '--defconfig=.tmp_defconfig', 'Kconfig'], 364 stdout=self.devnull, 365 cwd=self.build_dir, 366 env=self.env) 367 368 self.defconfig = defconfig 369 self.occupied = True 370 return True 371 372 def wait(self): 373 """Wait until the current subprocess finishes.""" 374 while self.occupied and self.ps.poll() == None: 375 time.sleep(SLEEP_TIME) 376 self.occupied = False 377 378 def poll(self): 379 """Check if the subprocess is running and invoke the .config 380 parser if the subprocess is terminated. 381 382 Returns: 383 Return True if the subprocess is terminated, False otherwise 384 """ 385 if not self.occupied: 386 return True 387 if self.ps.poll() == None: 388 return False 389 if self.ps.poll() == 0: 390 self.parser.parse(self.defconfig) 391 else: 392 print >> sys.stderr, ("WARNING: failed to process '%s'. skip." % 393 self.defconfig) 394 self.occupied = False 395 return True 396 397class Slots: 398 399 """Controller of the array of subprocess slots.""" 400 401 def __init__(self, jobs, output, maintainers_database): 402 """Create a new slots controller. 403 404 Arguments: 405 jobs: A number of slots to instantiate 406 output: File object which the result is written to 407 maintainers_database: An instance of class MaintainersDatabase 408 """ 409 self.slots = [] 410 devnull = get_devnull() 411 make_cmd = get_make_cmd() 412 for i in range(jobs): 413 self.slots.append(Slot(output, maintainers_database, 414 devnull, make_cmd)) 415 for slot in self.slots: 416 slot.wait() 417 418 def add(self, defconfig): 419 """Add a new subprocess if a vacant slot is available. 420 421 Arguments: 422 defconfig: Board (defconfig) name 423 424 Returns: 425 Return True on success or False on fail 426 """ 427 for slot in self.slots: 428 if slot.add(defconfig): 429 return True 430 return False 431 432 def available(self): 433 """Check if there is a vacant slot. 434 435 Returns: 436 Return True if a vacant slot is found, False if all slots are full 437 """ 438 for slot in self.slots: 439 if slot.poll(): 440 return True 441 return False 442 443 def empty(self): 444 """Check if all slots are vacant. 445 446 Returns: 447 Return True if all slots are vacant, False if at least one slot 448 is running 449 """ 450 ret = True 451 for slot in self.slots: 452 if not slot.poll(): 453 ret = False 454 return ret 455 456class Indicator: 457 458 """A class to control the progress indicator.""" 459 460 MIN_WIDTH = 15 461 MAX_WIDTH = 70 462 463 def __init__(self, total): 464 """Create an instance. 465 466 Arguments: 467 total: A number of boards 468 """ 469 self.total = total 470 self.cur = 0 471 width = get_terminal_columns() 472 width = min(width, self.MAX_WIDTH) 473 width -= self.MIN_WIDTH 474 if width > 0: 475 self.enabled = True 476 else: 477 self.enabled = False 478 self.width = width 479 480 def inc(self): 481 """Increment the counter and show the progress bar.""" 482 if not self.enabled: 483 return 484 self.cur += 1 485 arrow_len = self.width * self.cur // self.total 486 msg = '%4d/%d [' % (self.cur, self.total) 487 msg += '=' * arrow_len + '>' + ' ' * (self.width - arrow_len) + ']' 488 sys.stdout.write('\r' + msg) 489 sys.stdout.flush() 490 491class BoardsFileGenerator: 492 493 """Generator of boards.cfg.""" 494 495 def __init__(self): 496 """Prepare basic things for generating boards.cfg.""" 497 # All the defconfig files to be processed 498 defconfigs = [] 499 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): 500 dirpath = dirpath[len(CONFIG_DIR) + 1:] 501 for filename in fnmatch.filter(filenames, '*_defconfig'): 502 if fnmatch.fnmatch(filename, '.*'): 503 continue 504 defconfigs.append(os.path.join(dirpath, filename)) 505 self.defconfigs = defconfigs 506 self.indicator = Indicator(len(defconfigs)) 507 508 # Parse all the MAINTAINERS files 509 maintainers_database = MaintainersDatabase() 510 for (dirpath, dirnames, filenames) in os.walk('.'): 511 if 'MAINTAINERS' in filenames: 512 maintainers_database.parse_file(os.path.join(dirpath, 513 'MAINTAINERS')) 514 self.maintainers_database = maintainers_database 515 516 def __del__(self): 517 """Delete the incomplete boards.cfg 518 519 This destructor deletes boards.cfg if the private member 'in_progress' 520 is defined as True. The 'in_progress' member is set to True at the 521 beginning of the generate() method and set to False at its end. 522 So, in_progress==True means generating boards.cfg was terminated 523 on the way. 524 """ 525 526 if hasattr(self, 'in_progress') and self.in_progress: 527 try: 528 os.remove(BOARD_FILE) 529 except OSError as exception: 530 # Ignore 'No such file or directory' error 531 if exception.errno != errno.ENOENT: 532 raise 533 print 'Removed incomplete %s' % BOARD_FILE 534 535 def generate(self, jobs): 536 """Generate boards.cfg 537 538 This method sets the 'in_progress' member to True at the beginning 539 and sets it to False on success. The boards.cfg should not be 540 touched before/after this method because 'in_progress' is used 541 to detect the incomplete boards.cfg. 542 543 Arguments: 544 jobs: The number of jobs to run simultaneously 545 """ 546 547 self.in_progress = True 548 print 'Generating %s ... (jobs: %d)' % (BOARD_FILE, jobs) 549 550 # Output lines should be piped into the reformat tool 551 reformat_process = subprocess.Popen(REFORMAT_CMD, 552 stdin=subprocess.PIPE, 553 stdout=open(BOARD_FILE, 'w')) 554 pipe = reformat_process.stdin 555 pipe.write(COMMENT_BLOCK) 556 557 slots = Slots(jobs, pipe, self.maintainers_database) 558 559 # Main loop to process defconfig files: 560 # Add a new subprocess into a vacant slot. 561 # Sleep if there is no available slot. 562 for defconfig in self.defconfigs: 563 while not slots.add(defconfig): 564 while not slots.available(): 565 # No available slot: sleep for a while 566 time.sleep(SLEEP_TIME) 567 self.indicator.inc() 568 569 # wait until all the subprocesses finish 570 while not slots.empty(): 571 time.sleep(SLEEP_TIME) 572 print '' 573 574 # wait until the reformat tool finishes 575 reformat_process.communicate() 576 if reformat_process.returncode != 0: 577 sys.exit('"%s" failed' % REFORMAT_CMD[0]) 578 579 self.in_progress = False 580 581def gen_boards_cfg(jobs=1, force=False): 582 """Generate boards.cfg file. 583 584 The incomplete boards.cfg is deleted if an error (including 585 the termination by the keyboard interrupt) occurs on the halfway. 586 587 Arguments: 588 jobs: The number of jobs to run simultaneously 589 """ 590 check_top_directory() 591 if not force and output_is_new(): 592 print "%s is up to date. Nothing to do." % BOARD_FILE 593 sys.exit(0) 594 595 generator = BoardsFileGenerator() 596 generator.generate(jobs) 597 598def main(): 599 parser = optparse.OptionParser() 600 # Add options here 601 parser.add_option('-j', '--jobs', 602 help='the number of jobs to run simultaneously') 603 parser.add_option('-f', '--force', action="store_true", default=False, 604 help='regenerate the output even if it is new') 605 (options, args) = parser.parse_args() 606 607 if options.jobs: 608 try: 609 jobs = int(options.jobs) 610 except ValueError: 611 sys.exit('Option -j (--jobs) takes a number') 612 else: 613 try: 614 jobs = int(subprocess.Popen(['getconf', '_NPROCESSORS_ONLN'], 615 stdout=subprocess.PIPE).communicate()[0]) 616 except (OSError, ValueError): 617 print 'info: failed to get the number of CPUs. Set jobs to 1' 618 jobs = 1 619 620 gen_boards_cfg(jobs, force=options.force) 621 622if __name__ == '__main__': 623 main() 624