1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. 111 112The default per-arch CROSS_COMPILE used by this tool is specified by 113the list below, CROSS_COMPILE. You may wish to update the list to 114use your own. Instead of modifying the list directly, you can give 115them via environments. 116 117 118Available options 119----------------- 120 121 -c, --color 122 Surround each portion of the log with escape sequences to display it 123 in color on the terminal. 124 125 -d, --defconfigs 126 Specify a file containing a list of defconfigs to move 127 128 -n, --dry-run 129 Perform a trial run that does not make any changes. It is useful to 130 see what is going to happen before one actually runs it. 131 132 -e, --exit-on-error 133 Exit immediately if Make exits with a non-zero status while processing 134 a defconfig file. 135 136 -s, --force-sync 137 Do "make savedefconfig" forcibly for all the defconfig files. 138 If not specified, "make savedefconfig" only occurs for cases 139 where at least one CONFIG was moved. 140 141 -S, --spl 142 Look for moved config options in spl/include/autoconf.mk instead of 143 include/autoconf.mk. This is useful for moving options for SPL build 144 because SPL related options (mostly prefixed with CONFIG_SPL_) are 145 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 146 147 -H, --headers-only 148 Only cleanup the headers; skip the defconfig processing 149 150 -j, --jobs 151 Specify the number of threads to run simultaneously. If not specified, 152 the number of threads is the same as the number of CPU cores. 153 154 -r, --git-ref 155 Specify the git ref to clone for building the autoconf.mk. If unspecified 156 use the CWD. This is useful for when changes to the Kconfig affect the 157 default values and you want to capture the state of the defconfig from 158 before that change was in effect. If in doubt, specify a ref pre-Kconfig 159 changes (use HEAD if Kconfig changes are not committed). Worst case it will 160 take a bit longer to run, but will always do the right thing. 161 162 -v, --verbose 163 Show any build errors as boards are built 164 165To see the complete list of supported options, run 166 167 $ tools/moveconfig.py -h 168 169""" 170 171import copy 172import difflib 173import filecmp 174import fnmatch 175import multiprocessing 176import optparse 177import os 178import re 179import shutil 180import subprocess 181import sys 182import tempfile 183import time 184 185SHOW_GNU_MAKE = 'scripts/show-gnu-make' 186SLEEP_TIME=0.03 187 188# Here is the list of cross-tools I use. 189# Most of them are available at kernel.org 190# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following: 191# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 192# blackfin: http://sourceforge.net/projects/adi-toolchain/files/ 193# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 194# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 195# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 196# 197# openrisc kernel.org toolchain is out of date, download latest one from 198# http://opencores.org/or1k/OpenRISC_GNU_tool_chain#Prebuilt_versions 199CROSS_COMPILE = { 200 'arc': 'arc-linux-', 201 'aarch64': 'aarch64-linux-', 202 'arm': 'arm-unknown-linux-gnueabi-', 203 'avr32': 'avr32-linux-', 204 'blackfin': 'bfin-elf-', 205 'm68k': 'm68k-linux-', 206 'microblaze': 'microblaze-linux-', 207 'mips': 'mips-linux-', 208 'nds32': 'nds32le-linux-', 209 'nios2': 'nios2-linux-gnu-', 210 'openrisc': 'or1k-elf-', 211 'powerpc': 'powerpc-linux-', 212 'sh': 'sh-linux-gnu-', 213 'sparc': 'sparc-linux-', 214 'x86': 'i386-linux-', 215 'xtensa': 'xtensa-linux-' 216} 217 218STATE_IDLE = 0 219STATE_DEFCONFIG = 1 220STATE_AUTOCONF = 2 221STATE_SAVEDEFCONFIG = 3 222 223ACTION_MOVE = 0 224ACTION_NO_ENTRY = 1 225ACTION_NO_ENTRY_WARN = 2 226ACTION_NO_CHANGE = 3 227 228COLOR_BLACK = '0;30' 229COLOR_RED = '0;31' 230COLOR_GREEN = '0;32' 231COLOR_BROWN = '0;33' 232COLOR_BLUE = '0;34' 233COLOR_PURPLE = '0;35' 234COLOR_CYAN = '0;36' 235COLOR_LIGHT_GRAY = '0;37' 236COLOR_DARK_GRAY = '1;30' 237COLOR_LIGHT_RED = '1;31' 238COLOR_LIGHT_GREEN = '1;32' 239COLOR_YELLOW = '1;33' 240COLOR_LIGHT_BLUE = '1;34' 241COLOR_LIGHT_PURPLE = '1;35' 242COLOR_LIGHT_CYAN = '1;36' 243COLOR_WHITE = '1;37' 244 245### helper functions ### 246def get_devnull(): 247 """Get the file object of '/dev/null' device.""" 248 try: 249 devnull = subprocess.DEVNULL # py3k 250 except AttributeError: 251 devnull = open(os.devnull, 'wb') 252 return devnull 253 254def check_top_directory(): 255 """Exit if we are not at the top of source directory.""" 256 for f in ('README', 'Licenses'): 257 if not os.path.exists(f): 258 sys.exit('Please run at the top of source directory.') 259 260def check_clean_directory(): 261 """Exit if the source tree is not clean.""" 262 for f in ('.config', 'include/config'): 263 if os.path.exists(f): 264 sys.exit("source tree is not clean, please run 'make mrproper'") 265 266def get_make_cmd(): 267 """Get the command name of GNU Make. 268 269 U-Boot needs GNU Make for building, but the command name is not 270 necessarily "make". (for example, "gmake" on FreeBSD). 271 Returns the most appropriate command name on your system. 272 """ 273 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 274 ret = process.communicate() 275 if process.returncode: 276 sys.exit('GNU Make not found') 277 return ret[0].rstrip() 278 279def get_all_defconfigs(): 280 """Get all the defconfig files under the configs/ directory.""" 281 defconfigs = [] 282 for (dirpath, dirnames, filenames) in os.walk('configs'): 283 dirpath = dirpath[len('configs') + 1:] 284 for filename in fnmatch.filter(filenames, '*_defconfig'): 285 defconfigs.append(os.path.join(dirpath, filename)) 286 287 return defconfigs 288 289def color_text(color_enabled, color, string): 290 """Return colored string.""" 291 if color_enabled: 292 # LF should not be surrounded by the escape sequence. 293 # Otherwise, additional whitespace or line-feed might be printed. 294 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 295 for s in string.split('\n') ]) 296 else: 297 return string 298 299def show_diff(a, b, file_path, color_enabled): 300 """Show unidified diff. 301 302 Arguments: 303 a: A list of lines (before) 304 b: A list of lines (after) 305 file_path: Path to the file 306 color_enabled: Display the diff in color 307 """ 308 309 diff = difflib.unified_diff(a, b, 310 fromfile=os.path.join('a', file_path), 311 tofile=os.path.join('b', file_path)) 312 313 for line in diff: 314 if line[0] == '-' and line[1] != '-': 315 print color_text(color_enabled, COLOR_RED, line), 316 elif line[0] == '+' and line[1] != '+': 317 print color_text(color_enabled, COLOR_GREEN, line), 318 else: 319 print line, 320 321def update_cross_compile(color_enabled): 322 """Update per-arch CROSS_COMPILE via environment variables 323 324 The default CROSS_COMPILE values are available 325 in the CROSS_COMPILE list above. 326 327 You can override them via environment variables 328 CROSS_COMPILE_{ARCH}. 329 330 For example, if you want to override toolchain prefixes 331 for ARM and PowerPC, you can do as follows in your shell: 332 333 export CROSS_COMPILE_ARM=... 334 export CROSS_COMPILE_POWERPC=... 335 336 Then, this function checks if specified compilers really exist in your 337 PATH environment. 338 """ 339 archs = [] 340 341 for arch in os.listdir('arch'): 342 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 343 archs.append(arch) 344 345 # arm64 is a special case 346 archs.append('aarch64') 347 348 for arch in archs: 349 env = 'CROSS_COMPILE_' + arch.upper() 350 cross_compile = os.environ.get(env) 351 if not cross_compile: 352 cross_compile = CROSS_COMPILE.get(arch, '') 353 354 for path in os.environ["PATH"].split(os.pathsep): 355 gcc_path = os.path.join(path, cross_compile + 'gcc') 356 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 357 break 358 else: 359 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 360 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 361 % (cross_compile, arch)) 362 cross_compile = None 363 364 CROSS_COMPILE[arch] = cross_compile 365 366def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 367 extend_post): 368 """Extend matched lines if desired patterns are found before/after already 369 matched lines. 370 371 Arguments: 372 lines: A list of lines handled. 373 matched: A list of line numbers that have been already matched. 374 (will be updated by this function) 375 pre_patterns: A list of regular expression that should be matched as 376 preamble. 377 post_patterns: A list of regular expression that should be matched as 378 postamble. 379 extend_pre: Add the line number of matched preamble to the matched list. 380 extend_post: Add the line number of matched postamble to the matched list. 381 """ 382 extended_matched = [] 383 384 j = matched[0] 385 386 for i in matched: 387 if i == 0 or i < j: 388 continue 389 j = i 390 while j in matched: 391 j += 1 392 if j >= len(lines): 393 break 394 395 for p in pre_patterns: 396 if p.search(lines[i - 1]): 397 break 398 else: 399 # not matched 400 continue 401 402 for p in post_patterns: 403 if p.search(lines[j]): 404 break 405 else: 406 # not matched 407 continue 408 409 if extend_pre: 410 extended_matched.append(i - 1) 411 if extend_post: 412 extended_matched.append(j) 413 414 matched += extended_matched 415 matched.sort() 416 417def cleanup_one_header(header_path, patterns, options): 418 """Clean regex-matched lines away from a file. 419 420 Arguments: 421 header_path: path to the cleaned file. 422 patterns: list of regex patterns. Any lines matching to these 423 patterns are deleted. 424 options: option flags. 425 """ 426 with open(header_path) as f: 427 lines = f.readlines() 428 429 matched = [] 430 for i, line in enumerate(lines): 431 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 432 matched.append(i) 433 continue 434 for pattern in patterns: 435 if pattern.search(line): 436 matched.append(i) 437 break 438 439 if not matched: 440 return 441 442 # remove empty #ifdef ... #endif, successive blank lines 443 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 444 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 445 pattern_endif = re.compile(r'#\s*endif\W') # #endif 446 pattern_blank = re.compile(r'^\s*$') # empty line 447 448 while True: 449 old_matched = copy.copy(matched) 450 extend_matched_lines(lines, matched, [pattern_if], 451 [pattern_endif], True, True) 452 extend_matched_lines(lines, matched, [pattern_elif], 453 [pattern_elif, pattern_endif], True, False) 454 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 455 [pattern_blank], False, True) 456 extend_matched_lines(lines, matched, [pattern_blank], 457 [pattern_elif, pattern_endif], True, False) 458 extend_matched_lines(lines, matched, [pattern_blank], 459 [pattern_blank], True, False) 460 if matched == old_matched: 461 break 462 463 tolines = copy.copy(lines) 464 465 for i in reversed(matched): 466 tolines.pop(i) 467 468 show_diff(lines, tolines, header_path, options.color) 469 470 if options.dry_run: 471 return 472 473 with open(header_path, 'w') as f: 474 for line in tolines: 475 f.write(line) 476 477def cleanup_headers(configs, options): 478 """Delete config defines from board headers. 479 480 Arguments: 481 configs: A list of CONFIGs to remove. 482 options: option flags. 483 """ 484 while True: 485 choice = raw_input('Clean up headers? [y/n]: ').lower() 486 print choice 487 if choice == 'y' or choice == 'n': 488 break 489 490 if choice == 'n': 491 return 492 493 patterns = [] 494 for config in configs: 495 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 496 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 497 498 for dir in 'include', 'arch', 'board': 499 for (dirpath, dirnames, filenames) in os.walk(dir): 500 if dirpath == os.path.join('include', 'generated'): 501 continue 502 for filename in filenames: 503 if not fnmatch.fnmatch(filename, '*~'): 504 cleanup_one_header(os.path.join(dirpath, filename), 505 patterns, options) 506 507def cleanup_one_extra_option(defconfig_path, configs, options): 508 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 509 510 Arguments: 511 defconfig_path: path to the cleaned defconfig file. 512 configs: A list of CONFIGs to remove. 513 options: option flags. 514 """ 515 516 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 517 end = '"\n' 518 519 with open(defconfig_path) as f: 520 lines = f.readlines() 521 522 for i, line in enumerate(lines): 523 if line.startswith(start) and line.endswith(end): 524 break 525 else: 526 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 527 return 528 529 old_tokens = line[len(start):-len(end)].split(',') 530 new_tokens = [] 531 532 for token in old_tokens: 533 pos = token.find('=') 534 if not (token[:pos] if pos >= 0 else token) in configs: 535 new_tokens.append(token) 536 537 if new_tokens == old_tokens: 538 return 539 540 tolines = copy.copy(lines) 541 542 if new_tokens: 543 tolines[i] = start + ','.join(new_tokens) + end 544 else: 545 tolines.pop(i) 546 547 show_diff(lines, tolines, defconfig_path, options.color) 548 549 if options.dry_run: 550 return 551 552 with open(defconfig_path, 'w') as f: 553 for line in tolines: 554 f.write(line) 555 556def cleanup_extra_options(configs, options): 557 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 558 559 Arguments: 560 configs: A list of CONFIGs to remove. 561 options: option flags. 562 """ 563 while True: 564 choice = raw_input('Clean up CONFIG_SYS_EXTRA_OPTIONS? [y/n]: ').lower() 565 print choice 566 if choice == 'y' or choice == 'n': 567 break 568 569 if choice == 'n': 570 return 571 572 configs = [ config[len('CONFIG_'):] for config in configs ] 573 574 defconfigs = get_all_defconfigs() 575 576 for defconfig in defconfigs: 577 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 578 options) 579 580### classes ### 581class Progress: 582 583 """Progress Indicator""" 584 585 def __init__(self, total): 586 """Create a new progress indicator. 587 588 Arguments: 589 total: A number of defconfig files to process. 590 """ 591 self.current = 0 592 self.total = total 593 594 def inc(self): 595 """Increment the number of processed defconfig files.""" 596 597 self.current += 1 598 599 def show(self): 600 """Display the progress.""" 601 print ' %d defconfigs out of %d\r' % (self.current, self.total), 602 sys.stdout.flush() 603 604class KconfigParser: 605 606 """A parser of .config and include/autoconf.mk.""" 607 608 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 609 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 610 611 def __init__(self, configs, options, build_dir): 612 """Create a new parser. 613 614 Arguments: 615 configs: A list of CONFIGs to move. 616 options: option flags. 617 build_dir: Build directory. 618 """ 619 self.configs = configs 620 self.options = options 621 self.dotconfig = os.path.join(build_dir, '.config') 622 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 623 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 624 'autoconf.mk') 625 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 626 'auto.conf') 627 self.defconfig = os.path.join(build_dir, 'defconfig') 628 629 def get_cross_compile(self): 630 """Parse .config file and return CROSS_COMPILE. 631 632 Returns: 633 A string storing the compiler prefix for the architecture. 634 Return a NULL string for architectures that do not require 635 compiler prefix (Sandbox and native build is the case). 636 Return None if the specified compiler is missing in your PATH. 637 Caller should distinguish '' and None. 638 """ 639 arch = '' 640 cpu = '' 641 for line in open(self.dotconfig): 642 m = self.re_arch.match(line) 643 if m: 644 arch = m.group(1) 645 continue 646 m = self.re_cpu.match(line) 647 if m: 648 cpu = m.group(1) 649 650 if not arch: 651 return None 652 653 # fix-up for aarch64 654 if arch == 'arm' and cpu == 'armv8': 655 arch = 'aarch64' 656 657 return CROSS_COMPILE.get(arch, None) 658 659 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 660 """Parse .config, defconfig, include/autoconf.mk for one config. 661 662 This function looks for the config options in the lines from 663 defconfig, .config, and include/autoconf.mk in order to decide 664 which action should be taken for this defconfig. 665 666 Arguments: 667 config: CONFIG name to parse. 668 dotconfig_lines: lines from the .config file. 669 autoconf_lines: lines from the include/autoconf.mk file. 670 671 Returns: 672 A tupple of the action for this defconfig and the line 673 matched for the config. 674 """ 675 not_set = '# %s is not set' % config 676 677 for line in autoconf_lines: 678 line = line.rstrip() 679 if line.startswith(config + '='): 680 new_val = line 681 break 682 else: 683 new_val = not_set 684 685 for line in dotconfig_lines: 686 line = line.rstrip() 687 if line.startswith(config + '=') or line == not_set: 688 old_val = line 689 break 690 else: 691 if new_val == not_set: 692 return (ACTION_NO_ENTRY, config) 693 else: 694 return (ACTION_NO_ENTRY_WARN, config) 695 696 # If this CONFIG is neither bool nor trisate 697 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 698 # tools/scripts/define2mk.sed changes '1' to 'y'. 699 # This is a problem if the CONFIG is int type. 700 # Check the type in Kconfig and handle it correctly. 701 if new_val[-2:] == '=y': 702 new_val = new_val[:-1] + '1' 703 704 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 705 new_val) 706 707 def update_dotconfig(self): 708 """Parse files for the config options and update the .config. 709 710 This function parses the generated .config and include/autoconf.mk 711 searching the target options. 712 Move the config option(s) to the .config as needed. 713 714 Arguments: 715 defconfig: defconfig name. 716 717 Returns: 718 Return a tuple of (updated flag, log string). 719 The "updated flag" is True if the .config was updated, False 720 otherwise. The "log string" shows what happend to the .config. 721 """ 722 723 results = [] 724 updated = False 725 suspicious = False 726 rm_files = [self.config_autoconf, self.autoconf] 727 728 if self.options.spl: 729 if os.path.exists(self.spl_autoconf): 730 autoconf_path = self.spl_autoconf 731 rm_files.append(self.spl_autoconf) 732 else: 733 for f in rm_files: 734 os.remove(f) 735 return (updated, suspicious, 736 color_text(self.options.color, COLOR_BROWN, 737 "SPL is not enabled. Skipped.") + '\n') 738 else: 739 autoconf_path = self.autoconf 740 741 with open(self.dotconfig) as f: 742 dotconfig_lines = f.readlines() 743 744 with open(autoconf_path) as f: 745 autoconf_lines = f.readlines() 746 747 for config in self.configs: 748 result = self.parse_one_config(config, dotconfig_lines, 749 autoconf_lines) 750 results.append(result) 751 752 log = '' 753 754 for (action, value) in results: 755 if action == ACTION_MOVE: 756 actlog = "Move '%s'" % value 757 log_color = COLOR_LIGHT_GREEN 758 elif action == ACTION_NO_ENTRY: 759 actlog = "%s is not defined in Kconfig. Do nothing." % value 760 log_color = COLOR_LIGHT_BLUE 761 elif action == ACTION_NO_ENTRY_WARN: 762 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 763 log_color = COLOR_YELLOW 764 suspicious = True 765 elif action == ACTION_NO_CHANGE: 766 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 767 % value 768 log_color = COLOR_LIGHT_PURPLE 769 elif action == ACTION_SPL_NOT_EXIST: 770 actlog = "SPL is not enabled for this defconfig. Skip." 771 log_color = COLOR_PURPLE 772 else: 773 sys.exit("Internal Error. This should not happen.") 774 775 log += color_text(self.options.color, log_color, actlog) + '\n' 776 777 with open(self.dotconfig, 'a') as f: 778 for (action, value) in results: 779 if action == ACTION_MOVE: 780 f.write(value + '\n') 781 updated = True 782 783 self.results = results 784 for f in rm_files: 785 os.remove(f) 786 787 return (updated, suspicious, log) 788 789 def check_defconfig(self): 790 """Check the defconfig after savedefconfig 791 792 Returns: 793 Return additional log if moved CONFIGs were removed again by 794 'make savedefconfig'. 795 """ 796 797 log = '' 798 799 with open(self.defconfig) as f: 800 defconfig_lines = f.readlines() 801 802 for (action, value) in self.results: 803 if action != ACTION_MOVE: 804 continue 805 if not value + '\n' in defconfig_lines: 806 log += color_text(self.options.color, COLOR_YELLOW, 807 "'%s' was removed by savedefconfig.\n" % 808 value) 809 810 return log 811 812class Slot: 813 814 """A slot to store a subprocess. 815 816 Each instance of this class handles one subprocess. 817 This class is useful to control multiple threads 818 for faster processing. 819 """ 820 821 def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): 822 """Create a new process slot. 823 824 Arguments: 825 configs: A list of CONFIGs to move. 826 options: option flags. 827 progress: A progress indicator. 828 devnull: A file object of '/dev/null'. 829 make_cmd: command name of GNU Make. 830 reference_src_dir: Determine the true starting config state from this 831 source tree. 832 """ 833 self.options = options 834 self.progress = progress 835 self.build_dir = tempfile.mkdtemp() 836 self.devnull = devnull 837 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 838 self.reference_src_dir = reference_src_dir 839 self.parser = KconfigParser(configs, options, self.build_dir) 840 self.state = STATE_IDLE 841 self.failed_boards = set() 842 self.suspicious_boards = set() 843 844 def __del__(self): 845 """Delete the working directory 846 847 This function makes sure the temporary directory is cleaned away 848 even if Python suddenly dies due to error. It should be done in here 849 because it is guaranteed the destructor is always invoked when the 850 instance of the class gets unreferenced. 851 852 If the subprocess is still running, wait until it finishes. 853 """ 854 if self.state != STATE_IDLE: 855 while self.ps.poll() == None: 856 pass 857 shutil.rmtree(self.build_dir) 858 859 def add(self, defconfig): 860 """Assign a new subprocess for defconfig and add it to the slot. 861 862 If the slot is vacant, create a new subprocess for processing the 863 given defconfig and add it to the slot. Just returns False if 864 the slot is occupied (i.e. the current subprocess is still running). 865 866 Arguments: 867 defconfig: defconfig name. 868 869 Returns: 870 Return True on success or False on failure 871 """ 872 if self.state != STATE_IDLE: 873 return False 874 875 self.defconfig = defconfig 876 self.log = '' 877 self.current_src_dir = self.reference_src_dir 878 self.do_defconfig() 879 return True 880 881 def poll(self): 882 """Check the status of the subprocess and handle it as needed. 883 884 Returns True if the slot is vacant (i.e. in idle state). 885 If the configuration is successfully finished, assign a new 886 subprocess to build include/autoconf.mk. 887 If include/autoconf.mk is generated, invoke the parser to 888 parse the .config and the include/autoconf.mk, moving 889 config options to the .config as needed. 890 If the .config was updated, run "make savedefconfig" to sync 891 it, update the original defconfig, and then set the slot back 892 to the idle state. 893 894 Returns: 895 Return True if the subprocess is terminated, False otherwise 896 """ 897 if self.state == STATE_IDLE: 898 return True 899 900 if self.ps.poll() == None: 901 return False 902 903 if self.ps.poll() != 0: 904 self.handle_error() 905 elif self.state == STATE_DEFCONFIG: 906 if self.reference_src_dir and not self.current_src_dir: 907 self.do_savedefconfig() 908 else: 909 self.do_autoconf() 910 elif self.state == STATE_AUTOCONF: 911 if self.current_src_dir: 912 self.current_src_dir = None 913 self.do_defconfig() 914 else: 915 self.do_savedefconfig() 916 elif self.state == STATE_SAVEDEFCONFIG: 917 self.update_defconfig() 918 else: 919 sys.exit("Internal Error. This should not happen.") 920 921 return True if self.state == STATE_IDLE else False 922 923 def handle_error(self): 924 """Handle error cases.""" 925 926 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 927 "Failed to process.\n") 928 if self.options.verbose: 929 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 930 self.ps.stderr.read()) 931 self.finish(False) 932 933 def do_defconfig(self): 934 """Run 'make <board>_defconfig' to create the .config file.""" 935 936 cmd = list(self.make_cmd) 937 cmd.append(self.defconfig) 938 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 939 stderr=subprocess.PIPE, 940 cwd=self.current_src_dir) 941 self.state = STATE_DEFCONFIG 942 943 def do_autoconf(self): 944 """Run 'make include/config/auto.conf'.""" 945 946 self.cross_compile = self.parser.get_cross_compile() 947 if self.cross_compile is None: 948 self.log += color_text(self.options.color, COLOR_YELLOW, 949 "Compiler is missing. Do nothing.\n") 950 self.finish(False) 951 return 952 953 cmd = list(self.make_cmd) 954 if self.cross_compile: 955 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 956 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 957 cmd.append('include/config/auto.conf') 958 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 959 stderr=subprocess.PIPE, 960 cwd=self.current_src_dir) 961 self.state = STATE_AUTOCONF 962 963 def do_savedefconfig(self): 964 """Update the .config and run 'make savedefconfig'.""" 965 966 (updated, suspicious, log) = self.parser.update_dotconfig() 967 if suspicious: 968 self.suspicious_boards.add(self.defconfig) 969 self.log += log 970 971 if not self.options.force_sync and not updated: 972 self.finish(True) 973 return 974 if updated: 975 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 976 "Syncing by savedefconfig...\n") 977 else: 978 self.log += "Syncing by savedefconfig (forced by option)...\n" 979 980 cmd = list(self.make_cmd) 981 cmd.append('savedefconfig') 982 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 983 stderr=subprocess.PIPE) 984 self.state = STATE_SAVEDEFCONFIG 985 986 def update_defconfig(self): 987 """Update the input defconfig and go back to the idle state.""" 988 989 log = self.parser.check_defconfig() 990 if log: 991 self.suspicious_boards.add(self.defconfig) 992 self.log += log 993 orig_defconfig = os.path.join('configs', self.defconfig) 994 new_defconfig = os.path.join(self.build_dir, 'defconfig') 995 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 996 997 if updated: 998 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 999 "defconfig was updated.\n") 1000 1001 if not self.options.dry_run and updated: 1002 shutil.move(new_defconfig, orig_defconfig) 1003 self.finish(True) 1004 1005 def finish(self, success): 1006 """Display log along with progress and go to the idle state. 1007 1008 Arguments: 1009 success: Should be True when the defconfig was processed 1010 successfully, or False when it fails. 1011 """ 1012 # output at least 30 characters to hide the "* defconfigs out of *". 1013 log = self.defconfig.ljust(30) + '\n' 1014 1015 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1016 # Some threads are running in parallel. 1017 # Print log atomically to not mix up logs from different threads. 1018 print >> (sys.stdout if success else sys.stderr), log 1019 1020 if not success: 1021 if self.options.exit_on_error: 1022 sys.exit("Exit on error.") 1023 # If --exit-on-error flag is not set, skip this board and continue. 1024 # Record the failed board. 1025 self.failed_boards.add(self.defconfig) 1026 1027 self.progress.inc() 1028 self.progress.show() 1029 self.state = STATE_IDLE 1030 1031 def get_failed_boards(self): 1032 """Returns a set of failed boards (defconfigs) in this slot. 1033 """ 1034 return self.failed_boards 1035 1036 def get_suspicious_boards(self): 1037 """Returns a set of boards (defconfigs) with possible misconversion. 1038 """ 1039 return self.suspicious_boards - self.failed_boards 1040 1041class Slots: 1042 1043 """Controller of the array of subprocess slots.""" 1044 1045 def __init__(self, configs, options, progress, reference_src_dir): 1046 """Create a new slots controller. 1047 1048 Arguments: 1049 configs: A list of CONFIGs to move. 1050 options: option flags. 1051 progress: A progress indicator. 1052 reference_src_dir: Determine the true starting config state from this 1053 source tree. 1054 """ 1055 self.options = options 1056 self.slots = [] 1057 devnull = get_devnull() 1058 make_cmd = get_make_cmd() 1059 for i in range(options.jobs): 1060 self.slots.append(Slot(configs, options, progress, devnull, 1061 make_cmd, reference_src_dir)) 1062 1063 def add(self, defconfig): 1064 """Add a new subprocess if a vacant slot is found. 1065 1066 Arguments: 1067 defconfig: defconfig name to be put into. 1068 1069 Returns: 1070 Return True on success or False on failure 1071 """ 1072 for slot in self.slots: 1073 if slot.add(defconfig): 1074 return True 1075 return False 1076 1077 def available(self): 1078 """Check if there is a vacant slot. 1079 1080 Returns: 1081 Return True if at lease one vacant slot is found, False otherwise. 1082 """ 1083 for slot in self.slots: 1084 if slot.poll(): 1085 return True 1086 return False 1087 1088 def empty(self): 1089 """Check if all slots are vacant. 1090 1091 Returns: 1092 Return True if all the slots are vacant, False otherwise. 1093 """ 1094 ret = True 1095 for slot in self.slots: 1096 if not slot.poll(): 1097 ret = False 1098 return ret 1099 1100 def show_failed_boards(self): 1101 """Display all of the failed boards (defconfigs).""" 1102 boards = set() 1103 output_file = 'moveconfig.failed' 1104 1105 for slot in self.slots: 1106 boards |= slot.get_failed_boards() 1107 1108 if boards: 1109 boards = '\n'.join(boards) + '\n' 1110 msg = "The following boards were not processed due to error:\n" 1111 msg += boards 1112 msg += "(the list has been saved in %s)\n" % output_file 1113 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1114 msg) 1115 1116 with open(output_file, 'w') as f: 1117 f.write(boards) 1118 1119 def show_suspicious_boards(self): 1120 """Display all boards (defconfigs) with possible misconversion.""" 1121 boards = set() 1122 output_file = 'moveconfig.suspicious' 1123 1124 for slot in self.slots: 1125 boards |= slot.get_suspicious_boards() 1126 1127 if boards: 1128 boards = '\n'.join(boards) + '\n' 1129 msg = "The following boards might have been converted incorrectly.\n" 1130 msg += "It is highly recommended to check them manually:\n" 1131 msg += boards 1132 msg += "(the list has been saved in %s)\n" % output_file 1133 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1134 msg) 1135 1136 with open(output_file, 'w') as f: 1137 f.write(boards) 1138 1139class ReferenceSource: 1140 1141 """Reference source against which original configs should be parsed.""" 1142 1143 def __init__(self, commit): 1144 """Create a reference source directory based on a specified commit. 1145 1146 Arguments: 1147 commit: commit to git-clone 1148 """ 1149 self.src_dir = tempfile.mkdtemp() 1150 print "Cloning git repo to a separate work directory..." 1151 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1152 cwd=self.src_dir) 1153 print "Checkout '%s' to build the original autoconf.mk." % \ 1154 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1155 subprocess.check_output(['git', 'checkout', commit], 1156 stderr=subprocess.STDOUT, cwd=self.src_dir) 1157 1158 def __del__(self): 1159 """Delete the reference source directory 1160 1161 This function makes sure the temporary directory is cleaned away 1162 even if Python suddenly dies due to error. It should be done in here 1163 because it is guaranteed the destructor is always invoked when the 1164 instance of the class gets unreferenced. 1165 """ 1166 shutil.rmtree(self.src_dir) 1167 1168 def get_dir(self): 1169 """Return the absolute path to the reference source directory.""" 1170 1171 return self.src_dir 1172 1173def move_config(configs, options): 1174 """Move config options to defconfig files. 1175 1176 Arguments: 1177 configs: A list of CONFIGs to move. 1178 options: option flags 1179 """ 1180 if len(configs) == 0: 1181 if options.force_sync: 1182 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1183 else: 1184 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1185 else: 1186 print 'Move ' + ', '.join(configs), 1187 print '(jobs: %d)\n' % options.jobs 1188 1189 if options.git_ref: 1190 reference_src = ReferenceSource(options.git_ref) 1191 reference_src_dir = reference_src.get_dir() 1192 else: 1193 reference_src_dir = None 1194 1195 if options.defconfigs: 1196 defconfigs = [line.strip() for line in open(options.defconfigs)] 1197 for i, defconfig in enumerate(defconfigs): 1198 if not defconfig.endswith('_defconfig'): 1199 defconfigs[i] = defconfig + '_defconfig' 1200 if not os.path.exists(os.path.join('configs', defconfigs[i])): 1201 sys.exit('%s - defconfig does not exist. Stopping.' % 1202 defconfigs[i]) 1203 else: 1204 defconfigs = get_all_defconfigs() 1205 1206 progress = Progress(len(defconfigs)) 1207 slots = Slots(configs, options, progress, reference_src_dir) 1208 1209 # Main loop to process defconfig files: 1210 # Add a new subprocess into a vacant slot. 1211 # Sleep if there is no available slot. 1212 for defconfig in defconfigs: 1213 while not slots.add(defconfig): 1214 while not slots.available(): 1215 # No available slot: sleep for a while 1216 time.sleep(SLEEP_TIME) 1217 1218 # wait until all the subprocesses finish 1219 while not slots.empty(): 1220 time.sleep(SLEEP_TIME) 1221 1222 print '' 1223 slots.show_failed_boards() 1224 slots.show_suspicious_boards() 1225 1226def main(): 1227 try: 1228 cpu_count = multiprocessing.cpu_count() 1229 except NotImplementedError: 1230 cpu_count = 1 1231 1232 parser = optparse.OptionParser() 1233 # Add options here 1234 parser.add_option('-c', '--color', action='store_true', default=False, 1235 help='display the log in color') 1236 parser.add_option('-d', '--defconfigs', type='string', 1237 help='a file containing a list of defconfigs to move') 1238 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1239 help='perform a trial run (show log with no changes)') 1240 parser.add_option('-e', '--exit-on-error', action='store_true', 1241 default=False, 1242 help='exit immediately on any error') 1243 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1244 help='force sync by savedefconfig') 1245 parser.add_option('-S', '--spl', action='store_true', default=False, 1246 help='parse config options defined for SPL build') 1247 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1248 action='store_true', default=False, 1249 help='only cleanup the headers') 1250 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1251 help='the number of jobs to run simultaneously') 1252 parser.add_option('-r', '--git-ref', type='string', 1253 help='the git ref to clone for building the autoconf.mk') 1254 parser.add_option('-v', '--verbose', action='store_true', default=False, 1255 help='show any build errors as boards are built') 1256 parser.usage += ' CONFIG ...' 1257 1258 (options, configs) = parser.parse_args() 1259 1260 if len(configs) == 0 and not options.force_sync: 1261 parser.print_usage() 1262 sys.exit(1) 1263 1264 # prefix the option name with CONFIG_ if missing 1265 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1266 for config in configs ] 1267 1268 check_top_directory() 1269 1270 if not options.cleanup_headers_only: 1271 check_clean_directory() 1272 update_cross_compile(options.color) 1273 move_config(configs, options) 1274 1275 if configs: 1276 cleanup_headers(configs, options) 1277 cleanup_extra_options(configs, options) 1278 1279if __name__ == '__main__': 1280 main() 1281