1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. 111 112The default per-arch CROSS_COMPILE used by this tool is specified by 113the list below, CROSS_COMPILE. You may wish to update the list to 114use your own. Instead of modifying the list directly, you can give 115them via environments. 116 117 118Available options 119----------------- 120 121 -c, --color 122 Surround each portion of the log with escape sequences to display it 123 in color on the terminal. 124 125 -C, --commit 126 Create a git commit with the changes when the operation is complete. A 127 standard commit message is used which may need to be edited. 128 129 -d, --defconfigs 130 Specify a file containing a list of defconfigs to move. The defconfig 131 files can be given with shell-style wildcards. 132 133 -n, --dry-run 134 Perform a trial run that does not make any changes. It is useful to 135 see what is going to happen before one actually runs it. 136 137 -e, --exit-on-error 138 Exit immediately if Make exits with a non-zero status while processing 139 a defconfig file. 140 141 -s, --force-sync 142 Do "make savedefconfig" forcibly for all the defconfig files. 143 If not specified, "make savedefconfig" only occurs for cases 144 where at least one CONFIG was moved. 145 146 -S, --spl 147 Look for moved config options in spl/include/autoconf.mk instead of 148 include/autoconf.mk. This is useful for moving options for SPL build 149 because SPL related options (mostly prefixed with CONFIG_SPL_) are 150 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 151 152 -H, --headers-only 153 Only cleanup the headers; skip the defconfig processing 154 155 -j, --jobs 156 Specify the number of threads to run simultaneously. If not specified, 157 the number of threads is the same as the number of CPU cores. 158 159 -r, --git-ref 160 Specify the git ref to clone for building the autoconf.mk. If unspecified 161 use the CWD. This is useful for when changes to the Kconfig affect the 162 default values and you want to capture the state of the defconfig from 163 before that change was in effect. If in doubt, specify a ref pre-Kconfig 164 changes (use HEAD if Kconfig changes are not committed). Worst case it will 165 take a bit longer to run, but will always do the right thing. 166 167 -v, --verbose 168 Show any build errors as boards are built 169 170 -y, --yes 171 Instead of prompting, automatically go ahead with all operations. This 172 includes cleaning up headers and CONFIG_SYS_EXTRA_OPTIONS. 173 174To see the complete list of supported options, run 175 176 $ tools/moveconfig.py -h 177 178""" 179 180import copy 181import difflib 182import filecmp 183import fnmatch 184import glob 185import multiprocessing 186import optparse 187import os 188import re 189import shutil 190import subprocess 191import sys 192import tempfile 193import time 194 195SHOW_GNU_MAKE = 'scripts/show-gnu-make' 196SLEEP_TIME=0.03 197 198# Here is the list of cross-tools I use. 199# Most of them are available at kernel.org 200# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following: 201# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 202# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 203# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 204# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 205CROSS_COMPILE = { 206 'arc': 'arc-linux-', 207 'aarch64': 'aarch64-linux-', 208 'arm': 'arm-unknown-linux-gnueabi-', 209 'm68k': 'm68k-linux-', 210 'microblaze': 'microblaze-linux-', 211 'mips': 'mips-linux-', 212 'nds32': 'nds32le-linux-', 213 'nios2': 'nios2-linux-gnu-', 214 'powerpc': 'powerpc-linux-', 215 'sh': 'sh-linux-gnu-', 216 'x86': 'i386-linux-', 217 'xtensa': 'xtensa-linux-' 218} 219 220STATE_IDLE = 0 221STATE_DEFCONFIG = 1 222STATE_AUTOCONF = 2 223STATE_SAVEDEFCONFIG = 3 224 225ACTION_MOVE = 0 226ACTION_NO_ENTRY = 1 227ACTION_NO_ENTRY_WARN = 2 228ACTION_NO_CHANGE = 3 229 230COLOR_BLACK = '0;30' 231COLOR_RED = '0;31' 232COLOR_GREEN = '0;32' 233COLOR_BROWN = '0;33' 234COLOR_BLUE = '0;34' 235COLOR_PURPLE = '0;35' 236COLOR_CYAN = '0;36' 237COLOR_LIGHT_GRAY = '0;37' 238COLOR_DARK_GRAY = '1;30' 239COLOR_LIGHT_RED = '1;31' 240COLOR_LIGHT_GREEN = '1;32' 241COLOR_YELLOW = '1;33' 242COLOR_LIGHT_BLUE = '1;34' 243COLOR_LIGHT_PURPLE = '1;35' 244COLOR_LIGHT_CYAN = '1;36' 245COLOR_WHITE = '1;37' 246 247### helper functions ### 248def get_devnull(): 249 """Get the file object of '/dev/null' device.""" 250 try: 251 devnull = subprocess.DEVNULL # py3k 252 except AttributeError: 253 devnull = open(os.devnull, 'wb') 254 return devnull 255 256def check_top_directory(): 257 """Exit if we are not at the top of source directory.""" 258 for f in ('README', 'Licenses'): 259 if not os.path.exists(f): 260 sys.exit('Please run at the top of source directory.') 261 262def check_clean_directory(): 263 """Exit if the source tree is not clean.""" 264 for f in ('.config', 'include/config'): 265 if os.path.exists(f): 266 sys.exit("source tree is not clean, please run 'make mrproper'") 267 268def get_make_cmd(): 269 """Get the command name of GNU Make. 270 271 U-Boot needs GNU Make for building, but the command name is not 272 necessarily "make". (for example, "gmake" on FreeBSD). 273 Returns the most appropriate command name on your system. 274 """ 275 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 276 ret = process.communicate() 277 if process.returncode: 278 sys.exit('GNU Make not found') 279 return ret[0].rstrip() 280 281def get_matched_defconfigs(defconfigs_file): 282 """Get all the defconfig files that match the patterns in a file.""" 283 defconfigs = [] 284 for i, line in enumerate(open(defconfigs_file)): 285 line = line.strip() 286 if not line: 287 continue # skip blank lines silently 288 pattern = os.path.join('configs', line) 289 matched = glob.glob(pattern) + glob.glob(pattern + '_defconfig') 290 if not matched: 291 print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \ 292 (defconfigs_file, i + 1, line) 293 294 defconfigs += matched 295 296 # use set() to drop multiple matching 297 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 298 299def get_all_defconfigs(): 300 """Get all the defconfig files under the configs/ directory.""" 301 defconfigs = [] 302 for (dirpath, dirnames, filenames) in os.walk('configs'): 303 dirpath = dirpath[len('configs') + 1:] 304 for filename in fnmatch.filter(filenames, '*_defconfig'): 305 defconfigs.append(os.path.join(dirpath, filename)) 306 307 return defconfigs 308 309def color_text(color_enabled, color, string): 310 """Return colored string.""" 311 if color_enabled: 312 # LF should not be surrounded by the escape sequence. 313 # Otherwise, additional whitespace or line-feed might be printed. 314 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 315 for s in string.split('\n') ]) 316 else: 317 return string 318 319def show_diff(a, b, file_path, color_enabled): 320 """Show unidified diff. 321 322 Arguments: 323 a: A list of lines (before) 324 b: A list of lines (after) 325 file_path: Path to the file 326 color_enabled: Display the diff in color 327 """ 328 329 diff = difflib.unified_diff(a, b, 330 fromfile=os.path.join('a', file_path), 331 tofile=os.path.join('b', file_path)) 332 333 for line in diff: 334 if line[0] == '-' and line[1] != '-': 335 print color_text(color_enabled, COLOR_RED, line), 336 elif line[0] == '+' and line[1] != '+': 337 print color_text(color_enabled, COLOR_GREEN, line), 338 else: 339 print line, 340 341def update_cross_compile(color_enabled): 342 """Update per-arch CROSS_COMPILE via environment variables 343 344 The default CROSS_COMPILE values are available 345 in the CROSS_COMPILE list above. 346 347 You can override them via environment variables 348 CROSS_COMPILE_{ARCH}. 349 350 For example, if you want to override toolchain prefixes 351 for ARM and PowerPC, you can do as follows in your shell: 352 353 export CROSS_COMPILE_ARM=... 354 export CROSS_COMPILE_POWERPC=... 355 356 Then, this function checks if specified compilers really exist in your 357 PATH environment. 358 """ 359 archs = [] 360 361 for arch in os.listdir('arch'): 362 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 363 archs.append(arch) 364 365 # arm64 is a special case 366 archs.append('aarch64') 367 368 for arch in archs: 369 env = 'CROSS_COMPILE_' + arch.upper() 370 cross_compile = os.environ.get(env) 371 if not cross_compile: 372 cross_compile = CROSS_COMPILE.get(arch, '') 373 374 for path in os.environ["PATH"].split(os.pathsep): 375 gcc_path = os.path.join(path, cross_compile + 'gcc') 376 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 377 break 378 else: 379 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 380 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 381 % (cross_compile, arch)) 382 cross_compile = None 383 384 CROSS_COMPILE[arch] = cross_compile 385 386def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 387 extend_post): 388 """Extend matched lines if desired patterns are found before/after already 389 matched lines. 390 391 Arguments: 392 lines: A list of lines handled. 393 matched: A list of line numbers that have been already matched. 394 (will be updated by this function) 395 pre_patterns: A list of regular expression that should be matched as 396 preamble. 397 post_patterns: A list of regular expression that should be matched as 398 postamble. 399 extend_pre: Add the line number of matched preamble to the matched list. 400 extend_post: Add the line number of matched postamble to the matched list. 401 """ 402 extended_matched = [] 403 404 j = matched[0] 405 406 for i in matched: 407 if i == 0 or i < j: 408 continue 409 j = i 410 while j in matched: 411 j += 1 412 if j >= len(lines): 413 break 414 415 for p in pre_patterns: 416 if p.search(lines[i - 1]): 417 break 418 else: 419 # not matched 420 continue 421 422 for p in post_patterns: 423 if p.search(lines[j]): 424 break 425 else: 426 # not matched 427 continue 428 429 if extend_pre: 430 extended_matched.append(i - 1) 431 if extend_post: 432 extended_matched.append(j) 433 434 matched += extended_matched 435 matched.sort() 436 437def confirm(options, prompt): 438 if not options.yes: 439 while True: 440 choice = raw_input('{} [y/n]: '.format(prompt)) 441 choice = choice.lower() 442 print choice 443 if choice == 'y' or choice == 'n': 444 break 445 446 if choice == 'n': 447 return False 448 449 return True 450 451def cleanup_one_header(header_path, patterns, options): 452 """Clean regex-matched lines away from a file. 453 454 Arguments: 455 header_path: path to the cleaned file. 456 patterns: list of regex patterns. Any lines matching to these 457 patterns are deleted. 458 options: option flags. 459 """ 460 with open(header_path) as f: 461 lines = f.readlines() 462 463 matched = [] 464 for i, line in enumerate(lines): 465 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 466 matched.append(i) 467 continue 468 for pattern in patterns: 469 if pattern.search(line): 470 matched.append(i) 471 break 472 473 if not matched: 474 return 475 476 # remove empty #ifdef ... #endif, successive blank lines 477 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 478 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 479 pattern_endif = re.compile(r'#\s*endif\W') # #endif 480 pattern_blank = re.compile(r'^\s*$') # empty line 481 482 while True: 483 old_matched = copy.copy(matched) 484 extend_matched_lines(lines, matched, [pattern_if], 485 [pattern_endif], True, True) 486 extend_matched_lines(lines, matched, [pattern_elif], 487 [pattern_elif, pattern_endif], True, False) 488 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 489 [pattern_blank], False, True) 490 extend_matched_lines(lines, matched, [pattern_blank], 491 [pattern_elif, pattern_endif], True, False) 492 extend_matched_lines(lines, matched, [pattern_blank], 493 [pattern_blank], True, False) 494 if matched == old_matched: 495 break 496 497 tolines = copy.copy(lines) 498 499 for i in reversed(matched): 500 tolines.pop(i) 501 502 show_diff(lines, tolines, header_path, options.color) 503 504 if options.dry_run: 505 return 506 507 with open(header_path, 'w') as f: 508 for line in tolines: 509 f.write(line) 510 511def cleanup_headers(configs, options): 512 """Delete config defines from board headers. 513 514 Arguments: 515 configs: A list of CONFIGs to remove. 516 options: option flags. 517 """ 518 if not confirm(options, 'Clean up headers?'): 519 return 520 521 patterns = [] 522 for config in configs: 523 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 524 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 525 526 for dir in 'include', 'arch', 'board': 527 for (dirpath, dirnames, filenames) in os.walk(dir): 528 if dirpath == os.path.join('include', 'generated'): 529 continue 530 for filename in filenames: 531 if not fnmatch.fnmatch(filename, '*~'): 532 cleanup_one_header(os.path.join(dirpath, filename), 533 patterns, options) 534 535def cleanup_one_extra_option(defconfig_path, configs, options): 536 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 537 538 Arguments: 539 defconfig_path: path to the cleaned defconfig file. 540 configs: A list of CONFIGs to remove. 541 options: option flags. 542 """ 543 544 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 545 end = '"\n' 546 547 with open(defconfig_path) as f: 548 lines = f.readlines() 549 550 for i, line in enumerate(lines): 551 if line.startswith(start) and line.endswith(end): 552 break 553 else: 554 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 555 return 556 557 old_tokens = line[len(start):-len(end)].split(',') 558 new_tokens = [] 559 560 for token in old_tokens: 561 pos = token.find('=') 562 if not (token[:pos] if pos >= 0 else token) in configs: 563 new_tokens.append(token) 564 565 if new_tokens == old_tokens: 566 return 567 568 tolines = copy.copy(lines) 569 570 if new_tokens: 571 tolines[i] = start + ','.join(new_tokens) + end 572 else: 573 tolines.pop(i) 574 575 show_diff(lines, tolines, defconfig_path, options.color) 576 577 if options.dry_run: 578 return 579 580 with open(defconfig_path, 'w') as f: 581 for line in tolines: 582 f.write(line) 583 584def cleanup_extra_options(configs, options): 585 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 586 587 Arguments: 588 configs: A list of CONFIGs to remove. 589 options: option flags. 590 """ 591 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 592 return 593 594 configs = [ config[len('CONFIG_'):] for config in configs ] 595 596 defconfigs = get_all_defconfigs() 597 598 for defconfig in defconfigs: 599 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 600 options) 601 602def cleanup_whitelist(configs, options): 603 """Delete config whitelist entries 604 605 Arguments: 606 configs: A list of CONFIGs to remove. 607 options: option flags. 608 """ 609 if not confirm(options, 'Clean up whitelist entries?'): 610 return 611 612 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 613 lines = f.readlines() 614 615 lines = [x for x in lines if x.strip() not in configs] 616 617 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 618 f.write(''.join(lines)) 619 620def find_matching(patterns, line): 621 for pat in patterns: 622 if pat.search(line): 623 return True 624 return False 625 626def cleanup_readme(configs, options): 627 """Delete config description in README 628 629 Arguments: 630 configs: A list of CONFIGs to remove. 631 options: option flags. 632 """ 633 if not confirm(options, 'Clean up README?'): 634 return 635 636 patterns = [] 637 for config in configs: 638 patterns.append(re.compile(r'^\s+%s' % config)) 639 640 with open('README') as f: 641 lines = f.readlines() 642 643 found = False 644 newlines = [] 645 for line in lines: 646 if not found: 647 found = find_matching(patterns, line) 648 if found: 649 continue 650 651 if found and re.search(r'^\s+CONFIG', line): 652 found = False 653 654 if not found: 655 newlines.append(line) 656 657 with open('README', 'w') as f: 658 f.write(''.join(newlines)) 659 660 661### classes ### 662class Progress: 663 664 """Progress Indicator""" 665 666 def __init__(self, total): 667 """Create a new progress indicator. 668 669 Arguments: 670 total: A number of defconfig files to process. 671 """ 672 self.current = 0 673 self.total = total 674 675 def inc(self): 676 """Increment the number of processed defconfig files.""" 677 678 self.current += 1 679 680 def show(self): 681 """Display the progress.""" 682 print ' %d defconfigs out of %d\r' % (self.current, self.total), 683 sys.stdout.flush() 684 685class KconfigParser: 686 687 """A parser of .config and include/autoconf.mk.""" 688 689 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 690 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 691 692 def __init__(self, configs, options, build_dir): 693 """Create a new parser. 694 695 Arguments: 696 configs: A list of CONFIGs to move. 697 options: option flags. 698 build_dir: Build directory. 699 """ 700 self.configs = configs 701 self.options = options 702 self.dotconfig = os.path.join(build_dir, '.config') 703 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 704 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 705 'autoconf.mk') 706 self.config_autoconf = os.path.join(build_dir, 'include', 'config', 707 'auto.conf') 708 self.defconfig = os.path.join(build_dir, 'defconfig') 709 710 def get_cross_compile(self): 711 """Parse .config file and return CROSS_COMPILE. 712 713 Returns: 714 A string storing the compiler prefix for the architecture. 715 Return a NULL string for architectures that do not require 716 compiler prefix (Sandbox and native build is the case). 717 Return None if the specified compiler is missing in your PATH. 718 Caller should distinguish '' and None. 719 """ 720 arch = '' 721 cpu = '' 722 for line in open(self.dotconfig): 723 m = self.re_arch.match(line) 724 if m: 725 arch = m.group(1) 726 continue 727 m = self.re_cpu.match(line) 728 if m: 729 cpu = m.group(1) 730 731 if not arch: 732 return None 733 734 # fix-up for aarch64 735 if arch == 'arm' and cpu == 'armv8': 736 arch = 'aarch64' 737 738 return CROSS_COMPILE.get(arch, None) 739 740 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 741 """Parse .config, defconfig, include/autoconf.mk for one config. 742 743 This function looks for the config options in the lines from 744 defconfig, .config, and include/autoconf.mk in order to decide 745 which action should be taken for this defconfig. 746 747 Arguments: 748 config: CONFIG name to parse. 749 dotconfig_lines: lines from the .config file. 750 autoconf_lines: lines from the include/autoconf.mk file. 751 752 Returns: 753 A tupple of the action for this defconfig and the line 754 matched for the config. 755 """ 756 not_set = '# %s is not set' % config 757 758 for line in autoconf_lines: 759 line = line.rstrip() 760 if line.startswith(config + '='): 761 new_val = line 762 break 763 else: 764 new_val = not_set 765 766 for line in dotconfig_lines: 767 line = line.rstrip() 768 if line.startswith(config + '=') or line == not_set: 769 old_val = line 770 break 771 else: 772 if new_val == not_set: 773 return (ACTION_NO_ENTRY, config) 774 else: 775 return (ACTION_NO_ENTRY_WARN, config) 776 777 # If this CONFIG is neither bool nor trisate 778 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 779 # tools/scripts/define2mk.sed changes '1' to 'y'. 780 # This is a problem if the CONFIG is int type. 781 # Check the type in Kconfig and handle it correctly. 782 if new_val[-2:] == '=y': 783 new_val = new_val[:-1] + '1' 784 785 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 786 new_val) 787 788 def update_dotconfig(self): 789 """Parse files for the config options and update the .config. 790 791 This function parses the generated .config and include/autoconf.mk 792 searching the target options. 793 Move the config option(s) to the .config as needed. 794 795 Arguments: 796 defconfig: defconfig name. 797 798 Returns: 799 Return a tuple of (updated flag, log string). 800 The "updated flag" is True if the .config was updated, False 801 otherwise. The "log string" shows what happend to the .config. 802 """ 803 804 results = [] 805 updated = False 806 suspicious = False 807 rm_files = [self.config_autoconf, self.autoconf] 808 809 if self.options.spl: 810 if os.path.exists(self.spl_autoconf): 811 autoconf_path = self.spl_autoconf 812 rm_files.append(self.spl_autoconf) 813 else: 814 for f in rm_files: 815 os.remove(f) 816 return (updated, suspicious, 817 color_text(self.options.color, COLOR_BROWN, 818 "SPL is not enabled. Skipped.") + '\n') 819 else: 820 autoconf_path = self.autoconf 821 822 with open(self.dotconfig) as f: 823 dotconfig_lines = f.readlines() 824 825 with open(autoconf_path) as f: 826 autoconf_lines = f.readlines() 827 828 for config in self.configs: 829 result = self.parse_one_config(config, dotconfig_lines, 830 autoconf_lines) 831 results.append(result) 832 833 log = '' 834 835 for (action, value) in results: 836 if action == ACTION_MOVE: 837 actlog = "Move '%s'" % value 838 log_color = COLOR_LIGHT_GREEN 839 elif action == ACTION_NO_ENTRY: 840 actlog = "%s is not defined in Kconfig. Do nothing." % value 841 log_color = COLOR_LIGHT_BLUE 842 elif action == ACTION_NO_ENTRY_WARN: 843 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 844 log_color = COLOR_YELLOW 845 suspicious = True 846 elif action == ACTION_NO_CHANGE: 847 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 848 % value 849 log_color = COLOR_LIGHT_PURPLE 850 elif action == ACTION_SPL_NOT_EXIST: 851 actlog = "SPL is not enabled for this defconfig. Skip." 852 log_color = COLOR_PURPLE 853 else: 854 sys.exit("Internal Error. This should not happen.") 855 856 log += color_text(self.options.color, log_color, actlog) + '\n' 857 858 with open(self.dotconfig, 'a') as f: 859 for (action, value) in results: 860 if action == ACTION_MOVE: 861 f.write(value + '\n') 862 updated = True 863 864 self.results = results 865 for f in rm_files: 866 os.remove(f) 867 868 return (updated, suspicious, log) 869 870 def check_defconfig(self): 871 """Check the defconfig after savedefconfig 872 873 Returns: 874 Return additional log if moved CONFIGs were removed again by 875 'make savedefconfig'. 876 """ 877 878 log = '' 879 880 with open(self.defconfig) as f: 881 defconfig_lines = f.readlines() 882 883 for (action, value) in self.results: 884 if action != ACTION_MOVE: 885 continue 886 if not value + '\n' in defconfig_lines: 887 log += color_text(self.options.color, COLOR_YELLOW, 888 "'%s' was removed by savedefconfig.\n" % 889 value) 890 891 return log 892 893class Slot: 894 895 """A slot to store a subprocess. 896 897 Each instance of this class handles one subprocess. 898 This class is useful to control multiple threads 899 for faster processing. 900 """ 901 902 def __init__(self, configs, options, progress, devnull, make_cmd, reference_src_dir): 903 """Create a new process slot. 904 905 Arguments: 906 configs: A list of CONFIGs to move. 907 options: option flags. 908 progress: A progress indicator. 909 devnull: A file object of '/dev/null'. 910 make_cmd: command name of GNU Make. 911 reference_src_dir: Determine the true starting config state from this 912 source tree. 913 """ 914 self.options = options 915 self.progress = progress 916 self.build_dir = tempfile.mkdtemp() 917 self.devnull = devnull 918 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 919 self.reference_src_dir = reference_src_dir 920 self.parser = KconfigParser(configs, options, self.build_dir) 921 self.state = STATE_IDLE 922 self.failed_boards = set() 923 self.suspicious_boards = set() 924 925 def __del__(self): 926 """Delete the working directory 927 928 This function makes sure the temporary directory is cleaned away 929 even if Python suddenly dies due to error. It should be done in here 930 because it is guaranteed the destructor is always invoked when the 931 instance of the class gets unreferenced. 932 933 If the subprocess is still running, wait until it finishes. 934 """ 935 if self.state != STATE_IDLE: 936 while self.ps.poll() == None: 937 pass 938 shutil.rmtree(self.build_dir) 939 940 def add(self, defconfig): 941 """Assign a new subprocess for defconfig and add it to the slot. 942 943 If the slot is vacant, create a new subprocess for processing the 944 given defconfig and add it to the slot. Just returns False if 945 the slot is occupied (i.e. the current subprocess is still running). 946 947 Arguments: 948 defconfig: defconfig name. 949 950 Returns: 951 Return True on success or False on failure 952 """ 953 if self.state != STATE_IDLE: 954 return False 955 956 self.defconfig = defconfig 957 self.log = '' 958 self.current_src_dir = self.reference_src_dir 959 self.do_defconfig() 960 return True 961 962 def poll(self): 963 """Check the status of the subprocess and handle it as needed. 964 965 Returns True if the slot is vacant (i.e. in idle state). 966 If the configuration is successfully finished, assign a new 967 subprocess to build include/autoconf.mk. 968 If include/autoconf.mk is generated, invoke the parser to 969 parse the .config and the include/autoconf.mk, moving 970 config options to the .config as needed. 971 If the .config was updated, run "make savedefconfig" to sync 972 it, update the original defconfig, and then set the slot back 973 to the idle state. 974 975 Returns: 976 Return True if the subprocess is terminated, False otherwise 977 """ 978 if self.state == STATE_IDLE: 979 return True 980 981 if self.ps.poll() == None: 982 return False 983 984 if self.ps.poll() != 0: 985 self.handle_error() 986 elif self.state == STATE_DEFCONFIG: 987 if self.reference_src_dir and not self.current_src_dir: 988 self.do_savedefconfig() 989 else: 990 self.do_autoconf() 991 elif self.state == STATE_AUTOCONF: 992 if self.current_src_dir: 993 self.current_src_dir = None 994 self.do_defconfig() 995 else: 996 self.do_savedefconfig() 997 elif self.state == STATE_SAVEDEFCONFIG: 998 self.update_defconfig() 999 else: 1000 sys.exit("Internal Error. This should not happen.") 1001 1002 return True if self.state == STATE_IDLE else False 1003 1004 def handle_error(self): 1005 """Handle error cases.""" 1006 1007 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1008 "Failed to process.\n") 1009 if self.options.verbose: 1010 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1011 self.ps.stderr.read()) 1012 self.finish(False) 1013 1014 def do_defconfig(self): 1015 """Run 'make <board>_defconfig' to create the .config file.""" 1016 1017 cmd = list(self.make_cmd) 1018 cmd.append(self.defconfig) 1019 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1020 stderr=subprocess.PIPE, 1021 cwd=self.current_src_dir) 1022 self.state = STATE_DEFCONFIG 1023 1024 def do_autoconf(self): 1025 """Run 'make include/config/auto.conf'.""" 1026 1027 self.cross_compile = self.parser.get_cross_compile() 1028 if self.cross_compile is None: 1029 self.log += color_text(self.options.color, COLOR_YELLOW, 1030 "Compiler is missing. Do nothing.\n") 1031 self.finish(False) 1032 return 1033 1034 cmd = list(self.make_cmd) 1035 if self.cross_compile: 1036 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 1037 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1038 cmd.append('include/config/auto.conf') 1039 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1040 stderr=subprocess.PIPE, 1041 cwd=self.current_src_dir) 1042 self.state = STATE_AUTOCONF 1043 1044 def do_savedefconfig(self): 1045 """Update the .config and run 'make savedefconfig'.""" 1046 1047 (updated, suspicious, log) = self.parser.update_dotconfig() 1048 if suspicious: 1049 self.suspicious_boards.add(self.defconfig) 1050 self.log += log 1051 1052 if not self.options.force_sync and not updated: 1053 self.finish(True) 1054 return 1055 if updated: 1056 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1057 "Syncing by savedefconfig...\n") 1058 else: 1059 self.log += "Syncing by savedefconfig (forced by option)...\n" 1060 1061 cmd = list(self.make_cmd) 1062 cmd.append('savedefconfig') 1063 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1064 stderr=subprocess.PIPE) 1065 self.state = STATE_SAVEDEFCONFIG 1066 1067 def update_defconfig(self): 1068 """Update the input defconfig and go back to the idle state.""" 1069 1070 log = self.parser.check_defconfig() 1071 if log: 1072 self.suspicious_boards.add(self.defconfig) 1073 self.log += log 1074 orig_defconfig = os.path.join('configs', self.defconfig) 1075 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1076 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1077 1078 if updated: 1079 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1080 "defconfig was updated.\n") 1081 1082 if not self.options.dry_run and updated: 1083 shutil.move(new_defconfig, orig_defconfig) 1084 self.finish(True) 1085 1086 def finish(self, success): 1087 """Display log along with progress and go to the idle state. 1088 1089 Arguments: 1090 success: Should be True when the defconfig was processed 1091 successfully, or False when it fails. 1092 """ 1093 # output at least 30 characters to hide the "* defconfigs out of *". 1094 log = self.defconfig.ljust(30) + '\n' 1095 1096 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1097 # Some threads are running in parallel. 1098 # Print log atomically to not mix up logs from different threads. 1099 print >> (sys.stdout if success else sys.stderr), log 1100 1101 if not success: 1102 if self.options.exit_on_error: 1103 sys.exit("Exit on error.") 1104 # If --exit-on-error flag is not set, skip this board and continue. 1105 # Record the failed board. 1106 self.failed_boards.add(self.defconfig) 1107 1108 self.progress.inc() 1109 self.progress.show() 1110 self.state = STATE_IDLE 1111 1112 def get_failed_boards(self): 1113 """Returns a set of failed boards (defconfigs) in this slot. 1114 """ 1115 return self.failed_boards 1116 1117 def get_suspicious_boards(self): 1118 """Returns a set of boards (defconfigs) with possible misconversion. 1119 """ 1120 return self.suspicious_boards - self.failed_boards 1121 1122class Slots: 1123 1124 """Controller of the array of subprocess slots.""" 1125 1126 def __init__(self, configs, options, progress, reference_src_dir): 1127 """Create a new slots controller. 1128 1129 Arguments: 1130 configs: A list of CONFIGs to move. 1131 options: option flags. 1132 progress: A progress indicator. 1133 reference_src_dir: Determine the true starting config state from this 1134 source tree. 1135 """ 1136 self.options = options 1137 self.slots = [] 1138 devnull = get_devnull() 1139 make_cmd = get_make_cmd() 1140 for i in range(options.jobs): 1141 self.slots.append(Slot(configs, options, progress, devnull, 1142 make_cmd, reference_src_dir)) 1143 1144 def add(self, defconfig): 1145 """Add a new subprocess if a vacant slot is found. 1146 1147 Arguments: 1148 defconfig: defconfig name to be put into. 1149 1150 Returns: 1151 Return True on success or False on failure 1152 """ 1153 for slot in self.slots: 1154 if slot.add(defconfig): 1155 return True 1156 return False 1157 1158 def available(self): 1159 """Check if there is a vacant slot. 1160 1161 Returns: 1162 Return True if at lease one vacant slot is found, False otherwise. 1163 """ 1164 for slot in self.slots: 1165 if slot.poll(): 1166 return True 1167 return False 1168 1169 def empty(self): 1170 """Check if all slots are vacant. 1171 1172 Returns: 1173 Return True if all the slots are vacant, False otherwise. 1174 """ 1175 ret = True 1176 for slot in self.slots: 1177 if not slot.poll(): 1178 ret = False 1179 return ret 1180 1181 def show_failed_boards(self): 1182 """Display all of the failed boards (defconfigs).""" 1183 boards = set() 1184 output_file = 'moveconfig.failed' 1185 1186 for slot in self.slots: 1187 boards |= slot.get_failed_boards() 1188 1189 if boards: 1190 boards = '\n'.join(boards) + '\n' 1191 msg = "The following boards were not processed due to error:\n" 1192 msg += boards 1193 msg += "(the list has been saved in %s)\n" % output_file 1194 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1195 msg) 1196 1197 with open(output_file, 'w') as f: 1198 f.write(boards) 1199 1200 def show_suspicious_boards(self): 1201 """Display all boards (defconfigs) with possible misconversion.""" 1202 boards = set() 1203 output_file = 'moveconfig.suspicious' 1204 1205 for slot in self.slots: 1206 boards |= slot.get_suspicious_boards() 1207 1208 if boards: 1209 boards = '\n'.join(boards) + '\n' 1210 msg = "The following boards might have been converted incorrectly.\n" 1211 msg += "It is highly recommended to check them manually:\n" 1212 msg += boards 1213 msg += "(the list has been saved in %s)\n" % output_file 1214 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1215 msg) 1216 1217 with open(output_file, 'w') as f: 1218 f.write(boards) 1219 1220class ReferenceSource: 1221 1222 """Reference source against which original configs should be parsed.""" 1223 1224 def __init__(self, commit): 1225 """Create a reference source directory based on a specified commit. 1226 1227 Arguments: 1228 commit: commit to git-clone 1229 """ 1230 self.src_dir = tempfile.mkdtemp() 1231 print "Cloning git repo to a separate work directory..." 1232 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1233 cwd=self.src_dir) 1234 print "Checkout '%s' to build the original autoconf.mk." % \ 1235 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1236 subprocess.check_output(['git', 'checkout', commit], 1237 stderr=subprocess.STDOUT, cwd=self.src_dir) 1238 1239 def __del__(self): 1240 """Delete the reference source directory 1241 1242 This function makes sure the temporary directory is cleaned away 1243 even if Python suddenly dies due to error. It should be done in here 1244 because it is guaranteed the destructor is always invoked when the 1245 instance of the class gets unreferenced. 1246 """ 1247 shutil.rmtree(self.src_dir) 1248 1249 def get_dir(self): 1250 """Return the absolute path to the reference source directory.""" 1251 1252 return self.src_dir 1253 1254def move_config(configs, options): 1255 """Move config options to defconfig files. 1256 1257 Arguments: 1258 configs: A list of CONFIGs to move. 1259 options: option flags 1260 """ 1261 if len(configs) == 0: 1262 if options.force_sync: 1263 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1264 else: 1265 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1266 else: 1267 print 'Move ' + ', '.join(configs), 1268 print '(jobs: %d)\n' % options.jobs 1269 1270 if options.git_ref: 1271 reference_src = ReferenceSource(options.git_ref) 1272 reference_src_dir = reference_src.get_dir() 1273 else: 1274 reference_src_dir = None 1275 1276 if options.defconfigs: 1277 defconfigs = get_matched_defconfigs(options.defconfigs) 1278 else: 1279 defconfigs = get_all_defconfigs() 1280 1281 progress = Progress(len(defconfigs)) 1282 slots = Slots(configs, options, progress, reference_src_dir) 1283 1284 # Main loop to process defconfig files: 1285 # Add a new subprocess into a vacant slot. 1286 # Sleep if there is no available slot. 1287 for defconfig in defconfigs: 1288 while not slots.add(defconfig): 1289 while not slots.available(): 1290 # No available slot: sleep for a while 1291 time.sleep(SLEEP_TIME) 1292 1293 # wait until all the subprocesses finish 1294 while not slots.empty(): 1295 time.sleep(SLEEP_TIME) 1296 1297 print '' 1298 slots.show_failed_boards() 1299 slots.show_suspicious_boards() 1300 1301def main(): 1302 try: 1303 cpu_count = multiprocessing.cpu_count() 1304 except NotImplementedError: 1305 cpu_count = 1 1306 1307 parser = optparse.OptionParser() 1308 # Add options here 1309 parser.add_option('-c', '--color', action='store_true', default=False, 1310 help='display the log in color') 1311 parser.add_option('-C', '--commit', action='store_true', default=False, 1312 help='Create a git commit for the operation') 1313 parser.add_option('-d', '--defconfigs', type='string', 1314 help='a file containing a list of defconfigs to move') 1315 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1316 help='perform a trial run (show log with no changes)') 1317 parser.add_option('-e', '--exit-on-error', action='store_true', 1318 default=False, 1319 help='exit immediately on any error') 1320 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1321 help='force sync by savedefconfig') 1322 parser.add_option('-S', '--spl', action='store_true', default=False, 1323 help='parse config options defined for SPL build') 1324 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1325 action='store_true', default=False, 1326 help='only cleanup the headers') 1327 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1328 help='the number of jobs to run simultaneously') 1329 parser.add_option('-r', '--git-ref', type='string', 1330 help='the git ref to clone for building the autoconf.mk') 1331 parser.add_option('-y', '--yes', action='store_true', default=False, 1332 help="respond 'yes' to any prompts") 1333 parser.add_option('-v', '--verbose', action='store_true', default=False, 1334 help='show any build errors as boards are built') 1335 parser.usage += ' CONFIG ...' 1336 1337 (options, configs) = parser.parse_args() 1338 1339 if len(configs) == 0 and not options.force_sync: 1340 parser.print_usage() 1341 sys.exit(1) 1342 1343 # prefix the option name with CONFIG_ if missing 1344 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1345 for config in configs ] 1346 1347 check_top_directory() 1348 1349 if not options.cleanup_headers_only: 1350 check_clean_directory() 1351 update_cross_compile(options.color) 1352 move_config(configs, options) 1353 1354 if configs: 1355 cleanup_headers(configs, options) 1356 cleanup_extra_options(configs, options) 1357 cleanup_whitelist(configs, options) 1358 cleanup_readme(configs, options) 1359 1360 if options.commit: 1361 subprocess.call(['git', 'add', '-u']) 1362 if configs: 1363 msg = 'Convert %s %sto Kconfig' % (configs[0], 1364 'et al ' if len(configs) > 1 else '') 1365 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1366 '\n '.join(configs)) 1367 else: 1368 msg = 'configs: Resync with savedefconfig' 1369 msg += '\n\nRsync all defconfig files using moveconfig.py' 1370 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1371 1372if __name__ == '__main__': 1373 main() 1374