1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. 111 112The default per-arch CROSS_COMPILE used by this tool is specified by 113the list below, CROSS_COMPILE. You may wish to update the list to 114use your own. Instead of modifying the list directly, you can give 115them via environments. 116 117 118Tips and trips 119-------------- 120 121To sync only X86 defconfigs: 122 123 ./tools/moveconfig.py -s -d <(grep -l X86 configs/*) 124 125or: 126 127 grep -l X86 configs/* | ./tools/moveconfig.py -s -d - 128 129To process CONFIG_CMD_FPGAD only for a subset of configs based on path match: 130 131 ls configs/{hrcon*,iocon*,strider*} | \ 132 ./tools/moveconfig.py -Cy CONFIG_CMD_FPGAD -d - 133 134 135Finding implied CONFIGs 136----------------------- 137 138Some CONFIG options can be implied by others and this can help to reduce 139the size of the defconfig files. For example, CONFIG_X86 implies 140CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 141all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 142each of the x86 defconfig files. 143 144This tool can help find such configs. To use it, first build a database: 145 146 ./tools/moveconfig.py -b 147 148Then try to query it: 149 150 ./tools/moveconfig.py -i CONFIG_CMD_IRQ 151 CONFIG_CMD_IRQ found in 311/2384 defconfigs 152 44 : CONFIG_SYS_FSL_ERRATUM_IFC_A002769 153 41 : CONFIG_SYS_FSL_ERRATUM_A007075 154 31 : CONFIG_SYS_FSL_DDR_VER_44 155 28 : CONFIG_ARCH_P1010 156 28 : CONFIG_SYS_FSL_ERRATUM_P1010_A003549 157 28 : CONFIG_SYS_FSL_ERRATUM_SEC_A003571 158 28 : CONFIG_SYS_FSL_ERRATUM_IFC_A003399 159 25 : CONFIG_SYS_FSL_ERRATUM_A008044 160 22 : CONFIG_ARCH_P1020 161 21 : CONFIG_SYS_FSL_DDR_VER_46 162 20 : CONFIG_MAX_PIRQ_LINKS 163 20 : CONFIG_HPET_ADDRESS 164 20 : CONFIG_X86 165 20 : CONFIG_PCIE_ECAM_SIZE 166 20 : CONFIG_IRQ_SLOT_COUNT 167 20 : CONFIG_I8259_PIC 168 20 : CONFIG_CPU_ADDR_BITS 169 20 : CONFIG_RAMBASE 170 20 : CONFIG_SYS_FSL_ERRATUM_A005871 171 20 : CONFIG_PCIE_ECAM_BASE 172 20 : CONFIG_X86_TSC_TIMER 173 20 : CONFIG_I8254_TIMER 174 20 : CONFIG_CMD_GETTIME 175 19 : CONFIG_SYS_FSL_ERRATUM_A005812 176 18 : CONFIG_X86_RUN_32BIT 177 17 : CONFIG_CMD_CHIP_CONFIG 178 ... 179 180This shows a list of config options which might imply CONFIG_CMD_EEPROM along 181with how many defconfigs they cover. From this you can see that CONFIG_X86 182implies CONFIG_CMD_EEPROM. Therefore, instead of adding CONFIG_CMD_EEPROM to 183the defconfig of every x86 board, you could add a single imply line to the 184Kconfig file: 185 186 config X86 187 bool "x86 architecture" 188 ... 189 imply CMD_EEPROM 190 191That will cover 20 defconfigs. Many of the options listed are not suitable as 192they are not related. E.g. it would be odd for CONFIG_CMD_GETTIME to imply 193CMD_EEPROM. 194 195Using this search you can reduce the size of moveconfig patches. 196 197 198Available options 199----------------- 200 201 -c, --color 202 Surround each portion of the log with escape sequences to display it 203 in color on the terminal. 204 205 -C, --commit 206 Create a git commit with the changes when the operation is complete. A 207 standard commit message is used which may need to be edited. 208 209 -d, --defconfigs 210 Specify a file containing a list of defconfigs to move. The defconfig 211 files can be given with shell-style wildcards. Use '-' to read from stdin. 212 213 -n, --dry-run 214 Perform a trial run that does not make any changes. It is useful to 215 see what is going to happen before one actually runs it. 216 217 -e, --exit-on-error 218 Exit immediately if Make exits with a non-zero status while processing 219 a defconfig file. 220 221 -s, --force-sync 222 Do "make savedefconfig" forcibly for all the defconfig files. 223 If not specified, "make savedefconfig" only occurs for cases 224 where at least one CONFIG was moved. 225 226 -S, --spl 227 Look for moved config options in spl/include/autoconf.mk instead of 228 include/autoconf.mk. This is useful for moving options for SPL build 229 because SPL related options (mostly prefixed with CONFIG_SPL_) are 230 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 231 232 -H, --headers-only 233 Only cleanup the headers; skip the defconfig processing 234 235 -j, --jobs 236 Specify the number of threads to run simultaneously. If not specified, 237 the number of threads is the same as the number of CPU cores. 238 239 -r, --git-ref 240 Specify the git ref to clone for building the autoconf.mk. If unspecified 241 use the CWD. This is useful for when changes to the Kconfig affect the 242 default values and you want to capture the state of the defconfig from 243 before that change was in effect. If in doubt, specify a ref pre-Kconfig 244 changes (use HEAD if Kconfig changes are not committed). Worst case it will 245 take a bit longer to run, but will always do the right thing. 246 247 -v, --verbose 248 Show any build errors as boards are built 249 250 -y, --yes 251 Instead of prompting, automatically go ahead with all operations. This 252 includes cleaning up headers, CONFIG_SYS_EXTRA_OPTIONS, the config whitelist 253 and the README. 254 255To see the complete list of supported options, run 256 257 $ tools/moveconfig.py -h 258 259""" 260 261import collections 262import copy 263import difflib 264import filecmp 265import fnmatch 266import glob 267import multiprocessing 268import optparse 269import os 270import Queue 271import re 272import shutil 273import subprocess 274import sys 275import tempfile 276import threading 277import time 278 279SHOW_GNU_MAKE = 'scripts/show-gnu-make' 280SLEEP_TIME=0.03 281 282# Here is the list of cross-tools I use. 283# Most of them are available at kernel.org 284# (https://www.kernel.org/pub/tools/crosstool/files/bin/), except the following: 285# arc: https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases 286# nds32: http://osdk.andestech.com/packages/nds32le-linux-glibc-v1.tgz 287# nios2: https://sourcery.mentor.com/GNUToolchain/subscription42545 288# sh: http://sourcery.mentor.com/public/gnu_toolchain/sh-linux-gnu 289CROSS_COMPILE = { 290 'arc': 'arc-linux-', 291 'aarch64': 'aarch64-linux-', 292 'arm': 'arm-unknown-linux-gnueabi-', 293 'm68k': 'm68k-linux-', 294 'microblaze': 'microblaze-linux-', 295 'mips': 'mips-linux-', 296 'nds32': 'nds32le-linux-', 297 'nios2': 'nios2-linux-gnu-', 298 'powerpc': 'powerpc-linux-', 299 'sh': 'sh-linux-gnu-', 300 'x86': 'i386-linux-', 301 'xtensa': 'xtensa-linux-' 302} 303 304STATE_IDLE = 0 305STATE_DEFCONFIG = 1 306STATE_AUTOCONF = 2 307STATE_SAVEDEFCONFIG = 3 308 309ACTION_MOVE = 0 310ACTION_NO_ENTRY = 1 311ACTION_NO_ENTRY_WARN = 2 312ACTION_NO_CHANGE = 3 313 314COLOR_BLACK = '0;30' 315COLOR_RED = '0;31' 316COLOR_GREEN = '0;32' 317COLOR_BROWN = '0;33' 318COLOR_BLUE = '0;34' 319COLOR_PURPLE = '0;35' 320COLOR_CYAN = '0;36' 321COLOR_LIGHT_GRAY = '0;37' 322COLOR_DARK_GRAY = '1;30' 323COLOR_LIGHT_RED = '1;31' 324COLOR_LIGHT_GREEN = '1;32' 325COLOR_YELLOW = '1;33' 326COLOR_LIGHT_BLUE = '1;34' 327COLOR_LIGHT_PURPLE = '1;35' 328COLOR_LIGHT_CYAN = '1;36' 329COLOR_WHITE = '1;37' 330 331AUTO_CONF_PATH = 'include/config/auto.conf' 332CONFIG_DATABASE = 'moveconfig.db' 333 334 335### helper functions ### 336def get_devnull(): 337 """Get the file object of '/dev/null' device.""" 338 try: 339 devnull = subprocess.DEVNULL # py3k 340 except AttributeError: 341 devnull = open(os.devnull, 'wb') 342 return devnull 343 344def check_top_directory(): 345 """Exit if we are not at the top of source directory.""" 346 for f in ('README', 'Licenses'): 347 if not os.path.exists(f): 348 sys.exit('Please run at the top of source directory.') 349 350def check_clean_directory(): 351 """Exit if the source tree is not clean.""" 352 for f in ('.config', 'include/config'): 353 if os.path.exists(f): 354 sys.exit("source tree is not clean, please run 'make mrproper'") 355 356def get_make_cmd(): 357 """Get the command name of GNU Make. 358 359 U-Boot needs GNU Make for building, but the command name is not 360 necessarily "make". (for example, "gmake" on FreeBSD). 361 Returns the most appropriate command name on your system. 362 """ 363 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 364 ret = process.communicate() 365 if process.returncode: 366 sys.exit('GNU Make not found') 367 return ret[0].rstrip() 368 369def get_matched_defconfig(line): 370 """Get the defconfig files that match a pattern 371 372 Args: 373 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 374 'k2*_defconfig'. If no directory is provided, 'configs/' is 375 prepended 376 377 Returns: 378 a list of matching defconfig files 379 """ 380 dirname = os.path.dirname(line) 381 if dirname: 382 pattern = line 383 else: 384 pattern = os.path.join('configs', line) 385 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 386 387def get_matched_defconfigs(defconfigs_file): 388 """Get all the defconfig files that match the patterns in a file. 389 390 Args: 391 defconfigs_file: File containing a list of defconfigs to process, or 392 '-' to read the list from stdin 393 394 Returns: 395 A list of paths to defconfig files, with no duplicates 396 """ 397 defconfigs = [] 398 if defconfigs_file == '-': 399 fd = sys.stdin 400 defconfigs_file = 'stdin' 401 else: 402 fd = open(defconfigs_file) 403 for i, line in enumerate(fd): 404 line = line.strip() 405 if not line: 406 continue # skip blank lines silently 407 matched = get_matched_defconfig(line) 408 if not matched: 409 print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \ 410 (defconfigs_file, i + 1, line) 411 412 defconfigs += matched 413 414 # use set() to drop multiple matching 415 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 416 417def get_all_defconfigs(): 418 """Get all the defconfig files under the configs/ directory.""" 419 defconfigs = [] 420 for (dirpath, dirnames, filenames) in os.walk('configs'): 421 dirpath = dirpath[len('configs') + 1:] 422 for filename in fnmatch.filter(filenames, '*_defconfig'): 423 defconfigs.append(os.path.join(dirpath, filename)) 424 425 return defconfigs 426 427def color_text(color_enabled, color, string): 428 """Return colored string.""" 429 if color_enabled: 430 # LF should not be surrounded by the escape sequence. 431 # Otherwise, additional whitespace or line-feed might be printed. 432 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 433 for s in string.split('\n') ]) 434 else: 435 return string 436 437def show_diff(a, b, file_path, color_enabled): 438 """Show unidified diff. 439 440 Arguments: 441 a: A list of lines (before) 442 b: A list of lines (after) 443 file_path: Path to the file 444 color_enabled: Display the diff in color 445 """ 446 447 diff = difflib.unified_diff(a, b, 448 fromfile=os.path.join('a', file_path), 449 tofile=os.path.join('b', file_path)) 450 451 for line in diff: 452 if line[0] == '-' and line[1] != '-': 453 print color_text(color_enabled, COLOR_RED, line), 454 elif line[0] == '+' and line[1] != '+': 455 print color_text(color_enabled, COLOR_GREEN, line), 456 else: 457 print line, 458 459def update_cross_compile(color_enabled): 460 """Update per-arch CROSS_COMPILE via environment variables 461 462 The default CROSS_COMPILE values are available 463 in the CROSS_COMPILE list above. 464 465 You can override them via environment variables 466 CROSS_COMPILE_{ARCH}. 467 468 For example, if you want to override toolchain prefixes 469 for ARM and PowerPC, you can do as follows in your shell: 470 471 export CROSS_COMPILE_ARM=... 472 export CROSS_COMPILE_POWERPC=... 473 474 Then, this function checks if specified compilers really exist in your 475 PATH environment. 476 """ 477 archs = [] 478 479 for arch in os.listdir('arch'): 480 if os.path.exists(os.path.join('arch', arch, 'Makefile')): 481 archs.append(arch) 482 483 # arm64 is a special case 484 archs.append('aarch64') 485 486 for arch in archs: 487 env = 'CROSS_COMPILE_' + arch.upper() 488 cross_compile = os.environ.get(env) 489 if not cross_compile: 490 cross_compile = CROSS_COMPILE.get(arch, '') 491 492 for path in os.environ["PATH"].split(os.pathsep): 493 gcc_path = os.path.join(path, cross_compile + 'gcc') 494 if os.path.isfile(gcc_path) and os.access(gcc_path, os.X_OK): 495 break 496 else: 497 print >> sys.stderr, color_text(color_enabled, COLOR_YELLOW, 498 'warning: %sgcc: not found in PATH. %s architecture boards will be skipped' 499 % (cross_compile, arch)) 500 cross_compile = None 501 502 CROSS_COMPILE[arch] = cross_compile 503 504def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 505 extend_post): 506 """Extend matched lines if desired patterns are found before/after already 507 matched lines. 508 509 Arguments: 510 lines: A list of lines handled. 511 matched: A list of line numbers that have been already matched. 512 (will be updated by this function) 513 pre_patterns: A list of regular expression that should be matched as 514 preamble. 515 post_patterns: A list of regular expression that should be matched as 516 postamble. 517 extend_pre: Add the line number of matched preamble to the matched list. 518 extend_post: Add the line number of matched postamble to the matched list. 519 """ 520 extended_matched = [] 521 522 j = matched[0] 523 524 for i in matched: 525 if i == 0 or i < j: 526 continue 527 j = i 528 while j in matched: 529 j += 1 530 if j >= len(lines): 531 break 532 533 for p in pre_patterns: 534 if p.search(lines[i - 1]): 535 break 536 else: 537 # not matched 538 continue 539 540 for p in post_patterns: 541 if p.search(lines[j]): 542 break 543 else: 544 # not matched 545 continue 546 547 if extend_pre: 548 extended_matched.append(i - 1) 549 if extend_post: 550 extended_matched.append(j) 551 552 matched += extended_matched 553 matched.sort() 554 555def confirm(options, prompt): 556 if not options.yes: 557 while True: 558 choice = raw_input('{} [y/n]: '.format(prompt)) 559 choice = choice.lower() 560 print choice 561 if choice == 'y' or choice == 'n': 562 break 563 564 if choice == 'n': 565 return False 566 567 return True 568 569def cleanup_one_header(header_path, patterns, options): 570 """Clean regex-matched lines away from a file. 571 572 Arguments: 573 header_path: path to the cleaned file. 574 patterns: list of regex patterns. Any lines matching to these 575 patterns are deleted. 576 options: option flags. 577 """ 578 with open(header_path) as f: 579 lines = f.readlines() 580 581 matched = [] 582 for i, line in enumerate(lines): 583 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 584 matched.append(i) 585 continue 586 for pattern in patterns: 587 if pattern.search(line): 588 matched.append(i) 589 break 590 591 if not matched: 592 return 593 594 # remove empty #ifdef ... #endif, successive blank lines 595 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 596 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 597 pattern_endif = re.compile(r'#\s*endif\W') # #endif 598 pattern_blank = re.compile(r'^\s*$') # empty line 599 600 while True: 601 old_matched = copy.copy(matched) 602 extend_matched_lines(lines, matched, [pattern_if], 603 [pattern_endif], True, True) 604 extend_matched_lines(lines, matched, [pattern_elif], 605 [pattern_elif, pattern_endif], True, False) 606 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 607 [pattern_blank], False, True) 608 extend_matched_lines(lines, matched, [pattern_blank], 609 [pattern_elif, pattern_endif], True, False) 610 extend_matched_lines(lines, matched, [pattern_blank], 611 [pattern_blank], True, False) 612 if matched == old_matched: 613 break 614 615 tolines = copy.copy(lines) 616 617 for i in reversed(matched): 618 tolines.pop(i) 619 620 show_diff(lines, tolines, header_path, options.color) 621 622 if options.dry_run: 623 return 624 625 with open(header_path, 'w') as f: 626 for line in tolines: 627 f.write(line) 628 629def cleanup_headers(configs, options): 630 """Delete config defines from board headers. 631 632 Arguments: 633 configs: A list of CONFIGs to remove. 634 options: option flags. 635 """ 636 if not confirm(options, 'Clean up headers?'): 637 return 638 639 patterns = [] 640 for config in configs: 641 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 642 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 643 644 for dir in 'include', 'arch', 'board': 645 for (dirpath, dirnames, filenames) in os.walk(dir): 646 if dirpath == os.path.join('include', 'generated'): 647 continue 648 for filename in filenames: 649 if not fnmatch.fnmatch(filename, '*~'): 650 cleanup_one_header(os.path.join(dirpath, filename), 651 patterns, options) 652 653def cleanup_one_extra_option(defconfig_path, configs, options): 654 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 655 656 Arguments: 657 defconfig_path: path to the cleaned defconfig file. 658 configs: A list of CONFIGs to remove. 659 options: option flags. 660 """ 661 662 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 663 end = '"\n' 664 665 with open(defconfig_path) as f: 666 lines = f.readlines() 667 668 for i, line in enumerate(lines): 669 if line.startswith(start) and line.endswith(end): 670 break 671 else: 672 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 673 return 674 675 old_tokens = line[len(start):-len(end)].split(',') 676 new_tokens = [] 677 678 for token in old_tokens: 679 pos = token.find('=') 680 if not (token[:pos] if pos >= 0 else token) in configs: 681 new_tokens.append(token) 682 683 if new_tokens == old_tokens: 684 return 685 686 tolines = copy.copy(lines) 687 688 if new_tokens: 689 tolines[i] = start + ','.join(new_tokens) + end 690 else: 691 tolines.pop(i) 692 693 show_diff(lines, tolines, defconfig_path, options.color) 694 695 if options.dry_run: 696 return 697 698 with open(defconfig_path, 'w') as f: 699 for line in tolines: 700 f.write(line) 701 702def cleanup_extra_options(configs, options): 703 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 704 705 Arguments: 706 configs: A list of CONFIGs to remove. 707 options: option flags. 708 """ 709 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 710 return 711 712 configs = [ config[len('CONFIG_'):] for config in configs ] 713 714 defconfigs = get_all_defconfigs() 715 716 for defconfig in defconfigs: 717 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 718 options) 719 720def cleanup_whitelist(configs, options): 721 """Delete config whitelist entries 722 723 Arguments: 724 configs: A list of CONFIGs to remove. 725 options: option flags. 726 """ 727 if not confirm(options, 'Clean up whitelist entries?'): 728 return 729 730 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 731 lines = f.readlines() 732 733 lines = [x for x in lines if x.strip() not in configs] 734 735 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 736 f.write(''.join(lines)) 737 738def find_matching(patterns, line): 739 for pat in patterns: 740 if pat.search(line): 741 return True 742 return False 743 744def cleanup_readme(configs, options): 745 """Delete config description in README 746 747 Arguments: 748 configs: A list of CONFIGs to remove. 749 options: option flags. 750 """ 751 if not confirm(options, 'Clean up README?'): 752 return 753 754 patterns = [] 755 for config in configs: 756 patterns.append(re.compile(r'^\s+%s' % config)) 757 758 with open('README') as f: 759 lines = f.readlines() 760 761 found = False 762 newlines = [] 763 for line in lines: 764 if not found: 765 found = find_matching(patterns, line) 766 if found: 767 continue 768 769 if found and re.search(r'^\s+CONFIG', line): 770 found = False 771 772 if not found: 773 newlines.append(line) 774 775 with open('README', 'w') as f: 776 f.write(''.join(newlines)) 777 778 779### classes ### 780class Progress: 781 782 """Progress Indicator""" 783 784 def __init__(self, total): 785 """Create a new progress indicator. 786 787 Arguments: 788 total: A number of defconfig files to process. 789 """ 790 self.current = 0 791 self.total = total 792 793 def inc(self): 794 """Increment the number of processed defconfig files.""" 795 796 self.current += 1 797 798 def show(self): 799 """Display the progress.""" 800 print ' %d defconfigs out of %d\r' % (self.current, self.total), 801 sys.stdout.flush() 802 803class KconfigParser: 804 805 """A parser of .config and include/autoconf.mk.""" 806 807 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 808 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 809 810 def __init__(self, configs, options, build_dir): 811 """Create a new parser. 812 813 Arguments: 814 configs: A list of CONFIGs to move. 815 options: option flags. 816 build_dir: Build directory. 817 """ 818 self.configs = configs 819 self.options = options 820 self.dotconfig = os.path.join(build_dir, '.config') 821 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 822 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 823 'autoconf.mk') 824 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 825 self.defconfig = os.path.join(build_dir, 'defconfig') 826 827 def get_cross_compile(self): 828 """Parse .config file and return CROSS_COMPILE. 829 830 Returns: 831 A string storing the compiler prefix for the architecture. 832 Return a NULL string for architectures that do not require 833 compiler prefix (Sandbox and native build is the case). 834 Return None if the specified compiler is missing in your PATH. 835 Caller should distinguish '' and None. 836 """ 837 arch = '' 838 cpu = '' 839 for line in open(self.dotconfig): 840 m = self.re_arch.match(line) 841 if m: 842 arch = m.group(1) 843 continue 844 m = self.re_cpu.match(line) 845 if m: 846 cpu = m.group(1) 847 848 if not arch: 849 return None 850 851 # fix-up for aarch64 852 if arch == 'arm' and cpu == 'armv8': 853 arch = 'aarch64' 854 855 return CROSS_COMPILE.get(arch, None) 856 857 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 858 """Parse .config, defconfig, include/autoconf.mk for one config. 859 860 This function looks for the config options in the lines from 861 defconfig, .config, and include/autoconf.mk in order to decide 862 which action should be taken for this defconfig. 863 864 Arguments: 865 config: CONFIG name to parse. 866 dotconfig_lines: lines from the .config file. 867 autoconf_lines: lines from the include/autoconf.mk file. 868 869 Returns: 870 A tupple of the action for this defconfig and the line 871 matched for the config. 872 """ 873 not_set = '# %s is not set' % config 874 875 for line in autoconf_lines: 876 line = line.rstrip() 877 if line.startswith(config + '='): 878 new_val = line 879 break 880 else: 881 new_val = not_set 882 883 for line in dotconfig_lines: 884 line = line.rstrip() 885 if line.startswith(config + '=') or line == not_set: 886 old_val = line 887 break 888 else: 889 if new_val == not_set: 890 return (ACTION_NO_ENTRY, config) 891 else: 892 return (ACTION_NO_ENTRY_WARN, config) 893 894 # If this CONFIG is neither bool nor trisate 895 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 896 # tools/scripts/define2mk.sed changes '1' to 'y'. 897 # This is a problem if the CONFIG is int type. 898 # Check the type in Kconfig and handle it correctly. 899 if new_val[-2:] == '=y': 900 new_val = new_val[:-1] + '1' 901 902 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 903 new_val) 904 905 def update_dotconfig(self): 906 """Parse files for the config options and update the .config. 907 908 This function parses the generated .config and include/autoconf.mk 909 searching the target options. 910 Move the config option(s) to the .config as needed. 911 912 Arguments: 913 defconfig: defconfig name. 914 915 Returns: 916 Return a tuple of (updated flag, log string). 917 The "updated flag" is True if the .config was updated, False 918 otherwise. The "log string" shows what happend to the .config. 919 """ 920 921 results = [] 922 updated = False 923 suspicious = False 924 rm_files = [self.config_autoconf, self.autoconf] 925 926 if self.options.spl: 927 if os.path.exists(self.spl_autoconf): 928 autoconf_path = self.spl_autoconf 929 rm_files.append(self.spl_autoconf) 930 else: 931 for f in rm_files: 932 os.remove(f) 933 return (updated, suspicious, 934 color_text(self.options.color, COLOR_BROWN, 935 "SPL is not enabled. Skipped.") + '\n') 936 else: 937 autoconf_path = self.autoconf 938 939 with open(self.dotconfig) as f: 940 dotconfig_lines = f.readlines() 941 942 with open(autoconf_path) as f: 943 autoconf_lines = f.readlines() 944 945 for config in self.configs: 946 result = self.parse_one_config(config, dotconfig_lines, 947 autoconf_lines) 948 results.append(result) 949 950 log = '' 951 952 for (action, value) in results: 953 if action == ACTION_MOVE: 954 actlog = "Move '%s'" % value 955 log_color = COLOR_LIGHT_GREEN 956 elif action == ACTION_NO_ENTRY: 957 actlog = "%s is not defined in Kconfig. Do nothing." % value 958 log_color = COLOR_LIGHT_BLUE 959 elif action == ACTION_NO_ENTRY_WARN: 960 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 961 log_color = COLOR_YELLOW 962 suspicious = True 963 elif action == ACTION_NO_CHANGE: 964 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 965 % value 966 log_color = COLOR_LIGHT_PURPLE 967 elif action == ACTION_SPL_NOT_EXIST: 968 actlog = "SPL is not enabled for this defconfig. Skip." 969 log_color = COLOR_PURPLE 970 else: 971 sys.exit("Internal Error. This should not happen.") 972 973 log += color_text(self.options.color, log_color, actlog) + '\n' 974 975 with open(self.dotconfig, 'a') as f: 976 for (action, value) in results: 977 if action == ACTION_MOVE: 978 f.write(value + '\n') 979 updated = True 980 981 self.results = results 982 for f in rm_files: 983 os.remove(f) 984 985 return (updated, suspicious, log) 986 987 def check_defconfig(self): 988 """Check the defconfig after savedefconfig 989 990 Returns: 991 Return additional log if moved CONFIGs were removed again by 992 'make savedefconfig'. 993 """ 994 995 log = '' 996 997 with open(self.defconfig) as f: 998 defconfig_lines = f.readlines() 999 1000 for (action, value) in self.results: 1001 if action != ACTION_MOVE: 1002 continue 1003 if not value + '\n' in defconfig_lines: 1004 log += color_text(self.options.color, COLOR_YELLOW, 1005 "'%s' was removed by savedefconfig.\n" % 1006 value) 1007 1008 return log 1009 1010 1011class DatabaseThread(threading.Thread): 1012 """This thread processes results from Slot threads. 1013 1014 It collects the data in the master config directary. There is only one 1015 result thread, and this helps to serialise the build output. 1016 """ 1017 def __init__(self, config_db, db_queue): 1018 """Set up a new result thread 1019 1020 Args: 1021 builder: Builder which will be sent each result 1022 """ 1023 threading.Thread.__init__(self) 1024 self.config_db = config_db 1025 self.db_queue= db_queue 1026 1027 def run(self): 1028 """Called to start up the result thread. 1029 1030 We collect the next result job and pass it on to the build. 1031 """ 1032 while True: 1033 defconfig, configs = self.db_queue.get() 1034 self.config_db[defconfig] = configs 1035 self.db_queue.task_done() 1036 1037 1038class Slot: 1039 1040 """A slot to store a subprocess. 1041 1042 Each instance of this class handles one subprocess. 1043 This class is useful to control multiple threads 1044 for faster processing. 1045 """ 1046 1047 def __init__(self, configs, options, progress, devnull, make_cmd, 1048 reference_src_dir, db_queue): 1049 """Create a new process slot. 1050 1051 Arguments: 1052 configs: A list of CONFIGs to move. 1053 options: option flags. 1054 progress: A progress indicator. 1055 devnull: A file object of '/dev/null'. 1056 make_cmd: command name of GNU Make. 1057 reference_src_dir: Determine the true starting config state from this 1058 source tree. 1059 db_queue: output queue to write config info for the database 1060 """ 1061 self.options = options 1062 self.progress = progress 1063 self.build_dir = tempfile.mkdtemp() 1064 self.devnull = devnull 1065 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 1066 self.reference_src_dir = reference_src_dir 1067 self.db_queue = db_queue 1068 self.parser = KconfigParser(configs, options, self.build_dir) 1069 self.state = STATE_IDLE 1070 self.failed_boards = set() 1071 self.suspicious_boards = set() 1072 1073 def __del__(self): 1074 """Delete the working directory 1075 1076 This function makes sure the temporary directory is cleaned away 1077 even if Python suddenly dies due to error. It should be done in here 1078 because it is guaranteed the destructor is always invoked when the 1079 instance of the class gets unreferenced. 1080 1081 If the subprocess is still running, wait until it finishes. 1082 """ 1083 if self.state != STATE_IDLE: 1084 while self.ps.poll() == None: 1085 pass 1086 shutil.rmtree(self.build_dir) 1087 1088 def add(self, defconfig): 1089 """Assign a new subprocess for defconfig and add it to the slot. 1090 1091 If the slot is vacant, create a new subprocess for processing the 1092 given defconfig and add it to the slot. Just returns False if 1093 the slot is occupied (i.e. the current subprocess is still running). 1094 1095 Arguments: 1096 defconfig: defconfig name. 1097 1098 Returns: 1099 Return True on success or False on failure 1100 """ 1101 if self.state != STATE_IDLE: 1102 return False 1103 1104 self.defconfig = defconfig 1105 self.log = '' 1106 self.current_src_dir = self.reference_src_dir 1107 self.do_defconfig() 1108 return True 1109 1110 def poll(self): 1111 """Check the status of the subprocess and handle it as needed. 1112 1113 Returns True if the slot is vacant (i.e. in idle state). 1114 If the configuration is successfully finished, assign a new 1115 subprocess to build include/autoconf.mk. 1116 If include/autoconf.mk is generated, invoke the parser to 1117 parse the .config and the include/autoconf.mk, moving 1118 config options to the .config as needed. 1119 If the .config was updated, run "make savedefconfig" to sync 1120 it, update the original defconfig, and then set the slot back 1121 to the idle state. 1122 1123 Returns: 1124 Return True if the subprocess is terminated, False otherwise 1125 """ 1126 if self.state == STATE_IDLE: 1127 return True 1128 1129 if self.ps.poll() == None: 1130 return False 1131 1132 if self.ps.poll() != 0: 1133 self.handle_error() 1134 elif self.state == STATE_DEFCONFIG: 1135 if self.reference_src_dir and not self.current_src_dir: 1136 self.do_savedefconfig() 1137 else: 1138 self.do_autoconf() 1139 elif self.state == STATE_AUTOCONF: 1140 if self.current_src_dir: 1141 self.current_src_dir = None 1142 self.do_defconfig() 1143 elif self.options.build_db: 1144 self.do_build_db() 1145 else: 1146 self.do_savedefconfig() 1147 elif self.state == STATE_SAVEDEFCONFIG: 1148 self.update_defconfig() 1149 else: 1150 sys.exit("Internal Error. This should not happen.") 1151 1152 return True if self.state == STATE_IDLE else False 1153 1154 def handle_error(self): 1155 """Handle error cases.""" 1156 1157 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1158 "Failed to process.\n") 1159 if self.options.verbose: 1160 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1161 self.ps.stderr.read()) 1162 self.finish(False) 1163 1164 def do_defconfig(self): 1165 """Run 'make <board>_defconfig' to create the .config file.""" 1166 1167 cmd = list(self.make_cmd) 1168 cmd.append(self.defconfig) 1169 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1170 stderr=subprocess.PIPE, 1171 cwd=self.current_src_dir) 1172 self.state = STATE_DEFCONFIG 1173 1174 def do_autoconf(self): 1175 """Run 'make AUTO_CONF_PATH'.""" 1176 1177 self.cross_compile = self.parser.get_cross_compile() 1178 if self.cross_compile is None: 1179 self.log += color_text(self.options.color, COLOR_YELLOW, 1180 "Compiler is missing. Do nothing.\n") 1181 self.finish(False) 1182 return 1183 1184 cmd = list(self.make_cmd) 1185 if self.cross_compile: 1186 cmd.append('CROSS_COMPILE=%s' % self.cross_compile) 1187 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1188 cmd.append(AUTO_CONF_PATH) 1189 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1190 stderr=subprocess.PIPE, 1191 cwd=self.current_src_dir) 1192 self.state = STATE_AUTOCONF 1193 1194 def do_build_db(self): 1195 """Add the board to the database""" 1196 configs = {} 1197 with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: 1198 for line in fd.readlines(): 1199 if line.startswith('CONFIG'): 1200 config, value = line.split('=', 1) 1201 configs[config] = value.rstrip() 1202 self.db_queue.put([self.defconfig, configs]) 1203 self.finish(True) 1204 1205 def do_savedefconfig(self): 1206 """Update the .config and run 'make savedefconfig'.""" 1207 1208 (updated, suspicious, log) = self.parser.update_dotconfig() 1209 if suspicious: 1210 self.suspicious_boards.add(self.defconfig) 1211 self.log += log 1212 1213 if not self.options.force_sync and not updated: 1214 self.finish(True) 1215 return 1216 if updated: 1217 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1218 "Syncing by savedefconfig...\n") 1219 else: 1220 self.log += "Syncing by savedefconfig (forced by option)...\n" 1221 1222 cmd = list(self.make_cmd) 1223 cmd.append('savedefconfig') 1224 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1225 stderr=subprocess.PIPE) 1226 self.state = STATE_SAVEDEFCONFIG 1227 1228 def update_defconfig(self): 1229 """Update the input defconfig and go back to the idle state.""" 1230 1231 log = self.parser.check_defconfig() 1232 if log: 1233 self.suspicious_boards.add(self.defconfig) 1234 self.log += log 1235 orig_defconfig = os.path.join('configs', self.defconfig) 1236 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1237 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1238 1239 if updated: 1240 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1241 "defconfig was updated.\n") 1242 1243 if not self.options.dry_run and updated: 1244 shutil.move(new_defconfig, orig_defconfig) 1245 self.finish(True) 1246 1247 def finish(self, success): 1248 """Display log along with progress and go to the idle state. 1249 1250 Arguments: 1251 success: Should be True when the defconfig was processed 1252 successfully, or False when it fails. 1253 """ 1254 # output at least 30 characters to hide the "* defconfigs out of *". 1255 log = self.defconfig.ljust(30) + '\n' 1256 1257 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1258 # Some threads are running in parallel. 1259 # Print log atomically to not mix up logs from different threads. 1260 print >> (sys.stdout if success else sys.stderr), log 1261 1262 if not success: 1263 if self.options.exit_on_error: 1264 sys.exit("Exit on error.") 1265 # If --exit-on-error flag is not set, skip this board and continue. 1266 # Record the failed board. 1267 self.failed_boards.add(self.defconfig) 1268 1269 self.progress.inc() 1270 self.progress.show() 1271 self.state = STATE_IDLE 1272 1273 def get_failed_boards(self): 1274 """Returns a set of failed boards (defconfigs) in this slot. 1275 """ 1276 return self.failed_boards 1277 1278 def get_suspicious_boards(self): 1279 """Returns a set of boards (defconfigs) with possible misconversion. 1280 """ 1281 return self.suspicious_boards - self.failed_boards 1282 1283class Slots: 1284 1285 """Controller of the array of subprocess slots.""" 1286 1287 def __init__(self, configs, options, progress, reference_src_dir, db_queue): 1288 """Create a new slots controller. 1289 1290 Arguments: 1291 configs: A list of CONFIGs to move. 1292 options: option flags. 1293 progress: A progress indicator. 1294 reference_src_dir: Determine the true starting config state from this 1295 source tree. 1296 db_queue: output queue to write config info for the database 1297 """ 1298 self.options = options 1299 self.slots = [] 1300 devnull = get_devnull() 1301 make_cmd = get_make_cmd() 1302 for i in range(options.jobs): 1303 self.slots.append(Slot(configs, options, progress, devnull, 1304 make_cmd, reference_src_dir, db_queue)) 1305 1306 def add(self, defconfig): 1307 """Add a new subprocess if a vacant slot is found. 1308 1309 Arguments: 1310 defconfig: defconfig name to be put into. 1311 1312 Returns: 1313 Return True on success or False on failure 1314 """ 1315 for slot in self.slots: 1316 if slot.add(defconfig): 1317 return True 1318 return False 1319 1320 def available(self): 1321 """Check if there is a vacant slot. 1322 1323 Returns: 1324 Return True if at lease one vacant slot is found, False otherwise. 1325 """ 1326 for slot in self.slots: 1327 if slot.poll(): 1328 return True 1329 return False 1330 1331 def empty(self): 1332 """Check if all slots are vacant. 1333 1334 Returns: 1335 Return True if all the slots are vacant, False otherwise. 1336 """ 1337 ret = True 1338 for slot in self.slots: 1339 if not slot.poll(): 1340 ret = False 1341 return ret 1342 1343 def show_failed_boards(self): 1344 """Display all of the failed boards (defconfigs).""" 1345 boards = set() 1346 output_file = 'moveconfig.failed' 1347 1348 for slot in self.slots: 1349 boards |= slot.get_failed_boards() 1350 1351 if boards: 1352 boards = '\n'.join(boards) + '\n' 1353 msg = "The following boards were not processed due to error:\n" 1354 msg += boards 1355 msg += "(the list has been saved in %s)\n" % output_file 1356 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1357 msg) 1358 1359 with open(output_file, 'w') as f: 1360 f.write(boards) 1361 1362 def show_suspicious_boards(self): 1363 """Display all boards (defconfigs) with possible misconversion.""" 1364 boards = set() 1365 output_file = 'moveconfig.suspicious' 1366 1367 for slot in self.slots: 1368 boards |= slot.get_suspicious_boards() 1369 1370 if boards: 1371 boards = '\n'.join(boards) + '\n' 1372 msg = "The following boards might have been converted incorrectly.\n" 1373 msg += "It is highly recommended to check them manually:\n" 1374 msg += boards 1375 msg += "(the list has been saved in %s)\n" % output_file 1376 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1377 msg) 1378 1379 with open(output_file, 'w') as f: 1380 f.write(boards) 1381 1382class ReferenceSource: 1383 1384 """Reference source against which original configs should be parsed.""" 1385 1386 def __init__(self, commit): 1387 """Create a reference source directory based on a specified commit. 1388 1389 Arguments: 1390 commit: commit to git-clone 1391 """ 1392 self.src_dir = tempfile.mkdtemp() 1393 print "Cloning git repo to a separate work directory..." 1394 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1395 cwd=self.src_dir) 1396 print "Checkout '%s' to build the original autoconf.mk." % \ 1397 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1398 subprocess.check_output(['git', 'checkout', commit], 1399 stderr=subprocess.STDOUT, cwd=self.src_dir) 1400 1401 def __del__(self): 1402 """Delete the reference source directory 1403 1404 This function makes sure the temporary directory is cleaned away 1405 even if Python suddenly dies due to error. It should be done in here 1406 because it is guaranteed the destructor is always invoked when the 1407 instance of the class gets unreferenced. 1408 """ 1409 shutil.rmtree(self.src_dir) 1410 1411 def get_dir(self): 1412 """Return the absolute path to the reference source directory.""" 1413 1414 return self.src_dir 1415 1416def move_config(configs, options, db_queue): 1417 """Move config options to defconfig files. 1418 1419 Arguments: 1420 configs: A list of CONFIGs to move. 1421 options: option flags 1422 """ 1423 if len(configs) == 0: 1424 if options.force_sync: 1425 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1426 elif options.build_db: 1427 print 'Building %s database' % CONFIG_DATABASE 1428 else: 1429 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1430 else: 1431 print 'Move ' + ', '.join(configs), 1432 print '(jobs: %d)\n' % options.jobs 1433 1434 if options.git_ref: 1435 reference_src = ReferenceSource(options.git_ref) 1436 reference_src_dir = reference_src.get_dir() 1437 else: 1438 reference_src_dir = None 1439 1440 if options.defconfigs: 1441 defconfigs = get_matched_defconfigs(options.defconfigs) 1442 else: 1443 defconfigs = get_all_defconfigs() 1444 1445 progress = Progress(len(defconfigs)) 1446 slots = Slots(configs, options, progress, reference_src_dir, db_queue) 1447 1448 # Main loop to process defconfig files: 1449 # Add a new subprocess into a vacant slot. 1450 # Sleep if there is no available slot. 1451 for defconfig in defconfigs: 1452 while not slots.add(defconfig): 1453 while not slots.available(): 1454 # No available slot: sleep for a while 1455 time.sleep(SLEEP_TIME) 1456 1457 # wait until all the subprocesses finish 1458 while not slots.empty(): 1459 time.sleep(SLEEP_TIME) 1460 1461 print '' 1462 slots.show_failed_boards() 1463 slots.show_suspicious_boards() 1464 1465def imply_config(config_list, find_superset=False): 1466 """Find CONFIG options which imply those in the list 1467 1468 Some CONFIG options can be implied by others and this can help to reduce 1469 the size of the defconfig files. For example, CONFIG_X86 implies 1470 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 1471 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 1472 each of the x86 defconfig files. 1473 1474 This function uses the moveconfig database to find such options. It 1475 displays a list of things that could possibly imply those in the list. 1476 The algorithm ignores any that start with CONFIG_TARGET since these 1477 typically refer to only a few defconfigs (often one). It also does not 1478 display a config with less than 5 defconfigs. 1479 1480 The algorithm works using sets. For each target config in config_list: 1481 - Get the set 'defconfigs' which use that target config 1482 - For each config (from a list of all configs): 1483 - Get the set 'imply_defconfig' of defconfigs which use that config 1484 - 1485 - If imply_defconfigs contains anything not in defconfigs then 1486 this config does not imply the target config 1487 1488 Params: 1489 config_list: List of CONFIG options to check (each a string) 1490 find_superset: True to look for configs which are a superset of those 1491 already found. So for example if CONFIG_EXYNOS5 implies an option, 1492 but CONFIG_EXYNOS covers a larger set of defconfigs and also 1493 implies that option, this will drop the former in favour of the 1494 latter. In practice this option has not proved very used. 1495 1496 Note the terminoloy: 1497 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 1498 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 1499 """ 1500 # key is defconfig name, value is dict of (CONFIG_xxx, value) 1501 config_db = {} 1502 1503 # Holds a dict containing the set of defconfigs that contain each config 1504 # key is config, value is set of defconfigs using that config 1505 defconfig_db = collections.defaultdict(set) 1506 1507 # Set of all config options we have seen 1508 all_configs = set() 1509 1510 # Set of all defconfigs we have seen 1511 all_defconfigs = set() 1512 1513 # Read in the database 1514 configs = {} 1515 with open(CONFIG_DATABASE) as fd: 1516 for line in fd.readlines(): 1517 line = line.rstrip() 1518 if not line: # Separator between defconfigs 1519 config_db[defconfig] = configs 1520 all_defconfigs.add(defconfig) 1521 configs = {} 1522 elif line[0] == ' ': # CONFIG line 1523 config, value = line.strip().split('=', 1) 1524 configs[config] = value 1525 defconfig_db[config].add(defconfig) 1526 all_configs.add(config) 1527 else: # New defconfig 1528 defconfig = line 1529 1530 # Work through each target config option in tern, independently 1531 for config in config_list: 1532 defconfigs = defconfig_db.get(config) 1533 if not defconfigs: 1534 print '%s not found in any defconfig' % config 1535 continue 1536 1537 # Get the set of defconfigs without this one (since a config cannot 1538 # imply itself) 1539 non_defconfigs = all_defconfigs - defconfigs 1540 num_defconfigs = len(defconfigs) 1541 print '%s found in %d/%d defconfigs' % (config, num_defconfigs, 1542 len(all_configs)) 1543 1544 # This will hold the results: key=config, value=defconfigs containing it 1545 imply_configs = {} 1546 rest_configs = all_configs - set([config]) 1547 1548 # Look at every possible config, except the target one 1549 for imply_config in rest_configs: 1550 if 'CONFIG_TARGET' in imply_config: 1551 continue 1552 1553 # Find set of defconfigs that have this config 1554 imply_defconfig = defconfig_db[imply_config] 1555 1556 # Get the intersection of this with defconfigs containing the 1557 # target config 1558 common_defconfigs = imply_defconfig & defconfigs 1559 1560 # Get the set of defconfigs containing this config which DO NOT 1561 # also contain the taret config. If this set is non-empty it means 1562 # that this config affects other defconfigs as well as (possibly) 1563 # the ones affected by the target config. This means it implies 1564 # things we don't want to imply. 1565 not_common_defconfigs = imply_defconfig & non_defconfigs 1566 if not_common_defconfigs: 1567 continue 1568 1569 # If there are common defconfigs, imply_config may be useful 1570 if common_defconfigs: 1571 skip = False 1572 if find_superset: 1573 for prev in imply_configs.keys(): 1574 prev_count = len(imply_configs[prev]) 1575 count = len(common_defconfigs) 1576 if (prev_count > count and 1577 (imply_configs[prev] & common_defconfigs == 1578 common_defconfigs)): 1579 # skip imply_config because prev is a superset 1580 skip = True 1581 break 1582 elif count > prev_count: 1583 # delete prev because imply_config is a superset 1584 del imply_configs[prev] 1585 if not skip: 1586 imply_configs[imply_config] = common_defconfigs 1587 1588 # Now we have a dict imply_configs of configs which imply each config 1589 # The value of each dict item is the set of defconfigs containing that 1590 # config. Rank them so that we print the configs that imply the largest 1591 # number of defconfigs first. 1592 ranked_configs = sorted(imply_configs, 1593 key=lambda k: len(imply_configs[k]), reverse=True) 1594 for config in ranked_configs: 1595 num_common = len(imply_configs[config]) 1596 1597 # Don't bother if there are less than 5 defconfigs affected. 1598 if num_common < 5: 1599 continue 1600 missing = defconfigs - imply_configs[config] 1601 missing_str = ', '.join(missing) if missing else 'all' 1602 missing_str = '' 1603 print ' %d : %-30s%s' % (num_common, config.ljust(30), 1604 missing_str) 1605 1606 1607def main(): 1608 try: 1609 cpu_count = multiprocessing.cpu_count() 1610 except NotImplementedError: 1611 cpu_count = 1 1612 1613 parser = optparse.OptionParser() 1614 # Add options here 1615 parser.add_option('-b', '--build-db', action='store_true', default=False, 1616 help='build a CONFIG database') 1617 parser.add_option('-c', '--color', action='store_true', default=False, 1618 help='display the log in color') 1619 parser.add_option('-C', '--commit', action='store_true', default=False, 1620 help='Create a git commit for the operation') 1621 parser.add_option('-d', '--defconfigs', type='string', 1622 help='a file containing a list of defconfigs to move, ' 1623 "one per line (for example 'snow_defconfig') " 1624 "or '-' to read from stdin") 1625 parser.add_option('-i', '--imply', action='store_true', default=False, 1626 help='find options which imply others') 1627 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1628 help='perform a trial run (show log with no changes)') 1629 parser.add_option('-e', '--exit-on-error', action='store_true', 1630 default=False, 1631 help='exit immediately on any error') 1632 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1633 help='force sync by savedefconfig') 1634 parser.add_option('-S', '--spl', action='store_true', default=False, 1635 help='parse config options defined for SPL build') 1636 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1637 action='store_true', default=False, 1638 help='only cleanup the headers') 1639 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1640 help='the number of jobs to run simultaneously') 1641 parser.add_option('-r', '--git-ref', type='string', 1642 help='the git ref to clone for building the autoconf.mk') 1643 parser.add_option('-y', '--yes', action='store_true', default=False, 1644 help="respond 'yes' to any prompts") 1645 parser.add_option('-v', '--verbose', action='store_true', default=False, 1646 help='show any build errors as boards are built') 1647 parser.usage += ' CONFIG ...' 1648 1649 (options, configs) = parser.parse_args() 1650 1651 if len(configs) == 0 and not any((options.force_sync, options.build_db, 1652 options.imply)): 1653 parser.print_usage() 1654 sys.exit(1) 1655 1656 # prefix the option name with CONFIG_ if missing 1657 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1658 for config in configs ] 1659 1660 check_top_directory() 1661 1662 if options.imply: 1663 imply_config(configs) 1664 return 1665 1666 config_db = {} 1667 db_queue = Queue.Queue() 1668 t = DatabaseThread(config_db, db_queue) 1669 t.setDaemon(True) 1670 t.start() 1671 1672 if not options.cleanup_headers_only: 1673 check_clean_directory() 1674 update_cross_compile(options.color) 1675 move_config(configs, options, db_queue) 1676 db_queue.join() 1677 1678 if configs: 1679 cleanup_headers(configs, options) 1680 cleanup_extra_options(configs, options) 1681 cleanup_whitelist(configs, options) 1682 cleanup_readme(configs, options) 1683 1684 if options.commit: 1685 subprocess.call(['git', 'add', '-u']) 1686 if configs: 1687 msg = 'Convert %s %sto Kconfig' % (configs[0], 1688 'et al ' if len(configs) > 1 else '') 1689 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1690 '\n '.join(configs)) 1691 else: 1692 msg = 'configs: Resync with savedefconfig' 1693 msg += '\n\nRsync all defconfig files using moveconfig.py' 1694 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1695 1696 if options.build_db: 1697 with open(CONFIG_DATABASE, 'w') as fd: 1698 for defconfig, configs in config_db.iteritems(): 1699 print >>fd, '%s' % defconfig 1700 for config in sorted(configs.keys()): 1701 print >>fd, ' %s=%s' % (config, configs[config]) 1702 print >>fd 1703 1704if __name__ == '__main__': 1705 main() 1706