1#!/usr/bin/env python2 2# SPDX-License-Identifier: GPL-2.0+ 3# 4# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 5# 6 7""" 8Move config options from headers to defconfig files. 9 10Since Kconfig was introduced to U-Boot, we have worked on moving 11config options from headers to Kconfig (defconfig). 12 13This tool intends to help this tremendous work. 14 15 16Usage 17----- 18 19First, you must edit the Kconfig to add the menu entries for the configs 20you are moving. 21 22And then run this tool giving CONFIG names you want to move. 23For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 24simply type as follows: 25 26 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 27 28The tool walks through all the defconfig files and move the given CONFIGs. 29 30The log is also displayed on the terminal. 31 32The log is printed for each defconfig as follows: 33 34<defconfig_name> 35 <action1> 36 <action2> 37 <action3> 38 ... 39 40<defconfig_name> is the name of the defconfig. 41 42<action*> shows what the tool did for that defconfig. 43It looks like one of the following: 44 45 - Move 'CONFIG_... ' 46 This config option was moved to the defconfig 47 48 - CONFIG_... is not defined in Kconfig. Do nothing. 49 The entry for this CONFIG was not found in Kconfig. The option is not 50 defined in the config header, either. So, this case can be just skipped. 51 52 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 53 This option is defined in the config header, but its entry was not found 54 in Kconfig. 55 There are two common cases: 56 - You forgot to create an entry for the CONFIG before running 57 this tool, or made a typo in a CONFIG passed to this tool. 58 - The entry was hidden due to unmet 'depends on'. 59 The tool does not know if the result is reasonable, so please check it 60 manually. 61 62 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 63 The define in the config header matched the one in Kconfig. 64 We do not need to touch it. 65 66 - Compiler is missing. Do nothing. 67 The compiler specified for this architecture was not found 68 in your PATH environment. 69 (If -e option is passed, the tool exits immediately.) 70 71 - Failed to process. 72 An error occurred during processing this defconfig. Skipped. 73 (If -e option is passed, the tool exits immediately on error.) 74 75Finally, you will be asked, Clean up headers? [y/n]: 76 77If you say 'y' here, the unnecessary config defines are removed 78from the config headers (include/configs/*.h). 79It just uses the regex method, so you should not rely on it. 80Just in case, please do 'git diff' to see what happened. 81 82 83How does it work? 84----------------- 85 86This tool runs configuration and builds include/autoconf.mk for every 87defconfig. The config options defined in Kconfig appear in the .config 88file (unless they are hidden because of unmet dependency.) 89On the other hand, the config options defined by board headers are seen 90in include/autoconf.mk. The tool looks for the specified options in both 91of them to decide the appropriate action for the options. If the given 92config option is found in the .config, but its value does not match the 93one from the board header, the config option in the .config is replaced 94with the define in the board header. Then, the .config is synced by 95"make savedefconfig" and the defconfig is updated with it. 96 97For faster processing, this tool handles multi-threading. It creates 98separate build directories where the out-of-tree build is run. The 99temporary build directories are automatically created and deleted as 100needed. The number of threads are chosen based on the number of the CPU 101cores of your system although you can change it via -j (--jobs) option. 102 103 104Toolchains 105---------- 106 107Appropriate toolchain are necessary to generate include/autoconf.mk 108for all the architectures supported by U-Boot. Most of them are available 109at the kernel.org site, some are not provided by kernel.org. This tool uses 110the same tools as buildman, so see that tool for setup (e.g. --fetch-arch). 111 112 113Tips and trips 114-------------- 115 116To sync only X86 defconfigs: 117 118 ./tools/moveconfig.py -s -d <(grep -l X86 configs/*) 119 120or: 121 122 grep -l X86 configs/* | ./tools/moveconfig.py -s -d - 123 124To process CONFIG_CMD_FPGAD only for a subset of configs based on path match: 125 126 ls configs/{hrcon*,iocon*,strider*} | \ 127 ./tools/moveconfig.py -Cy CONFIG_CMD_FPGAD -d - 128 129 130Finding implied CONFIGs 131----------------------- 132 133Some CONFIG options can be implied by others and this can help to reduce 134the size of the defconfig files. For example, CONFIG_X86 implies 135CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 136all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 137each of the x86 defconfig files. 138 139This tool can help find such configs. To use it, first build a database: 140 141 ./tools/moveconfig.py -b 142 143Then try to query it: 144 145 ./tools/moveconfig.py -i CONFIG_CMD_IRQ 146 CONFIG_CMD_IRQ found in 311/2384 defconfigs 147 44 : CONFIG_SYS_FSL_ERRATUM_IFC_A002769 148 41 : CONFIG_SYS_FSL_ERRATUM_A007075 149 31 : CONFIG_SYS_FSL_DDR_VER_44 150 28 : CONFIG_ARCH_P1010 151 28 : CONFIG_SYS_FSL_ERRATUM_P1010_A003549 152 28 : CONFIG_SYS_FSL_ERRATUM_SEC_A003571 153 28 : CONFIG_SYS_FSL_ERRATUM_IFC_A003399 154 25 : CONFIG_SYS_FSL_ERRATUM_A008044 155 22 : CONFIG_ARCH_P1020 156 21 : CONFIG_SYS_FSL_DDR_VER_46 157 20 : CONFIG_MAX_PIRQ_LINKS 158 20 : CONFIG_HPET_ADDRESS 159 20 : CONFIG_X86 160 20 : CONFIG_PCIE_ECAM_SIZE 161 20 : CONFIG_IRQ_SLOT_COUNT 162 20 : CONFIG_I8259_PIC 163 20 : CONFIG_CPU_ADDR_BITS 164 20 : CONFIG_RAMBASE 165 20 : CONFIG_SYS_FSL_ERRATUM_A005871 166 20 : CONFIG_PCIE_ECAM_BASE 167 20 : CONFIG_X86_TSC_TIMER 168 20 : CONFIG_I8254_TIMER 169 20 : CONFIG_CMD_GETTIME 170 19 : CONFIG_SYS_FSL_ERRATUM_A005812 171 18 : CONFIG_X86_RUN_32BIT 172 17 : CONFIG_CMD_CHIP_CONFIG 173 ... 174 175This shows a list of config options which might imply CONFIG_CMD_EEPROM along 176with how many defconfigs they cover. From this you can see that CONFIG_X86 177implies CONFIG_CMD_EEPROM. Therefore, instead of adding CONFIG_CMD_EEPROM to 178the defconfig of every x86 board, you could add a single imply line to the 179Kconfig file: 180 181 config X86 182 bool "x86 architecture" 183 ... 184 imply CMD_EEPROM 185 186That will cover 20 defconfigs. Many of the options listed are not suitable as 187they are not related. E.g. it would be odd for CONFIG_CMD_GETTIME to imply 188CMD_EEPROM. 189 190Using this search you can reduce the size of moveconfig patches. 191 192You can automatically add 'imply' statements in the Kconfig with the -a 193option: 194 195 ./tools/moveconfig.py -s -i CONFIG_SCSI \ 196 -a CONFIG_ARCH_LS1021A,CONFIG_ARCH_LS1043A 197 198This will add 'imply SCSI' to the two CONFIG options mentioned, assuming that 199the database indicates that they do actually imply CONFIG_SCSI and do not 200already have an 'imply SCSI'. 201 202The output shows where the imply is added: 203 204 18 : CONFIG_ARCH_LS1021A arch/arm/cpu/armv7/ls102xa/Kconfig:1 205 13 : CONFIG_ARCH_LS1043A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:11 206 12 : CONFIG_ARCH_LS1046A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:31 207 208The first number is the number of boards which can avoid having a special 209CONFIG_SCSI option in their defconfig file if this 'imply' is added. 210The location at the right is the Kconfig file and line number where the config 211appears. For example, adding 'imply CONFIG_SCSI' to the 'config ARCH_LS1021A' 212in arch/arm/cpu/armv7/ls102xa/Kconfig at line 1 will help 18 boards to reduce 213the size of their defconfig files. 214 215If you want to add an 'imply' to every imply config in the list, you can use 216 217 ./tools/moveconfig.py -s -i CONFIG_SCSI -a all 218 219To control which ones are displayed, use -I <list> where list is a list of 220options (use '-I help' to see possible options and their meaning). 221 222To skip showing you options that already have an 'imply' attached, use -A. 223 224When you have finished adding 'imply' options you can regenerate the 225defconfig files for affected boards with something like: 226 227 git show --stat | ./tools/moveconfig.py -s -d - 228 229This will regenerate only those defconfigs changed in the current commit. 230If you start with (say) 100 defconfigs being changed in the commit, and add 231a few 'imply' options as above, then regenerate, hopefully you can reduce the 232number of defconfigs changed in the commit. 233 234 235Available options 236----------------- 237 238 -c, --color 239 Surround each portion of the log with escape sequences to display it 240 in color on the terminal. 241 242 -C, --commit 243 Create a git commit with the changes when the operation is complete. A 244 standard commit message is used which may need to be edited. 245 246 -d, --defconfigs 247 Specify a file containing a list of defconfigs to move. The defconfig 248 files can be given with shell-style wildcards. Use '-' to read from stdin. 249 250 -n, --dry-run 251 Perform a trial run that does not make any changes. It is useful to 252 see what is going to happen before one actually runs it. 253 254 -e, --exit-on-error 255 Exit immediately if Make exits with a non-zero status while processing 256 a defconfig file. 257 258 -s, --force-sync 259 Do "make savedefconfig" forcibly for all the defconfig files. 260 If not specified, "make savedefconfig" only occurs for cases 261 where at least one CONFIG was moved. 262 263 -S, --spl 264 Look for moved config options in spl/include/autoconf.mk instead of 265 include/autoconf.mk. This is useful for moving options for SPL build 266 because SPL related options (mostly prefixed with CONFIG_SPL_) are 267 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 268 269 -H, --headers-only 270 Only cleanup the headers; skip the defconfig processing 271 272 -j, --jobs 273 Specify the number of threads to run simultaneously. If not specified, 274 the number of threads is the same as the number of CPU cores. 275 276 -r, --git-ref 277 Specify the git ref to clone for building the autoconf.mk. If unspecified 278 use the CWD. This is useful for when changes to the Kconfig affect the 279 default values and you want to capture the state of the defconfig from 280 before that change was in effect. If in doubt, specify a ref pre-Kconfig 281 changes (use HEAD if Kconfig changes are not committed). Worst case it will 282 take a bit longer to run, but will always do the right thing. 283 284 -v, --verbose 285 Show any build errors as boards are built 286 287 -y, --yes 288 Instead of prompting, automatically go ahead with all operations. This 289 includes cleaning up headers, CONFIG_SYS_EXTRA_OPTIONS, the config whitelist 290 and the README. 291 292To see the complete list of supported options, run 293 294 $ tools/moveconfig.py -h 295 296""" 297 298import collections 299import copy 300import difflib 301import filecmp 302import fnmatch 303import glob 304import multiprocessing 305import optparse 306import os 307import Queue 308import re 309import shutil 310import subprocess 311import sys 312import tempfile 313import threading 314import time 315 316sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman')) 317sys.path.append(os.path.join(os.path.dirname(__file__), 'patman')) 318import bsettings 319import kconfiglib 320import toolchain 321 322SHOW_GNU_MAKE = 'scripts/show-gnu-make' 323SLEEP_TIME=0.03 324 325STATE_IDLE = 0 326STATE_DEFCONFIG = 1 327STATE_AUTOCONF = 2 328STATE_SAVEDEFCONFIG = 3 329 330ACTION_MOVE = 0 331ACTION_NO_ENTRY = 1 332ACTION_NO_ENTRY_WARN = 2 333ACTION_NO_CHANGE = 3 334 335COLOR_BLACK = '0;30' 336COLOR_RED = '0;31' 337COLOR_GREEN = '0;32' 338COLOR_BROWN = '0;33' 339COLOR_BLUE = '0;34' 340COLOR_PURPLE = '0;35' 341COLOR_CYAN = '0;36' 342COLOR_LIGHT_GRAY = '0;37' 343COLOR_DARK_GRAY = '1;30' 344COLOR_LIGHT_RED = '1;31' 345COLOR_LIGHT_GREEN = '1;32' 346COLOR_YELLOW = '1;33' 347COLOR_LIGHT_BLUE = '1;34' 348COLOR_LIGHT_PURPLE = '1;35' 349COLOR_LIGHT_CYAN = '1;36' 350COLOR_WHITE = '1;37' 351 352AUTO_CONF_PATH = 'include/config/auto.conf' 353CONFIG_DATABASE = 'moveconfig.db' 354 355CONFIG_LEN = len('CONFIG_') 356 357### helper functions ### 358def get_devnull(): 359 """Get the file object of '/dev/null' device.""" 360 try: 361 devnull = subprocess.DEVNULL # py3k 362 except AttributeError: 363 devnull = open(os.devnull, 'wb') 364 return devnull 365 366def check_top_directory(): 367 """Exit if we are not at the top of source directory.""" 368 for f in ('README', 'Licenses'): 369 if not os.path.exists(f): 370 sys.exit('Please run at the top of source directory.') 371 372def check_clean_directory(): 373 """Exit if the source tree is not clean.""" 374 for f in ('.config', 'include/config'): 375 if os.path.exists(f): 376 sys.exit("source tree is not clean, please run 'make mrproper'") 377 378def get_make_cmd(): 379 """Get the command name of GNU Make. 380 381 U-Boot needs GNU Make for building, but the command name is not 382 necessarily "make". (for example, "gmake" on FreeBSD). 383 Returns the most appropriate command name on your system. 384 """ 385 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 386 ret = process.communicate() 387 if process.returncode: 388 sys.exit('GNU Make not found') 389 return ret[0].rstrip() 390 391def get_matched_defconfig(line): 392 """Get the defconfig files that match a pattern 393 394 Args: 395 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 396 'k2*_defconfig'. If no directory is provided, 'configs/' is 397 prepended 398 399 Returns: 400 a list of matching defconfig files 401 """ 402 dirname = os.path.dirname(line) 403 if dirname: 404 pattern = line 405 else: 406 pattern = os.path.join('configs', line) 407 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 408 409def get_matched_defconfigs(defconfigs_file): 410 """Get all the defconfig files that match the patterns in a file. 411 412 Args: 413 defconfigs_file: File containing a list of defconfigs to process, or 414 '-' to read the list from stdin 415 416 Returns: 417 A list of paths to defconfig files, with no duplicates 418 """ 419 defconfigs = [] 420 if defconfigs_file == '-': 421 fd = sys.stdin 422 defconfigs_file = 'stdin' 423 else: 424 fd = open(defconfigs_file) 425 for i, line in enumerate(fd): 426 line = line.strip() 427 if not line: 428 continue # skip blank lines silently 429 if ' ' in line: 430 line = line.split(' ')[0] # handle 'git log' input 431 matched = get_matched_defconfig(line) 432 if not matched: 433 print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \ 434 (defconfigs_file, i + 1, line) 435 436 defconfigs += matched 437 438 # use set() to drop multiple matching 439 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 440 441def get_all_defconfigs(): 442 """Get all the defconfig files under the configs/ directory.""" 443 defconfigs = [] 444 for (dirpath, dirnames, filenames) in os.walk('configs'): 445 dirpath = dirpath[len('configs') + 1:] 446 for filename in fnmatch.filter(filenames, '*_defconfig'): 447 defconfigs.append(os.path.join(dirpath, filename)) 448 449 return defconfigs 450 451def color_text(color_enabled, color, string): 452 """Return colored string.""" 453 if color_enabled: 454 # LF should not be surrounded by the escape sequence. 455 # Otherwise, additional whitespace or line-feed might be printed. 456 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 457 for s in string.split('\n') ]) 458 else: 459 return string 460 461def show_diff(a, b, file_path, color_enabled): 462 """Show unidified diff. 463 464 Arguments: 465 a: A list of lines (before) 466 b: A list of lines (after) 467 file_path: Path to the file 468 color_enabled: Display the diff in color 469 """ 470 471 diff = difflib.unified_diff(a, b, 472 fromfile=os.path.join('a', file_path), 473 tofile=os.path.join('b', file_path)) 474 475 for line in diff: 476 if line[0] == '-' and line[1] != '-': 477 print color_text(color_enabled, COLOR_RED, line), 478 elif line[0] == '+' and line[1] != '+': 479 print color_text(color_enabled, COLOR_GREEN, line), 480 else: 481 print line, 482 483def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 484 extend_post): 485 """Extend matched lines if desired patterns are found before/after already 486 matched lines. 487 488 Arguments: 489 lines: A list of lines handled. 490 matched: A list of line numbers that have been already matched. 491 (will be updated by this function) 492 pre_patterns: A list of regular expression that should be matched as 493 preamble. 494 post_patterns: A list of regular expression that should be matched as 495 postamble. 496 extend_pre: Add the line number of matched preamble to the matched list. 497 extend_post: Add the line number of matched postamble to the matched list. 498 """ 499 extended_matched = [] 500 501 j = matched[0] 502 503 for i in matched: 504 if i == 0 or i < j: 505 continue 506 j = i 507 while j in matched: 508 j += 1 509 if j >= len(lines): 510 break 511 512 for p in pre_patterns: 513 if p.search(lines[i - 1]): 514 break 515 else: 516 # not matched 517 continue 518 519 for p in post_patterns: 520 if p.search(lines[j]): 521 break 522 else: 523 # not matched 524 continue 525 526 if extend_pre: 527 extended_matched.append(i - 1) 528 if extend_post: 529 extended_matched.append(j) 530 531 matched += extended_matched 532 matched.sort() 533 534def confirm(options, prompt): 535 if not options.yes: 536 while True: 537 choice = raw_input('{} [y/n]: '.format(prompt)) 538 choice = choice.lower() 539 print choice 540 if choice == 'y' or choice == 'n': 541 break 542 543 if choice == 'n': 544 return False 545 546 return True 547 548def cleanup_one_header(header_path, patterns, options): 549 """Clean regex-matched lines away from a file. 550 551 Arguments: 552 header_path: path to the cleaned file. 553 patterns: list of regex patterns. Any lines matching to these 554 patterns are deleted. 555 options: option flags. 556 """ 557 with open(header_path) as f: 558 lines = f.readlines() 559 560 matched = [] 561 for i, line in enumerate(lines): 562 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 563 matched.append(i) 564 continue 565 for pattern in patterns: 566 if pattern.search(line): 567 matched.append(i) 568 break 569 570 if not matched: 571 return 572 573 # remove empty #ifdef ... #endif, successive blank lines 574 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 575 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 576 pattern_endif = re.compile(r'#\s*endif\W') # #endif 577 pattern_blank = re.compile(r'^\s*$') # empty line 578 579 while True: 580 old_matched = copy.copy(matched) 581 extend_matched_lines(lines, matched, [pattern_if], 582 [pattern_endif], True, True) 583 extend_matched_lines(lines, matched, [pattern_elif], 584 [pattern_elif, pattern_endif], True, False) 585 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 586 [pattern_blank], False, True) 587 extend_matched_lines(lines, matched, [pattern_blank], 588 [pattern_elif, pattern_endif], True, False) 589 extend_matched_lines(lines, matched, [pattern_blank], 590 [pattern_blank], True, False) 591 if matched == old_matched: 592 break 593 594 tolines = copy.copy(lines) 595 596 for i in reversed(matched): 597 tolines.pop(i) 598 599 show_diff(lines, tolines, header_path, options.color) 600 601 if options.dry_run: 602 return 603 604 with open(header_path, 'w') as f: 605 for line in tolines: 606 f.write(line) 607 608def cleanup_headers(configs, options): 609 """Delete config defines from board headers. 610 611 Arguments: 612 configs: A list of CONFIGs to remove. 613 options: option flags. 614 """ 615 if not confirm(options, 'Clean up headers?'): 616 return 617 618 patterns = [] 619 for config in configs: 620 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 621 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 622 623 for dir in 'include', 'arch', 'board': 624 for (dirpath, dirnames, filenames) in os.walk(dir): 625 if dirpath == os.path.join('include', 'generated'): 626 continue 627 for filename in filenames: 628 if not fnmatch.fnmatch(filename, '*~'): 629 cleanup_one_header(os.path.join(dirpath, filename), 630 patterns, options) 631 632def cleanup_one_extra_option(defconfig_path, configs, options): 633 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 634 635 Arguments: 636 defconfig_path: path to the cleaned defconfig file. 637 configs: A list of CONFIGs to remove. 638 options: option flags. 639 """ 640 641 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 642 end = '"\n' 643 644 with open(defconfig_path) as f: 645 lines = f.readlines() 646 647 for i, line in enumerate(lines): 648 if line.startswith(start) and line.endswith(end): 649 break 650 else: 651 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 652 return 653 654 old_tokens = line[len(start):-len(end)].split(',') 655 new_tokens = [] 656 657 for token in old_tokens: 658 pos = token.find('=') 659 if not (token[:pos] if pos >= 0 else token) in configs: 660 new_tokens.append(token) 661 662 if new_tokens == old_tokens: 663 return 664 665 tolines = copy.copy(lines) 666 667 if new_tokens: 668 tolines[i] = start + ','.join(new_tokens) + end 669 else: 670 tolines.pop(i) 671 672 show_diff(lines, tolines, defconfig_path, options.color) 673 674 if options.dry_run: 675 return 676 677 with open(defconfig_path, 'w') as f: 678 for line in tolines: 679 f.write(line) 680 681def cleanup_extra_options(configs, options): 682 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 683 684 Arguments: 685 configs: A list of CONFIGs to remove. 686 options: option flags. 687 """ 688 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 689 return 690 691 configs = [ config[len('CONFIG_'):] for config in configs ] 692 693 defconfigs = get_all_defconfigs() 694 695 for defconfig in defconfigs: 696 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 697 options) 698 699def cleanup_whitelist(configs, options): 700 """Delete config whitelist entries 701 702 Arguments: 703 configs: A list of CONFIGs to remove. 704 options: option flags. 705 """ 706 if not confirm(options, 'Clean up whitelist entries?'): 707 return 708 709 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 710 lines = f.readlines() 711 712 lines = [x for x in lines if x.strip() not in configs] 713 714 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 715 f.write(''.join(lines)) 716 717def find_matching(patterns, line): 718 for pat in patterns: 719 if pat.search(line): 720 return True 721 return False 722 723def cleanup_readme(configs, options): 724 """Delete config description in README 725 726 Arguments: 727 configs: A list of CONFIGs to remove. 728 options: option flags. 729 """ 730 if not confirm(options, 'Clean up README?'): 731 return 732 733 patterns = [] 734 for config in configs: 735 patterns.append(re.compile(r'^\s+%s' % config)) 736 737 with open('README') as f: 738 lines = f.readlines() 739 740 found = False 741 newlines = [] 742 for line in lines: 743 if not found: 744 found = find_matching(patterns, line) 745 if found: 746 continue 747 748 if found and re.search(r'^\s+CONFIG', line): 749 found = False 750 751 if not found: 752 newlines.append(line) 753 754 with open('README', 'w') as f: 755 f.write(''.join(newlines)) 756 757 758### classes ### 759class Progress: 760 761 """Progress Indicator""" 762 763 def __init__(self, total): 764 """Create a new progress indicator. 765 766 Arguments: 767 total: A number of defconfig files to process. 768 """ 769 self.current = 0 770 self.total = total 771 772 def inc(self): 773 """Increment the number of processed defconfig files.""" 774 775 self.current += 1 776 777 def show(self): 778 """Display the progress.""" 779 print ' %d defconfigs out of %d\r' % (self.current, self.total), 780 sys.stdout.flush() 781 782 783class KconfigScanner: 784 """Kconfig scanner.""" 785 786 def __init__(self): 787 """Scan all the Kconfig files and create a Config object.""" 788 # Define environment variables referenced from Kconfig 789 os.environ['srctree'] = os.getcwd() 790 os.environ['UBOOTVERSION'] = 'dummy' 791 os.environ['KCONFIG_OBJDIR'] = '' 792 self.conf = kconfiglib.Config() 793 794 795class KconfigParser: 796 797 """A parser of .config and include/autoconf.mk.""" 798 799 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 800 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 801 802 def __init__(self, configs, options, build_dir): 803 """Create a new parser. 804 805 Arguments: 806 configs: A list of CONFIGs to move. 807 options: option flags. 808 build_dir: Build directory. 809 """ 810 self.configs = configs 811 self.options = options 812 self.dotconfig = os.path.join(build_dir, '.config') 813 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 814 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 815 'autoconf.mk') 816 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 817 self.defconfig = os.path.join(build_dir, 'defconfig') 818 819 def get_arch(self): 820 """Parse .config file and return the architecture. 821 822 Returns: 823 Architecture name (e.g. 'arm'). 824 """ 825 arch = '' 826 cpu = '' 827 for line in open(self.dotconfig): 828 m = self.re_arch.match(line) 829 if m: 830 arch = m.group(1) 831 continue 832 m = self.re_cpu.match(line) 833 if m: 834 cpu = m.group(1) 835 836 if not arch: 837 return None 838 839 # fix-up for aarch64 840 if arch == 'arm' and cpu == 'armv8': 841 arch = 'aarch64' 842 843 return arch 844 845 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 846 """Parse .config, defconfig, include/autoconf.mk for one config. 847 848 This function looks for the config options in the lines from 849 defconfig, .config, and include/autoconf.mk in order to decide 850 which action should be taken for this defconfig. 851 852 Arguments: 853 config: CONFIG name to parse. 854 dotconfig_lines: lines from the .config file. 855 autoconf_lines: lines from the include/autoconf.mk file. 856 857 Returns: 858 A tupple of the action for this defconfig and the line 859 matched for the config. 860 """ 861 not_set = '# %s is not set' % config 862 863 for line in autoconf_lines: 864 line = line.rstrip() 865 if line.startswith(config + '='): 866 new_val = line 867 break 868 else: 869 new_val = not_set 870 871 for line in dotconfig_lines: 872 line = line.rstrip() 873 if line.startswith(config + '=') or line == not_set: 874 old_val = line 875 break 876 else: 877 if new_val == not_set: 878 return (ACTION_NO_ENTRY, config) 879 else: 880 return (ACTION_NO_ENTRY_WARN, config) 881 882 # If this CONFIG is neither bool nor trisate 883 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 884 # tools/scripts/define2mk.sed changes '1' to 'y'. 885 # This is a problem if the CONFIG is int type. 886 # Check the type in Kconfig and handle it correctly. 887 if new_val[-2:] == '=y': 888 new_val = new_val[:-1] + '1' 889 890 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 891 new_val) 892 893 def update_dotconfig(self): 894 """Parse files for the config options and update the .config. 895 896 This function parses the generated .config and include/autoconf.mk 897 searching the target options. 898 Move the config option(s) to the .config as needed. 899 900 Arguments: 901 defconfig: defconfig name. 902 903 Returns: 904 Return a tuple of (updated flag, log string). 905 The "updated flag" is True if the .config was updated, False 906 otherwise. The "log string" shows what happend to the .config. 907 """ 908 909 results = [] 910 updated = False 911 suspicious = False 912 rm_files = [self.config_autoconf, self.autoconf] 913 914 if self.options.spl: 915 if os.path.exists(self.spl_autoconf): 916 autoconf_path = self.spl_autoconf 917 rm_files.append(self.spl_autoconf) 918 else: 919 for f in rm_files: 920 os.remove(f) 921 return (updated, suspicious, 922 color_text(self.options.color, COLOR_BROWN, 923 "SPL is not enabled. Skipped.") + '\n') 924 else: 925 autoconf_path = self.autoconf 926 927 with open(self.dotconfig) as f: 928 dotconfig_lines = f.readlines() 929 930 with open(autoconf_path) as f: 931 autoconf_lines = f.readlines() 932 933 for config in self.configs: 934 result = self.parse_one_config(config, dotconfig_lines, 935 autoconf_lines) 936 results.append(result) 937 938 log = '' 939 940 for (action, value) in results: 941 if action == ACTION_MOVE: 942 actlog = "Move '%s'" % value 943 log_color = COLOR_LIGHT_GREEN 944 elif action == ACTION_NO_ENTRY: 945 actlog = "%s is not defined in Kconfig. Do nothing." % value 946 log_color = COLOR_LIGHT_BLUE 947 elif action == ACTION_NO_ENTRY_WARN: 948 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 949 log_color = COLOR_YELLOW 950 suspicious = True 951 elif action == ACTION_NO_CHANGE: 952 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 953 % value 954 log_color = COLOR_LIGHT_PURPLE 955 elif action == ACTION_SPL_NOT_EXIST: 956 actlog = "SPL is not enabled for this defconfig. Skip." 957 log_color = COLOR_PURPLE 958 else: 959 sys.exit("Internal Error. This should not happen.") 960 961 log += color_text(self.options.color, log_color, actlog) + '\n' 962 963 with open(self.dotconfig, 'a') as f: 964 for (action, value) in results: 965 if action == ACTION_MOVE: 966 f.write(value + '\n') 967 updated = True 968 969 self.results = results 970 for f in rm_files: 971 os.remove(f) 972 973 return (updated, suspicious, log) 974 975 def check_defconfig(self): 976 """Check the defconfig after savedefconfig 977 978 Returns: 979 Return additional log if moved CONFIGs were removed again by 980 'make savedefconfig'. 981 """ 982 983 log = '' 984 985 with open(self.defconfig) as f: 986 defconfig_lines = f.readlines() 987 988 for (action, value) in self.results: 989 if action != ACTION_MOVE: 990 continue 991 if not value + '\n' in defconfig_lines: 992 log += color_text(self.options.color, COLOR_YELLOW, 993 "'%s' was removed by savedefconfig.\n" % 994 value) 995 996 return log 997 998 999class DatabaseThread(threading.Thread): 1000 """This thread processes results from Slot threads. 1001 1002 It collects the data in the master config directary. There is only one 1003 result thread, and this helps to serialise the build output. 1004 """ 1005 def __init__(self, config_db, db_queue): 1006 """Set up a new result thread 1007 1008 Args: 1009 builder: Builder which will be sent each result 1010 """ 1011 threading.Thread.__init__(self) 1012 self.config_db = config_db 1013 self.db_queue= db_queue 1014 1015 def run(self): 1016 """Called to start up the result thread. 1017 1018 We collect the next result job and pass it on to the build. 1019 """ 1020 while True: 1021 defconfig, configs = self.db_queue.get() 1022 self.config_db[defconfig] = configs 1023 self.db_queue.task_done() 1024 1025 1026class Slot: 1027 1028 """A slot to store a subprocess. 1029 1030 Each instance of this class handles one subprocess. 1031 This class is useful to control multiple threads 1032 for faster processing. 1033 """ 1034 1035 def __init__(self, toolchains, configs, options, progress, devnull, 1036 make_cmd, reference_src_dir, db_queue): 1037 """Create a new process slot. 1038 1039 Arguments: 1040 toolchains: Toolchains object containing toolchains. 1041 configs: A list of CONFIGs to move. 1042 options: option flags. 1043 progress: A progress indicator. 1044 devnull: A file object of '/dev/null'. 1045 make_cmd: command name of GNU Make. 1046 reference_src_dir: Determine the true starting config state from this 1047 source tree. 1048 db_queue: output queue to write config info for the database 1049 """ 1050 self.toolchains = toolchains 1051 self.options = options 1052 self.progress = progress 1053 self.build_dir = tempfile.mkdtemp() 1054 self.devnull = devnull 1055 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 1056 self.reference_src_dir = reference_src_dir 1057 self.db_queue = db_queue 1058 self.parser = KconfigParser(configs, options, self.build_dir) 1059 self.state = STATE_IDLE 1060 self.failed_boards = set() 1061 self.suspicious_boards = set() 1062 1063 def __del__(self): 1064 """Delete the working directory 1065 1066 This function makes sure the temporary directory is cleaned away 1067 even if Python suddenly dies due to error. It should be done in here 1068 because it is guaranteed the destructor is always invoked when the 1069 instance of the class gets unreferenced. 1070 1071 If the subprocess is still running, wait until it finishes. 1072 """ 1073 if self.state != STATE_IDLE: 1074 while self.ps.poll() == None: 1075 pass 1076 shutil.rmtree(self.build_dir) 1077 1078 def add(self, defconfig): 1079 """Assign a new subprocess for defconfig and add it to the slot. 1080 1081 If the slot is vacant, create a new subprocess for processing the 1082 given defconfig and add it to the slot. Just returns False if 1083 the slot is occupied (i.e. the current subprocess is still running). 1084 1085 Arguments: 1086 defconfig: defconfig name. 1087 1088 Returns: 1089 Return True on success or False on failure 1090 """ 1091 if self.state != STATE_IDLE: 1092 return False 1093 1094 self.defconfig = defconfig 1095 self.log = '' 1096 self.current_src_dir = self.reference_src_dir 1097 self.do_defconfig() 1098 return True 1099 1100 def poll(self): 1101 """Check the status of the subprocess and handle it as needed. 1102 1103 Returns True if the slot is vacant (i.e. in idle state). 1104 If the configuration is successfully finished, assign a new 1105 subprocess to build include/autoconf.mk. 1106 If include/autoconf.mk is generated, invoke the parser to 1107 parse the .config and the include/autoconf.mk, moving 1108 config options to the .config as needed. 1109 If the .config was updated, run "make savedefconfig" to sync 1110 it, update the original defconfig, and then set the slot back 1111 to the idle state. 1112 1113 Returns: 1114 Return True if the subprocess is terminated, False otherwise 1115 """ 1116 if self.state == STATE_IDLE: 1117 return True 1118 1119 if self.ps.poll() == None: 1120 return False 1121 1122 if self.ps.poll() != 0: 1123 self.handle_error() 1124 elif self.state == STATE_DEFCONFIG: 1125 if self.reference_src_dir and not self.current_src_dir: 1126 self.do_savedefconfig() 1127 else: 1128 self.do_autoconf() 1129 elif self.state == STATE_AUTOCONF: 1130 if self.current_src_dir: 1131 self.current_src_dir = None 1132 self.do_defconfig() 1133 elif self.options.build_db: 1134 self.do_build_db() 1135 else: 1136 self.do_savedefconfig() 1137 elif self.state == STATE_SAVEDEFCONFIG: 1138 self.update_defconfig() 1139 else: 1140 sys.exit("Internal Error. This should not happen.") 1141 1142 return True if self.state == STATE_IDLE else False 1143 1144 def handle_error(self): 1145 """Handle error cases.""" 1146 1147 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1148 "Failed to process.\n") 1149 if self.options.verbose: 1150 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1151 self.ps.stderr.read()) 1152 self.finish(False) 1153 1154 def do_defconfig(self): 1155 """Run 'make <board>_defconfig' to create the .config file.""" 1156 1157 cmd = list(self.make_cmd) 1158 cmd.append(self.defconfig) 1159 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1160 stderr=subprocess.PIPE, 1161 cwd=self.current_src_dir) 1162 self.state = STATE_DEFCONFIG 1163 1164 def do_autoconf(self): 1165 """Run 'make AUTO_CONF_PATH'.""" 1166 1167 arch = self.parser.get_arch() 1168 try: 1169 toolchain = self.toolchains.Select(arch) 1170 except ValueError: 1171 self.log += color_text(self.options.color, COLOR_YELLOW, 1172 "Tool chain for '%s' is missing. Do nothing.\n" % arch) 1173 self.finish(False) 1174 return 1175 env = toolchain.MakeEnvironment(False) 1176 1177 cmd = list(self.make_cmd) 1178 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1179 cmd.append(AUTO_CONF_PATH) 1180 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env, 1181 stderr=subprocess.PIPE, 1182 cwd=self.current_src_dir) 1183 self.state = STATE_AUTOCONF 1184 1185 def do_build_db(self): 1186 """Add the board to the database""" 1187 configs = {} 1188 with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: 1189 for line in fd.readlines(): 1190 if line.startswith('CONFIG'): 1191 config, value = line.split('=', 1) 1192 configs[config] = value.rstrip() 1193 self.db_queue.put([self.defconfig, configs]) 1194 self.finish(True) 1195 1196 def do_savedefconfig(self): 1197 """Update the .config and run 'make savedefconfig'.""" 1198 1199 (updated, suspicious, log) = self.parser.update_dotconfig() 1200 if suspicious: 1201 self.suspicious_boards.add(self.defconfig) 1202 self.log += log 1203 1204 if not self.options.force_sync and not updated: 1205 self.finish(True) 1206 return 1207 if updated: 1208 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1209 "Syncing by savedefconfig...\n") 1210 else: 1211 self.log += "Syncing by savedefconfig (forced by option)...\n" 1212 1213 cmd = list(self.make_cmd) 1214 cmd.append('savedefconfig') 1215 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1216 stderr=subprocess.PIPE) 1217 self.state = STATE_SAVEDEFCONFIG 1218 1219 def update_defconfig(self): 1220 """Update the input defconfig and go back to the idle state.""" 1221 1222 log = self.parser.check_defconfig() 1223 if log: 1224 self.suspicious_boards.add(self.defconfig) 1225 self.log += log 1226 orig_defconfig = os.path.join('configs', self.defconfig) 1227 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1228 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1229 1230 if updated: 1231 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1232 "defconfig was updated.\n") 1233 1234 if not self.options.dry_run and updated: 1235 shutil.move(new_defconfig, orig_defconfig) 1236 self.finish(True) 1237 1238 def finish(self, success): 1239 """Display log along with progress and go to the idle state. 1240 1241 Arguments: 1242 success: Should be True when the defconfig was processed 1243 successfully, or False when it fails. 1244 """ 1245 # output at least 30 characters to hide the "* defconfigs out of *". 1246 log = self.defconfig.ljust(30) + '\n' 1247 1248 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1249 # Some threads are running in parallel. 1250 # Print log atomically to not mix up logs from different threads. 1251 print >> (sys.stdout if success else sys.stderr), log 1252 1253 if not success: 1254 if self.options.exit_on_error: 1255 sys.exit("Exit on error.") 1256 # If --exit-on-error flag is not set, skip this board and continue. 1257 # Record the failed board. 1258 self.failed_boards.add(self.defconfig) 1259 1260 self.progress.inc() 1261 self.progress.show() 1262 self.state = STATE_IDLE 1263 1264 def get_failed_boards(self): 1265 """Returns a set of failed boards (defconfigs) in this slot. 1266 """ 1267 return self.failed_boards 1268 1269 def get_suspicious_boards(self): 1270 """Returns a set of boards (defconfigs) with possible misconversion. 1271 """ 1272 return self.suspicious_boards - self.failed_boards 1273 1274class Slots: 1275 1276 """Controller of the array of subprocess slots.""" 1277 1278 def __init__(self, toolchains, configs, options, progress, 1279 reference_src_dir, db_queue): 1280 """Create a new slots controller. 1281 1282 Arguments: 1283 toolchains: Toolchains object containing toolchains. 1284 configs: A list of CONFIGs to move. 1285 options: option flags. 1286 progress: A progress indicator. 1287 reference_src_dir: Determine the true starting config state from this 1288 source tree. 1289 db_queue: output queue to write config info for the database 1290 """ 1291 self.options = options 1292 self.slots = [] 1293 devnull = get_devnull() 1294 make_cmd = get_make_cmd() 1295 for i in range(options.jobs): 1296 self.slots.append(Slot(toolchains, configs, options, progress, 1297 devnull, make_cmd, reference_src_dir, 1298 db_queue)) 1299 1300 def add(self, defconfig): 1301 """Add a new subprocess if a vacant slot is found. 1302 1303 Arguments: 1304 defconfig: defconfig name to be put into. 1305 1306 Returns: 1307 Return True on success or False on failure 1308 """ 1309 for slot in self.slots: 1310 if slot.add(defconfig): 1311 return True 1312 return False 1313 1314 def available(self): 1315 """Check if there is a vacant slot. 1316 1317 Returns: 1318 Return True if at lease one vacant slot is found, False otherwise. 1319 """ 1320 for slot in self.slots: 1321 if slot.poll(): 1322 return True 1323 return False 1324 1325 def empty(self): 1326 """Check if all slots are vacant. 1327 1328 Returns: 1329 Return True if all the slots are vacant, False otherwise. 1330 """ 1331 ret = True 1332 for slot in self.slots: 1333 if not slot.poll(): 1334 ret = False 1335 return ret 1336 1337 def show_failed_boards(self): 1338 """Display all of the failed boards (defconfigs).""" 1339 boards = set() 1340 output_file = 'moveconfig.failed' 1341 1342 for slot in self.slots: 1343 boards |= slot.get_failed_boards() 1344 1345 if boards: 1346 boards = '\n'.join(boards) + '\n' 1347 msg = "The following boards were not processed due to error:\n" 1348 msg += boards 1349 msg += "(the list has been saved in %s)\n" % output_file 1350 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1351 msg) 1352 1353 with open(output_file, 'w') as f: 1354 f.write(boards) 1355 1356 def show_suspicious_boards(self): 1357 """Display all boards (defconfigs) with possible misconversion.""" 1358 boards = set() 1359 output_file = 'moveconfig.suspicious' 1360 1361 for slot in self.slots: 1362 boards |= slot.get_suspicious_boards() 1363 1364 if boards: 1365 boards = '\n'.join(boards) + '\n' 1366 msg = "The following boards might have been converted incorrectly.\n" 1367 msg += "It is highly recommended to check them manually:\n" 1368 msg += boards 1369 msg += "(the list has been saved in %s)\n" % output_file 1370 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1371 msg) 1372 1373 with open(output_file, 'w') as f: 1374 f.write(boards) 1375 1376class ReferenceSource: 1377 1378 """Reference source against which original configs should be parsed.""" 1379 1380 def __init__(self, commit): 1381 """Create a reference source directory based on a specified commit. 1382 1383 Arguments: 1384 commit: commit to git-clone 1385 """ 1386 self.src_dir = tempfile.mkdtemp() 1387 print "Cloning git repo to a separate work directory..." 1388 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1389 cwd=self.src_dir) 1390 print "Checkout '%s' to build the original autoconf.mk." % \ 1391 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1392 subprocess.check_output(['git', 'checkout', commit], 1393 stderr=subprocess.STDOUT, cwd=self.src_dir) 1394 1395 def __del__(self): 1396 """Delete the reference source directory 1397 1398 This function makes sure the temporary directory is cleaned away 1399 even if Python suddenly dies due to error. It should be done in here 1400 because it is guaranteed the destructor is always invoked when the 1401 instance of the class gets unreferenced. 1402 """ 1403 shutil.rmtree(self.src_dir) 1404 1405 def get_dir(self): 1406 """Return the absolute path to the reference source directory.""" 1407 1408 return self.src_dir 1409 1410def move_config(toolchains, configs, options, db_queue): 1411 """Move config options to defconfig files. 1412 1413 Arguments: 1414 configs: A list of CONFIGs to move. 1415 options: option flags 1416 """ 1417 if len(configs) == 0: 1418 if options.force_sync: 1419 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1420 elif options.build_db: 1421 print 'Building %s database' % CONFIG_DATABASE 1422 else: 1423 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1424 else: 1425 print 'Move ' + ', '.join(configs), 1426 print '(jobs: %d)\n' % options.jobs 1427 1428 if options.git_ref: 1429 reference_src = ReferenceSource(options.git_ref) 1430 reference_src_dir = reference_src.get_dir() 1431 else: 1432 reference_src_dir = None 1433 1434 if options.defconfigs: 1435 defconfigs = get_matched_defconfigs(options.defconfigs) 1436 else: 1437 defconfigs = get_all_defconfigs() 1438 1439 progress = Progress(len(defconfigs)) 1440 slots = Slots(toolchains, configs, options, progress, reference_src_dir, 1441 db_queue) 1442 1443 # Main loop to process defconfig files: 1444 # Add a new subprocess into a vacant slot. 1445 # Sleep if there is no available slot. 1446 for defconfig in defconfigs: 1447 while not slots.add(defconfig): 1448 while not slots.available(): 1449 # No available slot: sleep for a while 1450 time.sleep(SLEEP_TIME) 1451 1452 # wait until all the subprocesses finish 1453 while not slots.empty(): 1454 time.sleep(SLEEP_TIME) 1455 1456 print '' 1457 slots.show_failed_boards() 1458 slots.show_suspicious_boards() 1459 1460def find_kconfig_rules(kconf, config, imply_config): 1461 """Check whether a config has a 'select' or 'imply' keyword 1462 1463 Args: 1464 kconf: Kconfig.Config object 1465 config: Name of config to check (without CONFIG_ prefix) 1466 imply_config: Implying config (without CONFIG_ prefix) which may or 1467 may not have an 'imply' for 'config') 1468 1469 Returns: 1470 Symbol object for 'config' if found, else None 1471 """ 1472 sym = kconf.get_symbol(imply_config) 1473 if sym: 1474 for sel in sym.get_selected_symbols() | sym.get_implied_symbols(): 1475 if sel.get_name() == config: 1476 return sym 1477 return None 1478 1479def check_imply_rule(kconf, config, imply_config): 1480 """Check if we can add an 'imply' option 1481 1482 This finds imply_config in the Kconfig and looks to see if it is possible 1483 to add an 'imply' for 'config' to that part of the Kconfig. 1484 1485 Args: 1486 kconf: Kconfig.Config object 1487 config: Name of config to check (without CONFIG_ prefix) 1488 imply_config: Implying config (without CONFIG_ prefix) which may or 1489 may not have an 'imply' for 'config') 1490 1491 Returns: 1492 tuple: 1493 filename of Kconfig file containing imply_config, or None if none 1494 line number within the Kconfig file, or 0 if none 1495 message indicating the result 1496 """ 1497 sym = kconf.get_symbol(imply_config) 1498 if not sym: 1499 return 'cannot find sym' 1500 locs = sym.get_def_locations() 1501 if len(locs) != 1: 1502 return '%d locations' % len(locs) 1503 fname, linenum = locs[0] 1504 cwd = os.getcwd() 1505 if cwd and fname.startswith(cwd): 1506 fname = fname[len(cwd) + 1:] 1507 file_line = ' at %s:%d' % (fname, linenum) 1508 with open(fname) as fd: 1509 data = fd.read().splitlines() 1510 if data[linenum - 1] != 'config %s' % imply_config: 1511 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line) 1512 return fname, linenum, 'adding%s' % file_line 1513 1514def add_imply_rule(config, fname, linenum): 1515 """Add a new 'imply' option to a Kconfig 1516 1517 Args: 1518 config: config option to add an imply for (without CONFIG_ prefix) 1519 fname: Kconfig filename to update 1520 linenum: Line number to place the 'imply' before 1521 1522 Returns: 1523 Message indicating the result 1524 """ 1525 file_line = ' at %s:%d' % (fname, linenum) 1526 data = open(fname).read().splitlines() 1527 linenum -= 1 1528 1529 for offset, line in enumerate(data[linenum:]): 1530 if line.strip().startswith('help') or not line: 1531 data.insert(linenum + offset, '\timply %s' % config) 1532 with open(fname, 'w') as fd: 1533 fd.write('\n'.join(data) + '\n') 1534 return 'added%s' % file_line 1535 1536 return 'could not insert%s' 1537 1538(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( 1539 1, 2, 4, 8) 1540 1541IMPLY_FLAGS = { 1542 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], 1543 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], 1544 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], 1545 'non-arch-board': [ 1546 IMPLY_NON_ARCH_BOARD, 1547 'Allow Kconfig options outside arch/ and /board/ to imply'], 1548}; 1549 1550def do_imply_config(config_list, add_imply, imply_flags, skip_added, 1551 check_kconfig=True, find_superset=False): 1552 """Find CONFIG options which imply those in the list 1553 1554 Some CONFIG options can be implied by others and this can help to reduce 1555 the size of the defconfig files. For example, CONFIG_X86 implies 1556 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 1557 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 1558 each of the x86 defconfig files. 1559 1560 This function uses the moveconfig database to find such options. It 1561 displays a list of things that could possibly imply those in the list. 1562 The algorithm ignores any that start with CONFIG_TARGET since these 1563 typically refer to only a few defconfigs (often one). It also does not 1564 display a config with less than 5 defconfigs. 1565 1566 The algorithm works using sets. For each target config in config_list: 1567 - Get the set 'defconfigs' which use that target config 1568 - For each config (from a list of all configs): 1569 - Get the set 'imply_defconfig' of defconfigs which use that config 1570 - 1571 - If imply_defconfigs contains anything not in defconfigs then 1572 this config does not imply the target config 1573 1574 Params: 1575 config_list: List of CONFIG options to check (each a string) 1576 add_imply: Automatically add an 'imply' for each config. 1577 imply_flags: Flags which control which implying configs are allowed 1578 (IMPLY_...) 1579 skip_added: Don't show options which already have an imply added. 1580 check_kconfig: Check if implied symbols already have an 'imply' or 1581 'select' for the target config, and show this information if so. 1582 find_superset: True to look for configs which are a superset of those 1583 already found. So for example if CONFIG_EXYNOS5 implies an option, 1584 but CONFIG_EXYNOS covers a larger set of defconfigs and also 1585 implies that option, this will drop the former in favour of the 1586 latter. In practice this option has not proved very used. 1587 1588 Note the terminoloy: 1589 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 1590 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 1591 """ 1592 kconf = KconfigScanner().conf if check_kconfig else None 1593 if add_imply and add_imply != 'all': 1594 add_imply = add_imply.split() 1595 1596 # key is defconfig name, value is dict of (CONFIG_xxx, value) 1597 config_db = {} 1598 1599 # Holds a dict containing the set of defconfigs that contain each config 1600 # key is config, value is set of defconfigs using that config 1601 defconfig_db = collections.defaultdict(set) 1602 1603 # Set of all config options we have seen 1604 all_configs = set() 1605 1606 # Set of all defconfigs we have seen 1607 all_defconfigs = set() 1608 1609 # Read in the database 1610 configs = {} 1611 with open(CONFIG_DATABASE) as fd: 1612 for line in fd.readlines(): 1613 line = line.rstrip() 1614 if not line: # Separator between defconfigs 1615 config_db[defconfig] = configs 1616 all_defconfigs.add(defconfig) 1617 configs = {} 1618 elif line[0] == ' ': # CONFIG line 1619 config, value = line.strip().split('=', 1) 1620 configs[config] = value 1621 defconfig_db[config].add(defconfig) 1622 all_configs.add(config) 1623 else: # New defconfig 1624 defconfig = line 1625 1626 # Work through each target config option in tern, independently 1627 for config in config_list: 1628 defconfigs = defconfig_db.get(config) 1629 if not defconfigs: 1630 print '%s not found in any defconfig' % config 1631 continue 1632 1633 # Get the set of defconfigs without this one (since a config cannot 1634 # imply itself) 1635 non_defconfigs = all_defconfigs - defconfigs 1636 num_defconfigs = len(defconfigs) 1637 print '%s found in %d/%d defconfigs' % (config, num_defconfigs, 1638 len(all_configs)) 1639 1640 # This will hold the results: key=config, value=defconfigs containing it 1641 imply_configs = {} 1642 rest_configs = all_configs - set([config]) 1643 1644 # Look at every possible config, except the target one 1645 for imply_config in rest_configs: 1646 if 'ERRATUM' in imply_config: 1647 continue 1648 if not (imply_flags & IMPLY_CMD): 1649 if 'CONFIG_CMD' in imply_config: 1650 continue 1651 if not (imply_flags & IMPLY_TARGET): 1652 if 'CONFIG_TARGET' in imply_config: 1653 continue 1654 1655 # Find set of defconfigs that have this config 1656 imply_defconfig = defconfig_db[imply_config] 1657 1658 # Get the intersection of this with defconfigs containing the 1659 # target config 1660 common_defconfigs = imply_defconfig & defconfigs 1661 1662 # Get the set of defconfigs containing this config which DO NOT 1663 # also contain the taret config. If this set is non-empty it means 1664 # that this config affects other defconfigs as well as (possibly) 1665 # the ones affected by the target config. This means it implies 1666 # things we don't want to imply. 1667 not_common_defconfigs = imply_defconfig & non_defconfigs 1668 if not_common_defconfigs: 1669 continue 1670 1671 # If there are common defconfigs, imply_config may be useful 1672 if common_defconfigs: 1673 skip = False 1674 if find_superset: 1675 for prev in imply_configs.keys(): 1676 prev_count = len(imply_configs[prev]) 1677 count = len(common_defconfigs) 1678 if (prev_count > count and 1679 (imply_configs[prev] & common_defconfigs == 1680 common_defconfigs)): 1681 # skip imply_config because prev is a superset 1682 skip = True 1683 break 1684 elif count > prev_count: 1685 # delete prev because imply_config is a superset 1686 del imply_configs[prev] 1687 if not skip: 1688 imply_configs[imply_config] = common_defconfigs 1689 1690 # Now we have a dict imply_configs of configs which imply each config 1691 # The value of each dict item is the set of defconfigs containing that 1692 # config. Rank them so that we print the configs that imply the largest 1693 # number of defconfigs first. 1694 ranked_iconfigs = sorted(imply_configs, 1695 key=lambda k: len(imply_configs[k]), reverse=True) 1696 kconfig_info = '' 1697 cwd = os.getcwd() 1698 add_list = collections.defaultdict(list) 1699 for iconfig in ranked_iconfigs: 1700 num_common = len(imply_configs[iconfig]) 1701 1702 # Don't bother if there are less than 5 defconfigs affected. 1703 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): 1704 continue 1705 missing = defconfigs - imply_configs[iconfig] 1706 missing_str = ', '.join(missing) if missing else 'all' 1707 missing_str = '' 1708 show = True 1709 if kconf: 1710 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], 1711 iconfig[CONFIG_LEN:]) 1712 kconfig_info = '' 1713 if sym: 1714 locs = sym.get_def_locations() 1715 if len(locs) == 1: 1716 fname, linenum = locs[0] 1717 if cwd and fname.startswith(cwd): 1718 fname = fname[len(cwd) + 1:] 1719 kconfig_info = '%s:%d' % (fname, linenum) 1720 if skip_added: 1721 show = False 1722 else: 1723 sym = kconf.get_symbol(iconfig[CONFIG_LEN:]) 1724 fname = '' 1725 if sym: 1726 locs = sym.get_def_locations() 1727 if len(locs) == 1: 1728 fname, linenum = locs[0] 1729 if cwd and fname.startswith(cwd): 1730 fname = fname[len(cwd) + 1:] 1731 in_arch_board = not sym or (fname.startswith('arch') or 1732 fname.startswith('board')) 1733 if (not in_arch_board and 1734 not (imply_flags & IMPLY_NON_ARCH_BOARD)): 1735 continue 1736 1737 if add_imply and (add_imply == 'all' or 1738 iconfig in add_imply): 1739 fname, linenum, kconfig_info = (check_imply_rule(kconf, 1740 config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) 1741 if fname: 1742 add_list[fname].append(linenum) 1743 1744 if show and kconfig_info != 'skip': 1745 print '%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30), 1746 kconfig_info, missing_str) 1747 1748 # Having collected a list of things to add, now we add them. We process 1749 # each file from the largest line number to the smallest so that 1750 # earlier additions do not affect our line numbers. E.g. if we added an 1751 # imply at line 20 it would change the position of each line after 1752 # that. 1753 for fname, linenums in add_list.iteritems(): 1754 for linenum in sorted(linenums, reverse=True): 1755 add_imply_rule(config[CONFIG_LEN:], fname, linenum) 1756 1757 1758def main(): 1759 try: 1760 cpu_count = multiprocessing.cpu_count() 1761 except NotImplementedError: 1762 cpu_count = 1 1763 1764 parser = optparse.OptionParser() 1765 # Add options here 1766 parser.add_option('-a', '--add-imply', type='string', default='', 1767 help='comma-separated list of CONFIG options to add ' 1768 "an 'imply' statement to for the CONFIG in -i") 1769 parser.add_option('-A', '--skip-added', action='store_true', default=False, 1770 help="don't show options which are already marked as " 1771 'implying others') 1772 parser.add_option('-b', '--build-db', action='store_true', default=False, 1773 help='build a CONFIG database') 1774 parser.add_option('-c', '--color', action='store_true', default=False, 1775 help='display the log in color') 1776 parser.add_option('-C', '--commit', action='store_true', default=False, 1777 help='Create a git commit for the operation') 1778 parser.add_option('-d', '--defconfigs', type='string', 1779 help='a file containing a list of defconfigs to move, ' 1780 "one per line (for example 'snow_defconfig') " 1781 "or '-' to read from stdin") 1782 parser.add_option('-i', '--imply', action='store_true', default=False, 1783 help='find options which imply others') 1784 parser.add_option('-I', '--imply-flags', type='string', default='', 1785 help="control the -i option ('help' for help") 1786 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1787 help='perform a trial run (show log with no changes)') 1788 parser.add_option('-e', '--exit-on-error', action='store_true', 1789 default=False, 1790 help='exit immediately on any error') 1791 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1792 help='force sync by savedefconfig') 1793 parser.add_option('-S', '--spl', action='store_true', default=False, 1794 help='parse config options defined for SPL build') 1795 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1796 action='store_true', default=False, 1797 help='only cleanup the headers') 1798 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1799 help='the number of jobs to run simultaneously') 1800 parser.add_option('-r', '--git-ref', type='string', 1801 help='the git ref to clone for building the autoconf.mk') 1802 parser.add_option('-y', '--yes', action='store_true', default=False, 1803 help="respond 'yes' to any prompts") 1804 parser.add_option('-v', '--verbose', action='store_true', default=False, 1805 help='show any build errors as boards are built') 1806 parser.usage += ' CONFIG ...' 1807 1808 (options, configs) = parser.parse_args() 1809 1810 if len(configs) == 0 and not any((options.force_sync, options.build_db, 1811 options.imply)): 1812 parser.print_usage() 1813 sys.exit(1) 1814 1815 # prefix the option name with CONFIG_ if missing 1816 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1817 for config in configs ] 1818 1819 check_top_directory() 1820 1821 if options.imply: 1822 imply_flags = 0 1823 if options.imply_flags == 'all': 1824 imply_flags = -1 1825 1826 elif options.imply_flags: 1827 for flag in options.imply_flags.split(','): 1828 bad = flag not in IMPLY_FLAGS 1829 if bad: 1830 print "Invalid flag '%s'" % flag 1831 if flag == 'help' or bad: 1832 print "Imply flags: (separate with ',')" 1833 for name, info in IMPLY_FLAGS.iteritems(): 1834 print ' %-15s: %s' % (name, info[1]) 1835 parser.print_usage() 1836 sys.exit(1) 1837 imply_flags |= IMPLY_FLAGS[flag][0] 1838 1839 do_imply_config(configs, options.add_imply, imply_flags, 1840 options.skip_added) 1841 return 1842 1843 config_db = {} 1844 db_queue = Queue.Queue() 1845 t = DatabaseThread(config_db, db_queue) 1846 t.setDaemon(True) 1847 t.start() 1848 1849 if not options.cleanup_headers_only: 1850 check_clean_directory() 1851 bsettings.Setup('') 1852 toolchains = toolchain.Toolchains() 1853 toolchains.GetSettings() 1854 toolchains.Scan(verbose=False) 1855 move_config(toolchains, configs, options, db_queue) 1856 db_queue.join() 1857 1858 if configs: 1859 cleanup_headers(configs, options) 1860 cleanup_extra_options(configs, options) 1861 cleanup_whitelist(configs, options) 1862 cleanup_readme(configs, options) 1863 1864 if options.commit: 1865 subprocess.call(['git', 'add', '-u']) 1866 if configs: 1867 msg = 'Convert %s %sto Kconfig' % (configs[0], 1868 'et al ' if len(configs) > 1 else '') 1869 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1870 '\n '.join(configs)) 1871 else: 1872 msg = 'configs: Resync with savedefconfig' 1873 msg += '\n\nRsync all defconfig files using moveconfig.py' 1874 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1875 1876 if options.build_db: 1877 with open(CONFIG_DATABASE, 'w') as fd: 1878 for defconfig, configs in config_db.iteritems(): 1879 fd.write('%s\n' % defconfig) 1880 for config in sorted(configs.keys()): 1881 fd.write(' %s=%s\n' % (config, configs[config])) 1882 fd.write('\n') 1883 1884if __name__ == '__main__': 1885 main() 1886