1#!/usr/bin/env python2 2# 3# Author: Masahiro Yamada <yamada.masahiro@socionext.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8""" 9Move config options from headers to defconfig files. 10 11Since Kconfig was introduced to U-Boot, we have worked on moving 12config options from headers to Kconfig (defconfig). 13 14This tool intends to help this tremendous work. 15 16 17Usage 18----- 19 20First, you must edit the Kconfig to add the menu entries for the configs 21you are moving. 22 23And then run this tool giving CONFIG names you want to move. 24For example, if you want to move CONFIG_CMD_USB and CONFIG_SYS_TEXT_BASE, 25simply type as follows: 26 27 $ tools/moveconfig.py CONFIG_CMD_USB CONFIG_SYS_TEXT_BASE 28 29The tool walks through all the defconfig files and move the given CONFIGs. 30 31The log is also displayed on the terminal. 32 33The log is printed for each defconfig as follows: 34 35<defconfig_name> 36 <action1> 37 <action2> 38 <action3> 39 ... 40 41<defconfig_name> is the name of the defconfig. 42 43<action*> shows what the tool did for that defconfig. 44It looks like one of the following: 45 46 - Move 'CONFIG_... ' 47 This config option was moved to the defconfig 48 49 - CONFIG_... is not defined in Kconfig. Do nothing. 50 The entry for this CONFIG was not found in Kconfig. The option is not 51 defined in the config header, either. So, this case can be just skipped. 52 53 - CONFIG_... is not defined in Kconfig (suspicious). Do nothing. 54 This option is defined in the config header, but its entry was not found 55 in Kconfig. 56 There are two common cases: 57 - You forgot to create an entry for the CONFIG before running 58 this tool, or made a typo in a CONFIG passed to this tool. 59 - The entry was hidden due to unmet 'depends on'. 60 The tool does not know if the result is reasonable, so please check it 61 manually. 62 63 - 'CONFIG_...' is the same as the define in Kconfig. Do nothing. 64 The define in the config header matched the one in Kconfig. 65 We do not need to touch it. 66 67 - Compiler is missing. Do nothing. 68 The compiler specified for this architecture was not found 69 in your PATH environment. 70 (If -e option is passed, the tool exits immediately.) 71 72 - Failed to process. 73 An error occurred during processing this defconfig. Skipped. 74 (If -e option is passed, the tool exits immediately on error.) 75 76Finally, you will be asked, Clean up headers? [y/n]: 77 78If you say 'y' here, the unnecessary config defines are removed 79from the config headers (include/configs/*.h). 80It just uses the regex method, so you should not rely on it. 81Just in case, please do 'git diff' to see what happened. 82 83 84How does it work? 85----------------- 86 87This tool runs configuration and builds include/autoconf.mk for every 88defconfig. The config options defined in Kconfig appear in the .config 89file (unless they are hidden because of unmet dependency.) 90On the other hand, the config options defined by board headers are seen 91in include/autoconf.mk. The tool looks for the specified options in both 92of them to decide the appropriate action for the options. If the given 93config option is found in the .config, but its value does not match the 94one from the board header, the config option in the .config is replaced 95with the define in the board header. Then, the .config is synced by 96"make savedefconfig" and the defconfig is updated with it. 97 98For faster processing, this tool handles multi-threading. It creates 99separate build directories where the out-of-tree build is run. The 100temporary build directories are automatically created and deleted as 101needed. The number of threads are chosen based on the number of the CPU 102cores of your system although you can change it via -j (--jobs) option. 103 104 105Toolchains 106---------- 107 108Appropriate toolchain are necessary to generate include/autoconf.mk 109for all the architectures supported by U-Boot. Most of them are available 110at the kernel.org site, some are not provided by kernel.org. This tool uses 111the same tools as buildman, so see that tool for setup (e.g. --fetch-arch). 112 113 114Tips and trips 115-------------- 116 117To sync only X86 defconfigs: 118 119 ./tools/moveconfig.py -s -d <(grep -l X86 configs/*) 120 121or: 122 123 grep -l X86 configs/* | ./tools/moveconfig.py -s -d - 124 125To process CONFIG_CMD_FPGAD only for a subset of configs based on path match: 126 127 ls configs/{hrcon*,iocon*,strider*} | \ 128 ./tools/moveconfig.py -Cy CONFIG_CMD_FPGAD -d - 129 130 131Finding implied CONFIGs 132----------------------- 133 134Some CONFIG options can be implied by others and this can help to reduce 135the size of the defconfig files. For example, CONFIG_X86 implies 136CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 137all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 138each of the x86 defconfig files. 139 140This tool can help find such configs. To use it, first build a database: 141 142 ./tools/moveconfig.py -b 143 144Then try to query it: 145 146 ./tools/moveconfig.py -i CONFIG_CMD_IRQ 147 CONFIG_CMD_IRQ found in 311/2384 defconfigs 148 44 : CONFIG_SYS_FSL_ERRATUM_IFC_A002769 149 41 : CONFIG_SYS_FSL_ERRATUM_A007075 150 31 : CONFIG_SYS_FSL_DDR_VER_44 151 28 : CONFIG_ARCH_P1010 152 28 : CONFIG_SYS_FSL_ERRATUM_P1010_A003549 153 28 : CONFIG_SYS_FSL_ERRATUM_SEC_A003571 154 28 : CONFIG_SYS_FSL_ERRATUM_IFC_A003399 155 25 : CONFIG_SYS_FSL_ERRATUM_A008044 156 22 : CONFIG_ARCH_P1020 157 21 : CONFIG_SYS_FSL_DDR_VER_46 158 20 : CONFIG_MAX_PIRQ_LINKS 159 20 : CONFIG_HPET_ADDRESS 160 20 : CONFIG_X86 161 20 : CONFIG_PCIE_ECAM_SIZE 162 20 : CONFIG_IRQ_SLOT_COUNT 163 20 : CONFIG_I8259_PIC 164 20 : CONFIG_CPU_ADDR_BITS 165 20 : CONFIG_RAMBASE 166 20 : CONFIG_SYS_FSL_ERRATUM_A005871 167 20 : CONFIG_PCIE_ECAM_BASE 168 20 : CONFIG_X86_TSC_TIMER 169 20 : CONFIG_I8254_TIMER 170 20 : CONFIG_CMD_GETTIME 171 19 : CONFIG_SYS_FSL_ERRATUM_A005812 172 18 : CONFIG_X86_RUN_32BIT 173 17 : CONFIG_CMD_CHIP_CONFIG 174 ... 175 176This shows a list of config options which might imply CONFIG_CMD_EEPROM along 177with how many defconfigs they cover. From this you can see that CONFIG_X86 178implies CONFIG_CMD_EEPROM. Therefore, instead of adding CONFIG_CMD_EEPROM to 179the defconfig of every x86 board, you could add a single imply line to the 180Kconfig file: 181 182 config X86 183 bool "x86 architecture" 184 ... 185 imply CMD_EEPROM 186 187That will cover 20 defconfigs. Many of the options listed are not suitable as 188they are not related. E.g. it would be odd for CONFIG_CMD_GETTIME to imply 189CMD_EEPROM. 190 191Using this search you can reduce the size of moveconfig patches. 192 193You can automatically add 'imply' statements in the Kconfig with the -a 194option: 195 196 ./tools/moveconfig.py -s -i CONFIG_SCSI \ 197 -a CONFIG_ARCH_LS1021A,CONFIG_ARCH_LS1043A 198 199This will add 'imply SCSI' to the two CONFIG options mentioned, assuming that 200the database indicates that they do actually imply CONFIG_SCSI and do not 201already have an 'imply SCSI'. 202 203The output shows where the imply is added: 204 205 18 : CONFIG_ARCH_LS1021A arch/arm/cpu/armv7/ls102xa/Kconfig:1 206 13 : CONFIG_ARCH_LS1043A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:11 207 12 : CONFIG_ARCH_LS1046A arch/arm/cpu/armv8/fsl-layerscape/Kconfig:31 208 209The first number is the number of boards which can avoid having a special 210CONFIG_SCSI option in their defconfig file if this 'imply' is added. 211The location at the right is the Kconfig file and line number where the config 212appears. For example, adding 'imply CONFIG_SCSI' to the 'config ARCH_LS1021A' 213in arch/arm/cpu/armv7/ls102xa/Kconfig at line 1 will help 18 boards to reduce 214the size of their defconfig files. 215 216If you want to add an 'imply' to every imply config in the list, you can use 217 218 ./tools/moveconfig.py -s -i CONFIG_SCSI -a all 219 220To control which ones are displayed, use -I <list> where list is a list of 221options (use '-I help' to see possible options and their meaning). 222 223To skip showing you options that already have an 'imply' attached, use -A. 224 225When you have finished adding 'imply' options you can regenerate the 226defconfig files for affected boards with something like: 227 228 git show --stat | ./tools/moveconfig.py -s -d - 229 230This will regenerate only those defconfigs changed in the current commit. 231If you start with (say) 100 defconfigs being changed in the commit, and add 232a few 'imply' options as above, then regenerate, hopefully you can reduce the 233number of defconfigs changed in the commit. 234 235 236Available options 237----------------- 238 239 -c, --color 240 Surround each portion of the log with escape sequences to display it 241 in color on the terminal. 242 243 -C, --commit 244 Create a git commit with the changes when the operation is complete. A 245 standard commit message is used which may need to be edited. 246 247 -d, --defconfigs 248 Specify a file containing a list of defconfigs to move. The defconfig 249 files can be given with shell-style wildcards. Use '-' to read from stdin. 250 251 -n, --dry-run 252 Perform a trial run that does not make any changes. It is useful to 253 see what is going to happen before one actually runs it. 254 255 -e, --exit-on-error 256 Exit immediately if Make exits with a non-zero status while processing 257 a defconfig file. 258 259 -s, --force-sync 260 Do "make savedefconfig" forcibly for all the defconfig files. 261 If not specified, "make savedefconfig" only occurs for cases 262 where at least one CONFIG was moved. 263 264 -S, --spl 265 Look for moved config options in spl/include/autoconf.mk instead of 266 include/autoconf.mk. This is useful for moving options for SPL build 267 because SPL related options (mostly prefixed with CONFIG_SPL_) are 268 sometimes blocked by CONFIG_SPL_BUILD ifdef conditionals. 269 270 -H, --headers-only 271 Only cleanup the headers; skip the defconfig processing 272 273 -j, --jobs 274 Specify the number of threads to run simultaneously. If not specified, 275 the number of threads is the same as the number of CPU cores. 276 277 -r, --git-ref 278 Specify the git ref to clone for building the autoconf.mk. If unspecified 279 use the CWD. This is useful for when changes to the Kconfig affect the 280 default values and you want to capture the state of the defconfig from 281 before that change was in effect. If in doubt, specify a ref pre-Kconfig 282 changes (use HEAD if Kconfig changes are not committed). Worst case it will 283 take a bit longer to run, but will always do the right thing. 284 285 -v, --verbose 286 Show any build errors as boards are built 287 288 -y, --yes 289 Instead of prompting, automatically go ahead with all operations. This 290 includes cleaning up headers, CONFIG_SYS_EXTRA_OPTIONS, the config whitelist 291 and the README. 292 293To see the complete list of supported options, run 294 295 $ tools/moveconfig.py -h 296 297""" 298 299import collections 300import copy 301import difflib 302import filecmp 303import fnmatch 304import glob 305import multiprocessing 306import optparse 307import os 308import Queue 309import re 310import shutil 311import subprocess 312import sys 313import tempfile 314import threading 315import time 316 317sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman')) 318sys.path.append(os.path.join(os.path.dirname(__file__), 'patman')) 319import bsettings 320import kconfiglib 321import toolchain 322 323SHOW_GNU_MAKE = 'scripts/show-gnu-make' 324SLEEP_TIME=0.03 325 326STATE_IDLE = 0 327STATE_DEFCONFIG = 1 328STATE_AUTOCONF = 2 329STATE_SAVEDEFCONFIG = 3 330 331ACTION_MOVE = 0 332ACTION_NO_ENTRY = 1 333ACTION_NO_ENTRY_WARN = 2 334ACTION_NO_CHANGE = 3 335 336COLOR_BLACK = '0;30' 337COLOR_RED = '0;31' 338COLOR_GREEN = '0;32' 339COLOR_BROWN = '0;33' 340COLOR_BLUE = '0;34' 341COLOR_PURPLE = '0;35' 342COLOR_CYAN = '0;36' 343COLOR_LIGHT_GRAY = '0;37' 344COLOR_DARK_GRAY = '1;30' 345COLOR_LIGHT_RED = '1;31' 346COLOR_LIGHT_GREEN = '1;32' 347COLOR_YELLOW = '1;33' 348COLOR_LIGHT_BLUE = '1;34' 349COLOR_LIGHT_PURPLE = '1;35' 350COLOR_LIGHT_CYAN = '1;36' 351COLOR_WHITE = '1;37' 352 353AUTO_CONF_PATH = 'include/config/auto.conf' 354CONFIG_DATABASE = 'moveconfig.db' 355 356CONFIG_LEN = len('CONFIG_') 357 358### helper functions ### 359def get_devnull(): 360 """Get the file object of '/dev/null' device.""" 361 try: 362 devnull = subprocess.DEVNULL # py3k 363 except AttributeError: 364 devnull = open(os.devnull, 'wb') 365 return devnull 366 367def check_top_directory(): 368 """Exit if we are not at the top of source directory.""" 369 for f in ('README', 'Licenses'): 370 if not os.path.exists(f): 371 sys.exit('Please run at the top of source directory.') 372 373def check_clean_directory(): 374 """Exit if the source tree is not clean.""" 375 for f in ('.config', 'include/config'): 376 if os.path.exists(f): 377 sys.exit("source tree is not clean, please run 'make mrproper'") 378 379def get_make_cmd(): 380 """Get the command name of GNU Make. 381 382 U-Boot needs GNU Make for building, but the command name is not 383 necessarily "make". (for example, "gmake" on FreeBSD). 384 Returns the most appropriate command name on your system. 385 """ 386 process = subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) 387 ret = process.communicate() 388 if process.returncode: 389 sys.exit('GNU Make not found') 390 return ret[0].rstrip() 391 392def get_matched_defconfig(line): 393 """Get the defconfig files that match a pattern 394 395 Args: 396 line: Path or filename to match, e.g. 'configs/snow_defconfig' or 397 'k2*_defconfig'. If no directory is provided, 'configs/' is 398 prepended 399 400 Returns: 401 a list of matching defconfig files 402 """ 403 dirname = os.path.dirname(line) 404 if dirname: 405 pattern = line 406 else: 407 pattern = os.path.join('configs', line) 408 return glob.glob(pattern) + glob.glob(pattern + '_defconfig') 409 410def get_matched_defconfigs(defconfigs_file): 411 """Get all the defconfig files that match the patterns in a file. 412 413 Args: 414 defconfigs_file: File containing a list of defconfigs to process, or 415 '-' to read the list from stdin 416 417 Returns: 418 A list of paths to defconfig files, with no duplicates 419 """ 420 defconfigs = [] 421 if defconfigs_file == '-': 422 fd = sys.stdin 423 defconfigs_file = 'stdin' 424 else: 425 fd = open(defconfigs_file) 426 for i, line in enumerate(fd): 427 line = line.strip() 428 if not line: 429 continue # skip blank lines silently 430 if ' ' in line: 431 line = line.split(' ')[0] # handle 'git log' input 432 matched = get_matched_defconfig(line) 433 if not matched: 434 print >> sys.stderr, "warning: %s:%d: no defconfig matched '%s'" % \ 435 (defconfigs_file, i + 1, line) 436 437 defconfigs += matched 438 439 # use set() to drop multiple matching 440 return [ defconfig[len('configs') + 1:] for defconfig in set(defconfigs) ] 441 442def get_all_defconfigs(): 443 """Get all the defconfig files under the configs/ directory.""" 444 defconfigs = [] 445 for (dirpath, dirnames, filenames) in os.walk('configs'): 446 dirpath = dirpath[len('configs') + 1:] 447 for filename in fnmatch.filter(filenames, '*_defconfig'): 448 defconfigs.append(os.path.join(dirpath, filename)) 449 450 return defconfigs 451 452def color_text(color_enabled, color, string): 453 """Return colored string.""" 454 if color_enabled: 455 # LF should not be surrounded by the escape sequence. 456 # Otherwise, additional whitespace or line-feed might be printed. 457 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else '' 458 for s in string.split('\n') ]) 459 else: 460 return string 461 462def show_diff(a, b, file_path, color_enabled): 463 """Show unidified diff. 464 465 Arguments: 466 a: A list of lines (before) 467 b: A list of lines (after) 468 file_path: Path to the file 469 color_enabled: Display the diff in color 470 """ 471 472 diff = difflib.unified_diff(a, b, 473 fromfile=os.path.join('a', file_path), 474 tofile=os.path.join('b', file_path)) 475 476 for line in diff: 477 if line[0] == '-' and line[1] != '-': 478 print color_text(color_enabled, COLOR_RED, line), 479 elif line[0] == '+' and line[1] != '+': 480 print color_text(color_enabled, COLOR_GREEN, line), 481 else: 482 print line, 483 484def extend_matched_lines(lines, matched, pre_patterns, post_patterns, extend_pre, 485 extend_post): 486 """Extend matched lines if desired patterns are found before/after already 487 matched lines. 488 489 Arguments: 490 lines: A list of lines handled. 491 matched: A list of line numbers that have been already matched. 492 (will be updated by this function) 493 pre_patterns: A list of regular expression that should be matched as 494 preamble. 495 post_patterns: A list of regular expression that should be matched as 496 postamble. 497 extend_pre: Add the line number of matched preamble to the matched list. 498 extend_post: Add the line number of matched postamble to the matched list. 499 """ 500 extended_matched = [] 501 502 j = matched[0] 503 504 for i in matched: 505 if i == 0 or i < j: 506 continue 507 j = i 508 while j in matched: 509 j += 1 510 if j >= len(lines): 511 break 512 513 for p in pre_patterns: 514 if p.search(lines[i - 1]): 515 break 516 else: 517 # not matched 518 continue 519 520 for p in post_patterns: 521 if p.search(lines[j]): 522 break 523 else: 524 # not matched 525 continue 526 527 if extend_pre: 528 extended_matched.append(i - 1) 529 if extend_post: 530 extended_matched.append(j) 531 532 matched += extended_matched 533 matched.sort() 534 535def confirm(options, prompt): 536 if not options.yes: 537 while True: 538 choice = raw_input('{} [y/n]: '.format(prompt)) 539 choice = choice.lower() 540 print choice 541 if choice == 'y' or choice == 'n': 542 break 543 544 if choice == 'n': 545 return False 546 547 return True 548 549def cleanup_one_header(header_path, patterns, options): 550 """Clean regex-matched lines away from a file. 551 552 Arguments: 553 header_path: path to the cleaned file. 554 patterns: list of regex patterns. Any lines matching to these 555 patterns are deleted. 556 options: option flags. 557 """ 558 with open(header_path) as f: 559 lines = f.readlines() 560 561 matched = [] 562 for i, line in enumerate(lines): 563 if i - 1 in matched and lines[i - 1][-2:] == '\\\n': 564 matched.append(i) 565 continue 566 for pattern in patterns: 567 if pattern.search(line): 568 matched.append(i) 569 break 570 571 if not matched: 572 return 573 574 # remove empty #ifdef ... #endif, successive blank lines 575 pattern_if = re.compile(r'#\s*if(def|ndef)?\W') # #if, #ifdef, #ifndef 576 pattern_elif = re.compile(r'#\s*el(if|se)\W') # #elif, #else 577 pattern_endif = re.compile(r'#\s*endif\W') # #endif 578 pattern_blank = re.compile(r'^\s*$') # empty line 579 580 while True: 581 old_matched = copy.copy(matched) 582 extend_matched_lines(lines, matched, [pattern_if], 583 [pattern_endif], True, True) 584 extend_matched_lines(lines, matched, [pattern_elif], 585 [pattern_elif, pattern_endif], True, False) 586 extend_matched_lines(lines, matched, [pattern_if, pattern_elif], 587 [pattern_blank], False, True) 588 extend_matched_lines(lines, matched, [pattern_blank], 589 [pattern_elif, pattern_endif], True, False) 590 extend_matched_lines(lines, matched, [pattern_blank], 591 [pattern_blank], True, False) 592 if matched == old_matched: 593 break 594 595 tolines = copy.copy(lines) 596 597 for i in reversed(matched): 598 tolines.pop(i) 599 600 show_diff(lines, tolines, header_path, options.color) 601 602 if options.dry_run: 603 return 604 605 with open(header_path, 'w') as f: 606 for line in tolines: 607 f.write(line) 608 609def cleanup_headers(configs, options): 610 """Delete config defines from board headers. 611 612 Arguments: 613 configs: A list of CONFIGs to remove. 614 options: option flags. 615 """ 616 if not confirm(options, 'Clean up headers?'): 617 return 618 619 patterns = [] 620 for config in configs: 621 patterns.append(re.compile(r'#\s*define\s+%s\W' % config)) 622 patterns.append(re.compile(r'#\s*undef\s+%s\W' % config)) 623 624 for dir in 'include', 'arch', 'board': 625 for (dirpath, dirnames, filenames) in os.walk(dir): 626 if dirpath == os.path.join('include', 'generated'): 627 continue 628 for filename in filenames: 629 if not fnmatch.fnmatch(filename, '*~'): 630 cleanup_one_header(os.path.join(dirpath, filename), 631 patterns, options) 632 633def cleanup_one_extra_option(defconfig_path, configs, options): 634 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in one defconfig file. 635 636 Arguments: 637 defconfig_path: path to the cleaned defconfig file. 638 configs: A list of CONFIGs to remove. 639 options: option flags. 640 """ 641 642 start = 'CONFIG_SYS_EXTRA_OPTIONS="' 643 end = '"\n' 644 645 with open(defconfig_path) as f: 646 lines = f.readlines() 647 648 for i, line in enumerate(lines): 649 if line.startswith(start) and line.endswith(end): 650 break 651 else: 652 # CONFIG_SYS_EXTRA_OPTIONS was not found in this defconfig 653 return 654 655 old_tokens = line[len(start):-len(end)].split(',') 656 new_tokens = [] 657 658 for token in old_tokens: 659 pos = token.find('=') 660 if not (token[:pos] if pos >= 0 else token) in configs: 661 new_tokens.append(token) 662 663 if new_tokens == old_tokens: 664 return 665 666 tolines = copy.copy(lines) 667 668 if new_tokens: 669 tolines[i] = start + ','.join(new_tokens) + end 670 else: 671 tolines.pop(i) 672 673 show_diff(lines, tolines, defconfig_path, options.color) 674 675 if options.dry_run: 676 return 677 678 with open(defconfig_path, 'w') as f: 679 for line in tolines: 680 f.write(line) 681 682def cleanup_extra_options(configs, options): 683 """Delete config defines in CONFIG_SYS_EXTRA_OPTIONS in defconfig files. 684 685 Arguments: 686 configs: A list of CONFIGs to remove. 687 options: option flags. 688 """ 689 if not confirm(options, 'Clean up CONFIG_SYS_EXTRA_OPTIONS?'): 690 return 691 692 configs = [ config[len('CONFIG_'):] for config in configs ] 693 694 defconfigs = get_all_defconfigs() 695 696 for defconfig in defconfigs: 697 cleanup_one_extra_option(os.path.join('configs', defconfig), configs, 698 options) 699 700def cleanup_whitelist(configs, options): 701 """Delete config whitelist entries 702 703 Arguments: 704 configs: A list of CONFIGs to remove. 705 options: option flags. 706 """ 707 if not confirm(options, 'Clean up whitelist entries?'): 708 return 709 710 with open(os.path.join('scripts', 'config_whitelist.txt')) as f: 711 lines = f.readlines() 712 713 lines = [x for x in lines if x.strip() not in configs] 714 715 with open(os.path.join('scripts', 'config_whitelist.txt'), 'w') as f: 716 f.write(''.join(lines)) 717 718def find_matching(patterns, line): 719 for pat in patterns: 720 if pat.search(line): 721 return True 722 return False 723 724def cleanup_readme(configs, options): 725 """Delete config description in README 726 727 Arguments: 728 configs: A list of CONFIGs to remove. 729 options: option flags. 730 """ 731 if not confirm(options, 'Clean up README?'): 732 return 733 734 patterns = [] 735 for config in configs: 736 patterns.append(re.compile(r'^\s+%s' % config)) 737 738 with open('README') as f: 739 lines = f.readlines() 740 741 found = False 742 newlines = [] 743 for line in lines: 744 if not found: 745 found = find_matching(patterns, line) 746 if found: 747 continue 748 749 if found and re.search(r'^\s+CONFIG', line): 750 found = False 751 752 if not found: 753 newlines.append(line) 754 755 with open('README', 'w') as f: 756 f.write(''.join(newlines)) 757 758 759### classes ### 760class Progress: 761 762 """Progress Indicator""" 763 764 def __init__(self, total): 765 """Create a new progress indicator. 766 767 Arguments: 768 total: A number of defconfig files to process. 769 """ 770 self.current = 0 771 self.total = total 772 773 def inc(self): 774 """Increment the number of processed defconfig files.""" 775 776 self.current += 1 777 778 def show(self): 779 """Display the progress.""" 780 print ' %d defconfigs out of %d\r' % (self.current, self.total), 781 sys.stdout.flush() 782 783 784class KconfigScanner: 785 """Kconfig scanner.""" 786 787 def __init__(self): 788 """Scan all the Kconfig files and create a Config object.""" 789 # Define environment variables referenced from Kconfig 790 os.environ['srctree'] = os.getcwd() 791 os.environ['UBOOTVERSION'] = 'dummy' 792 os.environ['KCONFIG_OBJDIR'] = '' 793 self.conf = kconfiglib.Config() 794 795 796class KconfigParser: 797 798 """A parser of .config and include/autoconf.mk.""" 799 800 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') 801 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') 802 803 def __init__(self, configs, options, build_dir): 804 """Create a new parser. 805 806 Arguments: 807 configs: A list of CONFIGs to move. 808 options: option flags. 809 build_dir: Build directory. 810 """ 811 self.configs = configs 812 self.options = options 813 self.dotconfig = os.path.join(build_dir, '.config') 814 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') 815 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', 816 'autoconf.mk') 817 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) 818 self.defconfig = os.path.join(build_dir, 'defconfig') 819 820 def get_arch(self): 821 """Parse .config file and return the architecture. 822 823 Returns: 824 Architecture name (e.g. 'arm'). 825 """ 826 arch = '' 827 cpu = '' 828 for line in open(self.dotconfig): 829 m = self.re_arch.match(line) 830 if m: 831 arch = m.group(1) 832 continue 833 m = self.re_cpu.match(line) 834 if m: 835 cpu = m.group(1) 836 837 if not arch: 838 return None 839 840 # fix-up for aarch64 841 if arch == 'arm' and cpu == 'armv8': 842 arch = 'aarch64' 843 844 return arch 845 846 def parse_one_config(self, config, dotconfig_lines, autoconf_lines): 847 """Parse .config, defconfig, include/autoconf.mk for one config. 848 849 This function looks for the config options in the lines from 850 defconfig, .config, and include/autoconf.mk in order to decide 851 which action should be taken for this defconfig. 852 853 Arguments: 854 config: CONFIG name to parse. 855 dotconfig_lines: lines from the .config file. 856 autoconf_lines: lines from the include/autoconf.mk file. 857 858 Returns: 859 A tupple of the action for this defconfig and the line 860 matched for the config. 861 """ 862 not_set = '# %s is not set' % config 863 864 for line in autoconf_lines: 865 line = line.rstrip() 866 if line.startswith(config + '='): 867 new_val = line 868 break 869 else: 870 new_val = not_set 871 872 for line in dotconfig_lines: 873 line = line.rstrip() 874 if line.startswith(config + '=') or line == not_set: 875 old_val = line 876 break 877 else: 878 if new_val == not_set: 879 return (ACTION_NO_ENTRY, config) 880 else: 881 return (ACTION_NO_ENTRY_WARN, config) 882 883 # If this CONFIG is neither bool nor trisate 884 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set: 885 # tools/scripts/define2mk.sed changes '1' to 'y'. 886 # This is a problem if the CONFIG is int type. 887 # Check the type in Kconfig and handle it correctly. 888 if new_val[-2:] == '=y': 889 new_val = new_val[:-1] + '1' 890 891 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE, 892 new_val) 893 894 def update_dotconfig(self): 895 """Parse files for the config options and update the .config. 896 897 This function parses the generated .config and include/autoconf.mk 898 searching the target options. 899 Move the config option(s) to the .config as needed. 900 901 Arguments: 902 defconfig: defconfig name. 903 904 Returns: 905 Return a tuple of (updated flag, log string). 906 The "updated flag" is True if the .config was updated, False 907 otherwise. The "log string" shows what happend to the .config. 908 """ 909 910 results = [] 911 updated = False 912 suspicious = False 913 rm_files = [self.config_autoconf, self.autoconf] 914 915 if self.options.spl: 916 if os.path.exists(self.spl_autoconf): 917 autoconf_path = self.spl_autoconf 918 rm_files.append(self.spl_autoconf) 919 else: 920 for f in rm_files: 921 os.remove(f) 922 return (updated, suspicious, 923 color_text(self.options.color, COLOR_BROWN, 924 "SPL is not enabled. Skipped.") + '\n') 925 else: 926 autoconf_path = self.autoconf 927 928 with open(self.dotconfig) as f: 929 dotconfig_lines = f.readlines() 930 931 with open(autoconf_path) as f: 932 autoconf_lines = f.readlines() 933 934 for config in self.configs: 935 result = self.parse_one_config(config, dotconfig_lines, 936 autoconf_lines) 937 results.append(result) 938 939 log = '' 940 941 for (action, value) in results: 942 if action == ACTION_MOVE: 943 actlog = "Move '%s'" % value 944 log_color = COLOR_LIGHT_GREEN 945 elif action == ACTION_NO_ENTRY: 946 actlog = "%s is not defined in Kconfig. Do nothing." % value 947 log_color = COLOR_LIGHT_BLUE 948 elif action == ACTION_NO_ENTRY_WARN: 949 actlog = "%s is not defined in Kconfig (suspicious). Do nothing." % value 950 log_color = COLOR_YELLOW 951 suspicious = True 952 elif action == ACTION_NO_CHANGE: 953 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \ 954 % value 955 log_color = COLOR_LIGHT_PURPLE 956 elif action == ACTION_SPL_NOT_EXIST: 957 actlog = "SPL is not enabled for this defconfig. Skip." 958 log_color = COLOR_PURPLE 959 else: 960 sys.exit("Internal Error. This should not happen.") 961 962 log += color_text(self.options.color, log_color, actlog) + '\n' 963 964 with open(self.dotconfig, 'a') as f: 965 for (action, value) in results: 966 if action == ACTION_MOVE: 967 f.write(value + '\n') 968 updated = True 969 970 self.results = results 971 for f in rm_files: 972 os.remove(f) 973 974 return (updated, suspicious, log) 975 976 def check_defconfig(self): 977 """Check the defconfig after savedefconfig 978 979 Returns: 980 Return additional log if moved CONFIGs were removed again by 981 'make savedefconfig'. 982 """ 983 984 log = '' 985 986 with open(self.defconfig) as f: 987 defconfig_lines = f.readlines() 988 989 for (action, value) in self.results: 990 if action != ACTION_MOVE: 991 continue 992 if not value + '\n' in defconfig_lines: 993 log += color_text(self.options.color, COLOR_YELLOW, 994 "'%s' was removed by savedefconfig.\n" % 995 value) 996 997 return log 998 999 1000class DatabaseThread(threading.Thread): 1001 """This thread processes results from Slot threads. 1002 1003 It collects the data in the master config directary. There is only one 1004 result thread, and this helps to serialise the build output. 1005 """ 1006 def __init__(self, config_db, db_queue): 1007 """Set up a new result thread 1008 1009 Args: 1010 builder: Builder which will be sent each result 1011 """ 1012 threading.Thread.__init__(self) 1013 self.config_db = config_db 1014 self.db_queue= db_queue 1015 1016 def run(self): 1017 """Called to start up the result thread. 1018 1019 We collect the next result job and pass it on to the build. 1020 """ 1021 while True: 1022 defconfig, configs = self.db_queue.get() 1023 self.config_db[defconfig] = configs 1024 self.db_queue.task_done() 1025 1026 1027class Slot: 1028 1029 """A slot to store a subprocess. 1030 1031 Each instance of this class handles one subprocess. 1032 This class is useful to control multiple threads 1033 for faster processing. 1034 """ 1035 1036 def __init__(self, toolchains, configs, options, progress, devnull, 1037 make_cmd, reference_src_dir, db_queue): 1038 """Create a new process slot. 1039 1040 Arguments: 1041 toolchains: Toolchains object containing toolchains. 1042 configs: A list of CONFIGs to move. 1043 options: option flags. 1044 progress: A progress indicator. 1045 devnull: A file object of '/dev/null'. 1046 make_cmd: command name of GNU Make. 1047 reference_src_dir: Determine the true starting config state from this 1048 source tree. 1049 db_queue: output queue to write config info for the database 1050 """ 1051 self.toolchains = toolchains 1052 self.options = options 1053 self.progress = progress 1054 self.build_dir = tempfile.mkdtemp() 1055 self.devnull = devnull 1056 self.make_cmd = (make_cmd, 'O=' + self.build_dir) 1057 self.reference_src_dir = reference_src_dir 1058 self.db_queue = db_queue 1059 self.parser = KconfigParser(configs, options, self.build_dir) 1060 self.state = STATE_IDLE 1061 self.failed_boards = set() 1062 self.suspicious_boards = set() 1063 1064 def __del__(self): 1065 """Delete the working directory 1066 1067 This function makes sure the temporary directory is cleaned away 1068 even if Python suddenly dies due to error. It should be done in here 1069 because it is guaranteed the destructor is always invoked when the 1070 instance of the class gets unreferenced. 1071 1072 If the subprocess is still running, wait until it finishes. 1073 """ 1074 if self.state != STATE_IDLE: 1075 while self.ps.poll() == None: 1076 pass 1077 shutil.rmtree(self.build_dir) 1078 1079 def add(self, defconfig): 1080 """Assign a new subprocess for defconfig and add it to the slot. 1081 1082 If the slot is vacant, create a new subprocess for processing the 1083 given defconfig and add it to the slot. Just returns False if 1084 the slot is occupied (i.e. the current subprocess is still running). 1085 1086 Arguments: 1087 defconfig: defconfig name. 1088 1089 Returns: 1090 Return True on success or False on failure 1091 """ 1092 if self.state != STATE_IDLE: 1093 return False 1094 1095 self.defconfig = defconfig 1096 self.log = '' 1097 self.current_src_dir = self.reference_src_dir 1098 self.do_defconfig() 1099 return True 1100 1101 def poll(self): 1102 """Check the status of the subprocess and handle it as needed. 1103 1104 Returns True if the slot is vacant (i.e. in idle state). 1105 If the configuration is successfully finished, assign a new 1106 subprocess to build include/autoconf.mk. 1107 If include/autoconf.mk is generated, invoke the parser to 1108 parse the .config and the include/autoconf.mk, moving 1109 config options to the .config as needed. 1110 If the .config was updated, run "make savedefconfig" to sync 1111 it, update the original defconfig, and then set the slot back 1112 to the idle state. 1113 1114 Returns: 1115 Return True if the subprocess is terminated, False otherwise 1116 """ 1117 if self.state == STATE_IDLE: 1118 return True 1119 1120 if self.ps.poll() == None: 1121 return False 1122 1123 if self.ps.poll() != 0: 1124 self.handle_error() 1125 elif self.state == STATE_DEFCONFIG: 1126 if self.reference_src_dir and not self.current_src_dir: 1127 self.do_savedefconfig() 1128 else: 1129 self.do_autoconf() 1130 elif self.state == STATE_AUTOCONF: 1131 if self.current_src_dir: 1132 self.current_src_dir = None 1133 self.do_defconfig() 1134 elif self.options.build_db: 1135 self.do_build_db() 1136 else: 1137 self.do_savedefconfig() 1138 elif self.state == STATE_SAVEDEFCONFIG: 1139 self.update_defconfig() 1140 else: 1141 sys.exit("Internal Error. This should not happen.") 1142 1143 return True if self.state == STATE_IDLE else False 1144 1145 def handle_error(self): 1146 """Handle error cases.""" 1147 1148 self.log += color_text(self.options.color, COLOR_LIGHT_RED, 1149 "Failed to process.\n") 1150 if self.options.verbose: 1151 self.log += color_text(self.options.color, COLOR_LIGHT_CYAN, 1152 self.ps.stderr.read()) 1153 self.finish(False) 1154 1155 def do_defconfig(self): 1156 """Run 'make <board>_defconfig' to create the .config file.""" 1157 1158 cmd = list(self.make_cmd) 1159 cmd.append(self.defconfig) 1160 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1161 stderr=subprocess.PIPE, 1162 cwd=self.current_src_dir) 1163 self.state = STATE_DEFCONFIG 1164 1165 def do_autoconf(self): 1166 """Run 'make AUTO_CONF_PATH'.""" 1167 1168 arch = self.parser.get_arch() 1169 try: 1170 toolchain = self.toolchains.Select(arch) 1171 except ValueError: 1172 self.log += color_text(self.options.color, COLOR_YELLOW, 1173 "Tool chain for '%s' is missing. Do nothing.\n % arch") 1174 self.finish(False) 1175 return 1176 env = toolchain.MakeEnvironment(False) 1177 1178 cmd = list(self.make_cmd) 1179 cmd.append('KCONFIG_IGNORE_DUPLICATES=1') 1180 cmd.append(AUTO_CONF_PATH) 1181 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env, 1182 stderr=subprocess.PIPE, 1183 cwd=self.current_src_dir) 1184 self.state = STATE_AUTOCONF 1185 1186 def do_build_db(self): 1187 """Add the board to the database""" 1188 configs = {} 1189 with open(os.path.join(self.build_dir, AUTO_CONF_PATH)) as fd: 1190 for line in fd.readlines(): 1191 if line.startswith('CONFIG'): 1192 config, value = line.split('=', 1) 1193 configs[config] = value.rstrip() 1194 self.db_queue.put([self.defconfig, configs]) 1195 self.finish(True) 1196 1197 def do_savedefconfig(self): 1198 """Update the .config and run 'make savedefconfig'.""" 1199 1200 (updated, suspicious, log) = self.parser.update_dotconfig() 1201 if suspicious: 1202 self.suspicious_boards.add(self.defconfig) 1203 self.log += log 1204 1205 if not self.options.force_sync and not updated: 1206 self.finish(True) 1207 return 1208 if updated: 1209 self.log += color_text(self.options.color, COLOR_LIGHT_GREEN, 1210 "Syncing by savedefconfig...\n") 1211 else: 1212 self.log += "Syncing by savedefconfig (forced by option)...\n" 1213 1214 cmd = list(self.make_cmd) 1215 cmd.append('savedefconfig') 1216 self.ps = subprocess.Popen(cmd, stdout=self.devnull, 1217 stderr=subprocess.PIPE) 1218 self.state = STATE_SAVEDEFCONFIG 1219 1220 def update_defconfig(self): 1221 """Update the input defconfig and go back to the idle state.""" 1222 1223 log = self.parser.check_defconfig() 1224 if log: 1225 self.suspicious_boards.add(self.defconfig) 1226 self.log += log 1227 orig_defconfig = os.path.join('configs', self.defconfig) 1228 new_defconfig = os.path.join(self.build_dir, 'defconfig') 1229 updated = not filecmp.cmp(orig_defconfig, new_defconfig) 1230 1231 if updated: 1232 self.log += color_text(self.options.color, COLOR_LIGHT_BLUE, 1233 "defconfig was updated.\n") 1234 1235 if not self.options.dry_run and updated: 1236 shutil.move(new_defconfig, orig_defconfig) 1237 self.finish(True) 1238 1239 def finish(self, success): 1240 """Display log along with progress and go to the idle state. 1241 1242 Arguments: 1243 success: Should be True when the defconfig was processed 1244 successfully, or False when it fails. 1245 """ 1246 # output at least 30 characters to hide the "* defconfigs out of *". 1247 log = self.defconfig.ljust(30) + '\n' 1248 1249 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ]) 1250 # Some threads are running in parallel. 1251 # Print log atomically to not mix up logs from different threads. 1252 print >> (sys.stdout if success else sys.stderr), log 1253 1254 if not success: 1255 if self.options.exit_on_error: 1256 sys.exit("Exit on error.") 1257 # If --exit-on-error flag is not set, skip this board and continue. 1258 # Record the failed board. 1259 self.failed_boards.add(self.defconfig) 1260 1261 self.progress.inc() 1262 self.progress.show() 1263 self.state = STATE_IDLE 1264 1265 def get_failed_boards(self): 1266 """Returns a set of failed boards (defconfigs) in this slot. 1267 """ 1268 return self.failed_boards 1269 1270 def get_suspicious_boards(self): 1271 """Returns a set of boards (defconfigs) with possible misconversion. 1272 """ 1273 return self.suspicious_boards - self.failed_boards 1274 1275class Slots: 1276 1277 """Controller of the array of subprocess slots.""" 1278 1279 def __init__(self, toolchains, configs, options, progress, 1280 reference_src_dir, db_queue): 1281 """Create a new slots controller. 1282 1283 Arguments: 1284 toolchains: Toolchains object containing toolchains. 1285 configs: A list of CONFIGs to move. 1286 options: option flags. 1287 progress: A progress indicator. 1288 reference_src_dir: Determine the true starting config state from this 1289 source tree. 1290 db_queue: output queue to write config info for the database 1291 """ 1292 self.options = options 1293 self.slots = [] 1294 devnull = get_devnull() 1295 make_cmd = get_make_cmd() 1296 for i in range(options.jobs): 1297 self.slots.append(Slot(toolchains, configs, options, progress, 1298 devnull, make_cmd, reference_src_dir, 1299 db_queue)) 1300 1301 def add(self, defconfig): 1302 """Add a new subprocess if a vacant slot is found. 1303 1304 Arguments: 1305 defconfig: defconfig name to be put into. 1306 1307 Returns: 1308 Return True on success or False on failure 1309 """ 1310 for slot in self.slots: 1311 if slot.add(defconfig): 1312 return True 1313 return False 1314 1315 def available(self): 1316 """Check if there is a vacant slot. 1317 1318 Returns: 1319 Return True if at lease one vacant slot is found, False otherwise. 1320 """ 1321 for slot in self.slots: 1322 if slot.poll(): 1323 return True 1324 return False 1325 1326 def empty(self): 1327 """Check if all slots are vacant. 1328 1329 Returns: 1330 Return True if all the slots are vacant, False otherwise. 1331 """ 1332 ret = True 1333 for slot in self.slots: 1334 if not slot.poll(): 1335 ret = False 1336 return ret 1337 1338 def show_failed_boards(self): 1339 """Display all of the failed boards (defconfigs).""" 1340 boards = set() 1341 output_file = 'moveconfig.failed' 1342 1343 for slot in self.slots: 1344 boards |= slot.get_failed_boards() 1345 1346 if boards: 1347 boards = '\n'.join(boards) + '\n' 1348 msg = "The following boards were not processed due to error:\n" 1349 msg += boards 1350 msg += "(the list has been saved in %s)\n" % output_file 1351 print >> sys.stderr, color_text(self.options.color, COLOR_LIGHT_RED, 1352 msg) 1353 1354 with open(output_file, 'w') as f: 1355 f.write(boards) 1356 1357 def show_suspicious_boards(self): 1358 """Display all boards (defconfigs) with possible misconversion.""" 1359 boards = set() 1360 output_file = 'moveconfig.suspicious' 1361 1362 for slot in self.slots: 1363 boards |= slot.get_suspicious_boards() 1364 1365 if boards: 1366 boards = '\n'.join(boards) + '\n' 1367 msg = "The following boards might have been converted incorrectly.\n" 1368 msg += "It is highly recommended to check them manually:\n" 1369 msg += boards 1370 msg += "(the list has been saved in %s)\n" % output_file 1371 print >> sys.stderr, color_text(self.options.color, COLOR_YELLOW, 1372 msg) 1373 1374 with open(output_file, 'w') as f: 1375 f.write(boards) 1376 1377class ReferenceSource: 1378 1379 """Reference source against which original configs should be parsed.""" 1380 1381 def __init__(self, commit): 1382 """Create a reference source directory based on a specified commit. 1383 1384 Arguments: 1385 commit: commit to git-clone 1386 """ 1387 self.src_dir = tempfile.mkdtemp() 1388 print "Cloning git repo to a separate work directory..." 1389 subprocess.check_output(['git', 'clone', os.getcwd(), '.'], 1390 cwd=self.src_dir) 1391 print "Checkout '%s' to build the original autoconf.mk." % \ 1392 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip() 1393 subprocess.check_output(['git', 'checkout', commit], 1394 stderr=subprocess.STDOUT, cwd=self.src_dir) 1395 1396 def __del__(self): 1397 """Delete the reference source directory 1398 1399 This function makes sure the temporary directory is cleaned away 1400 even if Python suddenly dies due to error. It should be done in here 1401 because it is guaranteed the destructor is always invoked when the 1402 instance of the class gets unreferenced. 1403 """ 1404 shutil.rmtree(self.src_dir) 1405 1406 def get_dir(self): 1407 """Return the absolute path to the reference source directory.""" 1408 1409 return self.src_dir 1410 1411def move_config(toolchains, configs, options, db_queue): 1412 """Move config options to defconfig files. 1413 1414 Arguments: 1415 configs: A list of CONFIGs to move. 1416 options: option flags 1417 """ 1418 if len(configs) == 0: 1419 if options.force_sync: 1420 print 'No CONFIG is specified. You are probably syncing defconfigs.', 1421 elif options.build_db: 1422 print 'Building %s database' % CONFIG_DATABASE 1423 else: 1424 print 'Neither CONFIG nor --force-sync is specified. Nothing will happen.', 1425 else: 1426 print 'Move ' + ', '.join(configs), 1427 print '(jobs: %d)\n' % options.jobs 1428 1429 if options.git_ref: 1430 reference_src = ReferenceSource(options.git_ref) 1431 reference_src_dir = reference_src.get_dir() 1432 else: 1433 reference_src_dir = None 1434 1435 if options.defconfigs: 1436 defconfigs = get_matched_defconfigs(options.defconfigs) 1437 else: 1438 defconfigs = get_all_defconfigs() 1439 1440 progress = Progress(len(defconfigs)) 1441 slots = Slots(toolchains, configs, options, progress, reference_src_dir, 1442 db_queue) 1443 1444 # Main loop to process defconfig files: 1445 # Add a new subprocess into a vacant slot. 1446 # Sleep if there is no available slot. 1447 for defconfig in defconfigs: 1448 while not slots.add(defconfig): 1449 while not slots.available(): 1450 # No available slot: sleep for a while 1451 time.sleep(SLEEP_TIME) 1452 1453 # wait until all the subprocesses finish 1454 while not slots.empty(): 1455 time.sleep(SLEEP_TIME) 1456 1457 print '' 1458 slots.show_failed_boards() 1459 slots.show_suspicious_boards() 1460 1461def find_kconfig_rules(kconf, config, imply_config): 1462 """Check whether a config has a 'select' or 'imply' keyword 1463 1464 Args: 1465 kconf: Kconfig.Config object 1466 config: Name of config to check (without CONFIG_ prefix) 1467 imply_config: Implying config (without CONFIG_ prefix) which may or 1468 may not have an 'imply' for 'config') 1469 1470 Returns: 1471 Symbol object for 'config' if found, else None 1472 """ 1473 sym = kconf.get_symbol(imply_config) 1474 if sym: 1475 for sel in sym.get_selected_symbols(): 1476 if sel.get_name() == config: 1477 return sym 1478 return None 1479 1480def check_imply_rule(kconf, config, imply_config): 1481 """Check if we can add an 'imply' option 1482 1483 This finds imply_config in the Kconfig and looks to see if it is possible 1484 to add an 'imply' for 'config' to that part of the Kconfig. 1485 1486 Args: 1487 kconf: Kconfig.Config object 1488 config: Name of config to check (without CONFIG_ prefix) 1489 imply_config: Implying config (without CONFIG_ prefix) which may or 1490 may not have an 'imply' for 'config') 1491 1492 Returns: 1493 tuple: 1494 filename of Kconfig file containing imply_config, or None if none 1495 line number within the Kconfig file, or 0 if none 1496 message indicating the result 1497 """ 1498 sym = kconf.get_symbol(imply_config) 1499 if not sym: 1500 return 'cannot find sym' 1501 locs = sym.get_def_locations() 1502 if len(locs) != 1: 1503 return '%d locations' % len(locs) 1504 fname, linenum = locs[0] 1505 cwd = os.getcwd() 1506 if cwd and fname.startswith(cwd): 1507 fname = fname[len(cwd) + 1:] 1508 file_line = ' at %s:%d' % (fname, linenum) 1509 with open(fname) as fd: 1510 data = fd.read().splitlines() 1511 if data[linenum - 1] != 'config %s' % imply_config: 1512 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line) 1513 return fname, linenum, 'adding%s' % file_line 1514 1515def add_imply_rule(config, fname, linenum): 1516 """Add a new 'imply' option to a Kconfig 1517 1518 Args: 1519 config: config option to add an imply for (without CONFIG_ prefix) 1520 fname: Kconfig filename to update 1521 linenum: Line number to place the 'imply' before 1522 1523 Returns: 1524 Message indicating the result 1525 """ 1526 file_line = ' at %s:%d' % (fname, linenum) 1527 data = open(fname).read().splitlines() 1528 linenum -= 1 1529 1530 for offset, line in enumerate(data[linenum:]): 1531 if line.strip().startswith('help') or not line: 1532 data.insert(linenum + offset, '\timply %s' % config) 1533 with open(fname, 'w') as fd: 1534 fd.write('\n'.join(data) + '\n') 1535 return 'added%s' % file_line 1536 1537 return 'could not insert%s' 1538 1539(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( 1540 1, 2, 4, 8) 1541 1542IMPLY_FLAGS = { 1543 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], 1544 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], 1545 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], 1546 'non-arch-board': [ 1547 IMPLY_NON_ARCH_BOARD, 1548 'Allow Kconfig options outside arch/ and /board/ to imply'], 1549}; 1550 1551def do_imply_config(config_list, add_imply, imply_flags, skip_added, 1552 check_kconfig=True, find_superset=False): 1553 """Find CONFIG options which imply those in the list 1554 1555 Some CONFIG options can be implied by others and this can help to reduce 1556 the size of the defconfig files. For example, CONFIG_X86 implies 1557 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and 1558 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to 1559 each of the x86 defconfig files. 1560 1561 This function uses the moveconfig database to find such options. It 1562 displays a list of things that could possibly imply those in the list. 1563 The algorithm ignores any that start with CONFIG_TARGET since these 1564 typically refer to only a few defconfigs (often one). It also does not 1565 display a config with less than 5 defconfigs. 1566 1567 The algorithm works using sets. For each target config in config_list: 1568 - Get the set 'defconfigs' which use that target config 1569 - For each config (from a list of all configs): 1570 - Get the set 'imply_defconfig' of defconfigs which use that config 1571 - 1572 - If imply_defconfigs contains anything not in defconfigs then 1573 this config does not imply the target config 1574 1575 Params: 1576 config_list: List of CONFIG options to check (each a string) 1577 add_imply: Automatically add an 'imply' for each config. 1578 imply_flags: Flags which control which implying configs are allowed 1579 (IMPLY_...) 1580 skip_added: Don't show options which already have an imply added. 1581 check_kconfig: Check if implied symbols already have an 'imply' or 1582 'select' for the target config, and show this information if so. 1583 find_superset: True to look for configs which are a superset of those 1584 already found. So for example if CONFIG_EXYNOS5 implies an option, 1585 but CONFIG_EXYNOS covers a larger set of defconfigs and also 1586 implies that option, this will drop the former in favour of the 1587 latter. In practice this option has not proved very used. 1588 1589 Note the terminoloy: 1590 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') 1591 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') 1592 """ 1593 kconf = KconfigScanner().conf if check_kconfig else None 1594 if add_imply and add_imply != 'all': 1595 add_imply = add_imply.split() 1596 1597 # key is defconfig name, value is dict of (CONFIG_xxx, value) 1598 config_db = {} 1599 1600 # Holds a dict containing the set of defconfigs that contain each config 1601 # key is config, value is set of defconfigs using that config 1602 defconfig_db = collections.defaultdict(set) 1603 1604 # Set of all config options we have seen 1605 all_configs = set() 1606 1607 # Set of all defconfigs we have seen 1608 all_defconfigs = set() 1609 1610 # Read in the database 1611 configs = {} 1612 with open(CONFIG_DATABASE) as fd: 1613 for line in fd.readlines(): 1614 line = line.rstrip() 1615 if not line: # Separator between defconfigs 1616 config_db[defconfig] = configs 1617 all_defconfigs.add(defconfig) 1618 configs = {} 1619 elif line[0] == ' ': # CONFIG line 1620 config, value = line.strip().split('=', 1) 1621 configs[config] = value 1622 defconfig_db[config].add(defconfig) 1623 all_configs.add(config) 1624 else: # New defconfig 1625 defconfig = line 1626 1627 # Work through each target config option in tern, independently 1628 for config in config_list: 1629 defconfigs = defconfig_db.get(config) 1630 if not defconfigs: 1631 print '%s not found in any defconfig' % config 1632 continue 1633 1634 # Get the set of defconfigs without this one (since a config cannot 1635 # imply itself) 1636 non_defconfigs = all_defconfigs - defconfigs 1637 num_defconfigs = len(defconfigs) 1638 print '%s found in %d/%d defconfigs' % (config, num_defconfigs, 1639 len(all_configs)) 1640 1641 # This will hold the results: key=config, value=defconfigs containing it 1642 imply_configs = {} 1643 rest_configs = all_configs - set([config]) 1644 1645 # Look at every possible config, except the target one 1646 for imply_config in rest_configs: 1647 if 'ERRATUM' in imply_config: 1648 continue 1649 if not (imply_flags & IMPLY_CMD): 1650 if 'CONFIG_CMD' in imply_config: 1651 continue 1652 if not (imply_flags & IMPLY_TARGET): 1653 if 'CONFIG_TARGET' in imply_config: 1654 continue 1655 1656 # Find set of defconfigs that have this config 1657 imply_defconfig = defconfig_db[imply_config] 1658 1659 # Get the intersection of this with defconfigs containing the 1660 # target config 1661 common_defconfigs = imply_defconfig & defconfigs 1662 1663 # Get the set of defconfigs containing this config which DO NOT 1664 # also contain the taret config. If this set is non-empty it means 1665 # that this config affects other defconfigs as well as (possibly) 1666 # the ones affected by the target config. This means it implies 1667 # things we don't want to imply. 1668 not_common_defconfigs = imply_defconfig & non_defconfigs 1669 if not_common_defconfigs: 1670 continue 1671 1672 # If there are common defconfigs, imply_config may be useful 1673 if common_defconfigs: 1674 skip = False 1675 if find_superset: 1676 for prev in imply_configs.keys(): 1677 prev_count = len(imply_configs[prev]) 1678 count = len(common_defconfigs) 1679 if (prev_count > count and 1680 (imply_configs[prev] & common_defconfigs == 1681 common_defconfigs)): 1682 # skip imply_config because prev is a superset 1683 skip = True 1684 break 1685 elif count > prev_count: 1686 # delete prev because imply_config is a superset 1687 del imply_configs[prev] 1688 if not skip: 1689 imply_configs[imply_config] = common_defconfigs 1690 1691 # Now we have a dict imply_configs of configs which imply each config 1692 # The value of each dict item is the set of defconfigs containing that 1693 # config. Rank them so that we print the configs that imply the largest 1694 # number of defconfigs first. 1695 ranked_iconfigs = sorted(imply_configs, 1696 key=lambda k: len(imply_configs[k]), reverse=True) 1697 kconfig_info = '' 1698 cwd = os.getcwd() 1699 add_list = collections.defaultdict(list) 1700 for iconfig in ranked_iconfigs: 1701 num_common = len(imply_configs[iconfig]) 1702 1703 # Don't bother if there are less than 5 defconfigs affected. 1704 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): 1705 continue 1706 missing = defconfigs - imply_configs[iconfig] 1707 missing_str = ', '.join(missing) if missing else 'all' 1708 missing_str = '' 1709 show = True 1710 if kconf: 1711 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], 1712 iconfig[CONFIG_LEN:]) 1713 kconfig_info = '' 1714 if sym: 1715 locs = sym.get_def_locations() 1716 if len(locs) == 1: 1717 fname, linenum = locs[0] 1718 if cwd and fname.startswith(cwd): 1719 fname = fname[len(cwd) + 1:] 1720 kconfig_info = '%s:%d' % (fname, linenum) 1721 if skip_added: 1722 show = False 1723 else: 1724 sym = kconf.get_symbol(iconfig[CONFIG_LEN:]) 1725 fname = '' 1726 if sym: 1727 locs = sym.get_def_locations() 1728 if len(locs) == 1: 1729 fname, linenum = locs[0] 1730 if cwd and fname.startswith(cwd): 1731 fname = fname[len(cwd) + 1:] 1732 in_arch_board = not sym or (fname.startswith('arch') or 1733 fname.startswith('board')) 1734 if (not in_arch_board and 1735 not (imply_flags & IMPLY_NON_ARCH_BOARD)): 1736 continue 1737 1738 if add_imply and (add_imply == 'all' or 1739 iconfig in add_imply): 1740 fname, linenum, kconfig_info = (check_imply_rule(kconf, 1741 config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) 1742 if fname: 1743 add_list[fname].append(linenum) 1744 1745 if show and kconfig_info != 'skip': 1746 print '%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30), 1747 kconfig_info, missing_str) 1748 1749 # Having collected a list of things to add, now we add them. We process 1750 # each file from the largest line number to the smallest so that 1751 # earlier additions do not affect our line numbers. E.g. if we added an 1752 # imply at line 20 it would change the position of each line after 1753 # that. 1754 for fname, linenums in add_list.iteritems(): 1755 for linenum in sorted(linenums, reverse=True): 1756 add_imply_rule(config[CONFIG_LEN:], fname, linenum) 1757 1758 1759def main(): 1760 try: 1761 cpu_count = multiprocessing.cpu_count() 1762 except NotImplementedError: 1763 cpu_count = 1 1764 1765 parser = optparse.OptionParser() 1766 # Add options here 1767 parser.add_option('-a', '--add-imply', type='string', default='', 1768 help='comma-separated list of CONFIG options to add ' 1769 "an 'imply' statement to for the CONFIG in -i") 1770 parser.add_option('-A', '--skip-added', action='store_true', default=False, 1771 help="don't show options which are already marked as " 1772 'implying others') 1773 parser.add_option('-b', '--build-db', action='store_true', default=False, 1774 help='build a CONFIG database') 1775 parser.add_option('-c', '--color', action='store_true', default=False, 1776 help='display the log in color') 1777 parser.add_option('-C', '--commit', action='store_true', default=False, 1778 help='Create a git commit for the operation') 1779 parser.add_option('-d', '--defconfigs', type='string', 1780 help='a file containing a list of defconfigs to move, ' 1781 "one per line (for example 'snow_defconfig') " 1782 "or '-' to read from stdin") 1783 parser.add_option('-i', '--imply', action='store_true', default=False, 1784 help='find options which imply others') 1785 parser.add_option('-I', '--imply-flags', type='string', default='', 1786 help="control the -i option ('help' for help") 1787 parser.add_option('-n', '--dry-run', action='store_true', default=False, 1788 help='perform a trial run (show log with no changes)') 1789 parser.add_option('-e', '--exit-on-error', action='store_true', 1790 default=False, 1791 help='exit immediately on any error') 1792 parser.add_option('-s', '--force-sync', action='store_true', default=False, 1793 help='force sync by savedefconfig') 1794 parser.add_option('-S', '--spl', action='store_true', default=False, 1795 help='parse config options defined for SPL build') 1796 parser.add_option('-H', '--headers-only', dest='cleanup_headers_only', 1797 action='store_true', default=False, 1798 help='only cleanup the headers') 1799 parser.add_option('-j', '--jobs', type='int', default=cpu_count, 1800 help='the number of jobs to run simultaneously') 1801 parser.add_option('-r', '--git-ref', type='string', 1802 help='the git ref to clone for building the autoconf.mk') 1803 parser.add_option('-y', '--yes', action='store_true', default=False, 1804 help="respond 'yes' to any prompts") 1805 parser.add_option('-v', '--verbose', action='store_true', default=False, 1806 help='show any build errors as boards are built') 1807 parser.usage += ' CONFIG ...' 1808 1809 (options, configs) = parser.parse_args() 1810 1811 if len(configs) == 0 and not any((options.force_sync, options.build_db, 1812 options.imply)): 1813 parser.print_usage() 1814 sys.exit(1) 1815 1816 # prefix the option name with CONFIG_ if missing 1817 configs = [ config if config.startswith('CONFIG_') else 'CONFIG_' + config 1818 for config in configs ] 1819 1820 check_top_directory() 1821 1822 if options.imply: 1823 imply_flags = 0 1824 if options.imply_flags == 'all': 1825 imply_flags = -1 1826 1827 elif options.imply_flags: 1828 for flag in options.imply_flags.split(','): 1829 bad = flag not in IMPLY_FLAGS 1830 if bad: 1831 print "Invalid flag '%s'" % flag 1832 if flag == 'help' or bad: 1833 print "Imply flags: (separate with ',')" 1834 for name, info in IMPLY_FLAGS.iteritems(): 1835 print ' %-15s: %s' % (name, info[1]) 1836 parser.print_usage() 1837 sys.exit(1) 1838 imply_flags |= IMPLY_FLAGS[flag][0] 1839 1840 do_imply_config(configs, options.add_imply, imply_flags, 1841 options.skip_added) 1842 return 1843 1844 config_db = {} 1845 db_queue = Queue.Queue() 1846 t = DatabaseThread(config_db, db_queue) 1847 t.setDaemon(True) 1848 t.start() 1849 1850 if not options.cleanup_headers_only: 1851 check_clean_directory() 1852 bsettings.Setup('') 1853 toolchains = toolchain.Toolchains() 1854 toolchains.GetSettings() 1855 toolchains.Scan(verbose=False) 1856 move_config(toolchains, configs, options, db_queue) 1857 db_queue.join() 1858 1859 if configs: 1860 cleanup_headers(configs, options) 1861 cleanup_extra_options(configs, options) 1862 cleanup_whitelist(configs, options) 1863 cleanup_readme(configs, options) 1864 1865 if options.commit: 1866 subprocess.call(['git', 'add', '-u']) 1867 if configs: 1868 msg = 'Convert %s %sto Kconfig' % (configs[0], 1869 'et al ' if len(configs) > 1 else '') 1870 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % 1871 '\n '.join(configs)) 1872 else: 1873 msg = 'configs: Resync with savedefconfig' 1874 msg += '\n\nRsync all defconfig files using moveconfig.py' 1875 subprocess.call(['git', 'commit', '-s', '-m', msg]) 1876 1877 if options.build_db: 1878 with open(CONFIG_DATABASE, 'w') as fd: 1879 for defconfig, configs in config_db.iteritems(): 1880 fd.write('%s\n' % defconfig) 1881 for config in sorted(configs.keys()): 1882 fd.write(' %s=%s\n' % (config, configs[config])) 1883 fd.write('\n') 1884 1885if __name__ == '__main__': 1886 main() 1887