11a459660SWolfgang Denk# SPDX-License-Identifier: GPL-2.0+ 283d290c5STom Rini# Copyright (c) 2013 The Chromium OS Authors. 3fc3fe1c2SSimon Glass# 4fc3fe1c2SSimon Glass 5fc3fe1c2SSimon Glassimport multiprocessing 6fc3fe1c2SSimon Glassimport os 7883a321aSSimon Glassimport shutil 8fc3fe1c2SSimon Glassimport sys 9fc3fe1c2SSimon Glass 10fc3fe1c2SSimon Glassimport board 11fc3fe1c2SSimon Glassimport bsettings 12fc3fe1c2SSimon Glassfrom builder import Builder 13fc3fe1c2SSimon Glassimport gitutil 14fc3fe1c2SSimon Glassimport patchstream 15fc3fe1c2SSimon Glassimport terminal 16d4144e45SSimon Glassfrom terminal import Print 17fc3fe1c2SSimon Glassimport toolchain 1899796923SMasahiro Yamadaimport command 1973f30b9bSMasahiro Yamadaimport subprocess 20fc3fe1c2SSimon Glass 21fc3fe1c2SSimon Glassdef GetPlural(count): 22fc3fe1c2SSimon Glass """Returns a plural 's' if count is not 1""" 23fc3fe1c2SSimon Glass return 's' if count != 1 else '' 24fc3fe1c2SSimon Glass 25fea5858eSSimon Glassdef GetActionSummary(is_summary, commits, selected, options): 26fc3fe1c2SSimon Glass """Return a string summarising the intended action. 27fc3fe1c2SSimon Glass 28fc3fe1c2SSimon Glass Returns: 29fc3fe1c2SSimon Glass Summary string. 30fc3fe1c2SSimon Glass """ 31fea5858eSSimon Glass if commits: 32fea5858eSSimon Glass count = len(commits) 33fc3fe1c2SSimon Glass count = (count + options.step - 1) / options.step 34fea5858eSSimon Glass commit_str = '%d commit%s' % (count, GetPlural(count)) 35fea5858eSSimon Glass else: 36fea5858eSSimon Glass commit_str = 'current source' 37fea5858eSSimon Glass str = '%s %s for %d boards' % ( 38fea5858eSSimon Glass 'Summary of' if is_summary else 'Building', commit_str, 39fc3fe1c2SSimon Glass len(selected)) 40fc3fe1c2SSimon Glass str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 41fc3fe1c2SSimon Glass GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 42fc3fe1c2SSimon Glass return str 43fc3fe1c2SSimon Glass 440689036aSSimon Glassdef ShowActions(series, why_selected, boards_selected, builder, options, 450689036aSSimon Glass board_warnings): 46fc3fe1c2SSimon Glass """Display a list of actions that we would take, if not a dry run. 47fc3fe1c2SSimon Glass 48fc3fe1c2SSimon Glass Args: 49fc3fe1c2SSimon Glass series: Series object 50fc3fe1c2SSimon Glass why_selected: Dictionary where each key is a buildman argument 518d7523c5SSimon Glass provided by the user, and the value is the list of boards 528d7523c5SSimon Glass brought in by that argument. For example, 'arm' might bring 538d7523c5SSimon Glass in 400 boards, so in this case the key would be 'arm' and 54fc3fe1c2SSimon Glass the value would be a list of board names. 55fc3fe1c2SSimon Glass boards_selected: Dict of selected boards, key is target name, 56fc3fe1c2SSimon Glass value is Board object 57fc3fe1c2SSimon Glass builder: The builder that will be used to build the commits 58fc3fe1c2SSimon Glass options: Command line options object 590689036aSSimon Glass board_warnings: List of warnings obtained from board selected 60fc3fe1c2SSimon Glass """ 61fc3fe1c2SSimon Glass col = terminal.Color() 62fc3fe1c2SSimon Glass print 'Dry run, so not doing much. But I would do this:' 63fc3fe1c2SSimon Glass print 64fea5858eSSimon Glass if series: 65fea5858eSSimon Glass commits = series.commits 66fea5858eSSimon Glass else: 67fea5858eSSimon Glass commits = None 68fea5858eSSimon Glass print GetActionSummary(False, commits, boards_selected, 69fc3fe1c2SSimon Glass options) 70fc3fe1c2SSimon Glass print 'Build directory: %s' % builder.base_dir 71fea5858eSSimon Glass if commits: 72fc3fe1c2SSimon Glass for upto in range(0, len(series.commits), options.step): 73fc3fe1c2SSimon Glass commit = series.commits[upto] 741ddda1b3SSimon Glass print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 75fc3fe1c2SSimon Glass print commit.subject 76fc3fe1c2SSimon Glass print 77fc3fe1c2SSimon Glass for arg in why_selected: 78fc3fe1c2SSimon Glass if arg != 'all': 798d7523c5SSimon Glass print arg, ': %d boards' % len(why_selected[arg]) 808d7523c5SSimon Glass if options.verbose: 818d7523c5SSimon Glass print ' %s' % ' '.join(why_selected[arg]) 82fc3fe1c2SSimon Glass print ('Total boards to build for each commit: %d\n' % 838d7523c5SSimon Glass len(why_selected['all'])) 840689036aSSimon Glass if board_warnings: 850689036aSSimon Glass for warning in board_warnings: 860689036aSSimon Glass print col.Color(col.YELLOW, warning) 87fc3fe1c2SSimon Glass 88409fc029SLothar Waßmanndef CheckOutputDir(output_dir): 89409fc029SLothar Waßmann """Make sure that the output directory is not within the current directory 90409fc029SLothar Waßmann 91409fc029SLothar Waßmann If we try to use an output directory which is within the current directory 92409fc029SLothar Waßmann (which is assumed to hold the U-Boot source) we may end up deleting the 93409fc029SLothar Waßmann U-Boot source code. Detect this and print an error in this case. 94409fc029SLothar Waßmann 95409fc029SLothar Waßmann Args: 96409fc029SLothar Waßmann output_dir: Output directory path to check 97409fc029SLothar Waßmann """ 98409fc029SLothar Waßmann path = os.path.realpath(output_dir) 99409fc029SLothar Waßmann cwd_path = os.path.realpath('.') 100409fc029SLothar Waßmann while True: 101409fc029SLothar Waßmann if os.path.realpath(path) == cwd_path: 102*58804b8cSChris Packham Print("Cannot use output directory '%s' since it is within the current directory '%s'" % 103409fc029SLothar Waßmann (path, cwd_path)) 104409fc029SLothar Waßmann sys.exit(1) 105409fc029SLothar Waßmann parent = os.path.dirname(path) 106409fc029SLothar Waßmann if parent == path: 107409fc029SLothar Waßmann break 108409fc029SLothar Waßmann path = parent 109409fc029SLothar Waßmann 110883a321aSSimon Glassdef DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 111883a321aSSimon Glass clean_dir=False): 112fc3fe1c2SSimon Glass """The main control code for buildman 113fc3fe1c2SSimon Glass 114fc3fe1c2SSimon Glass Args: 115fc3fe1c2SSimon Glass options: Command line options object 116fc3fe1c2SSimon Glass args: Command line arguments (list of strings) 117d4144e45SSimon Glass toolchains: Toolchains to use - this should be a Toolchains() 118d4144e45SSimon Glass object. If None, then it will be created and scanned 119d4144e45SSimon Glass make_func: Make function to use for the builder. This is called 120d4144e45SSimon Glass to execute 'make'. If this is None, the normal function 121d4144e45SSimon Glass will be used, which calls the 'make' tool with suitable 122d4144e45SSimon Glass arguments. This setting is useful for tests. 123823e60b6SSimon Glass board: Boards() object to use, containing a list of available 124823e60b6SSimon Glass boards. If this is None it will be created and scanned. 125fc3fe1c2SSimon Glass """ 126883a321aSSimon Glass global builder 127883a321aSSimon Glass 12848ba5856SSimon Glass if options.full_help: 12948ba5856SSimon Glass pager = os.getenv('PAGER') 13048ba5856SSimon Glass if not pager: 13148ba5856SSimon Glass pager = 'more' 1322bdeade0SSimon Glass fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 1332bdeade0SSimon Glass 'README') 13448ba5856SSimon Glass command.Run(pager, fname) 13548ba5856SSimon Glass return 0 13648ba5856SSimon Glass 137fc3fe1c2SSimon Glass gitutil.Setup() 138713bea38SSimon Glass col = terminal.Color() 139fc3fe1c2SSimon Glass 140fc3fe1c2SSimon Glass options.git_dir = os.path.join(options.git, '.git') 141fc3fe1c2SSimon Glass 1427e92e46eSSimon Glass no_toolchains = toolchains is None 1437e92e46eSSimon Glass if no_toolchains: 14400beb248SSimon Glass toolchains = toolchain.Toolchains(options.override_toolchain) 145fc3fe1c2SSimon Glass 146827e37b5SSimon Glass if options.fetch_arch: 147827e37b5SSimon Glass if options.fetch_arch == 'list': 148827e37b5SSimon Glass sorted_list = toolchains.ListArchs() 149713bea38SSimon Glass print col.Color(col.BLUE, 'Available architectures: %s\n' % 150713bea38SSimon Glass ' '.join(sorted_list)) 151827e37b5SSimon Glass return 0 152827e37b5SSimon Glass else: 153827e37b5SSimon Glass fetch_arch = options.fetch_arch 154827e37b5SSimon Glass if fetch_arch == 'all': 155827e37b5SSimon Glass fetch_arch = ','.join(toolchains.ListArchs()) 156713bea38SSimon Glass print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 157713bea38SSimon Glass fetch_arch) 158827e37b5SSimon Glass for arch in fetch_arch.split(','): 159713bea38SSimon Glass print 160827e37b5SSimon Glass ret = toolchains.FetchAndInstall(arch) 161827e37b5SSimon Glass if ret: 162827e37b5SSimon Glass return ret 163827e37b5SSimon Glass return 0 164827e37b5SSimon Glass 1657e92e46eSSimon Glass if no_toolchains: 1667e92e46eSSimon Glass toolchains.GetSettings() 16740232c91SSimon Glass toolchains.Scan(options.list_tool_chains and options.verbose) 1687e92e46eSSimon Glass if options.list_tool_chains: 1697e92e46eSSimon Glass toolchains.List() 1707e92e46eSSimon Glass print 1717e92e46eSSimon Glass return 0 1727e92e46eSSimon Glass 173fc3fe1c2SSimon Glass # Work out how many commits to build. We want to build everything on the 174fc3fe1c2SSimon Glass # branch. We also build the upstream commit as a control so we can see 175fc3fe1c2SSimon Glass # problems introduced by the first commit on the branch. 176fc3fe1c2SSimon Glass count = options.count 1775abab20dSSimon Glass has_range = options.branch and '..' in options.branch 178fc3fe1c2SSimon Glass if count == -1: 179fc3fe1c2SSimon Glass if not options.branch: 180fea5858eSSimon Glass count = 1 181fea5858eSSimon Glass else: 1825abab20dSSimon Glass if has_range: 1835abab20dSSimon Glass count, msg = gitutil.CountCommitsInRange(options.git_dir, 1845abab20dSSimon Glass options.branch) 1855abab20dSSimon Glass else: 1862a9e2c6aSSimon Glass count, msg = gitutil.CountCommitsInBranch(options.git_dir, 187fea5858eSSimon Glass options.branch) 188cce717a9SSimon Glass if count is None: 1892a9e2c6aSSimon Glass sys.exit(col.Color(col.RED, msg)) 1905abab20dSSimon Glass elif count == 0: 1915abab20dSSimon Glass sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 1925abab20dSSimon Glass options.branch)) 1932a9e2c6aSSimon Glass if msg: 1942a9e2c6aSSimon Glass print col.Color(col.YELLOW, msg) 195fc3fe1c2SSimon Glass count += 1 # Build upstream commit also 196fc3fe1c2SSimon Glass 197fc3fe1c2SSimon Glass if not count: 198fc3fe1c2SSimon Glass str = ("No commits found to process in branch '%s': " 199fc3fe1c2SSimon Glass "set branch's upstream or use -c flag" % options.branch) 20031e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, str)) 201fc3fe1c2SSimon Glass 202fc3fe1c2SSimon Glass # Work out what subset of the boards we are building 203823e60b6SSimon Glass if not boards: 20473f30b9bSMasahiro Yamada board_file = os.path.join(options.git, 'boards.cfg') 20573f30b9bSMasahiro Yamada status = subprocess.call([os.path.join(options.git, 20673f30b9bSMasahiro Yamada 'tools/genboardscfg.py')]) 20773f30b9bSMasahiro Yamada if status != 0: 20831e2141dSMasahiro Yamada sys.exit("Failed to generate boards.cfg") 20973f30b9bSMasahiro Yamada 210fc3fe1c2SSimon Glass boards = board.Boards() 211fc3fe1c2SSimon Glass boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 2123cf4ae6fSSimon Glass 2133cf4ae6fSSimon Glass exclude = [] 2143cf4ae6fSSimon Glass if options.exclude: 2153cf4ae6fSSimon Glass for arg in options.exclude: 2163cf4ae6fSSimon Glass exclude += arg.split(',') 2173cf4ae6fSSimon Glass 2180689036aSSimon Glass 2190689036aSSimon Glass if options.boards: 2200689036aSSimon Glass requested_boards = [] 2210689036aSSimon Glass for b in options.boards: 2220689036aSSimon Glass requested_boards += b.split(',') 2230689036aSSimon Glass else: 2240689036aSSimon Glass requested_boards = None 2250689036aSSimon Glass why_selected, board_warnings = boards.SelectBoards(args, exclude, 2260689036aSSimon Glass requested_boards) 227fc3fe1c2SSimon Glass selected = boards.GetSelected() 228fc3fe1c2SSimon Glass if not len(selected): 22931e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, 'No matching boards found')) 230fc3fe1c2SSimon Glass 231fc3fe1c2SSimon Glass # Read the metadata from the commits. First look at the upstream commit, 232fc3fe1c2SSimon Glass # then the ones in the branch. We would like to do something like 233fc3fe1c2SSimon Glass # upstream/master~..branch but that isn't possible if upstream/master is 234fc3fe1c2SSimon Glass # a merge commit (it will list all the commits that form part of the 235fc3fe1c2SSimon Glass # merge) 236950a2313SSimon Glass # Conflicting tags are not a problem for buildman, since it does not use 237950a2313SSimon Glass # them. For example, Series-version is not useful for buildman. On the 238950a2313SSimon Glass # other hand conflicting tags will cause an error. So allow later tags 239950a2313SSimon Glass # to overwrite earlier ones by setting allow_overwrite=True 240fea5858eSSimon Glass if options.branch: 2413b74ba5fSSimon Glass if count == -1: 2425abab20dSSimon Glass if has_range: 2435abab20dSSimon Glass range_expr = options.branch 2445abab20dSSimon Glass else: 2453b74ba5fSSimon Glass range_expr = gitutil.GetRangeInBranch(options.git_dir, 2463b74ba5fSSimon Glass options.branch) 2473b74ba5fSSimon Glass upstream_commit = gitutil.GetUpstream(options.git_dir, 2483b74ba5fSSimon Glass options.branch) 249fea5858eSSimon Glass series = patchstream.GetMetaDataForList(upstream_commit, 250950a2313SSimon Glass options.git_dir, 1, series=None, allow_overwrite=True) 251fea5858eSSimon Glass 2523b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(range_expr, 253950a2313SSimon Glass options.git_dir, None, series, allow_overwrite=True) 2543b74ba5fSSimon Glass else: 2553b74ba5fSSimon Glass # Honour the count 2563b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(options.branch, 257950a2313SSimon Glass options.git_dir, count, series=None, allow_overwrite=True) 258fea5858eSSimon Glass else: 259fea5858eSSimon Glass series = None 2608d7523c5SSimon Glass if not options.dry_run: 261e5a0e5d8SSimon Glass options.verbose = True 26258d818f1SSimon Glass if not options.summary: 263e5a0e5d8SSimon Glass options.show_errors = True 264fc3fe1c2SSimon Glass 265fc3fe1c2SSimon Glass # By default we have one thread per CPU. But if there are not enough jobs 266fc3fe1c2SSimon Glass # we can have fewer threads and use a high '-j' value for make. 267fc3fe1c2SSimon Glass if not options.threads: 268fc3fe1c2SSimon Glass options.threads = min(multiprocessing.cpu_count(), len(selected)) 269fc3fe1c2SSimon Glass if not options.jobs: 270fc3fe1c2SSimon Glass options.jobs = max(1, (multiprocessing.cpu_count() + 271fc3fe1c2SSimon Glass len(selected) - 1) / len(selected)) 272fc3fe1c2SSimon Glass 273fc3fe1c2SSimon Glass if not options.step: 274fc3fe1c2SSimon Glass options.step = len(series.commits) - 1 275fc3fe1c2SSimon Glass 27699796923SMasahiro Yamada gnu_make = command.Output(os.path.join(options.git, 277785f1548SSimon Glass 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 27899796923SMasahiro Yamada if not gnu_make: 27931e2141dSMasahiro Yamada sys.exit('GNU Make not found') 28099796923SMasahiro Yamada 28105c96b18SSimon Glass # Create a new builder with the selected options. 28205c96b18SSimon Glass output_dir = options.output_dir 283fea5858eSSimon Glass if options.branch: 284f7582ce8SSimon Glass dirname = options.branch.replace('/', '_') 2855971ab5cSSimon Glass # As a special case allow the board directory to be placed in the 2865971ab5cSSimon Glass # output directory itself rather than any subdirectory. 2875971ab5cSSimon Glass if not options.no_subdirs: 288fea5858eSSimon Glass output_dir = os.path.join(options.output_dir, dirname) 289409fc029SLothar Waßmann if clean_dir and os.path.exists(output_dir): 290883a321aSSimon Glass shutil.rmtree(output_dir) 291409fc029SLothar Waßmann CheckOutputDir(output_dir) 292fc3fe1c2SSimon Glass builder = Builder(toolchains, output_dir, options.git_dir, 29399796923SMasahiro Yamada options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 2945971ab5cSSimon Glass show_unknown=options.show_unknown, step=options.step, 295d2ce658dSSimon Glass no_subdirs=options.no_subdirs, full_path=options.full_path, 296f79f1e0cSStephen Warren verbose_build=options.verbose_build, 297f79f1e0cSStephen Warren incremental=options.incremental, 298b50113f3SSimon Glass per_board_out_dir=options.per_board_out_dir, 299b464f8e7SSimon Glass config_only=options.config_only, 3002371d1bcSDaniel Schwierzeck squash_config_y=not options.preserve_config_y, 3012371d1bcSDaniel Schwierzeck warnings_as_errors=options.warnings_as_errors) 302fc3fe1c2SSimon Glass builder.force_config_on_failure = not options.quick 303d4144e45SSimon Glass if make_func: 304d4144e45SSimon Glass builder.do_make = make_func 305fc3fe1c2SSimon Glass 306fc3fe1c2SSimon Glass # For a dry run, just show our actions as a sanity check 307fc3fe1c2SSimon Glass if options.dry_run: 3080689036aSSimon Glass ShowActions(series, why_selected, selected, builder, options, 3090689036aSSimon Glass board_warnings) 310fc3fe1c2SSimon Glass else: 311fc3fe1c2SSimon Glass builder.force_build = options.force_build 3124266dc28SSimon Glass builder.force_build_failures = options.force_build_failures 31397e91526SSimon Glass builder.force_reconfig = options.force_reconfig 314189a4968SSimon Glass builder.in_tree = options.in_tree 315fc3fe1c2SSimon Glass 316fc3fe1c2SSimon Glass # Work out which boards to build 317fc3fe1c2SSimon Glass board_selected = boards.GetSelectedDict() 318fc3fe1c2SSimon Glass 319fea5858eSSimon Glass if series: 320fea5858eSSimon Glass commits = series.commits 321883a321aSSimon Glass # Number the commits for test purposes 322883a321aSSimon Glass for commit in range(len(commits)): 323883a321aSSimon Glass commits[commit].sequence = commit 324fea5858eSSimon Glass else: 325fea5858eSSimon Glass commits = None 326fea5858eSSimon Glass 327d4144e45SSimon Glass Print(GetActionSummary(options.summary, commits, board_selected, 328d4144e45SSimon Glass options)) 329fc3fe1c2SSimon Glass 3307798e228SSimon Glass # We can't show function sizes without board details at present 3317798e228SSimon Glass if options.show_bloat: 3327798e228SSimon Glass options.show_detail = True 333b2ea7ab2SSimon Glass builder.SetDisplayOptions(options.show_errors, options.show_sizes, 334ed966657SSimon Glass options.show_detail, options.show_bloat, 335843312dcSSimon Glass options.list_error_boards, 33648ae4124SAlex Kiernan options.show_config, 33748ae4124SAlex Kiernan options.show_environment) 338fc3fe1c2SSimon Glass if options.summary: 339b2ea7ab2SSimon Glass builder.ShowSummary(commits, board_selected) 340fc3fe1c2SSimon Glass else: 3412c3deb97SSimon Glass fail, warned = builder.BuildBoards(commits, board_selected, 342e5a0e5d8SSimon Glass options.keep_outputs, options.verbose) 3432c3deb97SSimon Glass if fail: 3442c3deb97SSimon Glass return 128 3452c3deb97SSimon Glass elif warned: 3462c3deb97SSimon Glass return 129 3472c3deb97SSimon Glass return 0 348