1fc3fe1c2SSimon Glass# Copyright (c) 2013 The Chromium OS Authors. 2fc3fe1c2SSimon Glass# 31a459660SWolfgang Denk# SPDX-License-Identifier: GPL-2.0+ 4fc3fe1c2SSimon Glass# 5fc3fe1c2SSimon Glass 6fc3fe1c2SSimon Glassimport multiprocessing 7fc3fe1c2SSimon Glassimport os 8883a321aSSimon Glassimport shutil 9fc3fe1c2SSimon Glassimport sys 10fc3fe1c2SSimon Glass 11fc3fe1c2SSimon Glassimport board 12fc3fe1c2SSimon Glassimport bsettings 13fc3fe1c2SSimon Glassfrom builder import Builder 14fc3fe1c2SSimon Glassimport gitutil 15fc3fe1c2SSimon Glassimport patchstream 16fc3fe1c2SSimon Glassimport terminal 17d4144e45SSimon Glassfrom terminal import Print 18fc3fe1c2SSimon Glassimport toolchain 1999796923SMasahiro Yamadaimport command 2073f30b9bSMasahiro Yamadaimport subprocess 21fc3fe1c2SSimon Glass 22fc3fe1c2SSimon Glassdef GetPlural(count): 23fc3fe1c2SSimon Glass """Returns a plural 's' if count is not 1""" 24fc3fe1c2SSimon Glass return 's' if count != 1 else '' 25fc3fe1c2SSimon Glass 26fea5858eSSimon Glassdef GetActionSummary(is_summary, commits, selected, options): 27fc3fe1c2SSimon Glass """Return a string summarising the intended action. 28fc3fe1c2SSimon Glass 29fc3fe1c2SSimon Glass Returns: 30fc3fe1c2SSimon Glass Summary string. 31fc3fe1c2SSimon Glass """ 32fea5858eSSimon Glass if commits: 33fea5858eSSimon Glass count = len(commits) 34fc3fe1c2SSimon Glass count = (count + options.step - 1) / options.step 35fea5858eSSimon Glass commit_str = '%d commit%s' % (count, GetPlural(count)) 36fea5858eSSimon Glass else: 37fea5858eSSimon Glass commit_str = 'current source' 38fea5858eSSimon Glass str = '%s %s for %d boards' % ( 39fea5858eSSimon Glass 'Summary of' if is_summary else 'Building', commit_str, 40fc3fe1c2SSimon Glass len(selected)) 41fc3fe1c2SSimon Glass str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42fc3fe1c2SSimon Glass GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43fc3fe1c2SSimon Glass return str 44fc3fe1c2SSimon Glass 45fc3fe1c2SSimon Glassdef ShowActions(series, why_selected, boards_selected, builder, options): 46fc3fe1c2SSimon Glass """Display a list of actions that we would take, if not a dry run. 47fc3fe1c2SSimon Glass 48fc3fe1c2SSimon Glass Args: 49fc3fe1c2SSimon Glass series: Series object 50fc3fe1c2SSimon Glass why_selected: Dictionary where each key is a buildman argument 51fc3fe1c2SSimon Glass provided by the user, and the value is the boards brought 52fc3fe1c2SSimon Glass in by that argument. For example, 'arm' might bring in 53fc3fe1c2SSimon Glass 400 boards, so in this case the key would be 'arm' and 54fc3fe1c2SSimon Glass the value would be a list of board names. 55fc3fe1c2SSimon Glass boards_selected: Dict of selected boards, key is target name, 56fc3fe1c2SSimon Glass value is Board object 57fc3fe1c2SSimon Glass builder: The builder that will be used to build the commits 58fc3fe1c2SSimon Glass options: Command line options object 59fc3fe1c2SSimon Glass """ 60fc3fe1c2SSimon Glass col = terminal.Color() 61fc3fe1c2SSimon Glass print 'Dry run, so not doing much. But I would do this:' 62fc3fe1c2SSimon Glass print 63fea5858eSSimon Glass if series: 64fea5858eSSimon Glass commits = series.commits 65fea5858eSSimon Glass else: 66fea5858eSSimon Glass commits = None 67fea5858eSSimon Glass print GetActionSummary(False, commits, boards_selected, 68fc3fe1c2SSimon Glass options) 69fc3fe1c2SSimon Glass print 'Build directory: %s' % builder.base_dir 70fea5858eSSimon Glass if commits: 71fc3fe1c2SSimon Glass for upto in range(0, len(series.commits), options.step): 72fc3fe1c2SSimon Glass commit = series.commits[upto] 731ddda1b3SSimon Glass print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74fc3fe1c2SSimon Glass print commit.subject 75fc3fe1c2SSimon Glass print 76fc3fe1c2SSimon Glass for arg in why_selected: 77fc3fe1c2SSimon Glass if arg != 'all': 78fc3fe1c2SSimon Glass print arg, ': %d boards' % why_selected[arg] 79fc3fe1c2SSimon Glass print ('Total boards to build for each commit: %d\n' % 80fc3fe1c2SSimon Glass why_selected['all']) 81fc3fe1c2SSimon Glass 82883a321aSSimon Glassdef DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83883a321aSSimon Glass clean_dir=False): 84fc3fe1c2SSimon Glass """The main control code for buildman 85fc3fe1c2SSimon Glass 86fc3fe1c2SSimon Glass Args: 87fc3fe1c2SSimon Glass options: Command line options object 88fc3fe1c2SSimon Glass args: Command line arguments (list of strings) 89d4144e45SSimon Glass toolchains: Toolchains to use - this should be a Toolchains() 90d4144e45SSimon Glass object. If None, then it will be created and scanned 91d4144e45SSimon Glass make_func: Make function to use for the builder. This is called 92d4144e45SSimon Glass to execute 'make'. If this is None, the normal function 93d4144e45SSimon Glass will be used, which calls the 'make' tool with suitable 94d4144e45SSimon Glass arguments. This setting is useful for tests. 95823e60b6SSimon Glass board: Boards() object to use, containing a list of available 96823e60b6SSimon Glass boards. If this is None it will be created and scanned. 97fc3fe1c2SSimon Glass """ 98883a321aSSimon Glass global builder 99883a321aSSimon Glass 10048ba5856SSimon Glass if options.full_help: 10148ba5856SSimon Glass pager = os.getenv('PAGER') 10248ba5856SSimon Glass if not pager: 10348ba5856SSimon Glass pager = 'more' 1042bdeade0SSimon Glass fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 1052bdeade0SSimon Glass 'README') 10648ba5856SSimon Glass command.Run(pager, fname) 10748ba5856SSimon Glass return 0 10848ba5856SSimon Glass 109fc3fe1c2SSimon Glass gitutil.Setup() 110713bea38SSimon Glass col = terminal.Color() 111fc3fe1c2SSimon Glass 112fc3fe1c2SSimon Glass options.git_dir = os.path.join(options.git, '.git') 113fc3fe1c2SSimon Glass 1147e92e46eSSimon Glass no_toolchains = toolchains is None 1157e92e46eSSimon Glass if no_toolchains: 116fc3fe1c2SSimon Glass toolchains = toolchain.Toolchains() 117fc3fe1c2SSimon Glass 118827e37b5SSimon Glass if options.fetch_arch: 119827e37b5SSimon Glass if options.fetch_arch == 'list': 120827e37b5SSimon Glass sorted_list = toolchains.ListArchs() 121713bea38SSimon Glass print col.Color(col.BLUE, 'Available architectures: %s\n' % 122713bea38SSimon Glass ' '.join(sorted_list)) 123827e37b5SSimon Glass return 0 124827e37b5SSimon Glass else: 125827e37b5SSimon Glass fetch_arch = options.fetch_arch 126827e37b5SSimon Glass if fetch_arch == 'all': 127827e37b5SSimon Glass fetch_arch = ','.join(toolchains.ListArchs()) 128713bea38SSimon Glass print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 129713bea38SSimon Glass fetch_arch) 130827e37b5SSimon Glass for arch in fetch_arch.split(','): 131713bea38SSimon Glass print 132827e37b5SSimon Glass ret = toolchains.FetchAndInstall(arch) 133827e37b5SSimon Glass if ret: 134827e37b5SSimon Glass return ret 135827e37b5SSimon Glass return 0 136827e37b5SSimon Glass 1377e92e46eSSimon Glass if no_toolchains: 1387e92e46eSSimon Glass toolchains.GetSettings() 1397e92e46eSSimon Glass toolchains.Scan(options.list_tool_chains) 1407e92e46eSSimon Glass if options.list_tool_chains: 1417e92e46eSSimon Glass toolchains.List() 1427e92e46eSSimon Glass print 1437e92e46eSSimon Glass return 0 1447e92e46eSSimon Glass 145fc3fe1c2SSimon Glass # Work out how many commits to build. We want to build everything on the 146fc3fe1c2SSimon Glass # branch. We also build the upstream commit as a control so we can see 147fc3fe1c2SSimon Glass # problems introduced by the first commit on the branch. 148fc3fe1c2SSimon Glass count = options.count 1495abab20dSSimon Glass has_range = options.branch and '..' in options.branch 150fc3fe1c2SSimon Glass if count == -1: 151fc3fe1c2SSimon Glass if not options.branch: 152fea5858eSSimon Glass count = 1 153fea5858eSSimon Glass else: 1545abab20dSSimon Glass if has_range: 1555abab20dSSimon Glass count, msg = gitutil.CountCommitsInRange(options.git_dir, 1565abab20dSSimon Glass options.branch) 1575abab20dSSimon Glass else: 1582a9e2c6aSSimon Glass count, msg = gitutil.CountCommitsInBranch(options.git_dir, 159fea5858eSSimon Glass options.branch) 160cce717a9SSimon Glass if count is None: 1612a9e2c6aSSimon Glass sys.exit(col.Color(col.RED, msg)) 1625abab20dSSimon Glass elif count == 0: 1635abab20dSSimon Glass sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 1645abab20dSSimon Glass options.branch)) 1652a9e2c6aSSimon Glass if msg: 1662a9e2c6aSSimon Glass print col.Color(col.YELLOW, msg) 167fc3fe1c2SSimon Glass count += 1 # Build upstream commit also 168fc3fe1c2SSimon Glass 169fc3fe1c2SSimon Glass if not count: 170fc3fe1c2SSimon Glass str = ("No commits found to process in branch '%s': " 171fc3fe1c2SSimon Glass "set branch's upstream or use -c flag" % options.branch) 17231e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, str)) 173fc3fe1c2SSimon Glass 174fc3fe1c2SSimon Glass # Work out what subset of the boards we are building 175823e60b6SSimon Glass if not boards: 17673f30b9bSMasahiro Yamada board_file = os.path.join(options.git, 'boards.cfg') 17773f30b9bSMasahiro Yamada status = subprocess.call([os.path.join(options.git, 17873f30b9bSMasahiro Yamada 'tools/genboardscfg.py')]) 17973f30b9bSMasahiro Yamada if status != 0: 18031e2141dSMasahiro Yamada sys.exit("Failed to generate boards.cfg") 18173f30b9bSMasahiro Yamada 182fc3fe1c2SSimon Glass boards = board.Boards() 183fc3fe1c2SSimon Glass boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 1843cf4ae6fSSimon Glass 1853cf4ae6fSSimon Glass exclude = [] 1863cf4ae6fSSimon Glass if options.exclude: 1873cf4ae6fSSimon Glass for arg in options.exclude: 1883cf4ae6fSSimon Glass exclude += arg.split(',') 1893cf4ae6fSSimon Glass 1903cf4ae6fSSimon Glass why_selected = boards.SelectBoards(args, exclude) 191fc3fe1c2SSimon Glass selected = boards.GetSelected() 192fc3fe1c2SSimon Glass if not len(selected): 19331e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, 'No matching boards found')) 194fc3fe1c2SSimon Glass 195fc3fe1c2SSimon Glass # Read the metadata from the commits. First look at the upstream commit, 196fc3fe1c2SSimon Glass # then the ones in the branch. We would like to do something like 197fc3fe1c2SSimon Glass # upstream/master~..branch but that isn't possible if upstream/master is 198fc3fe1c2SSimon Glass # a merge commit (it will list all the commits that form part of the 199fc3fe1c2SSimon Glass # merge) 200950a2313SSimon Glass # Conflicting tags are not a problem for buildman, since it does not use 201950a2313SSimon Glass # them. For example, Series-version is not useful for buildman. On the 202950a2313SSimon Glass # other hand conflicting tags will cause an error. So allow later tags 203950a2313SSimon Glass # to overwrite earlier ones by setting allow_overwrite=True 204fea5858eSSimon Glass if options.branch: 2053b74ba5fSSimon Glass if count == -1: 2065abab20dSSimon Glass if has_range: 2075abab20dSSimon Glass range_expr = options.branch 2085abab20dSSimon Glass else: 2093b74ba5fSSimon Glass range_expr = gitutil.GetRangeInBranch(options.git_dir, 2103b74ba5fSSimon Glass options.branch) 2113b74ba5fSSimon Glass upstream_commit = gitutil.GetUpstream(options.git_dir, 2123b74ba5fSSimon Glass options.branch) 213fea5858eSSimon Glass series = patchstream.GetMetaDataForList(upstream_commit, 214950a2313SSimon Glass options.git_dir, 1, series=None, allow_overwrite=True) 215fea5858eSSimon Glass 2163b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(range_expr, 217950a2313SSimon Glass options.git_dir, None, series, allow_overwrite=True) 2183b74ba5fSSimon Glass else: 2193b74ba5fSSimon Glass # Honour the count 2203b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(options.branch, 221950a2313SSimon Glass options.git_dir, count, series=None, allow_overwrite=True) 222fea5858eSSimon Glass else: 223fea5858eSSimon Glass series = None 224e5a0e5d8SSimon Glass options.verbose = True 22558d818f1SSimon Glass if not options.summary: 226e5a0e5d8SSimon Glass options.show_errors = True 227fc3fe1c2SSimon Glass 228fc3fe1c2SSimon Glass # By default we have one thread per CPU. But if there are not enough jobs 229fc3fe1c2SSimon Glass # we can have fewer threads and use a high '-j' value for make. 230fc3fe1c2SSimon Glass if not options.threads: 231fc3fe1c2SSimon Glass options.threads = min(multiprocessing.cpu_count(), len(selected)) 232fc3fe1c2SSimon Glass if not options.jobs: 233fc3fe1c2SSimon Glass options.jobs = max(1, (multiprocessing.cpu_count() + 234fc3fe1c2SSimon Glass len(selected) - 1) / len(selected)) 235fc3fe1c2SSimon Glass 236fc3fe1c2SSimon Glass if not options.step: 237fc3fe1c2SSimon Glass options.step = len(series.commits) - 1 238fc3fe1c2SSimon Glass 23999796923SMasahiro Yamada gnu_make = command.Output(os.path.join(options.git, 240*785f1548SSimon Glass 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 24199796923SMasahiro Yamada if not gnu_make: 24231e2141dSMasahiro Yamada sys.exit('GNU Make not found') 24399796923SMasahiro Yamada 24405c96b18SSimon Glass # Create a new builder with the selected options. 24505c96b18SSimon Glass output_dir = options.output_dir 246fea5858eSSimon Glass if options.branch: 247f7582ce8SSimon Glass dirname = options.branch.replace('/', '_') 2485971ab5cSSimon Glass # As a special case allow the board directory to be placed in the 2495971ab5cSSimon Glass # output directory itself rather than any subdirectory. 2505971ab5cSSimon Glass if not options.no_subdirs: 251fea5858eSSimon Glass output_dir = os.path.join(options.output_dir, dirname) 2520740127fSSimon Glass if (clean_dir and output_dir != options.output_dir and 2530740127fSSimon Glass os.path.exists(output_dir)): 254883a321aSSimon Glass shutil.rmtree(output_dir) 255fc3fe1c2SSimon Glass builder = Builder(toolchains, output_dir, options.git_dir, 25699796923SMasahiro Yamada options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 2575971ab5cSSimon Glass show_unknown=options.show_unknown, step=options.step, 258d2ce658dSSimon Glass no_subdirs=options.no_subdirs, full_path=options.full_path, 259f79f1e0cSStephen Warren verbose_build=options.verbose_build, 260f79f1e0cSStephen Warren incremental=options.incremental, 261f79f1e0cSStephen Warren per_board_out_dir=options.per_board_out_dir,) 262fc3fe1c2SSimon Glass builder.force_config_on_failure = not options.quick 263d4144e45SSimon Glass if make_func: 264d4144e45SSimon Glass builder.do_make = make_func 265fc3fe1c2SSimon Glass 266fc3fe1c2SSimon Glass # For a dry run, just show our actions as a sanity check 267fc3fe1c2SSimon Glass if options.dry_run: 268fc3fe1c2SSimon Glass ShowActions(series, why_selected, selected, builder, options) 269fc3fe1c2SSimon Glass else: 270fc3fe1c2SSimon Glass builder.force_build = options.force_build 2714266dc28SSimon Glass builder.force_build_failures = options.force_build_failures 27297e91526SSimon Glass builder.force_reconfig = options.force_reconfig 273189a4968SSimon Glass builder.in_tree = options.in_tree 274fc3fe1c2SSimon Glass 275fc3fe1c2SSimon Glass # Work out which boards to build 276fc3fe1c2SSimon Glass board_selected = boards.GetSelectedDict() 277fc3fe1c2SSimon Glass 278fea5858eSSimon Glass if series: 279fea5858eSSimon Glass commits = series.commits 280883a321aSSimon Glass # Number the commits for test purposes 281883a321aSSimon Glass for commit in range(len(commits)): 282883a321aSSimon Glass commits[commit].sequence = commit 283fea5858eSSimon Glass else: 284fea5858eSSimon Glass commits = None 285fea5858eSSimon Glass 286d4144e45SSimon Glass Print(GetActionSummary(options.summary, commits, board_selected, 287d4144e45SSimon Glass options)) 288fc3fe1c2SSimon Glass 2897798e228SSimon Glass # We can't show function sizes without board details at present 2907798e228SSimon Glass if options.show_bloat: 2917798e228SSimon Glass options.show_detail = True 292b2ea7ab2SSimon Glass builder.SetDisplayOptions(options.show_errors, options.show_sizes, 293ed966657SSimon Glass options.show_detail, options.show_bloat, 294843312dcSSimon Glass options.list_error_boards, 295843312dcSSimon Glass options.show_config) 296fc3fe1c2SSimon Glass if options.summary: 297b2ea7ab2SSimon Glass builder.ShowSummary(commits, board_selected) 298fc3fe1c2SSimon Glass else: 2992c3deb97SSimon Glass fail, warned = builder.BuildBoards(commits, board_selected, 300e5a0e5d8SSimon Glass options.keep_outputs, options.verbose) 3012c3deb97SSimon Glass if fail: 3022c3deb97SSimon Glass return 128 3032c3deb97SSimon Glass elif warned: 3042c3deb97SSimon Glass return 129 3052c3deb97SSimon Glass return 0 306