1fc3fe1c2SSimon Glass# Copyright (c) 2013 The Chromium OS Authors. 2fc3fe1c2SSimon Glass# 31a459660SWolfgang Denk# SPDX-License-Identifier: GPL-2.0+ 4fc3fe1c2SSimon Glass# 5fc3fe1c2SSimon Glass 6fc3fe1c2SSimon Glassimport multiprocessing 7fc3fe1c2SSimon Glassimport os 8fc3fe1c2SSimon Glassimport sys 9fc3fe1c2SSimon Glass 10fc3fe1c2SSimon Glassimport board 11fc3fe1c2SSimon Glassimport bsettings 12fc3fe1c2SSimon Glassfrom builder import Builder 13fc3fe1c2SSimon Glassimport gitutil 14fc3fe1c2SSimon Glassimport patchstream 15fc3fe1c2SSimon Glassimport terminal 16*d4144e45SSimon Glassfrom terminal import Print 17fc3fe1c2SSimon Glassimport toolchain 1899796923SMasahiro Yamadaimport command 1973f30b9bSMasahiro Yamadaimport subprocess 20fc3fe1c2SSimon Glass 21fc3fe1c2SSimon Glassdef GetPlural(count): 22fc3fe1c2SSimon Glass """Returns a plural 's' if count is not 1""" 23fc3fe1c2SSimon Glass return 's' if count != 1 else '' 24fc3fe1c2SSimon Glass 25fea5858eSSimon Glassdef GetActionSummary(is_summary, commits, selected, options): 26fc3fe1c2SSimon Glass """Return a string summarising the intended action. 27fc3fe1c2SSimon Glass 28fc3fe1c2SSimon Glass Returns: 29fc3fe1c2SSimon Glass Summary string. 30fc3fe1c2SSimon Glass """ 31fea5858eSSimon Glass if commits: 32fea5858eSSimon Glass count = len(commits) 33fc3fe1c2SSimon Glass count = (count + options.step - 1) / options.step 34fea5858eSSimon Glass commit_str = '%d commit%s' % (count, GetPlural(count)) 35fea5858eSSimon Glass else: 36fea5858eSSimon Glass commit_str = 'current source' 37fea5858eSSimon Glass str = '%s %s for %d boards' % ( 38fea5858eSSimon Glass 'Summary of' if is_summary else 'Building', commit_str, 39fc3fe1c2SSimon Glass len(selected)) 40fc3fe1c2SSimon Glass str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 41fc3fe1c2SSimon Glass GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 42fc3fe1c2SSimon Glass return str 43fc3fe1c2SSimon Glass 44fc3fe1c2SSimon Glassdef ShowActions(series, why_selected, boards_selected, builder, options): 45fc3fe1c2SSimon Glass """Display a list of actions that we would take, if not a dry run. 46fc3fe1c2SSimon Glass 47fc3fe1c2SSimon Glass Args: 48fc3fe1c2SSimon Glass series: Series object 49fc3fe1c2SSimon Glass why_selected: Dictionary where each key is a buildman argument 50fc3fe1c2SSimon Glass provided by the user, and the value is the boards brought 51fc3fe1c2SSimon Glass in by that argument. For example, 'arm' might bring in 52fc3fe1c2SSimon Glass 400 boards, so in this case the key would be 'arm' and 53fc3fe1c2SSimon Glass the value would be a list of board names. 54fc3fe1c2SSimon Glass boards_selected: Dict of selected boards, key is target name, 55fc3fe1c2SSimon Glass value is Board object 56fc3fe1c2SSimon Glass builder: The builder that will be used to build the commits 57fc3fe1c2SSimon Glass options: Command line options object 58fc3fe1c2SSimon Glass """ 59fc3fe1c2SSimon Glass col = terminal.Color() 60fc3fe1c2SSimon Glass print 'Dry run, so not doing much. But I would do this:' 61fc3fe1c2SSimon Glass print 62fea5858eSSimon Glass if series: 63fea5858eSSimon Glass commits = series.commits 64fea5858eSSimon Glass else: 65fea5858eSSimon Glass commits = None 66fea5858eSSimon Glass print GetActionSummary(False, commits, boards_selected, 67fc3fe1c2SSimon Glass options) 68fc3fe1c2SSimon Glass print 'Build directory: %s' % builder.base_dir 69fea5858eSSimon Glass if commits: 70fc3fe1c2SSimon Glass for upto in range(0, len(series.commits), options.step): 71fc3fe1c2SSimon Glass commit = series.commits[upto] 72fc3fe1c2SSimon Glass print ' ', col.Color(col.YELLOW, commit.hash, bright=False), 73fc3fe1c2SSimon Glass print commit.subject 74fc3fe1c2SSimon Glass print 75fc3fe1c2SSimon Glass for arg in why_selected: 76fc3fe1c2SSimon Glass if arg != 'all': 77fc3fe1c2SSimon Glass print arg, ': %d boards' % why_selected[arg] 78fc3fe1c2SSimon Glass print ('Total boards to build for each commit: %d\n' % 79fc3fe1c2SSimon Glass why_selected['all']) 80fc3fe1c2SSimon Glass 81*d4144e45SSimon Glassdef DoBuildman(options, args, toolchains=None, make_func=None): 82fc3fe1c2SSimon Glass """The main control code for buildman 83fc3fe1c2SSimon Glass 84fc3fe1c2SSimon Glass Args: 85fc3fe1c2SSimon Glass options: Command line options object 86fc3fe1c2SSimon Glass args: Command line arguments (list of strings) 87*d4144e45SSimon Glass toolchains: Toolchains to use - this should be a Toolchains() 88*d4144e45SSimon Glass object. If None, then it will be created and scanned 89*d4144e45SSimon Glass make_func: Make function to use for the builder. This is called 90*d4144e45SSimon Glass to execute 'make'. If this is None, the normal function 91*d4144e45SSimon Glass will be used, which calls the 'make' tool with suitable 92*d4144e45SSimon Glass arguments. This setting is useful for tests. 93fc3fe1c2SSimon Glass """ 9448ba5856SSimon Glass if options.full_help: 9548ba5856SSimon Glass pager = os.getenv('PAGER') 9648ba5856SSimon Glass if not pager: 9748ba5856SSimon Glass pager = 'more' 9848ba5856SSimon Glass fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') 9948ba5856SSimon Glass command.Run(pager, fname) 10048ba5856SSimon Glass return 0 10148ba5856SSimon Glass 102fc3fe1c2SSimon Glass gitutil.Setup() 103fc3fe1c2SSimon Glass 1040f7c9ddaSSimon Glass bsettings.Setup(options.config_file) 105fc3fe1c2SSimon Glass options.git_dir = os.path.join(options.git, '.git') 106fc3fe1c2SSimon Glass 107*d4144e45SSimon Glass if not toolchains: 108fc3fe1c2SSimon Glass toolchains = toolchain.Toolchains() 109*d4144e45SSimon Glass toolchains.GetSettings() 110fc3fe1c2SSimon Glass toolchains.Scan(options.list_tool_chains) 111fc3fe1c2SSimon Glass if options.list_tool_chains: 112fc3fe1c2SSimon Glass toolchains.List() 113fc3fe1c2SSimon Glass print 1142c3deb97SSimon Glass return 0 115fc3fe1c2SSimon Glass 116fc3fe1c2SSimon Glass # Work out how many commits to build. We want to build everything on the 117fc3fe1c2SSimon Glass # branch. We also build the upstream commit as a control so we can see 118fc3fe1c2SSimon Glass # problems introduced by the first commit on the branch. 119fc3fe1c2SSimon Glass col = terminal.Color() 120fc3fe1c2SSimon Glass count = options.count 121fc3fe1c2SSimon Glass if count == -1: 122fc3fe1c2SSimon Glass if not options.branch: 123fea5858eSSimon Glass count = 1 124fea5858eSSimon Glass else: 125fea5858eSSimon Glass count = gitutil.CountCommitsInBranch(options.git_dir, 126fea5858eSSimon Glass options.branch) 127cce717a9SSimon Glass if count is None: 128fea5858eSSimon Glass str = ("Branch '%s' not found or has no upstream" % 129fea5858eSSimon Glass options.branch) 13031e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, str)) 131fc3fe1c2SSimon Glass count += 1 # Build upstream commit also 132fc3fe1c2SSimon Glass 133fc3fe1c2SSimon Glass if not count: 134fc3fe1c2SSimon Glass str = ("No commits found to process in branch '%s': " 135fc3fe1c2SSimon Glass "set branch's upstream or use -c flag" % options.branch) 13631e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, str)) 137fc3fe1c2SSimon Glass 138fc3fe1c2SSimon Glass # Work out what subset of the boards we are building 13973f30b9bSMasahiro Yamada board_file = os.path.join(options.git, 'boards.cfg') 14073f30b9bSMasahiro Yamada status = subprocess.call([os.path.join(options.git, 14173f30b9bSMasahiro Yamada 'tools/genboardscfg.py')]) 14273f30b9bSMasahiro Yamada if status != 0: 14331e2141dSMasahiro Yamada sys.exit("Failed to generate boards.cfg") 14473f30b9bSMasahiro Yamada 145fc3fe1c2SSimon Glass boards = board.Boards() 146fc3fe1c2SSimon Glass boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 1473cf4ae6fSSimon Glass 1483cf4ae6fSSimon Glass exclude = [] 1493cf4ae6fSSimon Glass if options.exclude: 1503cf4ae6fSSimon Glass for arg in options.exclude: 1513cf4ae6fSSimon Glass exclude += arg.split(',') 1523cf4ae6fSSimon Glass 1533cf4ae6fSSimon Glass why_selected = boards.SelectBoards(args, exclude) 154fc3fe1c2SSimon Glass selected = boards.GetSelected() 155fc3fe1c2SSimon Glass if not len(selected): 15631e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, 'No matching boards found')) 157fc3fe1c2SSimon Glass 158fc3fe1c2SSimon Glass # Read the metadata from the commits. First look at the upstream commit, 159fc3fe1c2SSimon Glass # then the ones in the branch. We would like to do something like 160fc3fe1c2SSimon Glass # upstream/master~..branch but that isn't possible if upstream/master is 161fc3fe1c2SSimon Glass # a merge commit (it will list all the commits that form part of the 162fc3fe1c2SSimon Glass # merge) 163fea5858eSSimon Glass if options.branch: 1643b74ba5fSSimon Glass if count == -1: 1653b74ba5fSSimon Glass range_expr = gitutil.GetRangeInBranch(options.git_dir, 1663b74ba5fSSimon Glass options.branch) 1673b74ba5fSSimon Glass upstream_commit = gitutil.GetUpstream(options.git_dir, 1683b74ba5fSSimon Glass options.branch) 169fea5858eSSimon Glass series = patchstream.GetMetaDataForList(upstream_commit, 170fea5858eSSimon Glass options.git_dir, 1) 171fea5858eSSimon Glass 1723b74ba5fSSimon Glass # Conflicting tags are not a problem for buildman, since it does 1733b74ba5fSSimon Glass # not use them. For example, Series-version is not useful for 1743b74ba5fSSimon Glass # buildman. On the other hand conflicting tags will cause an 1753b74ba5fSSimon Glass # error. So allow later tags to overwrite earlier ones. 176f0b739f1SSimon Glass series.allow_overwrite = True 1773b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(range_expr, 1783b74ba5fSSimon Glass options.git_dir, None, series) 1793b74ba5fSSimon Glass else: 1803b74ba5fSSimon Glass # Honour the count 1813b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(options.branch, 1823b74ba5fSSimon Glass options.git_dir, count) 183fea5858eSSimon Glass else: 184fea5858eSSimon Glass series = None 185e5a0e5d8SSimon Glass options.verbose = True 186e5a0e5d8SSimon Glass options.show_errors = True 187fc3fe1c2SSimon Glass 188fc3fe1c2SSimon Glass # By default we have one thread per CPU. But if there are not enough jobs 189fc3fe1c2SSimon Glass # we can have fewer threads and use a high '-j' value for make. 190fc3fe1c2SSimon Glass if not options.threads: 191fc3fe1c2SSimon Glass options.threads = min(multiprocessing.cpu_count(), len(selected)) 192fc3fe1c2SSimon Glass if not options.jobs: 193fc3fe1c2SSimon Glass options.jobs = max(1, (multiprocessing.cpu_count() + 194fc3fe1c2SSimon Glass len(selected) - 1) / len(selected)) 195fc3fe1c2SSimon Glass 196fc3fe1c2SSimon Glass if not options.step: 197fc3fe1c2SSimon Glass options.step = len(series.commits) - 1 198fc3fe1c2SSimon Glass 19999796923SMasahiro Yamada gnu_make = command.Output(os.path.join(options.git, 20099796923SMasahiro Yamada 'scripts/show-gnu-make')).rstrip() 20199796923SMasahiro Yamada if not gnu_make: 20231e2141dSMasahiro Yamada sys.exit('GNU Make not found') 20399796923SMasahiro Yamada 204fc3fe1c2SSimon Glass # Create a new builder with the selected options 205fea5858eSSimon Glass if options.branch: 206fea5858eSSimon Glass dirname = options.branch 207fea5858eSSimon Glass else: 208fea5858eSSimon Glass dirname = 'current' 209fea5858eSSimon Glass output_dir = os.path.join(options.output_dir, dirname) 210fc3fe1c2SSimon Glass builder = Builder(toolchains, output_dir, options.git_dir, 21199796923SMasahiro Yamada options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 212fc3fe1c2SSimon Glass show_unknown=options.show_unknown, step=options.step) 213fc3fe1c2SSimon Glass builder.force_config_on_failure = not options.quick 214*d4144e45SSimon Glass if make_func: 215*d4144e45SSimon Glass builder.do_make = make_func 216fc3fe1c2SSimon Glass 217fc3fe1c2SSimon Glass # For a dry run, just show our actions as a sanity check 218fc3fe1c2SSimon Glass if options.dry_run: 219fc3fe1c2SSimon Glass ShowActions(series, why_selected, selected, builder, options) 220fc3fe1c2SSimon Glass else: 221fc3fe1c2SSimon Glass builder.force_build = options.force_build 2224266dc28SSimon Glass builder.force_build_failures = options.force_build_failures 22397e91526SSimon Glass builder.force_reconfig = options.force_reconfig 224189a4968SSimon Glass builder.in_tree = options.in_tree 225fc3fe1c2SSimon Glass 226fc3fe1c2SSimon Glass # Work out which boards to build 227fc3fe1c2SSimon Glass board_selected = boards.GetSelectedDict() 228fc3fe1c2SSimon Glass 229fea5858eSSimon Glass if series: 230fea5858eSSimon Glass commits = series.commits 231fea5858eSSimon Glass else: 232fea5858eSSimon Glass commits = None 233fea5858eSSimon Glass 234*d4144e45SSimon Glass Print(GetActionSummary(options.summary, commits, board_selected, 235*d4144e45SSimon Glass options)) 236fc3fe1c2SSimon Glass 237b2ea7ab2SSimon Glass builder.SetDisplayOptions(options.show_errors, options.show_sizes, 238ed966657SSimon Glass options.show_detail, options.show_bloat, 239ed966657SSimon Glass options.list_error_boards) 240fc3fe1c2SSimon Glass if options.summary: 241fc3fe1c2SSimon Glass # We can't show function sizes without board details at present 242fc3fe1c2SSimon Glass if options.show_bloat: 243fc3fe1c2SSimon Glass options.show_detail = True 244b2ea7ab2SSimon Glass builder.ShowSummary(commits, board_selected) 245fc3fe1c2SSimon Glass else: 2462c3deb97SSimon Glass fail, warned = builder.BuildBoards(commits, board_selected, 247e5a0e5d8SSimon Glass options.keep_outputs, options.verbose) 2482c3deb97SSimon Glass if fail: 2492c3deb97SSimon Glass return 128 2502c3deb97SSimon Glass elif warned: 2512c3deb97SSimon Glass return 129 2522c3deb97SSimon Glass return 0 253