1fc3fe1c2SSimon Glass# Copyright (c) 2013 The Chromium OS Authors. 2fc3fe1c2SSimon Glass# 31a459660SWolfgang Denk# SPDX-License-Identifier: GPL-2.0+ 4fc3fe1c2SSimon Glass# 5fc3fe1c2SSimon Glass 6fc3fe1c2SSimon Glassimport multiprocessing 7fc3fe1c2SSimon Glassimport os 8883a321aSSimon Glassimport shutil 9fc3fe1c2SSimon Glassimport sys 10fc3fe1c2SSimon Glass 11fc3fe1c2SSimon Glassimport board 12fc3fe1c2SSimon Glassimport bsettings 13fc3fe1c2SSimon Glassfrom builder import Builder 14fc3fe1c2SSimon Glassimport gitutil 15fc3fe1c2SSimon Glassimport patchstream 16fc3fe1c2SSimon Glassimport terminal 17d4144e45SSimon Glassfrom terminal import Print 18fc3fe1c2SSimon Glassimport toolchain 1999796923SMasahiro Yamadaimport command 2073f30b9bSMasahiro Yamadaimport subprocess 21fc3fe1c2SSimon Glass 22fc3fe1c2SSimon Glassdef GetPlural(count): 23fc3fe1c2SSimon Glass """Returns a plural 's' if count is not 1""" 24fc3fe1c2SSimon Glass return 's' if count != 1 else '' 25fc3fe1c2SSimon Glass 26fea5858eSSimon Glassdef GetActionSummary(is_summary, commits, selected, options): 27fc3fe1c2SSimon Glass """Return a string summarising the intended action. 28fc3fe1c2SSimon Glass 29fc3fe1c2SSimon Glass Returns: 30fc3fe1c2SSimon Glass Summary string. 31fc3fe1c2SSimon Glass """ 32fea5858eSSimon Glass if commits: 33fea5858eSSimon Glass count = len(commits) 34fc3fe1c2SSimon Glass count = (count + options.step - 1) / options.step 35fea5858eSSimon Glass commit_str = '%d commit%s' % (count, GetPlural(count)) 36fea5858eSSimon Glass else: 37fea5858eSSimon Glass commit_str = 'current source' 38fea5858eSSimon Glass str = '%s %s for %d boards' % ( 39fea5858eSSimon Glass 'Summary of' if is_summary else 'Building', commit_str, 40fc3fe1c2SSimon Glass len(selected)) 41fc3fe1c2SSimon Glass str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42fc3fe1c2SSimon Glass GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43fc3fe1c2SSimon Glass return str 44fc3fe1c2SSimon Glass 45fc3fe1c2SSimon Glassdef ShowActions(series, why_selected, boards_selected, builder, options): 46fc3fe1c2SSimon Glass """Display a list of actions that we would take, if not a dry run. 47fc3fe1c2SSimon Glass 48fc3fe1c2SSimon Glass Args: 49fc3fe1c2SSimon Glass series: Series object 50fc3fe1c2SSimon Glass why_selected: Dictionary where each key is a buildman argument 51fc3fe1c2SSimon Glass provided by the user, and the value is the boards brought 52fc3fe1c2SSimon Glass in by that argument. For example, 'arm' might bring in 53fc3fe1c2SSimon Glass 400 boards, so in this case the key would be 'arm' and 54fc3fe1c2SSimon Glass the value would be a list of board names. 55fc3fe1c2SSimon Glass boards_selected: Dict of selected boards, key is target name, 56fc3fe1c2SSimon Glass value is Board object 57fc3fe1c2SSimon Glass builder: The builder that will be used to build the commits 58fc3fe1c2SSimon Glass options: Command line options object 59fc3fe1c2SSimon Glass """ 60fc3fe1c2SSimon Glass col = terminal.Color() 61fc3fe1c2SSimon Glass print 'Dry run, so not doing much. But I would do this:' 62fc3fe1c2SSimon Glass print 63fea5858eSSimon Glass if series: 64fea5858eSSimon Glass commits = series.commits 65fea5858eSSimon Glass else: 66fea5858eSSimon Glass commits = None 67fea5858eSSimon Glass print GetActionSummary(False, commits, boards_selected, 68fc3fe1c2SSimon Glass options) 69fc3fe1c2SSimon Glass print 'Build directory: %s' % builder.base_dir 70fea5858eSSimon Glass if commits: 71fc3fe1c2SSimon Glass for upto in range(0, len(series.commits), options.step): 72fc3fe1c2SSimon Glass commit = series.commits[upto] 73fc3fe1c2SSimon Glass print ' ', col.Color(col.YELLOW, commit.hash, bright=False), 74fc3fe1c2SSimon Glass print commit.subject 75fc3fe1c2SSimon Glass print 76fc3fe1c2SSimon Glass for arg in why_selected: 77fc3fe1c2SSimon Glass if arg != 'all': 78fc3fe1c2SSimon Glass print arg, ': %d boards' % why_selected[arg] 79fc3fe1c2SSimon Glass print ('Total boards to build for each commit: %d\n' % 80fc3fe1c2SSimon Glass why_selected['all']) 81fc3fe1c2SSimon Glass 82883a321aSSimon Glassdef DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83883a321aSSimon Glass clean_dir=False): 84fc3fe1c2SSimon Glass """The main control code for buildman 85fc3fe1c2SSimon Glass 86fc3fe1c2SSimon Glass Args: 87fc3fe1c2SSimon Glass options: Command line options object 88fc3fe1c2SSimon Glass args: Command line arguments (list of strings) 89d4144e45SSimon Glass toolchains: Toolchains to use - this should be a Toolchains() 90d4144e45SSimon Glass object. If None, then it will be created and scanned 91d4144e45SSimon Glass make_func: Make function to use for the builder. This is called 92d4144e45SSimon Glass to execute 'make'. If this is None, the normal function 93d4144e45SSimon Glass will be used, which calls the 'make' tool with suitable 94d4144e45SSimon Glass arguments. This setting is useful for tests. 95823e60b6SSimon Glass board: Boards() object to use, containing a list of available 96823e60b6SSimon Glass boards. If this is None it will be created and scanned. 97fc3fe1c2SSimon Glass """ 98883a321aSSimon Glass global builder 99883a321aSSimon Glass 10048ba5856SSimon Glass if options.full_help: 10148ba5856SSimon Glass pager = os.getenv('PAGER') 10248ba5856SSimon Glass if not pager: 10348ba5856SSimon Glass pager = 'more' 10448ba5856SSimon Glass fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') 10548ba5856SSimon Glass command.Run(pager, fname) 10648ba5856SSimon Glass return 0 10748ba5856SSimon Glass 108fc3fe1c2SSimon Glass gitutil.Setup() 109fc3fe1c2SSimon Glass 110fc3fe1c2SSimon Glass options.git_dir = os.path.join(options.git, '.git') 111fc3fe1c2SSimon Glass 112d4144e45SSimon Glass if not toolchains: 113fc3fe1c2SSimon Glass toolchains = toolchain.Toolchains() 114d4144e45SSimon Glass toolchains.GetSettings() 115fc3fe1c2SSimon Glass toolchains.Scan(options.list_tool_chains) 116fc3fe1c2SSimon Glass if options.list_tool_chains: 117fc3fe1c2SSimon Glass toolchains.List() 118fc3fe1c2SSimon Glass print 1192c3deb97SSimon Glass return 0 120fc3fe1c2SSimon Glass 121fc3fe1c2SSimon Glass # Work out how many commits to build. We want to build everything on the 122fc3fe1c2SSimon Glass # branch. We also build the upstream commit as a control so we can see 123fc3fe1c2SSimon Glass # problems introduced by the first commit on the branch. 124fc3fe1c2SSimon Glass col = terminal.Color() 125fc3fe1c2SSimon Glass count = options.count 126fc3fe1c2SSimon Glass if count == -1: 127fc3fe1c2SSimon Glass if not options.branch: 128fea5858eSSimon Glass count = 1 129fea5858eSSimon Glass else: 130fea5858eSSimon Glass count = gitutil.CountCommitsInBranch(options.git_dir, 131fea5858eSSimon Glass options.branch) 132cce717a9SSimon Glass if count is None: 133fea5858eSSimon Glass str = ("Branch '%s' not found or has no upstream" % 134fea5858eSSimon Glass options.branch) 13531e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, str)) 136fc3fe1c2SSimon Glass count += 1 # Build upstream commit also 137fc3fe1c2SSimon Glass 138fc3fe1c2SSimon Glass if not count: 139fc3fe1c2SSimon Glass str = ("No commits found to process in branch '%s': " 140fc3fe1c2SSimon Glass "set branch's upstream or use -c flag" % options.branch) 14131e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, str)) 142fc3fe1c2SSimon Glass 143fc3fe1c2SSimon Glass # Work out what subset of the boards we are building 144823e60b6SSimon Glass if not boards: 14573f30b9bSMasahiro Yamada board_file = os.path.join(options.git, 'boards.cfg') 14673f30b9bSMasahiro Yamada status = subprocess.call([os.path.join(options.git, 14773f30b9bSMasahiro Yamada 'tools/genboardscfg.py')]) 14873f30b9bSMasahiro Yamada if status != 0: 14931e2141dSMasahiro Yamada sys.exit("Failed to generate boards.cfg") 15073f30b9bSMasahiro Yamada 151fc3fe1c2SSimon Glass boards = board.Boards() 152fc3fe1c2SSimon Glass boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 1533cf4ae6fSSimon Glass 1543cf4ae6fSSimon Glass exclude = [] 1553cf4ae6fSSimon Glass if options.exclude: 1563cf4ae6fSSimon Glass for arg in options.exclude: 1573cf4ae6fSSimon Glass exclude += arg.split(',') 1583cf4ae6fSSimon Glass 1593cf4ae6fSSimon Glass why_selected = boards.SelectBoards(args, exclude) 160fc3fe1c2SSimon Glass selected = boards.GetSelected() 161fc3fe1c2SSimon Glass if not len(selected): 16231e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, 'No matching boards found')) 163fc3fe1c2SSimon Glass 164fc3fe1c2SSimon Glass # Read the metadata from the commits. First look at the upstream commit, 165fc3fe1c2SSimon Glass # then the ones in the branch. We would like to do something like 166fc3fe1c2SSimon Glass # upstream/master~..branch but that isn't possible if upstream/master is 167fc3fe1c2SSimon Glass # a merge commit (it will list all the commits that form part of the 168fc3fe1c2SSimon Glass # merge) 169950a2313SSimon Glass # Conflicting tags are not a problem for buildman, since it does not use 170950a2313SSimon Glass # them. For example, Series-version is not useful for buildman. On the 171950a2313SSimon Glass # other hand conflicting tags will cause an error. So allow later tags 172950a2313SSimon Glass # to overwrite earlier ones by setting allow_overwrite=True 173fea5858eSSimon Glass if options.branch: 1743b74ba5fSSimon Glass if count == -1: 1753b74ba5fSSimon Glass range_expr = gitutil.GetRangeInBranch(options.git_dir, 1763b74ba5fSSimon Glass options.branch) 1773b74ba5fSSimon Glass upstream_commit = gitutil.GetUpstream(options.git_dir, 1783b74ba5fSSimon Glass options.branch) 179fea5858eSSimon Glass series = patchstream.GetMetaDataForList(upstream_commit, 180950a2313SSimon Glass options.git_dir, 1, series=None, allow_overwrite=True) 181fea5858eSSimon Glass 1823b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(range_expr, 183950a2313SSimon Glass options.git_dir, None, series, allow_overwrite=True) 1843b74ba5fSSimon Glass else: 1853b74ba5fSSimon Glass # Honour the count 1863b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(options.branch, 187950a2313SSimon Glass options.git_dir, count, series=None, allow_overwrite=True) 188fea5858eSSimon Glass else: 189fea5858eSSimon Glass series = None 190e5a0e5d8SSimon Glass options.verbose = True 191*58d818f1SSimon Glass if not options.summary: 192e5a0e5d8SSimon Glass options.show_errors = True 193fc3fe1c2SSimon Glass 194fc3fe1c2SSimon Glass # By default we have one thread per CPU. But if there are not enough jobs 195fc3fe1c2SSimon Glass # we can have fewer threads and use a high '-j' value for make. 196fc3fe1c2SSimon Glass if not options.threads: 197fc3fe1c2SSimon Glass options.threads = min(multiprocessing.cpu_count(), len(selected)) 198fc3fe1c2SSimon Glass if not options.jobs: 199fc3fe1c2SSimon Glass options.jobs = max(1, (multiprocessing.cpu_count() + 200fc3fe1c2SSimon Glass len(selected) - 1) / len(selected)) 201fc3fe1c2SSimon Glass 202fc3fe1c2SSimon Glass if not options.step: 203fc3fe1c2SSimon Glass options.step = len(series.commits) - 1 204fc3fe1c2SSimon Glass 20599796923SMasahiro Yamada gnu_make = command.Output(os.path.join(options.git, 20699796923SMasahiro Yamada 'scripts/show-gnu-make')).rstrip() 20799796923SMasahiro Yamada if not gnu_make: 20831e2141dSMasahiro Yamada sys.exit('GNU Make not found') 20999796923SMasahiro Yamada 210fc3fe1c2SSimon Glass # Create a new builder with the selected options 211fea5858eSSimon Glass if options.branch: 212f7582ce8SSimon Glass dirname = options.branch.replace('/', '_') 213fea5858eSSimon Glass else: 214fea5858eSSimon Glass dirname = 'current' 215fea5858eSSimon Glass output_dir = os.path.join(options.output_dir, dirname) 216883a321aSSimon Glass if clean_dir and os.path.exists(output_dir): 217883a321aSSimon Glass shutil.rmtree(output_dir) 218fc3fe1c2SSimon Glass builder = Builder(toolchains, output_dir, options.git_dir, 21999796923SMasahiro Yamada options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 220fc3fe1c2SSimon Glass show_unknown=options.show_unknown, step=options.step) 221fc3fe1c2SSimon Glass builder.force_config_on_failure = not options.quick 222d4144e45SSimon Glass if make_func: 223d4144e45SSimon Glass builder.do_make = make_func 224fc3fe1c2SSimon Glass 225fc3fe1c2SSimon Glass # For a dry run, just show our actions as a sanity check 226fc3fe1c2SSimon Glass if options.dry_run: 227fc3fe1c2SSimon Glass ShowActions(series, why_selected, selected, builder, options) 228fc3fe1c2SSimon Glass else: 229fc3fe1c2SSimon Glass builder.force_build = options.force_build 2304266dc28SSimon Glass builder.force_build_failures = options.force_build_failures 23197e91526SSimon Glass builder.force_reconfig = options.force_reconfig 232189a4968SSimon Glass builder.in_tree = options.in_tree 233fc3fe1c2SSimon Glass 234fc3fe1c2SSimon Glass # Work out which boards to build 235fc3fe1c2SSimon Glass board_selected = boards.GetSelectedDict() 236fc3fe1c2SSimon Glass 237fea5858eSSimon Glass if series: 238fea5858eSSimon Glass commits = series.commits 239883a321aSSimon Glass # Number the commits for test purposes 240883a321aSSimon Glass for commit in range(len(commits)): 241883a321aSSimon Glass commits[commit].sequence = commit 242fea5858eSSimon Glass else: 243fea5858eSSimon Glass commits = None 244fea5858eSSimon Glass 245d4144e45SSimon Glass Print(GetActionSummary(options.summary, commits, board_selected, 246d4144e45SSimon Glass options)) 247fc3fe1c2SSimon Glass 2487798e228SSimon Glass # We can't show function sizes without board details at present 2497798e228SSimon Glass if options.show_bloat: 2507798e228SSimon Glass options.show_detail = True 251b2ea7ab2SSimon Glass builder.SetDisplayOptions(options.show_errors, options.show_sizes, 252ed966657SSimon Glass options.show_detail, options.show_bloat, 253ed966657SSimon Glass options.list_error_boards) 254fc3fe1c2SSimon Glass if options.summary: 255b2ea7ab2SSimon Glass builder.ShowSummary(commits, board_selected) 256fc3fe1c2SSimon Glass else: 2572c3deb97SSimon Glass fail, warned = builder.BuildBoards(commits, board_selected, 258e5a0e5d8SSimon Glass options.keep_outputs, options.verbose) 2592c3deb97SSimon Glass if fail: 2602c3deb97SSimon Glass return 128 2612c3deb97SSimon Glass elif warned: 2622c3deb97SSimon Glass return 129 2632c3deb97SSimon Glass return 0 264