1fc3fe1c2SSimon Glass# Copyright (c) 2013 The Chromium OS Authors. 2fc3fe1c2SSimon Glass# 31a459660SWolfgang Denk# SPDX-License-Identifier: GPL-2.0+ 4fc3fe1c2SSimon Glass# 5fc3fe1c2SSimon Glass 6fc3fe1c2SSimon Glassimport multiprocessing 7fc3fe1c2SSimon Glassimport os 8883a321aSSimon Glassimport shutil 9fc3fe1c2SSimon Glassimport sys 10fc3fe1c2SSimon Glass 11fc3fe1c2SSimon Glassimport board 12fc3fe1c2SSimon Glassimport bsettings 13fc3fe1c2SSimon Glassfrom builder import Builder 14fc3fe1c2SSimon Glassimport gitutil 15fc3fe1c2SSimon Glassimport patchstream 16fc3fe1c2SSimon Glassimport terminal 17d4144e45SSimon Glassfrom terminal import Print 18fc3fe1c2SSimon Glassimport toolchain 1999796923SMasahiro Yamadaimport command 2073f30b9bSMasahiro Yamadaimport subprocess 21fc3fe1c2SSimon Glass 22fc3fe1c2SSimon Glassdef GetPlural(count): 23fc3fe1c2SSimon Glass """Returns a plural 's' if count is not 1""" 24fc3fe1c2SSimon Glass return 's' if count != 1 else '' 25fc3fe1c2SSimon Glass 26fea5858eSSimon Glassdef GetActionSummary(is_summary, commits, selected, options): 27fc3fe1c2SSimon Glass """Return a string summarising the intended action. 28fc3fe1c2SSimon Glass 29fc3fe1c2SSimon Glass Returns: 30fc3fe1c2SSimon Glass Summary string. 31fc3fe1c2SSimon Glass """ 32fea5858eSSimon Glass if commits: 33fea5858eSSimon Glass count = len(commits) 34fc3fe1c2SSimon Glass count = (count + options.step - 1) / options.step 35fea5858eSSimon Glass commit_str = '%d commit%s' % (count, GetPlural(count)) 36fea5858eSSimon Glass else: 37fea5858eSSimon Glass commit_str = 'current source' 38fea5858eSSimon Glass str = '%s %s for %d boards' % ( 39fea5858eSSimon Glass 'Summary of' if is_summary else 'Building', commit_str, 40fc3fe1c2SSimon Glass len(selected)) 41fc3fe1c2SSimon Glass str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42fc3fe1c2SSimon Glass GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43fc3fe1c2SSimon Glass return str 44fc3fe1c2SSimon Glass 45fc3fe1c2SSimon Glassdef ShowActions(series, why_selected, boards_selected, builder, options): 46fc3fe1c2SSimon Glass """Display a list of actions that we would take, if not a dry run. 47fc3fe1c2SSimon Glass 48fc3fe1c2SSimon Glass Args: 49fc3fe1c2SSimon Glass series: Series object 50fc3fe1c2SSimon Glass why_selected: Dictionary where each key is a buildman argument 51fc3fe1c2SSimon Glass provided by the user, and the value is the boards brought 52fc3fe1c2SSimon Glass in by that argument. For example, 'arm' might bring in 53fc3fe1c2SSimon Glass 400 boards, so in this case the key would be 'arm' and 54fc3fe1c2SSimon Glass the value would be a list of board names. 55fc3fe1c2SSimon Glass boards_selected: Dict of selected boards, key is target name, 56fc3fe1c2SSimon Glass value is Board object 57fc3fe1c2SSimon Glass builder: The builder that will be used to build the commits 58fc3fe1c2SSimon Glass options: Command line options object 59fc3fe1c2SSimon Glass """ 60fc3fe1c2SSimon Glass col = terminal.Color() 61fc3fe1c2SSimon Glass print 'Dry run, so not doing much. But I would do this:' 62fc3fe1c2SSimon Glass print 63fea5858eSSimon Glass if series: 64fea5858eSSimon Glass commits = series.commits 65fea5858eSSimon Glass else: 66fea5858eSSimon Glass commits = None 67fea5858eSSimon Glass print GetActionSummary(False, commits, boards_selected, 68fc3fe1c2SSimon Glass options) 69fc3fe1c2SSimon Glass print 'Build directory: %s' % builder.base_dir 70fea5858eSSimon Glass if commits: 71fc3fe1c2SSimon Glass for upto in range(0, len(series.commits), options.step): 72fc3fe1c2SSimon Glass commit = series.commits[upto] 731ddda1b3SSimon Glass print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74fc3fe1c2SSimon Glass print commit.subject 75fc3fe1c2SSimon Glass print 76fc3fe1c2SSimon Glass for arg in why_selected: 77fc3fe1c2SSimon Glass if arg != 'all': 78fc3fe1c2SSimon Glass print arg, ': %d boards' % why_selected[arg] 79fc3fe1c2SSimon Glass print ('Total boards to build for each commit: %d\n' % 80fc3fe1c2SSimon Glass why_selected['all']) 81fc3fe1c2SSimon Glass 82883a321aSSimon Glassdef DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83883a321aSSimon Glass clean_dir=False): 84fc3fe1c2SSimon Glass """The main control code for buildman 85fc3fe1c2SSimon Glass 86fc3fe1c2SSimon Glass Args: 87fc3fe1c2SSimon Glass options: Command line options object 88fc3fe1c2SSimon Glass args: Command line arguments (list of strings) 89d4144e45SSimon Glass toolchains: Toolchains to use - this should be a Toolchains() 90d4144e45SSimon Glass object. If None, then it will be created and scanned 91d4144e45SSimon Glass make_func: Make function to use for the builder. This is called 92d4144e45SSimon Glass to execute 'make'. If this is None, the normal function 93d4144e45SSimon Glass will be used, which calls the 'make' tool with suitable 94d4144e45SSimon Glass arguments. This setting is useful for tests. 95823e60b6SSimon Glass board: Boards() object to use, containing a list of available 96823e60b6SSimon Glass boards. If this is None it will be created and scanned. 97fc3fe1c2SSimon Glass """ 98883a321aSSimon Glass global builder 99883a321aSSimon Glass 10048ba5856SSimon Glass if options.full_help: 10148ba5856SSimon Glass pager = os.getenv('PAGER') 10248ba5856SSimon Glass if not pager: 10348ba5856SSimon Glass pager = 'more' 10448ba5856SSimon Glass fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') 10548ba5856SSimon Glass command.Run(pager, fname) 10648ba5856SSimon Glass return 0 10748ba5856SSimon Glass 108fc3fe1c2SSimon Glass gitutil.Setup() 109fc3fe1c2SSimon Glass 110fc3fe1c2SSimon Glass options.git_dir = os.path.join(options.git, '.git') 111fc3fe1c2SSimon Glass 112d4144e45SSimon Glass if not toolchains: 113fc3fe1c2SSimon Glass toolchains = toolchain.Toolchains() 114d4144e45SSimon Glass toolchains.GetSettings() 115fc3fe1c2SSimon Glass toolchains.Scan(options.list_tool_chains) 116fc3fe1c2SSimon Glass if options.list_tool_chains: 117fc3fe1c2SSimon Glass toolchains.List() 118fc3fe1c2SSimon Glass print 1192c3deb97SSimon Glass return 0 120fc3fe1c2SSimon Glass 121827e37b5SSimon Glass if options.fetch_arch: 122827e37b5SSimon Glass if options.fetch_arch == 'list': 123827e37b5SSimon Glass sorted_list = toolchains.ListArchs() 124827e37b5SSimon Glass print 'Available architectures: %s\n' % ' '.join(sorted_list) 125827e37b5SSimon Glass return 0 126827e37b5SSimon Glass else: 127827e37b5SSimon Glass fetch_arch = options.fetch_arch 128827e37b5SSimon Glass if fetch_arch == 'all': 129827e37b5SSimon Glass fetch_arch = ','.join(toolchains.ListArchs()) 130827e37b5SSimon Glass print 'Downloading toolchains: %s\n' % fetch_arch 131827e37b5SSimon Glass for arch in fetch_arch.split(','): 132827e37b5SSimon Glass ret = toolchains.FetchAndInstall(arch) 133827e37b5SSimon Glass if ret: 134827e37b5SSimon Glass return ret 135827e37b5SSimon Glass return 0 136827e37b5SSimon Glass 137fc3fe1c2SSimon Glass # Work out how many commits to build. We want to build everything on the 138fc3fe1c2SSimon Glass # branch. We also build the upstream commit as a control so we can see 139fc3fe1c2SSimon Glass # problems introduced by the first commit on the branch. 140fc3fe1c2SSimon Glass col = terminal.Color() 141fc3fe1c2SSimon Glass count = options.count 1425abab20dSSimon Glass has_range = options.branch and '..' in options.branch 143fc3fe1c2SSimon Glass if count == -1: 144fc3fe1c2SSimon Glass if not options.branch: 145fea5858eSSimon Glass count = 1 146fea5858eSSimon Glass else: 1475abab20dSSimon Glass if has_range: 1485abab20dSSimon Glass count, msg = gitutil.CountCommitsInRange(options.git_dir, 1495abab20dSSimon Glass options.branch) 1505abab20dSSimon Glass else: 1512a9e2c6aSSimon Glass count, msg = gitutil.CountCommitsInBranch(options.git_dir, 152fea5858eSSimon Glass options.branch) 153cce717a9SSimon Glass if count is None: 1542a9e2c6aSSimon Glass sys.exit(col.Color(col.RED, msg)) 1555abab20dSSimon Glass elif count == 0: 1565abab20dSSimon Glass sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 1575abab20dSSimon Glass options.branch)) 1582a9e2c6aSSimon Glass if msg: 1592a9e2c6aSSimon Glass print col.Color(col.YELLOW, msg) 160fc3fe1c2SSimon Glass count += 1 # Build upstream commit also 161fc3fe1c2SSimon Glass 162fc3fe1c2SSimon Glass if not count: 163fc3fe1c2SSimon Glass str = ("No commits found to process in branch '%s': " 164fc3fe1c2SSimon Glass "set branch's upstream or use -c flag" % options.branch) 16531e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, str)) 166fc3fe1c2SSimon Glass 167fc3fe1c2SSimon Glass # Work out what subset of the boards we are building 168823e60b6SSimon Glass if not boards: 16973f30b9bSMasahiro Yamada board_file = os.path.join(options.git, 'boards.cfg') 17073f30b9bSMasahiro Yamada status = subprocess.call([os.path.join(options.git, 17173f30b9bSMasahiro Yamada 'tools/genboardscfg.py')]) 17273f30b9bSMasahiro Yamada if status != 0: 17331e2141dSMasahiro Yamada sys.exit("Failed to generate boards.cfg") 17473f30b9bSMasahiro Yamada 175fc3fe1c2SSimon Glass boards = board.Boards() 176fc3fe1c2SSimon Glass boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 1773cf4ae6fSSimon Glass 1783cf4ae6fSSimon Glass exclude = [] 1793cf4ae6fSSimon Glass if options.exclude: 1803cf4ae6fSSimon Glass for arg in options.exclude: 1813cf4ae6fSSimon Glass exclude += arg.split(',') 1823cf4ae6fSSimon Glass 1833cf4ae6fSSimon Glass why_selected = boards.SelectBoards(args, exclude) 184fc3fe1c2SSimon Glass selected = boards.GetSelected() 185fc3fe1c2SSimon Glass if not len(selected): 18631e2141dSMasahiro Yamada sys.exit(col.Color(col.RED, 'No matching boards found')) 187fc3fe1c2SSimon Glass 188fc3fe1c2SSimon Glass # Read the metadata from the commits. First look at the upstream commit, 189fc3fe1c2SSimon Glass # then the ones in the branch. We would like to do something like 190fc3fe1c2SSimon Glass # upstream/master~..branch but that isn't possible if upstream/master is 191fc3fe1c2SSimon Glass # a merge commit (it will list all the commits that form part of the 192fc3fe1c2SSimon Glass # merge) 193950a2313SSimon Glass # Conflicting tags are not a problem for buildman, since it does not use 194950a2313SSimon Glass # them. For example, Series-version is not useful for buildman. On the 195950a2313SSimon Glass # other hand conflicting tags will cause an error. So allow later tags 196950a2313SSimon Glass # to overwrite earlier ones by setting allow_overwrite=True 197fea5858eSSimon Glass if options.branch: 1983b74ba5fSSimon Glass if count == -1: 1995abab20dSSimon Glass if has_range: 2005abab20dSSimon Glass range_expr = options.branch 2015abab20dSSimon Glass else: 2023b74ba5fSSimon Glass range_expr = gitutil.GetRangeInBranch(options.git_dir, 2033b74ba5fSSimon Glass options.branch) 2043b74ba5fSSimon Glass upstream_commit = gitutil.GetUpstream(options.git_dir, 2053b74ba5fSSimon Glass options.branch) 206fea5858eSSimon Glass series = patchstream.GetMetaDataForList(upstream_commit, 207950a2313SSimon Glass options.git_dir, 1, series=None, allow_overwrite=True) 208fea5858eSSimon Glass 2093b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(range_expr, 210950a2313SSimon Glass options.git_dir, None, series, allow_overwrite=True) 2113b74ba5fSSimon Glass else: 2123b74ba5fSSimon Glass # Honour the count 2133b74ba5fSSimon Glass series = patchstream.GetMetaDataForList(options.branch, 214950a2313SSimon Glass options.git_dir, count, series=None, allow_overwrite=True) 215fea5858eSSimon Glass else: 216fea5858eSSimon Glass series = None 217e5a0e5d8SSimon Glass options.verbose = True 21858d818f1SSimon Glass if not options.summary: 219e5a0e5d8SSimon Glass options.show_errors = True 220fc3fe1c2SSimon Glass 221fc3fe1c2SSimon Glass # By default we have one thread per CPU. But if there are not enough jobs 222fc3fe1c2SSimon Glass # we can have fewer threads and use a high '-j' value for make. 223fc3fe1c2SSimon Glass if not options.threads: 224fc3fe1c2SSimon Glass options.threads = min(multiprocessing.cpu_count(), len(selected)) 225fc3fe1c2SSimon Glass if not options.jobs: 226fc3fe1c2SSimon Glass options.jobs = max(1, (multiprocessing.cpu_count() + 227fc3fe1c2SSimon Glass len(selected) - 1) / len(selected)) 228fc3fe1c2SSimon Glass 229fc3fe1c2SSimon Glass if not options.step: 230fc3fe1c2SSimon Glass options.step = len(series.commits) - 1 231fc3fe1c2SSimon Glass 23299796923SMasahiro Yamada gnu_make = command.Output(os.path.join(options.git, 23399796923SMasahiro Yamada 'scripts/show-gnu-make')).rstrip() 23499796923SMasahiro Yamada if not gnu_make: 23531e2141dSMasahiro Yamada sys.exit('GNU Make not found') 23699796923SMasahiro Yamada 23705c96b18SSimon Glass # Create a new builder with the selected options. 23805c96b18SSimon Glass output_dir = options.output_dir 239fea5858eSSimon Glass if options.branch: 240f7582ce8SSimon Glass dirname = options.branch.replace('/', '_') 2415971ab5cSSimon Glass # As a special case allow the board directory to be placed in the 2425971ab5cSSimon Glass # output directory itself rather than any subdirectory. 2435971ab5cSSimon Glass if not options.no_subdirs: 244fea5858eSSimon Glass output_dir = os.path.join(options.output_dir, dirname) 2450740127fSSimon Glass if (clean_dir and output_dir != options.output_dir and 2460740127fSSimon Glass os.path.exists(output_dir)): 247883a321aSSimon Glass shutil.rmtree(output_dir) 248fc3fe1c2SSimon Glass builder = Builder(toolchains, output_dir, options.git_dir, 24999796923SMasahiro Yamada options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 2505971ab5cSSimon Glass show_unknown=options.show_unknown, step=options.step, 251d2ce658dSSimon Glass no_subdirs=options.no_subdirs, full_path=options.full_path, 252d2ce658dSSimon Glass verbose_build=options.verbose_build) 253fc3fe1c2SSimon Glass builder.force_config_on_failure = not options.quick 254d4144e45SSimon Glass if make_func: 255d4144e45SSimon Glass builder.do_make = make_func 256fc3fe1c2SSimon Glass 257fc3fe1c2SSimon Glass # For a dry run, just show our actions as a sanity check 258fc3fe1c2SSimon Glass if options.dry_run: 259fc3fe1c2SSimon Glass ShowActions(series, why_selected, selected, builder, options) 260fc3fe1c2SSimon Glass else: 261fc3fe1c2SSimon Glass builder.force_build = options.force_build 2624266dc28SSimon Glass builder.force_build_failures = options.force_build_failures 26397e91526SSimon Glass builder.force_reconfig = options.force_reconfig 264189a4968SSimon Glass builder.in_tree = options.in_tree 265fc3fe1c2SSimon Glass 266fc3fe1c2SSimon Glass # Work out which boards to build 267fc3fe1c2SSimon Glass board_selected = boards.GetSelectedDict() 268fc3fe1c2SSimon Glass 269fea5858eSSimon Glass if series: 270fea5858eSSimon Glass commits = series.commits 271883a321aSSimon Glass # Number the commits for test purposes 272883a321aSSimon Glass for commit in range(len(commits)): 273883a321aSSimon Glass commits[commit].sequence = commit 274fea5858eSSimon Glass else: 275fea5858eSSimon Glass commits = None 276fea5858eSSimon Glass 277d4144e45SSimon Glass Print(GetActionSummary(options.summary, commits, board_selected, 278d4144e45SSimon Glass options)) 279fc3fe1c2SSimon Glass 2807798e228SSimon Glass # We can't show function sizes without board details at present 2817798e228SSimon Glass if options.show_bloat: 2827798e228SSimon Glass options.show_detail = True 283b2ea7ab2SSimon Glass builder.SetDisplayOptions(options.show_errors, options.show_sizes, 284ed966657SSimon Glass options.show_detail, options.show_bloat, 285*843312dcSSimon Glass options.list_error_boards, 286*843312dcSSimon Glass options.show_config) 287fc3fe1c2SSimon Glass if options.summary: 288b2ea7ab2SSimon Glass builder.ShowSummary(commits, board_selected) 289fc3fe1c2SSimon Glass else: 2902c3deb97SSimon Glass fail, warned = builder.BuildBoards(commits, board_selected, 291e5a0e5d8SSimon Glass options.keep_outputs, options.verbose) 2922c3deb97SSimon Glass if fail: 2932c3deb97SSimon Glass return 128 2942c3deb97SSimon Glass elif warned: 2952c3deb97SSimon Glass return 129 2962c3deb97SSimon Glass return 0 297