1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import multiprocessing 7import os 8import shutil 9import sys 10 11import board 12import bsettings 13from builder import Builder 14import gitutil 15import patchstream 16import terminal 17from terminal import Print 18import toolchain 19import command 20import subprocess 21 22def GetPlural(count): 23 """Returns a plural 's' if count is not 1""" 24 return 's' if count != 1 else '' 25 26def GetActionSummary(is_summary, commits, selected, options): 27 """Return a string summarising the intended action. 28 29 Returns: 30 Summary string. 31 """ 32 if commits: 33 count = len(commits) 34 count = (count + options.step - 1) / options.step 35 commit_str = '%d commit%s' % (count, GetPlural(count)) 36 else: 37 commit_str = 'current source' 38 str = '%s %s for %d boards' % ( 39 'Summary of' if is_summary else 'Building', commit_str, 40 len(selected)) 41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43 return str 44 45def ShowActions(series, why_selected, boards_selected, builder, options): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the boards brought 52 in by that argument. For example, 'arm' might bring in 53 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 """ 60 col = terminal.Color() 61 print 'Dry run, so not doing much. But I would do this:' 62 print 63 if series: 64 commits = series.commits 65 else: 66 commits = None 67 print GetActionSummary(False, commits, boards_selected, 68 options) 69 print 'Build directory: %s' % builder.base_dir 70 if commits: 71 for upto in range(0, len(series.commits), options.step): 72 commit = series.commits[upto] 73 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74 print commit.subject 75 print 76 for arg in why_selected: 77 if arg != 'all': 78 print arg, ': %d boards' % why_selected[arg] 79 print ('Total boards to build for each commit: %d\n' % 80 why_selected['all']) 81 82def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83 clean_dir=False): 84 """The main control code for buildman 85 86 Args: 87 options: Command line options object 88 args: Command line arguments (list of strings) 89 toolchains: Toolchains to use - this should be a Toolchains() 90 object. If None, then it will be created and scanned 91 make_func: Make function to use for the builder. This is called 92 to execute 'make'. If this is None, the normal function 93 will be used, which calls the 'make' tool with suitable 94 arguments. This setting is useful for tests. 95 board: Boards() object to use, containing a list of available 96 boards. If this is None it will be created and scanned. 97 """ 98 global builder 99 100 if options.full_help: 101 pager = os.getenv('PAGER') 102 if not pager: 103 pager = 'more' 104 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 105 'README') 106 command.Run(pager, fname) 107 return 0 108 109 gitutil.Setup() 110 111 options.git_dir = os.path.join(options.git, '.git') 112 113 if not toolchains: 114 toolchains = toolchain.Toolchains() 115 toolchains.GetSettings() 116 toolchains.Scan(options.list_tool_chains) 117 if options.list_tool_chains: 118 toolchains.List() 119 print 120 return 0 121 122 if options.fetch_arch: 123 if options.fetch_arch == 'list': 124 sorted_list = toolchains.ListArchs() 125 print 'Available architectures: %s\n' % ' '.join(sorted_list) 126 return 0 127 else: 128 fetch_arch = options.fetch_arch 129 if fetch_arch == 'all': 130 fetch_arch = ','.join(toolchains.ListArchs()) 131 print 'Downloading toolchains: %s\n' % fetch_arch 132 for arch in fetch_arch.split(','): 133 ret = toolchains.FetchAndInstall(arch) 134 if ret: 135 return ret 136 return 0 137 138 # Work out how many commits to build. We want to build everything on the 139 # branch. We also build the upstream commit as a control so we can see 140 # problems introduced by the first commit on the branch. 141 col = terminal.Color() 142 count = options.count 143 has_range = options.branch and '..' in options.branch 144 if count == -1: 145 if not options.branch: 146 count = 1 147 else: 148 if has_range: 149 count, msg = gitutil.CountCommitsInRange(options.git_dir, 150 options.branch) 151 else: 152 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 153 options.branch) 154 if count is None: 155 sys.exit(col.Color(col.RED, msg)) 156 elif count == 0: 157 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 158 options.branch)) 159 if msg: 160 print col.Color(col.YELLOW, msg) 161 count += 1 # Build upstream commit also 162 163 if not count: 164 str = ("No commits found to process in branch '%s': " 165 "set branch's upstream or use -c flag" % options.branch) 166 sys.exit(col.Color(col.RED, str)) 167 168 # Work out what subset of the boards we are building 169 if not boards: 170 board_file = os.path.join(options.git, 'boards.cfg') 171 status = subprocess.call([os.path.join(options.git, 172 'tools/genboardscfg.py')]) 173 if status != 0: 174 sys.exit("Failed to generate boards.cfg") 175 176 boards = board.Boards() 177 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 178 179 exclude = [] 180 if options.exclude: 181 for arg in options.exclude: 182 exclude += arg.split(',') 183 184 why_selected = boards.SelectBoards(args, exclude) 185 selected = boards.GetSelected() 186 if not len(selected): 187 sys.exit(col.Color(col.RED, 'No matching boards found')) 188 189 # Read the metadata from the commits. First look at the upstream commit, 190 # then the ones in the branch. We would like to do something like 191 # upstream/master~..branch but that isn't possible if upstream/master is 192 # a merge commit (it will list all the commits that form part of the 193 # merge) 194 # Conflicting tags are not a problem for buildman, since it does not use 195 # them. For example, Series-version is not useful for buildman. On the 196 # other hand conflicting tags will cause an error. So allow later tags 197 # to overwrite earlier ones by setting allow_overwrite=True 198 if options.branch: 199 if count == -1: 200 if has_range: 201 range_expr = options.branch 202 else: 203 range_expr = gitutil.GetRangeInBranch(options.git_dir, 204 options.branch) 205 upstream_commit = gitutil.GetUpstream(options.git_dir, 206 options.branch) 207 series = patchstream.GetMetaDataForList(upstream_commit, 208 options.git_dir, 1, series=None, allow_overwrite=True) 209 210 series = patchstream.GetMetaDataForList(range_expr, 211 options.git_dir, None, series, allow_overwrite=True) 212 else: 213 # Honour the count 214 series = patchstream.GetMetaDataForList(options.branch, 215 options.git_dir, count, series=None, allow_overwrite=True) 216 else: 217 series = None 218 options.verbose = True 219 if not options.summary: 220 options.show_errors = True 221 222 # By default we have one thread per CPU. But if there are not enough jobs 223 # we can have fewer threads and use a high '-j' value for make. 224 if not options.threads: 225 options.threads = min(multiprocessing.cpu_count(), len(selected)) 226 if not options.jobs: 227 options.jobs = max(1, (multiprocessing.cpu_count() + 228 len(selected) - 1) / len(selected)) 229 230 if not options.step: 231 options.step = len(series.commits) - 1 232 233 gnu_make = command.Output(os.path.join(options.git, 234 'scripts/show-gnu-make')).rstrip() 235 if not gnu_make: 236 sys.exit('GNU Make not found') 237 238 # Create a new builder with the selected options. 239 output_dir = options.output_dir 240 if options.branch: 241 dirname = options.branch.replace('/', '_') 242 # As a special case allow the board directory to be placed in the 243 # output directory itself rather than any subdirectory. 244 if not options.no_subdirs: 245 output_dir = os.path.join(options.output_dir, dirname) 246 if (clean_dir and output_dir != options.output_dir and 247 os.path.exists(output_dir)): 248 shutil.rmtree(output_dir) 249 builder = Builder(toolchains, output_dir, options.git_dir, 250 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 251 show_unknown=options.show_unknown, step=options.step, 252 no_subdirs=options.no_subdirs, full_path=options.full_path, 253 verbose_build=options.verbose_build, 254 incremental=options.incremental, 255 per_board_out_dir=options.per_board_out_dir,) 256 builder.force_config_on_failure = not options.quick 257 if make_func: 258 builder.do_make = make_func 259 260 # For a dry run, just show our actions as a sanity check 261 if options.dry_run: 262 ShowActions(series, why_selected, selected, builder, options) 263 else: 264 builder.force_build = options.force_build 265 builder.force_build_failures = options.force_build_failures 266 builder.force_reconfig = options.force_reconfig 267 builder.in_tree = options.in_tree 268 269 # Work out which boards to build 270 board_selected = boards.GetSelectedDict() 271 272 if series: 273 commits = series.commits 274 # Number the commits for test purposes 275 for commit in range(len(commits)): 276 commits[commit].sequence = commit 277 else: 278 commits = None 279 280 Print(GetActionSummary(options.summary, commits, board_selected, 281 options)) 282 283 # We can't show function sizes without board details at present 284 if options.show_bloat: 285 options.show_detail = True 286 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 287 options.show_detail, options.show_bloat, 288 options.list_error_boards, 289 options.show_config) 290 if options.summary: 291 builder.ShowSummary(commits, board_selected) 292 else: 293 fail, warned = builder.BuildBoards(commits, board_selected, 294 options.keep_outputs, options.verbose) 295 if fail: 296 return 128 297 elif warned: 298 return 129 299 return 0 300