1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2013 The Chromium OS Authors. 3# 4 5import multiprocessing 6import os 7import shutil 8import sys 9 10import board 11import bsettings 12from builder import Builder 13import gitutil 14import patchstream 15import terminal 16from terminal import Print 17import toolchain 18import command 19import subprocess 20 21def GetPlural(count): 22 """Returns a plural 's' if count is not 1""" 23 return 's' if count != 1 else '' 24 25def GetActionSummary(is_summary, commits, selected, options): 26 """Return a string summarising the intended action. 27 28 Returns: 29 Summary string. 30 """ 31 if commits: 32 count = len(commits) 33 count = (count + options.step - 1) / options.step 34 commit_str = '%d commit%s' % (count, GetPlural(count)) 35 else: 36 commit_str = 'current source' 37 str = '%s %s for %d boards' % ( 38 'Summary of' if is_summary else 'Building', commit_str, 39 len(selected)) 40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 42 return str 43 44def ShowActions(series, why_selected, boards_selected, builder, options): 45 """Display a list of actions that we would take, if not a dry run. 46 47 Args: 48 series: Series object 49 why_selected: Dictionary where each key is a buildman argument 50 provided by the user, and the value is the list of boards 51 brought in by that argument. For example, 'arm' might bring 52 in 400 boards, so in this case the key would be 'arm' and 53 the value would be a list of board names. 54 boards_selected: Dict of selected boards, key is target name, 55 value is Board object 56 builder: The builder that will be used to build the commits 57 options: Command line options object 58 """ 59 col = terminal.Color() 60 print 'Dry run, so not doing much. But I would do this:' 61 print 62 if series: 63 commits = series.commits 64 else: 65 commits = None 66 print GetActionSummary(False, commits, boards_selected, 67 options) 68 print 'Build directory: %s' % builder.base_dir 69 if commits: 70 for upto in range(0, len(series.commits), options.step): 71 commit = series.commits[upto] 72 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 73 print commit.subject 74 print 75 for arg in why_selected: 76 if arg != 'all': 77 print arg, ': %d boards' % len(why_selected[arg]) 78 if options.verbose: 79 print ' %s' % ' '.join(why_selected[arg]) 80 print ('Total boards to build for each commit: %d\n' % 81 len(why_selected['all'])) 82 83def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 84 clean_dir=False): 85 """The main control code for buildman 86 87 Args: 88 options: Command line options object 89 args: Command line arguments (list of strings) 90 toolchains: Toolchains to use - this should be a Toolchains() 91 object. If None, then it will be created and scanned 92 make_func: Make function to use for the builder. This is called 93 to execute 'make'. If this is None, the normal function 94 will be used, which calls the 'make' tool with suitable 95 arguments. This setting is useful for tests. 96 board: Boards() object to use, containing a list of available 97 boards. If this is None it will be created and scanned. 98 """ 99 global builder 100 101 if options.full_help: 102 pager = os.getenv('PAGER') 103 if not pager: 104 pager = 'more' 105 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 106 'README') 107 command.Run(pager, fname) 108 return 0 109 110 gitutil.Setup() 111 col = terminal.Color() 112 113 options.git_dir = os.path.join(options.git, '.git') 114 115 no_toolchains = toolchains is None 116 if no_toolchains: 117 toolchains = toolchain.Toolchains() 118 119 if options.fetch_arch: 120 if options.fetch_arch == 'list': 121 sorted_list = toolchains.ListArchs() 122 print col.Color(col.BLUE, 'Available architectures: %s\n' % 123 ' '.join(sorted_list)) 124 return 0 125 else: 126 fetch_arch = options.fetch_arch 127 if fetch_arch == 'all': 128 fetch_arch = ','.join(toolchains.ListArchs()) 129 print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 130 fetch_arch) 131 for arch in fetch_arch.split(','): 132 print 133 ret = toolchains.FetchAndInstall(arch) 134 if ret: 135 return ret 136 return 0 137 138 if no_toolchains: 139 toolchains.GetSettings() 140 toolchains.Scan(options.list_tool_chains) 141 if options.list_tool_chains: 142 toolchains.List() 143 print 144 return 0 145 146 # Work out how many commits to build. We want to build everything on the 147 # branch. We also build the upstream commit as a control so we can see 148 # problems introduced by the first commit on the branch. 149 count = options.count 150 has_range = options.branch and '..' in options.branch 151 if count == -1: 152 if not options.branch: 153 count = 1 154 else: 155 if has_range: 156 count, msg = gitutil.CountCommitsInRange(options.git_dir, 157 options.branch) 158 else: 159 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 160 options.branch) 161 if count is None: 162 sys.exit(col.Color(col.RED, msg)) 163 elif count == 0: 164 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 165 options.branch)) 166 if msg: 167 print col.Color(col.YELLOW, msg) 168 count += 1 # Build upstream commit also 169 170 if not count: 171 str = ("No commits found to process in branch '%s': " 172 "set branch's upstream or use -c flag" % options.branch) 173 sys.exit(col.Color(col.RED, str)) 174 175 # Work out what subset of the boards we are building 176 if not boards: 177 board_file = os.path.join(options.git, 'boards.cfg') 178 status = subprocess.call([os.path.join(options.git, 179 'tools/genboardscfg.py')]) 180 if status != 0: 181 sys.exit("Failed to generate boards.cfg") 182 183 boards = board.Boards() 184 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 185 186 exclude = [] 187 if options.exclude: 188 for arg in options.exclude: 189 exclude += arg.split(',') 190 191 why_selected = boards.SelectBoards(args, exclude) 192 selected = boards.GetSelected() 193 if not len(selected): 194 sys.exit(col.Color(col.RED, 'No matching boards found')) 195 196 # Read the metadata from the commits. First look at the upstream commit, 197 # then the ones in the branch. We would like to do something like 198 # upstream/master~..branch but that isn't possible if upstream/master is 199 # a merge commit (it will list all the commits that form part of the 200 # merge) 201 # Conflicting tags are not a problem for buildman, since it does not use 202 # them. For example, Series-version is not useful for buildman. On the 203 # other hand conflicting tags will cause an error. So allow later tags 204 # to overwrite earlier ones by setting allow_overwrite=True 205 if options.branch: 206 if count == -1: 207 if has_range: 208 range_expr = options.branch 209 else: 210 range_expr = gitutil.GetRangeInBranch(options.git_dir, 211 options.branch) 212 upstream_commit = gitutil.GetUpstream(options.git_dir, 213 options.branch) 214 series = patchstream.GetMetaDataForList(upstream_commit, 215 options.git_dir, 1, series=None, allow_overwrite=True) 216 217 series = patchstream.GetMetaDataForList(range_expr, 218 options.git_dir, None, series, allow_overwrite=True) 219 else: 220 # Honour the count 221 series = patchstream.GetMetaDataForList(options.branch, 222 options.git_dir, count, series=None, allow_overwrite=True) 223 else: 224 series = None 225 if not options.dry_run: 226 options.verbose = True 227 if not options.summary: 228 options.show_errors = True 229 230 # By default we have one thread per CPU. But if there are not enough jobs 231 # we can have fewer threads and use a high '-j' value for make. 232 if not options.threads: 233 options.threads = min(multiprocessing.cpu_count(), len(selected)) 234 if not options.jobs: 235 options.jobs = max(1, (multiprocessing.cpu_count() + 236 len(selected) - 1) / len(selected)) 237 238 if not options.step: 239 options.step = len(series.commits) - 1 240 241 gnu_make = command.Output(os.path.join(options.git, 242 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 243 if not gnu_make: 244 sys.exit('GNU Make not found') 245 246 # Create a new builder with the selected options. 247 output_dir = options.output_dir 248 if options.branch: 249 dirname = options.branch.replace('/', '_') 250 # As a special case allow the board directory to be placed in the 251 # output directory itself rather than any subdirectory. 252 if not options.no_subdirs: 253 output_dir = os.path.join(options.output_dir, dirname) 254 if (clean_dir and output_dir != options.output_dir and 255 os.path.exists(output_dir)): 256 shutil.rmtree(output_dir) 257 builder = Builder(toolchains, output_dir, options.git_dir, 258 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 259 show_unknown=options.show_unknown, step=options.step, 260 no_subdirs=options.no_subdirs, full_path=options.full_path, 261 verbose_build=options.verbose_build, 262 incremental=options.incremental, 263 per_board_out_dir=options.per_board_out_dir, 264 config_only=options.config_only, 265 squash_config_y=not options.preserve_config_y, 266 warnings_as_errors=options.warnings_as_errors) 267 builder.force_config_on_failure = not options.quick 268 if make_func: 269 builder.do_make = make_func 270 271 # For a dry run, just show our actions as a sanity check 272 if options.dry_run: 273 ShowActions(series, why_selected, selected, builder, options) 274 else: 275 builder.force_build = options.force_build 276 builder.force_build_failures = options.force_build_failures 277 builder.force_reconfig = options.force_reconfig 278 builder.in_tree = options.in_tree 279 280 # Work out which boards to build 281 board_selected = boards.GetSelectedDict() 282 283 if series: 284 commits = series.commits 285 # Number the commits for test purposes 286 for commit in range(len(commits)): 287 commits[commit].sequence = commit 288 else: 289 commits = None 290 291 Print(GetActionSummary(options.summary, commits, board_selected, 292 options)) 293 294 # We can't show function sizes without board details at present 295 if options.show_bloat: 296 options.show_detail = True 297 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 298 options.show_detail, options.show_bloat, 299 options.list_error_boards, 300 options.show_config) 301 if options.summary: 302 builder.ShowSummary(commits, board_selected) 303 else: 304 fail, warned = builder.BuildBoards(commits, board_selected, 305 options.keep_outputs, options.verbose) 306 if fail: 307 return 128 308 elif warned: 309 return 129 310 return 0 311