1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2013 The Chromium OS Authors. 3# 4 5import multiprocessing 6import os 7import shutil 8import sys 9 10import board 11import bsettings 12from builder import Builder 13import gitutil 14import patchstream 15import terminal 16from terminal import Print 17import toolchain 18import command 19import subprocess 20 21def GetPlural(count): 22 """Returns a plural 's' if count is not 1""" 23 return 's' if count != 1 else '' 24 25def GetActionSummary(is_summary, commits, selected, options): 26 """Return a string summarising the intended action. 27 28 Returns: 29 Summary string. 30 """ 31 if commits: 32 count = len(commits) 33 count = (count + options.step - 1) / options.step 34 commit_str = '%d commit%s' % (count, GetPlural(count)) 35 else: 36 commit_str = 'current source' 37 str = '%s %s for %d boards' % ( 38 'Summary of' if is_summary else 'Building', commit_str, 39 len(selected)) 40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 42 return str 43 44def ShowActions(series, why_selected, boards_selected, builder, options, 45 board_warnings): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the list of boards 52 brought in by that argument. For example, 'arm' might bring 53 in 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 board_warnings: List of warnings obtained from board selected 60 """ 61 col = terminal.Color() 62 print 'Dry run, so not doing much. But I would do this:' 63 print 64 if series: 65 commits = series.commits 66 else: 67 commits = None 68 print GetActionSummary(False, commits, boards_selected, 69 options) 70 print 'Build directory: %s' % builder.base_dir 71 if commits: 72 for upto in range(0, len(series.commits), options.step): 73 commit = series.commits[upto] 74 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 75 print commit.subject 76 print 77 for arg in why_selected: 78 if arg != 'all': 79 print arg, ': %d boards' % len(why_selected[arg]) 80 if options.verbose: 81 print ' %s' % ' '.join(why_selected[arg]) 82 print ('Total boards to build for each commit: %d\n' % 83 len(why_selected['all'])) 84 if board_warnings: 85 for warning in board_warnings: 86 print col.Color(col.YELLOW, warning) 87 88def CheckOutputDir(output_dir): 89 """Make sure that the output directory is not within the current directory 90 91 If we try to use an output directory which is within the current directory 92 (which is assumed to hold the U-Boot source) we may end up deleting the 93 U-Boot source code. Detect this and print an error in this case. 94 95 Args: 96 output_dir: Output directory path to check 97 """ 98 path = os.path.realpath(output_dir) 99 cwd_path = os.path.realpath('.') 100 while True: 101 if os.path.realpath(path) == cwd_path: 102 Print("Cannot use output directory '%s' since it is within the current directtory '%s'" % 103 (path, cwd_path)) 104 sys.exit(1) 105 parent = os.path.dirname(path) 106 if parent == path: 107 break 108 path = parent 109 110def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 111 clean_dir=False): 112 """The main control code for buildman 113 114 Args: 115 options: Command line options object 116 args: Command line arguments (list of strings) 117 toolchains: Toolchains to use - this should be a Toolchains() 118 object. If None, then it will be created and scanned 119 make_func: Make function to use for the builder. This is called 120 to execute 'make'. If this is None, the normal function 121 will be used, which calls the 'make' tool with suitable 122 arguments. This setting is useful for tests. 123 board: Boards() object to use, containing a list of available 124 boards. If this is None it will be created and scanned. 125 """ 126 global builder 127 128 if options.full_help: 129 pager = os.getenv('PAGER') 130 if not pager: 131 pager = 'more' 132 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 133 'README') 134 command.Run(pager, fname) 135 return 0 136 137 gitutil.Setup() 138 col = terminal.Color() 139 140 options.git_dir = os.path.join(options.git, '.git') 141 142 no_toolchains = toolchains is None 143 if no_toolchains: 144 toolchains = toolchain.Toolchains(options.override_toolchain) 145 146 if options.fetch_arch: 147 if options.fetch_arch == 'list': 148 sorted_list = toolchains.ListArchs() 149 print col.Color(col.BLUE, 'Available architectures: %s\n' % 150 ' '.join(sorted_list)) 151 return 0 152 else: 153 fetch_arch = options.fetch_arch 154 if fetch_arch == 'all': 155 fetch_arch = ','.join(toolchains.ListArchs()) 156 print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 157 fetch_arch) 158 for arch in fetch_arch.split(','): 159 print 160 ret = toolchains.FetchAndInstall(arch) 161 if ret: 162 return ret 163 return 0 164 165 if no_toolchains: 166 toolchains.GetSettings() 167 toolchains.Scan(options.list_tool_chains and options.verbose) 168 if options.list_tool_chains: 169 toolchains.List() 170 print 171 return 0 172 173 # Work out how many commits to build. We want to build everything on the 174 # branch. We also build the upstream commit as a control so we can see 175 # problems introduced by the first commit on the branch. 176 count = options.count 177 has_range = options.branch and '..' in options.branch 178 if count == -1: 179 if not options.branch: 180 count = 1 181 else: 182 if has_range: 183 count, msg = gitutil.CountCommitsInRange(options.git_dir, 184 options.branch) 185 else: 186 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 187 options.branch) 188 if count is None: 189 sys.exit(col.Color(col.RED, msg)) 190 elif count == 0: 191 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 192 options.branch)) 193 if msg: 194 print col.Color(col.YELLOW, msg) 195 count += 1 # Build upstream commit also 196 197 if not count: 198 str = ("No commits found to process in branch '%s': " 199 "set branch's upstream or use -c flag" % options.branch) 200 sys.exit(col.Color(col.RED, str)) 201 202 # Work out what subset of the boards we are building 203 if not boards: 204 board_file = os.path.join(options.git, 'boards.cfg') 205 status = subprocess.call([os.path.join(options.git, 206 'tools/genboardscfg.py')]) 207 if status != 0: 208 sys.exit("Failed to generate boards.cfg") 209 210 boards = board.Boards() 211 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 212 213 exclude = [] 214 if options.exclude: 215 for arg in options.exclude: 216 exclude += arg.split(',') 217 218 219 if options.boards: 220 requested_boards = [] 221 for b in options.boards: 222 requested_boards += b.split(',') 223 else: 224 requested_boards = None 225 why_selected, board_warnings = boards.SelectBoards(args, exclude, 226 requested_boards) 227 selected = boards.GetSelected() 228 if not len(selected): 229 sys.exit(col.Color(col.RED, 'No matching boards found')) 230 231 # Read the metadata from the commits. First look at the upstream commit, 232 # then the ones in the branch. We would like to do something like 233 # upstream/master~..branch but that isn't possible if upstream/master is 234 # a merge commit (it will list all the commits that form part of the 235 # merge) 236 # Conflicting tags are not a problem for buildman, since it does not use 237 # them. For example, Series-version is not useful for buildman. On the 238 # other hand conflicting tags will cause an error. So allow later tags 239 # to overwrite earlier ones by setting allow_overwrite=True 240 if options.branch: 241 if count == -1: 242 if has_range: 243 range_expr = options.branch 244 else: 245 range_expr = gitutil.GetRangeInBranch(options.git_dir, 246 options.branch) 247 upstream_commit = gitutil.GetUpstream(options.git_dir, 248 options.branch) 249 series = patchstream.GetMetaDataForList(upstream_commit, 250 options.git_dir, 1, series=None, allow_overwrite=True) 251 252 series = patchstream.GetMetaDataForList(range_expr, 253 options.git_dir, None, series, allow_overwrite=True) 254 else: 255 # Honour the count 256 series = patchstream.GetMetaDataForList(options.branch, 257 options.git_dir, count, series=None, allow_overwrite=True) 258 else: 259 series = None 260 if not options.dry_run: 261 options.verbose = True 262 if not options.summary: 263 options.show_errors = True 264 265 # By default we have one thread per CPU. But if there are not enough jobs 266 # we can have fewer threads and use a high '-j' value for make. 267 if not options.threads: 268 options.threads = min(multiprocessing.cpu_count(), len(selected)) 269 if not options.jobs: 270 options.jobs = max(1, (multiprocessing.cpu_count() + 271 len(selected) - 1) / len(selected)) 272 273 if not options.step: 274 options.step = len(series.commits) - 1 275 276 gnu_make = command.Output(os.path.join(options.git, 277 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 278 if not gnu_make: 279 sys.exit('GNU Make not found') 280 281 # Create a new builder with the selected options. 282 output_dir = options.output_dir 283 if options.branch: 284 dirname = options.branch.replace('/', '_') 285 # As a special case allow the board directory to be placed in the 286 # output directory itself rather than any subdirectory. 287 if not options.no_subdirs: 288 output_dir = os.path.join(options.output_dir, dirname) 289 if clean_dir and os.path.exists(output_dir): 290 shutil.rmtree(output_dir) 291 CheckOutputDir(output_dir) 292 builder = Builder(toolchains, output_dir, options.git_dir, 293 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 294 show_unknown=options.show_unknown, step=options.step, 295 no_subdirs=options.no_subdirs, full_path=options.full_path, 296 verbose_build=options.verbose_build, 297 incremental=options.incremental, 298 per_board_out_dir=options.per_board_out_dir, 299 config_only=options.config_only, 300 squash_config_y=not options.preserve_config_y, 301 warnings_as_errors=options.warnings_as_errors) 302 builder.force_config_on_failure = not options.quick 303 if make_func: 304 builder.do_make = make_func 305 306 # For a dry run, just show our actions as a sanity check 307 if options.dry_run: 308 ShowActions(series, why_selected, selected, builder, options, 309 board_warnings) 310 else: 311 builder.force_build = options.force_build 312 builder.force_build_failures = options.force_build_failures 313 builder.force_reconfig = options.force_reconfig 314 builder.in_tree = options.in_tree 315 316 # Work out which boards to build 317 board_selected = boards.GetSelectedDict() 318 319 if series: 320 commits = series.commits 321 # Number the commits for test purposes 322 for commit in range(len(commits)): 323 commits[commit].sequence = commit 324 else: 325 commits = None 326 327 Print(GetActionSummary(options.summary, commits, board_selected, 328 options)) 329 330 # We can't show function sizes without board details at present 331 if options.show_bloat: 332 options.show_detail = True 333 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 334 options.show_detail, options.show_bloat, 335 options.list_error_boards, 336 options.show_config, 337 options.show_environment) 338 if options.summary: 339 builder.ShowSummary(commits, board_selected) 340 else: 341 fail, warned = builder.BuildBoards(commits, board_selected, 342 options.keep_outputs, options.verbose) 343 if fail: 344 return 128 345 elif warned: 346 return 129 347 return 0 348