1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2013 The Chromium OS Authors. 3# 4 5import multiprocessing 6import os 7import shutil 8import sys 9 10import board 11import bsettings 12from builder import Builder 13import gitutil 14import patchstream 15import terminal 16from terminal import Print 17import toolchain 18import command 19import subprocess 20 21def GetPlural(count): 22 """Returns a plural 's' if count is not 1""" 23 return 's' if count != 1 else '' 24 25def GetActionSummary(is_summary, commits, selected, options): 26 """Return a string summarising the intended action. 27 28 Returns: 29 Summary string. 30 """ 31 if commits: 32 count = len(commits) 33 count = (count + options.step - 1) / options.step 34 commit_str = '%d commit%s' % (count, GetPlural(count)) 35 else: 36 commit_str = 'current source' 37 str = '%s %s for %d boards' % ( 38 'Summary of' if is_summary else 'Building', commit_str, 39 len(selected)) 40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 42 return str 43 44def ShowActions(series, why_selected, boards_selected, builder, options): 45 """Display a list of actions that we would take, if not a dry run. 46 47 Args: 48 series: Series object 49 why_selected: Dictionary where each key is a buildman argument 50 provided by the user, and the value is the list of boards 51 brought in by that argument. For example, 'arm' might bring 52 in 400 boards, so in this case the key would be 'arm' and 53 the value would be a list of board names. 54 boards_selected: Dict of selected boards, key is target name, 55 value is Board object 56 builder: The builder that will be used to build the commits 57 options: Command line options object 58 """ 59 col = terminal.Color() 60 print 'Dry run, so not doing much. But I would do this:' 61 print 62 if series: 63 commits = series.commits 64 else: 65 commits = None 66 print GetActionSummary(False, commits, boards_selected, 67 options) 68 print 'Build directory: %s' % builder.base_dir 69 if commits: 70 for upto in range(0, len(series.commits), options.step): 71 commit = series.commits[upto] 72 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 73 print commit.subject 74 print 75 for arg in why_selected: 76 if arg != 'all': 77 print arg, ': %d boards' % len(why_selected[arg]) 78 if options.verbose: 79 print ' %s' % ' '.join(why_selected[arg]) 80 print ('Total boards to build for each commit: %d\n' % 81 len(why_selected['all'])) 82 83def CheckOutputDir(output_dir): 84 """Make sure that the output directory is not within the current directory 85 86 If we try to use an output directory which is within the current directory 87 (which is assumed to hold the U-Boot source) we may end up deleting the 88 U-Boot source code. Detect this and print an error in this case. 89 90 Args: 91 output_dir: Output directory path to check 92 """ 93 path = os.path.realpath(output_dir) 94 cwd_path = os.path.realpath('.') 95 while True: 96 if os.path.realpath(path) == cwd_path: 97 Print("Cannot use output directory '%s' since it is within the current directtory '%s'" % 98 (path, cwd_path)) 99 sys.exit(1) 100 parent = os.path.dirname(path) 101 if parent == path: 102 break 103 path = parent 104 105def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 106 clean_dir=False): 107 """The main control code for buildman 108 109 Args: 110 options: Command line options object 111 args: Command line arguments (list of strings) 112 toolchains: Toolchains to use - this should be a Toolchains() 113 object. If None, then it will be created and scanned 114 make_func: Make function to use for the builder. This is called 115 to execute 'make'. If this is None, the normal function 116 will be used, which calls the 'make' tool with suitable 117 arguments. This setting is useful for tests. 118 board: Boards() object to use, containing a list of available 119 boards. If this is None it will be created and scanned. 120 """ 121 global builder 122 123 if options.full_help: 124 pager = os.getenv('PAGER') 125 if not pager: 126 pager = 'more' 127 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 128 'README') 129 command.Run(pager, fname) 130 return 0 131 132 gitutil.Setup() 133 col = terminal.Color() 134 135 options.git_dir = os.path.join(options.git, '.git') 136 137 no_toolchains = toolchains is None 138 if no_toolchains: 139 toolchains = toolchain.Toolchains() 140 141 if options.fetch_arch: 142 if options.fetch_arch == 'list': 143 sorted_list = toolchains.ListArchs() 144 print col.Color(col.BLUE, 'Available architectures: %s\n' % 145 ' '.join(sorted_list)) 146 return 0 147 else: 148 fetch_arch = options.fetch_arch 149 if fetch_arch == 'all': 150 fetch_arch = ','.join(toolchains.ListArchs()) 151 print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 152 fetch_arch) 153 for arch in fetch_arch.split(','): 154 print 155 ret = toolchains.FetchAndInstall(arch) 156 if ret: 157 return ret 158 return 0 159 160 if no_toolchains: 161 toolchains.GetSettings() 162 toolchains.Scan(options.list_tool_chains) 163 if options.list_tool_chains: 164 toolchains.List() 165 print 166 return 0 167 168 # Work out how many commits to build. We want to build everything on the 169 # branch. We also build the upstream commit as a control so we can see 170 # problems introduced by the first commit on the branch. 171 count = options.count 172 has_range = options.branch and '..' in options.branch 173 if count == -1: 174 if not options.branch: 175 count = 1 176 else: 177 if has_range: 178 count, msg = gitutil.CountCommitsInRange(options.git_dir, 179 options.branch) 180 else: 181 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 182 options.branch) 183 if count is None: 184 sys.exit(col.Color(col.RED, msg)) 185 elif count == 0: 186 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 187 options.branch)) 188 if msg: 189 print col.Color(col.YELLOW, msg) 190 count += 1 # Build upstream commit also 191 192 if not count: 193 str = ("No commits found to process in branch '%s': " 194 "set branch's upstream or use -c flag" % options.branch) 195 sys.exit(col.Color(col.RED, str)) 196 197 # Work out what subset of the boards we are building 198 if not boards: 199 board_file = os.path.join(options.git, 'boards.cfg') 200 status = subprocess.call([os.path.join(options.git, 201 'tools/genboardscfg.py')]) 202 if status != 0: 203 sys.exit("Failed to generate boards.cfg") 204 205 boards = board.Boards() 206 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 207 208 exclude = [] 209 if options.exclude: 210 for arg in options.exclude: 211 exclude += arg.split(',') 212 213 why_selected = boards.SelectBoards(args, exclude) 214 selected = boards.GetSelected() 215 if not len(selected): 216 sys.exit(col.Color(col.RED, 'No matching boards found')) 217 218 # Read the metadata from the commits. First look at the upstream commit, 219 # then the ones in the branch. We would like to do something like 220 # upstream/master~..branch but that isn't possible if upstream/master is 221 # a merge commit (it will list all the commits that form part of the 222 # merge) 223 # Conflicting tags are not a problem for buildman, since it does not use 224 # them. For example, Series-version is not useful for buildman. On the 225 # other hand conflicting tags will cause an error. So allow later tags 226 # to overwrite earlier ones by setting allow_overwrite=True 227 if options.branch: 228 if count == -1: 229 if has_range: 230 range_expr = options.branch 231 else: 232 range_expr = gitutil.GetRangeInBranch(options.git_dir, 233 options.branch) 234 upstream_commit = gitutil.GetUpstream(options.git_dir, 235 options.branch) 236 series = patchstream.GetMetaDataForList(upstream_commit, 237 options.git_dir, 1, series=None, allow_overwrite=True) 238 239 series = patchstream.GetMetaDataForList(range_expr, 240 options.git_dir, None, series, allow_overwrite=True) 241 else: 242 # Honour the count 243 series = patchstream.GetMetaDataForList(options.branch, 244 options.git_dir, count, series=None, allow_overwrite=True) 245 else: 246 series = None 247 if not options.dry_run: 248 options.verbose = True 249 if not options.summary: 250 options.show_errors = True 251 252 # By default we have one thread per CPU. But if there are not enough jobs 253 # we can have fewer threads and use a high '-j' value for make. 254 if not options.threads: 255 options.threads = min(multiprocessing.cpu_count(), len(selected)) 256 if not options.jobs: 257 options.jobs = max(1, (multiprocessing.cpu_count() + 258 len(selected) - 1) / len(selected)) 259 260 if not options.step: 261 options.step = len(series.commits) - 1 262 263 gnu_make = command.Output(os.path.join(options.git, 264 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 265 if not gnu_make: 266 sys.exit('GNU Make not found') 267 268 # Create a new builder with the selected options. 269 output_dir = options.output_dir 270 if options.branch: 271 dirname = options.branch.replace('/', '_') 272 # As a special case allow the board directory to be placed in the 273 # output directory itself rather than any subdirectory. 274 if not options.no_subdirs: 275 output_dir = os.path.join(options.output_dir, dirname) 276 if clean_dir and os.path.exists(output_dir): 277 shutil.rmtree(output_dir) 278 CheckOutputDir(output_dir) 279 builder = Builder(toolchains, output_dir, options.git_dir, 280 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 281 show_unknown=options.show_unknown, step=options.step, 282 no_subdirs=options.no_subdirs, full_path=options.full_path, 283 verbose_build=options.verbose_build, 284 incremental=options.incremental, 285 per_board_out_dir=options.per_board_out_dir, 286 config_only=options.config_only, 287 squash_config_y=not options.preserve_config_y, 288 warnings_as_errors=options.warnings_as_errors) 289 builder.force_config_on_failure = not options.quick 290 if make_func: 291 builder.do_make = make_func 292 293 # For a dry run, just show our actions as a sanity check 294 if options.dry_run: 295 ShowActions(series, why_selected, selected, builder, options) 296 else: 297 builder.force_build = options.force_build 298 builder.force_build_failures = options.force_build_failures 299 builder.force_reconfig = options.force_reconfig 300 builder.in_tree = options.in_tree 301 302 # Work out which boards to build 303 board_selected = boards.GetSelectedDict() 304 305 if series: 306 commits = series.commits 307 # Number the commits for test purposes 308 for commit in range(len(commits)): 309 commits[commit].sequence = commit 310 else: 311 commits = None 312 313 Print(GetActionSummary(options.summary, commits, board_selected, 314 options)) 315 316 # We can't show function sizes without board details at present 317 if options.show_bloat: 318 options.show_detail = True 319 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 320 options.show_detail, options.show_bloat, 321 options.list_error_boards, 322 options.show_config, 323 options.show_environment) 324 if options.summary: 325 builder.ShowSummary(commits, board_selected) 326 else: 327 fail, warned = builder.BuildBoards(commits, board_selected, 328 options.keep_outputs, options.verbose) 329 if fail: 330 return 128 331 elif warned: 332 return 129 333 return 0 334