1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import multiprocessing 7import os 8import shutil 9import sys 10 11import board 12import bsettings 13from builder import Builder 14import gitutil 15import patchstream 16import terminal 17from terminal import Print 18import toolchain 19import command 20import subprocess 21 22def GetPlural(count): 23 """Returns a plural 's' if count is not 1""" 24 return 's' if count != 1 else '' 25 26def GetActionSummary(is_summary, commits, selected, options): 27 """Return a string summarising the intended action. 28 29 Returns: 30 Summary string. 31 """ 32 if commits: 33 count = len(commits) 34 count = (count + options.step - 1) / options.step 35 commit_str = '%d commit%s' % (count, GetPlural(count)) 36 else: 37 commit_str = 'current source' 38 str = '%s %s for %d boards' % ( 39 'Summary of' if is_summary else 'Building', commit_str, 40 len(selected)) 41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43 return str 44 45def ShowActions(series, why_selected, boards_selected, builder, options): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the list of boards 52 brought in by that argument. For example, 'arm' might bring 53 in 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 """ 60 col = terminal.Color() 61 print 'Dry run, so not doing much. But I would do this:' 62 print 63 if series: 64 commits = series.commits 65 else: 66 commits = None 67 print GetActionSummary(False, commits, boards_selected, 68 options) 69 print 'Build directory: %s' % builder.base_dir 70 if commits: 71 for upto in range(0, len(series.commits), options.step): 72 commit = series.commits[upto] 73 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74 print commit.subject 75 print 76 for arg in why_selected: 77 if arg != 'all': 78 print arg, ': %d boards' % len(why_selected[arg]) 79 if options.verbose: 80 print ' %s' % ' '.join(why_selected[arg]) 81 print ('Total boards to build for each commit: %d\n' % 82 len(why_selected['all'])) 83 84def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 85 clean_dir=False): 86 """The main control code for buildman 87 88 Args: 89 options: Command line options object 90 args: Command line arguments (list of strings) 91 toolchains: Toolchains to use - this should be a Toolchains() 92 object. If None, then it will be created and scanned 93 make_func: Make function to use for the builder. This is called 94 to execute 'make'. If this is None, the normal function 95 will be used, which calls the 'make' tool with suitable 96 arguments. This setting is useful for tests. 97 board: Boards() object to use, containing a list of available 98 boards. If this is None it will be created and scanned. 99 """ 100 global builder 101 102 if options.full_help: 103 pager = os.getenv('PAGER') 104 if not pager: 105 pager = 'more' 106 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 107 'README') 108 command.Run(pager, fname) 109 return 0 110 111 gitutil.Setup() 112 col = terminal.Color() 113 114 options.git_dir = os.path.join(options.git, '.git') 115 116 no_toolchains = toolchains is None 117 if no_toolchains: 118 toolchains = toolchain.Toolchains() 119 120 if options.fetch_arch: 121 if options.fetch_arch == 'list': 122 sorted_list = toolchains.ListArchs() 123 print col.Color(col.BLUE, 'Available architectures: %s\n' % 124 ' '.join(sorted_list)) 125 return 0 126 else: 127 fetch_arch = options.fetch_arch 128 if fetch_arch == 'all': 129 fetch_arch = ','.join(toolchains.ListArchs()) 130 print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 131 fetch_arch) 132 for arch in fetch_arch.split(','): 133 print 134 ret = toolchains.FetchAndInstall(arch) 135 if ret: 136 return ret 137 return 0 138 139 if no_toolchains: 140 toolchains.GetSettings() 141 toolchains.Scan(options.list_tool_chains) 142 if options.list_tool_chains: 143 toolchains.List() 144 print 145 return 0 146 147 # Work out how many commits to build. We want to build everything on the 148 # branch. We also build the upstream commit as a control so we can see 149 # problems introduced by the first commit on the branch. 150 count = options.count 151 has_range = options.branch and '..' in options.branch 152 if count == -1: 153 if not options.branch: 154 count = 1 155 else: 156 if has_range: 157 count, msg = gitutil.CountCommitsInRange(options.git_dir, 158 options.branch) 159 else: 160 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 161 options.branch) 162 if count is None: 163 sys.exit(col.Color(col.RED, msg)) 164 elif count == 0: 165 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 166 options.branch)) 167 if msg: 168 print col.Color(col.YELLOW, msg) 169 count += 1 # Build upstream commit also 170 171 if not count: 172 str = ("No commits found to process in branch '%s': " 173 "set branch's upstream or use -c flag" % options.branch) 174 sys.exit(col.Color(col.RED, str)) 175 176 # Work out what subset of the boards we are building 177 if not boards: 178 board_file = os.path.join(options.git, 'boards.cfg') 179 status = subprocess.call([os.path.join(options.git, 180 'tools/genboardscfg.py')]) 181 if status != 0: 182 sys.exit("Failed to generate boards.cfg") 183 184 boards = board.Boards() 185 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 186 187 exclude = [] 188 if options.exclude: 189 for arg in options.exclude: 190 exclude += arg.split(',') 191 192 why_selected = boards.SelectBoards(args, exclude) 193 selected = boards.GetSelected() 194 if not len(selected): 195 sys.exit(col.Color(col.RED, 'No matching boards found')) 196 197 # Read the metadata from the commits. First look at the upstream commit, 198 # then the ones in the branch. We would like to do something like 199 # upstream/master~..branch but that isn't possible if upstream/master is 200 # a merge commit (it will list all the commits that form part of the 201 # merge) 202 # Conflicting tags are not a problem for buildman, since it does not use 203 # them. For example, Series-version is not useful for buildman. On the 204 # other hand conflicting tags will cause an error. So allow later tags 205 # to overwrite earlier ones by setting allow_overwrite=True 206 if options.branch: 207 if count == -1: 208 if has_range: 209 range_expr = options.branch 210 else: 211 range_expr = gitutil.GetRangeInBranch(options.git_dir, 212 options.branch) 213 upstream_commit = gitutil.GetUpstream(options.git_dir, 214 options.branch) 215 series = patchstream.GetMetaDataForList(upstream_commit, 216 options.git_dir, 1, series=None, allow_overwrite=True) 217 218 series = patchstream.GetMetaDataForList(range_expr, 219 options.git_dir, None, series, allow_overwrite=True) 220 else: 221 # Honour the count 222 series = patchstream.GetMetaDataForList(options.branch, 223 options.git_dir, count, series=None, allow_overwrite=True) 224 else: 225 series = None 226 if not options.dry_run: 227 options.verbose = True 228 if not options.summary: 229 options.show_errors = True 230 231 # By default we have one thread per CPU. But if there are not enough jobs 232 # we can have fewer threads and use a high '-j' value for make. 233 if not options.threads: 234 options.threads = min(multiprocessing.cpu_count(), len(selected)) 235 if not options.jobs: 236 options.jobs = max(1, (multiprocessing.cpu_count() + 237 len(selected) - 1) / len(selected)) 238 239 if not options.step: 240 options.step = len(series.commits) - 1 241 242 gnu_make = command.Output(os.path.join(options.git, 243 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 244 if not gnu_make: 245 sys.exit('GNU Make not found') 246 247 # Create a new builder with the selected options. 248 output_dir = options.output_dir 249 if options.branch: 250 dirname = options.branch.replace('/', '_') 251 # As a special case allow the board directory to be placed in the 252 # output directory itself rather than any subdirectory. 253 if not options.no_subdirs: 254 output_dir = os.path.join(options.output_dir, dirname) 255 if (clean_dir and output_dir != options.output_dir and 256 os.path.exists(output_dir)): 257 shutil.rmtree(output_dir) 258 builder = Builder(toolchains, output_dir, options.git_dir, 259 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 260 show_unknown=options.show_unknown, step=options.step, 261 no_subdirs=options.no_subdirs, full_path=options.full_path, 262 verbose_build=options.verbose_build, 263 incremental=options.incremental, 264 per_board_out_dir=options.per_board_out_dir, 265 config_only=options.config_only, 266 squash_config_y=not options.preserve_config_y) 267 builder.force_config_on_failure = not options.quick 268 if make_func: 269 builder.do_make = make_func 270 271 # For a dry run, just show our actions as a sanity check 272 if options.dry_run: 273 ShowActions(series, why_selected, selected, builder, options) 274 else: 275 builder.force_build = options.force_build 276 builder.force_build_failures = options.force_build_failures 277 builder.force_reconfig = options.force_reconfig 278 builder.in_tree = options.in_tree 279 280 # Work out which boards to build 281 board_selected = boards.GetSelectedDict() 282 283 if series: 284 commits = series.commits 285 # Number the commits for test purposes 286 for commit in range(len(commits)): 287 commits[commit].sequence = commit 288 else: 289 commits = None 290 291 Print(GetActionSummary(options.summary, commits, board_selected, 292 options)) 293 294 # We can't show function sizes without board details at present 295 if options.show_bloat: 296 options.show_detail = True 297 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 298 options.show_detail, options.show_bloat, 299 options.list_error_boards, 300 options.show_config) 301 if options.summary: 302 builder.ShowSummary(commits, board_selected) 303 else: 304 fail, warned = builder.BuildBoards(commits, board_selected, 305 options.keep_outputs, options.verbose) 306 if fail: 307 return 128 308 elif warned: 309 return 129 310 return 0 311