1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import multiprocessing 7import os 8import shutil 9import sys 10 11import board 12import bsettings 13from builder import Builder 14import gitutil 15import patchstream 16import terminal 17from terminal import Print 18import toolchain 19import command 20import subprocess 21 22def GetPlural(count): 23 """Returns a plural 's' if count is not 1""" 24 return 's' if count != 1 else '' 25 26def GetActionSummary(is_summary, commits, selected, options): 27 """Return a string summarising the intended action. 28 29 Returns: 30 Summary string. 31 """ 32 if commits: 33 count = len(commits) 34 count = (count + options.step - 1) / options.step 35 commit_str = '%d commit%s' % (count, GetPlural(count)) 36 else: 37 commit_str = 'current source' 38 str = '%s %s for %d boards' % ( 39 'Summary of' if is_summary else 'Building', commit_str, 40 len(selected)) 41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43 return str 44 45def ShowActions(series, why_selected, boards_selected, builder, options): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the boards brought 52 in by that argument. For example, 'arm' might bring in 53 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 """ 60 col = terminal.Color() 61 print 'Dry run, so not doing much. But I would do this:' 62 print 63 if series: 64 commits = series.commits 65 else: 66 commits = None 67 print GetActionSummary(False, commits, boards_selected, 68 options) 69 print 'Build directory: %s' % builder.base_dir 70 if commits: 71 for upto in range(0, len(series.commits), options.step): 72 commit = series.commits[upto] 73 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74 print commit.subject 75 print 76 for arg in why_selected: 77 if arg != 'all': 78 print arg, ': %d boards' % why_selected[arg] 79 print ('Total boards to build for each commit: %d\n' % 80 why_selected['all']) 81 82def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83 clean_dir=False): 84 """The main control code for buildman 85 86 Args: 87 options: Command line options object 88 args: Command line arguments (list of strings) 89 toolchains: Toolchains to use - this should be a Toolchains() 90 object. If None, then it will be created and scanned 91 make_func: Make function to use for the builder. This is called 92 to execute 'make'. If this is None, the normal function 93 will be used, which calls the 'make' tool with suitable 94 arguments. This setting is useful for tests. 95 board: Boards() object to use, containing a list of available 96 boards. If this is None it will be created and scanned. 97 """ 98 global builder 99 100 if options.full_help: 101 pager = os.getenv('PAGER') 102 if not pager: 103 pager = 'more' 104 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 105 'README') 106 command.Run(pager, fname) 107 return 0 108 109 gitutil.Setup() 110 col = terminal.Color() 111 112 options.git_dir = os.path.join(options.git, '.git') 113 114 no_toolchains = toolchains is None 115 if no_toolchains: 116 toolchains = toolchain.Toolchains() 117 118 if options.fetch_arch: 119 if options.fetch_arch == 'list': 120 sorted_list = toolchains.ListArchs() 121 print col.Color(col.BLUE, 'Available architectures: %s\n' % 122 ' '.join(sorted_list)) 123 return 0 124 else: 125 fetch_arch = options.fetch_arch 126 if fetch_arch == 'all': 127 fetch_arch = ','.join(toolchains.ListArchs()) 128 print col.Color(col.CYAN, '\nDownloading toolchains: %s' % 129 fetch_arch) 130 for arch in fetch_arch.split(','): 131 print 132 ret = toolchains.FetchAndInstall(arch) 133 if ret: 134 return ret 135 return 0 136 137 if no_toolchains: 138 toolchains.GetSettings() 139 toolchains.Scan(options.list_tool_chains) 140 if options.list_tool_chains: 141 toolchains.List() 142 print 143 return 0 144 145 # Work out how many commits to build. We want to build everything on the 146 # branch. We also build the upstream commit as a control so we can see 147 # problems introduced by the first commit on the branch. 148 count = options.count 149 has_range = options.branch and '..' in options.branch 150 if count == -1: 151 if not options.branch: 152 count = 1 153 else: 154 if has_range: 155 count, msg = gitutil.CountCommitsInRange(options.git_dir, 156 options.branch) 157 else: 158 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 159 options.branch) 160 if count is None: 161 sys.exit(col.Color(col.RED, msg)) 162 elif count == 0: 163 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 164 options.branch)) 165 if msg: 166 print col.Color(col.YELLOW, msg) 167 count += 1 # Build upstream commit also 168 169 if not count: 170 str = ("No commits found to process in branch '%s': " 171 "set branch's upstream or use -c flag" % options.branch) 172 sys.exit(col.Color(col.RED, str)) 173 174 # Work out what subset of the boards we are building 175 if not boards: 176 board_file = os.path.join(options.git, 'boards.cfg') 177 status = subprocess.call([os.path.join(options.git, 178 'tools/genboardscfg.py')]) 179 if status != 0: 180 sys.exit("Failed to generate boards.cfg") 181 182 boards = board.Boards() 183 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 184 185 exclude = [] 186 if options.exclude: 187 for arg in options.exclude: 188 exclude += arg.split(',') 189 190 why_selected = boards.SelectBoards(args, exclude) 191 selected = boards.GetSelected() 192 if not len(selected): 193 sys.exit(col.Color(col.RED, 'No matching boards found')) 194 195 # Read the metadata from the commits. First look at the upstream commit, 196 # then the ones in the branch. We would like to do something like 197 # upstream/master~..branch but that isn't possible if upstream/master is 198 # a merge commit (it will list all the commits that form part of the 199 # merge) 200 # Conflicting tags are not a problem for buildman, since it does not use 201 # them. For example, Series-version is not useful for buildman. On the 202 # other hand conflicting tags will cause an error. So allow later tags 203 # to overwrite earlier ones by setting allow_overwrite=True 204 if options.branch: 205 if count == -1: 206 if has_range: 207 range_expr = options.branch 208 else: 209 range_expr = gitutil.GetRangeInBranch(options.git_dir, 210 options.branch) 211 upstream_commit = gitutil.GetUpstream(options.git_dir, 212 options.branch) 213 series = patchstream.GetMetaDataForList(upstream_commit, 214 options.git_dir, 1, series=None, allow_overwrite=True) 215 216 series = patchstream.GetMetaDataForList(range_expr, 217 options.git_dir, None, series, allow_overwrite=True) 218 else: 219 # Honour the count 220 series = patchstream.GetMetaDataForList(options.branch, 221 options.git_dir, count, series=None, allow_overwrite=True) 222 else: 223 series = None 224 options.verbose = True 225 if not options.summary: 226 options.show_errors = True 227 228 # By default we have one thread per CPU. But if there are not enough jobs 229 # we can have fewer threads and use a high '-j' value for make. 230 if not options.threads: 231 options.threads = min(multiprocessing.cpu_count(), len(selected)) 232 if not options.jobs: 233 options.jobs = max(1, (multiprocessing.cpu_count() + 234 len(selected) - 1) / len(selected)) 235 236 if not options.step: 237 options.step = len(series.commits) - 1 238 239 gnu_make = command.Output(os.path.join(options.git, 240 'scripts/show-gnu-make'), raise_on_error=False).rstrip() 241 if not gnu_make: 242 sys.exit('GNU Make not found') 243 244 # Create a new builder with the selected options. 245 output_dir = options.output_dir 246 if options.branch: 247 dirname = options.branch.replace('/', '_') 248 # As a special case allow the board directory to be placed in the 249 # output directory itself rather than any subdirectory. 250 if not options.no_subdirs: 251 output_dir = os.path.join(options.output_dir, dirname) 252 if (clean_dir and output_dir != options.output_dir and 253 os.path.exists(output_dir)): 254 shutil.rmtree(output_dir) 255 builder = Builder(toolchains, output_dir, options.git_dir, 256 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 257 show_unknown=options.show_unknown, step=options.step, 258 no_subdirs=options.no_subdirs, full_path=options.full_path, 259 verbose_build=options.verbose_build, 260 incremental=options.incremental, 261 per_board_out_dir=options.per_board_out_dir,) 262 builder.force_config_on_failure = not options.quick 263 if make_func: 264 builder.do_make = make_func 265 266 # For a dry run, just show our actions as a sanity check 267 if options.dry_run: 268 ShowActions(series, why_selected, selected, builder, options) 269 else: 270 builder.force_build = options.force_build 271 builder.force_build_failures = options.force_build_failures 272 builder.force_reconfig = options.force_reconfig 273 builder.in_tree = options.in_tree 274 275 # Work out which boards to build 276 board_selected = boards.GetSelectedDict() 277 278 if series: 279 commits = series.commits 280 # Number the commits for test purposes 281 for commit in range(len(commits)): 282 commits[commit].sequence = commit 283 else: 284 commits = None 285 286 Print(GetActionSummary(options.summary, commits, board_selected, 287 options)) 288 289 # We can't show function sizes without board details at present 290 if options.show_bloat: 291 options.show_detail = True 292 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 293 options.show_detail, options.show_bloat, 294 options.list_error_boards, 295 options.show_config) 296 if options.summary: 297 builder.ShowSummary(commits, board_selected) 298 else: 299 fail, warned = builder.BuildBoards(commits, board_selected, 300 options.keep_outputs, options.verbose) 301 if fail: 302 return 128 303 elif warned: 304 return 129 305 return 0 306