1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# SPDX-License-Identifier: GPL-2.0+ 4# 5 6import multiprocessing 7import os 8import shutil 9import sys 10 11import board 12import bsettings 13from builder import Builder 14import gitutil 15import patchstream 16import terminal 17from terminal import Print 18import toolchain 19import command 20import subprocess 21 22def GetPlural(count): 23 """Returns a plural 's' if count is not 1""" 24 return 's' if count != 1 else '' 25 26def GetActionSummary(is_summary, commits, selected, options): 27 """Return a string summarising the intended action. 28 29 Returns: 30 Summary string. 31 """ 32 if commits: 33 count = len(commits) 34 count = (count + options.step - 1) / options.step 35 commit_str = '%d commit%s' % (count, GetPlural(count)) 36 else: 37 commit_str = 'current source' 38 str = '%s %s for %d boards' % ( 39 'Summary of' if is_summary else 'Building', commit_str, 40 len(selected)) 41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads, 42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs)) 43 return str 44 45def ShowActions(series, why_selected, boards_selected, builder, options): 46 """Display a list of actions that we would take, if not a dry run. 47 48 Args: 49 series: Series object 50 why_selected: Dictionary where each key is a buildman argument 51 provided by the user, and the value is the boards brought 52 in by that argument. For example, 'arm' might bring in 53 400 boards, so in this case the key would be 'arm' and 54 the value would be a list of board names. 55 boards_selected: Dict of selected boards, key is target name, 56 value is Board object 57 builder: The builder that will be used to build the commits 58 options: Command line options object 59 """ 60 col = terminal.Color() 61 print 'Dry run, so not doing much. But I would do this:' 62 print 63 if series: 64 commits = series.commits 65 else: 66 commits = None 67 print GetActionSummary(False, commits, boards_selected, 68 options) 69 print 'Build directory: %s' % builder.base_dir 70 if commits: 71 for upto in range(0, len(series.commits), options.step): 72 commit = series.commits[upto] 73 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), 74 print commit.subject 75 print 76 for arg in why_selected: 77 if arg != 'all': 78 print arg, ': %d boards' % why_selected[arg] 79 print ('Total boards to build for each commit: %d\n' % 80 why_selected['all']) 81 82def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, 83 clean_dir=False): 84 """The main control code for buildman 85 86 Args: 87 options: Command line options object 88 args: Command line arguments (list of strings) 89 toolchains: Toolchains to use - this should be a Toolchains() 90 object. If None, then it will be created and scanned 91 make_func: Make function to use for the builder. This is called 92 to execute 'make'. If this is None, the normal function 93 will be used, which calls the 'make' tool with suitable 94 arguments. This setting is useful for tests. 95 board: Boards() object to use, containing a list of available 96 boards. If this is None it will be created and scanned. 97 """ 98 global builder 99 100 if options.full_help: 101 pager = os.getenv('PAGER') 102 if not pager: 103 pager = 'more' 104 fname = os.path.join(os.path.dirname(sys.argv[0]), 'README') 105 command.Run(pager, fname) 106 return 0 107 108 gitutil.Setup() 109 110 options.git_dir = os.path.join(options.git, '.git') 111 112 if not toolchains: 113 toolchains = toolchain.Toolchains() 114 toolchains.GetSettings() 115 toolchains.Scan(options.list_tool_chains) 116 if options.list_tool_chains: 117 toolchains.List() 118 print 119 return 0 120 121 if options.fetch_arch: 122 if options.fetch_arch == 'list': 123 sorted_list = toolchains.ListArchs() 124 print 'Available architectures: %s\n' % ' '.join(sorted_list) 125 return 0 126 else: 127 fetch_arch = options.fetch_arch 128 if fetch_arch == 'all': 129 fetch_arch = ','.join(toolchains.ListArchs()) 130 print 'Downloading toolchains: %s\n' % fetch_arch 131 for arch in fetch_arch.split(','): 132 ret = toolchains.FetchAndInstall(arch) 133 if ret: 134 return ret 135 return 0 136 137 # Work out how many commits to build. We want to build everything on the 138 # branch. We also build the upstream commit as a control so we can see 139 # problems introduced by the first commit on the branch. 140 col = terminal.Color() 141 count = options.count 142 has_range = options.branch and '..' in options.branch 143 if count == -1: 144 if not options.branch: 145 count = 1 146 else: 147 if has_range: 148 count, msg = gitutil.CountCommitsInRange(options.git_dir, 149 options.branch) 150 else: 151 count, msg = gitutil.CountCommitsInBranch(options.git_dir, 152 options.branch) 153 if count is None: 154 sys.exit(col.Color(col.RED, msg)) 155 elif count == 0: 156 sys.exit(col.Color(col.RED, "Range '%s' has no commits" % 157 options.branch)) 158 if msg: 159 print col.Color(col.YELLOW, msg) 160 count += 1 # Build upstream commit also 161 162 if not count: 163 str = ("No commits found to process in branch '%s': " 164 "set branch's upstream or use -c flag" % options.branch) 165 sys.exit(col.Color(col.RED, str)) 166 167 # Work out what subset of the boards we are building 168 if not boards: 169 board_file = os.path.join(options.git, 'boards.cfg') 170 status = subprocess.call([os.path.join(options.git, 171 'tools/genboardscfg.py')]) 172 if status != 0: 173 sys.exit("Failed to generate boards.cfg") 174 175 boards = board.Boards() 176 boards.ReadBoards(os.path.join(options.git, 'boards.cfg')) 177 178 exclude = [] 179 if options.exclude: 180 for arg in options.exclude: 181 exclude += arg.split(',') 182 183 why_selected = boards.SelectBoards(args, exclude) 184 selected = boards.GetSelected() 185 if not len(selected): 186 sys.exit(col.Color(col.RED, 'No matching boards found')) 187 188 # Read the metadata from the commits. First look at the upstream commit, 189 # then the ones in the branch. We would like to do something like 190 # upstream/master~..branch but that isn't possible if upstream/master is 191 # a merge commit (it will list all the commits that form part of the 192 # merge) 193 # Conflicting tags are not a problem for buildman, since it does not use 194 # them. For example, Series-version is not useful for buildman. On the 195 # other hand conflicting tags will cause an error. So allow later tags 196 # to overwrite earlier ones by setting allow_overwrite=True 197 if options.branch: 198 if count == -1: 199 if has_range: 200 range_expr = options.branch 201 else: 202 range_expr = gitutil.GetRangeInBranch(options.git_dir, 203 options.branch) 204 upstream_commit = gitutil.GetUpstream(options.git_dir, 205 options.branch) 206 series = patchstream.GetMetaDataForList(upstream_commit, 207 options.git_dir, 1, series=None, allow_overwrite=True) 208 209 series = patchstream.GetMetaDataForList(range_expr, 210 options.git_dir, None, series, allow_overwrite=True) 211 else: 212 # Honour the count 213 series = patchstream.GetMetaDataForList(options.branch, 214 options.git_dir, count, series=None, allow_overwrite=True) 215 else: 216 series = None 217 options.verbose = True 218 if not options.summary: 219 options.show_errors = True 220 221 # By default we have one thread per CPU. But if there are not enough jobs 222 # we can have fewer threads and use a high '-j' value for make. 223 if not options.threads: 224 options.threads = min(multiprocessing.cpu_count(), len(selected)) 225 if not options.jobs: 226 options.jobs = max(1, (multiprocessing.cpu_count() + 227 len(selected) - 1) / len(selected)) 228 229 if not options.step: 230 options.step = len(series.commits) - 1 231 232 gnu_make = command.Output(os.path.join(options.git, 233 'scripts/show-gnu-make')).rstrip() 234 if not gnu_make: 235 sys.exit('GNU Make not found') 236 237 # Create a new builder with the selected options. 238 output_dir = options.output_dir 239 if options.branch: 240 dirname = options.branch.replace('/', '_') 241 # As a special case allow the board directory to be placed in the 242 # output directory itself rather than any subdirectory. 243 if not options.no_subdirs: 244 output_dir = os.path.join(options.output_dir, dirname) 245 if (clean_dir and output_dir != options.output_dir and 246 os.path.exists(output_dir)): 247 shutil.rmtree(output_dir) 248 builder = Builder(toolchains, output_dir, options.git_dir, 249 options.threads, options.jobs, gnu_make=gnu_make, checkout=True, 250 show_unknown=options.show_unknown, step=options.step, 251 no_subdirs=options.no_subdirs, full_path=options.full_path, 252 verbose_build=options.verbose_build) 253 builder.force_config_on_failure = not options.quick 254 if make_func: 255 builder.do_make = make_func 256 257 # For a dry run, just show our actions as a sanity check 258 if options.dry_run: 259 ShowActions(series, why_selected, selected, builder, options) 260 else: 261 builder.force_build = options.force_build 262 builder.force_build_failures = options.force_build_failures 263 builder.force_reconfig = options.force_reconfig 264 builder.in_tree = options.in_tree 265 266 # Work out which boards to build 267 board_selected = boards.GetSelectedDict() 268 269 if series: 270 commits = series.commits 271 # Number the commits for test purposes 272 for commit in range(len(commits)): 273 commits[commit].sequence = commit 274 else: 275 commits = None 276 277 Print(GetActionSummary(options.summary, commits, board_selected, 278 options)) 279 280 # We can't show function sizes without board details at present 281 if options.show_bloat: 282 options.show_detail = True 283 builder.SetDisplayOptions(options.show_errors, options.show_sizes, 284 options.show_detail, options.show_bloat, 285 options.list_error_boards, 286 options.show_config) 287 if options.summary: 288 builder.ShowSummary(commits, board_selected) 289 else: 290 fail, warned = builder.BuildBoards(commits, board_selected, 291 options.keep_outputs, options.verbose) 292 if fail: 293 return 128 294 elif warned: 295 return 129 296 return 0 297