1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import signal 16import string 17import sys 18import threading 19import time 20 21import builderthread 22import command 23import gitutil 24import terminal 25from terminal import Print 26import toolchain 27 28 29""" 30Theory of Operation 31 32Please see README for user documentation, and you should be familiar with 33that before trying to make sense of this. 34 35Buildman works by keeping the machine as busy as possible, building different 36commits for different boards on multiple CPUs at once. 37 38The source repo (self.git_dir) contains all the commits to be built. Each 39thread works on a single board at a time. It checks out the first commit, 40configures it for that board, then builds it. Then it checks out the next 41commit and builds it (typically without re-configuring). When it runs out 42of commits, it gets another job from the builder and starts again with that 43board. 44 45Clearly the builder threads could work either way - they could check out a 46commit and then built it for all boards. Using separate directories for each 47commit/board pair they could leave their build product around afterwards 48also. 49 50The intent behind building a single board for multiple commits, is to make 51use of incremental builds. Since each commit is built incrementally from 52the previous one, builds are faster. Reconfiguring for a different board 53removes all intermediate object files. 54 55Many threads can be working at once, but each has its own working directory. 56When a thread finishes a build, it puts the output files into a result 57directory. 58 59The base directory used by buildman is normally '../<branch>', i.e. 60a directory higher than the source repository and named after the branch 61being built. 62 63Within the base directory, we have one subdirectory for each commit. Within 64that is one subdirectory for each board. Within that is the build output for 65that commit/board combination. 66 67Buildman also create working directories for each thread, in a .bm-work/ 68subdirectory in the base dir. 69 70As an example, say we are building branch 'us-net' for boards 'sandbox' and 71'seaboard', and say that us-net has two commits. We will have directories 72like this: 73 74us-net/ base directory 75 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 76 sandbox/ 77 u-boot.bin 78 seaboard/ 79 u-boot.bin 80 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 81 sandbox/ 82 u-boot.bin 83 seaboard/ 84 u-boot.bin 85 .bm-work/ 86 00/ working directory for thread 0 (contains source checkout) 87 build/ build output 88 01/ working directory for thread 1 89 build/ build output 90 ... 91u-boot/ source directory 92 .git/ repository 93""" 94 95# Possible build outcomes 96OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 97 98# Translate a commit subject into a valid filename (and handle unicode) 99trans_valid_chars = string.maketrans('/: ', '---') 100trans_valid_chars = trans_valid_chars.decode('latin-1') 101 102BASE_CONFIG_FILENAMES = [ 103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 104] 105 106EXTRA_CONFIG_FILENAMES = [ 107 '.config', '.config-spl', '.config-tpl', 108 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 109 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 110] 111 112class Config: 113 """Holds information about configuration settings for a board.""" 114 def __init__(self, config_filename, target): 115 self.target = target 116 self.config = {} 117 for fname in config_filename: 118 self.config[fname] = {} 119 120 def Add(self, fname, key, value): 121 self.config[fname][key] = value 122 123 def __hash__(self): 124 val = 0 125 for fname in self.config: 126 for key, value in self.config[fname].iteritems(): 127 print key, value 128 val = val ^ hash(key) & hash(value) 129 return val 130 131class Builder: 132 """Class for building U-Boot for a particular commit. 133 134 Public members: (many should ->private) 135 already_done: Number of builds already completed 136 base_dir: Base directory to use for builder 137 checkout: True to check out source, False to skip that step. 138 This is used for testing. 139 col: terminal.Color() object 140 count: Number of commits to build 141 do_make: Method to call to invoke Make 142 fail: Number of builds that failed due to error 143 force_build: Force building even if a build already exists 144 force_config_on_failure: If a commit fails for a board, disable 145 incremental building for the next commit we build for that 146 board, so that we will see all warnings/errors again. 147 force_build_failures: If a previously-built build (i.e. built on 148 a previous run of buildman) is marked as failed, rebuild it. 149 git_dir: Git directory containing source repository 150 last_line_len: Length of the last line we printed (used for erasing 151 it with new progress information) 152 num_jobs: Number of jobs to run at once (passed to make as -j) 153 num_threads: Number of builder threads to run 154 out_queue: Queue of results to process 155 re_make_err: Compiled regular expression for ignore_lines 156 queue: Queue of jobs to run 157 threads: List of active threads 158 toolchains: Toolchains object to use for building 159 upto: Current commit number we are building (0.count-1) 160 warned: Number of builds that produced at least one warning 161 force_reconfig: Reconfigure U-Boot on each comiit. This disables 162 incremental building, where buildman reconfigures on the first 163 commit for a baord, and then just does an incremental build for 164 the following commits. In fact buildman will reconfigure and 165 retry for any failing commits, so generally the only effect of 166 this option is to slow things down. 167 in_tree: Build U-Boot in-tree instead of specifying an output 168 directory separate from the source code. This option is really 169 only useful for testing in-tree builds. 170 171 Private members: 172 _base_board_dict: Last-summarised Dict of boards 173 _base_err_lines: Last-summarised list of errors 174 _base_warn_lines: Last-summarised list of warnings 175 _build_period_us: Time taken for a single build (float object). 176 _complete_delay: Expected delay until completion (timedelta) 177 _next_delay_update: Next time we plan to display a progress update 178 (datatime) 179 _show_unknown: Show unknown boards (those not built) in summary 180 _timestamps: List of timestamps for the completion of the last 181 last _timestamp_count builds. Each is a datetime object. 182 _timestamp_count: Number of timestamps to keep in our list. 183 _working_dir: Base working directory containing all threads 184 """ 185 class Outcome: 186 """Records a build outcome for a single make invocation 187 188 Public Members: 189 rc: Outcome value (OUTCOME_...) 190 err_lines: List of error lines or [] if none 191 sizes: Dictionary of image size information, keyed by filename 192 - Each value is itself a dictionary containing 193 values for 'text', 'data' and 'bss', being the integer 194 size in bytes of each section. 195 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 196 value is itself a dictionary: 197 key: function name 198 value: Size of function in bytes 199 config: Dictionary keyed by filename - e.g. '.config'. Each 200 value is itself a dictionary: 201 key: config name 202 value: config value 203 """ 204 def __init__(self, rc, err_lines, sizes, func_sizes, config): 205 self.rc = rc 206 self.err_lines = err_lines 207 self.sizes = sizes 208 self.func_sizes = func_sizes 209 self.config = config 210 211 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 212 gnu_make='make', checkout=True, show_unknown=True, step=1, 213 no_subdirs=False, full_path=False, verbose_build=False, 214 incremental=False, per_board_out_dir=False, 215 config_only=False, squash_config_y=False): 216 """Create a new Builder object 217 218 Args: 219 toolchains: Toolchains object to use for building 220 base_dir: Base directory to use for builder 221 git_dir: Git directory containing source repository 222 num_threads: Number of builder threads to run 223 num_jobs: Number of jobs to run at once (passed to make as -j) 224 gnu_make: the command name of GNU Make. 225 checkout: True to check out source, False to skip that step. 226 This is used for testing. 227 show_unknown: Show unknown boards (those not built) in summary 228 step: 1 to process every commit, n to process every nth commit 229 no_subdirs: Don't create subdirectories when building current 230 source for a single board 231 full_path: Return the full path in CROSS_COMPILE and don't set 232 PATH 233 verbose_build: Run build with V=1 and don't use 'make -s' 234 incremental: Always perform incremental builds; don't run make 235 mrproper when configuring 236 per_board_out_dir: Build in a separate persistent directory per 237 board rather than a thread-specific directory 238 config_only: Only configure each build, don't build it 239 squash_config_y: Convert CONFIG options with the value 'y' to '1' 240 """ 241 self.toolchains = toolchains 242 self.base_dir = base_dir 243 self._working_dir = os.path.join(base_dir, '.bm-work') 244 self.threads = [] 245 self.do_make = self.Make 246 self.gnu_make = gnu_make 247 self.checkout = checkout 248 self.num_threads = num_threads 249 self.num_jobs = num_jobs 250 self.already_done = 0 251 self.force_build = False 252 self.git_dir = git_dir 253 self._show_unknown = show_unknown 254 self._timestamp_count = 10 255 self._build_period_us = None 256 self._complete_delay = None 257 self._next_delay_update = datetime.now() 258 self.force_config_on_failure = True 259 self.force_build_failures = False 260 self.force_reconfig = False 261 self._step = step 262 self.in_tree = False 263 self._error_lines = 0 264 self.no_subdirs = no_subdirs 265 self.full_path = full_path 266 self.verbose_build = verbose_build 267 self.config_only = config_only 268 self.squash_config_y = squash_config_y 269 self.config_filenames = BASE_CONFIG_FILENAMES 270 if not self.squash_config_y: 271 self.config_filenames += EXTRA_CONFIG_FILENAMES 272 273 self.col = terminal.Color() 274 275 self._re_function = re.compile('(.*): In function.*') 276 self._re_files = re.compile('In file included from.*') 277 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 278 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 279 280 self.queue = Queue.Queue() 281 self.out_queue = Queue.Queue() 282 for i in range(self.num_threads): 283 t = builderthread.BuilderThread(self, i, incremental, 284 per_board_out_dir) 285 t.setDaemon(True) 286 t.start() 287 self.threads.append(t) 288 289 self.last_line_len = 0 290 t = builderthread.ResultThread(self) 291 t.setDaemon(True) 292 t.start() 293 self.threads.append(t) 294 295 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 296 self.re_make_err = re.compile('|'.join(ignore_lines)) 297 298 # Handle existing graceful with SIGINT / Ctrl-C 299 signal.signal(signal.SIGINT, self.signal_handler) 300 301 def __del__(self): 302 """Get rid of all threads created by the builder""" 303 for t in self.threads: 304 del t 305 306 def signal_handler(self, signal, frame): 307 sys.exit(1) 308 309 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 310 show_detail=False, show_bloat=False, 311 list_error_boards=False, show_config=False): 312 """Setup display options for the builder. 313 314 show_errors: True to show summarised error/warning info 315 show_sizes: Show size deltas 316 show_detail: Show detail for each board 317 show_bloat: Show detail for each function 318 list_error_boards: Show the boards which caused each error/warning 319 show_config: Show config deltas 320 """ 321 self._show_errors = show_errors 322 self._show_sizes = show_sizes 323 self._show_detail = show_detail 324 self._show_bloat = show_bloat 325 self._list_error_boards = list_error_boards 326 self._show_config = show_config 327 328 def _AddTimestamp(self): 329 """Add a new timestamp to the list and record the build period. 330 331 The build period is the length of time taken to perform a single 332 build (one board, one commit). 333 """ 334 now = datetime.now() 335 self._timestamps.append(now) 336 count = len(self._timestamps) 337 delta = self._timestamps[-1] - self._timestamps[0] 338 seconds = delta.total_seconds() 339 340 # If we have enough data, estimate build period (time taken for a 341 # single build) and therefore completion time. 342 if count > 1 and self._next_delay_update < now: 343 self._next_delay_update = now + timedelta(seconds=2) 344 if seconds > 0: 345 self._build_period = float(seconds) / count 346 todo = self.count - self.upto 347 self._complete_delay = timedelta(microseconds= 348 self._build_period * todo * 1000000) 349 # Round it 350 self._complete_delay -= timedelta( 351 microseconds=self._complete_delay.microseconds) 352 353 if seconds > 60: 354 self._timestamps.popleft() 355 count -= 1 356 357 def ClearLine(self, length): 358 """Clear any characters on the current line 359 360 Make way for a new line of length 'length', by outputting enough 361 spaces to clear out the old line. Then remember the new length for 362 next time. 363 364 Args: 365 length: Length of new line, in characters 366 """ 367 if length < self.last_line_len: 368 Print(' ' * (self.last_line_len - length), newline=False) 369 Print('\r', newline=False) 370 self.last_line_len = length 371 sys.stdout.flush() 372 373 def SelectCommit(self, commit, checkout=True): 374 """Checkout the selected commit for this build 375 """ 376 self.commit = commit 377 if checkout and self.checkout: 378 gitutil.Checkout(commit.hash) 379 380 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 381 """Run make 382 383 Args: 384 commit: Commit object that is being built 385 brd: Board object that is being built 386 stage: Stage that we are at (mrproper, config, build) 387 cwd: Directory where make should be run 388 args: Arguments to pass to make 389 kwargs: Arguments to pass to command.RunPipe() 390 """ 391 cmd = [self.gnu_make] + list(args) 392 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 393 cwd=cwd, raise_on_error=False, **kwargs) 394 if self.verbose_build: 395 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 396 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 397 return result 398 399 def ProcessResult(self, result): 400 """Process the result of a build, showing progress information 401 402 Args: 403 result: A CommandResult object, which indicates the result for 404 a single build 405 """ 406 col = terminal.Color() 407 if result: 408 target = result.brd.target 409 410 self.upto += 1 411 if result.return_code != 0: 412 self.fail += 1 413 elif result.stderr: 414 self.warned += 1 415 if result.already_done: 416 self.already_done += 1 417 if self._verbose: 418 Print('\r', newline=False) 419 self.ClearLine(0) 420 boards_selected = {target : result.brd} 421 self.ResetResultSummary(boards_selected) 422 self.ProduceResultSummary(result.commit_upto, self.commits, 423 boards_selected) 424 else: 425 target = '(starting)' 426 427 # Display separate counts for ok, warned and fail 428 ok = self.upto - self.warned - self.fail 429 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 430 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 431 line += self.col.Color(self.col.RED, '%5d' % self.fail) 432 433 name = ' /%-5d ' % self.count 434 435 # Add our current completion time estimate 436 self._AddTimestamp() 437 if self._complete_delay: 438 name += '%s : ' % self._complete_delay 439 # When building all boards for a commit, we can print a commit 440 # progress message. 441 if result and result.commit_upto is None: 442 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 443 self.commit_count) 444 445 name += target 446 Print(line + name, newline=False) 447 length = 16 + len(name) 448 self.ClearLine(length) 449 450 def _GetOutputDir(self, commit_upto): 451 """Get the name of the output directory for a commit number 452 453 The output directory is typically .../<branch>/<commit>. 454 455 Args: 456 commit_upto: Commit number to use (0..self.count-1) 457 """ 458 commit_dir = None 459 if self.commits: 460 commit = self.commits[commit_upto] 461 subject = commit.subject.translate(trans_valid_chars) 462 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 463 self.commit_count, commit.hash, subject[:20])) 464 elif not self.no_subdirs: 465 commit_dir = 'current' 466 if not commit_dir: 467 return self.base_dir 468 return os.path.join(self.base_dir, commit_dir) 469 470 def GetBuildDir(self, commit_upto, target): 471 """Get the name of the build directory for a commit number 472 473 The build directory is typically .../<branch>/<commit>/<target>. 474 475 Args: 476 commit_upto: Commit number to use (0..self.count-1) 477 target: Target name 478 """ 479 output_dir = self._GetOutputDir(commit_upto) 480 return os.path.join(output_dir, target) 481 482 def GetDoneFile(self, commit_upto, target): 483 """Get the name of the done file for a commit number 484 485 Args: 486 commit_upto: Commit number to use (0..self.count-1) 487 target: Target name 488 """ 489 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 490 491 def GetSizesFile(self, commit_upto, target): 492 """Get the name of the sizes file for a commit number 493 494 Args: 495 commit_upto: Commit number to use (0..self.count-1) 496 target: Target name 497 """ 498 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 499 500 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 501 """Get the name of the funcsizes file for a commit number and ELF file 502 503 Args: 504 commit_upto: Commit number to use (0..self.count-1) 505 target: Target name 506 elf_fname: Filename of elf image 507 """ 508 return os.path.join(self.GetBuildDir(commit_upto, target), 509 '%s.sizes' % elf_fname.replace('/', '-')) 510 511 def GetObjdumpFile(self, commit_upto, target, elf_fname): 512 """Get the name of the objdump file for a commit number and ELF file 513 514 Args: 515 commit_upto: Commit number to use (0..self.count-1) 516 target: Target name 517 elf_fname: Filename of elf image 518 """ 519 return os.path.join(self.GetBuildDir(commit_upto, target), 520 '%s.objdump' % elf_fname.replace('/', '-')) 521 522 def GetErrFile(self, commit_upto, target): 523 """Get the name of the err file for a commit number 524 525 Args: 526 commit_upto: Commit number to use (0..self.count-1) 527 target: Target name 528 """ 529 output_dir = self.GetBuildDir(commit_upto, target) 530 return os.path.join(output_dir, 'err') 531 532 def FilterErrors(self, lines): 533 """Filter out errors in which we have no interest 534 535 We should probably use map(). 536 537 Args: 538 lines: List of error lines, each a string 539 Returns: 540 New list with only interesting lines included 541 """ 542 out_lines = [] 543 for line in lines: 544 if not self.re_make_err.search(line): 545 out_lines.append(line) 546 return out_lines 547 548 def ReadFuncSizes(self, fname, fd): 549 """Read function sizes from the output of 'nm' 550 551 Args: 552 fd: File containing data to read 553 fname: Filename we are reading from (just for errors) 554 555 Returns: 556 Dictionary containing size of each function in bytes, indexed by 557 function name. 558 """ 559 sym = {} 560 for line in fd.readlines(): 561 try: 562 size, type, name = line[:-1].split() 563 except: 564 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 565 continue 566 if type in 'tTdDbB': 567 # function names begin with '.' on 64-bit powerpc 568 if '.' in name[1:]: 569 name = 'static.' + name.split('.')[0] 570 sym[name] = sym.get(name, 0) + int(size, 16) 571 return sym 572 573 def _ProcessConfig(self, fname): 574 """Read in a .config, autoconf.mk or autoconf.h file 575 576 This function handles all config file types. It ignores comments and 577 any #defines which don't start with CONFIG_. 578 579 Args: 580 fname: Filename to read 581 582 Returns: 583 Dictionary: 584 key: Config name (e.g. CONFIG_DM) 585 value: Config value (e.g. 1) 586 """ 587 config = {} 588 if os.path.exists(fname): 589 with open(fname) as fd: 590 for line in fd: 591 line = line.strip() 592 if line.startswith('#define'): 593 values = line[8:].split(' ', 1) 594 if len(values) > 1: 595 key, value = values 596 else: 597 key = values[0] 598 value = '1' if self.squash_config_y else '' 599 if not key.startswith('CONFIG_'): 600 continue 601 elif not line or line[0] in ['#', '*', '/']: 602 continue 603 else: 604 key, value = line.split('=', 1) 605 if self.squash_config_y and value == 'y': 606 value = '1' 607 config[key] = value 608 return config 609 610 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 611 read_config): 612 """Work out the outcome of a build. 613 614 Args: 615 commit_upto: Commit number to check (0..n-1) 616 target: Target board to check 617 read_func_sizes: True to read function size information 618 read_config: True to read .config and autoconf.h files 619 620 Returns: 621 Outcome object 622 """ 623 done_file = self.GetDoneFile(commit_upto, target) 624 sizes_file = self.GetSizesFile(commit_upto, target) 625 sizes = {} 626 func_sizes = {} 627 config = {} 628 if os.path.exists(done_file): 629 with open(done_file, 'r') as fd: 630 return_code = int(fd.readline()) 631 err_lines = [] 632 err_file = self.GetErrFile(commit_upto, target) 633 if os.path.exists(err_file): 634 with open(err_file, 'r') as fd: 635 err_lines = self.FilterErrors(fd.readlines()) 636 637 # Decide whether the build was ok, failed or created warnings 638 if return_code: 639 rc = OUTCOME_ERROR 640 elif len(err_lines): 641 rc = OUTCOME_WARNING 642 else: 643 rc = OUTCOME_OK 644 645 # Convert size information to our simple format 646 if os.path.exists(sizes_file): 647 with open(sizes_file, 'r') as fd: 648 for line in fd.readlines(): 649 values = line.split() 650 rodata = 0 651 if len(values) > 6: 652 rodata = int(values[6], 16) 653 size_dict = { 654 'all' : int(values[0]) + int(values[1]) + 655 int(values[2]), 656 'text' : int(values[0]) - rodata, 657 'data' : int(values[1]), 658 'bss' : int(values[2]), 659 'rodata' : rodata, 660 } 661 sizes[values[5]] = size_dict 662 663 if read_func_sizes: 664 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 665 for fname in glob.glob(pattern): 666 with open(fname, 'r') as fd: 667 dict_name = os.path.basename(fname).replace('.sizes', 668 '') 669 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 670 671 if read_config: 672 output_dir = self.GetBuildDir(commit_upto, target) 673 for name in self.config_filenames: 674 fname = os.path.join(output_dir, name) 675 config[name] = self._ProcessConfig(fname) 676 677 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 678 679 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 680 681 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 682 read_config): 683 """Calculate a summary of the results of building a commit. 684 685 Args: 686 board_selected: Dict containing boards to summarise 687 commit_upto: Commit number to summarize (0..self.count-1) 688 read_func_sizes: True to read function size information 689 read_config: True to read .config and autoconf.h files 690 691 Returns: 692 Tuple: 693 Dict containing boards which passed building this commit. 694 keyed by board.target 695 List containing a summary of error lines 696 Dict keyed by error line, containing a list of the Board 697 objects with that error 698 List containing a summary of warning lines 699 Dict keyed by error line, containing a list of the Board 700 objects with that warning 701 Dictionary keyed by board.target. Each value is a dictionary: 702 key: filename - e.g. '.config' 703 value is itself a dictionary: 704 key: config name 705 value: config value 706 """ 707 def AddLine(lines_summary, lines_boards, line, board): 708 line = line.rstrip() 709 if line in lines_boards: 710 lines_boards[line].append(board) 711 else: 712 lines_boards[line] = [board] 713 lines_summary.append(line) 714 715 board_dict = {} 716 err_lines_summary = [] 717 err_lines_boards = {} 718 warn_lines_summary = [] 719 warn_lines_boards = {} 720 config = {} 721 722 for board in boards_selected.itervalues(): 723 outcome = self.GetBuildOutcome(commit_upto, board.target, 724 read_func_sizes, read_config) 725 board_dict[board.target] = outcome 726 last_func = None 727 last_was_warning = False 728 for line in outcome.err_lines: 729 if line: 730 if (self._re_function.match(line) or 731 self._re_files.match(line)): 732 last_func = line 733 else: 734 is_warning = self._re_warning.match(line) 735 is_note = self._re_note.match(line) 736 if is_warning or (last_was_warning and is_note): 737 if last_func: 738 AddLine(warn_lines_summary, warn_lines_boards, 739 last_func, board) 740 AddLine(warn_lines_summary, warn_lines_boards, 741 line, board) 742 else: 743 if last_func: 744 AddLine(err_lines_summary, err_lines_boards, 745 last_func, board) 746 AddLine(err_lines_summary, err_lines_boards, 747 line, board) 748 last_was_warning = is_warning 749 last_func = None 750 tconfig = Config(self.config_filenames, board.target) 751 for fname in self.config_filenames: 752 if outcome.config: 753 for key, value in outcome.config[fname].iteritems(): 754 tconfig.Add(fname, key, value) 755 config[board.target] = tconfig 756 757 return (board_dict, err_lines_summary, err_lines_boards, 758 warn_lines_summary, warn_lines_boards, config) 759 760 def AddOutcome(self, board_dict, arch_list, changes, char, color): 761 """Add an output to our list of outcomes for each architecture 762 763 This simple function adds failing boards (changes) to the 764 relevant architecture string, so we can print the results out 765 sorted by architecture. 766 767 Args: 768 board_dict: Dict containing all boards 769 arch_list: Dict keyed by arch name. Value is a string containing 770 a list of board names which failed for that arch. 771 changes: List of boards to add to arch_list 772 color: terminal.Colour object 773 """ 774 done_arch = {} 775 for target in changes: 776 if target in board_dict: 777 arch = board_dict[target].arch 778 else: 779 arch = 'unknown' 780 str = self.col.Color(color, ' ' + target) 781 if not arch in done_arch: 782 str = ' %s %s' % (self.col.Color(color, char), str) 783 done_arch[arch] = True 784 if not arch in arch_list: 785 arch_list[arch] = str 786 else: 787 arch_list[arch] += str 788 789 790 def ColourNum(self, num): 791 color = self.col.RED if num > 0 else self.col.GREEN 792 if num == 0: 793 return '0' 794 return self.col.Color(color, str(num)) 795 796 def ResetResultSummary(self, board_selected): 797 """Reset the results summary ready for use. 798 799 Set up the base board list to be all those selected, and set the 800 error lines to empty. 801 802 Following this, calls to PrintResultSummary() will use this 803 information to work out what has changed. 804 805 Args: 806 board_selected: Dict containing boards to summarise, keyed by 807 board.target 808 """ 809 self._base_board_dict = {} 810 for board in board_selected: 811 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 812 self._base_err_lines = [] 813 self._base_warn_lines = [] 814 self._base_err_line_boards = {} 815 self._base_warn_line_boards = {} 816 self._base_config = None 817 818 def PrintFuncSizeDetail(self, fname, old, new): 819 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 820 delta, common = [], {} 821 822 for a in old: 823 if a in new: 824 common[a] = 1 825 826 for name in old: 827 if name not in common: 828 remove += 1 829 down += old[name] 830 delta.append([-old[name], name]) 831 832 for name in new: 833 if name not in common: 834 add += 1 835 up += new[name] 836 delta.append([new[name], name]) 837 838 for name in common: 839 diff = new.get(name, 0) - old.get(name, 0) 840 if diff > 0: 841 grow, up = grow + 1, up + diff 842 elif diff < 0: 843 shrink, down = shrink + 1, down - diff 844 delta.append([diff, name]) 845 846 delta.sort() 847 delta.reverse() 848 849 args = [add, -remove, grow, -shrink, up, -down, up - down] 850 if max(args) == 0 and min(args) == 0: 851 return 852 args = [self.ColourNum(x) for x in args] 853 indent = ' ' * 15 854 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 855 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 856 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 857 'delta')) 858 for diff, name in delta: 859 if diff: 860 color = self.col.RED if diff > 0 else self.col.GREEN 861 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 862 old.get(name, '-'), new.get(name,'-'), diff) 863 Print(msg, colour=color) 864 865 866 def PrintSizeDetail(self, target_list, show_bloat): 867 """Show details size information for each board 868 869 Args: 870 target_list: List of targets, each a dict containing: 871 'target': Target name 872 'total_diff': Total difference in bytes across all areas 873 <part_name>: Difference for that part 874 show_bloat: Show detail for each function 875 """ 876 targets_by_diff = sorted(target_list, reverse=True, 877 key=lambda x: x['_total_diff']) 878 for result in targets_by_diff: 879 printed_target = False 880 for name in sorted(result): 881 diff = result[name] 882 if name.startswith('_'): 883 continue 884 if diff != 0: 885 color = self.col.RED if diff > 0 else self.col.GREEN 886 msg = ' %s %+d' % (name, diff) 887 if not printed_target: 888 Print('%10s %-15s:' % ('', result['_target']), 889 newline=False) 890 printed_target = True 891 Print(msg, colour=color, newline=False) 892 if printed_target: 893 Print() 894 if show_bloat: 895 target = result['_target'] 896 outcome = result['_outcome'] 897 base_outcome = self._base_board_dict[target] 898 for fname in outcome.func_sizes: 899 self.PrintFuncSizeDetail(fname, 900 base_outcome.func_sizes[fname], 901 outcome.func_sizes[fname]) 902 903 904 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 905 show_bloat): 906 """Print a summary of image sizes broken down by section. 907 908 The summary takes the form of one line per architecture. The 909 line contains deltas for each of the sections (+ means the section 910 got bigger, - means smaller). The nunmbers are the average number 911 of bytes that a board in this section increased by. 912 913 For example: 914 powerpc: (622 boards) text -0.0 915 arm: (285 boards) text -0.0 916 nds32: (3 boards) text -8.0 917 918 Args: 919 board_selected: Dict containing boards to summarise, keyed by 920 board.target 921 board_dict: Dict containing boards for which we built this 922 commit, keyed by board.target. The value is an Outcome object. 923 show_detail: Show detail for each board 924 show_bloat: Show detail for each function 925 """ 926 arch_list = {} 927 arch_count = {} 928 929 # Calculate changes in size for different image parts 930 # The previous sizes are in Board.sizes, for each board 931 for target in board_dict: 932 if target not in board_selected: 933 continue 934 base_sizes = self._base_board_dict[target].sizes 935 outcome = board_dict[target] 936 sizes = outcome.sizes 937 938 # Loop through the list of images, creating a dict of size 939 # changes for each image/part. We end up with something like 940 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 941 # which means that U-Boot data increased by 5 bytes and SPL 942 # text decreased by 4. 943 err = {'_target' : target} 944 for image in sizes: 945 if image in base_sizes: 946 base_image = base_sizes[image] 947 # Loop through the text, data, bss parts 948 for part in sorted(sizes[image]): 949 diff = sizes[image][part] - base_image[part] 950 col = None 951 if diff: 952 if image == 'u-boot': 953 name = part 954 else: 955 name = image + ':' + part 956 err[name] = diff 957 arch = board_selected[target].arch 958 if not arch in arch_count: 959 arch_count[arch] = 1 960 else: 961 arch_count[arch] += 1 962 if not sizes: 963 pass # Only add to our list when we have some stats 964 elif not arch in arch_list: 965 arch_list[arch] = [err] 966 else: 967 arch_list[arch].append(err) 968 969 # We now have a list of image size changes sorted by arch 970 # Print out a summary of these 971 for arch, target_list in arch_list.iteritems(): 972 # Get total difference for each type 973 totals = {} 974 for result in target_list: 975 total = 0 976 for name, diff in result.iteritems(): 977 if name.startswith('_'): 978 continue 979 total += diff 980 if name in totals: 981 totals[name] += diff 982 else: 983 totals[name] = diff 984 result['_total_diff'] = total 985 result['_outcome'] = board_dict[result['_target']] 986 987 count = len(target_list) 988 printed_arch = False 989 for name in sorted(totals): 990 diff = totals[name] 991 if diff: 992 # Display the average difference in this name for this 993 # architecture 994 avg_diff = float(diff) / count 995 color = self.col.RED if avg_diff > 0 else self.col.GREEN 996 msg = ' %s %+1.1f' % (name, avg_diff) 997 if not printed_arch: 998 Print('%10s: (for %d/%d boards)' % (arch, count, 999 arch_count[arch]), newline=False) 1000 printed_arch = True 1001 Print(msg, colour=color, newline=False) 1002 1003 if printed_arch: 1004 Print() 1005 if show_detail: 1006 self.PrintSizeDetail(target_list, show_bloat) 1007 1008 1009 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1010 err_line_boards, warn_lines, warn_line_boards, 1011 config, show_sizes, show_detail, show_bloat, 1012 show_config): 1013 """Compare results with the base results and display delta. 1014 1015 Only boards mentioned in board_selected will be considered. This 1016 function is intended to be called repeatedly with the results of 1017 each commit. It therefore shows a 'diff' between what it saw in 1018 the last call and what it sees now. 1019 1020 Args: 1021 board_selected: Dict containing boards to summarise, keyed by 1022 board.target 1023 board_dict: Dict containing boards for which we built this 1024 commit, keyed by board.target. The value is an Outcome object. 1025 err_lines: A list of errors for this commit, or [] if there is 1026 none, or we don't want to print errors 1027 err_line_boards: Dict keyed by error line, containing a list of 1028 the Board objects with that error 1029 warn_lines: A list of warnings for this commit, or [] if there is 1030 none, or we don't want to print errors 1031 warn_line_boards: Dict keyed by warning line, containing a list of 1032 the Board objects with that warning 1033 config: Dictionary keyed by filename - e.g. '.config'. Each 1034 value is itself a dictionary: 1035 key: config name 1036 value: config value 1037 show_sizes: Show image size deltas 1038 show_detail: Show detail for each board 1039 show_bloat: Show detail for each function 1040 show_config: Show config changes 1041 """ 1042 def _BoardList(line, line_boards): 1043 """Helper function to get a line of boards containing a line 1044 1045 Args: 1046 line: Error line to search for 1047 Return: 1048 String containing a list of boards with that error line, or 1049 '' if the user has not requested such a list 1050 """ 1051 if self._list_error_boards: 1052 names = [] 1053 for board in line_boards[line]: 1054 if not board.target in names: 1055 names.append(board.target) 1056 names_str = '(%s) ' % ','.join(names) 1057 else: 1058 names_str = '' 1059 return names_str 1060 1061 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1062 char): 1063 better_lines = [] 1064 worse_lines = [] 1065 for line in lines: 1066 if line not in base_lines: 1067 worse_lines.append(char + '+' + 1068 _BoardList(line, line_boards) + line) 1069 for line in base_lines: 1070 if line not in lines: 1071 better_lines.append(char + '-' + 1072 _BoardList(line, base_line_boards) + line) 1073 return better_lines, worse_lines 1074 1075 def _CalcConfig(delta, name, config): 1076 """Calculate configuration changes 1077 1078 Args: 1079 delta: Type of the delta, e.g. '+' 1080 name: name of the file which changed (e.g. .config) 1081 config: configuration change dictionary 1082 key: config name 1083 value: config value 1084 Returns: 1085 String containing the configuration changes which can be 1086 printed 1087 """ 1088 out = '' 1089 for key in sorted(config.keys()): 1090 out += '%s=%s ' % (key, config[key]) 1091 return '%s %s: %s' % (delta, name, out) 1092 1093 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1094 """Add changes in configuration to a list 1095 1096 Args: 1097 lines: list to add to 1098 name: config file name 1099 config_plus: configurations added, dictionary 1100 key: config name 1101 value: config value 1102 config_minus: configurations removed, dictionary 1103 key: config name 1104 value: config value 1105 config_change: configurations changed, dictionary 1106 key: config name 1107 value: config value 1108 """ 1109 if config_plus: 1110 lines.append(_CalcConfig('+', name, config_plus)) 1111 if config_minus: 1112 lines.append(_CalcConfig('-', name, config_minus)) 1113 if config_change: 1114 lines.append(_CalcConfig('c', name, config_change)) 1115 1116 def _OutputConfigInfo(lines): 1117 for line in lines: 1118 if not line: 1119 continue 1120 if line[0] == '+': 1121 col = self.col.GREEN 1122 elif line[0] == '-': 1123 col = self.col.RED 1124 elif line[0] == 'c': 1125 col = self.col.YELLOW 1126 Print(' ' + line, newline=True, colour=col) 1127 1128 1129 better = [] # List of boards fixed since last commit 1130 worse = [] # List of new broken boards since last commit 1131 new = [] # List of boards that didn't exist last time 1132 unknown = [] # List of boards that were not built 1133 1134 for target in board_dict: 1135 if target not in board_selected: 1136 continue 1137 1138 # If the board was built last time, add its outcome to a list 1139 if target in self._base_board_dict: 1140 base_outcome = self._base_board_dict[target].rc 1141 outcome = board_dict[target] 1142 if outcome.rc == OUTCOME_UNKNOWN: 1143 unknown.append(target) 1144 elif outcome.rc < base_outcome: 1145 better.append(target) 1146 elif outcome.rc > base_outcome: 1147 worse.append(target) 1148 else: 1149 new.append(target) 1150 1151 # Get a list of errors that have appeared, and disappeared 1152 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1153 self._base_err_line_boards, err_lines, err_line_boards, '') 1154 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1155 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1156 1157 # Display results by arch 1158 if (better or worse or unknown or new or worse_err or better_err 1159 or worse_warn or better_warn): 1160 arch_list = {} 1161 self.AddOutcome(board_selected, arch_list, better, '', 1162 self.col.GREEN) 1163 self.AddOutcome(board_selected, arch_list, worse, '+', 1164 self.col.RED) 1165 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1166 if self._show_unknown: 1167 self.AddOutcome(board_selected, arch_list, unknown, '?', 1168 self.col.MAGENTA) 1169 for arch, target_list in arch_list.iteritems(): 1170 Print('%10s: %s' % (arch, target_list)) 1171 self._error_lines += 1 1172 if better_err: 1173 Print('\n'.join(better_err), colour=self.col.GREEN) 1174 self._error_lines += 1 1175 if worse_err: 1176 Print('\n'.join(worse_err), colour=self.col.RED) 1177 self._error_lines += 1 1178 if better_warn: 1179 Print('\n'.join(better_warn), colour=self.col.CYAN) 1180 self._error_lines += 1 1181 if worse_warn: 1182 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1183 self._error_lines += 1 1184 1185 if show_sizes: 1186 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1187 show_bloat) 1188 1189 if show_config and self._base_config: 1190 summary = {} 1191 arch_config_plus = {} 1192 arch_config_minus = {} 1193 arch_config_change = {} 1194 arch_list = [] 1195 1196 for target in board_dict: 1197 if target not in board_selected: 1198 continue 1199 arch = board_selected[target].arch 1200 if arch not in arch_list: 1201 arch_list.append(arch) 1202 1203 for arch in arch_list: 1204 arch_config_plus[arch] = {} 1205 arch_config_minus[arch] = {} 1206 arch_config_change[arch] = {} 1207 for name in self.config_filenames: 1208 arch_config_plus[arch][name] = {} 1209 arch_config_minus[arch][name] = {} 1210 arch_config_change[arch][name] = {} 1211 1212 for target in board_dict: 1213 if target not in board_selected: 1214 continue 1215 1216 arch = board_selected[target].arch 1217 1218 all_config_plus = {} 1219 all_config_minus = {} 1220 all_config_change = {} 1221 tbase = self._base_config[target] 1222 tconfig = config[target] 1223 lines = [] 1224 for name in self.config_filenames: 1225 if not tconfig.config[name]: 1226 continue 1227 config_plus = {} 1228 config_minus = {} 1229 config_change = {} 1230 base = tbase.config[name] 1231 for key, value in tconfig.config[name].iteritems(): 1232 if key not in base: 1233 config_plus[key] = value 1234 all_config_plus[key] = value 1235 for key, value in base.iteritems(): 1236 if key not in tconfig.config[name]: 1237 config_minus[key] = value 1238 all_config_minus[key] = value 1239 for key, value in base.iteritems(): 1240 new_value = tconfig.config.get(key) 1241 if new_value and value != new_value: 1242 desc = '%s -> %s' % (value, new_value) 1243 config_change[key] = desc 1244 all_config_change[key] = desc 1245 1246 arch_config_plus[arch][name].update(config_plus) 1247 arch_config_minus[arch][name].update(config_minus) 1248 arch_config_change[arch][name].update(config_change) 1249 1250 _AddConfig(lines, name, config_plus, config_minus, 1251 config_change) 1252 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1253 all_config_change) 1254 summary[target] = '\n'.join(lines) 1255 1256 lines_by_target = {} 1257 for target, lines in summary.iteritems(): 1258 if lines in lines_by_target: 1259 lines_by_target[lines].append(target) 1260 else: 1261 lines_by_target[lines] = [target] 1262 1263 for arch in arch_list: 1264 lines = [] 1265 all_plus = {} 1266 all_minus = {} 1267 all_change = {} 1268 for name in self.config_filenames: 1269 all_plus.update(arch_config_plus[arch][name]) 1270 all_minus.update(arch_config_minus[arch][name]) 1271 all_change.update(arch_config_change[arch][name]) 1272 _AddConfig(lines, name, arch_config_plus[arch][name], 1273 arch_config_minus[arch][name], 1274 arch_config_change[arch][name]) 1275 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1276 #arch_summary[target] = '\n'.join(lines) 1277 if lines: 1278 Print('%s:' % arch) 1279 _OutputConfigInfo(lines) 1280 1281 for lines, targets in lines_by_target.iteritems(): 1282 if not lines: 1283 continue 1284 Print('%s :' % ' '.join(sorted(targets))) 1285 _OutputConfigInfo(lines.split('\n')) 1286 1287 1288 # Save our updated information for the next call to this function 1289 self._base_board_dict = board_dict 1290 self._base_err_lines = err_lines 1291 self._base_warn_lines = warn_lines 1292 self._base_err_line_boards = err_line_boards 1293 self._base_warn_line_boards = warn_line_boards 1294 self._base_config = config 1295 1296 # Get a list of boards that did not get built, if needed 1297 not_built = [] 1298 for board in board_selected: 1299 if not board in board_dict: 1300 not_built.append(board) 1301 if not_built: 1302 Print("Boards not built (%d): %s" % (len(not_built), 1303 ', '.join(not_built))) 1304 1305 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1306 (board_dict, err_lines, err_line_boards, warn_lines, 1307 warn_line_boards, config) = self.GetResultSummary( 1308 board_selected, commit_upto, 1309 read_func_sizes=self._show_bloat, 1310 read_config=self._show_config) 1311 if commits: 1312 msg = '%02d: %s' % (commit_upto + 1, 1313 commits[commit_upto].subject) 1314 Print(msg, colour=self.col.BLUE) 1315 self.PrintResultSummary(board_selected, board_dict, 1316 err_lines if self._show_errors else [], err_line_boards, 1317 warn_lines if self._show_errors else [], warn_line_boards, 1318 config, self._show_sizes, self._show_detail, 1319 self._show_bloat, self._show_config) 1320 1321 def ShowSummary(self, commits, board_selected): 1322 """Show a build summary for U-Boot for a given board list. 1323 1324 Reset the result summary, then repeatedly call GetResultSummary on 1325 each commit's results, then display the differences we see. 1326 1327 Args: 1328 commit: Commit objects to summarise 1329 board_selected: Dict containing boards to summarise 1330 """ 1331 self.commit_count = len(commits) if commits else 1 1332 self.commits = commits 1333 self.ResetResultSummary(board_selected) 1334 self._error_lines = 0 1335 1336 for commit_upto in range(0, self.commit_count, self._step): 1337 self.ProduceResultSummary(commit_upto, commits, board_selected) 1338 if not self._error_lines: 1339 Print('(no errors to report)', colour=self.col.GREEN) 1340 1341 1342 def SetupBuild(self, board_selected, commits): 1343 """Set up ready to start a build. 1344 1345 Args: 1346 board_selected: Selected boards to build 1347 commits: Selected commits to build 1348 """ 1349 # First work out how many commits we will build 1350 count = (self.commit_count + self._step - 1) / self._step 1351 self.count = len(board_selected) * count 1352 self.upto = self.warned = self.fail = 0 1353 self._timestamps = collections.deque() 1354 1355 def GetThreadDir(self, thread_num): 1356 """Get the directory path to the working dir for a thread. 1357 1358 Args: 1359 thread_num: Number of thread to check. 1360 """ 1361 return os.path.join(self._working_dir, '%02d' % thread_num) 1362 1363 def _PrepareThread(self, thread_num, setup_git): 1364 """Prepare the working directory for a thread. 1365 1366 This clones or fetches the repo into the thread's work directory. 1367 1368 Args: 1369 thread_num: Thread number (0, 1, ...) 1370 setup_git: True to set up a git repo clone 1371 """ 1372 thread_dir = self.GetThreadDir(thread_num) 1373 builderthread.Mkdir(thread_dir) 1374 git_dir = os.path.join(thread_dir, '.git') 1375 1376 # Clone the repo if it doesn't already exist 1377 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1378 # we have a private index but uses the origin repo's contents? 1379 if setup_git and self.git_dir: 1380 src_dir = os.path.abspath(self.git_dir) 1381 if os.path.exists(git_dir): 1382 gitutil.Fetch(git_dir, thread_dir) 1383 else: 1384 Print('\rCloning repo for thread %d' % thread_num, 1385 newline=False) 1386 gitutil.Clone(src_dir, thread_dir) 1387 Print('\r%s\r' % (' ' * 30), newline=False) 1388 1389 def _PrepareWorkingSpace(self, max_threads, setup_git): 1390 """Prepare the working directory for use. 1391 1392 Set up the git repo for each thread. 1393 1394 Args: 1395 max_threads: Maximum number of threads we expect to need. 1396 setup_git: True to set up a git repo clone 1397 """ 1398 builderthread.Mkdir(self._working_dir) 1399 for thread in range(max_threads): 1400 self._PrepareThread(thread, setup_git) 1401 1402 def _PrepareOutputSpace(self): 1403 """Get the output directories ready to receive files. 1404 1405 We delete any output directories which look like ones we need to 1406 create. Having left over directories is confusing when the user wants 1407 to check the output manually. 1408 """ 1409 if not self.commits: 1410 return 1411 dir_list = [] 1412 for commit_upto in range(self.commit_count): 1413 dir_list.append(self._GetOutputDir(commit_upto)) 1414 1415 to_remove = [] 1416 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1417 if dirname not in dir_list: 1418 to_remove.append(dirname) 1419 if to_remove: 1420 Print('Removing %d old build directories' % len(to_remove), 1421 newline=False) 1422 for dirname in to_remove: 1423 shutil.rmtree(dirname) 1424 1425 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1426 """Build all commits for a list of boards 1427 1428 Args: 1429 commits: List of commits to be build, each a Commit object 1430 boards_selected: Dict of selected boards, key is target name, 1431 value is Board object 1432 keep_outputs: True to save build output files 1433 verbose: Display build results as they are completed 1434 Returns: 1435 Tuple containing: 1436 - number of boards that failed to build 1437 - number of boards that issued warnings 1438 """ 1439 self.commit_count = len(commits) if commits else 1 1440 self.commits = commits 1441 self._verbose = verbose 1442 1443 self.ResetResultSummary(board_selected) 1444 builderthread.Mkdir(self.base_dir, parents = True) 1445 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1446 commits is not None) 1447 self._PrepareOutputSpace() 1448 Print('\rStarting build...', newline=False) 1449 self.SetupBuild(board_selected, commits) 1450 self.ProcessResult(None) 1451 1452 # Create jobs to build all commits for each board 1453 for brd in board_selected.itervalues(): 1454 job = builderthread.BuilderJob() 1455 job.board = brd 1456 job.commits = commits 1457 job.keep_outputs = keep_outputs 1458 job.step = self._step 1459 self.queue.put(job) 1460 1461 term = threading.Thread(target=self.queue.join) 1462 term.setDaemon(True) 1463 term.start() 1464 while term.isAlive(): 1465 term.join(100) 1466 1467 # Wait until we have processed all output 1468 self.out_queue.join() 1469 Print() 1470 self.ClearLine(0) 1471 return (self.fail, self.warned) 1472