1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import signal 16import string 17import sys 18import threading 19import time 20 21import builderthread 22import command 23import gitutil 24import terminal 25from terminal import Print 26import toolchain 27 28 29""" 30Theory of Operation 31 32Please see README for user documentation, and you should be familiar with 33that before trying to make sense of this. 34 35Buildman works by keeping the machine as busy as possible, building different 36commits for different boards on multiple CPUs at once. 37 38The source repo (self.git_dir) contains all the commits to be built. Each 39thread works on a single board at a time. It checks out the first commit, 40configures it for that board, then builds it. Then it checks out the next 41commit and builds it (typically without re-configuring). When it runs out 42of commits, it gets another job from the builder and starts again with that 43board. 44 45Clearly the builder threads could work either way - they could check out a 46commit and then built it for all boards. Using separate directories for each 47commit/board pair they could leave their build product around afterwards 48also. 49 50The intent behind building a single board for multiple commits, is to make 51use of incremental builds. Since each commit is built incrementally from 52the previous one, builds are faster. Reconfiguring for a different board 53removes all intermediate object files. 54 55Many threads can be working at once, but each has its own working directory. 56When a thread finishes a build, it puts the output files into a result 57directory. 58 59The base directory used by buildman is normally '../<branch>', i.e. 60a directory higher than the source repository and named after the branch 61being built. 62 63Within the base directory, we have one subdirectory for each commit. Within 64that is one subdirectory for each board. Within that is the build output for 65that commit/board combination. 66 67Buildman also create working directories for each thread, in a .bm-work/ 68subdirectory in the base dir. 69 70As an example, say we are building branch 'us-net' for boards 'sandbox' and 71'seaboard', and say that us-net has two commits. We will have directories 72like this: 73 74us-net/ base directory 75 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 76 sandbox/ 77 u-boot.bin 78 seaboard/ 79 u-boot.bin 80 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 81 sandbox/ 82 u-boot.bin 83 seaboard/ 84 u-boot.bin 85 .bm-work/ 86 00/ working directory for thread 0 (contains source checkout) 87 build/ build output 88 01/ working directory for thread 1 89 build/ build output 90 ... 91u-boot/ source directory 92 .git/ repository 93""" 94 95# Possible build outcomes 96OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 97 98# Translate a commit subject into a valid filename 99trans_valid_chars = string.maketrans("/: ", "---") 100 101BASE_CONFIG_FILENAMES = [ 102 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 103] 104 105EXTRA_CONFIG_FILENAMES = [ 106 '.config', '.config-spl', '.config-tpl', 107 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 108 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 109] 110 111class Config: 112 """Holds information about configuration settings for a board.""" 113 def __init__(self, config_filename, target): 114 self.target = target 115 self.config = {} 116 for fname in config_filename: 117 self.config[fname] = {} 118 119 def Add(self, fname, key, value): 120 self.config[fname][key] = value 121 122 def __hash__(self): 123 val = 0 124 for fname in self.config: 125 for key, value in self.config[fname].iteritems(): 126 print key, value 127 val = val ^ hash(key) & hash(value) 128 return val 129 130class Builder: 131 """Class for building U-Boot for a particular commit. 132 133 Public members: (many should ->private) 134 already_done: Number of builds already completed 135 base_dir: Base directory to use for builder 136 checkout: True to check out source, False to skip that step. 137 This is used for testing. 138 col: terminal.Color() object 139 count: Number of commits to build 140 do_make: Method to call to invoke Make 141 fail: Number of builds that failed due to error 142 force_build: Force building even if a build already exists 143 force_config_on_failure: If a commit fails for a board, disable 144 incremental building for the next commit we build for that 145 board, so that we will see all warnings/errors again. 146 force_build_failures: If a previously-built build (i.e. built on 147 a previous run of buildman) is marked as failed, rebuild it. 148 git_dir: Git directory containing source repository 149 last_line_len: Length of the last line we printed (used for erasing 150 it with new progress information) 151 num_jobs: Number of jobs to run at once (passed to make as -j) 152 num_threads: Number of builder threads to run 153 out_queue: Queue of results to process 154 re_make_err: Compiled regular expression for ignore_lines 155 queue: Queue of jobs to run 156 threads: List of active threads 157 toolchains: Toolchains object to use for building 158 upto: Current commit number we are building (0.count-1) 159 warned: Number of builds that produced at least one warning 160 force_reconfig: Reconfigure U-Boot on each comiit. This disables 161 incremental building, where buildman reconfigures on the first 162 commit for a baord, and then just does an incremental build for 163 the following commits. In fact buildman will reconfigure and 164 retry for any failing commits, so generally the only effect of 165 this option is to slow things down. 166 in_tree: Build U-Boot in-tree instead of specifying an output 167 directory separate from the source code. This option is really 168 only useful for testing in-tree builds. 169 170 Private members: 171 _base_board_dict: Last-summarised Dict of boards 172 _base_err_lines: Last-summarised list of errors 173 _base_warn_lines: Last-summarised list of warnings 174 _build_period_us: Time taken for a single build (float object). 175 _complete_delay: Expected delay until completion (timedelta) 176 _next_delay_update: Next time we plan to display a progress update 177 (datatime) 178 _show_unknown: Show unknown boards (those not built) in summary 179 _timestamps: List of timestamps for the completion of the last 180 last _timestamp_count builds. Each is a datetime object. 181 _timestamp_count: Number of timestamps to keep in our list. 182 _working_dir: Base working directory containing all threads 183 """ 184 class Outcome: 185 """Records a build outcome for a single make invocation 186 187 Public Members: 188 rc: Outcome value (OUTCOME_...) 189 err_lines: List of error lines or [] if none 190 sizes: Dictionary of image size information, keyed by filename 191 - Each value is itself a dictionary containing 192 values for 'text', 'data' and 'bss', being the integer 193 size in bytes of each section. 194 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 195 value is itself a dictionary: 196 key: function name 197 value: Size of function in bytes 198 config: Dictionary keyed by filename - e.g. '.config'. Each 199 value is itself a dictionary: 200 key: config name 201 value: config value 202 """ 203 def __init__(self, rc, err_lines, sizes, func_sizes, config): 204 self.rc = rc 205 self.err_lines = err_lines 206 self.sizes = sizes 207 self.func_sizes = func_sizes 208 self.config = config 209 210 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 211 gnu_make='make', checkout=True, show_unknown=True, step=1, 212 no_subdirs=False, full_path=False, verbose_build=False, 213 incremental=False, per_board_out_dir=False, 214 config_only=False, squash_config_y=False): 215 """Create a new Builder object 216 217 Args: 218 toolchains: Toolchains object to use for building 219 base_dir: Base directory to use for builder 220 git_dir: Git directory containing source repository 221 num_threads: Number of builder threads to run 222 num_jobs: Number of jobs to run at once (passed to make as -j) 223 gnu_make: the command name of GNU Make. 224 checkout: True to check out source, False to skip that step. 225 This is used for testing. 226 show_unknown: Show unknown boards (those not built) in summary 227 step: 1 to process every commit, n to process every nth commit 228 no_subdirs: Don't create subdirectories when building current 229 source for a single board 230 full_path: Return the full path in CROSS_COMPILE and don't set 231 PATH 232 verbose_build: Run build with V=1 and don't use 'make -s' 233 incremental: Always perform incremental builds; don't run make 234 mrproper when configuring 235 per_board_out_dir: Build in a separate persistent directory per 236 board rather than a thread-specific directory 237 config_only: Only configure each build, don't build it 238 squash_config_y: Convert CONFIG options with the value 'y' to '1' 239 """ 240 self.toolchains = toolchains 241 self.base_dir = base_dir 242 self._working_dir = os.path.join(base_dir, '.bm-work') 243 self.threads = [] 244 self.do_make = self.Make 245 self.gnu_make = gnu_make 246 self.checkout = checkout 247 self.num_threads = num_threads 248 self.num_jobs = num_jobs 249 self.already_done = 0 250 self.force_build = False 251 self.git_dir = git_dir 252 self._show_unknown = show_unknown 253 self._timestamp_count = 10 254 self._build_period_us = None 255 self._complete_delay = None 256 self._next_delay_update = datetime.now() 257 self.force_config_on_failure = True 258 self.force_build_failures = False 259 self.force_reconfig = False 260 self._step = step 261 self.in_tree = False 262 self._error_lines = 0 263 self.no_subdirs = no_subdirs 264 self.full_path = full_path 265 self.verbose_build = verbose_build 266 self.config_only = config_only 267 self.squash_config_y = squash_config_y 268 self.config_filenames = BASE_CONFIG_FILENAMES 269 if not self.squash_config_y: 270 self.config_filenames += EXTRA_CONFIG_FILENAMES 271 272 self.col = terminal.Color() 273 274 self._re_function = re.compile('(.*): In function.*') 275 self._re_files = re.compile('In file included from.*') 276 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 277 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 278 279 self.queue = Queue.Queue() 280 self.out_queue = Queue.Queue() 281 for i in range(self.num_threads): 282 t = builderthread.BuilderThread(self, i, incremental, 283 per_board_out_dir) 284 t.setDaemon(True) 285 t.start() 286 self.threads.append(t) 287 288 self.last_line_len = 0 289 t = builderthread.ResultThread(self) 290 t.setDaemon(True) 291 t.start() 292 self.threads.append(t) 293 294 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 295 self.re_make_err = re.compile('|'.join(ignore_lines)) 296 297 # Handle existing graceful with SIGINT / Ctrl-C 298 signal.signal(signal.SIGINT, self.signal_handler) 299 300 def __del__(self): 301 """Get rid of all threads created by the builder""" 302 for t in self.threads: 303 del t 304 305 def signal_handler(self, signal, frame): 306 sys.exit(1) 307 308 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 309 show_detail=False, show_bloat=False, 310 list_error_boards=False, show_config=False): 311 """Setup display options for the builder. 312 313 show_errors: True to show summarised error/warning info 314 show_sizes: Show size deltas 315 show_detail: Show detail for each board 316 show_bloat: Show detail for each function 317 list_error_boards: Show the boards which caused each error/warning 318 show_config: Show config deltas 319 """ 320 self._show_errors = show_errors 321 self._show_sizes = show_sizes 322 self._show_detail = show_detail 323 self._show_bloat = show_bloat 324 self._list_error_boards = list_error_boards 325 self._show_config = show_config 326 327 def _AddTimestamp(self): 328 """Add a new timestamp to the list and record the build period. 329 330 The build period is the length of time taken to perform a single 331 build (one board, one commit). 332 """ 333 now = datetime.now() 334 self._timestamps.append(now) 335 count = len(self._timestamps) 336 delta = self._timestamps[-1] - self._timestamps[0] 337 seconds = delta.total_seconds() 338 339 # If we have enough data, estimate build period (time taken for a 340 # single build) and therefore completion time. 341 if count > 1 and self._next_delay_update < now: 342 self._next_delay_update = now + timedelta(seconds=2) 343 if seconds > 0: 344 self._build_period = float(seconds) / count 345 todo = self.count - self.upto 346 self._complete_delay = timedelta(microseconds= 347 self._build_period * todo * 1000000) 348 # Round it 349 self._complete_delay -= timedelta( 350 microseconds=self._complete_delay.microseconds) 351 352 if seconds > 60: 353 self._timestamps.popleft() 354 count -= 1 355 356 def ClearLine(self, length): 357 """Clear any characters on the current line 358 359 Make way for a new line of length 'length', by outputting enough 360 spaces to clear out the old line. Then remember the new length for 361 next time. 362 363 Args: 364 length: Length of new line, in characters 365 """ 366 if length < self.last_line_len: 367 Print(' ' * (self.last_line_len - length), newline=False) 368 Print('\r', newline=False) 369 self.last_line_len = length 370 sys.stdout.flush() 371 372 def SelectCommit(self, commit, checkout=True): 373 """Checkout the selected commit for this build 374 """ 375 self.commit = commit 376 if checkout and self.checkout: 377 gitutil.Checkout(commit.hash) 378 379 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 380 """Run make 381 382 Args: 383 commit: Commit object that is being built 384 brd: Board object that is being built 385 stage: Stage that we are at (mrproper, config, build) 386 cwd: Directory where make should be run 387 args: Arguments to pass to make 388 kwargs: Arguments to pass to command.RunPipe() 389 """ 390 cmd = [self.gnu_make] + list(args) 391 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 392 cwd=cwd, raise_on_error=False, **kwargs) 393 if self.verbose_build: 394 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 395 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 396 return result 397 398 def ProcessResult(self, result): 399 """Process the result of a build, showing progress information 400 401 Args: 402 result: A CommandResult object, which indicates the result for 403 a single build 404 """ 405 col = terminal.Color() 406 if result: 407 target = result.brd.target 408 409 self.upto += 1 410 if result.return_code != 0: 411 self.fail += 1 412 elif result.stderr: 413 self.warned += 1 414 if result.already_done: 415 self.already_done += 1 416 if self._verbose: 417 Print('\r', newline=False) 418 self.ClearLine(0) 419 boards_selected = {target : result.brd} 420 self.ResetResultSummary(boards_selected) 421 self.ProduceResultSummary(result.commit_upto, self.commits, 422 boards_selected) 423 else: 424 target = '(starting)' 425 426 # Display separate counts for ok, warned and fail 427 ok = self.upto - self.warned - self.fail 428 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 429 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 430 line += self.col.Color(self.col.RED, '%5d' % self.fail) 431 432 name = ' /%-5d ' % self.count 433 434 # Add our current completion time estimate 435 self._AddTimestamp() 436 if self._complete_delay: 437 name += '%s : ' % self._complete_delay 438 # When building all boards for a commit, we can print a commit 439 # progress message. 440 if result and result.commit_upto is None: 441 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 442 self.commit_count) 443 444 name += target 445 Print(line + name, newline=False) 446 length = 16 + len(name) 447 self.ClearLine(length) 448 449 def _GetOutputDir(self, commit_upto): 450 """Get the name of the output directory for a commit number 451 452 The output directory is typically .../<branch>/<commit>. 453 454 Args: 455 commit_upto: Commit number to use (0..self.count-1) 456 """ 457 commit_dir = None 458 if self.commits: 459 commit = self.commits[commit_upto] 460 subject = commit.subject.translate(trans_valid_chars) 461 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 462 self.commit_count, commit.hash, subject[:20])) 463 elif not self.no_subdirs: 464 commit_dir = 'current' 465 if not commit_dir: 466 return self.base_dir 467 return os.path.join(self.base_dir, commit_dir) 468 469 def GetBuildDir(self, commit_upto, target): 470 """Get the name of the build directory for a commit number 471 472 The build directory is typically .../<branch>/<commit>/<target>. 473 474 Args: 475 commit_upto: Commit number to use (0..self.count-1) 476 target: Target name 477 """ 478 output_dir = self._GetOutputDir(commit_upto) 479 return os.path.join(output_dir, target) 480 481 def GetDoneFile(self, commit_upto, target): 482 """Get the name of the done file for a commit number 483 484 Args: 485 commit_upto: Commit number to use (0..self.count-1) 486 target: Target name 487 """ 488 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 489 490 def GetSizesFile(self, commit_upto, target): 491 """Get the name of the sizes file for a commit number 492 493 Args: 494 commit_upto: Commit number to use (0..self.count-1) 495 target: Target name 496 """ 497 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 498 499 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 500 """Get the name of the funcsizes file for a commit number and ELF file 501 502 Args: 503 commit_upto: Commit number to use (0..self.count-1) 504 target: Target name 505 elf_fname: Filename of elf image 506 """ 507 return os.path.join(self.GetBuildDir(commit_upto, target), 508 '%s.sizes' % elf_fname.replace('/', '-')) 509 510 def GetObjdumpFile(self, commit_upto, target, elf_fname): 511 """Get the name of the objdump file for a commit number and ELF file 512 513 Args: 514 commit_upto: Commit number to use (0..self.count-1) 515 target: Target name 516 elf_fname: Filename of elf image 517 """ 518 return os.path.join(self.GetBuildDir(commit_upto, target), 519 '%s.objdump' % elf_fname.replace('/', '-')) 520 521 def GetErrFile(self, commit_upto, target): 522 """Get the name of the err file for a commit number 523 524 Args: 525 commit_upto: Commit number to use (0..self.count-1) 526 target: Target name 527 """ 528 output_dir = self.GetBuildDir(commit_upto, target) 529 return os.path.join(output_dir, 'err') 530 531 def FilterErrors(self, lines): 532 """Filter out errors in which we have no interest 533 534 We should probably use map(). 535 536 Args: 537 lines: List of error lines, each a string 538 Returns: 539 New list with only interesting lines included 540 """ 541 out_lines = [] 542 for line in lines: 543 if not self.re_make_err.search(line): 544 out_lines.append(line) 545 return out_lines 546 547 def ReadFuncSizes(self, fname, fd): 548 """Read function sizes from the output of 'nm' 549 550 Args: 551 fd: File containing data to read 552 fname: Filename we are reading from (just for errors) 553 554 Returns: 555 Dictionary containing size of each function in bytes, indexed by 556 function name. 557 """ 558 sym = {} 559 for line in fd.readlines(): 560 try: 561 size, type, name = line[:-1].split() 562 except: 563 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 564 continue 565 if type in 'tTdDbB': 566 # function names begin with '.' on 64-bit powerpc 567 if '.' in name[1:]: 568 name = 'static.' + name.split('.')[0] 569 sym[name] = sym.get(name, 0) + int(size, 16) 570 return sym 571 572 def _ProcessConfig(self, fname): 573 """Read in a .config, autoconf.mk or autoconf.h file 574 575 This function handles all config file types. It ignores comments and 576 any #defines which don't start with CONFIG_. 577 578 Args: 579 fname: Filename to read 580 581 Returns: 582 Dictionary: 583 key: Config name (e.g. CONFIG_DM) 584 value: Config value (e.g. 1) 585 """ 586 config = {} 587 if os.path.exists(fname): 588 with open(fname) as fd: 589 for line in fd: 590 line = line.strip() 591 if line.startswith('#define'): 592 values = line[8:].split(' ', 1) 593 if len(values) > 1: 594 key, value = values 595 else: 596 key = values[0] 597 value = '1' if self.squash_config_y else '' 598 if not key.startswith('CONFIG_'): 599 continue 600 elif not line or line[0] in ['#', '*', '/']: 601 continue 602 else: 603 key, value = line.split('=', 1) 604 if self.squash_config_y and value == 'y': 605 value = '1' 606 config[key] = value 607 return config 608 609 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 610 read_config): 611 """Work out the outcome of a build. 612 613 Args: 614 commit_upto: Commit number to check (0..n-1) 615 target: Target board to check 616 read_func_sizes: True to read function size information 617 read_config: True to read .config and autoconf.h files 618 619 Returns: 620 Outcome object 621 """ 622 done_file = self.GetDoneFile(commit_upto, target) 623 sizes_file = self.GetSizesFile(commit_upto, target) 624 sizes = {} 625 func_sizes = {} 626 config = {} 627 if os.path.exists(done_file): 628 with open(done_file, 'r') as fd: 629 return_code = int(fd.readline()) 630 err_lines = [] 631 err_file = self.GetErrFile(commit_upto, target) 632 if os.path.exists(err_file): 633 with open(err_file, 'r') as fd: 634 err_lines = self.FilterErrors(fd.readlines()) 635 636 # Decide whether the build was ok, failed or created warnings 637 if return_code: 638 rc = OUTCOME_ERROR 639 elif len(err_lines): 640 rc = OUTCOME_WARNING 641 else: 642 rc = OUTCOME_OK 643 644 # Convert size information to our simple format 645 if os.path.exists(sizes_file): 646 with open(sizes_file, 'r') as fd: 647 for line in fd.readlines(): 648 values = line.split() 649 rodata = 0 650 if len(values) > 6: 651 rodata = int(values[6], 16) 652 size_dict = { 653 'all' : int(values[0]) + int(values[1]) + 654 int(values[2]), 655 'text' : int(values[0]) - rodata, 656 'data' : int(values[1]), 657 'bss' : int(values[2]), 658 'rodata' : rodata, 659 } 660 sizes[values[5]] = size_dict 661 662 if read_func_sizes: 663 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 664 for fname in glob.glob(pattern): 665 with open(fname, 'r') as fd: 666 dict_name = os.path.basename(fname).replace('.sizes', 667 '') 668 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 669 670 if read_config: 671 output_dir = self.GetBuildDir(commit_upto, target) 672 for name in self.config_filenames: 673 fname = os.path.join(output_dir, name) 674 config[name] = self._ProcessConfig(fname) 675 676 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 677 678 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 679 680 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 681 read_config): 682 """Calculate a summary of the results of building a commit. 683 684 Args: 685 board_selected: Dict containing boards to summarise 686 commit_upto: Commit number to summarize (0..self.count-1) 687 read_func_sizes: True to read function size information 688 read_config: True to read .config and autoconf.h files 689 690 Returns: 691 Tuple: 692 Dict containing boards which passed building this commit. 693 keyed by board.target 694 List containing a summary of error lines 695 Dict keyed by error line, containing a list of the Board 696 objects with that error 697 List containing a summary of warning lines 698 Dict keyed by error line, containing a list of the Board 699 objects with that warning 700 Dictionary keyed by board.target. Each value is a dictionary: 701 key: filename - e.g. '.config' 702 value is itself a dictionary: 703 key: config name 704 value: config value 705 """ 706 def AddLine(lines_summary, lines_boards, line, board): 707 line = line.rstrip() 708 if line in lines_boards: 709 lines_boards[line].append(board) 710 else: 711 lines_boards[line] = [board] 712 lines_summary.append(line) 713 714 board_dict = {} 715 err_lines_summary = [] 716 err_lines_boards = {} 717 warn_lines_summary = [] 718 warn_lines_boards = {} 719 config = {} 720 721 for board in boards_selected.itervalues(): 722 outcome = self.GetBuildOutcome(commit_upto, board.target, 723 read_func_sizes, read_config) 724 board_dict[board.target] = outcome 725 last_func = None 726 last_was_warning = False 727 for line in outcome.err_lines: 728 if line: 729 if (self._re_function.match(line) or 730 self._re_files.match(line)): 731 last_func = line 732 else: 733 is_warning = self._re_warning.match(line) 734 is_note = self._re_note.match(line) 735 if is_warning or (last_was_warning and is_note): 736 if last_func: 737 AddLine(warn_lines_summary, warn_lines_boards, 738 last_func, board) 739 AddLine(warn_lines_summary, warn_lines_boards, 740 line, board) 741 else: 742 if last_func: 743 AddLine(err_lines_summary, err_lines_boards, 744 last_func, board) 745 AddLine(err_lines_summary, err_lines_boards, 746 line, board) 747 last_was_warning = is_warning 748 last_func = None 749 tconfig = Config(self.config_filenames, board.target) 750 for fname in self.config_filenames: 751 if outcome.config: 752 for key, value in outcome.config[fname].iteritems(): 753 tconfig.Add(fname, key, value) 754 config[board.target] = tconfig 755 756 return (board_dict, err_lines_summary, err_lines_boards, 757 warn_lines_summary, warn_lines_boards, config) 758 759 def AddOutcome(self, board_dict, arch_list, changes, char, color): 760 """Add an output to our list of outcomes for each architecture 761 762 This simple function adds failing boards (changes) to the 763 relevant architecture string, so we can print the results out 764 sorted by architecture. 765 766 Args: 767 board_dict: Dict containing all boards 768 arch_list: Dict keyed by arch name. Value is a string containing 769 a list of board names which failed for that arch. 770 changes: List of boards to add to arch_list 771 color: terminal.Colour object 772 """ 773 done_arch = {} 774 for target in changes: 775 if target in board_dict: 776 arch = board_dict[target].arch 777 else: 778 arch = 'unknown' 779 str = self.col.Color(color, ' ' + target) 780 if not arch in done_arch: 781 str = ' %s %s' % (self.col.Color(color, char), str) 782 done_arch[arch] = True 783 if not arch in arch_list: 784 arch_list[arch] = str 785 else: 786 arch_list[arch] += str 787 788 789 def ColourNum(self, num): 790 color = self.col.RED if num > 0 else self.col.GREEN 791 if num == 0: 792 return '0' 793 return self.col.Color(color, str(num)) 794 795 def ResetResultSummary(self, board_selected): 796 """Reset the results summary ready for use. 797 798 Set up the base board list to be all those selected, and set the 799 error lines to empty. 800 801 Following this, calls to PrintResultSummary() will use this 802 information to work out what has changed. 803 804 Args: 805 board_selected: Dict containing boards to summarise, keyed by 806 board.target 807 """ 808 self._base_board_dict = {} 809 for board in board_selected: 810 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 811 self._base_err_lines = [] 812 self._base_warn_lines = [] 813 self._base_err_line_boards = {} 814 self._base_warn_line_boards = {} 815 self._base_config = None 816 817 def PrintFuncSizeDetail(self, fname, old, new): 818 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 819 delta, common = [], {} 820 821 for a in old: 822 if a in new: 823 common[a] = 1 824 825 for name in old: 826 if name not in common: 827 remove += 1 828 down += old[name] 829 delta.append([-old[name], name]) 830 831 for name in new: 832 if name not in common: 833 add += 1 834 up += new[name] 835 delta.append([new[name], name]) 836 837 for name in common: 838 diff = new.get(name, 0) - old.get(name, 0) 839 if diff > 0: 840 grow, up = grow + 1, up + diff 841 elif diff < 0: 842 shrink, down = shrink + 1, down - diff 843 delta.append([diff, name]) 844 845 delta.sort() 846 delta.reverse() 847 848 args = [add, -remove, grow, -shrink, up, -down, up - down] 849 if max(args) == 0: 850 return 851 args = [self.ColourNum(x) for x in args] 852 indent = ' ' * 15 853 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 854 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 855 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 856 'delta')) 857 for diff, name in delta: 858 if diff: 859 color = self.col.RED if diff > 0 else self.col.GREEN 860 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 861 old.get(name, '-'), new.get(name,'-'), diff) 862 Print(msg, colour=color) 863 864 865 def PrintSizeDetail(self, target_list, show_bloat): 866 """Show details size information for each board 867 868 Args: 869 target_list: List of targets, each a dict containing: 870 'target': Target name 871 'total_diff': Total difference in bytes across all areas 872 <part_name>: Difference for that part 873 show_bloat: Show detail for each function 874 """ 875 targets_by_diff = sorted(target_list, reverse=True, 876 key=lambda x: x['_total_diff']) 877 for result in targets_by_diff: 878 printed_target = False 879 for name in sorted(result): 880 diff = result[name] 881 if name.startswith('_'): 882 continue 883 if diff != 0: 884 color = self.col.RED if diff > 0 else self.col.GREEN 885 msg = ' %s %+d' % (name, diff) 886 if not printed_target: 887 Print('%10s %-15s:' % ('', result['_target']), 888 newline=False) 889 printed_target = True 890 Print(msg, colour=color, newline=False) 891 if printed_target: 892 Print() 893 if show_bloat: 894 target = result['_target'] 895 outcome = result['_outcome'] 896 base_outcome = self._base_board_dict[target] 897 for fname in outcome.func_sizes: 898 self.PrintFuncSizeDetail(fname, 899 base_outcome.func_sizes[fname], 900 outcome.func_sizes[fname]) 901 902 903 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 904 show_bloat): 905 """Print a summary of image sizes broken down by section. 906 907 The summary takes the form of one line per architecture. The 908 line contains deltas for each of the sections (+ means the section 909 got bigger, - means smaller). The nunmbers are the average number 910 of bytes that a board in this section increased by. 911 912 For example: 913 powerpc: (622 boards) text -0.0 914 arm: (285 boards) text -0.0 915 nds32: (3 boards) text -8.0 916 917 Args: 918 board_selected: Dict containing boards to summarise, keyed by 919 board.target 920 board_dict: Dict containing boards for which we built this 921 commit, keyed by board.target. The value is an Outcome object. 922 show_detail: Show detail for each board 923 show_bloat: Show detail for each function 924 """ 925 arch_list = {} 926 arch_count = {} 927 928 # Calculate changes in size for different image parts 929 # The previous sizes are in Board.sizes, for each board 930 for target in board_dict: 931 if target not in board_selected: 932 continue 933 base_sizes = self._base_board_dict[target].sizes 934 outcome = board_dict[target] 935 sizes = outcome.sizes 936 937 # Loop through the list of images, creating a dict of size 938 # changes for each image/part. We end up with something like 939 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 940 # which means that U-Boot data increased by 5 bytes and SPL 941 # text decreased by 4. 942 err = {'_target' : target} 943 for image in sizes: 944 if image in base_sizes: 945 base_image = base_sizes[image] 946 # Loop through the text, data, bss parts 947 for part in sorted(sizes[image]): 948 diff = sizes[image][part] - base_image[part] 949 col = None 950 if diff: 951 if image == 'u-boot': 952 name = part 953 else: 954 name = image + ':' + part 955 err[name] = diff 956 arch = board_selected[target].arch 957 if not arch in arch_count: 958 arch_count[arch] = 1 959 else: 960 arch_count[arch] += 1 961 if not sizes: 962 pass # Only add to our list when we have some stats 963 elif not arch in arch_list: 964 arch_list[arch] = [err] 965 else: 966 arch_list[arch].append(err) 967 968 # We now have a list of image size changes sorted by arch 969 # Print out a summary of these 970 for arch, target_list in arch_list.iteritems(): 971 # Get total difference for each type 972 totals = {} 973 for result in target_list: 974 total = 0 975 for name, diff in result.iteritems(): 976 if name.startswith('_'): 977 continue 978 total += diff 979 if name in totals: 980 totals[name] += diff 981 else: 982 totals[name] = diff 983 result['_total_diff'] = total 984 result['_outcome'] = board_dict[result['_target']] 985 986 count = len(target_list) 987 printed_arch = False 988 for name in sorted(totals): 989 diff = totals[name] 990 if diff: 991 # Display the average difference in this name for this 992 # architecture 993 avg_diff = float(diff) / count 994 color = self.col.RED if avg_diff > 0 else self.col.GREEN 995 msg = ' %s %+1.1f' % (name, avg_diff) 996 if not printed_arch: 997 Print('%10s: (for %d/%d boards)' % (arch, count, 998 arch_count[arch]), newline=False) 999 printed_arch = True 1000 Print(msg, colour=color, newline=False) 1001 1002 if printed_arch: 1003 Print() 1004 if show_detail: 1005 self.PrintSizeDetail(target_list, show_bloat) 1006 1007 1008 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1009 err_line_boards, warn_lines, warn_line_boards, 1010 config, show_sizes, show_detail, show_bloat, 1011 show_config): 1012 """Compare results with the base results and display delta. 1013 1014 Only boards mentioned in board_selected will be considered. This 1015 function is intended to be called repeatedly with the results of 1016 each commit. It therefore shows a 'diff' between what it saw in 1017 the last call and what it sees now. 1018 1019 Args: 1020 board_selected: Dict containing boards to summarise, keyed by 1021 board.target 1022 board_dict: Dict containing boards for which we built this 1023 commit, keyed by board.target. The value is an Outcome object. 1024 err_lines: A list of errors for this commit, or [] if there is 1025 none, or we don't want to print errors 1026 err_line_boards: Dict keyed by error line, containing a list of 1027 the Board objects with that error 1028 warn_lines: A list of warnings for this commit, or [] if there is 1029 none, or we don't want to print errors 1030 warn_line_boards: Dict keyed by warning line, containing a list of 1031 the Board objects with that warning 1032 config: Dictionary keyed by filename - e.g. '.config'. Each 1033 value is itself a dictionary: 1034 key: config name 1035 value: config value 1036 show_sizes: Show image size deltas 1037 show_detail: Show detail for each board 1038 show_bloat: Show detail for each function 1039 show_config: Show config changes 1040 """ 1041 def _BoardList(line, line_boards): 1042 """Helper function to get a line of boards containing a line 1043 1044 Args: 1045 line: Error line to search for 1046 Return: 1047 String containing a list of boards with that error line, or 1048 '' if the user has not requested such a list 1049 """ 1050 if self._list_error_boards: 1051 names = [] 1052 for board in line_boards[line]: 1053 if not board.target in names: 1054 names.append(board.target) 1055 names_str = '(%s) ' % ','.join(names) 1056 else: 1057 names_str = '' 1058 return names_str 1059 1060 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1061 char): 1062 better_lines = [] 1063 worse_lines = [] 1064 for line in lines: 1065 if line not in base_lines: 1066 worse_lines.append(char + '+' + 1067 _BoardList(line, line_boards) + line) 1068 for line in base_lines: 1069 if line not in lines: 1070 better_lines.append(char + '-' + 1071 _BoardList(line, base_line_boards) + line) 1072 return better_lines, worse_lines 1073 1074 def _CalcConfig(delta, name, config): 1075 """Calculate configuration changes 1076 1077 Args: 1078 delta: Type of the delta, e.g. '+' 1079 name: name of the file which changed (e.g. .config) 1080 config: configuration change dictionary 1081 key: config name 1082 value: config value 1083 Returns: 1084 String containing the configuration changes which can be 1085 printed 1086 """ 1087 out = '' 1088 for key in sorted(config.keys()): 1089 out += '%s=%s ' % (key, config[key]) 1090 return '%s %s: %s' % (delta, name, out) 1091 1092 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1093 """Add changes in configuration to a list 1094 1095 Args: 1096 lines: list to add to 1097 name: config file name 1098 config_plus: configurations added, dictionary 1099 key: config name 1100 value: config value 1101 config_minus: configurations removed, dictionary 1102 key: config name 1103 value: config value 1104 config_change: configurations changed, dictionary 1105 key: config name 1106 value: config value 1107 """ 1108 if config_plus: 1109 lines.append(_CalcConfig('+', name, config_plus)) 1110 if config_minus: 1111 lines.append(_CalcConfig('-', name, config_minus)) 1112 if config_change: 1113 lines.append(_CalcConfig('c', name, config_change)) 1114 1115 def _OutputConfigInfo(lines): 1116 for line in lines: 1117 if not line: 1118 continue 1119 if line[0] == '+': 1120 col = self.col.GREEN 1121 elif line[0] == '-': 1122 col = self.col.RED 1123 elif line[0] == 'c': 1124 col = self.col.YELLOW 1125 Print(' ' + line, newline=True, colour=col) 1126 1127 1128 better = [] # List of boards fixed since last commit 1129 worse = [] # List of new broken boards since last commit 1130 new = [] # List of boards that didn't exist last time 1131 unknown = [] # List of boards that were not built 1132 1133 for target in board_dict: 1134 if target not in board_selected: 1135 continue 1136 1137 # If the board was built last time, add its outcome to a list 1138 if target in self._base_board_dict: 1139 base_outcome = self._base_board_dict[target].rc 1140 outcome = board_dict[target] 1141 if outcome.rc == OUTCOME_UNKNOWN: 1142 unknown.append(target) 1143 elif outcome.rc < base_outcome: 1144 better.append(target) 1145 elif outcome.rc > base_outcome: 1146 worse.append(target) 1147 else: 1148 new.append(target) 1149 1150 # Get a list of errors that have appeared, and disappeared 1151 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1152 self._base_err_line_boards, err_lines, err_line_boards, '') 1153 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1154 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1155 1156 # Display results by arch 1157 if (better or worse or unknown or new or worse_err or better_err 1158 or worse_warn or better_warn): 1159 arch_list = {} 1160 self.AddOutcome(board_selected, arch_list, better, '', 1161 self.col.GREEN) 1162 self.AddOutcome(board_selected, arch_list, worse, '+', 1163 self.col.RED) 1164 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1165 if self._show_unknown: 1166 self.AddOutcome(board_selected, arch_list, unknown, '?', 1167 self.col.MAGENTA) 1168 for arch, target_list in arch_list.iteritems(): 1169 Print('%10s: %s' % (arch, target_list)) 1170 self._error_lines += 1 1171 if better_err: 1172 Print('\n'.join(better_err), colour=self.col.GREEN) 1173 self._error_lines += 1 1174 if worse_err: 1175 Print('\n'.join(worse_err), colour=self.col.RED) 1176 self._error_lines += 1 1177 if better_warn: 1178 Print('\n'.join(better_warn), colour=self.col.CYAN) 1179 self._error_lines += 1 1180 if worse_warn: 1181 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1182 self._error_lines += 1 1183 1184 if show_sizes: 1185 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1186 show_bloat) 1187 1188 if show_config and self._base_config: 1189 summary = {} 1190 arch_config_plus = {} 1191 arch_config_minus = {} 1192 arch_config_change = {} 1193 arch_list = [] 1194 1195 for target in board_dict: 1196 if target not in board_selected: 1197 continue 1198 arch = board_selected[target].arch 1199 if arch not in arch_list: 1200 arch_list.append(arch) 1201 1202 for arch in arch_list: 1203 arch_config_plus[arch] = {} 1204 arch_config_minus[arch] = {} 1205 arch_config_change[arch] = {} 1206 for name in self.config_filenames: 1207 arch_config_plus[arch][name] = {} 1208 arch_config_minus[arch][name] = {} 1209 arch_config_change[arch][name] = {} 1210 1211 for target in board_dict: 1212 if target not in board_selected: 1213 continue 1214 1215 arch = board_selected[target].arch 1216 1217 all_config_plus = {} 1218 all_config_minus = {} 1219 all_config_change = {} 1220 tbase = self._base_config[target] 1221 tconfig = config[target] 1222 lines = [] 1223 for name in self.config_filenames: 1224 if not tconfig.config[name]: 1225 continue 1226 config_plus = {} 1227 config_minus = {} 1228 config_change = {} 1229 base = tbase.config[name] 1230 for key, value in tconfig.config[name].iteritems(): 1231 if key not in base: 1232 config_plus[key] = value 1233 all_config_plus[key] = value 1234 for key, value in base.iteritems(): 1235 if key not in tconfig.config[name]: 1236 config_minus[key] = value 1237 all_config_minus[key] = value 1238 for key, value in base.iteritems(): 1239 new_value = tconfig.config.get(key) 1240 if new_value and value != new_value: 1241 desc = '%s -> %s' % (value, new_value) 1242 config_change[key] = desc 1243 all_config_change[key] = desc 1244 1245 arch_config_plus[arch][name].update(config_plus) 1246 arch_config_minus[arch][name].update(config_minus) 1247 arch_config_change[arch][name].update(config_change) 1248 1249 _AddConfig(lines, name, config_plus, config_minus, 1250 config_change) 1251 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1252 all_config_change) 1253 summary[target] = '\n'.join(lines) 1254 1255 lines_by_target = {} 1256 for target, lines in summary.iteritems(): 1257 if lines in lines_by_target: 1258 lines_by_target[lines].append(target) 1259 else: 1260 lines_by_target[lines] = [target] 1261 1262 for arch in arch_list: 1263 lines = [] 1264 all_plus = {} 1265 all_minus = {} 1266 all_change = {} 1267 for name in self.config_filenames: 1268 all_plus.update(arch_config_plus[arch][name]) 1269 all_minus.update(arch_config_minus[arch][name]) 1270 all_change.update(arch_config_change[arch][name]) 1271 _AddConfig(lines, name, arch_config_plus[arch][name], 1272 arch_config_minus[arch][name], 1273 arch_config_change[arch][name]) 1274 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1275 #arch_summary[target] = '\n'.join(lines) 1276 if lines: 1277 Print('%s:' % arch) 1278 _OutputConfigInfo(lines) 1279 1280 for lines, targets in lines_by_target.iteritems(): 1281 if not lines: 1282 continue 1283 Print('%s :' % ' '.join(sorted(targets))) 1284 _OutputConfigInfo(lines.split('\n')) 1285 1286 1287 # Save our updated information for the next call to this function 1288 self._base_board_dict = board_dict 1289 self._base_err_lines = err_lines 1290 self._base_warn_lines = warn_lines 1291 self._base_err_line_boards = err_line_boards 1292 self._base_warn_line_boards = warn_line_boards 1293 self._base_config = config 1294 1295 # Get a list of boards that did not get built, if needed 1296 not_built = [] 1297 for board in board_selected: 1298 if not board in board_dict: 1299 not_built.append(board) 1300 if not_built: 1301 Print("Boards not built (%d): %s" % (len(not_built), 1302 ', '.join(not_built))) 1303 1304 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1305 (board_dict, err_lines, err_line_boards, warn_lines, 1306 warn_line_boards, config) = self.GetResultSummary( 1307 board_selected, commit_upto, 1308 read_func_sizes=self._show_bloat, 1309 read_config=self._show_config) 1310 if commits: 1311 msg = '%02d: %s' % (commit_upto + 1, 1312 commits[commit_upto].subject) 1313 Print(msg, colour=self.col.BLUE) 1314 self.PrintResultSummary(board_selected, board_dict, 1315 err_lines if self._show_errors else [], err_line_boards, 1316 warn_lines if self._show_errors else [], warn_line_boards, 1317 config, self._show_sizes, self._show_detail, 1318 self._show_bloat, self._show_config) 1319 1320 def ShowSummary(self, commits, board_selected): 1321 """Show a build summary for U-Boot for a given board list. 1322 1323 Reset the result summary, then repeatedly call GetResultSummary on 1324 each commit's results, then display the differences we see. 1325 1326 Args: 1327 commit: Commit objects to summarise 1328 board_selected: Dict containing boards to summarise 1329 """ 1330 self.commit_count = len(commits) if commits else 1 1331 self.commits = commits 1332 self.ResetResultSummary(board_selected) 1333 self._error_lines = 0 1334 1335 for commit_upto in range(0, self.commit_count, self._step): 1336 self.ProduceResultSummary(commit_upto, commits, board_selected) 1337 if not self._error_lines: 1338 Print('(no errors to report)', colour=self.col.GREEN) 1339 1340 1341 def SetupBuild(self, board_selected, commits): 1342 """Set up ready to start a build. 1343 1344 Args: 1345 board_selected: Selected boards to build 1346 commits: Selected commits to build 1347 """ 1348 # First work out how many commits we will build 1349 count = (self.commit_count + self._step - 1) / self._step 1350 self.count = len(board_selected) * count 1351 self.upto = self.warned = self.fail = 0 1352 self._timestamps = collections.deque() 1353 1354 def GetThreadDir(self, thread_num): 1355 """Get the directory path to the working dir for a thread. 1356 1357 Args: 1358 thread_num: Number of thread to check. 1359 """ 1360 return os.path.join(self._working_dir, '%02d' % thread_num) 1361 1362 def _PrepareThread(self, thread_num, setup_git): 1363 """Prepare the working directory for a thread. 1364 1365 This clones or fetches the repo into the thread's work directory. 1366 1367 Args: 1368 thread_num: Thread number (0, 1, ...) 1369 setup_git: True to set up a git repo clone 1370 """ 1371 thread_dir = self.GetThreadDir(thread_num) 1372 builderthread.Mkdir(thread_dir) 1373 git_dir = os.path.join(thread_dir, '.git') 1374 1375 # Clone the repo if it doesn't already exist 1376 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1377 # we have a private index but uses the origin repo's contents? 1378 if setup_git and self.git_dir: 1379 src_dir = os.path.abspath(self.git_dir) 1380 if os.path.exists(git_dir): 1381 gitutil.Fetch(git_dir, thread_dir) 1382 else: 1383 Print('\rCloning repo for thread %d' % thread_num, 1384 newline=False) 1385 gitutil.Clone(src_dir, thread_dir) 1386 Print('\r%s\r' % (' ' * 30), newline=False) 1387 1388 def _PrepareWorkingSpace(self, max_threads, setup_git): 1389 """Prepare the working directory for use. 1390 1391 Set up the git repo for each thread. 1392 1393 Args: 1394 max_threads: Maximum number of threads we expect to need. 1395 setup_git: True to set up a git repo clone 1396 """ 1397 builderthread.Mkdir(self._working_dir) 1398 for thread in range(max_threads): 1399 self._PrepareThread(thread, setup_git) 1400 1401 def _PrepareOutputSpace(self): 1402 """Get the output directories ready to receive files. 1403 1404 We delete any output directories which look like ones we need to 1405 create. Having left over directories is confusing when the user wants 1406 to check the output manually. 1407 """ 1408 if not self.commits: 1409 return 1410 dir_list = [] 1411 for commit_upto in range(self.commit_count): 1412 dir_list.append(self._GetOutputDir(commit_upto)) 1413 1414 to_remove = [] 1415 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1416 if dirname not in dir_list: 1417 to_remove.append(dirname) 1418 if to_remove: 1419 Print('Removing %d old build directories' % len(to_remove), 1420 newline=False) 1421 for dirname in to_remove: 1422 shutil.rmtree(dirname) 1423 1424 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1425 """Build all commits for a list of boards 1426 1427 Args: 1428 commits: List of commits to be build, each a Commit object 1429 boards_selected: Dict of selected boards, key is target name, 1430 value is Board object 1431 keep_outputs: True to save build output files 1432 verbose: Display build results as they are completed 1433 Returns: 1434 Tuple containing: 1435 - number of boards that failed to build 1436 - number of boards that issued warnings 1437 """ 1438 self.commit_count = len(commits) if commits else 1 1439 self.commits = commits 1440 self._verbose = verbose 1441 1442 self.ResetResultSummary(board_selected) 1443 builderthread.Mkdir(self.base_dir, parents = True) 1444 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1445 commits is not None) 1446 self._PrepareOutputSpace() 1447 Print('\rStarting build...', newline=False) 1448 self.SetupBuild(board_selected, commits) 1449 self.ProcessResult(None) 1450 1451 # Create jobs to build all commits for each board 1452 for brd in board_selected.itervalues(): 1453 job = builderthread.BuilderJob() 1454 job.board = brd 1455 job.commits = commits 1456 job.keep_outputs = keep_outputs 1457 job.step = self._step 1458 self.queue.put(job) 1459 1460 term = threading.Thread(target=self.queue.join) 1461 term.setDaemon(True) 1462 term.start() 1463 while term.isAlive(): 1464 term.join(100) 1465 1466 # Wait until we have processed all output 1467 self.out_queue.join() 1468 Print() 1469 self.ClearLine(0) 1470 return (self.fail, self.warned) 1471