1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import signal 16import string 17import sys 18import threading 19import time 20 21import builderthread 22import command 23import gitutil 24import terminal 25from terminal import Print 26import toolchain 27 28 29""" 30Theory of Operation 31 32Please see README for user documentation, and you should be familiar with 33that before trying to make sense of this. 34 35Buildman works by keeping the machine as busy as possible, building different 36commits for different boards on multiple CPUs at once. 37 38The source repo (self.git_dir) contains all the commits to be built. Each 39thread works on a single board at a time. It checks out the first commit, 40configures it for that board, then builds it. Then it checks out the next 41commit and builds it (typically without re-configuring). When it runs out 42of commits, it gets another job from the builder and starts again with that 43board. 44 45Clearly the builder threads could work either way - they could check out a 46commit and then built it for all boards. Using separate directories for each 47commit/board pair they could leave their build product around afterwards 48also. 49 50The intent behind building a single board for multiple commits, is to make 51use of incremental builds. Since each commit is built incrementally from 52the previous one, builds are faster. Reconfiguring for a different board 53removes all intermediate object files. 54 55Many threads can be working at once, but each has its own working directory. 56When a thread finishes a build, it puts the output files into a result 57directory. 58 59The base directory used by buildman is normally '../<branch>', i.e. 60a directory higher than the source repository and named after the branch 61being built. 62 63Within the base directory, we have one subdirectory for each commit. Within 64that is one subdirectory for each board. Within that is the build output for 65that commit/board combination. 66 67Buildman also create working directories for each thread, in a .bm-work/ 68subdirectory in the base dir. 69 70As an example, say we are building branch 'us-net' for boards 'sandbox' and 71'seaboard', and say that us-net has two commits. We will have directories 72like this: 73 74us-net/ base directory 75 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 76 sandbox/ 77 u-boot.bin 78 seaboard/ 79 u-boot.bin 80 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 81 sandbox/ 82 u-boot.bin 83 seaboard/ 84 u-boot.bin 85 .bm-work/ 86 00/ working directory for thread 0 (contains source checkout) 87 build/ build output 88 01/ working directory for thread 1 89 build/ build output 90 ... 91u-boot/ source directory 92 .git/ repository 93""" 94 95# Possible build outcomes 96OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 97 98# Translate a commit subject into a valid filename (and handle unicode) 99trans_valid_chars = string.maketrans('/: ', '---') 100trans_valid_chars = trans_valid_chars.decode('latin-1') 101 102BASE_CONFIG_FILENAMES = [ 103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 104] 105 106EXTRA_CONFIG_FILENAMES = [ 107 '.config', '.config-spl', '.config-tpl', 108 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 109 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 110] 111 112class Config: 113 """Holds information about configuration settings for a board.""" 114 def __init__(self, config_filename, target): 115 self.target = target 116 self.config = {} 117 for fname in config_filename: 118 self.config[fname] = {} 119 120 def Add(self, fname, key, value): 121 self.config[fname][key] = value 122 123 def __hash__(self): 124 val = 0 125 for fname in self.config: 126 for key, value in self.config[fname].iteritems(): 127 print key, value 128 val = val ^ hash(key) & hash(value) 129 return val 130 131class Builder: 132 """Class for building U-Boot for a particular commit. 133 134 Public members: (many should ->private) 135 already_done: Number of builds already completed 136 base_dir: Base directory to use for builder 137 checkout: True to check out source, False to skip that step. 138 This is used for testing. 139 col: terminal.Color() object 140 count: Number of commits to build 141 do_make: Method to call to invoke Make 142 fail: Number of builds that failed due to error 143 force_build: Force building even if a build already exists 144 force_config_on_failure: If a commit fails for a board, disable 145 incremental building for the next commit we build for that 146 board, so that we will see all warnings/errors again. 147 force_build_failures: If a previously-built build (i.e. built on 148 a previous run of buildman) is marked as failed, rebuild it. 149 git_dir: Git directory containing source repository 150 last_line_len: Length of the last line we printed (used for erasing 151 it with new progress information) 152 num_jobs: Number of jobs to run at once (passed to make as -j) 153 num_threads: Number of builder threads to run 154 out_queue: Queue of results to process 155 re_make_err: Compiled regular expression for ignore_lines 156 queue: Queue of jobs to run 157 threads: List of active threads 158 toolchains: Toolchains object to use for building 159 upto: Current commit number we are building (0.count-1) 160 warned: Number of builds that produced at least one warning 161 force_reconfig: Reconfigure U-Boot on each comiit. This disables 162 incremental building, where buildman reconfigures on the first 163 commit for a baord, and then just does an incremental build for 164 the following commits. In fact buildman will reconfigure and 165 retry for any failing commits, so generally the only effect of 166 this option is to slow things down. 167 in_tree: Build U-Boot in-tree instead of specifying an output 168 directory separate from the source code. This option is really 169 only useful for testing in-tree builds. 170 171 Private members: 172 _base_board_dict: Last-summarised Dict of boards 173 _base_err_lines: Last-summarised list of errors 174 _base_warn_lines: Last-summarised list of warnings 175 _build_period_us: Time taken for a single build (float object). 176 _complete_delay: Expected delay until completion (timedelta) 177 _next_delay_update: Next time we plan to display a progress update 178 (datatime) 179 _show_unknown: Show unknown boards (those not built) in summary 180 _timestamps: List of timestamps for the completion of the last 181 last _timestamp_count builds. Each is a datetime object. 182 _timestamp_count: Number of timestamps to keep in our list. 183 _working_dir: Base working directory containing all threads 184 """ 185 class Outcome: 186 """Records a build outcome for a single make invocation 187 188 Public Members: 189 rc: Outcome value (OUTCOME_...) 190 err_lines: List of error lines or [] if none 191 sizes: Dictionary of image size information, keyed by filename 192 - Each value is itself a dictionary containing 193 values for 'text', 'data' and 'bss', being the integer 194 size in bytes of each section. 195 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 196 value is itself a dictionary: 197 key: function name 198 value: Size of function in bytes 199 config: Dictionary keyed by filename - e.g. '.config'. Each 200 value is itself a dictionary: 201 key: config name 202 value: config value 203 """ 204 def __init__(self, rc, err_lines, sizes, func_sizes, config): 205 self.rc = rc 206 self.err_lines = err_lines 207 self.sizes = sizes 208 self.func_sizes = func_sizes 209 self.config = config 210 211 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 212 gnu_make='make', checkout=True, show_unknown=True, step=1, 213 no_subdirs=False, full_path=False, verbose_build=False, 214 incremental=False, per_board_out_dir=False, 215 config_only=False, squash_config_y=False, 216 warnings_as_errors=False): 217 """Create a new Builder object 218 219 Args: 220 toolchains: Toolchains object to use for building 221 base_dir: Base directory to use for builder 222 git_dir: Git directory containing source repository 223 num_threads: Number of builder threads to run 224 num_jobs: Number of jobs to run at once (passed to make as -j) 225 gnu_make: the command name of GNU Make. 226 checkout: True to check out source, False to skip that step. 227 This is used for testing. 228 show_unknown: Show unknown boards (those not built) in summary 229 step: 1 to process every commit, n to process every nth commit 230 no_subdirs: Don't create subdirectories when building current 231 source for a single board 232 full_path: Return the full path in CROSS_COMPILE and don't set 233 PATH 234 verbose_build: Run build with V=1 and don't use 'make -s' 235 incremental: Always perform incremental builds; don't run make 236 mrproper when configuring 237 per_board_out_dir: Build in a separate persistent directory per 238 board rather than a thread-specific directory 239 config_only: Only configure each build, don't build it 240 squash_config_y: Convert CONFIG options with the value 'y' to '1' 241 warnings_as_errors: Treat all compiler warnings as errors 242 """ 243 self.toolchains = toolchains 244 self.base_dir = base_dir 245 self._working_dir = os.path.join(base_dir, '.bm-work') 246 self.threads = [] 247 self.do_make = self.Make 248 self.gnu_make = gnu_make 249 self.checkout = checkout 250 self.num_threads = num_threads 251 self.num_jobs = num_jobs 252 self.already_done = 0 253 self.force_build = False 254 self.git_dir = git_dir 255 self._show_unknown = show_unknown 256 self._timestamp_count = 10 257 self._build_period_us = None 258 self._complete_delay = None 259 self._next_delay_update = datetime.now() 260 self.force_config_on_failure = True 261 self.force_build_failures = False 262 self.force_reconfig = False 263 self._step = step 264 self.in_tree = False 265 self._error_lines = 0 266 self.no_subdirs = no_subdirs 267 self.full_path = full_path 268 self.verbose_build = verbose_build 269 self.config_only = config_only 270 self.squash_config_y = squash_config_y 271 self.config_filenames = BASE_CONFIG_FILENAMES 272 if not self.squash_config_y: 273 self.config_filenames += EXTRA_CONFIG_FILENAMES 274 275 self.warnings_as_errors = warnings_as_errors 276 self.col = terminal.Color() 277 278 self._re_function = re.compile('(.*): In function.*') 279 self._re_files = re.compile('In file included from.*') 280 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 281 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 282 283 self.queue = Queue.Queue() 284 self.out_queue = Queue.Queue() 285 for i in range(self.num_threads): 286 t = builderthread.BuilderThread(self, i, incremental, 287 per_board_out_dir) 288 t.setDaemon(True) 289 t.start() 290 self.threads.append(t) 291 292 self.last_line_len = 0 293 t = builderthread.ResultThread(self) 294 t.setDaemon(True) 295 t.start() 296 self.threads.append(t) 297 298 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 299 self.re_make_err = re.compile('|'.join(ignore_lines)) 300 301 # Handle existing graceful with SIGINT / Ctrl-C 302 signal.signal(signal.SIGINT, self.signal_handler) 303 304 def __del__(self): 305 """Get rid of all threads created by the builder""" 306 for t in self.threads: 307 del t 308 309 def signal_handler(self, signal, frame): 310 sys.exit(1) 311 312 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 313 show_detail=False, show_bloat=False, 314 list_error_boards=False, show_config=False): 315 """Setup display options for the builder. 316 317 show_errors: True to show summarised error/warning info 318 show_sizes: Show size deltas 319 show_detail: Show detail for each board 320 show_bloat: Show detail for each function 321 list_error_boards: Show the boards which caused each error/warning 322 show_config: Show config deltas 323 """ 324 self._show_errors = show_errors 325 self._show_sizes = show_sizes 326 self._show_detail = show_detail 327 self._show_bloat = show_bloat 328 self._list_error_boards = list_error_boards 329 self._show_config = show_config 330 331 def _AddTimestamp(self): 332 """Add a new timestamp to the list and record the build period. 333 334 The build period is the length of time taken to perform a single 335 build (one board, one commit). 336 """ 337 now = datetime.now() 338 self._timestamps.append(now) 339 count = len(self._timestamps) 340 delta = self._timestamps[-1] - self._timestamps[0] 341 seconds = delta.total_seconds() 342 343 # If we have enough data, estimate build period (time taken for a 344 # single build) and therefore completion time. 345 if count > 1 and self._next_delay_update < now: 346 self._next_delay_update = now + timedelta(seconds=2) 347 if seconds > 0: 348 self._build_period = float(seconds) / count 349 todo = self.count - self.upto 350 self._complete_delay = timedelta(microseconds= 351 self._build_period * todo * 1000000) 352 # Round it 353 self._complete_delay -= timedelta( 354 microseconds=self._complete_delay.microseconds) 355 356 if seconds > 60: 357 self._timestamps.popleft() 358 count -= 1 359 360 def ClearLine(self, length): 361 """Clear any characters on the current line 362 363 Make way for a new line of length 'length', by outputting enough 364 spaces to clear out the old line. Then remember the new length for 365 next time. 366 367 Args: 368 length: Length of new line, in characters 369 """ 370 if length < self.last_line_len: 371 Print(' ' * (self.last_line_len - length), newline=False) 372 Print('\r', newline=False) 373 self.last_line_len = length 374 sys.stdout.flush() 375 376 def SelectCommit(self, commit, checkout=True): 377 """Checkout the selected commit for this build 378 """ 379 self.commit = commit 380 if checkout and self.checkout: 381 gitutil.Checkout(commit.hash) 382 383 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 384 """Run make 385 386 Args: 387 commit: Commit object that is being built 388 brd: Board object that is being built 389 stage: Stage that we are at (mrproper, config, build) 390 cwd: Directory where make should be run 391 args: Arguments to pass to make 392 kwargs: Arguments to pass to command.RunPipe() 393 """ 394 cmd = [self.gnu_make] + list(args) 395 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 396 cwd=cwd, raise_on_error=False, **kwargs) 397 if self.verbose_build: 398 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 399 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 400 return result 401 402 def ProcessResult(self, result): 403 """Process the result of a build, showing progress information 404 405 Args: 406 result: A CommandResult object, which indicates the result for 407 a single build 408 """ 409 col = terminal.Color() 410 if result: 411 target = result.brd.target 412 413 self.upto += 1 414 if result.return_code != 0: 415 self.fail += 1 416 elif result.stderr: 417 self.warned += 1 418 if result.already_done: 419 self.already_done += 1 420 if self._verbose: 421 Print('\r', newline=False) 422 self.ClearLine(0) 423 boards_selected = {target : result.brd} 424 self.ResetResultSummary(boards_selected) 425 self.ProduceResultSummary(result.commit_upto, self.commits, 426 boards_selected) 427 else: 428 target = '(starting)' 429 430 # Display separate counts for ok, warned and fail 431 ok = self.upto - self.warned - self.fail 432 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 433 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 434 line += self.col.Color(self.col.RED, '%5d' % self.fail) 435 436 name = ' /%-5d ' % self.count 437 438 # Add our current completion time estimate 439 self._AddTimestamp() 440 if self._complete_delay: 441 name += '%s : ' % self._complete_delay 442 # When building all boards for a commit, we can print a commit 443 # progress message. 444 if result and result.commit_upto is None: 445 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 446 self.commit_count) 447 448 name += target 449 Print(line + name, newline=False) 450 length = 16 + len(name) 451 self.ClearLine(length) 452 453 def _GetOutputDir(self, commit_upto): 454 """Get the name of the output directory for a commit number 455 456 The output directory is typically .../<branch>/<commit>. 457 458 Args: 459 commit_upto: Commit number to use (0..self.count-1) 460 """ 461 commit_dir = None 462 if self.commits: 463 commit = self.commits[commit_upto] 464 subject = commit.subject.translate(trans_valid_chars) 465 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 466 self.commit_count, commit.hash, subject[:20])) 467 elif not self.no_subdirs: 468 commit_dir = 'current' 469 if not commit_dir: 470 return self.base_dir 471 return os.path.join(self.base_dir, commit_dir) 472 473 def GetBuildDir(self, commit_upto, target): 474 """Get the name of the build directory for a commit number 475 476 The build directory is typically .../<branch>/<commit>/<target>. 477 478 Args: 479 commit_upto: Commit number to use (0..self.count-1) 480 target: Target name 481 """ 482 output_dir = self._GetOutputDir(commit_upto) 483 return os.path.join(output_dir, target) 484 485 def GetDoneFile(self, commit_upto, target): 486 """Get the name of the done file for a commit number 487 488 Args: 489 commit_upto: Commit number to use (0..self.count-1) 490 target: Target name 491 """ 492 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 493 494 def GetSizesFile(self, commit_upto, target): 495 """Get the name of the sizes file for a commit number 496 497 Args: 498 commit_upto: Commit number to use (0..self.count-1) 499 target: Target name 500 """ 501 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 502 503 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 504 """Get the name of the funcsizes file for a commit number and ELF file 505 506 Args: 507 commit_upto: Commit number to use (0..self.count-1) 508 target: Target name 509 elf_fname: Filename of elf image 510 """ 511 return os.path.join(self.GetBuildDir(commit_upto, target), 512 '%s.sizes' % elf_fname.replace('/', '-')) 513 514 def GetObjdumpFile(self, commit_upto, target, elf_fname): 515 """Get the name of the objdump file for a commit number and ELF file 516 517 Args: 518 commit_upto: Commit number to use (0..self.count-1) 519 target: Target name 520 elf_fname: Filename of elf image 521 """ 522 return os.path.join(self.GetBuildDir(commit_upto, target), 523 '%s.objdump' % elf_fname.replace('/', '-')) 524 525 def GetErrFile(self, commit_upto, target): 526 """Get the name of the err file for a commit number 527 528 Args: 529 commit_upto: Commit number to use (0..self.count-1) 530 target: Target name 531 """ 532 output_dir = self.GetBuildDir(commit_upto, target) 533 return os.path.join(output_dir, 'err') 534 535 def FilterErrors(self, lines): 536 """Filter out errors in which we have no interest 537 538 We should probably use map(). 539 540 Args: 541 lines: List of error lines, each a string 542 Returns: 543 New list with only interesting lines included 544 """ 545 out_lines = [] 546 for line in lines: 547 if not self.re_make_err.search(line): 548 out_lines.append(line) 549 return out_lines 550 551 def ReadFuncSizes(self, fname, fd): 552 """Read function sizes from the output of 'nm' 553 554 Args: 555 fd: File containing data to read 556 fname: Filename we are reading from (just for errors) 557 558 Returns: 559 Dictionary containing size of each function in bytes, indexed by 560 function name. 561 """ 562 sym = {} 563 for line in fd.readlines(): 564 try: 565 size, type, name = line[:-1].split() 566 except: 567 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 568 continue 569 if type in 'tTdDbB': 570 # function names begin with '.' on 64-bit powerpc 571 if '.' in name[1:]: 572 name = 'static.' + name.split('.')[0] 573 sym[name] = sym.get(name, 0) + int(size, 16) 574 return sym 575 576 def _ProcessConfig(self, fname): 577 """Read in a .config, autoconf.mk or autoconf.h file 578 579 This function handles all config file types. It ignores comments and 580 any #defines which don't start with CONFIG_. 581 582 Args: 583 fname: Filename to read 584 585 Returns: 586 Dictionary: 587 key: Config name (e.g. CONFIG_DM) 588 value: Config value (e.g. 1) 589 """ 590 config = {} 591 if os.path.exists(fname): 592 with open(fname) as fd: 593 for line in fd: 594 line = line.strip() 595 if line.startswith('#define'): 596 values = line[8:].split(' ', 1) 597 if len(values) > 1: 598 key, value = values 599 else: 600 key = values[0] 601 value = '1' if self.squash_config_y else '' 602 if not key.startswith('CONFIG_'): 603 continue 604 elif not line or line[0] in ['#', '*', '/']: 605 continue 606 else: 607 key, value = line.split('=', 1) 608 if self.squash_config_y and value == 'y': 609 value = '1' 610 config[key] = value 611 return config 612 613 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 614 read_config): 615 """Work out the outcome of a build. 616 617 Args: 618 commit_upto: Commit number to check (0..n-1) 619 target: Target board to check 620 read_func_sizes: True to read function size information 621 read_config: True to read .config and autoconf.h files 622 623 Returns: 624 Outcome object 625 """ 626 done_file = self.GetDoneFile(commit_upto, target) 627 sizes_file = self.GetSizesFile(commit_upto, target) 628 sizes = {} 629 func_sizes = {} 630 config = {} 631 if os.path.exists(done_file): 632 with open(done_file, 'r') as fd: 633 return_code = int(fd.readline()) 634 err_lines = [] 635 err_file = self.GetErrFile(commit_upto, target) 636 if os.path.exists(err_file): 637 with open(err_file, 'r') as fd: 638 err_lines = self.FilterErrors(fd.readlines()) 639 640 # Decide whether the build was ok, failed or created warnings 641 if return_code: 642 rc = OUTCOME_ERROR 643 elif len(err_lines): 644 rc = OUTCOME_WARNING 645 else: 646 rc = OUTCOME_OK 647 648 # Convert size information to our simple format 649 if os.path.exists(sizes_file): 650 with open(sizes_file, 'r') as fd: 651 for line in fd.readlines(): 652 values = line.split() 653 rodata = 0 654 if len(values) > 6: 655 rodata = int(values[6], 16) 656 size_dict = { 657 'all' : int(values[0]) + int(values[1]) + 658 int(values[2]), 659 'text' : int(values[0]) - rodata, 660 'data' : int(values[1]), 661 'bss' : int(values[2]), 662 'rodata' : rodata, 663 } 664 sizes[values[5]] = size_dict 665 666 if read_func_sizes: 667 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 668 for fname in glob.glob(pattern): 669 with open(fname, 'r') as fd: 670 dict_name = os.path.basename(fname).replace('.sizes', 671 '') 672 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 673 674 if read_config: 675 output_dir = self.GetBuildDir(commit_upto, target) 676 for name in self.config_filenames: 677 fname = os.path.join(output_dir, name) 678 config[name] = self._ProcessConfig(fname) 679 680 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 681 682 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 683 684 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 685 read_config): 686 """Calculate a summary of the results of building a commit. 687 688 Args: 689 board_selected: Dict containing boards to summarise 690 commit_upto: Commit number to summarize (0..self.count-1) 691 read_func_sizes: True to read function size information 692 read_config: True to read .config and autoconf.h files 693 694 Returns: 695 Tuple: 696 Dict containing boards which passed building this commit. 697 keyed by board.target 698 List containing a summary of error lines 699 Dict keyed by error line, containing a list of the Board 700 objects with that error 701 List containing a summary of warning lines 702 Dict keyed by error line, containing a list of the Board 703 objects with that warning 704 Dictionary keyed by board.target. Each value is a dictionary: 705 key: filename - e.g. '.config' 706 value is itself a dictionary: 707 key: config name 708 value: config value 709 """ 710 def AddLine(lines_summary, lines_boards, line, board): 711 line = line.rstrip() 712 if line in lines_boards: 713 lines_boards[line].append(board) 714 else: 715 lines_boards[line] = [board] 716 lines_summary.append(line) 717 718 board_dict = {} 719 err_lines_summary = [] 720 err_lines_boards = {} 721 warn_lines_summary = [] 722 warn_lines_boards = {} 723 config = {} 724 725 for board in boards_selected.itervalues(): 726 outcome = self.GetBuildOutcome(commit_upto, board.target, 727 read_func_sizes, read_config) 728 board_dict[board.target] = outcome 729 last_func = None 730 last_was_warning = False 731 for line in outcome.err_lines: 732 if line: 733 if (self._re_function.match(line) or 734 self._re_files.match(line)): 735 last_func = line 736 else: 737 is_warning = self._re_warning.match(line) 738 is_note = self._re_note.match(line) 739 if is_warning or (last_was_warning and is_note): 740 if last_func: 741 AddLine(warn_lines_summary, warn_lines_boards, 742 last_func, board) 743 AddLine(warn_lines_summary, warn_lines_boards, 744 line, board) 745 else: 746 if last_func: 747 AddLine(err_lines_summary, err_lines_boards, 748 last_func, board) 749 AddLine(err_lines_summary, err_lines_boards, 750 line, board) 751 last_was_warning = is_warning 752 last_func = None 753 tconfig = Config(self.config_filenames, board.target) 754 for fname in self.config_filenames: 755 if outcome.config: 756 for key, value in outcome.config[fname].iteritems(): 757 tconfig.Add(fname, key, value) 758 config[board.target] = tconfig 759 760 return (board_dict, err_lines_summary, err_lines_boards, 761 warn_lines_summary, warn_lines_boards, config) 762 763 def AddOutcome(self, board_dict, arch_list, changes, char, color): 764 """Add an output to our list of outcomes for each architecture 765 766 This simple function adds failing boards (changes) to the 767 relevant architecture string, so we can print the results out 768 sorted by architecture. 769 770 Args: 771 board_dict: Dict containing all boards 772 arch_list: Dict keyed by arch name. Value is a string containing 773 a list of board names which failed for that arch. 774 changes: List of boards to add to arch_list 775 color: terminal.Colour object 776 """ 777 done_arch = {} 778 for target in changes: 779 if target in board_dict: 780 arch = board_dict[target].arch 781 else: 782 arch = 'unknown' 783 str = self.col.Color(color, ' ' + target) 784 if not arch in done_arch: 785 str = ' %s %s' % (self.col.Color(color, char), str) 786 done_arch[arch] = True 787 if not arch in arch_list: 788 arch_list[arch] = str 789 else: 790 arch_list[arch] += str 791 792 793 def ColourNum(self, num): 794 color = self.col.RED if num > 0 else self.col.GREEN 795 if num == 0: 796 return '0' 797 return self.col.Color(color, str(num)) 798 799 def ResetResultSummary(self, board_selected): 800 """Reset the results summary ready for use. 801 802 Set up the base board list to be all those selected, and set the 803 error lines to empty. 804 805 Following this, calls to PrintResultSummary() will use this 806 information to work out what has changed. 807 808 Args: 809 board_selected: Dict containing boards to summarise, keyed by 810 board.target 811 """ 812 self._base_board_dict = {} 813 for board in board_selected: 814 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 815 self._base_err_lines = [] 816 self._base_warn_lines = [] 817 self._base_err_line_boards = {} 818 self._base_warn_line_boards = {} 819 self._base_config = None 820 821 def PrintFuncSizeDetail(self, fname, old, new): 822 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 823 delta, common = [], {} 824 825 for a in old: 826 if a in new: 827 common[a] = 1 828 829 for name in old: 830 if name not in common: 831 remove += 1 832 down += old[name] 833 delta.append([-old[name], name]) 834 835 for name in new: 836 if name not in common: 837 add += 1 838 up += new[name] 839 delta.append([new[name], name]) 840 841 for name in common: 842 diff = new.get(name, 0) - old.get(name, 0) 843 if diff > 0: 844 grow, up = grow + 1, up + diff 845 elif diff < 0: 846 shrink, down = shrink + 1, down - diff 847 delta.append([diff, name]) 848 849 delta.sort() 850 delta.reverse() 851 852 args = [add, -remove, grow, -shrink, up, -down, up - down] 853 if max(args) == 0 and min(args) == 0: 854 return 855 args = [self.ColourNum(x) for x in args] 856 indent = ' ' * 15 857 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 858 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 859 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 860 'delta')) 861 for diff, name in delta: 862 if diff: 863 color = self.col.RED if diff > 0 else self.col.GREEN 864 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 865 old.get(name, '-'), new.get(name,'-'), diff) 866 Print(msg, colour=color) 867 868 869 def PrintSizeDetail(self, target_list, show_bloat): 870 """Show details size information for each board 871 872 Args: 873 target_list: List of targets, each a dict containing: 874 'target': Target name 875 'total_diff': Total difference in bytes across all areas 876 <part_name>: Difference for that part 877 show_bloat: Show detail for each function 878 """ 879 targets_by_diff = sorted(target_list, reverse=True, 880 key=lambda x: x['_total_diff']) 881 for result in targets_by_diff: 882 printed_target = False 883 for name in sorted(result): 884 diff = result[name] 885 if name.startswith('_'): 886 continue 887 if diff != 0: 888 color = self.col.RED if diff > 0 else self.col.GREEN 889 msg = ' %s %+d' % (name, diff) 890 if not printed_target: 891 Print('%10s %-15s:' % ('', result['_target']), 892 newline=False) 893 printed_target = True 894 Print(msg, colour=color, newline=False) 895 if printed_target: 896 Print() 897 if show_bloat: 898 target = result['_target'] 899 outcome = result['_outcome'] 900 base_outcome = self._base_board_dict[target] 901 for fname in outcome.func_sizes: 902 self.PrintFuncSizeDetail(fname, 903 base_outcome.func_sizes[fname], 904 outcome.func_sizes[fname]) 905 906 907 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 908 show_bloat): 909 """Print a summary of image sizes broken down by section. 910 911 The summary takes the form of one line per architecture. The 912 line contains deltas for each of the sections (+ means the section 913 got bigger, - means smaller). The nunmbers are the average number 914 of bytes that a board in this section increased by. 915 916 For example: 917 powerpc: (622 boards) text -0.0 918 arm: (285 boards) text -0.0 919 nds32: (3 boards) text -8.0 920 921 Args: 922 board_selected: Dict containing boards to summarise, keyed by 923 board.target 924 board_dict: Dict containing boards for which we built this 925 commit, keyed by board.target. The value is an Outcome object. 926 show_detail: Show detail for each board 927 show_bloat: Show detail for each function 928 """ 929 arch_list = {} 930 arch_count = {} 931 932 # Calculate changes in size for different image parts 933 # The previous sizes are in Board.sizes, for each board 934 for target in board_dict: 935 if target not in board_selected: 936 continue 937 base_sizes = self._base_board_dict[target].sizes 938 outcome = board_dict[target] 939 sizes = outcome.sizes 940 941 # Loop through the list of images, creating a dict of size 942 # changes for each image/part. We end up with something like 943 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 944 # which means that U-Boot data increased by 5 bytes and SPL 945 # text decreased by 4. 946 err = {'_target' : target} 947 for image in sizes: 948 if image in base_sizes: 949 base_image = base_sizes[image] 950 # Loop through the text, data, bss parts 951 for part in sorted(sizes[image]): 952 diff = sizes[image][part] - base_image[part] 953 col = None 954 if diff: 955 if image == 'u-boot': 956 name = part 957 else: 958 name = image + ':' + part 959 err[name] = diff 960 arch = board_selected[target].arch 961 if not arch in arch_count: 962 arch_count[arch] = 1 963 else: 964 arch_count[arch] += 1 965 if not sizes: 966 pass # Only add to our list when we have some stats 967 elif not arch in arch_list: 968 arch_list[arch] = [err] 969 else: 970 arch_list[arch].append(err) 971 972 # We now have a list of image size changes sorted by arch 973 # Print out a summary of these 974 for arch, target_list in arch_list.iteritems(): 975 # Get total difference for each type 976 totals = {} 977 for result in target_list: 978 total = 0 979 for name, diff in result.iteritems(): 980 if name.startswith('_'): 981 continue 982 total += diff 983 if name in totals: 984 totals[name] += diff 985 else: 986 totals[name] = diff 987 result['_total_diff'] = total 988 result['_outcome'] = board_dict[result['_target']] 989 990 count = len(target_list) 991 printed_arch = False 992 for name in sorted(totals): 993 diff = totals[name] 994 if diff: 995 # Display the average difference in this name for this 996 # architecture 997 avg_diff = float(diff) / count 998 color = self.col.RED if avg_diff > 0 else self.col.GREEN 999 msg = ' %s %+1.1f' % (name, avg_diff) 1000 if not printed_arch: 1001 Print('%10s: (for %d/%d boards)' % (arch, count, 1002 arch_count[arch]), newline=False) 1003 printed_arch = True 1004 Print(msg, colour=color, newline=False) 1005 1006 if printed_arch: 1007 Print() 1008 if show_detail: 1009 self.PrintSizeDetail(target_list, show_bloat) 1010 1011 1012 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1013 err_line_boards, warn_lines, warn_line_boards, 1014 config, show_sizes, show_detail, show_bloat, 1015 show_config): 1016 """Compare results with the base results and display delta. 1017 1018 Only boards mentioned in board_selected will be considered. This 1019 function is intended to be called repeatedly with the results of 1020 each commit. It therefore shows a 'diff' between what it saw in 1021 the last call and what it sees now. 1022 1023 Args: 1024 board_selected: Dict containing boards to summarise, keyed by 1025 board.target 1026 board_dict: Dict containing boards for which we built this 1027 commit, keyed by board.target. The value is an Outcome object. 1028 err_lines: A list of errors for this commit, or [] if there is 1029 none, or we don't want to print errors 1030 err_line_boards: Dict keyed by error line, containing a list of 1031 the Board objects with that error 1032 warn_lines: A list of warnings for this commit, or [] if there is 1033 none, or we don't want to print errors 1034 warn_line_boards: Dict keyed by warning line, containing a list of 1035 the Board objects with that warning 1036 config: Dictionary keyed by filename - e.g. '.config'. Each 1037 value is itself a dictionary: 1038 key: config name 1039 value: config value 1040 show_sizes: Show image size deltas 1041 show_detail: Show detail for each board 1042 show_bloat: Show detail for each function 1043 show_config: Show config changes 1044 """ 1045 def _BoardList(line, line_boards): 1046 """Helper function to get a line of boards containing a line 1047 1048 Args: 1049 line: Error line to search for 1050 Return: 1051 String containing a list of boards with that error line, or 1052 '' if the user has not requested such a list 1053 """ 1054 if self._list_error_boards: 1055 names = [] 1056 for board in line_boards[line]: 1057 if not board.target in names: 1058 names.append(board.target) 1059 names_str = '(%s) ' % ','.join(names) 1060 else: 1061 names_str = '' 1062 return names_str 1063 1064 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1065 char): 1066 better_lines = [] 1067 worse_lines = [] 1068 for line in lines: 1069 if line not in base_lines: 1070 worse_lines.append(char + '+' + 1071 _BoardList(line, line_boards) + line) 1072 for line in base_lines: 1073 if line not in lines: 1074 better_lines.append(char + '-' + 1075 _BoardList(line, base_line_boards) + line) 1076 return better_lines, worse_lines 1077 1078 def _CalcConfig(delta, name, config): 1079 """Calculate configuration changes 1080 1081 Args: 1082 delta: Type of the delta, e.g. '+' 1083 name: name of the file which changed (e.g. .config) 1084 config: configuration change dictionary 1085 key: config name 1086 value: config value 1087 Returns: 1088 String containing the configuration changes which can be 1089 printed 1090 """ 1091 out = '' 1092 for key in sorted(config.keys()): 1093 out += '%s=%s ' % (key, config[key]) 1094 return '%s %s: %s' % (delta, name, out) 1095 1096 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1097 """Add changes in configuration to a list 1098 1099 Args: 1100 lines: list to add to 1101 name: config file name 1102 config_plus: configurations added, dictionary 1103 key: config name 1104 value: config value 1105 config_minus: configurations removed, dictionary 1106 key: config name 1107 value: config value 1108 config_change: configurations changed, dictionary 1109 key: config name 1110 value: config value 1111 """ 1112 if config_plus: 1113 lines.append(_CalcConfig('+', name, config_plus)) 1114 if config_minus: 1115 lines.append(_CalcConfig('-', name, config_minus)) 1116 if config_change: 1117 lines.append(_CalcConfig('c', name, config_change)) 1118 1119 def _OutputConfigInfo(lines): 1120 for line in lines: 1121 if not line: 1122 continue 1123 if line[0] == '+': 1124 col = self.col.GREEN 1125 elif line[0] == '-': 1126 col = self.col.RED 1127 elif line[0] == 'c': 1128 col = self.col.YELLOW 1129 Print(' ' + line, newline=True, colour=col) 1130 1131 1132 better = [] # List of boards fixed since last commit 1133 worse = [] # List of new broken boards since last commit 1134 new = [] # List of boards that didn't exist last time 1135 unknown = [] # List of boards that were not built 1136 1137 for target in board_dict: 1138 if target not in board_selected: 1139 continue 1140 1141 # If the board was built last time, add its outcome to a list 1142 if target in self._base_board_dict: 1143 base_outcome = self._base_board_dict[target].rc 1144 outcome = board_dict[target] 1145 if outcome.rc == OUTCOME_UNKNOWN: 1146 unknown.append(target) 1147 elif outcome.rc < base_outcome: 1148 better.append(target) 1149 elif outcome.rc > base_outcome: 1150 worse.append(target) 1151 else: 1152 new.append(target) 1153 1154 # Get a list of errors that have appeared, and disappeared 1155 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1156 self._base_err_line_boards, err_lines, err_line_boards, '') 1157 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1158 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1159 1160 # Display results by arch 1161 if (better or worse or unknown or new or worse_err or better_err 1162 or worse_warn or better_warn): 1163 arch_list = {} 1164 self.AddOutcome(board_selected, arch_list, better, '', 1165 self.col.GREEN) 1166 self.AddOutcome(board_selected, arch_list, worse, '+', 1167 self.col.RED) 1168 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1169 if self._show_unknown: 1170 self.AddOutcome(board_selected, arch_list, unknown, '?', 1171 self.col.MAGENTA) 1172 for arch, target_list in arch_list.iteritems(): 1173 Print('%10s: %s' % (arch, target_list)) 1174 self._error_lines += 1 1175 if better_err: 1176 Print('\n'.join(better_err), colour=self.col.GREEN) 1177 self._error_lines += 1 1178 if worse_err: 1179 Print('\n'.join(worse_err), colour=self.col.RED) 1180 self._error_lines += 1 1181 if better_warn: 1182 Print('\n'.join(better_warn), colour=self.col.CYAN) 1183 self._error_lines += 1 1184 if worse_warn: 1185 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1186 self._error_lines += 1 1187 1188 if show_sizes: 1189 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1190 show_bloat) 1191 1192 if show_config and self._base_config: 1193 summary = {} 1194 arch_config_plus = {} 1195 arch_config_minus = {} 1196 arch_config_change = {} 1197 arch_list = [] 1198 1199 for target in board_dict: 1200 if target not in board_selected: 1201 continue 1202 arch = board_selected[target].arch 1203 if arch not in arch_list: 1204 arch_list.append(arch) 1205 1206 for arch in arch_list: 1207 arch_config_plus[arch] = {} 1208 arch_config_minus[arch] = {} 1209 arch_config_change[arch] = {} 1210 for name in self.config_filenames: 1211 arch_config_plus[arch][name] = {} 1212 arch_config_minus[arch][name] = {} 1213 arch_config_change[arch][name] = {} 1214 1215 for target in board_dict: 1216 if target not in board_selected: 1217 continue 1218 1219 arch = board_selected[target].arch 1220 1221 all_config_plus = {} 1222 all_config_minus = {} 1223 all_config_change = {} 1224 tbase = self._base_config[target] 1225 tconfig = config[target] 1226 lines = [] 1227 for name in self.config_filenames: 1228 if not tconfig.config[name]: 1229 continue 1230 config_plus = {} 1231 config_minus = {} 1232 config_change = {} 1233 base = tbase.config[name] 1234 for key, value in tconfig.config[name].iteritems(): 1235 if key not in base: 1236 config_plus[key] = value 1237 all_config_plus[key] = value 1238 for key, value in base.iteritems(): 1239 if key not in tconfig.config[name]: 1240 config_minus[key] = value 1241 all_config_minus[key] = value 1242 for key, value in base.iteritems(): 1243 new_value = tconfig.config.get(key) 1244 if new_value and value != new_value: 1245 desc = '%s -> %s' % (value, new_value) 1246 config_change[key] = desc 1247 all_config_change[key] = desc 1248 1249 arch_config_plus[arch][name].update(config_plus) 1250 arch_config_minus[arch][name].update(config_minus) 1251 arch_config_change[arch][name].update(config_change) 1252 1253 _AddConfig(lines, name, config_plus, config_minus, 1254 config_change) 1255 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1256 all_config_change) 1257 summary[target] = '\n'.join(lines) 1258 1259 lines_by_target = {} 1260 for target, lines in summary.iteritems(): 1261 if lines in lines_by_target: 1262 lines_by_target[lines].append(target) 1263 else: 1264 lines_by_target[lines] = [target] 1265 1266 for arch in arch_list: 1267 lines = [] 1268 all_plus = {} 1269 all_minus = {} 1270 all_change = {} 1271 for name in self.config_filenames: 1272 all_plus.update(arch_config_plus[arch][name]) 1273 all_minus.update(arch_config_minus[arch][name]) 1274 all_change.update(arch_config_change[arch][name]) 1275 _AddConfig(lines, name, arch_config_plus[arch][name], 1276 arch_config_minus[arch][name], 1277 arch_config_change[arch][name]) 1278 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1279 #arch_summary[target] = '\n'.join(lines) 1280 if lines: 1281 Print('%s:' % arch) 1282 _OutputConfigInfo(lines) 1283 1284 for lines, targets in lines_by_target.iteritems(): 1285 if not lines: 1286 continue 1287 Print('%s :' % ' '.join(sorted(targets))) 1288 _OutputConfigInfo(lines.split('\n')) 1289 1290 1291 # Save our updated information for the next call to this function 1292 self._base_board_dict = board_dict 1293 self._base_err_lines = err_lines 1294 self._base_warn_lines = warn_lines 1295 self._base_err_line_boards = err_line_boards 1296 self._base_warn_line_boards = warn_line_boards 1297 self._base_config = config 1298 1299 # Get a list of boards that did not get built, if needed 1300 not_built = [] 1301 for board in board_selected: 1302 if not board in board_dict: 1303 not_built.append(board) 1304 if not_built: 1305 Print("Boards not built (%d): %s" % (len(not_built), 1306 ', '.join(not_built))) 1307 1308 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1309 (board_dict, err_lines, err_line_boards, warn_lines, 1310 warn_line_boards, config) = self.GetResultSummary( 1311 board_selected, commit_upto, 1312 read_func_sizes=self._show_bloat, 1313 read_config=self._show_config) 1314 if commits: 1315 msg = '%02d: %s' % (commit_upto + 1, 1316 commits[commit_upto].subject) 1317 Print(msg, colour=self.col.BLUE) 1318 self.PrintResultSummary(board_selected, board_dict, 1319 err_lines if self._show_errors else [], err_line_boards, 1320 warn_lines if self._show_errors else [], warn_line_boards, 1321 config, self._show_sizes, self._show_detail, 1322 self._show_bloat, self._show_config) 1323 1324 def ShowSummary(self, commits, board_selected): 1325 """Show a build summary for U-Boot for a given board list. 1326 1327 Reset the result summary, then repeatedly call GetResultSummary on 1328 each commit's results, then display the differences we see. 1329 1330 Args: 1331 commit: Commit objects to summarise 1332 board_selected: Dict containing boards to summarise 1333 """ 1334 self.commit_count = len(commits) if commits else 1 1335 self.commits = commits 1336 self.ResetResultSummary(board_selected) 1337 self._error_lines = 0 1338 1339 for commit_upto in range(0, self.commit_count, self._step): 1340 self.ProduceResultSummary(commit_upto, commits, board_selected) 1341 if not self._error_lines: 1342 Print('(no errors to report)', colour=self.col.GREEN) 1343 1344 1345 def SetupBuild(self, board_selected, commits): 1346 """Set up ready to start a build. 1347 1348 Args: 1349 board_selected: Selected boards to build 1350 commits: Selected commits to build 1351 """ 1352 # First work out how many commits we will build 1353 count = (self.commit_count + self._step - 1) / self._step 1354 self.count = len(board_selected) * count 1355 self.upto = self.warned = self.fail = 0 1356 self._timestamps = collections.deque() 1357 1358 def GetThreadDir(self, thread_num): 1359 """Get the directory path to the working dir for a thread. 1360 1361 Args: 1362 thread_num: Number of thread to check. 1363 """ 1364 return os.path.join(self._working_dir, '%02d' % thread_num) 1365 1366 def _PrepareThread(self, thread_num, setup_git): 1367 """Prepare the working directory for a thread. 1368 1369 This clones or fetches the repo into the thread's work directory. 1370 1371 Args: 1372 thread_num: Thread number (0, 1, ...) 1373 setup_git: True to set up a git repo clone 1374 """ 1375 thread_dir = self.GetThreadDir(thread_num) 1376 builderthread.Mkdir(thread_dir) 1377 git_dir = os.path.join(thread_dir, '.git') 1378 1379 # Clone the repo if it doesn't already exist 1380 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1381 # we have a private index but uses the origin repo's contents? 1382 if setup_git and self.git_dir: 1383 src_dir = os.path.abspath(self.git_dir) 1384 if os.path.exists(git_dir): 1385 gitutil.Fetch(git_dir, thread_dir) 1386 else: 1387 Print('\rCloning repo for thread %d' % thread_num, 1388 newline=False) 1389 gitutil.Clone(src_dir, thread_dir) 1390 Print('\r%s\r' % (' ' * 30), newline=False) 1391 1392 def _PrepareWorkingSpace(self, max_threads, setup_git): 1393 """Prepare the working directory for use. 1394 1395 Set up the git repo for each thread. 1396 1397 Args: 1398 max_threads: Maximum number of threads we expect to need. 1399 setup_git: True to set up a git repo clone 1400 """ 1401 builderthread.Mkdir(self._working_dir) 1402 for thread in range(max_threads): 1403 self._PrepareThread(thread, setup_git) 1404 1405 def _PrepareOutputSpace(self): 1406 """Get the output directories ready to receive files. 1407 1408 We delete any output directories which look like ones we need to 1409 create. Having left over directories is confusing when the user wants 1410 to check the output manually. 1411 """ 1412 if not self.commits: 1413 return 1414 dir_list = [] 1415 for commit_upto in range(self.commit_count): 1416 dir_list.append(self._GetOutputDir(commit_upto)) 1417 1418 to_remove = [] 1419 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1420 if dirname not in dir_list: 1421 to_remove.append(dirname) 1422 if to_remove: 1423 Print('Removing %d old build directories' % len(to_remove), 1424 newline=False) 1425 for dirname in to_remove: 1426 shutil.rmtree(dirname) 1427 1428 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1429 """Build all commits for a list of boards 1430 1431 Args: 1432 commits: List of commits to be build, each a Commit object 1433 boards_selected: Dict of selected boards, key is target name, 1434 value is Board object 1435 keep_outputs: True to save build output files 1436 verbose: Display build results as they are completed 1437 Returns: 1438 Tuple containing: 1439 - number of boards that failed to build 1440 - number of boards that issued warnings 1441 """ 1442 self.commit_count = len(commits) if commits else 1 1443 self.commits = commits 1444 self._verbose = verbose 1445 1446 self.ResetResultSummary(board_selected) 1447 builderthread.Mkdir(self.base_dir, parents = True) 1448 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1449 commits is not None) 1450 self._PrepareOutputSpace() 1451 Print('\rStarting build...', newline=False) 1452 self.SetupBuild(board_selected, commits) 1453 self.ProcessResult(None) 1454 1455 # Create jobs to build all commits for each board 1456 for brd in board_selected.itervalues(): 1457 job = builderthread.BuilderJob() 1458 job.board = brd 1459 job.commits = commits 1460 job.keep_outputs = keep_outputs 1461 job.step = self._step 1462 self.queue.put(job) 1463 1464 term = threading.Thread(target=self.queue.join) 1465 term.setDaemon(True) 1466 term.start() 1467 while term.isAlive(): 1468 term.join(100) 1469 1470 # Wait until we have processed all output 1471 self.out_queue.join() 1472 Print() 1473 self.ClearLine(0) 1474 return (self.fail, self.warned) 1475