1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2013 The Chromium OS Authors. 3# 4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 5# 6 7import collections 8from datetime import datetime, timedelta 9import glob 10import os 11import re 12import Queue 13import shutil 14import signal 15import string 16import sys 17import threading 18import time 19 20import builderthread 21import command 22import gitutil 23import terminal 24from terminal import Print 25import toolchain 26 27 28""" 29Theory of Operation 30 31Please see README for user documentation, and you should be familiar with 32that before trying to make sense of this. 33 34Buildman works by keeping the machine as busy as possible, building different 35commits for different boards on multiple CPUs at once. 36 37The source repo (self.git_dir) contains all the commits to be built. Each 38thread works on a single board at a time. It checks out the first commit, 39configures it for that board, then builds it. Then it checks out the next 40commit and builds it (typically without re-configuring). When it runs out 41of commits, it gets another job from the builder and starts again with that 42board. 43 44Clearly the builder threads could work either way - they could check out a 45commit and then built it for all boards. Using separate directories for each 46commit/board pair they could leave their build product around afterwards 47also. 48 49The intent behind building a single board for multiple commits, is to make 50use of incremental builds. Since each commit is built incrementally from 51the previous one, builds are faster. Reconfiguring for a different board 52removes all intermediate object files. 53 54Many threads can be working at once, but each has its own working directory. 55When a thread finishes a build, it puts the output files into a result 56directory. 57 58The base directory used by buildman is normally '../<branch>', i.e. 59a directory higher than the source repository and named after the branch 60being built. 61 62Within the base directory, we have one subdirectory for each commit. Within 63that is one subdirectory for each board. Within that is the build output for 64that commit/board combination. 65 66Buildman also create working directories for each thread, in a .bm-work/ 67subdirectory in the base dir. 68 69As an example, say we are building branch 'us-net' for boards 'sandbox' and 70'seaboard', and say that us-net has two commits. We will have directories 71like this: 72 73us-net/ base directory 74 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 75 sandbox/ 76 u-boot.bin 77 seaboard/ 78 u-boot.bin 79 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 80 sandbox/ 81 u-boot.bin 82 seaboard/ 83 u-boot.bin 84 .bm-work/ 85 00/ working directory for thread 0 (contains source checkout) 86 build/ build output 87 01/ working directory for thread 1 88 build/ build output 89 ... 90u-boot/ source directory 91 .git/ repository 92""" 93 94# Possible build outcomes 95OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 96 97# Translate a commit subject into a valid filename (and handle unicode) 98trans_valid_chars = string.maketrans('/: ', '---') 99trans_valid_chars = trans_valid_chars.decode('latin-1') 100 101BASE_CONFIG_FILENAMES = [ 102 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 103] 104 105EXTRA_CONFIG_FILENAMES = [ 106 '.config', '.config-spl', '.config-tpl', 107 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 108 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 109] 110 111class Config: 112 """Holds information about configuration settings for a board.""" 113 def __init__(self, config_filename, target): 114 self.target = target 115 self.config = {} 116 for fname in config_filename: 117 self.config[fname] = {} 118 119 def Add(self, fname, key, value): 120 self.config[fname][key] = value 121 122 def __hash__(self): 123 val = 0 124 for fname in self.config: 125 for key, value in self.config[fname].iteritems(): 126 print key, value 127 val = val ^ hash(key) & hash(value) 128 return val 129 130class Builder: 131 """Class for building U-Boot for a particular commit. 132 133 Public members: (many should ->private) 134 already_done: Number of builds already completed 135 base_dir: Base directory to use for builder 136 checkout: True to check out source, False to skip that step. 137 This is used for testing. 138 col: terminal.Color() object 139 count: Number of commits to build 140 do_make: Method to call to invoke Make 141 fail: Number of builds that failed due to error 142 force_build: Force building even if a build already exists 143 force_config_on_failure: If a commit fails for a board, disable 144 incremental building for the next commit we build for that 145 board, so that we will see all warnings/errors again. 146 force_build_failures: If a previously-built build (i.e. built on 147 a previous run of buildman) is marked as failed, rebuild it. 148 git_dir: Git directory containing source repository 149 last_line_len: Length of the last line we printed (used for erasing 150 it with new progress information) 151 num_jobs: Number of jobs to run at once (passed to make as -j) 152 num_threads: Number of builder threads to run 153 out_queue: Queue of results to process 154 re_make_err: Compiled regular expression for ignore_lines 155 queue: Queue of jobs to run 156 threads: List of active threads 157 toolchains: Toolchains object to use for building 158 upto: Current commit number we are building (0.count-1) 159 warned: Number of builds that produced at least one warning 160 force_reconfig: Reconfigure U-Boot on each comiit. This disables 161 incremental building, where buildman reconfigures on the first 162 commit for a baord, and then just does an incremental build for 163 the following commits. In fact buildman will reconfigure and 164 retry for any failing commits, so generally the only effect of 165 this option is to slow things down. 166 in_tree: Build U-Boot in-tree instead of specifying an output 167 directory separate from the source code. This option is really 168 only useful for testing in-tree builds. 169 170 Private members: 171 _base_board_dict: Last-summarised Dict of boards 172 _base_err_lines: Last-summarised list of errors 173 _base_warn_lines: Last-summarised list of warnings 174 _build_period_us: Time taken for a single build (float object). 175 _complete_delay: Expected delay until completion (timedelta) 176 _next_delay_update: Next time we plan to display a progress update 177 (datatime) 178 _show_unknown: Show unknown boards (those not built) in summary 179 _timestamps: List of timestamps for the completion of the last 180 last _timestamp_count builds. Each is a datetime object. 181 _timestamp_count: Number of timestamps to keep in our list. 182 _working_dir: Base working directory containing all threads 183 """ 184 class Outcome: 185 """Records a build outcome for a single make invocation 186 187 Public Members: 188 rc: Outcome value (OUTCOME_...) 189 err_lines: List of error lines or [] if none 190 sizes: Dictionary of image size information, keyed by filename 191 - Each value is itself a dictionary containing 192 values for 'text', 'data' and 'bss', being the integer 193 size in bytes of each section. 194 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 195 value is itself a dictionary: 196 key: function name 197 value: Size of function in bytes 198 config: Dictionary keyed by filename - e.g. '.config'. Each 199 value is itself a dictionary: 200 key: config name 201 value: config value 202 """ 203 def __init__(self, rc, err_lines, sizes, func_sizes, config): 204 self.rc = rc 205 self.err_lines = err_lines 206 self.sizes = sizes 207 self.func_sizes = func_sizes 208 self.config = config 209 210 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 211 gnu_make='make', checkout=True, show_unknown=True, step=1, 212 no_subdirs=False, full_path=False, verbose_build=False, 213 incremental=False, per_board_out_dir=False, 214 config_only=False, squash_config_y=False, 215 warnings_as_errors=False): 216 """Create a new Builder object 217 218 Args: 219 toolchains: Toolchains object to use for building 220 base_dir: Base directory to use for builder 221 git_dir: Git directory containing source repository 222 num_threads: Number of builder threads to run 223 num_jobs: Number of jobs to run at once (passed to make as -j) 224 gnu_make: the command name of GNU Make. 225 checkout: True to check out source, False to skip that step. 226 This is used for testing. 227 show_unknown: Show unknown boards (those not built) in summary 228 step: 1 to process every commit, n to process every nth commit 229 no_subdirs: Don't create subdirectories when building current 230 source for a single board 231 full_path: Return the full path in CROSS_COMPILE and don't set 232 PATH 233 verbose_build: Run build with V=1 and don't use 'make -s' 234 incremental: Always perform incremental builds; don't run make 235 mrproper when configuring 236 per_board_out_dir: Build in a separate persistent directory per 237 board rather than a thread-specific directory 238 config_only: Only configure each build, don't build it 239 squash_config_y: Convert CONFIG options with the value 'y' to '1' 240 warnings_as_errors: Treat all compiler warnings as errors 241 """ 242 self.toolchains = toolchains 243 self.base_dir = base_dir 244 self._working_dir = os.path.join(base_dir, '.bm-work') 245 self.threads = [] 246 self.do_make = self.Make 247 self.gnu_make = gnu_make 248 self.checkout = checkout 249 self.num_threads = num_threads 250 self.num_jobs = num_jobs 251 self.already_done = 0 252 self.force_build = False 253 self.git_dir = git_dir 254 self._show_unknown = show_unknown 255 self._timestamp_count = 10 256 self._build_period_us = None 257 self._complete_delay = None 258 self._next_delay_update = datetime.now() 259 self.force_config_on_failure = True 260 self.force_build_failures = False 261 self.force_reconfig = False 262 self._step = step 263 self.in_tree = False 264 self._error_lines = 0 265 self.no_subdirs = no_subdirs 266 self.full_path = full_path 267 self.verbose_build = verbose_build 268 self.config_only = config_only 269 self.squash_config_y = squash_config_y 270 self.config_filenames = BASE_CONFIG_FILENAMES 271 if not self.squash_config_y: 272 self.config_filenames += EXTRA_CONFIG_FILENAMES 273 274 self.warnings_as_errors = warnings_as_errors 275 self.col = terminal.Color() 276 277 self._re_function = re.compile('(.*): In function.*') 278 self._re_files = re.compile('In file included from.*') 279 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 280 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 281 282 self.queue = Queue.Queue() 283 self.out_queue = Queue.Queue() 284 for i in range(self.num_threads): 285 t = builderthread.BuilderThread(self, i, incremental, 286 per_board_out_dir) 287 t.setDaemon(True) 288 t.start() 289 self.threads.append(t) 290 291 self.last_line_len = 0 292 t = builderthread.ResultThread(self) 293 t.setDaemon(True) 294 t.start() 295 self.threads.append(t) 296 297 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 298 self.re_make_err = re.compile('|'.join(ignore_lines)) 299 300 # Handle existing graceful with SIGINT / Ctrl-C 301 signal.signal(signal.SIGINT, self.signal_handler) 302 303 def __del__(self): 304 """Get rid of all threads created by the builder""" 305 for t in self.threads: 306 del t 307 308 def signal_handler(self, signal, frame): 309 sys.exit(1) 310 311 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 312 show_detail=False, show_bloat=False, 313 list_error_boards=False, show_config=False): 314 """Setup display options for the builder. 315 316 show_errors: True to show summarised error/warning info 317 show_sizes: Show size deltas 318 show_detail: Show detail for each board 319 show_bloat: Show detail for each function 320 list_error_boards: Show the boards which caused each error/warning 321 show_config: Show config deltas 322 """ 323 self._show_errors = show_errors 324 self._show_sizes = show_sizes 325 self._show_detail = show_detail 326 self._show_bloat = show_bloat 327 self._list_error_boards = list_error_boards 328 self._show_config = show_config 329 330 def _AddTimestamp(self): 331 """Add a new timestamp to the list and record the build period. 332 333 The build period is the length of time taken to perform a single 334 build (one board, one commit). 335 """ 336 now = datetime.now() 337 self._timestamps.append(now) 338 count = len(self._timestamps) 339 delta = self._timestamps[-1] - self._timestamps[0] 340 seconds = delta.total_seconds() 341 342 # If we have enough data, estimate build period (time taken for a 343 # single build) and therefore completion time. 344 if count > 1 and self._next_delay_update < now: 345 self._next_delay_update = now + timedelta(seconds=2) 346 if seconds > 0: 347 self._build_period = float(seconds) / count 348 todo = self.count - self.upto 349 self._complete_delay = timedelta(microseconds= 350 self._build_period * todo * 1000000) 351 # Round it 352 self._complete_delay -= timedelta( 353 microseconds=self._complete_delay.microseconds) 354 355 if seconds > 60: 356 self._timestamps.popleft() 357 count -= 1 358 359 def ClearLine(self, length): 360 """Clear any characters on the current line 361 362 Make way for a new line of length 'length', by outputting enough 363 spaces to clear out the old line. Then remember the new length for 364 next time. 365 366 Args: 367 length: Length of new line, in characters 368 """ 369 if length < self.last_line_len: 370 Print(' ' * (self.last_line_len - length), newline=False) 371 Print('\r', newline=False) 372 self.last_line_len = length 373 sys.stdout.flush() 374 375 def SelectCommit(self, commit, checkout=True): 376 """Checkout the selected commit for this build 377 """ 378 self.commit = commit 379 if checkout and self.checkout: 380 gitutil.Checkout(commit.hash) 381 382 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 383 """Run make 384 385 Args: 386 commit: Commit object that is being built 387 brd: Board object that is being built 388 stage: Stage that we are at (mrproper, config, build) 389 cwd: Directory where make should be run 390 args: Arguments to pass to make 391 kwargs: Arguments to pass to command.RunPipe() 392 """ 393 cmd = [self.gnu_make] + list(args) 394 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 395 cwd=cwd, raise_on_error=False, **kwargs) 396 if self.verbose_build: 397 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 398 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 399 return result 400 401 def ProcessResult(self, result): 402 """Process the result of a build, showing progress information 403 404 Args: 405 result: A CommandResult object, which indicates the result for 406 a single build 407 """ 408 col = terminal.Color() 409 if result: 410 target = result.brd.target 411 412 self.upto += 1 413 if result.return_code != 0: 414 self.fail += 1 415 elif result.stderr: 416 self.warned += 1 417 if result.already_done: 418 self.already_done += 1 419 if self._verbose: 420 Print('\r', newline=False) 421 self.ClearLine(0) 422 boards_selected = {target : result.brd} 423 self.ResetResultSummary(boards_selected) 424 self.ProduceResultSummary(result.commit_upto, self.commits, 425 boards_selected) 426 else: 427 target = '(starting)' 428 429 # Display separate counts for ok, warned and fail 430 ok = self.upto - self.warned - self.fail 431 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 432 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 433 line += self.col.Color(self.col.RED, '%5d' % self.fail) 434 435 name = ' /%-5d ' % self.count 436 437 # Add our current completion time estimate 438 self._AddTimestamp() 439 if self._complete_delay: 440 name += '%s : ' % self._complete_delay 441 # When building all boards for a commit, we can print a commit 442 # progress message. 443 if result and result.commit_upto is None: 444 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 445 self.commit_count) 446 447 name += target 448 Print(line + name, newline=False) 449 length = 16 + len(name) 450 self.ClearLine(length) 451 452 def _GetOutputDir(self, commit_upto): 453 """Get the name of the output directory for a commit number 454 455 The output directory is typically .../<branch>/<commit>. 456 457 Args: 458 commit_upto: Commit number to use (0..self.count-1) 459 """ 460 commit_dir = None 461 if self.commits: 462 commit = self.commits[commit_upto] 463 subject = commit.subject.translate(trans_valid_chars) 464 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 465 self.commit_count, commit.hash, subject[:20])) 466 elif not self.no_subdirs: 467 commit_dir = 'current' 468 if not commit_dir: 469 return self.base_dir 470 return os.path.join(self.base_dir, commit_dir) 471 472 def GetBuildDir(self, commit_upto, target): 473 """Get the name of the build directory for a commit number 474 475 The build directory is typically .../<branch>/<commit>/<target>. 476 477 Args: 478 commit_upto: Commit number to use (0..self.count-1) 479 target: Target name 480 """ 481 output_dir = self._GetOutputDir(commit_upto) 482 return os.path.join(output_dir, target) 483 484 def GetDoneFile(self, commit_upto, target): 485 """Get the name of the done file for a commit number 486 487 Args: 488 commit_upto: Commit number to use (0..self.count-1) 489 target: Target name 490 """ 491 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 492 493 def GetSizesFile(self, commit_upto, target): 494 """Get the name of the sizes file for a commit number 495 496 Args: 497 commit_upto: Commit number to use (0..self.count-1) 498 target: Target name 499 """ 500 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 501 502 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 503 """Get the name of the funcsizes file for a commit number and ELF file 504 505 Args: 506 commit_upto: Commit number to use (0..self.count-1) 507 target: Target name 508 elf_fname: Filename of elf image 509 """ 510 return os.path.join(self.GetBuildDir(commit_upto, target), 511 '%s.sizes' % elf_fname.replace('/', '-')) 512 513 def GetObjdumpFile(self, commit_upto, target, elf_fname): 514 """Get the name of the objdump file for a commit number and ELF file 515 516 Args: 517 commit_upto: Commit number to use (0..self.count-1) 518 target: Target name 519 elf_fname: Filename of elf image 520 """ 521 return os.path.join(self.GetBuildDir(commit_upto, target), 522 '%s.objdump' % elf_fname.replace('/', '-')) 523 524 def GetErrFile(self, commit_upto, target): 525 """Get the name of the err file for a commit number 526 527 Args: 528 commit_upto: Commit number to use (0..self.count-1) 529 target: Target name 530 """ 531 output_dir = self.GetBuildDir(commit_upto, target) 532 return os.path.join(output_dir, 'err') 533 534 def FilterErrors(self, lines): 535 """Filter out errors in which we have no interest 536 537 We should probably use map(). 538 539 Args: 540 lines: List of error lines, each a string 541 Returns: 542 New list with only interesting lines included 543 """ 544 out_lines = [] 545 for line in lines: 546 if not self.re_make_err.search(line): 547 out_lines.append(line) 548 return out_lines 549 550 def ReadFuncSizes(self, fname, fd): 551 """Read function sizes from the output of 'nm' 552 553 Args: 554 fd: File containing data to read 555 fname: Filename we are reading from (just for errors) 556 557 Returns: 558 Dictionary containing size of each function in bytes, indexed by 559 function name. 560 """ 561 sym = {} 562 for line in fd.readlines(): 563 try: 564 size, type, name = line[:-1].split() 565 except: 566 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 567 continue 568 if type in 'tTdDbB': 569 # function names begin with '.' on 64-bit powerpc 570 if '.' in name[1:]: 571 name = 'static.' + name.split('.')[0] 572 sym[name] = sym.get(name, 0) + int(size, 16) 573 return sym 574 575 def _ProcessConfig(self, fname): 576 """Read in a .config, autoconf.mk or autoconf.h file 577 578 This function handles all config file types. It ignores comments and 579 any #defines which don't start with CONFIG_. 580 581 Args: 582 fname: Filename to read 583 584 Returns: 585 Dictionary: 586 key: Config name (e.g. CONFIG_DM) 587 value: Config value (e.g. 1) 588 """ 589 config = {} 590 if os.path.exists(fname): 591 with open(fname) as fd: 592 for line in fd: 593 line = line.strip() 594 if line.startswith('#define'): 595 values = line[8:].split(' ', 1) 596 if len(values) > 1: 597 key, value = values 598 else: 599 key = values[0] 600 value = '1' if self.squash_config_y else '' 601 if not key.startswith('CONFIG_'): 602 continue 603 elif not line or line[0] in ['#', '*', '/']: 604 continue 605 else: 606 key, value = line.split('=', 1) 607 if self.squash_config_y and value == 'y': 608 value = '1' 609 config[key] = value 610 return config 611 612 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 613 read_config): 614 """Work out the outcome of a build. 615 616 Args: 617 commit_upto: Commit number to check (0..n-1) 618 target: Target board to check 619 read_func_sizes: True to read function size information 620 read_config: True to read .config and autoconf.h files 621 622 Returns: 623 Outcome object 624 """ 625 done_file = self.GetDoneFile(commit_upto, target) 626 sizes_file = self.GetSizesFile(commit_upto, target) 627 sizes = {} 628 func_sizes = {} 629 config = {} 630 if os.path.exists(done_file): 631 with open(done_file, 'r') as fd: 632 return_code = int(fd.readline()) 633 err_lines = [] 634 err_file = self.GetErrFile(commit_upto, target) 635 if os.path.exists(err_file): 636 with open(err_file, 'r') as fd: 637 err_lines = self.FilterErrors(fd.readlines()) 638 639 # Decide whether the build was ok, failed or created warnings 640 if return_code: 641 rc = OUTCOME_ERROR 642 elif len(err_lines): 643 rc = OUTCOME_WARNING 644 else: 645 rc = OUTCOME_OK 646 647 # Convert size information to our simple format 648 if os.path.exists(sizes_file): 649 with open(sizes_file, 'r') as fd: 650 for line in fd.readlines(): 651 values = line.split() 652 rodata = 0 653 if len(values) > 6: 654 rodata = int(values[6], 16) 655 size_dict = { 656 'all' : int(values[0]) + int(values[1]) + 657 int(values[2]), 658 'text' : int(values[0]) - rodata, 659 'data' : int(values[1]), 660 'bss' : int(values[2]), 661 'rodata' : rodata, 662 } 663 sizes[values[5]] = size_dict 664 665 if read_func_sizes: 666 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 667 for fname in glob.glob(pattern): 668 with open(fname, 'r') as fd: 669 dict_name = os.path.basename(fname).replace('.sizes', 670 '') 671 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 672 673 if read_config: 674 output_dir = self.GetBuildDir(commit_upto, target) 675 for name in self.config_filenames: 676 fname = os.path.join(output_dir, name) 677 config[name] = self._ProcessConfig(fname) 678 679 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 680 681 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 682 683 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 684 read_config): 685 """Calculate a summary of the results of building a commit. 686 687 Args: 688 board_selected: Dict containing boards to summarise 689 commit_upto: Commit number to summarize (0..self.count-1) 690 read_func_sizes: True to read function size information 691 read_config: True to read .config and autoconf.h files 692 693 Returns: 694 Tuple: 695 Dict containing boards which passed building this commit. 696 keyed by board.target 697 List containing a summary of error lines 698 Dict keyed by error line, containing a list of the Board 699 objects with that error 700 List containing a summary of warning lines 701 Dict keyed by error line, containing a list of the Board 702 objects with that warning 703 Dictionary keyed by board.target. Each value is a dictionary: 704 key: filename - e.g. '.config' 705 value is itself a dictionary: 706 key: config name 707 value: config value 708 """ 709 def AddLine(lines_summary, lines_boards, line, board): 710 line = line.rstrip() 711 if line in lines_boards: 712 lines_boards[line].append(board) 713 else: 714 lines_boards[line] = [board] 715 lines_summary.append(line) 716 717 board_dict = {} 718 err_lines_summary = [] 719 err_lines_boards = {} 720 warn_lines_summary = [] 721 warn_lines_boards = {} 722 config = {} 723 724 for board in boards_selected.itervalues(): 725 outcome = self.GetBuildOutcome(commit_upto, board.target, 726 read_func_sizes, read_config) 727 board_dict[board.target] = outcome 728 last_func = None 729 last_was_warning = False 730 for line in outcome.err_lines: 731 if line: 732 if (self._re_function.match(line) or 733 self._re_files.match(line)): 734 last_func = line 735 else: 736 is_warning = self._re_warning.match(line) 737 is_note = self._re_note.match(line) 738 if is_warning or (last_was_warning and is_note): 739 if last_func: 740 AddLine(warn_lines_summary, warn_lines_boards, 741 last_func, board) 742 AddLine(warn_lines_summary, warn_lines_boards, 743 line, board) 744 else: 745 if last_func: 746 AddLine(err_lines_summary, err_lines_boards, 747 last_func, board) 748 AddLine(err_lines_summary, err_lines_boards, 749 line, board) 750 last_was_warning = is_warning 751 last_func = None 752 tconfig = Config(self.config_filenames, board.target) 753 for fname in self.config_filenames: 754 if outcome.config: 755 for key, value in outcome.config[fname].iteritems(): 756 tconfig.Add(fname, key, value) 757 config[board.target] = tconfig 758 759 return (board_dict, err_lines_summary, err_lines_boards, 760 warn_lines_summary, warn_lines_boards, config) 761 762 def AddOutcome(self, board_dict, arch_list, changes, char, color): 763 """Add an output to our list of outcomes for each architecture 764 765 This simple function adds failing boards (changes) to the 766 relevant architecture string, so we can print the results out 767 sorted by architecture. 768 769 Args: 770 board_dict: Dict containing all boards 771 arch_list: Dict keyed by arch name. Value is a string containing 772 a list of board names which failed for that arch. 773 changes: List of boards to add to arch_list 774 color: terminal.Colour object 775 """ 776 done_arch = {} 777 for target in changes: 778 if target in board_dict: 779 arch = board_dict[target].arch 780 else: 781 arch = 'unknown' 782 str = self.col.Color(color, ' ' + target) 783 if not arch in done_arch: 784 str = ' %s %s' % (self.col.Color(color, char), str) 785 done_arch[arch] = True 786 if not arch in arch_list: 787 arch_list[arch] = str 788 else: 789 arch_list[arch] += str 790 791 792 def ColourNum(self, num): 793 color = self.col.RED if num > 0 else self.col.GREEN 794 if num == 0: 795 return '0' 796 return self.col.Color(color, str(num)) 797 798 def ResetResultSummary(self, board_selected): 799 """Reset the results summary ready for use. 800 801 Set up the base board list to be all those selected, and set the 802 error lines to empty. 803 804 Following this, calls to PrintResultSummary() will use this 805 information to work out what has changed. 806 807 Args: 808 board_selected: Dict containing boards to summarise, keyed by 809 board.target 810 """ 811 self._base_board_dict = {} 812 for board in board_selected: 813 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 814 self._base_err_lines = [] 815 self._base_warn_lines = [] 816 self._base_err_line_boards = {} 817 self._base_warn_line_boards = {} 818 self._base_config = None 819 820 def PrintFuncSizeDetail(self, fname, old, new): 821 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 822 delta, common = [], {} 823 824 for a in old: 825 if a in new: 826 common[a] = 1 827 828 for name in old: 829 if name not in common: 830 remove += 1 831 down += old[name] 832 delta.append([-old[name], name]) 833 834 for name in new: 835 if name not in common: 836 add += 1 837 up += new[name] 838 delta.append([new[name], name]) 839 840 for name in common: 841 diff = new.get(name, 0) - old.get(name, 0) 842 if diff > 0: 843 grow, up = grow + 1, up + diff 844 elif diff < 0: 845 shrink, down = shrink + 1, down - diff 846 delta.append([diff, name]) 847 848 delta.sort() 849 delta.reverse() 850 851 args = [add, -remove, grow, -shrink, up, -down, up - down] 852 if max(args) == 0 and min(args) == 0: 853 return 854 args = [self.ColourNum(x) for x in args] 855 indent = ' ' * 15 856 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 857 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 858 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 859 'delta')) 860 for diff, name in delta: 861 if diff: 862 color = self.col.RED if diff > 0 else self.col.GREEN 863 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 864 old.get(name, '-'), new.get(name,'-'), diff) 865 Print(msg, colour=color) 866 867 868 def PrintSizeDetail(self, target_list, show_bloat): 869 """Show details size information for each board 870 871 Args: 872 target_list: List of targets, each a dict containing: 873 'target': Target name 874 'total_diff': Total difference in bytes across all areas 875 <part_name>: Difference for that part 876 show_bloat: Show detail for each function 877 """ 878 targets_by_diff = sorted(target_list, reverse=True, 879 key=lambda x: x['_total_diff']) 880 for result in targets_by_diff: 881 printed_target = False 882 for name in sorted(result): 883 diff = result[name] 884 if name.startswith('_'): 885 continue 886 if diff != 0: 887 color = self.col.RED if diff > 0 else self.col.GREEN 888 msg = ' %s %+d' % (name, diff) 889 if not printed_target: 890 Print('%10s %-15s:' % ('', result['_target']), 891 newline=False) 892 printed_target = True 893 Print(msg, colour=color, newline=False) 894 if printed_target: 895 Print() 896 if show_bloat: 897 target = result['_target'] 898 outcome = result['_outcome'] 899 base_outcome = self._base_board_dict[target] 900 for fname in outcome.func_sizes: 901 self.PrintFuncSizeDetail(fname, 902 base_outcome.func_sizes[fname], 903 outcome.func_sizes[fname]) 904 905 906 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 907 show_bloat): 908 """Print a summary of image sizes broken down by section. 909 910 The summary takes the form of one line per architecture. The 911 line contains deltas for each of the sections (+ means the section 912 got bigger, - means smaller). The nunmbers are the average number 913 of bytes that a board in this section increased by. 914 915 For example: 916 powerpc: (622 boards) text -0.0 917 arm: (285 boards) text -0.0 918 nds32: (3 boards) text -8.0 919 920 Args: 921 board_selected: Dict containing boards to summarise, keyed by 922 board.target 923 board_dict: Dict containing boards for which we built this 924 commit, keyed by board.target. The value is an Outcome object. 925 show_detail: Show detail for each board 926 show_bloat: Show detail for each function 927 """ 928 arch_list = {} 929 arch_count = {} 930 931 # Calculate changes in size for different image parts 932 # The previous sizes are in Board.sizes, for each board 933 for target in board_dict: 934 if target not in board_selected: 935 continue 936 base_sizes = self._base_board_dict[target].sizes 937 outcome = board_dict[target] 938 sizes = outcome.sizes 939 940 # Loop through the list of images, creating a dict of size 941 # changes for each image/part. We end up with something like 942 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 943 # which means that U-Boot data increased by 5 bytes and SPL 944 # text decreased by 4. 945 err = {'_target' : target} 946 for image in sizes: 947 if image in base_sizes: 948 base_image = base_sizes[image] 949 # Loop through the text, data, bss parts 950 for part in sorted(sizes[image]): 951 diff = sizes[image][part] - base_image[part] 952 col = None 953 if diff: 954 if image == 'u-boot': 955 name = part 956 else: 957 name = image + ':' + part 958 err[name] = diff 959 arch = board_selected[target].arch 960 if not arch in arch_count: 961 arch_count[arch] = 1 962 else: 963 arch_count[arch] += 1 964 if not sizes: 965 pass # Only add to our list when we have some stats 966 elif not arch in arch_list: 967 arch_list[arch] = [err] 968 else: 969 arch_list[arch].append(err) 970 971 # We now have a list of image size changes sorted by arch 972 # Print out a summary of these 973 for arch, target_list in arch_list.iteritems(): 974 # Get total difference for each type 975 totals = {} 976 for result in target_list: 977 total = 0 978 for name, diff in result.iteritems(): 979 if name.startswith('_'): 980 continue 981 total += diff 982 if name in totals: 983 totals[name] += diff 984 else: 985 totals[name] = diff 986 result['_total_diff'] = total 987 result['_outcome'] = board_dict[result['_target']] 988 989 count = len(target_list) 990 printed_arch = False 991 for name in sorted(totals): 992 diff = totals[name] 993 if diff: 994 # Display the average difference in this name for this 995 # architecture 996 avg_diff = float(diff) / count 997 color = self.col.RED if avg_diff > 0 else self.col.GREEN 998 msg = ' %s %+1.1f' % (name, avg_diff) 999 if not printed_arch: 1000 Print('%10s: (for %d/%d boards)' % (arch, count, 1001 arch_count[arch]), newline=False) 1002 printed_arch = True 1003 Print(msg, colour=color, newline=False) 1004 1005 if printed_arch: 1006 Print() 1007 if show_detail: 1008 self.PrintSizeDetail(target_list, show_bloat) 1009 1010 1011 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1012 err_line_boards, warn_lines, warn_line_boards, 1013 config, show_sizes, show_detail, show_bloat, 1014 show_config): 1015 """Compare results with the base results and display delta. 1016 1017 Only boards mentioned in board_selected will be considered. This 1018 function is intended to be called repeatedly with the results of 1019 each commit. It therefore shows a 'diff' between what it saw in 1020 the last call and what it sees now. 1021 1022 Args: 1023 board_selected: Dict containing boards to summarise, keyed by 1024 board.target 1025 board_dict: Dict containing boards for which we built this 1026 commit, keyed by board.target. The value is an Outcome object. 1027 err_lines: A list of errors for this commit, or [] if there is 1028 none, or we don't want to print errors 1029 err_line_boards: Dict keyed by error line, containing a list of 1030 the Board objects with that error 1031 warn_lines: A list of warnings for this commit, or [] if there is 1032 none, or we don't want to print errors 1033 warn_line_boards: Dict keyed by warning line, containing a list of 1034 the Board objects with that warning 1035 config: Dictionary keyed by filename - e.g. '.config'. Each 1036 value is itself a dictionary: 1037 key: config name 1038 value: config value 1039 show_sizes: Show image size deltas 1040 show_detail: Show detail for each board 1041 show_bloat: Show detail for each function 1042 show_config: Show config changes 1043 """ 1044 def _BoardList(line, line_boards): 1045 """Helper function to get a line of boards containing a line 1046 1047 Args: 1048 line: Error line to search for 1049 Return: 1050 String containing a list of boards with that error line, or 1051 '' if the user has not requested such a list 1052 """ 1053 if self._list_error_boards: 1054 names = [] 1055 for board in line_boards[line]: 1056 if not board.target in names: 1057 names.append(board.target) 1058 names_str = '(%s) ' % ','.join(names) 1059 else: 1060 names_str = '' 1061 return names_str 1062 1063 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1064 char): 1065 better_lines = [] 1066 worse_lines = [] 1067 for line in lines: 1068 if line not in base_lines: 1069 worse_lines.append(char + '+' + 1070 _BoardList(line, line_boards) + line) 1071 for line in base_lines: 1072 if line not in lines: 1073 better_lines.append(char + '-' + 1074 _BoardList(line, base_line_boards) + line) 1075 return better_lines, worse_lines 1076 1077 def _CalcConfig(delta, name, config): 1078 """Calculate configuration changes 1079 1080 Args: 1081 delta: Type of the delta, e.g. '+' 1082 name: name of the file which changed (e.g. .config) 1083 config: configuration change dictionary 1084 key: config name 1085 value: config value 1086 Returns: 1087 String containing the configuration changes which can be 1088 printed 1089 """ 1090 out = '' 1091 for key in sorted(config.keys()): 1092 out += '%s=%s ' % (key, config[key]) 1093 return '%s %s: %s' % (delta, name, out) 1094 1095 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1096 """Add changes in configuration to a list 1097 1098 Args: 1099 lines: list to add to 1100 name: config file name 1101 config_plus: configurations added, dictionary 1102 key: config name 1103 value: config value 1104 config_minus: configurations removed, dictionary 1105 key: config name 1106 value: config value 1107 config_change: configurations changed, dictionary 1108 key: config name 1109 value: config value 1110 """ 1111 if config_plus: 1112 lines.append(_CalcConfig('+', name, config_plus)) 1113 if config_minus: 1114 lines.append(_CalcConfig('-', name, config_minus)) 1115 if config_change: 1116 lines.append(_CalcConfig('c', name, config_change)) 1117 1118 def _OutputConfigInfo(lines): 1119 for line in lines: 1120 if not line: 1121 continue 1122 if line[0] == '+': 1123 col = self.col.GREEN 1124 elif line[0] == '-': 1125 col = self.col.RED 1126 elif line[0] == 'c': 1127 col = self.col.YELLOW 1128 Print(' ' + line, newline=True, colour=col) 1129 1130 1131 better = [] # List of boards fixed since last commit 1132 worse = [] # List of new broken boards since last commit 1133 new = [] # List of boards that didn't exist last time 1134 unknown = [] # List of boards that were not built 1135 1136 for target in board_dict: 1137 if target not in board_selected: 1138 continue 1139 1140 # If the board was built last time, add its outcome to a list 1141 if target in self._base_board_dict: 1142 base_outcome = self._base_board_dict[target].rc 1143 outcome = board_dict[target] 1144 if outcome.rc == OUTCOME_UNKNOWN: 1145 unknown.append(target) 1146 elif outcome.rc < base_outcome: 1147 better.append(target) 1148 elif outcome.rc > base_outcome: 1149 worse.append(target) 1150 else: 1151 new.append(target) 1152 1153 # Get a list of errors that have appeared, and disappeared 1154 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1155 self._base_err_line_boards, err_lines, err_line_boards, '') 1156 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1157 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1158 1159 # Display results by arch 1160 if (better or worse or unknown or new or worse_err or better_err 1161 or worse_warn or better_warn): 1162 arch_list = {} 1163 self.AddOutcome(board_selected, arch_list, better, '', 1164 self.col.GREEN) 1165 self.AddOutcome(board_selected, arch_list, worse, '+', 1166 self.col.RED) 1167 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1168 if self._show_unknown: 1169 self.AddOutcome(board_selected, arch_list, unknown, '?', 1170 self.col.MAGENTA) 1171 for arch, target_list in arch_list.iteritems(): 1172 Print('%10s: %s' % (arch, target_list)) 1173 self._error_lines += 1 1174 if better_err: 1175 Print('\n'.join(better_err), colour=self.col.GREEN) 1176 self._error_lines += 1 1177 if worse_err: 1178 Print('\n'.join(worse_err), colour=self.col.RED) 1179 self._error_lines += 1 1180 if better_warn: 1181 Print('\n'.join(better_warn), colour=self.col.CYAN) 1182 self._error_lines += 1 1183 if worse_warn: 1184 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1185 self._error_lines += 1 1186 1187 if show_sizes: 1188 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1189 show_bloat) 1190 1191 if show_config and self._base_config: 1192 summary = {} 1193 arch_config_plus = {} 1194 arch_config_minus = {} 1195 arch_config_change = {} 1196 arch_list = [] 1197 1198 for target in board_dict: 1199 if target not in board_selected: 1200 continue 1201 arch = board_selected[target].arch 1202 if arch not in arch_list: 1203 arch_list.append(arch) 1204 1205 for arch in arch_list: 1206 arch_config_plus[arch] = {} 1207 arch_config_minus[arch] = {} 1208 arch_config_change[arch] = {} 1209 for name in self.config_filenames: 1210 arch_config_plus[arch][name] = {} 1211 arch_config_minus[arch][name] = {} 1212 arch_config_change[arch][name] = {} 1213 1214 for target in board_dict: 1215 if target not in board_selected: 1216 continue 1217 1218 arch = board_selected[target].arch 1219 1220 all_config_plus = {} 1221 all_config_minus = {} 1222 all_config_change = {} 1223 tbase = self._base_config[target] 1224 tconfig = config[target] 1225 lines = [] 1226 for name in self.config_filenames: 1227 if not tconfig.config[name]: 1228 continue 1229 config_plus = {} 1230 config_minus = {} 1231 config_change = {} 1232 base = tbase.config[name] 1233 for key, value in tconfig.config[name].iteritems(): 1234 if key not in base: 1235 config_plus[key] = value 1236 all_config_plus[key] = value 1237 for key, value in base.iteritems(): 1238 if key not in tconfig.config[name]: 1239 config_minus[key] = value 1240 all_config_minus[key] = value 1241 for key, value in base.iteritems(): 1242 new_value = tconfig.config.get(key) 1243 if new_value and value != new_value: 1244 desc = '%s -> %s' % (value, new_value) 1245 config_change[key] = desc 1246 all_config_change[key] = desc 1247 1248 arch_config_plus[arch][name].update(config_plus) 1249 arch_config_minus[arch][name].update(config_minus) 1250 arch_config_change[arch][name].update(config_change) 1251 1252 _AddConfig(lines, name, config_plus, config_minus, 1253 config_change) 1254 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1255 all_config_change) 1256 summary[target] = '\n'.join(lines) 1257 1258 lines_by_target = {} 1259 for target, lines in summary.iteritems(): 1260 if lines in lines_by_target: 1261 lines_by_target[lines].append(target) 1262 else: 1263 lines_by_target[lines] = [target] 1264 1265 for arch in arch_list: 1266 lines = [] 1267 all_plus = {} 1268 all_minus = {} 1269 all_change = {} 1270 for name in self.config_filenames: 1271 all_plus.update(arch_config_plus[arch][name]) 1272 all_minus.update(arch_config_minus[arch][name]) 1273 all_change.update(arch_config_change[arch][name]) 1274 _AddConfig(lines, name, arch_config_plus[arch][name], 1275 arch_config_minus[arch][name], 1276 arch_config_change[arch][name]) 1277 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1278 #arch_summary[target] = '\n'.join(lines) 1279 if lines: 1280 Print('%s:' % arch) 1281 _OutputConfigInfo(lines) 1282 1283 for lines, targets in lines_by_target.iteritems(): 1284 if not lines: 1285 continue 1286 Print('%s :' % ' '.join(sorted(targets))) 1287 _OutputConfigInfo(lines.split('\n')) 1288 1289 1290 # Save our updated information for the next call to this function 1291 self._base_board_dict = board_dict 1292 self._base_err_lines = err_lines 1293 self._base_warn_lines = warn_lines 1294 self._base_err_line_boards = err_line_boards 1295 self._base_warn_line_boards = warn_line_boards 1296 self._base_config = config 1297 1298 # Get a list of boards that did not get built, if needed 1299 not_built = [] 1300 for board in board_selected: 1301 if not board in board_dict: 1302 not_built.append(board) 1303 if not_built: 1304 Print("Boards not built (%d): %s" % (len(not_built), 1305 ', '.join(not_built))) 1306 1307 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1308 (board_dict, err_lines, err_line_boards, warn_lines, 1309 warn_line_boards, config) = self.GetResultSummary( 1310 board_selected, commit_upto, 1311 read_func_sizes=self._show_bloat, 1312 read_config=self._show_config) 1313 if commits: 1314 msg = '%02d: %s' % (commit_upto + 1, 1315 commits[commit_upto].subject) 1316 Print(msg, colour=self.col.BLUE) 1317 self.PrintResultSummary(board_selected, board_dict, 1318 err_lines if self._show_errors else [], err_line_boards, 1319 warn_lines if self._show_errors else [], warn_line_boards, 1320 config, self._show_sizes, self._show_detail, 1321 self._show_bloat, self._show_config) 1322 1323 def ShowSummary(self, commits, board_selected): 1324 """Show a build summary for U-Boot for a given board list. 1325 1326 Reset the result summary, then repeatedly call GetResultSummary on 1327 each commit's results, then display the differences we see. 1328 1329 Args: 1330 commit: Commit objects to summarise 1331 board_selected: Dict containing boards to summarise 1332 """ 1333 self.commit_count = len(commits) if commits else 1 1334 self.commits = commits 1335 self.ResetResultSummary(board_selected) 1336 self._error_lines = 0 1337 1338 for commit_upto in range(0, self.commit_count, self._step): 1339 self.ProduceResultSummary(commit_upto, commits, board_selected) 1340 if not self._error_lines: 1341 Print('(no errors to report)', colour=self.col.GREEN) 1342 1343 1344 def SetupBuild(self, board_selected, commits): 1345 """Set up ready to start a build. 1346 1347 Args: 1348 board_selected: Selected boards to build 1349 commits: Selected commits to build 1350 """ 1351 # First work out how many commits we will build 1352 count = (self.commit_count + self._step - 1) / self._step 1353 self.count = len(board_selected) * count 1354 self.upto = self.warned = self.fail = 0 1355 self._timestamps = collections.deque() 1356 1357 def GetThreadDir(self, thread_num): 1358 """Get the directory path to the working dir for a thread. 1359 1360 Args: 1361 thread_num: Number of thread to check. 1362 """ 1363 return os.path.join(self._working_dir, '%02d' % thread_num) 1364 1365 def _PrepareThread(self, thread_num, setup_git): 1366 """Prepare the working directory for a thread. 1367 1368 This clones or fetches the repo into the thread's work directory. 1369 1370 Args: 1371 thread_num: Thread number (0, 1, ...) 1372 setup_git: True to set up a git repo clone 1373 """ 1374 thread_dir = self.GetThreadDir(thread_num) 1375 builderthread.Mkdir(thread_dir) 1376 git_dir = os.path.join(thread_dir, '.git') 1377 1378 # Clone the repo if it doesn't already exist 1379 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1380 # we have a private index but uses the origin repo's contents? 1381 if setup_git and self.git_dir: 1382 src_dir = os.path.abspath(self.git_dir) 1383 if os.path.exists(git_dir): 1384 gitutil.Fetch(git_dir, thread_dir) 1385 else: 1386 Print('\rCloning repo for thread %d' % thread_num, 1387 newline=False) 1388 gitutil.Clone(src_dir, thread_dir) 1389 Print('\r%s\r' % (' ' * 30), newline=False) 1390 1391 def _PrepareWorkingSpace(self, max_threads, setup_git): 1392 """Prepare the working directory for use. 1393 1394 Set up the git repo for each thread. 1395 1396 Args: 1397 max_threads: Maximum number of threads we expect to need. 1398 setup_git: True to set up a git repo clone 1399 """ 1400 builderthread.Mkdir(self._working_dir) 1401 for thread in range(max_threads): 1402 self._PrepareThread(thread, setup_git) 1403 1404 def _PrepareOutputSpace(self): 1405 """Get the output directories ready to receive files. 1406 1407 We delete any output directories which look like ones we need to 1408 create. Having left over directories is confusing when the user wants 1409 to check the output manually. 1410 """ 1411 if not self.commits: 1412 return 1413 dir_list = [] 1414 for commit_upto in range(self.commit_count): 1415 dir_list.append(self._GetOutputDir(commit_upto)) 1416 1417 to_remove = [] 1418 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1419 if dirname not in dir_list: 1420 to_remove.append(dirname) 1421 if to_remove: 1422 Print('Removing %d old build directories' % len(to_remove), 1423 newline=False) 1424 for dirname in to_remove: 1425 shutil.rmtree(dirname) 1426 1427 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1428 """Build all commits for a list of boards 1429 1430 Args: 1431 commits: List of commits to be build, each a Commit object 1432 boards_selected: Dict of selected boards, key is target name, 1433 value is Board object 1434 keep_outputs: True to save build output files 1435 verbose: Display build results as they are completed 1436 Returns: 1437 Tuple containing: 1438 - number of boards that failed to build 1439 - number of boards that issued warnings 1440 """ 1441 self.commit_count = len(commits) if commits else 1 1442 self.commits = commits 1443 self._verbose = verbose 1444 1445 self.ResetResultSummary(board_selected) 1446 builderthread.Mkdir(self.base_dir, parents = True) 1447 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1448 commits is not None) 1449 self._PrepareOutputSpace() 1450 Print('\rStarting build...', newline=False) 1451 self.SetupBuild(board_selected, commits) 1452 self.ProcessResult(None) 1453 1454 # Create jobs to build all commits for each board 1455 for brd in board_selected.itervalues(): 1456 job = builderthread.BuilderJob() 1457 job.board = brd 1458 job.commits = commits 1459 job.keep_outputs = keep_outputs 1460 job.step = self._step 1461 self.queue.put(job) 1462 1463 term = threading.Thread(target=self.queue.join) 1464 term.setDaemon(True) 1465 term.start() 1466 while term.isAlive(): 1467 term.join(100) 1468 1469 # Wait until we have processed all output 1470 self.out_queue.join() 1471 Print() 1472 self.ClearLine(0) 1473 return (self.fail, self.warned) 1474