1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, 192 force_build_failures): 193 """Build a particular commit. 194 195 If the build is already done, and we are not forcing a build, we skip 196 the build and just return the previously-saved results. 197 198 Args: 199 commit_upto: Commit number to build (0...n-1) 200 brd: Board object to build 201 work_dir: Directory to which the source will be checked out 202 do_config: True to run a make <board>_defconfig on the source 203 force_build: Force a build even if one was previously done 204 force_build_failures: Force a bulid if the previous result showed 205 failure 206 207 Returns: 208 tuple containing: 209 - CommandResult object containing the results of the build 210 - boolean indicating whether 'make config' is still needed 211 """ 212 # Create a default result - it will be overwritte by the call to 213 # self.Make() below, in the event that we do a build. 214 result = command.CommandResult() 215 result.return_code = 0 216 if self.builder.in_tree: 217 out_dir = work_dir 218 else: 219 out_dir = os.path.join(work_dir, 'build') 220 221 # Check if the job was already completed last time 222 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 223 result.already_done = os.path.exists(done_file) 224 will_build = (force_build or force_build_failures or 225 not result.already_done) 226 if result.already_done and will_build: 227 # Get the return code from that build and use it 228 with open(done_file, 'r') as fd: 229 result.return_code = int(fd.readline()) 230 err_file = self.builder.GetErrFile(commit_upto, brd.target) 231 if os.path.exists(err_file) and os.stat(err_file).st_size: 232 result.stderr = 'bad' 233 elif not force_build: 234 # The build passed, so no need to build it again 235 will_build = False 236 237 if will_build: 238 # We are going to have to build it. First, get a toolchain 239 if not self.toolchain: 240 try: 241 self.toolchain = self.builder.toolchains.Select(brd.arch) 242 except ValueError as err: 243 result.return_code = 10 244 result.stdout = '' 245 result.stderr = str(err) 246 # TODO(sjg@chromium.org): This gets swallowed, but needs 247 # to be reported. 248 249 if self.toolchain: 250 # Checkout the right commit 251 if commit_upto is not None: 252 commit = self.builder.commits[commit_upto] 253 if self.builder.checkout: 254 git_dir = os.path.join(work_dir, '.git') 255 gitutil.Checkout(commit.hash, git_dir, work_dir, 256 force=True) 257 else: 258 commit = self.builder.commit # Ick, fix this for BuildCommits() 259 260 # Set up the environment and command line 261 env = self.toolchain.MakeEnvironment() 262 Mkdir(out_dir) 263 args = [] 264 if not self.builder.in_tree: 265 args.append('O=build') 266 args.append('-s') 267 if self.builder.num_jobs is not None: 268 args.extend(['-j', str(self.builder.num_jobs)]) 269 config_args = ['%s_defconfig' % brd.target] 270 config_out = '' 271 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 272 273 # If we need to reconfigure, do that now 274 if do_config: 275 result = self.Make(commit, brd, 'distclean', work_dir, 276 'distclean', *args, env=env) 277 result = self.Make(commit, brd, 'config', work_dir, 278 *(args + config_args), env=env) 279 config_out = result.combined 280 do_config = False # No need to configure next time 281 if result.return_code == 0: 282 result = self.Make(commit, brd, 'build', work_dir, *args, 283 env=env) 284 result.stdout = config_out + result.stdout 285 else: 286 result.return_code = 1 287 result.stderr = 'No tool chain for %s\n' % brd.arch 288 result.already_done = False 289 290 result.toolchain = self.toolchain 291 result.brd = brd 292 result.commit_upto = commit_upto 293 result.out_dir = out_dir 294 return result, do_config 295 296 def _WriteResult(self, result, keep_outputs): 297 """Write a built result to the output directory. 298 299 Args: 300 result: CommandResult object containing result to write 301 keep_outputs: True to store the output binaries, False 302 to delete them 303 """ 304 # Fatal error 305 if result.return_code < 0: 306 return 307 308 # Aborted? 309 if result.stderr and 'No child processes' in result.stderr: 310 return 311 312 if result.already_done: 313 return 314 315 # Write the output and stderr 316 output_dir = self.builder._GetOutputDir(result.commit_upto) 317 Mkdir(output_dir) 318 build_dir = self.builder.GetBuildDir(result.commit_upto, 319 result.brd.target) 320 Mkdir(build_dir) 321 322 outfile = os.path.join(build_dir, 'log') 323 with open(outfile, 'w') as fd: 324 if result.stdout: 325 fd.write(result.stdout) 326 327 errfile = self.builder.GetErrFile(result.commit_upto, 328 result.brd.target) 329 if result.stderr: 330 with open(errfile, 'w') as fd: 331 fd.write(result.stderr) 332 elif os.path.exists(errfile): 333 os.remove(errfile) 334 335 if result.toolchain: 336 # Write the build result and toolchain information. 337 done_file = self.builder.GetDoneFile(result.commit_upto, 338 result.brd.target) 339 with open(done_file, 'w') as fd: 340 fd.write('%s' % result.return_code) 341 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 342 print >>fd, 'gcc', result.toolchain.gcc 343 print >>fd, 'path', result.toolchain.path 344 print >>fd, 'cross', result.toolchain.cross 345 print >>fd, 'arch', result.toolchain.arch 346 fd.write('%s' % result.return_code) 347 348 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 349 print >>fd, 'gcc', result.toolchain.gcc 350 print >>fd, 'path', result.toolchain.path 351 352 # Write out the image and function size information and an objdump 353 env = result.toolchain.MakeEnvironment() 354 lines = [] 355 for fname in ['u-boot', 'spl/u-boot-spl']: 356 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 357 nm_result = command.RunPipe([cmd], capture=True, 358 capture_stderr=True, cwd=result.out_dir, 359 raise_on_error=False, env=env) 360 if nm_result.stdout: 361 nm = self.builder.GetFuncSizesFile(result.commit_upto, 362 result.brd.target, fname) 363 with open(nm, 'w') as fd: 364 print >>fd, nm_result.stdout, 365 366 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 367 dump_result = command.RunPipe([cmd], capture=True, 368 capture_stderr=True, cwd=result.out_dir, 369 raise_on_error=False, env=env) 370 rodata_size = '' 371 if dump_result.stdout: 372 objdump = self.builder.GetObjdumpFile(result.commit_upto, 373 result.brd.target, fname) 374 with open(objdump, 'w') as fd: 375 print >>fd, dump_result.stdout, 376 for line in dump_result.stdout.splitlines(): 377 fields = line.split() 378 if len(fields) > 5 and fields[1] == '.rodata': 379 rodata_size = fields[2] 380 381 cmd = ['%ssize' % self.toolchain.cross, fname] 382 size_result = command.RunPipe([cmd], capture=True, 383 capture_stderr=True, cwd=result.out_dir, 384 raise_on_error=False, env=env) 385 if size_result.stdout: 386 lines.append(size_result.stdout.splitlines()[1] + ' ' + 387 rodata_size) 388 389 # Write out the image sizes file. This is similar to the output 390 # of binutil's 'size' utility, but it omits the header line and 391 # adds an additional hex value at the end of each line for the 392 # rodata size 393 if len(lines): 394 sizes = self.builder.GetSizesFile(result.commit_upto, 395 result.brd.target) 396 with open(sizes, 'w') as fd: 397 print >>fd, '\n'.join(lines) 398 399 # Now write the actual build output 400 if keep_outputs: 401 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 402 'include/autoconf.mk', 'spl/u-boot-spl', 403 'spl/u-boot-spl.bin'] 404 for pattern in patterns: 405 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 406 for fname in file_list: 407 shutil.copy(fname, build_dir) 408 409 410 def RunJob(self, job): 411 """Run a single job 412 413 A job consists of a building a list of commits for a particular board. 414 415 Args: 416 job: Job to build 417 """ 418 brd = job.board 419 work_dir = self.builder.GetThreadDir(self.thread_num) 420 self.toolchain = None 421 if job.commits: 422 # Run 'make board_defconfig' on the first commit 423 do_config = True 424 commit_upto = 0 425 force_build = False 426 for commit_upto in range(0, len(job.commits), job.step): 427 result, request_config = self.RunCommit(commit_upto, brd, 428 work_dir, do_config, 429 force_build or self.builder.force_build, 430 self.builder.force_build_failures) 431 failed = result.return_code or result.stderr 432 did_config = do_config 433 if failed and not do_config: 434 # If our incremental build failed, try building again 435 # with a reconfig. 436 if self.builder.force_config_on_failure: 437 result, request_config = self.RunCommit(commit_upto, 438 brd, work_dir, True, True, False) 439 did_config = True 440 if not self.builder.force_reconfig: 441 do_config = request_config 442 443 # If we built that commit, then config is done. But if we got 444 # an warning, reconfig next time to force it to build the same 445 # files that created warnings this time. Otherwise an 446 # incremental build may not build the same file, and we will 447 # think that the warning has gone away. 448 # We could avoid this by using -Werror everywhere... 449 # For errors, the problem doesn't happen, since presumably 450 # the build stopped and didn't generate output, so will retry 451 # that file next time. So we could detect warnings and deal 452 # with them specially here. For now, we just reconfigure if 453 # anything goes work. 454 # Of course this is substantially slower if there are build 455 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 456 # have problems). 457 if (failed and not result.already_done and not did_config and 458 self.builder.force_config_on_failure): 459 # If this build failed, try the next one with a 460 # reconfigure. 461 # Sometimes if the board_config.h file changes it can mess 462 # with dependencies, and we get: 463 # make: *** No rule to make target `include/autoconf.mk', 464 # needed by `depend'. 465 do_config = True 466 force_build = True 467 else: 468 force_build = False 469 if self.builder.force_config_on_failure: 470 if failed: 471 do_config = True 472 result.commit_upto = commit_upto 473 if result.return_code < 0: 474 raise ValueError('Interrupt') 475 476 # We have the build results, so output the result 477 self._WriteResult(result, job.keep_outputs) 478 self.builder.out_queue.put(result) 479 else: 480 # Just build the currently checked-out build 481 result = self.RunCommit(None, True) 482 result.commit_upto = self.builder.upto 483 self.builder.out_queue.put(result) 484 485 def run(self): 486 """Our thread's run function 487 488 This thread picks a job from the queue, runs it, and then goes to the 489 next job. 490 """ 491 alive = True 492 while True: 493 job = self.builder.queue.get() 494 try: 495 if self.builder.active and alive: 496 self.RunJob(job) 497 except Exception as err: 498 alive = False 499 print err 500 self.builder.queue.task_done() 501 502 503class Builder: 504 """Class for building U-Boot for a particular commit. 505 506 Public members: (many should ->private) 507 active: True if the builder is active and has not been stopped 508 already_done: Number of builds already completed 509 base_dir: Base directory to use for builder 510 checkout: True to check out source, False to skip that step. 511 This is used for testing. 512 col: terminal.Color() object 513 count: Number of commits to build 514 do_make: Method to call to invoke Make 515 fail: Number of builds that failed due to error 516 force_build: Force building even if a build already exists 517 force_config_on_failure: If a commit fails for a board, disable 518 incremental building for the next commit we build for that 519 board, so that we will see all warnings/errors again. 520 force_build_failures: If a previously-built build (i.e. built on 521 a previous run of buildman) is marked as failed, rebuild it. 522 git_dir: Git directory containing source repository 523 last_line_len: Length of the last line we printed (used for erasing 524 it with new progress information) 525 num_jobs: Number of jobs to run at once (passed to make as -j) 526 num_threads: Number of builder threads to run 527 out_queue: Queue of results to process 528 re_make_err: Compiled regular expression for ignore_lines 529 queue: Queue of jobs to run 530 threads: List of active threads 531 toolchains: Toolchains object to use for building 532 upto: Current commit number we are building (0.count-1) 533 warned: Number of builds that produced at least one warning 534 force_reconfig: Reconfigure U-Boot on each comiit. This disables 535 incremental building, where buildman reconfigures on the first 536 commit for a baord, and then just does an incremental build for 537 the following commits. In fact buildman will reconfigure and 538 retry for any failing commits, so generally the only effect of 539 this option is to slow things down. 540 in_tree: Build U-Boot in-tree instead of specifying an output 541 directory separate from the source code. This option is really 542 only useful for testing in-tree builds. 543 544 Private members: 545 _base_board_dict: Last-summarised Dict of boards 546 _base_err_lines: Last-summarised list of errors 547 _build_period_us: Time taken for a single build (float object). 548 _complete_delay: Expected delay until completion (timedelta) 549 _next_delay_update: Next time we plan to display a progress update 550 (datatime) 551 _show_unknown: Show unknown boards (those not built) in summary 552 _timestamps: List of timestamps for the completion of the last 553 last _timestamp_count builds. Each is a datetime object. 554 _timestamp_count: Number of timestamps to keep in our list. 555 _working_dir: Base working directory containing all threads 556 """ 557 class Outcome: 558 """Records a build outcome for a single make invocation 559 560 Public Members: 561 rc: Outcome value (OUTCOME_...) 562 err_lines: List of error lines or [] if none 563 sizes: Dictionary of image size information, keyed by filename 564 - Each value is itself a dictionary containing 565 values for 'text', 'data' and 'bss', being the integer 566 size in bytes of each section. 567 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 568 value is itself a dictionary: 569 key: function name 570 value: Size of function in bytes 571 """ 572 def __init__(self, rc, err_lines, sizes, func_sizes): 573 self.rc = rc 574 self.err_lines = err_lines 575 self.sizes = sizes 576 self.func_sizes = func_sizes 577 578 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 579 gnu_make='make', checkout=True, show_unknown=True, step=1): 580 """Create a new Builder object 581 582 Args: 583 toolchains: Toolchains object to use for building 584 base_dir: Base directory to use for builder 585 git_dir: Git directory containing source repository 586 num_threads: Number of builder threads to run 587 num_jobs: Number of jobs to run at once (passed to make as -j) 588 gnu_make: the command name of GNU Make. 589 checkout: True to check out source, False to skip that step. 590 This is used for testing. 591 show_unknown: Show unknown boards (those not built) in summary 592 step: 1 to process every commit, n to process every nth commit 593 """ 594 self.toolchains = toolchains 595 self.base_dir = base_dir 596 self._working_dir = os.path.join(base_dir, '.bm-work') 597 self.threads = [] 598 self.active = True 599 self.do_make = self.Make 600 self.gnu_make = gnu_make 601 self.checkout = checkout 602 self.num_threads = num_threads 603 self.num_jobs = num_jobs 604 self.already_done = 0 605 self.force_build = False 606 self.git_dir = git_dir 607 self._show_unknown = show_unknown 608 self._timestamp_count = 10 609 self._build_period_us = None 610 self._complete_delay = None 611 self._next_delay_update = datetime.now() 612 self.force_config_on_failure = True 613 self.force_build_failures = False 614 self.force_reconfig = False 615 self._step = step 616 self.in_tree = False 617 618 self.col = terminal.Color() 619 620 self.queue = Queue.Queue() 621 self.out_queue = Queue.Queue() 622 for i in range(self.num_threads): 623 t = BuilderThread(self, i) 624 t.setDaemon(True) 625 t.start() 626 self.threads.append(t) 627 628 self.last_line_len = 0 629 t = ResultThread(self) 630 t.setDaemon(True) 631 t.start() 632 self.threads.append(t) 633 634 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 635 self.re_make_err = re.compile('|'.join(ignore_lines)) 636 637 def __del__(self): 638 """Get rid of all threads created by the builder""" 639 for t in self.threads: 640 del t 641 642 def _AddTimestamp(self): 643 """Add a new timestamp to the list and record the build period. 644 645 The build period is the length of time taken to perform a single 646 build (one board, one commit). 647 """ 648 now = datetime.now() 649 self._timestamps.append(now) 650 count = len(self._timestamps) 651 delta = self._timestamps[-1] - self._timestamps[0] 652 seconds = delta.total_seconds() 653 654 # If we have enough data, estimate build period (time taken for a 655 # single build) and therefore completion time. 656 if count > 1 and self._next_delay_update < now: 657 self._next_delay_update = now + timedelta(seconds=2) 658 if seconds > 0: 659 self._build_period = float(seconds) / count 660 todo = self.count - self.upto 661 self._complete_delay = timedelta(microseconds= 662 self._build_period * todo * 1000000) 663 # Round it 664 self._complete_delay -= timedelta( 665 microseconds=self._complete_delay.microseconds) 666 667 if seconds > 60: 668 self._timestamps.popleft() 669 count -= 1 670 671 def ClearLine(self, length): 672 """Clear any characters on the current line 673 674 Make way for a new line of length 'length', by outputting enough 675 spaces to clear out the old line. Then remember the new length for 676 next time. 677 678 Args: 679 length: Length of new line, in characters 680 """ 681 if length < self.last_line_len: 682 print ' ' * (self.last_line_len - length), 683 print '\r', 684 self.last_line_len = length 685 sys.stdout.flush() 686 687 def SelectCommit(self, commit, checkout=True): 688 """Checkout the selected commit for this build 689 """ 690 self.commit = commit 691 if checkout and self.checkout: 692 gitutil.Checkout(commit.hash) 693 694 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 695 """Run make 696 697 Args: 698 commit: Commit object that is being built 699 brd: Board object that is being built 700 stage: Stage that we are at (distclean, config, build) 701 cwd: Directory where make should be run 702 args: Arguments to pass to make 703 kwargs: Arguments to pass to command.RunPipe() 704 """ 705 cmd = [self.gnu_make] + list(args) 706 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 707 cwd=cwd, raise_on_error=False, **kwargs) 708 return result 709 710 def ProcessResult(self, result): 711 """Process the result of a build, showing progress information 712 713 Args: 714 result: A CommandResult object 715 """ 716 col = terminal.Color() 717 if result: 718 target = result.brd.target 719 720 if result.return_code < 0: 721 self.active = False 722 command.StopAll() 723 return 724 725 self.upto += 1 726 if result.return_code != 0: 727 self.fail += 1 728 elif result.stderr: 729 self.warned += 1 730 if result.already_done: 731 self.already_done += 1 732 else: 733 target = '(starting)' 734 735 # Display separate counts for ok, warned and fail 736 ok = self.upto - self.warned - self.fail 737 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 738 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 739 line += self.col.Color(self.col.RED, '%5d' % self.fail) 740 741 name = ' /%-5d ' % self.count 742 743 # Add our current completion time estimate 744 self._AddTimestamp() 745 if self._complete_delay: 746 name += '%s : ' % self._complete_delay 747 # When building all boards for a commit, we can print a commit 748 # progress message. 749 if result and result.commit_upto is None: 750 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 751 self.commit_count) 752 753 name += target 754 print line + name, 755 length = 13 + len(name) 756 self.ClearLine(length) 757 758 def _GetOutputDir(self, commit_upto): 759 """Get the name of the output directory for a commit number 760 761 The output directory is typically .../<branch>/<commit>. 762 763 Args: 764 commit_upto: Commit number to use (0..self.count-1) 765 """ 766 commit = self.commits[commit_upto] 767 subject = commit.subject.translate(trans_valid_chars) 768 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 769 self.commit_count, commit.hash, subject[:20])) 770 output_dir = os.path.join(self.base_dir, commit_dir) 771 return output_dir 772 773 def GetBuildDir(self, commit_upto, target): 774 """Get the name of the build directory for a commit number 775 776 The build directory is typically .../<branch>/<commit>/<target>. 777 778 Args: 779 commit_upto: Commit number to use (0..self.count-1) 780 target: Target name 781 """ 782 output_dir = self._GetOutputDir(commit_upto) 783 return os.path.join(output_dir, target) 784 785 def GetDoneFile(self, commit_upto, target): 786 """Get the name of the done file for a commit number 787 788 Args: 789 commit_upto: Commit number to use (0..self.count-1) 790 target: Target name 791 """ 792 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 793 794 def GetSizesFile(self, commit_upto, target): 795 """Get the name of the sizes file for a commit number 796 797 Args: 798 commit_upto: Commit number to use (0..self.count-1) 799 target: Target name 800 """ 801 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 802 803 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 804 """Get the name of the funcsizes file for a commit number and ELF file 805 806 Args: 807 commit_upto: Commit number to use (0..self.count-1) 808 target: Target name 809 elf_fname: Filename of elf image 810 """ 811 return os.path.join(self.GetBuildDir(commit_upto, target), 812 '%s.sizes' % elf_fname.replace('/', '-')) 813 814 def GetObjdumpFile(self, commit_upto, target, elf_fname): 815 """Get the name of the objdump file for a commit number and ELF file 816 817 Args: 818 commit_upto: Commit number to use (0..self.count-1) 819 target: Target name 820 elf_fname: Filename of elf image 821 """ 822 return os.path.join(self.GetBuildDir(commit_upto, target), 823 '%s.objdump' % elf_fname.replace('/', '-')) 824 825 def GetErrFile(self, commit_upto, target): 826 """Get the name of the err file for a commit number 827 828 Args: 829 commit_upto: Commit number to use (0..self.count-1) 830 target: Target name 831 """ 832 output_dir = self.GetBuildDir(commit_upto, target) 833 return os.path.join(output_dir, 'err') 834 835 def FilterErrors(self, lines): 836 """Filter out errors in which we have no interest 837 838 We should probably use map(). 839 840 Args: 841 lines: List of error lines, each a string 842 Returns: 843 New list with only interesting lines included 844 """ 845 out_lines = [] 846 for line in lines: 847 if not self.re_make_err.search(line): 848 out_lines.append(line) 849 return out_lines 850 851 def ReadFuncSizes(self, fname, fd): 852 """Read function sizes from the output of 'nm' 853 854 Args: 855 fd: File containing data to read 856 fname: Filename we are reading from (just for errors) 857 858 Returns: 859 Dictionary containing size of each function in bytes, indexed by 860 function name. 861 """ 862 sym = {} 863 for line in fd.readlines(): 864 try: 865 size, type, name = line[:-1].split() 866 except: 867 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 868 continue 869 if type in 'tTdDbB': 870 # function names begin with '.' on 64-bit powerpc 871 if '.' in name[1:]: 872 name = 'static.' + name.split('.')[0] 873 sym[name] = sym.get(name, 0) + int(size, 16) 874 return sym 875 876 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 877 """Work out the outcome of a build. 878 879 Args: 880 commit_upto: Commit number to check (0..n-1) 881 target: Target board to check 882 read_func_sizes: True to read function size information 883 884 Returns: 885 Outcome object 886 """ 887 done_file = self.GetDoneFile(commit_upto, target) 888 sizes_file = self.GetSizesFile(commit_upto, target) 889 sizes = {} 890 func_sizes = {} 891 if os.path.exists(done_file): 892 with open(done_file, 'r') as fd: 893 return_code = int(fd.readline()) 894 err_lines = [] 895 err_file = self.GetErrFile(commit_upto, target) 896 if os.path.exists(err_file): 897 with open(err_file, 'r') as fd: 898 err_lines = self.FilterErrors(fd.readlines()) 899 900 # Decide whether the build was ok, failed or created warnings 901 if return_code: 902 rc = OUTCOME_ERROR 903 elif len(err_lines): 904 rc = OUTCOME_WARNING 905 else: 906 rc = OUTCOME_OK 907 908 # Convert size information to our simple format 909 if os.path.exists(sizes_file): 910 with open(sizes_file, 'r') as fd: 911 for line in fd.readlines(): 912 values = line.split() 913 rodata = 0 914 if len(values) > 6: 915 rodata = int(values[6], 16) 916 size_dict = { 917 'all' : int(values[0]) + int(values[1]) + 918 int(values[2]), 919 'text' : int(values[0]) - rodata, 920 'data' : int(values[1]), 921 'bss' : int(values[2]), 922 'rodata' : rodata, 923 } 924 sizes[values[5]] = size_dict 925 926 if read_func_sizes: 927 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 928 for fname in glob.glob(pattern): 929 with open(fname, 'r') as fd: 930 dict_name = os.path.basename(fname).replace('.sizes', 931 '') 932 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 933 934 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 935 936 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 937 938 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 939 """Calculate a summary of the results of building a commit. 940 941 Args: 942 board_selected: Dict containing boards to summarise 943 commit_upto: Commit number to summarize (0..self.count-1) 944 read_func_sizes: True to read function size information 945 946 Returns: 947 Tuple: 948 Dict containing boards which passed building this commit. 949 keyed by board.target 950 List containing a summary of error/warning lines 951 """ 952 board_dict = {} 953 err_lines_summary = [] 954 955 for board in boards_selected.itervalues(): 956 outcome = self.GetBuildOutcome(commit_upto, board.target, 957 read_func_sizes) 958 board_dict[board.target] = outcome 959 for err in outcome.err_lines: 960 if err and not err.rstrip() in err_lines_summary: 961 err_lines_summary.append(err.rstrip()) 962 return board_dict, err_lines_summary 963 964 def AddOutcome(self, board_dict, arch_list, changes, char, color): 965 """Add an output to our list of outcomes for each architecture 966 967 This simple function adds failing boards (changes) to the 968 relevant architecture string, so we can print the results out 969 sorted by architecture. 970 971 Args: 972 board_dict: Dict containing all boards 973 arch_list: Dict keyed by arch name. Value is a string containing 974 a list of board names which failed for that arch. 975 changes: List of boards to add to arch_list 976 color: terminal.Colour object 977 """ 978 done_arch = {} 979 for target in changes: 980 if target in board_dict: 981 arch = board_dict[target].arch 982 else: 983 arch = 'unknown' 984 str = self.col.Color(color, ' ' + target) 985 if not arch in done_arch: 986 str = self.col.Color(color, char) + ' ' + str 987 done_arch[arch] = True 988 if not arch in arch_list: 989 arch_list[arch] = str 990 else: 991 arch_list[arch] += str 992 993 994 def ColourNum(self, num): 995 color = self.col.RED if num > 0 else self.col.GREEN 996 if num == 0: 997 return '0' 998 return self.col.Color(color, str(num)) 999 1000 def ResetResultSummary(self, board_selected): 1001 """Reset the results summary ready for use. 1002 1003 Set up the base board list to be all those selected, and set the 1004 error lines to empty. 1005 1006 Following this, calls to PrintResultSummary() will use this 1007 information to work out what has changed. 1008 1009 Args: 1010 board_selected: Dict containing boards to summarise, keyed by 1011 board.target 1012 """ 1013 self._base_board_dict = {} 1014 for board in board_selected: 1015 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 1016 self._base_err_lines = [] 1017 1018 def PrintFuncSizeDetail(self, fname, old, new): 1019 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 1020 delta, common = [], {} 1021 1022 for a in old: 1023 if a in new: 1024 common[a] = 1 1025 1026 for name in old: 1027 if name not in common: 1028 remove += 1 1029 down += old[name] 1030 delta.append([-old[name], name]) 1031 1032 for name in new: 1033 if name not in common: 1034 add += 1 1035 up += new[name] 1036 delta.append([new[name], name]) 1037 1038 for name in common: 1039 diff = new.get(name, 0) - old.get(name, 0) 1040 if diff > 0: 1041 grow, up = grow + 1, up + diff 1042 elif diff < 0: 1043 shrink, down = shrink + 1, down - diff 1044 delta.append([diff, name]) 1045 1046 delta.sort() 1047 delta.reverse() 1048 1049 args = [add, -remove, grow, -shrink, up, -down, up - down] 1050 if max(args) == 0: 1051 return 1052 args = [self.ColourNum(x) for x in args] 1053 indent = ' ' * 15 1054 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1055 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1056 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1057 'delta') 1058 for diff, name in delta: 1059 if diff: 1060 color = self.col.RED if diff > 0 else self.col.GREEN 1061 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1062 old.get(name, '-'), new.get(name,'-'), diff) 1063 print self.col.Color(color, msg) 1064 1065 1066 def PrintSizeDetail(self, target_list, show_bloat): 1067 """Show details size information for each board 1068 1069 Args: 1070 target_list: List of targets, each a dict containing: 1071 'target': Target name 1072 'total_diff': Total difference in bytes across all areas 1073 <part_name>: Difference for that part 1074 show_bloat: Show detail for each function 1075 """ 1076 targets_by_diff = sorted(target_list, reverse=True, 1077 key=lambda x: x['_total_diff']) 1078 for result in targets_by_diff: 1079 printed_target = False 1080 for name in sorted(result): 1081 diff = result[name] 1082 if name.startswith('_'): 1083 continue 1084 if diff != 0: 1085 color = self.col.RED if diff > 0 else self.col.GREEN 1086 msg = ' %s %+d' % (name, diff) 1087 if not printed_target: 1088 print '%10s %-15s:' % ('', result['_target']), 1089 printed_target = True 1090 print self.col.Color(color, msg), 1091 if printed_target: 1092 print 1093 if show_bloat: 1094 target = result['_target'] 1095 outcome = result['_outcome'] 1096 base_outcome = self._base_board_dict[target] 1097 for fname in outcome.func_sizes: 1098 self.PrintFuncSizeDetail(fname, 1099 base_outcome.func_sizes[fname], 1100 outcome.func_sizes[fname]) 1101 1102 1103 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1104 show_bloat): 1105 """Print a summary of image sizes broken down by section. 1106 1107 The summary takes the form of one line per architecture. The 1108 line contains deltas for each of the sections (+ means the section 1109 got bigger, - means smaller). The nunmbers are the average number 1110 of bytes that a board in this section increased by. 1111 1112 For example: 1113 powerpc: (622 boards) text -0.0 1114 arm: (285 boards) text -0.0 1115 nds32: (3 boards) text -8.0 1116 1117 Args: 1118 board_selected: Dict containing boards to summarise, keyed by 1119 board.target 1120 board_dict: Dict containing boards for which we built this 1121 commit, keyed by board.target. The value is an Outcome object. 1122 show_detail: Show detail for each board 1123 show_bloat: Show detail for each function 1124 """ 1125 arch_list = {} 1126 arch_count = {} 1127 1128 # Calculate changes in size for different image parts 1129 # The previous sizes are in Board.sizes, for each board 1130 for target in board_dict: 1131 if target not in board_selected: 1132 continue 1133 base_sizes = self._base_board_dict[target].sizes 1134 outcome = board_dict[target] 1135 sizes = outcome.sizes 1136 1137 # Loop through the list of images, creating a dict of size 1138 # changes for each image/part. We end up with something like 1139 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1140 # which means that U-Boot data increased by 5 bytes and SPL 1141 # text decreased by 4. 1142 err = {'_target' : target} 1143 for image in sizes: 1144 if image in base_sizes: 1145 base_image = base_sizes[image] 1146 # Loop through the text, data, bss parts 1147 for part in sorted(sizes[image]): 1148 diff = sizes[image][part] - base_image[part] 1149 col = None 1150 if diff: 1151 if image == 'u-boot': 1152 name = part 1153 else: 1154 name = image + ':' + part 1155 err[name] = diff 1156 arch = board_selected[target].arch 1157 if not arch in arch_count: 1158 arch_count[arch] = 1 1159 else: 1160 arch_count[arch] += 1 1161 if not sizes: 1162 pass # Only add to our list when we have some stats 1163 elif not arch in arch_list: 1164 arch_list[arch] = [err] 1165 else: 1166 arch_list[arch].append(err) 1167 1168 # We now have a list of image size changes sorted by arch 1169 # Print out a summary of these 1170 for arch, target_list in arch_list.iteritems(): 1171 # Get total difference for each type 1172 totals = {} 1173 for result in target_list: 1174 total = 0 1175 for name, diff in result.iteritems(): 1176 if name.startswith('_'): 1177 continue 1178 total += diff 1179 if name in totals: 1180 totals[name] += diff 1181 else: 1182 totals[name] = diff 1183 result['_total_diff'] = total 1184 result['_outcome'] = board_dict[result['_target']] 1185 1186 count = len(target_list) 1187 printed_arch = False 1188 for name in sorted(totals): 1189 diff = totals[name] 1190 if diff: 1191 # Display the average difference in this name for this 1192 # architecture 1193 avg_diff = float(diff) / count 1194 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1195 msg = ' %s %+1.1f' % (name, avg_diff) 1196 if not printed_arch: 1197 print '%10s: (for %d/%d boards)' % (arch, count, 1198 arch_count[arch]), 1199 printed_arch = True 1200 print self.col.Color(color, msg), 1201 1202 if printed_arch: 1203 print 1204 if show_detail: 1205 self.PrintSizeDetail(target_list, show_bloat) 1206 1207 1208 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1209 show_sizes, show_detail, show_bloat): 1210 """Compare results with the base results and display delta. 1211 1212 Only boards mentioned in board_selected will be considered. This 1213 function is intended to be called repeatedly with the results of 1214 each commit. It therefore shows a 'diff' between what it saw in 1215 the last call and what it sees now. 1216 1217 Args: 1218 board_selected: Dict containing boards to summarise, keyed by 1219 board.target 1220 board_dict: Dict containing boards for which we built this 1221 commit, keyed by board.target. The value is an Outcome object. 1222 err_lines: A list of errors for this commit, or [] if there is 1223 none, or we don't want to print errors 1224 show_sizes: Show image size deltas 1225 show_detail: Show detail for each board 1226 show_bloat: Show detail for each function 1227 """ 1228 better = [] # List of boards fixed since last commit 1229 worse = [] # List of new broken boards since last commit 1230 new = [] # List of boards that didn't exist last time 1231 unknown = [] # List of boards that were not built 1232 1233 for target in board_dict: 1234 if target not in board_selected: 1235 continue 1236 1237 # If the board was built last time, add its outcome to a list 1238 if target in self._base_board_dict: 1239 base_outcome = self._base_board_dict[target].rc 1240 outcome = board_dict[target] 1241 if outcome.rc == OUTCOME_UNKNOWN: 1242 unknown.append(target) 1243 elif outcome.rc < base_outcome: 1244 better.append(target) 1245 elif outcome.rc > base_outcome: 1246 worse.append(target) 1247 else: 1248 new.append(target) 1249 1250 # Get a list of errors that have appeared, and disappeared 1251 better_err = [] 1252 worse_err = [] 1253 for line in err_lines: 1254 if line not in self._base_err_lines: 1255 worse_err.append('+' + line) 1256 for line in self._base_err_lines: 1257 if line not in err_lines: 1258 better_err.append('-' + line) 1259 1260 # Display results by arch 1261 if better or worse or unknown or new or worse_err or better_err: 1262 arch_list = {} 1263 self.AddOutcome(board_selected, arch_list, better, '', 1264 self.col.GREEN) 1265 self.AddOutcome(board_selected, arch_list, worse, '+', 1266 self.col.RED) 1267 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1268 if self._show_unknown: 1269 self.AddOutcome(board_selected, arch_list, unknown, '?', 1270 self.col.MAGENTA) 1271 for arch, target_list in arch_list.iteritems(): 1272 print '%10s: %s' % (arch, target_list) 1273 if better_err: 1274 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1275 if worse_err: 1276 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1277 1278 if show_sizes: 1279 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1280 show_bloat) 1281 1282 # Save our updated information for the next call to this function 1283 self._base_board_dict = board_dict 1284 self._base_err_lines = err_lines 1285 1286 # Get a list of boards that did not get built, if needed 1287 not_built = [] 1288 for board in board_selected: 1289 if not board in board_dict: 1290 not_built.append(board) 1291 if not_built: 1292 print "Boards not built (%d): %s" % (len(not_built), 1293 ', '.join(not_built)) 1294 1295 1296 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1297 show_detail, show_bloat): 1298 """Show a build summary for U-Boot for a given board list. 1299 1300 Reset the result summary, then repeatedly call GetResultSummary on 1301 each commit's results, then display the differences we see. 1302 1303 Args: 1304 commit: Commit objects to summarise 1305 board_selected: Dict containing boards to summarise 1306 show_errors: Show errors that occured 1307 show_sizes: Show size deltas 1308 show_detail: Show detail for each board 1309 show_bloat: Show detail for each function 1310 """ 1311 self.commit_count = len(commits) 1312 self.commits = commits 1313 self.ResetResultSummary(board_selected) 1314 1315 for commit_upto in range(0, self.commit_count, self._step): 1316 board_dict, err_lines = self.GetResultSummary(board_selected, 1317 commit_upto, read_func_sizes=show_bloat) 1318 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1319 print self.col.Color(self.col.BLUE, msg) 1320 self.PrintResultSummary(board_selected, board_dict, 1321 err_lines if show_errors else [], show_sizes, show_detail, 1322 show_bloat) 1323 1324 1325 def SetupBuild(self, board_selected, commits): 1326 """Set up ready to start a build. 1327 1328 Args: 1329 board_selected: Selected boards to build 1330 commits: Selected commits to build 1331 """ 1332 # First work out how many commits we will build 1333 count = (len(commits) + self._step - 1) / self._step 1334 self.count = len(board_selected) * count 1335 self.upto = self.warned = self.fail = 0 1336 self._timestamps = collections.deque() 1337 1338 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1339 """Build all boards for a single commit""" 1340 self.SetupBuild(board_selected) 1341 self.count = len(board_selected) 1342 for brd in board_selected.itervalues(): 1343 job = BuilderJob() 1344 job.board = brd 1345 job.commits = None 1346 job.keep_outputs = keep_outputs 1347 self.queue.put(brd) 1348 1349 self.queue.join() 1350 self.out_queue.join() 1351 print 1352 self.ClearLine(0) 1353 1354 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1355 """Build all boards for all commits (non-incremental)""" 1356 self.commit_count = len(commits) 1357 1358 self.ResetResultSummary(board_selected) 1359 for self.commit_upto in range(self.commit_count): 1360 self.SelectCommit(commits[self.commit_upto]) 1361 self.SelectOutputDir() 1362 Mkdir(self.output_dir) 1363 1364 self.BuildBoardsForCommit(board_selected, keep_outputs) 1365 board_dict, err_lines = self.GetResultSummary() 1366 self.PrintResultSummary(board_selected, board_dict, 1367 err_lines if show_errors else []) 1368 1369 if self.already_done: 1370 print '%d builds already done' % self.already_done 1371 1372 def GetThreadDir(self, thread_num): 1373 """Get the directory path to the working dir for a thread. 1374 1375 Args: 1376 thread_num: Number of thread to check. 1377 """ 1378 return os.path.join(self._working_dir, '%02d' % thread_num) 1379 1380 def _PrepareThread(self, thread_num): 1381 """Prepare the working directory for a thread. 1382 1383 This clones or fetches the repo into the thread's work directory. 1384 1385 Args: 1386 thread_num: Thread number (0, 1, ...) 1387 """ 1388 thread_dir = self.GetThreadDir(thread_num) 1389 Mkdir(thread_dir) 1390 git_dir = os.path.join(thread_dir, '.git') 1391 1392 # Clone the repo if it doesn't already exist 1393 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1394 # we have a private index but uses the origin repo's contents? 1395 if self.git_dir: 1396 src_dir = os.path.abspath(self.git_dir) 1397 if os.path.exists(git_dir): 1398 gitutil.Fetch(git_dir, thread_dir) 1399 else: 1400 print 'Cloning repo for thread %d' % thread_num 1401 gitutil.Clone(src_dir, thread_dir) 1402 1403 def _PrepareWorkingSpace(self, max_threads): 1404 """Prepare the working directory for use. 1405 1406 Set up the git repo for each thread. 1407 1408 Args: 1409 max_threads: Maximum number of threads we expect to need. 1410 """ 1411 Mkdir(self._working_dir) 1412 for thread in range(max_threads): 1413 self._PrepareThread(thread) 1414 1415 def _PrepareOutputSpace(self): 1416 """Get the output directories ready to receive files. 1417 1418 We delete any output directories which look like ones we need to 1419 create. Having left over directories is confusing when the user wants 1420 to check the output manually. 1421 """ 1422 dir_list = [] 1423 for commit_upto in range(self.commit_count): 1424 dir_list.append(self._GetOutputDir(commit_upto)) 1425 1426 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1427 if dirname not in dir_list: 1428 shutil.rmtree(dirname) 1429 1430 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1431 """Build all commits for a list of boards 1432 1433 Args: 1434 commits: List of commits to be build, each a Commit object 1435 boards_selected: Dict of selected boards, key is target name, 1436 value is Board object 1437 show_errors: True to show summarised error/warning info 1438 keep_outputs: True to save build output files 1439 """ 1440 self.commit_count = len(commits) 1441 self.commits = commits 1442 1443 self.ResetResultSummary(board_selected) 1444 Mkdir(self.base_dir) 1445 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1446 self._PrepareOutputSpace() 1447 self.SetupBuild(board_selected, commits) 1448 self.ProcessResult(None) 1449 1450 # Create jobs to build all commits for each board 1451 for brd in board_selected.itervalues(): 1452 job = BuilderJob() 1453 job.board = brd 1454 job.commits = commits 1455 job.keep_outputs = keep_outputs 1456 job.step = self._step 1457 self.queue.put(job) 1458 1459 # Wait until all jobs are started 1460 self.queue.join() 1461 1462 # Wait until we have processed all output 1463 self.out_queue.join() 1464 print 1465 self.ClearLine(0) 1466