1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# See file CREDITS for list of people who contributed to this 6# project. 7# 8# This program is free software; you can redistribute it and/or 9# modify it under the terms of the GNU General Public License as 10# published by the Free Software Foundation; either version 2 of 11# the License, or (at your option) any later version. 12# 13# This program is distributed in the hope that it will be useful, 14# but WITHOUT ANY WARRANTY; without even the implied warranty of 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16# GNU General Public License for more details. 17# 18# You should have received a copy of the GNU General Public License 19# along with this program; if not, write to the Free Software 20# Foundation, Inc., 59 Temple Place, Suite 330, Boston, 21# MA 02111-1307 USA 22# 23 24import collections 25import errno 26from datetime import datetime, timedelta 27import glob 28import os 29import re 30import Queue 31import shutil 32import string 33import sys 34import threading 35import time 36 37import command 38import gitutil 39import terminal 40import toolchain 41 42 43""" 44Theory of Operation 45 46Please see README for user documentation, and you should be familiar with 47that before trying to make sense of this. 48 49Buildman works by keeping the machine as busy as possible, building different 50commits for different boards on multiple CPUs at once. 51 52The source repo (self.git_dir) contains all the commits to be built. Each 53thread works on a single board at a time. It checks out the first commit, 54configures it for that board, then builds it. Then it checks out the next 55commit and builds it (typically without re-configuring). When it runs out 56of commits, it gets another job from the builder and starts again with that 57board. 58 59Clearly the builder threads could work either way - they could check out a 60commit and then built it for all boards. Using separate directories for each 61commit/board pair they could leave their build product around afterwards 62also. 63 64The intent behind building a single board for multiple commits, is to make 65use of incremental builds. Since each commit is built incrementally from 66the previous one, builds are faster. Reconfiguring for a different board 67removes all intermediate object files. 68 69Many threads can be working at once, but each has its own working directory. 70When a thread finishes a build, it puts the output files into a result 71directory. 72 73The base directory used by buildman is normally '../<branch>', i.e. 74a directory higher than the source repository and named after the branch 75being built. 76 77Within the base directory, we have one subdirectory for each commit. Within 78that is one subdirectory for each board. Within that is the build output for 79that commit/board combination. 80 81Buildman also create working directories for each thread, in a .bm-work/ 82subdirectory in the base dir. 83 84As an example, say we are building branch 'us-net' for boards 'sandbox' and 85'seaboard', and say that us-net has two commits. We will have directories 86like this: 87 88us-net/ base directory 89 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 90 sandbox/ 91 u-boot.bin 92 seaboard/ 93 u-boot.bin 94 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 95 sandbox/ 96 u-boot.bin 97 seaboard/ 98 u-boot.bin 99 .bm-work/ 100 00/ working directory for thread 0 (contains source checkout) 101 build/ build output 102 01/ working directory for thread 1 103 build/ build output 104 ... 105u-boot/ source directory 106 .git/ repository 107""" 108 109# Possible build outcomes 110OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 111 112# Translate a commit subject into a valid filename 113trans_valid_chars = string.maketrans("/: ", "---") 114 115 116def Mkdir(dirname): 117 """Make a directory if it doesn't already exist. 118 119 Args: 120 dirname: Directory to create 121 """ 122 try: 123 os.mkdir(dirname) 124 except OSError as err: 125 if err.errno == errno.EEXIST: 126 pass 127 else: 128 raise 129 130class BuilderJob: 131 """Holds information about a job to be performed by a thread 132 133 Members: 134 board: Board object to build 135 commits: List of commit options to build. 136 """ 137 def __init__(self): 138 self.board = None 139 self.commits = [] 140 141 142class ResultThread(threading.Thread): 143 """This thread processes results from builder threads. 144 145 It simply passes the results on to the builder. There is only one 146 result thread, and this helps to serialise the build output. 147 """ 148 def __init__(self, builder): 149 """Set up a new result thread 150 151 Args: 152 builder: Builder which will be sent each result 153 """ 154 threading.Thread.__init__(self) 155 self.builder = builder 156 157 def run(self): 158 """Called to start up the result thread. 159 160 We collect the next result job and pass it on to the build. 161 """ 162 while True: 163 result = self.builder.out_queue.get() 164 self.builder.ProcessResult(result) 165 self.builder.out_queue.task_done() 166 167 168class BuilderThread(threading.Thread): 169 """This thread builds U-Boot for a particular board. 170 171 An input queue provides each new job. We run 'make' to build U-Boot 172 and then pass the results on to the output queue. 173 174 Members: 175 builder: The builder which contains information we might need 176 thread_num: Our thread number (0-n-1), used to decide on a 177 temporary directory 178 """ 179 def __init__(self, builder, thread_num): 180 """Set up a new builder thread""" 181 threading.Thread.__init__(self) 182 self.builder = builder 183 self.thread_num = thread_num 184 185 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 186 """Run 'make' on a particular commit and board. 187 188 The source code will already be checked out, so the 'commit' 189 argument is only for information. 190 191 Args: 192 commit: Commit object that is being built 193 brd: Board object that is being built 194 stage: Stage of the build. Valid stages are: 195 distclean - can be called to clean source 196 config - called to configure for a board 197 build - the main make invocation - it does the build 198 args: A list of arguments to pass to 'make' 199 kwargs: A list of keyword arguments to pass to command.RunPipe() 200 201 Returns: 202 CommandResult object 203 """ 204 return self.builder.do_make(commit, brd, stage, cwd, *args, 205 **kwargs) 206 207 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build): 208 """Build a particular commit. 209 210 If the build is already done, and we are not forcing a build, we skip 211 the build and just return the previously-saved results. 212 213 Args: 214 commit_upto: Commit number to build (0...n-1) 215 brd: Board object to build 216 work_dir: Directory to which the source will be checked out 217 do_config: True to run a make <board>_config on the source 218 force_build: Force a build even if one was previously done 219 220 Returns: 221 tuple containing: 222 - CommandResult object containing the results of the build 223 - boolean indicating whether 'make config' is still needed 224 """ 225 # Create a default result - it will be overwritte by the call to 226 # self.Make() below, in the event that we do a build. 227 result = command.CommandResult() 228 result.return_code = 0 229 out_dir = os.path.join(work_dir, 'build') 230 231 # Check if the job was already completed last time 232 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 233 result.already_done = os.path.exists(done_file) 234 if result.already_done and not force_build: 235 # Get the return code from that build and use it 236 with open(done_file, 'r') as fd: 237 result.return_code = int(fd.readline()) 238 err_file = self.builder.GetErrFile(commit_upto, brd.target) 239 if os.path.exists(err_file) and os.stat(err_file).st_size: 240 result.stderr = 'bad' 241 else: 242 # We are going to have to build it. First, get a toolchain 243 if not self.toolchain: 244 try: 245 self.toolchain = self.builder.toolchains.Select(brd.arch) 246 except ValueError as err: 247 result.return_code = 10 248 result.stdout = '' 249 result.stderr = str(err) 250 # TODO(sjg@chromium.org): This gets swallowed, but needs 251 # to be reported. 252 253 if self.toolchain: 254 # Checkout the right commit 255 if commit_upto is not None: 256 commit = self.builder.commits[commit_upto] 257 if self.builder.checkout: 258 git_dir = os.path.join(work_dir, '.git') 259 gitutil.Checkout(commit.hash, git_dir, work_dir, 260 force=True) 261 else: 262 commit = self.builder.commit # Ick, fix this for BuildCommits() 263 264 # Set up the environment and command line 265 env = self.toolchain.MakeEnvironment() 266 Mkdir(out_dir) 267 args = ['O=build', '-s'] 268 if self.builder.num_jobs is not None: 269 args.extend(['-j', str(self.builder.num_jobs)]) 270 config_args = ['%s_config' % brd.target] 271 config_out = '' 272 273 # If we need to reconfigure, do that now 274 if do_config: 275 result = self.Make(commit, brd, 'distclean', work_dir, 276 'distclean', *args, env=env) 277 result = self.Make(commit, brd, 'config', work_dir, 278 *(args + config_args), env=env) 279 config_out = result.combined 280 do_config = False # No need to configure next time 281 if result.return_code == 0: 282 result = self.Make(commit, brd, 'build', work_dir, *args, 283 env=env) 284 result.stdout = config_out + result.stdout 285 else: 286 result.return_code = 1 287 result.stderr = 'No tool chain for %s\n' % brd.arch 288 result.already_done = False 289 290 result.toolchain = self.toolchain 291 result.brd = brd 292 result.commit_upto = commit_upto 293 result.out_dir = out_dir 294 return result, do_config 295 296 def _WriteResult(self, result, keep_outputs): 297 """Write a built result to the output directory. 298 299 Args: 300 result: CommandResult object containing result to write 301 keep_outputs: True to store the output binaries, False 302 to delete them 303 """ 304 # Fatal error 305 if result.return_code < 0: 306 return 307 308 # Aborted? 309 if result.stderr and 'No child processes' in result.stderr: 310 return 311 312 if result.already_done: 313 return 314 315 # Write the output and stderr 316 output_dir = self.builder._GetOutputDir(result.commit_upto) 317 Mkdir(output_dir) 318 build_dir = self.builder.GetBuildDir(result.commit_upto, 319 result.brd.target) 320 Mkdir(build_dir) 321 322 outfile = os.path.join(build_dir, 'log') 323 with open(outfile, 'w') as fd: 324 if result.stdout: 325 fd.write(result.stdout) 326 327 errfile = self.builder.GetErrFile(result.commit_upto, 328 result.brd.target) 329 if result.stderr: 330 with open(errfile, 'w') as fd: 331 fd.write(result.stderr) 332 elif os.path.exists(errfile): 333 os.remove(errfile) 334 335 if result.toolchain: 336 # Write the build result and toolchain information. 337 done_file = self.builder.GetDoneFile(result.commit_upto, 338 result.brd.target) 339 with open(done_file, 'w') as fd: 340 fd.write('%s' % result.return_code) 341 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 342 print >>fd, 'gcc', result.toolchain.gcc 343 print >>fd, 'path', result.toolchain.path 344 print >>fd, 'cross', result.toolchain.cross 345 print >>fd, 'arch', result.toolchain.arch 346 fd.write('%s' % result.return_code) 347 348 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 349 print >>fd, 'gcc', result.toolchain.gcc 350 print >>fd, 'path', result.toolchain.path 351 352 # Write out the image and function size information and an objdump 353 env = result.toolchain.MakeEnvironment() 354 lines = [] 355 for fname in ['u-boot', 'spl/u-boot-spl']: 356 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 357 nm_result = command.RunPipe([cmd], capture=True, 358 capture_stderr=True, cwd=result.out_dir, 359 raise_on_error=False, env=env) 360 if nm_result.stdout: 361 nm = self.builder.GetFuncSizesFile(result.commit_upto, 362 result.brd.target, fname) 363 with open(nm, 'w') as fd: 364 print >>fd, nm_result.stdout, 365 366 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 367 dump_result = command.RunPipe([cmd], capture=True, 368 capture_stderr=True, cwd=result.out_dir, 369 raise_on_error=False, env=env) 370 rodata_size = '' 371 if dump_result.stdout: 372 objdump = self.builder.GetObjdumpFile(result.commit_upto, 373 result.brd.target, fname) 374 with open(objdump, 'w') as fd: 375 print >>fd, dump_result.stdout, 376 for line in dump_result.stdout.splitlines(): 377 fields = line.split() 378 if len(fields) > 5 and fields[1] == '.rodata': 379 rodata_size = fields[2] 380 381 cmd = ['%ssize' % self.toolchain.cross, fname] 382 size_result = command.RunPipe([cmd], capture=True, 383 capture_stderr=True, cwd=result.out_dir, 384 raise_on_error=False, env=env) 385 if size_result.stdout: 386 lines.append(size_result.stdout.splitlines()[1] + ' ' + 387 rodata_size) 388 389 # Write out the image sizes file. This is similar to the output 390 # of binutil's 'size' utility, but it omits the header line and 391 # adds an additional hex value at the end of each line for the 392 # rodata size 393 if len(lines): 394 sizes = self.builder.GetSizesFile(result.commit_upto, 395 result.brd.target) 396 with open(sizes, 'w') as fd: 397 print >>fd, '\n'.join(lines) 398 399 # Now write the actual build output 400 if keep_outputs: 401 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 402 'include/autoconf.mk', 'spl/u-boot-spl', 403 'spl/u-boot-spl.bin'] 404 for pattern in patterns: 405 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 406 for fname in file_list: 407 shutil.copy(fname, build_dir) 408 409 410 def RunJob(self, job): 411 """Run a single job 412 413 A job consists of a building a list of commits for a particular board. 414 415 Args: 416 job: Job to build 417 """ 418 brd = job.board 419 work_dir = self.builder.GetThreadDir(self.thread_num) 420 self.toolchain = None 421 if job.commits: 422 # Run 'make board_config' on the first commit 423 do_config = True 424 commit_upto = 0 425 force_build = False 426 for commit_upto in range(0, len(job.commits), job.step): 427 result, request_config = self.RunCommit(commit_upto, brd, 428 work_dir, do_config, 429 force_build or self.builder.force_build) 430 failed = result.return_code or result.stderr 431 if failed and not do_config: 432 # If our incremental build failed, try building again 433 # with a reconfig. 434 if self.builder.force_config_on_failure: 435 result, request_config = self.RunCommit(commit_upto, 436 brd, work_dir, True, True) 437 do_config = request_config 438 439 # If we built that commit, then config is done. But if we got 440 # an warning, reconfig next time to force it to build the same 441 # files that created warnings this time. Otherwise an 442 # incremental build may not build the same file, and we will 443 # think that the warning has gone away. 444 # We could avoid this by using -Werror everywhere... 445 # For errors, the problem doesn't happen, since presumably 446 # the build stopped and didn't generate output, so will retry 447 # that file next time. So we could detect warnings and deal 448 # with them specially here. For now, we just reconfigure if 449 # anything goes work. 450 # Of course this is substantially slower if there are build 451 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 452 # have problems). 453 if (failed and not result.already_done and not do_config and 454 self.builder.force_config_on_failure): 455 # If this build failed, try the next one with a 456 # reconfigure. 457 # Sometimes if the board_config.h file changes it can mess 458 # with dependencies, and we get: 459 # make: *** No rule to make target `include/autoconf.mk', 460 # needed by `depend'. 461 do_config = True 462 force_build = True 463 else: 464 force_build = False 465 if self.builder.force_config_on_failure: 466 if failed: 467 do_config = True 468 result.commit_upto = commit_upto 469 if result.return_code < 0: 470 raise ValueError('Interrupt') 471 472 # We have the build results, so output the result 473 self._WriteResult(result, job.keep_outputs) 474 self.builder.out_queue.put(result) 475 else: 476 # Just build the currently checked-out build 477 result = self.RunCommit(None, True) 478 result.commit_upto = self.builder.upto 479 self.builder.out_queue.put(result) 480 481 def run(self): 482 """Our thread's run function 483 484 This thread picks a job from the queue, runs it, and then goes to the 485 next job. 486 """ 487 alive = True 488 while True: 489 job = self.builder.queue.get() 490 try: 491 if self.builder.active and alive: 492 self.RunJob(job) 493 except Exception as err: 494 alive = False 495 print err 496 self.builder.queue.task_done() 497 498 499class Builder: 500 """Class for building U-Boot for a particular commit. 501 502 Public members: (many should ->private) 503 active: True if the builder is active and has not been stopped 504 already_done: Number of builds already completed 505 base_dir: Base directory to use for builder 506 checkout: True to check out source, False to skip that step. 507 This is used for testing. 508 col: terminal.Color() object 509 count: Number of commits to build 510 do_make: Method to call to invoke Make 511 fail: Number of builds that failed due to error 512 force_build: Force building even if a build already exists 513 force_config_on_failure: If a commit fails for a board, disable 514 incremental building for the next commit we build for that 515 board, so that we will see all warnings/errors again. 516 git_dir: Git directory containing source repository 517 last_line_len: Length of the last line we printed (used for erasing 518 it with new progress information) 519 num_jobs: Number of jobs to run at once (passed to make as -j) 520 num_threads: Number of builder threads to run 521 out_queue: Queue of results to process 522 re_make_err: Compiled regular expression for ignore_lines 523 queue: Queue of jobs to run 524 threads: List of active threads 525 toolchains: Toolchains object to use for building 526 upto: Current commit number we are building (0.count-1) 527 warned: Number of builds that produced at least one warning 528 529 Private members: 530 _base_board_dict: Last-summarised Dict of boards 531 _base_err_lines: Last-summarised list of errors 532 _build_period_us: Time taken for a single build (float object). 533 _complete_delay: Expected delay until completion (timedelta) 534 _next_delay_update: Next time we plan to display a progress update 535 (datatime) 536 _show_unknown: Show unknown boards (those not built) in summary 537 _timestamps: List of timestamps for the completion of the last 538 last _timestamp_count builds. Each is a datetime object. 539 _timestamp_count: Number of timestamps to keep in our list. 540 _working_dir: Base working directory containing all threads 541 """ 542 class Outcome: 543 """Records a build outcome for a single make invocation 544 545 Public Members: 546 rc: Outcome value (OUTCOME_...) 547 err_lines: List of error lines or [] if none 548 sizes: Dictionary of image size information, keyed by filename 549 - Each value is itself a dictionary containing 550 values for 'text', 'data' and 'bss', being the integer 551 size in bytes of each section. 552 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 553 value is itself a dictionary: 554 key: function name 555 value: Size of function in bytes 556 """ 557 def __init__(self, rc, err_lines, sizes, func_sizes): 558 self.rc = rc 559 self.err_lines = err_lines 560 self.sizes = sizes 561 self.func_sizes = func_sizes 562 563 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 564 checkout=True, show_unknown=True, step=1): 565 """Create a new Builder object 566 567 Args: 568 toolchains: Toolchains object to use for building 569 base_dir: Base directory to use for builder 570 git_dir: Git directory containing source repository 571 num_threads: Number of builder threads to run 572 num_jobs: Number of jobs to run at once (passed to make as -j) 573 checkout: True to check out source, False to skip that step. 574 This is used for testing. 575 show_unknown: Show unknown boards (those not built) in summary 576 step: 1 to process every commit, n to process every nth commit 577 """ 578 self.toolchains = toolchains 579 self.base_dir = base_dir 580 self._working_dir = os.path.join(base_dir, '.bm-work') 581 self.threads = [] 582 self.active = True 583 self.do_make = self.Make 584 self.checkout = checkout 585 self.num_threads = num_threads 586 self.num_jobs = num_jobs 587 self.already_done = 0 588 self.force_build = False 589 self.git_dir = git_dir 590 self._show_unknown = show_unknown 591 self._timestamp_count = 10 592 self._build_period_us = None 593 self._complete_delay = None 594 self._next_delay_update = datetime.now() 595 self.force_config_on_failure = True 596 self._step = step 597 598 self.col = terminal.Color() 599 600 self.queue = Queue.Queue() 601 self.out_queue = Queue.Queue() 602 for i in range(self.num_threads): 603 t = BuilderThread(self, i) 604 t.setDaemon(True) 605 t.start() 606 self.threads.append(t) 607 608 self.last_line_len = 0 609 t = ResultThread(self) 610 t.setDaemon(True) 611 t.start() 612 self.threads.append(t) 613 614 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 615 self.re_make_err = re.compile('|'.join(ignore_lines)) 616 617 def __del__(self): 618 """Get rid of all threads created by the builder""" 619 for t in self.threads: 620 del t 621 622 def _AddTimestamp(self): 623 """Add a new timestamp to the list and record the build period. 624 625 The build period is the length of time taken to perform a single 626 build (one board, one commit). 627 """ 628 now = datetime.now() 629 self._timestamps.append(now) 630 count = len(self._timestamps) 631 delta = self._timestamps[-1] - self._timestamps[0] 632 seconds = delta.total_seconds() 633 634 # If we have enough data, estimate build period (time taken for a 635 # single build) and therefore completion time. 636 if count > 1 and self._next_delay_update < now: 637 self._next_delay_update = now + timedelta(seconds=2) 638 if seconds > 0: 639 self._build_period = float(seconds) / count 640 todo = self.count - self.upto 641 self._complete_delay = timedelta(microseconds= 642 self._build_period * todo * 1000000) 643 # Round it 644 self._complete_delay -= timedelta( 645 microseconds=self._complete_delay.microseconds) 646 647 if seconds > 60: 648 self._timestamps.popleft() 649 count -= 1 650 651 def ClearLine(self, length): 652 """Clear any characters on the current line 653 654 Make way for a new line of length 'length', by outputting enough 655 spaces to clear out the old line. Then remember the new length for 656 next time. 657 658 Args: 659 length: Length of new line, in characters 660 """ 661 if length < self.last_line_len: 662 print ' ' * (self.last_line_len - length), 663 print '\r', 664 self.last_line_len = length 665 sys.stdout.flush() 666 667 def SelectCommit(self, commit, checkout=True): 668 """Checkout the selected commit for this build 669 """ 670 self.commit = commit 671 if checkout and self.checkout: 672 gitutil.Checkout(commit.hash) 673 674 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 675 """Run make 676 677 Args: 678 commit: Commit object that is being built 679 brd: Board object that is being built 680 stage: Stage that we are at (distclean, config, build) 681 cwd: Directory where make should be run 682 args: Arguments to pass to make 683 kwargs: Arguments to pass to command.RunPipe() 684 """ 685 cmd = ['make'] + list(args) 686 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 687 cwd=cwd, raise_on_error=False, **kwargs) 688 return result 689 690 def ProcessResult(self, result): 691 """Process the result of a build, showing progress information 692 693 Args: 694 result: A CommandResult object 695 """ 696 col = terminal.Color() 697 if result: 698 target = result.brd.target 699 700 if result.return_code < 0: 701 self.active = False 702 command.StopAll() 703 return 704 705 self.upto += 1 706 if result.return_code != 0: 707 self.fail += 1 708 elif result.stderr: 709 self.warned += 1 710 if result.already_done: 711 self.already_done += 1 712 else: 713 target = '(starting)' 714 715 # Display separate counts for ok, warned and fail 716 ok = self.upto - self.warned - self.fail 717 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 718 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 719 line += self.col.Color(self.col.RED, '%5d' % self.fail) 720 721 name = ' /%-5d ' % self.count 722 723 # Add our current completion time estimate 724 self._AddTimestamp() 725 if self._complete_delay: 726 name += '%s : ' % self._complete_delay 727 # When building all boards for a commit, we can print a commit 728 # progress message. 729 if result and result.commit_upto is None: 730 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 731 self.commit_count) 732 733 name += target 734 print line + name, 735 length = 13 + len(name) 736 self.ClearLine(length) 737 738 def _GetOutputDir(self, commit_upto): 739 """Get the name of the output directory for a commit number 740 741 The output directory is typically .../<branch>/<commit>. 742 743 Args: 744 commit_upto: Commit number to use (0..self.count-1) 745 """ 746 commit = self.commits[commit_upto] 747 subject = commit.subject.translate(trans_valid_chars) 748 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 749 self.commit_count, commit.hash, subject[:20])) 750 output_dir = os.path.join(self.base_dir, commit_dir) 751 return output_dir 752 753 def GetBuildDir(self, commit_upto, target): 754 """Get the name of the build directory for a commit number 755 756 The build directory is typically .../<branch>/<commit>/<target>. 757 758 Args: 759 commit_upto: Commit number to use (0..self.count-1) 760 target: Target name 761 """ 762 output_dir = self._GetOutputDir(commit_upto) 763 return os.path.join(output_dir, target) 764 765 def GetDoneFile(self, commit_upto, target): 766 """Get the name of the done file for a commit number 767 768 Args: 769 commit_upto: Commit number to use (0..self.count-1) 770 target: Target name 771 """ 772 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 773 774 def GetSizesFile(self, commit_upto, target): 775 """Get the name of the sizes file for a commit number 776 777 Args: 778 commit_upto: Commit number to use (0..self.count-1) 779 target: Target name 780 """ 781 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 782 783 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 784 """Get the name of the funcsizes file for a commit number and ELF file 785 786 Args: 787 commit_upto: Commit number to use (0..self.count-1) 788 target: Target name 789 elf_fname: Filename of elf image 790 """ 791 return os.path.join(self.GetBuildDir(commit_upto, target), 792 '%s.sizes' % elf_fname.replace('/', '-')) 793 794 def GetObjdumpFile(self, commit_upto, target, elf_fname): 795 """Get the name of the objdump file for a commit number and ELF file 796 797 Args: 798 commit_upto: Commit number to use (0..self.count-1) 799 target: Target name 800 elf_fname: Filename of elf image 801 """ 802 return os.path.join(self.GetBuildDir(commit_upto, target), 803 '%s.objdump' % elf_fname.replace('/', '-')) 804 805 def GetErrFile(self, commit_upto, target): 806 """Get the name of the err file for a commit number 807 808 Args: 809 commit_upto: Commit number to use (0..self.count-1) 810 target: Target name 811 """ 812 output_dir = self.GetBuildDir(commit_upto, target) 813 return os.path.join(output_dir, 'err') 814 815 def FilterErrors(self, lines): 816 """Filter out errors in which we have no interest 817 818 We should probably use map(). 819 820 Args: 821 lines: List of error lines, each a string 822 Returns: 823 New list with only interesting lines included 824 """ 825 out_lines = [] 826 for line in lines: 827 if not self.re_make_err.search(line): 828 out_lines.append(line) 829 return out_lines 830 831 def ReadFuncSizes(self, fname, fd): 832 """Read function sizes from the output of 'nm' 833 834 Args: 835 fd: File containing data to read 836 fname: Filename we are reading from (just for errors) 837 838 Returns: 839 Dictionary containing size of each function in bytes, indexed by 840 function name. 841 """ 842 sym = {} 843 for line in fd.readlines(): 844 try: 845 size, type, name = line[:-1].split() 846 except: 847 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 848 continue 849 if type in 'tTdDbB': 850 # function names begin with '.' on 64-bit powerpc 851 if '.' in name[1:]: 852 name = 'static.' + name.split('.')[0] 853 sym[name] = sym.get(name, 0) + int(size, 16) 854 return sym 855 856 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 857 """Work out the outcome of a build. 858 859 Args: 860 commit_upto: Commit number to check (0..n-1) 861 target: Target board to check 862 read_func_sizes: True to read function size information 863 864 Returns: 865 Outcome object 866 """ 867 done_file = self.GetDoneFile(commit_upto, target) 868 sizes_file = self.GetSizesFile(commit_upto, target) 869 sizes = {} 870 func_sizes = {} 871 if os.path.exists(done_file): 872 with open(done_file, 'r') as fd: 873 return_code = int(fd.readline()) 874 err_lines = [] 875 err_file = self.GetErrFile(commit_upto, target) 876 if os.path.exists(err_file): 877 with open(err_file, 'r') as fd: 878 err_lines = self.FilterErrors(fd.readlines()) 879 880 # Decide whether the build was ok, failed or created warnings 881 if return_code: 882 rc = OUTCOME_ERROR 883 elif len(err_lines): 884 rc = OUTCOME_WARNING 885 else: 886 rc = OUTCOME_OK 887 888 # Convert size information to our simple format 889 if os.path.exists(sizes_file): 890 with open(sizes_file, 'r') as fd: 891 for line in fd.readlines(): 892 values = line.split() 893 rodata = 0 894 if len(values) > 6: 895 rodata = int(values[6], 16) 896 size_dict = { 897 'all' : int(values[0]) + int(values[1]) + 898 int(values[2]), 899 'text' : int(values[0]) - rodata, 900 'data' : int(values[1]), 901 'bss' : int(values[2]), 902 'rodata' : rodata, 903 } 904 sizes[values[5]] = size_dict 905 906 if read_func_sizes: 907 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 908 for fname in glob.glob(pattern): 909 with open(fname, 'r') as fd: 910 dict_name = os.path.basename(fname).replace('.sizes', 911 '') 912 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 913 914 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 915 916 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 917 918 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 919 """Calculate a summary of the results of building a commit. 920 921 Args: 922 board_selected: Dict containing boards to summarise 923 commit_upto: Commit number to summarize (0..self.count-1) 924 read_func_sizes: True to read function size information 925 926 Returns: 927 Tuple: 928 Dict containing boards which passed building this commit. 929 keyed by board.target 930 List containing a summary of error/warning lines 931 """ 932 board_dict = {} 933 err_lines_summary = [] 934 935 for board in boards_selected.itervalues(): 936 outcome = self.GetBuildOutcome(commit_upto, board.target, 937 read_func_sizes) 938 board_dict[board.target] = outcome 939 for err in outcome.err_lines: 940 if err and not err.rstrip() in err_lines_summary: 941 err_lines_summary.append(err.rstrip()) 942 return board_dict, err_lines_summary 943 944 def AddOutcome(self, board_dict, arch_list, changes, char, color): 945 """Add an output to our list of outcomes for each architecture 946 947 This simple function adds failing boards (changes) to the 948 relevant architecture string, so we can print the results out 949 sorted by architecture. 950 951 Args: 952 board_dict: Dict containing all boards 953 arch_list: Dict keyed by arch name. Value is a string containing 954 a list of board names which failed for that arch. 955 changes: List of boards to add to arch_list 956 color: terminal.Colour object 957 """ 958 done_arch = {} 959 for target in changes: 960 if target in board_dict: 961 arch = board_dict[target].arch 962 else: 963 arch = 'unknown' 964 str = self.col.Color(color, ' ' + target) 965 if not arch in done_arch: 966 str = self.col.Color(color, char) + ' ' + str 967 done_arch[arch] = True 968 if not arch in arch_list: 969 arch_list[arch] = str 970 else: 971 arch_list[arch] += str 972 973 974 def ColourNum(self, num): 975 color = self.col.RED if num > 0 else self.col.GREEN 976 if num == 0: 977 return '0' 978 return self.col.Color(color, str(num)) 979 980 def ResetResultSummary(self, board_selected): 981 """Reset the results summary ready for use. 982 983 Set up the base board list to be all those selected, and set the 984 error lines to empty. 985 986 Following this, calls to PrintResultSummary() will use this 987 information to work out what has changed. 988 989 Args: 990 board_selected: Dict containing boards to summarise, keyed by 991 board.target 992 """ 993 self._base_board_dict = {} 994 for board in board_selected: 995 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 996 self._base_err_lines = [] 997 998 def PrintFuncSizeDetail(self, fname, old, new): 999 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 1000 delta, common = [], {} 1001 1002 for a in old: 1003 if a in new: 1004 common[a] = 1 1005 1006 for name in old: 1007 if name not in common: 1008 remove += 1 1009 down += old[name] 1010 delta.append([-old[name], name]) 1011 1012 for name in new: 1013 if name not in common: 1014 add += 1 1015 up += new[name] 1016 delta.append([new[name], name]) 1017 1018 for name in common: 1019 diff = new.get(name, 0) - old.get(name, 0) 1020 if diff > 0: 1021 grow, up = grow + 1, up + diff 1022 elif diff < 0: 1023 shrink, down = shrink + 1, down - diff 1024 delta.append([diff, name]) 1025 1026 delta.sort() 1027 delta.reverse() 1028 1029 args = [add, -remove, grow, -shrink, up, -down, up - down] 1030 if max(args) == 0: 1031 return 1032 args = [self.ColourNum(x) for x in args] 1033 indent = ' ' * 15 1034 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1035 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1036 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1037 'delta') 1038 for diff, name in delta: 1039 if diff: 1040 color = self.col.RED if diff > 0 else self.col.GREEN 1041 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1042 old.get(name, '-'), new.get(name,'-'), diff) 1043 print self.col.Color(color, msg) 1044 1045 1046 def PrintSizeDetail(self, target_list, show_bloat): 1047 """Show details size information for each board 1048 1049 Args: 1050 target_list: List of targets, each a dict containing: 1051 'target': Target name 1052 'total_diff': Total difference in bytes across all areas 1053 <part_name>: Difference for that part 1054 show_bloat: Show detail for each function 1055 """ 1056 targets_by_diff = sorted(target_list, reverse=True, 1057 key=lambda x: x['_total_diff']) 1058 for result in targets_by_diff: 1059 printed_target = False 1060 for name in sorted(result): 1061 diff = result[name] 1062 if name.startswith('_'): 1063 continue 1064 if diff != 0: 1065 color = self.col.RED if diff > 0 else self.col.GREEN 1066 msg = ' %s %+d' % (name, diff) 1067 if not printed_target: 1068 print '%10s %-15s:' % ('', result['_target']), 1069 printed_target = True 1070 print self.col.Color(color, msg), 1071 if printed_target: 1072 print 1073 if show_bloat: 1074 target = result['_target'] 1075 outcome = result['_outcome'] 1076 base_outcome = self._base_board_dict[target] 1077 for fname in outcome.func_sizes: 1078 self.PrintFuncSizeDetail(fname, 1079 base_outcome.func_sizes[fname], 1080 outcome.func_sizes[fname]) 1081 1082 1083 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1084 show_bloat): 1085 """Print a summary of image sizes broken down by section. 1086 1087 The summary takes the form of one line per architecture. The 1088 line contains deltas for each of the sections (+ means the section 1089 got bigger, - means smaller). The nunmbers are the average number 1090 of bytes that a board in this section increased by. 1091 1092 For example: 1093 powerpc: (622 boards) text -0.0 1094 arm: (285 boards) text -0.0 1095 nds32: (3 boards) text -8.0 1096 1097 Args: 1098 board_selected: Dict containing boards to summarise, keyed by 1099 board.target 1100 board_dict: Dict containing boards for which we built this 1101 commit, keyed by board.target. The value is an Outcome object. 1102 show_detail: Show detail for each board 1103 show_bloat: Show detail for each function 1104 """ 1105 arch_list = {} 1106 arch_count = {} 1107 1108 # Calculate changes in size for different image parts 1109 # The previous sizes are in Board.sizes, for each board 1110 for target in board_dict: 1111 if target not in board_selected: 1112 continue 1113 base_sizes = self._base_board_dict[target].sizes 1114 outcome = board_dict[target] 1115 sizes = outcome.sizes 1116 1117 # Loop through the list of images, creating a dict of size 1118 # changes for each image/part. We end up with something like 1119 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1120 # which means that U-Boot data increased by 5 bytes and SPL 1121 # text decreased by 4. 1122 err = {'_target' : target} 1123 for image in sizes: 1124 if image in base_sizes: 1125 base_image = base_sizes[image] 1126 # Loop through the text, data, bss parts 1127 for part in sorted(sizes[image]): 1128 diff = sizes[image][part] - base_image[part] 1129 col = None 1130 if diff: 1131 if image == 'u-boot': 1132 name = part 1133 else: 1134 name = image + ':' + part 1135 err[name] = diff 1136 arch = board_selected[target].arch 1137 if not arch in arch_count: 1138 arch_count[arch] = 1 1139 else: 1140 arch_count[arch] += 1 1141 if not sizes: 1142 pass # Only add to our list when we have some stats 1143 elif not arch in arch_list: 1144 arch_list[arch] = [err] 1145 else: 1146 arch_list[arch].append(err) 1147 1148 # We now have a list of image size changes sorted by arch 1149 # Print out a summary of these 1150 for arch, target_list in arch_list.iteritems(): 1151 # Get total difference for each type 1152 totals = {} 1153 for result in target_list: 1154 total = 0 1155 for name, diff in result.iteritems(): 1156 if name.startswith('_'): 1157 continue 1158 total += diff 1159 if name in totals: 1160 totals[name] += diff 1161 else: 1162 totals[name] = diff 1163 result['_total_diff'] = total 1164 result['_outcome'] = board_dict[result['_target']] 1165 1166 count = len(target_list) 1167 printed_arch = False 1168 for name in sorted(totals): 1169 diff = totals[name] 1170 if diff: 1171 # Display the average difference in this name for this 1172 # architecture 1173 avg_diff = float(diff) / count 1174 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1175 msg = ' %s %+1.1f' % (name, avg_diff) 1176 if not printed_arch: 1177 print '%10s: (for %d/%d boards)' % (arch, count, 1178 arch_count[arch]), 1179 printed_arch = True 1180 print self.col.Color(color, msg), 1181 1182 if printed_arch: 1183 print 1184 if show_detail: 1185 self.PrintSizeDetail(target_list, show_bloat) 1186 1187 1188 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1189 show_sizes, show_detail, show_bloat): 1190 """Compare results with the base results and display delta. 1191 1192 Only boards mentioned in board_selected will be considered. This 1193 function is intended to be called repeatedly with the results of 1194 each commit. It therefore shows a 'diff' between what it saw in 1195 the last call and what it sees now. 1196 1197 Args: 1198 board_selected: Dict containing boards to summarise, keyed by 1199 board.target 1200 board_dict: Dict containing boards for which we built this 1201 commit, keyed by board.target. The value is an Outcome object. 1202 err_lines: A list of errors for this commit, or [] if there is 1203 none, or we don't want to print errors 1204 show_sizes: Show image size deltas 1205 show_detail: Show detail for each board 1206 show_bloat: Show detail for each function 1207 """ 1208 better = [] # List of boards fixed since last commit 1209 worse = [] # List of new broken boards since last commit 1210 new = [] # List of boards that didn't exist last time 1211 unknown = [] # List of boards that were not built 1212 1213 for target in board_dict: 1214 if target not in board_selected: 1215 continue 1216 1217 # If the board was built last time, add its outcome to a list 1218 if target in self._base_board_dict: 1219 base_outcome = self._base_board_dict[target].rc 1220 outcome = board_dict[target] 1221 if outcome.rc == OUTCOME_UNKNOWN: 1222 unknown.append(target) 1223 elif outcome.rc < base_outcome: 1224 better.append(target) 1225 elif outcome.rc > base_outcome: 1226 worse.append(target) 1227 else: 1228 new.append(target) 1229 1230 # Get a list of errors that have appeared, and disappeared 1231 better_err = [] 1232 worse_err = [] 1233 for line in err_lines: 1234 if line not in self._base_err_lines: 1235 worse_err.append('+' + line) 1236 for line in self._base_err_lines: 1237 if line not in err_lines: 1238 better_err.append('-' + line) 1239 1240 # Display results by arch 1241 if better or worse or unknown or new or worse_err or better_err: 1242 arch_list = {} 1243 self.AddOutcome(board_selected, arch_list, better, '', 1244 self.col.GREEN) 1245 self.AddOutcome(board_selected, arch_list, worse, '+', 1246 self.col.RED) 1247 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1248 if self._show_unknown: 1249 self.AddOutcome(board_selected, arch_list, unknown, '?', 1250 self.col.MAGENTA) 1251 for arch, target_list in arch_list.iteritems(): 1252 print '%10s: %s' % (arch, target_list) 1253 if better_err: 1254 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1255 if worse_err: 1256 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1257 1258 if show_sizes: 1259 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1260 show_bloat) 1261 1262 # Save our updated information for the next call to this function 1263 self._base_board_dict = board_dict 1264 self._base_err_lines = err_lines 1265 1266 # Get a list of boards that did not get built, if needed 1267 not_built = [] 1268 for board in board_selected: 1269 if not board in board_dict: 1270 not_built.append(board) 1271 if not_built: 1272 print "Boards not built (%d): %s" % (len(not_built), 1273 ', '.join(not_built)) 1274 1275 1276 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1277 show_detail, show_bloat): 1278 """Show a build summary for U-Boot for a given board list. 1279 1280 Reset the result summary, then repeatedly call GetResultSummary on 1281 each commit's results, then display the differences we see. 1282 1283 Args: 1284 commit: Commit objects to summarise 1285 board_selected: Dict containing boards to summarise 1286 show_errors: Show errors that occured 1287 show_sizes: Show size deltas 1288 show_detail: Show detail for each board 1289 show_bloat: Show detail for each function 1290 """ 1291 self.commit_count = len(commits) 1292 self.commits = commits 1293 self.ResetResultSummary(board_selected) 1294 1295 for commit_upto in range(0, self.commit_count, self._step): 1296 board_dict, err_lines = self.GetResultSummary(board_selected, 1297 commit_upto, read_func_sizes=show_bloat) 1298 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1299 print self.col.Color(self.col.BLUE, msg) 1300 self.PrintResultSummary(board_selected, board_dict, 1301 err_lines if show_errors else [], show_sizes, show_detail, 1302 show_bloat) 1303 1304 1305 def SetupBuild(self, board_selected, commits): 1306 """Set up ready to start a build. 1307 1308 Args: 1309 board_selected: Selected boards to build 1310 commits: Selected commits to build 1311 """ 1312 # First work out how many commits we will build 1313 count = (len(commits) + self._step - 1) / self._step 1314 self.count = len(board_selected) * count 1315 self.upto = self.warned = self.fail = 0 1316 self._timestamps = collections.deque() 1317 1318 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1319 """Build all boards for a single commit""" 1320 self.SetupBuild(board_selected) 1321 self.count = len(board_selected) 1322 for brd in board_selected.itervalues(): 1323 job = BuilderJob() 1324 job.board = brd 1325 job.commits = None 1326 job.keep_outputs = keep_outputs 1327 self.queue.put(brd) 1328 1329 self.queue.join() 1330 self.out_queue.join() 1331 print 1332 self.ClearLine(0) 1333 1334 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1335 """Build all boards for all commits (non-incremental)""" 1336 self.commit_count = len(commits) 1337 1338 self.ResetResultSummary(board_selected) 1339 for self.commit_upto in range(self.commit_count): 1340 self.SelectCommit(commits[self.commit_upto]) 1341 self.SelectOutputDir() 1342 Mkdir(self.output_dir) 1343 1344 self.BuildBoardsForCommit(board_selected, keep_outputs) 1345 board_dict, err_lines = self.GetResultSummary() 1346 self.PrintResultSummary(board_selected, board_dict, 1347 err_lines if show_errors else []) 1348 1349 if self.already_done: 1350 print '%d builds already done' % self.already_done 1351 1352 def GetThreadDir(self, thread_num): 1353 """Get the directory path to the working dir for a thread. 1354 1355 Args: 1356 thread_num: Number of thread to check. 1357 """ 1358 return os.path.join(self._working_dir, '%02d' % thread_num) 1359 1360 def _PrepareThread(self, thread_num): 1361 """Prepare the working directory for a thread. 1362 1363 This clones or fetches the repo into the thread's work directory. 1364 1365 Args: 1366 thread_num: Thread number (0, 1, ...) 1367 """ 1368 thread_dir = self.GetThreadDir(thread_num) 1369 Mkdir(thread_dir) 1370 git_dir = os.path.join(thread_dir, '.git') 1371 1372 # Clone the repo if it doesn't already exist 1373 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1374 # we have a private index but uses the origin repo's contents? 1375 if self.git_dir: 1376 src_dir = os.path.abspath(self.git_dir) 1377 if os.path.exists(git_dir): 1378 gitutil.Fetch(git_dir, thread_dir) 1379 else: 1380 print 'Cloning repo for thread %d' % thread_num 1381 gitutil.Clone(src_dir, thread_dir) 1382 1383 def _PrepareWorkingSpace(self, max_threads): 1384 """Prepare the working directory for use. 1385 1386 Set up the git repo for each thread. 1387 1388 Args: 1389 max_threads: Maximum number of threads we expect to need. 1390 """ 1391 Mkdir(self._working_dir) 1392 for thread in range(max_threads): 1393 self._PrepareThread(thread) 1394 1395 def _PrepareOutputSpace(self): 1396 """Get the output directories ready to receive files. 1397 1398 We delete any output directories which look like ones we need to 1399 create. Having left over directories is confusing when the user wants 1400 to check the output manually. 1401 """ 1402 dir_list = [] 1403 for commit_upto in range(self.commit_count): 1404 dir_list.append(self._GetOutputDir(commit_upto)) 1405 1406 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1407 if dirname not in dir_list: 1408 shutil.rmtree(dirname) 1409 1410 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1411 """Build all commits for a list of boards 1412 1413 Args: 1414 commits: List of commits to be build, each a Commit object 1415 boards_selected: Dict of selected boards, key is target name, 1416 value is Board object 1417 show_errors: True to show summarised error/warning info 1418 keep_outputs: True to save build output files 1419 """ 1420 self.commit_count = len(commits) 1421 self.commits = commits 1422 1423 self.ResetResultSummary(board_selected) 1424 Mkdir(self.base_dir) 1425 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1426 self._PrepareOutputSpace() 1427 self.SetupBuild(board_selected, commits) 1428 self.ProcessResult(None) 1429 1430 # Create jobs to build all commits for each board 1431 for brd in board_selected.itervalues(): 1432 job = BuilderJob() 1433 job.board = brd 1434 job.commits = commits 1435 job.keep_outputs = keep_outputs 1436 job.step = self._step 1437 self.queue.put(job) 1438 1439 # Wait until all jobs are started 1440 self.queue.join() 1441 1442 # Wait until we have processed all output 1443 self.out_queue.join() 1444 print 1445 self.ClearLine(0) 1446