1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build): 192 """Build a particular commit. 193 194 If the build is already done, and we are not forcing a build, we skip 195 the build and just return the previously-saved results. 196 197 Args: 198 commit_upto: Commit number to build (0...n-1) 199 brd: Board object to build 200 work_dir: Directory to which the source will be checked out 201 do_config: True to run a make <board>_config on the source 202 force_build: Force a build even if one was previously done 203 204 Returns: 205 tuple containing: 206 - CommandResult object containing the results of the build 207 - boolean indicating whether 'make config' is still needed 208 """ 209 # Create a default result - it will be overwritte by the call to 210 # self.Make() below, in the event that we do a build. 211 result = command.CommandResult() 212 result.return_code = 0 213 out_dir = os.path.join(work_dir, 'build') 214 215 # Check if the job was already completed last time 216 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 217 result.already_done = os.path.exists(done_file) 218 if result.already_done and not force_build: 219 # Get the return code from that build and use it 220 with open(done_file, 'r') as fd: 221 result.return_code = int(fd.readline()) 222 err_file = self.builder.GetErrFile(commit_upto, brd.target) 223 if os.path.exists(err_file) and os.stat(err_file).st_size: 224 result.stderr = 'bad' 225 else: 226 # We are going to have to build it. First, get a toolchain 227 if not self.toolchain: 228 try: 229 self.toolchain = self.builder.toolchains.Select(brd.arch) 230 except ValueError as err: 231 result.return_code = 10 232 result.stdout = '' 233 result.stderr = str(err) 234 # TODO(sjg@chromium.org): This gets swallowed, but needs 235 # to be reported. 236 237 if self.toolchain: 238 # Checkout the right commit 239 if commit_upto is not None: 240 commit = self.builder.commits[commit_upto] 241 if self.builder.checkout: 242 git_dir = os.path.join(work_dir, '.git') 243 gitutil.Checkout(commit.hash, git_dir, work_dir, 244 force=True) 245 else: 246 commit = self.builder.commit # Ick, fix this for BuildCommits() 247 248 # Set up the environment and command line 249 env = self.toolchain.MakeEnvironment() 250 Mkdir(out_dir) 251 args = ['O=build', '-s'] 252 if self.builder.num_jobs is not None: 253 args.extend(['-j', str(self.builder.num_jobs)]) 254 config_args = ['%s_config' % brd.target] 255 config_out = '' 256 257 # If we need to reconfigure, do that now 258 if do_config: 259 result = self.Make(commit, brd, 'distclean', work_dir, 260 'distclean', *args, env=env) 261 result = self.Make(commit, brd, 'config', work_dir, 262 *(args + config_args), env=env) 263 config_out = result.combined 264 do_config = False # No need to configure next time 265 if result.return_code == 0: 266 result = self.Make(commit, brd, 'build', work_dir, *args, 267 env=env) 268 result.stdout = config_out + result.stdout 269 else: 270 result.return_code = 1 271 result.stderr = 'No tool chain for %s\n' % brd.arch 272 result.already_done = False 273 274 result.toolchain = self.toolchain 275 result.brd = brd 276 result.commit_upto = commit_upto 277 result.out_dir = out_dir 278 return result, do_config 279 280 def _WriteResult(self, result, keep_outputs): 281 """Write a built result to the output directory. 282 283 Args: 284 result: CommandResult object containing result to write 285 keep_outputs: True to store the output binaries, False 286 to delete them 287 """ 288 # Fatal error 289 if result.return_code < 0: 290 return 291 292 # Aborted? 293 if result.stderr and 'No child processes' in result.stderr: 294 return 295 296 if result.already_done: 297 return 298 299 # Write the output and stderr 300 output_dir = self.builder._GetOutputDir(result.commit_upto) 301 Mkdir(output_dir) 302 build_dir = self.builder.GetBuildDir(result.commit_upto, 303 result.brd.target) 304 Mkdir(build_dir) 305 306 outfile = os.path.join(build_dir, 'log') 307 with open(outfile, 'w') as fd: 308 if result.stdout: 309 fd.write(result.stdout) 310 311 errfile = self.builder.GetErrFile(result.commit_upto, 312 result.brd.target) 313 if result.stderr: 314 with open(errfile, 'w') as fd: 315 fd.write(result.stderr) 316 elif os.path.exists(errfile): 317 os.remove(errfile) 318 319 if result.toolchain: 320 # Write the build result and toolchain information. 321 done_file = self.builder.GetDoneFile(result.commit_upto, 322 result.brd.target) 323 with open(done_file, 'w') as fd: 324 fd.write('%s' % result.return_code) 325 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 326 print >>fd, 'gcc', result.toolchain.gcc 327 print >>fd, 'path', result.toolchain.path 328 print >>fd, 'cross', result.toolchain.cross 329 print >>fd, 'arch', result.toolchain.arch 330 fd.write('%s' % result.return_code) 331 332 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 333 print >>fd, 'gcc', result.toolchain.gcc 334 print >>fd, 'path', result.toolchain.path 335 336 # Write out the image and function size information and an objdump 337 env = result.toolchain.MakeEnvironment() 338 lines = [] 339 for fname in ['u-boot', 'spl/u-boot-spl']: 340 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 341 nm_result = command.RunPipe([cmd], capture=True, 342 capture_stderr=True, cwd=result.out_dir, 343 raise_on_error=False, env=env) 344 if nm_result.stdout: 345 nm = self.builder.GetFuncSizesFile(result.commit_upto, 346 result.brd.target, fname) 347 with open(nm, 'w') as fd: 348 print >>fd, nm_result.stdout, 349 350 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 351 dump_result = command.RunPipe([cmd], capture=True, 352 capture_stderr=True, cwd=result.out_dir, 353 raise_on_error=False, env=env) 354 rodata_size = '' 355 if dump_result.stdout: 356 objdump = self.builder.GetObjdumpFile(result.commit_upto, 357 result.brd.target, fname) 358 with open(objdump, 'w') as fd: 359 print >>fd, dump_result.stdout, 360 for line in dump_result.stdout.splitlines(): 361 fields = line.split() 362 if len(fields) > 5 and fields[1] == '.rodata': 363 rodata_size = fields[2] 364 365 cmd = ['%ssize' % self.toolchain.cross, fname] 366 size_result = command.RunPipe([cmd], capture=True, 367 capture_stderr=True, cwd=result.out_dir, 368 raise_on_error=False, env=env) 369 if size_result.stdout: 370 lines.append(size_result.stdout.splitlines()[1] + ' ' + 371 rodata_size) 372 373 # Write out the image sizes file. This is similar to the output 374 # of binutil's 'size' utility, but it omits the header line and 375 # adds an additional hex value at the end of each line for the 376 # rodata size 377 if len(lines): 378 sizes = self.builder.GetSizesFile(result.commit_upto, 379 result.brd.target) 380 with open(sizes, 'w') as fd: 381 print >>fd, '\n'.join(lines) 382 383 # Now write the actual build output 384 if keep_outputs: 385 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 386 'include/autoconf.mk', 'spl/u-boot-spl', 387 'spl/u-boot-spl.bin'] 388 for pattern in patterns: 389 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 390 for fname in file_list: 391 shutil.copy(fname, build_dir) 392 393 394 def RunJob(self, job): 395 """Run a single job 396 397 A job consists of a building a list of commits for a particular board. 398 399 Args: 400 job: Job to build 401 """ 402 brd = job.board 403 work_dir = self.builder.GetThreadDir(self.thread_num) 404 self.toolchain = None 405 if job.commits: 406 # Run 'make board_config' on the first commit 407 do_config = True 408 commit_upto = 0 409 force_build = False 410 for commit_upto in range(0, len(job.commits), job.step): 411 result, request_config = self.RunCommit(commit_upto, brd, 412 work_dir, do_config, 413 force_build or self.builder.force_build) 414 failed = result.return_code or result.stderr 415 if failed and not do_config: 416 # If our incremental build failed, try building again 417 # with a reconfig. 418 if self.builder.force_config_on_failure: 419 result, request_config = self.RunCommit(commit_upto, 420 brd, work_dir, True, True) 421 do_config = request_config 422 423 # If we built that commit, then config is done. But if we got 424 # an warning, reconfig next time to force it to build the same 425 # files that created warnings this time. Otherwise an 426 # incremental build may not build the same file, and we will 427 # think that the warning has gone away. 428 # We could avoid this by using -Werror everywhere... 429 # For errors, the problem doesn't happen, since presumably 430 # the build stopped and didn't generate output, so will retry 431 # that file next time. So we could detect warnings and deal 432 # with them specially here. For now, we just reconfigure if 433 # anything goes work. 434 # Of course this is substantially slower if there are build 435 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 436 # have problems). 437 if (failed and not result.already_done and not do_config and 438 self.builder.force_config_on_failure): 439 # If this build failed, try the next one with a 440 # reconfigure. 441 # Sometimes if the board_config.h file changes it can mess 442 # with dependencies, and we get: 443 # make: *** No rule to make target `include/autoconf.mk', 444 # needed by `depend'. 445 do_config = True 446 force_build = True 447 else: 448 force_build = False 449 if self.builder.force_config_on_failure: 450 if failed: 451 do_config = True 452 result.commit_upto = commit_upto 453 if result.return_code < 0: 454 raise ValueError('Interrupt') 455 456 # We have the build results, so output the result 457 self._WriteResult(result, job.keep_outputs) 458 self.builder.out_queue.put(result) 459 else: 460 # Just build the currently checked-out build 461 result = self.RunCommit(None, True) 462 result.commit_upto = self.builder.upto 463 self.builder.out_queue.put(result) 464 465 def run(self): 466 """Our thread's run function 467 468 This thread picks a job from the queue, runs it, and then goes to the 469 next job. 470 """ 471 alive = True 472 while True: 473 job = self.builder.queue.get() 474 try: 475 if self.builder.active and alive: 476 self.RunJob(job) 477 except Exception as err: 478 alive = False 479 print err 480 self.builder.queue.task_done() 481 482 483class Builder: 484 """Class for building U-Boot for a particular commit. 485 486 Public members: (many should ->private) 487 active: True if the builder is active and has not been stopped 488 already_done: Number of builds already completed 489 base_dir: Base directory to use for builder 490 checkout: True to check out source, False to skip that step. 491 This is used for testing. 492 col: terminal.Color() object 493 count: Number of commits to build 494 do_make: Method to call to invoke Make 495 fail: Number of builds that failed due to error 496 force_build: Force building even if a build already exists 497 force_config_on_failure: If a commit fails for a board, disable 498 incremental building for the next commit we build for that 499 board, so that we will see all warnings/errors again. 500 git_dir: Git directory containing source repository 501 last_line_len: Length of the last line we printed (used for erasing 502 it with new progress information) 503 num_jobs: Number of jobs to run at once (passed to make as -j) 504 num_threads: Number of builder threads to run 505 out_queue: Queue of results to process 506 re_make_err: Compiled regular expression for ignore_lines 507 queue: Queue of jobs to run 508 threads: List of active threads 509 toolchains: Toolchains object to use for building 510 upto: Current commit number we are building (0.count-1) 511 warned: Number of builds that produced at least one warning 512 513 Private members: 514 _base_board_dict: Last-summarised Dict of boards 515 _base_err_lines: Last-summarised list of errors 516 _build_period_us: Time taken for a single build (float object). 517 _complete_delay: Expected delay until completion (timedelta) 518 _next_delay_update: Next time we plan to display a progress update 519 (datatime) 520 _show_unknown: Show unknown boards (those not built) in summary 521 _timestamps: List of timestamps for the completion of the last 522 last _timestamp_count builds. Each is a datetime object. 523 _timestamp_count: Number of timestamps to keep in our list. 524 _working_dir: Base working directory containing all threads 525 """ 526 class Outcome: 527 """Records a build outcome for a single make invocation 528 529 Public Members: 530 rc: Outcome value (OUTCOME_...) 531 err_lines: List of error lines or [] if none 532 sizes: Dictionary of image size information, keyed by filename 533 - Each value is itself a dictionary containing 534 values for 'text', 'data' and 'bss', being the integer 535 size in bytes of each section. 536 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 537 value is itself a dictionary: 538 key: function name 539 value: Size of function in bytes 540 """ 541 def __init__(self, rc, err_lines, sizes, func_sizes): 542 self.rc = rc 543 self.err_lines = err_lines 544 self.sizes = sizes 545 self.func_sizes = func_sizes 546 547 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 548 checkout=True, show_unknown=True, step=1): 549 """Create a new Builder object 550 551 Args: 552 toolchains: Toolchains object to use for building 553 base_dir: Base directory to use for builder 554 git_dir: Git directory containing source repository 555 num_threads: Number of builder threads to run 556 num_jobs: Number of jobs to run at once (passed to make as -j) 557 checkout: True to check out source, False to skip that step. 558 This is used for testing. 559 show_unknown: Show unknown boards (those not built) in summary 560 step: 1 to process every commit, n to process every nth commit 561 """ 562 self.toolchains = toolchains 563 self.base_dir = base_dir 564 self._working_dir = os.path.join(base_dir, '.bm-work') 565 self.threads = [] 566 self.active = True 567 self.do_make = self.Make 568 self.checkout = checkout 569 self.num_threads = num_threads 570 self.num_jobs = num_jobs 571 self.already_done = 0 572 self.force_build = False 573 self.git_dir = git_dir 574 self._show_unknown = show_unknown 575 self._timestamp_count = 10 576 self._build_period_us = None 577 self._complete_delay = None 578 self._next_delay_update = datetime.now() 579 self.force_config_on_failure = True 580 self._step = step 581 582 self.col = terminal.Color() 583 584 self.queue = Queue.Queue() 585 self.out_queue = Queue.Queue() 586 for i in range(self.num_threads): 587 t = BuilderThread(self, i) 588 t.setDaemon(True) 589 t.start() 590 self.threads.append(t) 591 592 self.last_line_len = 0 593 t = ResultThread(self) 594 t.setDaemon(True) 595 t.start() 596 self.threads.append(t) 597 598 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 599 self.re_make_err = re.compile('|'.join(ignore_lines)) 600 601 def __del__(self): 602 """Get rid of all threads created by the builder""" 603 for t in self.threads: 604 del t 605 606 def _AddTimestamp(self): 607 """Add a new timestamp to the list and record the build period. 608 609 The build period is the length of time taken to perform a single 610 build (one board, one commit). 611 """ 612 now = datetime.now() 613 self._timestamps.append(now) 614 count = len(self._timestamps) 615 delta = self._timestamps[-1] - self._timestamps[0] 616 seconds = delta.total_seconds() 617 618 # If we have enough data, estimate build period (time taken for a 619 # single build) and therefore completion time. 620 if count > 1 and self._next_delay_update < now: 621 self._next_delay_update = now + timedelta(seconds=2) 622 if seconds > 0: 623 self._build_period = float(seconds) / count 624 todo = self.count - self.upto 625 self._complete_delay = timedelta(microseconds= 626 self._build_period * todo * 1000000) 627 # Round it 628 self._complete_delay -= timedelta( 629 microseconds=self._complete_delay.microseconds) 630 631 if seconds > 60: 632 self._timestamps.popleft() 633 count -= 1 634 635 def ClearLine(self, length): 636 """Clear any characters on the current line 637 638 Make way for a new line of length 'length', by outputting enough 639 spaces to clear out the old line. Then remember the new length for 640 next time. 641 642 Args: 643 length: Length of new line, in characters 644 """ 645 if length < self.last_line_len: 646 print ' ' * (self.last_line_len - length), 647 print '\r', 648 self.last_line_len = length 649 sys.stdout.flush() 650 651 def SelectCommit(self, commit, checkout=True): 652 """Checkout the selected commit for this build 653 """ 654 self.commit = commit 655 if checkout and self.checkout: 656 gitutil.Checkout(commit.hash) 657 658 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 659 """Run make 660 661 Args: 662 commit: Commit object that is being built 663 brd: Board object that is being built 664 stage: Stage that we are at (distclean, config, build) 665 cwd: Directory where make should be run 666 args: Arguments to pass to make 667 kwargs: Arguments to pass to command.RunPipe() 668 """ 669 cmd = ['make'] + list(args) 670 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 671 cwd=cwd, raise_on_error=False, **kwargs) 672 return result 673 674 def ProcessResult(self, result): 675 """Process the result of a build, showing progress information 676 677 Args: 678 result: A CommandResult object 679 """ 680 col = terminal.Color() 681 if result: 682 target = result.brd.target 683 684 if result.return_code < 0: 685 self.active = False 686 command.StopAll() 687 return 688 689 self.upto += 1 690 if result.return_code != 0: 691 self.fail += 1 692 elif result.stderr: 693 self.warned += 1 694 if result.already_done: 695 self.already_done += 1 696 else: 697 target = '(starting)' 698 699 # Display separate counts for ok, warned and fail 700 ok = self.upto - self.warned - self.fail 701 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 702 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 703 line += self.col.Color(self.col.RED, '%5d' % self.fail) 704 705 name = ' /%-5d ' % self.count 706 707 # Add our current completion time estimate 708 self._AddTimestamp() 709 if self._complete_delay: 710 name += '%s : ' % self._complete_delay 711 # When building all boards for a commit, we can print a commit 712 # progress message. 713 if result and result.commit_upto is None: 714 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 715 self.commit_count) 716 717 name += target 718 print line + name, 719 length = 13 + len(name) 720 self.ClearLine(length) 721 722 def _GetOutputDir(self, commit_upto): 723 """Get the name of the output directory for a commit number 724 725 The output directory is typically .../<branch>/<commit>. 726 727 Args: 728 commit_upto: Commit number to use (0..self.count-1) 729 """ 730 commit = self.commits[commit_upto] 731 subject = commit.subject.translate(trans_valid_chars) 732 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 733 self.commit_count, commit.hash, subject[:20])) 734 output_dir = os.path.join(self.base_dir, commit_dir) 735 return output_dir 736 737 def GetBuildDir(self, commit_upto, target): 738 """Get the name of the build directory for a commit number 739 740 The build directory is typically .../<branch>/<commit>/<target>. 741 742 Args: 743 commit_upto: Commit number to use (0..self.count-1) 744 target: Target name 745 """ 746 output_dir = self._GetOutputDir(commit_upto) 747 return os.path.join(output_dir, target) 748 749 def GetDoneFile(self, commit_upto, target): 750 """Get the name of the done file for a commit number 751 752 Args: 753 commit_upto: Commit number to use (0..self.count-1) 754 target: Target name 755 """ 756 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 757 758 def GetSizesFile(self, commit_upto, target): 759 """Get the name of the sizes file for a commit number 760 761 Args: 762 commit_upto: Commit number to use (0..self.count-1) 763 target: Target name 764 """ 765 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 766 767 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 768 """Get the name of the funcsizes file for a commit number and ELF file 769 770 Args: 771 commit_upto: Commit number to use (0..self.count-1) 772 target: Target name 773 elf_fname: Filename of elf image 774 """ 775 return os.path.join(self.GetBuildDir(commit_upto, target), 776 '%s.sizes' % elf_fname.replace('/', '-')) 777 778 def GetObjdumpFile(self, commit_upto, target, elf_fname): 779 """Get the name of the objdump file for a commit number and ELF file 780 781 Args: 782 commit_upto: Commit number to use (0..self.count-1) 783 target: Target name 784 elf_fname: Filename of elf image 785 """ 786 return os.path.join(self.GetBuildDir(commit_upto, target), 787 '%s.objdump' % elf_fname.replace('/', '-')) 788 789 def GetErrFile(self, commit_upto, target): 790 """Get the name of the err file for a commit number 791 792 Args: 793 commit_upto: Commit number to use (0..self.count-1) 794 target: Target name 795 """ 796 output_dir = self.GetBuildDir(commit_upto, target) 797 return os.path.join(output_dir, 'err') 798 799 def FilterErrors(self, lines): 800 """Filter out errors in which we have no interest 801 802 We should probably use map(). 803 804 Args: 805 lines: List of error lines, each a string 806 Returns: 807 New list with only interesting lines included 808 """ 809 out_lines = [] 810 for line in lines: 811 if not self.re_make_err.search(line): 812 out_lines.append(line) 813 return out_lines 814 815 def ReadFuncSizes(self, fname, fd): 816 """Read function sizes from the output of 'nm' 817 818 Args: 819 fd: File containing data to read 820 fname: Filename we are reading from (just for errors) 821 822 Returns: 823 Dictionary containing size of each function in bytes, indexed by 824 function name. 825 """ 826 sym = {} 827 for line in fd.readlines(): 828 try: 829 size, type, name = line[:-1].split() 830 except: 831 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 832 continue 833 if type in 'tTdDbB': 834 # function names begin with '.' on 64-bit powerpc 835 if '.' in name[1:]: 836 name = 'static.' + name.split('.')[0] 837 sym[name] = sym.get(name, 0) + int(size, 16) 838 return sym 839 840 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 841 """Work out the outcome of a build. 842 843 Args: 844 commit_upto: Commit number to check (0..n-1) 845 target: Target board to check 846 read_func_sizes: True to read function size information 847 848 Returns: 849 Outcome object 850 """ 851 done_file = self.GetDoneFile(commit_upto, target) 852 sizes_file = self.GetSizesFile(commit_upto, target) 853 sizes = {} 854 func_sizes = {} 855 if os.path.exists(done_file): 856 with open(done_file, 'r') as fd: 857 return_code = int(fd.readline()) 858 err_lines = [] 859 err_file = self.GetErrFile(commit_upto, target) 860 if os.path.exists(err_file): 861 with open(err_file, 'r') as fd: 862 err_lines = self.FilterErrors(fd.readlines()) 863 864 # Decide whether the build was ok, failed or created warnings 865 if return_code: 866 rc = OUTCOME_ERROR 867 elif len(err_lines): 868 rc = OUTCOME_WARNING 869 else: 870 rc = OUTCOME_OK 871 872 # Convert size information to our simple format 873 if os.path.exists(sizes_file): 874 with open(sizes_file, 'r') as fd: 875 for line in fd.readlines(): 876 values = line.split() 877 rodata = 0 878 if len(values) > 6: 879 rodata = int(values[6], 16) 880 size_dict = { 881 'all' : int(values[0]) + int(values[1]) + 882 int(values[2]), 883 'text' : int(values[0]) - rodata, 884 'data' : int(values[1]), 885 'bss' : int(values[2]), 886 'rodata' : rodata, 887 } 888 sizes[values[5]] = size_dict 889 890 if read_func_sizes: 891 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 892 for fname in glob.glob(pattern): 893 with open(fname, 'r') as fd: 894 dict_name = os.path.basename(fname).replace('.sizes', 895 '') 896 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 897 898 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 899 900 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 901 902 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 903 """Calculate a summary of the results of building a commit. 904 905 Args: 906 board_selected: Dict containing boards to summarise 907 commit_upto: Commit number to summarize (0..self.count-1) 908 read_func_sizes: True to read function size information 909 910 Returns: 911 Tuple: 912 Dict containing boards which passed building this commit. 913 keyed by board.target 914 List containing a summary of error/warning lines 915 """ 916 board_dict = {} 917 err_lines_summary = [] 918 919 for board in boards_selected.itervalues(): 920 outcome = self.GetBuildOutcome(commit_upto, board.target, 921 read_func_sizes) 922 board_dict[board.target] = outcome 923 for err in outcome.err_lines: 924 if err and not err.rstrip() in err_lines_summary: 925 err_lines_summary.append(err.rstrip()) 926 return board_dict, err_lines_summary 927 928 def AddOutcome(self, board_dict, arch_list, changes, char, color): 929 """Add an output to our list of outcomes for each architecture 930 931 This simple function adds failing boards (changes) to the 932 relevant architecture string, so we can print the results out 933 sorted by architecture. 934 935 Args: 936 board_dict: Dict containing all boards 937 arch_list: Dict keyed by arch name. Value is a string containing 938 a list of board names which failed for that arch. 939 changes: List of boards to add to arch_list 940 color: terminal.Colour object 941 """ 942 done_arch = {} 943 for target in changes: 944 if target in board_dict: 945 arch = board_dict[target].arch 946 else: 947 arch = 'unknown' 948 str = self.col.Color(color, ' ' + target) 949 if not arch in done_arch: 950 str = self.col.Color(color, char) + ' ' + str 951 done_arch[arch] = True 952 if not arch in arch_list: 953 arch_list[arch] = str 954 else: 955 arch_list[arch] += str 956 957 958 def ColourNum(self, num): 959 color = self.col.RED if num > 0 else self.col.GREEN 960 if num == 0: 961 return '0' 962 return self.col.Color(color, str(num)) 963 964 def ResetResultSummary(self, board_selected): 965 """Reset the results summary ready for use. 966 967 Set up the base board list to be all those selected, and set the 968 error lines to empty. 969 970 Following this, calls to PrintResultSummary() will use this 971 information to work out what has changed. 972 973 Args: 974 board_selected: Dict containing boards to summarise, keyed by 975 board.target 976 """ 977 self._base_board_dict = {} 978 for board in board_selected: 979 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 980 self._base_err_lines = [] 981 982 def PrintFuncSizeDetail(self, fname, old, new): 983 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 984 delta, common = [], {} 985 986 for a in old: 987 if a in new: 988 common[a] = 1 989 990 for name in old: 991 if name not in common: 992 remove += 1 993 down += old[name] 994 delta.append([-old[name], name]) 995 996 for name in new: 997 if name not in common: 998 add += 1 999 up += new[name] 1000 delta.append([new[name], name]) 1001 1002 for name in common: 1003 diff = new.get(name, 0) - old.get(name, 0) 1004 if diff > 0: 1005 grow, up = grow + 1, up + diff 1006 elif diff < 0: 1007 shrink, down = shrink + 1, down - diff 1008 delta.append([diff, name]) 1009 1010 delta.sort() 1011 delta.reverse() 1012 1013 args = [add, -remove, grow, -shrink, up, -down, up - down] 1014 if max(args) == 0: 1015 return 1016 args = [self.ColourNum(x) for x in args] 1017 indent = ' ' * 15 1018 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1019 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1020 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1021 'delta') 1022 for diff, name in delta: 1023 if diff: 1024 color = self.col.RED if diff > 0 else self.col.GREEN 1025 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1026 old.get(name, '-'), new.get(name,'-'), diff) 1027 print self.col.Color(color, msg) 1028 1029 1030 def PrintSizeDetail(self, target_list, show_bloat): 1031 """Show details size information for each board 1032 1033 Args: 1034 target_list: List of targets, each a dict containing: 1035 'target': Target name 1036 'total_diff': Total difference in bytes across all areas 1037 <part_name>: Difference for that part 1038 show_bloat: Show detail for each function 1039 """ 1040 targets_by_diff = sorted(target_list, reverse=True, 1041 key=lambda x: x['_total_diff']) 1042 for result in targets_by_diff: 1043 printed_target = False 1044 for name in sorted(result): 1045 diff = result[name] 1046 if name.startswith('_'): 1047 continue 1048 if diff != 0: 1049 color = self.col.RED if diff > 0 else self.col.GREEN 1050 msg = ' %s %+d' % (name, diff) 1051 if not printed_target: 1052 print '%10s %-15s:' % ('', result['_target']), 1053 printed_target = True 1054 print self.col.Color(color, msg), 1055 if printed_target: 1056 print 1057 if show_bloat: 1058 target = result['_target'] 1059 outcome = result['_outcome'] 1060 base_outcome = self._base_board_dict[target] 1061 for fname in outcome.func_sizes: 1062 self.PrintFuncSizeDetail(fname, 1063 base_outcome.func_sizes[fname], 1064 outcome.func_sizes[fname]) 1065 1066 1067 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1068 show_bloat): 1069 """Print a summary of image sizes broken down by section. 1070 1071 The summary takes the form of one line per architecture. The 1072 line contains deltas for each of the sections (+ means the section 1073 got bigger, - means smaller). The nunmbers are the average number 1074 of bytes that a board in this section increased by. 1075 1076 For example: 1077 powerpc: (622 boards) text -0.0 1078 arm: (285 boards) text -0.0 1079 nds32: (3 boards) text -8.0 1080 1081 Args: 1082 board_selected: Dict containing boards to summarise, keyed by 1083 board.target 1084 board_dict: Dict containing boards for which we built this 1085 commit, keyed by board.target. The value is an Outcome object. 1086 show_detail: Show detail for each board 1087 show_bloat: Show detail for each function 1088 """ 1089 arch_list = {} 1090 arch_count = {} 1091 1092 # Calculate changes in size for different image parts 1093 # The previous sizes are in Board.sizes, for each board 1094 for target in board_dict: 1095 if target not in board_selected: 1096 continue 1097 base_sizes = self._base_board_dict[target].sizes 1098 outcome = board_dict[target] 1099 sizes = outcome.sizes 1100 1101 # Loop through the list of images, creating a dict of size 1102 # changes for each image/part. We end up with something like 1103 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1104 # which means that U-Boot data increased by 5 bytes and SPL 1105 # text decreased by 4. 1106 err = {'_target' : target} 1107 for image in sizes: 1108 if image in base_sizes: 1109 base_image = base_sizes[image] 1110 # Loop through the text, data, bss parts 1111 for part in sorted(sizes[image]): 1112 diff = sizes[image][part] - base_image[part] 1113 col = None 1114 if diff: 1115 if image == 'u-boot': 1116 name = part 1117 else: 1118 name = image + ':' + part 1119 err[name] = diff 1120 arch = board_selected[target].arch 1121 if not arch in arch_count: 1122 arch_count[arch] = 1 1123 else: 1124 arch_count[arch] += 1 1125 if not sizes: 1126 pass # Only add to our list when we have some stats 1127 elif not arch in arch_list: 1128 arch_list[arch] = [err] 1129 else: 1130 arch_list[arch].append(err) 1131 1132 # We now have a list of image size changes sorted by arch 1133 # Print out a summary of these 1134 for arch, target_list in arch_list.iteritems(): 1135 # Get total difference for each type 1136 totals = {} 1137 for result in target_list: 1138 total = 0 1139 for name, diff in result.iteritems(): 1140 if name.startswith('_'): 1141 continue 1142 total += diff 1143 if name in totals: 1144 totals[name] += diff 1145 else: 1146 totals[name] = diff 1147 result['_total_diff'] = total 1148 result['_outcome'] = board_dict[result['_target']] 1149 1150 count = len(target_list) 1151 printed_arch = False 1152 for name in sorted(totals): 1153 diff = totals[name] 1154 if diff: 1155 # Display the average difference in this name for this 1156 # architecture 1157 avg_diff = float(diff) / count 1158 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1159 msg = ' %s %+1.1f' % (name, avg_diff) 1160 if not printed_arch: 1161 print '%10s: (for %d/%d boards)' % (arch, count, 1162 arch_count[arch]), 1163 printed_arch = True 1164 print self.col.Color(color, msg), 1165 1166 if printed_arch: 1167 print 1168 if show_detail: 1169 self.PrintSizeDetail(target_list, show_bloat) 1170 1171 1172 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1173 show_sizes, show_detail, show_bloat): 1174 """Compare results with the base results and display delta. 1175 1176 Only boards mentioned in board_selected will be considered. This 1177 function is intended to be called repeatedly with the results of 1178 each commit. It therefore shows a 'diff' between what it saw in 1179 the last call and what it sees now. 1180 1181 Args: 1182 board_selected: Dict containing boards to summarise, keyed by 1183 board.target 1184 board_dict: Dict containing boards for which we built this 1185 commit, keyed by board.target. The value is an Outcome object. 1186 err_lines: A list of errors for this commit, or [] if there is 1187 none, or we don't want to print errors 1188 show_sizes: Show image size deltas 1189 show_detail: Show detail for each board 1190 show_bloat: Show detail for each function 1191 """ 1192 better = [] # List of boards fixed since last commit 1193 worse = [] # List of new broken boards since last commit 1194 new = [] # List of boards that didn't exist last time 1195 unknown = [] # List of boards that were not built 1196 1197 for target in board_dict: 1198 if target not in board_selected: 1199 continue 1200 1201 # If the board was built last time, add its outcome to a list 1202 if target in self._base_board_dict: 1203 base_outcome = self._base_board_dict[target].rc 1204 outcome = board_dict[target] 1205 if outcome.rc == OUTCOME_UNKNOWN: 1206 unknown.append(target) 1207 elif outcome.rc < base_outcome: 1208 better.append(target) 1209 elif outcome.rc > base_outcome: 1210 worse.append(target) 1211 else: 1212 new.append(target) 1213 1214 # Get a list of errors that have appeared, and disappeared 1215 better_err = [] 1216 worse_err = [] 1217 for line in err_lines: 1218 if line not in self._base_err_lines: 1219 worse_err.append('+' + line) 1220 for line in self._base_err_lines: 1221 if line not in err_lines: 1222 better_err.append('-' + line) 1223 1224 # Display results by arch 1225 if better or worse or unknown or new or worse_err or better_err: 1226 arch_list = {} 1227 self.AddOutcome(board_selected, arch_list, better, '', 1228 self.col.GREEN) 1229 self.AddOutcome(board_selected, arch_list, worse, '+', 1230 self.col.RED) 1231 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1232 if self._show_unknown: 1233 self.AddOutcome(board_selected, arch_list, unknown, '?', 1234 self.col.MAGENTA) 1235 for arch, target_list in arch_list.iteritems(): 1236 print '%10s: %s' % (arch, target_list) 1237 if better_err: 1238 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1239 if worse_err: 1240 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1241 1242 if show_sizes: 1243 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1244 show_bloat) 1245 1246 # Save our updated information for the next call to this function 1247 self._base_board_dict = board_dict 1248 self._base_err_lines = err_lines 1249 1250 # Get a list of boards that did not get built, if needed 1251 not_built = [] 1252 for board in board_selected: 1253 if not board in board_dict: 1254 not_built.append(board) 1255 if not_built: 1256 print "Boards not built (%d): %s" % (len(not_built), 1257 ', '.join(not_built)) 1258 1259 1260 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1261 show_detail, show_bloat): 1262 """Show a build summary for U-Boot for a given board list. 1263 1264 Reset the result summary, then repeatedly call GetResultSummary on 1265 each commit's results, then display the differences we see. 1266 1267 Args: 1268 commit: Commit objects to summarise 1269 board_selected: Dict containing boards to summarise 1270 show_errors: Show errors that occured 1271 show_sizes: Show size deltas 1272 show_detail: Show detail for each board 1273 show_bloat: Show detail for each function 1274 """ 1275 self.commit_count = len(commits) 1276 self.commits = commits 1277 self.ResetResultSummary(board_selected) 1278 1279 for commit_upto in range(0, self.commit_count, self._step): 1280 board_dict, err_lines = self.GetResultSummary(board_selected, 1281 commit_upto, read_func_sizes=show_bloat) 1282 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1283 print self.col.Color(self.col.BLUE, msg) 1284 self.PrintResultSummary(board_selected, board_dict, 1285 err_lines if show_errors else [], show_sizes, show_detail, 1286 show_bloat) 1287 1288 1289 def SetupBuild(self, board_selected, commits): 1290 """Set up ready to start a build. 1291 1292 Args: 1293 board_selected: Selected boards to build 1294 commits: Selected commits to build 1295 """ 1296 # First work out how many commits we will build 1297 count = (len(commits) + self._step - 1) / self._step 1298 self.count = len(board_selected) * count 1299 self.upto = self.warned = self.fail = 0 1300 self._timestamps = collections.deque() 1301 1302 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1303 """Build all boards for a single commit""" 1304 self.SetupBuild(board_selected) 1305 self.count = len(board_selected) 1306 for brd in board_selected.itervalues(): 1307 job = BuilderJob() 1308 job.board = brd 1309 job.commits = None 1310 job.keep_outputs = keep_outputs 1311 self.queue.put(brd) 1312 1313 self.queue.join() 1314 self.out_queue.join() 1315 print 1316 self.ClearLine(0) 1317 1318 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1319 """Build all boards for all commits (non-incremental)""" 1320 self.commit_count = len(commits) 1321 1322 self.ResetResultSummary(board_selected) 1323 for self.commit_upto in range(self.commit_count): 1324 self.SelectCommit(commits[self.commit_upto]) 1325 self.SelectOutputDir() 1326 Mkdir(self.output_dir) 1327 1328 self.BuildBoardsForCommit(board_selected, keep_outputs) 1329 board_dict, err_lines = self.GetResultSummary() 1330 self.PrintResultSummary(board_selected, board_dict, 1331 err_lines if show_errors else []) 1332 1333 if self.already_done: 1334 print '%d builds already done' % self.already_done 1335 1336 def GetThreadDir(self, thread_num): 1337 """Get the directory path to the working dir for a thread. 1338 1339 Args: 1340 thread_num: Number of thread to check. 1341 """ 1342 return os.path.join(self._working_dir, '%02d' % thread_num) 1343 1344 def _PrepareThread(self, thread_num): 1345 """Prepare the working directory for a thread. 1346 1347 This clones or fetches the repo into the thread's work directory. 1348 1349 Args: 1350 thread_num: Thread number (0, 1, ...) 1351 """ 1352 thread_dir = self.GetThreadDir(thread_num) 1353 Mkdir(thread_dir) 1354 git_dir = os.path.join(thread_dir, '.git') 1355 1356 # Clone the repo if it doesn't already exist 1357 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1358 # we have a private index but uses the origin repo's contents? 1359 if self.git_dir: 1360 src_dir = os.path.abspath(self.git_dir) 1361 if os.path.exists(git_dir): 1362 gitutil.Fetch(git_dir, thread_dir) 1363 else: 1364 print 'Cloning repo for thread %d' % thread_num 1365 gitutil.Clone(src_dir, thread_dir) 1366 1367 def _PrepareWorkingSpace(self, max_threads): 1368 """Prepare the working directory for use. 1369 1370 Set up the git repo for each thread. 1371 1372 Args: 1373 max_threads: Maximum number of threads we expect to need. 1374 """ 1375 Mkdir(self._working_dir) 1376 for thread in range(max_threads): 1377 self._PrepareThread(thread) 1378 1379 def _PrepareOutputSpace(self): 1380 """Get the output directories ready to receive files. 1381 1382 We delete any output directories which look like ones we need to 1383 create. Having left over directories is confusing when the user wants 1384 to check the output manually. 1385 """ 1386 dir_list = [] 1387 for commit_upto in range(self.commit_count): 1388 dir_list.append(self._GetOutputDir(commit_upto)) 1389 1390 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1391 if dirname not in dir_list: 1392 shutil.rmtree(dirname) 1393 1394 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1395 """Build all commits for a list of boards 1396 1397 Args: 1398 commits: List of commits to be build, each a Commit object 1399 boards_selected: Dict of selected boards, key is target name, 1400 value is Board object 1401 show_errors: True to show summarised error/warning info 1402 keep_outputs: True to save build output files 1403 """ 1404 self.commit_count = len(commits) 1405 self.commits = commits 1406 1407 self.ResetResultSummary(board_selected) 1408 Mkdir(self.base_dir) 1409 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1410 self._PrepareOutputSpace() 1411 self.SetupBuild(board_selected, commits) 1412 self.ProcessResult(None) 1413 1414 # Create jobs to build all commits for each board 1415 for brd in board_selected.itervalues(): 1416 job = BuilderJob() 1417 job.board = brd 1418 job.commits = commits 1419 job.keep_outputs = keep_outputs 1420 job.step = self._step 1421 self.queue.put(job) 1422 1423 # Wait until all jobs are started 1424 self.queue.join() 1425 1426 # Wait until we have processed all output 1427 self.out_queue.join() 1428 print 1429 self.ClearLine(0) 1430