1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9import errno 10from datetime import datetime, timedelta 11import glob 12import os 13import re 14import Queue 15import shutil 16import string 17import sys 18import threading 19import time 20 21import command 22import gitutil 23import terminal 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100def Mkdir(dirname): 101 """Make a directory if it doesn't already exist. 102 103 Args: 104 dirname: Directory to create 105 """ 106 try: 107 os.mkdir(dirname) 108 except OSError as err: 109 if err.errno == errno.EEXIST: 110 pass 111 else: 112 raise 113 114class BuilderJob: 115 """Holds information about a job to be performed by a thread 116 117 Members: 118 board: Board object to build 119 commits: List of commit options to build. 120 """ 121 def __init__(self): 122 self.board = None 123 self.commits = [] 124 125 126class ResultThread(threading.Thread): 127 """This thread processes results from builder threads. 128 129 It simply passes the results on to the builder. There is only one 130 result thread, and this helps to serialise the build output. 131 """ 132 def __init__(self, builder): 133 """Set up a new result thread 134 135 Args: 136 builder: Builder which will be sent each result 137 """ 138 threading.Thread.__init__(self) 139 self.builder = builder 140 141 def run(self): 142 """Called to start up the result thread. 143 144 We collect the next result job and pass it on to the build. 145 """ 146 while True: 147 result = self.builder.out_queue.get() 148 self.builder.ProcessResult(result) 149 self.builder.out_queue.task_done() 150 151 152class BuilderThread(threading.Thread): 153 """This thread builds U-Boot for a particular board. 154 155 An input queue provides each new job. We run 'make' to build U-Boot 156 and then pass the results on to the output queue. 157 158 Members: 159 builder: The builder which contains information we might need 160 thread_num: Our thread number (0-n-1), used to decide on a 161 temporary directory 162 """ 163 def __init__(self, builder, thread_num): 164 """Set up a new builder thread""" 165 threading.Thread.__init__(self) 166 self.builder = builder 167 self.thread_num = thread_num 168 169 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 170 """Run 'make' on a particular commit and board. 171 172 The source code will already be checked out, so the 'commit' 173 argument is only for information. 174 175 Args: 176 commit: Commit object that is being built 177 brd: Board object that is being built 178 stage: Stage of the build. Valid stages are: 179 distclean - can be called to clean source 180 config - called to configure for a board 181 build - the main make invocation - it does the build 182 args: A list of arguments to pass to 'make' 183 kwargs: A list of keyword arguments to pass to command.RunPipe() 184 185 Returns: 186 CommandResult object 187 """ 188 return self.builder.do_make(commit, brd, stage, cwd, *args, 189 **kwargs) 190 191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build): 192 """Build a particular commit. 193 194 If the build is already done, and we are not forcing a build, we skip 195 the build and just return the previously-saved results. 196 197 Args: 198 commit_upto: Commit number to build (0...n-1) 199 brd: Board object to build 200 work_dir: Directory to which the source will be checked out 201 do_config: True to run a make <board>_config on the source 202 force_build: Force a build even if one was previously done 203 204 Returns: 205 tuple containing: 206 - CommandResult object containing the results of the build 207 - boolean indicating whether 'make config' is still needed 208 """ 209 # Create a default result - it will be overwritte by the call to 210 # self.Make() below, in the event that we do a build. 211 result = command.CommandResult() 212 result.return_code = 0 213 out_dir = os.path.join(work_dir, 'build') 214 215 # Check if the job was already completed last time 216 done_file = self.builder.GetDoneFile(commit_upto, brd.target) 217 result.already_done = os.path.exists(done_file) 218 if result.already_done and not force_build: 219 # Get the return code from that build and use it 220 with open(done_file, 'r') as fd: 221 result.return_code = int(fd.readline()) 222 err_file = self.builder.GetErrFile(commit_upto, brd.target) 223 if os.path.exists(err_file) and os.stat(err_file).st_size: 224 result.stderr = 'bad' 225 else: 226 # We are going to have to build it. First, get a toolchain 227 if not self.toolchain: 228 try: 229 self.toolchain = self.builder.toolchains.Select(brd.arch) 230 except ValueError as err: 231 result.return_code = 10 232 result.stdout = '' 233 result.stderr = str(err) 234 # TODO(sjg@chromium.org): This gets swallowed, but needs 235 # to be reported. 236 237 if self.toolchain: 238 # Checkout the right commit 239 if commit_upto is not None: 240 commit = self.builder.commits[commit_upto] 241 if self.builder.checkout: 242 git_dir = os.path.join(work_dir, '.git') 243 gitutil.Checkout(commit.hash, git_dir, work_dir, 244 force=True) 245 else: 246 commit = self.builder.commit # Ick, fix this for BuildCommits() 247 248 # Set up the environment and command line 249 env = self.toolchain.MakeEnvironment() 250 Mkdir(out_dir) 251 args = ['O=build', '-s'] 252 if self.builder.num_jobs is not None: 253 args.extend(['-j', str(self.builder.num_jobs)]) 254 config_args = ['%s_config' % brd.target] 255 config_out = '' 256 args.extend(self.builder.toolchains.GetMakeArguments(brd)) 257 258 # If we need to reconfigure, do that now 259 if do_config: 260 result = self.Make(commit, brd, 'distclean', work_dir, 261 'distclean', *args, env=env) 262 result = self.Make(commit, brd, 'config', work_dir, 263 *(args + config_args), env=env) 264 config_out = result.combined 265 do_config = False # No need to configure next time 266 if result.return_code == 0: 267 result = self.Make(commit, brd, 'build', work_dir, *args, 268 env=env) 269 result.stdout = config_out + result.stdout 270 else: 271 result.return_code = 1 272 result.stderr = 'No tool chain for %s\n' % brd.arch 273 result.already_done = False 274 275 result.toolchain = self.toolchain 276 result.brd = brd 277 result.commit_upto = commit_upto 278 result.out_dir = out_dir 279 return result, do_config 280 281 def _WriteResult(self, result, keep_outputs): 282 """Write a built result to the output directory. 283 284 Args: 285 result: CommandResult object containing result to write 286 keep_outputs: True to store the output binaries, False 287 to delete them 288 """ 289 # Fatal error 290 if result.return_code < 0: 291 return 292 293 # Aborted? 294 if result.stderr and 'No child processes' in result.stderr: 295 return 296 297 if result.already_done: 298 return 299 300 # Write the output and stderr 301 output_dir = self.builder._GetOutputDir(result.commit_upto) 302 Mkdir(output_dir) 303 build_dir = self.builder.GetBuildDir(result.commit_upto, 304 result.brd.target) 305 Mkdir(build_dir) 306 307 outfile = os.path.join(build_dir, 'log') 308 with open(outfile, 'w') as fd: 309 if result.stdout: 310 fd.write(result.stdout) 311 312 errfile = self.builder.GetErrFile(result.commit_upto, 313 result.brd.target) 314 if result.stderr: 315 with open(errfile, 'w') as fd: 316 fd.write(result.stderr) 317 elif os.path.exists(errfile): 318 os.remove(errfile) 319 320 if result.toolchain: 321 # Write the build result and toolchain information. 322 done_file = self.builder.GetDoneFile(result.commit_upto, 323 result.brd.target) 324 with open(done_file, 'w') as fd: 325 fd.write('%s' % result.return_code) 326 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 327 print >>fd, 'gcc', result.toolchain.gcc 328 print >>fd, 'path', result.toolchain.path 329 print >>fd, 'cross', result.toolchain.cross 330 print >>fd, 'arch', result.toolchain.arch 331 fd.write('%s' % result.return_code) 332 333 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: 334 print >>fd, 'gcc', result.toolchain.gcc 335 print >>fd, 'path', result.toolchain.path 336 337 # Write out the image and function size information and an objdump 338 env = result.toolchain.MakeEnvironment() 339 lines = [] 340 for fname in ['u-boot', 'spl/u-boot-spl']: 341 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] 342 nm_result = command.RunPipe([cmd], capture=True, 343 capture_stderr=True, cwd=result.out_dir, 344 raise_on_error=False, env=env) 345 if nm_result.stdout: 346 nm = self.builder.GetFuncSizesFile(result.commit_upto, 347 result.brd.target, fname) 348 with open(nm, 'w') as fd: 349 print >>fd, nm_result.stdout, 350 351 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] 352 dump_result = command.RunPipe([cmd], capture=True, 353 capture_stderr=True, cwd=result.out_dir, 354 raise_on_error=False, env=env) 355 rodata_size = '' 356 if dump_result.stdout: 357 objdump = self.builder.GetObjdumpFile(result.commit_upto, 358 result.brd.target, fname) 359 with open(objdump, 'w') as fd: 360 print >>fd, dump_result.stdout, 361 for line in dump_result.stdout.splitlines(): 362 fields = line.split() 363 if len(fields) > 5 and fields[1] == '.rodata': 364 rodata_size = fields[2] 365 366 cmd = ['%ssize' % self.toolchain.cross, fname] 367 size_result = command.RunPipe([cmd], capture=True, 368 capture_stderr=True, cwd=result.out_dir, 369 raise_on_error=False, env=env) 370 if size_result.stdout: 371 lines.append(size_result.stdout.splitlines()[1] + ' ' + 372 rodata_size) 373 374 # Write out the image sizes file. This is similar to the output 375 # of binutil's 'size' utility, but it omits the header line and 376 # adds an additional hex value at the end of each line for the 377 # rodata size 378 if len(lines): 379 sizes = self.builder.GetSizesFile(result.commit_upto, 380 result.brd.target) 381 with open(sizes, 'w') as fd: 382 print >>fd, '\n'.join(lines) 383 384 # Now write the actual build output 385 if keep_outputs: 386 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', 387 'include/autoconf.mk', 'spl/u-boot-spl', 388 'spl/u-boot-spl.bin'] 389 for pattern in patterns: 390 file_list = glob.glob(os.path.join(result.out_dir, pattern)) 391 for fname in file_list: 392 shutil.copy(fname, build_dir) 393 394 395 def RunJob(self, job): 396 """Run a single job 397 398 A job consists of a building a list of commits for a particular board. 399 400 Args: 401 job: Job to build 402 """ 403 brd = job.board 404 work_dir = self.builder.GetThreadDir(self.thread_num) 405 self.toolchain = None 406 if job.commits: 407 # Run 'make board_config' on the first commit 408 do_config = True 409 commit_upto = 0 410 force_build = False 411 for commit_upto in range(0, len(job.commits), job.step): 412 result, request_config = self.RunCommit(commit_upto, brd, 413 work_dir, do_config, 414 force_build or self.builder.force_build) 415 failed = result.return_code or result.stderr 416 if failed and not do_config: 417 # If our incremental build failed, try building again 418 # with a reconfig. 419 if self.builder.force_config_on_failure: 420 result, request_config = self.RunCommit(commit_upto, 421 brd, work_dir, True, True) 422 do_config = request_config 423 424 # If we built that commit, then config is done. But if we got 425 # an warning, reconfig next time to force it to build the same 426 # files that created warnings this time. Otherwise an 427 # incremental build may not build the same file, and we will 428 # think that the warning has gone away. 429 # We could avoid this by using -Werror everywhere... 430 # For errors, the problem doesn't happen, since presumably 431 # the build stopped and didn't generate output, so will retry 432 # that file next time. So we could detect warnings and deal 433 # with them specially here. For now, we just reconfigure if 434 # anything goes work. 435 # Of course this is substantially slower if there are build 436 # errors/warnings (e.g. 2-3x slower even if only 10% of builds 437 # have problems). 438 if (failed and not result.already_done and not do_config and 439 self.builder.force_config_on_failure): 440 # If this build failed, try the next one with a 441 # reconfigure. 442 # Sometimes if the board_config.h file changes it can mess 443 # with dependencies, and we get: 444 # make: *** No rule to make target `include/autoconf.mk', 445 # needed by `depend'. 446 do_config = True 447 force_build = True 448 else: 449 force_build = False 450 if self.builder.force_config_on_failure: 451 if failed: 452 do_config = True 453 result.commit_upto = commit_upto 454 if result.return_code < 0: 455 raise ValueError('Interrupt') 456 457 # We have the build results, so output the result 458 self._WriteResult(result, job.keep_outputs) 459 self.builder.out_queue.put(result) 460 else: 461 # Just build the currently checked-out build 462 result = self.RunCommit(None, True) 463 result.commit_upto = self.builder.upto 464 self.builder.out_queue.put(result) 465 466 def run(self): 467 """Our thread's run function 468 469 This thread picks a job from the queue, runs it, and then goes to the 470 next job. 471 """ 472 alive = True 473 while True: 474 job = self.builder.queue.get() 475 try: 476 if self.builder.active and alive: 477 self.RunJob(job) 478 except Exception as err: 479 alive = False 480 print err 481 self.builder.queue.task_done() 482 483 484class Builder: 485 """Class for building U-Boot for a particular commit. 486 487 Public members: (many should ->private) 488 active: True if the builder is active and has not been stopped 489 already_done: Number of builds already completed 490 base_dir: Base directory to use for builder 491 checkout: True to check out source, False to skip that step. 492 This is used for testing. 493 col: terminal.Color() object 494 count: Number of commits to build 495 do_make: Method to call to invoke Make 496 fail: Number of builds that failed due to error 497 force_build: Force building even if a build already exists 498 force_config_on_failure: If a commit fails for a board, disable 499 incremental building for the next commit we build for that 500 board, so that we will see all warnings/errors again. 501 git_dir: Git directory containing source repository 502 last_line_len: Length of the last line we printed (used for erasing 503 it with new progress information) 504 num_jobs: Number of jobs to run at once (passed to make as -j) 505 num_threads: Number of builder threads to run 506 out_queue: Queue of results to process 507 re_make_err: Compiled regular expression for ignore_lines 508 queue: Queue of jobs to run 509 threads: List of active threads 510 toolchains: Toolchains object to use for building 511 upto: Current commit number we are building (0.count-1) 512 warned: Number of builds that produced at least one warning 513 514 Private members: 515 _base_board_dict: Last-summarised Dict of boards 516 _base_err_lines: Last-summarised list of errors 517 _build_period_us: Time taken for a single build (float object). 518 _complete_delay: Expected delay until completion (timedelta) 519 _next_delay_update: Next time we plan to display a progress update 520 (datatime) 521 _show_unknown: Show unknown boards (those not built) in summary 522 _timestamps: List of timestamps for the completion of the last 523 last _timestamp_count builds. Each is a datetime object. 524 _timestamp_count: Number of timestamps to keep in our list. 525 _working_dir: Base working directory containing all threads 526 """ 527 class Outcome: 528 """Records a build outcome for a single make invocation 529 530 Public Members: 531 rc: Outcome value (OUTCOME_...) 532 err_lines: List of error lines or [] if none 533 sizes: Dictionary of image size information, keyed by filename 534 - Each value is itself a dictionary containing 535 values for 'text', 'data' and 'bss', being the integer 536 size in bytes of each section. 537 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 538 value is itself a dictionary: 539 key: function name 540 value: Size of function in bytes 541 """ 542 def __init__(self, rc, err_lines, sizes, func_sizes): 543 self.rc = rc 544 self.err_lines = err_lines 545 self.sizes = sizes 546 self.func_sizes = func_sizes 547 548 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 549 checkout=True, show_unknown=True, step=1): 550 """Create a new Builder object 551 552 Args: 553 toolchains: Toolchains object to use for building 554 base_dir: Base directory to use for builder 555 git_dir: Git directory containing source repository 556 num_threads: Number of builder threads to run 557 num_jobs: Number of jobs to run at once (passed to make as -j) 558 checkout: True to check out source, False to skip that step. 559 This is used for testing. 560 show_unknown: Show unknown boards (those not built) in summary 561 step: 1 to process every commit, n to process every nth commit 562 """ 563 self.toolchains = toolchains 564 self.base_dir = base_dir 565 self._working_dir = os.path.join(base_dir, '.bm-work') 566 self.threads = [] 567 self.active = True 568 self.do_make = self.Make 569 self.checkout = checkout 570 self.num_threads = num_threads 571 self.num_jobs = num_jobs 572 self.already_done = 0 573 self.force_build = False 574 self.git_dir = git_dir 575 self._show_unknown = show_unknown 576 self._timestamp_count = 10 577 self._build_period_us = None 578 self._complete_delay = None 579 self._next_delay_update = datetime.now() 580 self.force_config_on_failure = True 581 self._step = step 582 583 self.col = terminal.Color() 584 585 self.queue = Queue.Queue() 586 self.out_queue = Queue.Queue() 587 for i in range(self.num_threads): 588 t = BuilderThread(self, i) 589 t.setDaemon(True) 590 t.start() 591 self.threads.append(t) 592 593 self.last_line_len = 0 594 t = ResultThread(self) 595 t.setDaemon(True) 596 t.start() 597 self.threads.append(t) 598 599 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 600 self.re_make_err = re.compile('|'.join(ignore_lines)) 601 602 def __del__(self): 603 """Get rid of all threads created by the builder""" 604 for t in self.threads: 605 del t 606 607 def _AddTimestamp(self): 608 """Add a new timestamp to the list and record the build period. 609 610 The build period is the length of time taken to perform a single 611 build (one board, one commit). 612 """ 613 now = datetime.now() 614 self._timestamps.append(now) 615 count = len(self._timestamps) 616 delta = self._timestamps[-1] - self._timestamps[0] 617 seconds = delta.total_seconds() 618 619 # If we have enough data, estimate build period (time taken for a 620 # single build) and therefore completion time. 621 if count > 1 and self._next_delay_update < now: 622 self._next_delay_update = now + timedelta(seconds=2) 623 if seconds > 0: 624 self._build_period = float(seconds) / count 625 todo = self.count - self.upto 626 self._complete_delay = timedelta(microseconds= 627 self._build_period * todo * 1000000) 628 # Round it 629 self._complete_delay -= timedelta( 630 microseconds=self._complete_delay.microseconds) 631 632 if seconds > 60: 633 self._timestamps.popleft() 634 count -= 1 635 636 def ClearLine(self, length): 637 """Clear any characters on the current line 638 639 Make way for a new line of length 'length', by outputting enough 640 spaces to clear out the old line. Then remember the new length for 641 next time. 642 643 Args: 644 length: Length of new line, in characters 645 """ 646 if length < self.last_line_len: 647 print ' ' * (self.last_line_len - length), 648 print '\r', 649 self.last_line_len = length 650 sys.stdout.flush() 651 652 def SelectCommit(self, commit, checkout=True): 653 """Checkout the selected commit for this build 654 """ 655 self.commit = commit 656 if checkout and self.checkout: 657 gitutil.Checkout(commit.hash) 658 659 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 660 """Run make 661 662 Args: 663 commit: Commit object that is being built 664 brd: Board object that is being built 665 stage: Stage that we are at (distclean, config, build) 666 cwd: Directory where make should be run 667 args: Arguments to pass to make 668 kwargs: Arguments to pass to command.RunPipe() 669 """ 670 cmd = ['make'] + list(args) 671 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 672 cwd=cwd, raise_on_error=False, **kwargs) 673 return result 674 675 def ProcessResult(self, result): 676 """Process the result of a build, showing progress information 677 678 Args: 679 result: A CommandResult object 680 """ 681 col = terminal.Color() 682 if result: 683 target = result.brd.target 684 685 if result.return_code < 0: 686 self.active = False 687 command.StopAll() 688 return 689 690 self.upto += 1 691 if result.return_code != 0: 692 self.fail += 1 693 elif result.stderr: 694 self.warned += 1 695 if result.already_done: 696 self.already_done += 1 697 else: 698 target = '(starting)' 699 700 # Display separate counts for ok, warned and fail 701 ok = self.upto - self.warned - self.fail 702 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 703 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 704 line += self.col.Color(self.col.RED, '%5d' % self.fail) 705 706 name = ' /%-5d ' % self.count 707 708 # Add our current completion time estimate 709 self._AddTimestamp() 710 if self._complete_delay: 711 name += '%s : ' % self._complete_delay 712 # When building all boards for a commit, we can print a commit 713 # progress message. 714 if result and result.commit_upto is None: 715 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 716 self.commit_count) 717 718 name += target 719 print line + name, 720 length = 13 + len(name) 721 self.ClearLine(length) 722 723 def _GetOutputDir(self, commit_upto): 724 """Get the name of the output directory for a commit number 725 726 The output directory is typically .../<branch>/<commit>. 727 728 Args: 729 commit_upto: Commit number to use (0..self.count-1) 730 """ 731 commit = self.commits[commit_upto] 732 subject = commit.subject.translate(trans_valid_chars) 733 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 734 self.commit_count, commit.hash, subject[:20])) 735 output_dir = os.path.join(self.base_dir, commit_dir) 736 return output_dir 737 738 def GetBuildDir(self, commit_upto, target): 739 """Get the name of the build directory for a commit number 740 741 The build directory is typically .../<branch>/<commit>/<target>. 742 743 Args: 744 commit_upto: Commit number to use (0..self.count-1) 745 target: Target name 746 """ 747 output_dir = self._GetOutputDir(commit_upto) 748 return os.path.join(output_dir, target) 749 750 def GetDoneFile(self, commit_upto, target): 751 """Get the name of the done file for a commit number 752 753 Args: 754 commit_upto: Commit number to use (0..self.count-1) 755 target: Target name 756 """ 757 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 758 759 def GetSizesFile(self, commit_upto, target): 760 """Get the name of the sizes file for a commit number 761 762 Args: 763 commit_upto: Commit number to use (0..self.count-1) 764 target: Target name 765 """ 766 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 767 768 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 769 """Get the name of the funcsizes file for a commit number and ELF file 770 771 Args: 772 commit_upto: Commit number to use (0..self.count-1) 773 target: Target name 774 elf_fname: Filename of elf image 775 """ 776 return os.path.join(self.GetBuildDir(commit_upto, target), 777 '%s.sizes' % elf_fname.replace('/', '-')) 778 779 def GetObjdumpFile(self, commit_upto, target, elf_fname): 780 """Get the name of the objdump file for a commit number and ELF file 781 782 Args: 783 commit_upto: Commit number to use (0..self.count-1) 784 target: Target name 785 elf_fname: Filename of elf image 786 """ 787 return os.path.join(self.GetBuildDir(commit_upto, target), 788 '%s.objdump' % elf_fname.replace('/', '-')) 789 790 def GetErrFile(self, commit_upto, target): 791 """Get the name of the err file for a commit number 792 793 Args: 794 commit_upto: Commit number to use (0..self.count-1) 795 target: Target name 796 """ 797 output_dir = self.GetBuildDir(commit_upto, target) 798 return os.path.join(output_dir, 'err') 799 800 def FilterErrors(self, lines): 801 """Filter out errors in which we have no interest 802 803 We should probably use map(). 804 805 Args: 806 lines: List of error lines, each a string 807 Returns: 808 New list with only interesting lines included 809 """ 810 out_lines = [] 811 for line in lines: 812 if not self.re_make_err.search(line): 813 out_lines.append(line) 814 return out_lines 815 816 def ReadFuncSizes(self, fname, fd): 817 """Read function sizes from the output of 'nm' 818 819 Args: 820 fd: File containing data to read 821 fname: Filename we are reading from (just for errors) 822 823 Returns: 824 Dictionary containing size of each function in bytes, indexed by 825 function name. 826 """ 827 sym = {} 828 for line in fd.readlines(): 829 try: 830 size, type, name = line[:-1].split() 831 except: 832 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 833 continue 834 if type in 'tTdDbB': 835 # function names begin with '.' on 64-bit powerpc 836 if '.' in name[1:]: 837 name = 'static.' + name.split('.')[0] 838 sym[name] = sym.get(name, 0) + int(size, 16) 839 return sym 840 841 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 842 """Work out the outcome of a build. 843 844 Args: 845 commit_upto: Commit number to check (0..n-1) 846 target: Target board to check 847 read_func_sizes: True to read function size information 848 849 Returns: 850 Outcome object 851 """ 852 done_file = self.GetDoneFile(commit_upto, target) 853 sizes_file = self.GetSizesFile(commit_upto, target) 854 sizes = {} 855 func_sizes = {} 856 if os.path.exists(done_file): 857 with open(done_file, 'r') as fd: 858 return_code = int(fd.readline()) 859 err_lines = [] 860 err_file = self.GetErrFile(commit_upto, target) 861 if os.path.exists(err_file): 862 with open(err_file, 'r') as fd: 863 err_lines = self.FilterErrors(fd.readlines()) 864 865 # Decide whether the build was ok, failed or created warnings 866 if return_code: 867 rc = OUTCOME_ERROR 868 elif len(err_lines): 869 rc = OUTCOME_WARNING 870 else: 871 rc = OUTCOME_OK 872 873 # Convert size information to our simple format 874 if os.path.exists(sizes_file): 875 with open(sizes_file, 'r') as fd: 876 for line in fd.readlines(): 877 values = line.split() 878 rodata = 0 879 if len(values) > 6: 880 rodata = int(values[6], 16) 881 size_dict = { 882 'all' : int(values[0]) + int(values[1]) + 883 int(values[2]), 884 'text' : int(values[0]) - rodata, 885 'data' : int(values[1]), 886 'bss' : int(values[2]), 887 'rodata' : rodata, 888 } 889 sizes[values[5]] = size_dict 890 891 if read_func_sizes: 892 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 893 for fname in glob.glob(pattern): 894 with open(fname, 'r') as fd: 895 dict_name = os.path.basename(fname).replace('.sizes', 896 '') 897 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 898 899 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 900 901 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 902 903 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 904 """Calculate a summary of the results of building a commit. 905 906 Args: 907 board_selected: Dict containing boards to summarise 908 commit_upto: Commit number to summarize (0..self.count-1) 909 read_func_sizes: True to read function size information 910 911 Returns: 912 Tuple: 913 Dict containing boards which passed building this commit. 914 keyed by board.target 915 List containing a summary of error/warning lines 916 """ 917 board_dict = {} 918 err_lines_summary = [] 919 920 for board in boards_selected.itervalues(): 921 outcome = self.GetBuildOutcome(commit_upto, board.target, 922 read_func_sizes) 923 board_dict[board.target] = outcome 924 for err in outcome.err_lines: 925 if err and not err.rstrip() in err_lines_summary: 926 err_lines_summary.append(err.rstrip()) 927 return board_dict, err_lines_summary 928 929 def AddOutcome(self, board_dict, arch_list, changes, char, color): 930 """Add an output to our list of outcomes for each architecture 931 932 This simple function adds failing boards (changes) to the 933 relevant architecture string, so we can print the results out 934 sorted by architecture. 935 936 Args: 937 board_dict: Dict containing all boards 938 arch_list: Dict keyed by arch name. Value is a string containing 939 a list of board names which failed for that arch. 940 changes: List of boards to add to arch_list 941 color: terminal.Colour object 942 """ 943 done_arch = {} 944 for target in changes: 945 if target in board_dict: 946 arch = board_dict[target].arch 947 else: 948 arch = 'unknown' 949 str = self.col.Color(color, ' ' + target) 950 if not arch in done_arch: 951 str = self.col.Color(color, char) + ' ' + str 952 done_arch[arch] = True 953 if not arch in arch_list: 954 arch_list[arch] = str 955 else: 956 arch_list[arch] += str 957 958 959 def ColourNum(self, num): 960 color = self.col.RED if num > 0 else self.col.GREEN 961 if num == 0: 962 return '0' 963 return self.col.Color(color, str(num)) 964 965 def ResetResultSummary(self, board_selected): 966 """Reset the results summary ready for use. 967 968 Set up the base board list to be all those selected, and set the 969 error lines to empty. 970 971 Following this, calls to PrintResultSummary() will use this 972 information to work out what has changed. 973 974 Args: 975 board_selected: Dict containing boards to summarise, keyed by 976 board.target 977 """ 978 self._base_board_dict = {} 979 for board in board_selected: 980 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 981 self._base_err_lines = [] 982 983 def PrintFuncSizeDetail(self, fname, old, new): 984 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 985 delta, common = [], {} 986 987 for a in old: 988 if a in new: 989 common[a] = 1 990 991 for name in old: 992 if name not in common: 993 remove += 1 994 down += old[name] 995 delta.append([-old[name], name]) 996 997 for name in new: 998 if name not in common: 999 add += 1 1000 up += new[name] 1001 delta.append([new[name], name]) 1002 1003 for name in common: 1004 diff = new.get(name, 0) - old.get(name, 0) 1005 if diff > 0: 1006 grow, up = grow + 1, up + diff 1007 elif diff < 0: 1008 shrink, down = shrink + 1, down - diff 1009 delta.append([diff, name]) 1010 1011 delta.sort() 1012 delta.reverse() 1013 1014 args = [add, -remove, grow, -shrink, up, -down, up - down] 1015 if max(args) == 0: 1016 return 1017 args = [self.ColourNum(x) for x in args] 1018 indent = ' ' * 15 1019 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 1020 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 1021 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 1022 'delta') 1023 for diff, name in delta: 1024 if diff: 1025 color = self.col.RED if diff > 0 else self.col.GREEN 1026 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 1027 old.get(name, '-'), new.get(name,'-'), diff) 1028 print self.col.Color(color, msg) 1029 1030 1031 def PrintSizeDetail(self, target_list, show_bloat): 1032 """Show details size information for each board 1033 1034 Args: 1035 target_list: List of targets, each a dict containing: 1036 'target': Target name 1037 'total_diff': Total difference in bytes across all areas 1038 <part_name>: Difference for that part 1039 show_bloat: Show detail for each function 1040 """ 1041 targets_by_diff = sorted(target_list, reverse=True, 1042 key=lambda x: x['_total_diff']) 1043 for result in targets_by_diff: 1044 printed_target = False 1045 for name in sorted(result): 1046 diff = result[name] 1047 if name.startswith('_'): 1048 continue 1049 if diff != 0: 1050 color = self.col.RED if diff > 0 else self.col.GREEN 1051 msg = ' %s %+d' % (name, diff) 1052 if not printed_target: 1053 print '%10s %-15s:' % ('', result['_target']), 1054 printed_target = True 1055 print self.col.Color(color, msg), 1056 if printed_target: 1057 print 1058 if show_bloat: 1059 target = result['_target'] 1060 outcome = result['_outcome'] 1061 base_outcome = self._base_board_dict[target] 1062 for fname in outcome.func_sizes: 1063 self.PrintFuncSizeDetail(fname, 1064 base_outcome.func_sizes[fname], 1065 outcome.func_sizes[fname]) 1066 1067 1068 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 1069 show_bloat): 1070 """Print a summary of image sizes broken down by section. 1071 1072 The summary takes the form of one line per architecture. The 1073 line contains deltas for each of the sections (+ means the section 1074 got bigger, - means smaller). The nunmbers are the average number 1075 of bytes that a board in this section increased by. 1076 1077 For example: 1078 powerpc: (622 boards) text -0.0 1079 arm: (285 boards) text -0.0 1080 nds32: (3 boards) text -8.0 1081 1082 Args: 1083 board_selected: Dict containing boards to summarise, keyed by 1084 board.target 1085 board_dict: Dict containing boards for which we built this 1086 commit, keyed by board.target. The value is an Outcome object. 1087 show_detail: Show detail for each board 1088 show_bloat: Show detail for each function 1089 """ 1090 arch_list = {} 1091 arch_count = {} 1092 1093 # Calculate changes in size for different image parts 1094 # The previous sizes are in Board.sizes, for each board 1095 for target in board_dict: 1096 if target not in board_selected: 1097 continue 1098 base_sizes = self._base_board_dict[target].sizes 1099 outcome = board_dict[target] 1100 sizes = outcome.sizes 1101 1102 # Loop through the list of images, creating a dict of size 1103 # changes for each image/part. We end up with something like 1104 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1105 # which means that U-Boot data increased by 5 bytes and SPL 1106 # text decreased by 4. 1107 err = {'_target' : target} 1108 for image in sizes: 1109 if image in base_sizes: 1110 base_image = base_sizes[image] 1111 # Loop through the text, data, bss parts 1112 for part in sorted(sizes[image]): 1113 diff = sizes[image][part] - base_image[part] 1114 col = None 1115 if diff: 1116 if image == 'u-boot': 1117 name = part 1118 else: 1119 name = image + ':' + part 1120 err[name] = diff 1121 arch = board_selected[target].arch 1122 if not arch in arch_count: 1123 arch_count[arch] = 1 1124 else: 1125 arch_count[arch] += 1 1126 if not sizes: 1127 pass # Only add to our list when we have some stats 1128 elif not arch in arch_list: 1129 arch_list[arch] = [err] 1130 else: 1131 arch_list[arch].append(err) 1132 1133 # We now have a list of image size changes sorted by arch 1134 # Print out a summary of these 1135 for arch, target_list in arch_list.iteritems(): 1136 # Get total difference for each type 1137 totals = {} 1138 for result in target_list: 1139 total = 0 1140 for name, diff in result.iteritems(): 1141 if name.startswith('_'): 1142 continue 1143 total += diff 1144 if name in totals: 1145 totals[name] += diff 1146 else: 1147 totals[name] = diff 1148 result['_total_diff'] = total 1149 result['_outcome'] = board_dict[result['_target']] 1150 1151 count = len(target_list) 1152 printed_arch = False 1153 for name in sorted(totals): 1154 diff = totals[name] 1155 if diff: 1156 # Display the average difference in this name for this 1157 # architecture 1158 avg_diff = float(diff) / count 1159 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1160 msg = ' %s %+1.1f' % (name, avg_diff) 1161 if not printed_arch: 1162 print '%10s: (for %d/%d boards)' % (arch, count, 1163 arch_count[arch]), 1164 printed_arch = True 1165 print self.col.Color(color, msg), 1166 1167 if printed_arch: 1168 print 1169 if show_detail: 1170 self.PrintSizeDetail(target_list, show_bloat) 1171 1172 1173 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1174 show_sizes, show_detail, show_bloat): 1175 """Compare results with the base results and display delta. 1176 1177 Only boards mentioned in board_selected will be considered. This 1178 function is intended to be called repeatedly with the results of 1179 each commit. It therefore shows a 'diff' between what it saw in 1180 the last call and what it sees now. 1181 1182 Args: 1183 board_selected: Dict containing boards to summarise, keyed by 1184 board.target 1185 board_dict: Dict containing boards for which we built this 1186 commit, keyed by board.target. The value is an Outcome object. 1187 err_lines: A list of errors for this commit, or [] if there is 1188 none, or we don't want to print errors 1189 show_sizes: Show image size deltas 1190 show_detail: Show detail for each board 1191 show_bloat: Show detail for each function 1192 """ 1193 better = [] # List of boards fixed since last commit 1194 worse = [] # List of new broken boards since last commit 1195 new = [] # List of boards that didn't exist last time 1196 unknown = [] # List of boards that were not built 1197 1198 for target in board_dict: 1199 if target not in board_selected: 1200 continue 1201 1202 # If the board was built last time, add its outcome to a list 1203 if target in self._base_board_dict: 1204 base_outcome = self._base_board_dict[target].rc 1205 outcome = board_dict[target] 1206 if outcome.rc == OUTCOME_UNKNOWN: 1207 unknown.append(target) 1208 elif outcome.rc < base_outcome: 1209 better.append(target) 1210 elif outcome.rc > base_outcome: 1211 worse.append(target) 1212 else: 1213 new.append(target) 1214 1215 # Get a list of errors that have appeared, and disappeared 1216 better_err = [] 1217 worse_err = [] 1218 for line in err_lines: 1219 if line not in self._base_err_lines: 1220 worse_err.append('+' + line) 1221 for line in self._base_err_lines: 1222 if line not in err_lines: 1223 better_err.append('-' + line) 1224 1225 # Display results by arch 1226 if better or worse or unknown or new or worse_err or better_err: 1227 arch_list = {} 1228 self.AddOutcome(board_selected, arch_list, better, '', 1229 self.col.GREEN) 1230 self.AddOutcome(board_selected, arch_list, worse, '+', 1231 self.col.RED) 1232 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1233 if self._show_unknown: 1234 self.AddOutcome(board_selected, arch_list, unknown, '?', 1235 self.col.MAGENTA) 1236 for arch, target_list in arch_list.iteritems(): 1237 print '%10s: %s' % (arch, target_list) 1238 if better_err: 1239 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 1240 if worse_err: 1241 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 1242 1243 if show_sizes: 1244 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1245 show_bloat) 1246 1247 # Save our updated information for the next call to this function 1248 self._base_board_dict = board_dict 1249 self._base_err_lines = err_lines 1250 1251 # Get a list of boards that did not get built, if needed 1252 not_built = [] 1253 for board in board_selected: 1254 if not board in board_dict: 1255 not_built.append(board) 1256 if not_built: 1257 print "Boards not built (%d): %s" % (len(not_built), 1258 ', '.join(not_built)) 1259 1260 1261 def ShowSummary(self, commits, board_selected, show_errors, show_sizes, 1262 show_detail, show_bloat): 1263 """Show a build summary for U-Boot for a given board list. 1264 1265 Reset the result summary, then repeatedly call GetResultSummary on 1266 each commit's results, then display the differences we see. 1267 1268 Args: 1269 commit: Commit objects to summarise 1270 board_selected: Dict containing boards to summarise 1271 show_errors: Show errors that occured 1272 show_sizes: Show size deltas 1273 show_detail: Show detail for each board 1274 show_bloat: Show detail for each function 1275 """ 1276 self.commit_count = len(commits) 1277 self.commits = commits 1278 self.ResetResultSummary(board_selected) 1279 1280 for commit_upto in range(0, self.commit_count, self._step): 1281 board_dict, err_lines = self.GetResultSummary(board_selected, 1282 commit_upto, read_func_sizes=show_bloat) 1283 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) 1284 print self.col.Color(self.col.BLUE, msg) 1285 self.PrintResultSummary(board_selected, board_dict, 1286 err_lines if show_errors else [], show_sizes, show_detail, 1287 show_bloat) 1288 1289 1290 def SetupBuild(self, board_selected, commits): 1291 """Set up ready to start a build. 1292 1293 Args: 1294 board_selected: Selected boards to build 1295 commits: Selected commits to build 1296 """ 1297 # First work out how many commits we will build 1298 count = (len(commits) + self._step - 1) / self._step 1299 self.count = len(board_selected) * count 1300 self.upto = self.warned = self.fail = 0 1301 self._timestamps = collections.deque() 1302 1303 def BuildBoardsForCommit(self, board_selected, keep_outputs): 1304 """Build all boards for a single commit""" 1305 self.SetupBuild(board_selected) 1306 self.count = len(board_selected) 1307 for brd in board_selected.itervalues(): 1308 job = BuilderJob() 1309 job.board = brd 1310 job.commits = None 1311 job.keep_outputs = keep_outputs 1312 self.queue.put(brd) 1313 1314 self.queue.join() 1315 self.out_queue.join() 1316 print 1317 self.ClearLine(0) 1318 1319 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): 1320 """Build all boards for all commits (non-incremental)""" 1321 self.commit_count = len(commits) 1322 1323 self.ResetResultSummary(board_selected) 1324 for self.commit_upto in range(self.commit_count): 1325 self.SelectCommit(commits[self.commit_upto]) 1326 self.SelectOutputDir() 1327 Mkdir(self.output_dir) 1328 1329 self.BuildBoardsForCommit(board_selected, keep_outputs) 1330 board_dict, err_lines = self.GetResultSummary() 1331 self.PrintResultSummary(board_selected, board_dict, 1332 err_lines if show_errors else []) 1333 1334 if self.already_done: 1335 print '%d builds already done' % self.already_done 1336 1337 def GetThreadDir(self, thread_num): 1338 """Get the directory path to the working dir for a thread. 1339 1340 Args: 1341 thread_num: Number of thread to check. 1342 """ 1343 return os.path.join(self._working_dir, '%02d' % thread_num) 1344 1345 def _PrepareThread(self, thread_num): 1346 """Prepare the working directory for a thread. 1347 1348 This clones or fetches the repo into the thread's work directory. 1349 1350 Args: 1351 thread_num: Thread number (0, 1, ...) 1352 """ 1353 thread_dir = self.GetThreadDir(thread_num) 1354 Mkdir(thread_dir) 1355 git_dir = os.path.join(thread_dir, '.git') 1356 1357 # Clone the repo if it doesn't already exist 1358 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1359 # we have a private index but uses the origin repo's contents? 1360 if self.git_dir: 1361 src_dir = os.path.abspath(self.git_dir) 1362 if os.path.exists(git_dir): 1363 gitutil.Fetch(git_dir, thread_dir) 1364 else: 1365 print 'Cloning repo for thread %d' % thread_num 1366 gitutil.Clone(src_dir, thread_dir) 1367 1368 def _PrepareWorkingSpace(self, max_threads): 1369 """Prepare the working directory for use. 1370 1371 Set up the git repo for each thread. 1372 1373 Args: 1374 max_threads: Maximum number of threads we expect to need. 1375 """ 1376 Mkdir(self._working_dir) 1377 for thread in range(max_threads): 1378 self._PrepareThread(thread) 1379 1380 def _PrepareOutputSpace(self): 1381 """Get the output directories ready to receive files. 1382 1383 We delete any output directories which look like ones we need to 1384 create. Having left over directories is confusing when the user wants 1385 to check the output manually. 1386 """ 1387 dir_list = [] 1388 for commit_upto in range(self.commit_count): 1389 dir_list.append(self._GetOutputDir(commit_upto)) 1390 1391 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1392 if dirname not in dir_list: 1393 shutil.rmtree(dirname) 1394 1395 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): 1396 """Build all commits for a list of boards 1397 1398 Args: 1399 commits: List of commits to be build, each a Commit object 1400 boards_selected: Dict of selected boards, key is target name, 1401 value is Board object 1402 show_errors: True to show summarised error/warning info 1403 keep_outputs: True to save build output files 1404 """ 1405 self.commit_count = len(commits) 1406 self.commits = commits 1407 1408 self.ResetResultSummary(board_selected) 1409 Mkdir(self.base_dir) 1410 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) 1411 self._PrepareOutputSpace() 1412 self.SetupBuild(board_selected, commits) 1413 self.ProcessResult(None) 1414 1415 # Create jobs to build all commits for each board 1416 for brd in board_selected.itervalues(): 1417 job = BuilderJob() 1418 job.board = brd 1419 job.commits = commits 1420 job.keep_outputs = keep_outputs 1421 job.step = self._step 1422 self.queue.put(job) 1423 1424 # Wait until all jobs are started 1425 self.queue.join() 1426 1427 # Wait until we have processed all output 1428 self.out_queue.join() 1429 print 1430 self.ClearLine(0) 1431