1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23import toolchain 24 25 26""" 27Theory of Operation 28 29Please see README for user documentation, and you should be familiar with 30that before trying to make sense of this. 31 32Buildman works by keeping the machine as busy as possible, building different 33commits for different boards on multiple CPUs at once. 34 35The source repo (self.git_dir) contains all the commits to be built. Each 36thread works on a single board at a time. It checks out the first commit, 37configures it for that board, then builds it. Then it checks out the next 38commit and builds it (typically without re-configuring). When it runs out 39of commits, it gets another job from the builder and starts again with that 40board. 41 42Clearly the builder threads could work either way - they could check out a 43commit and then built it for all boards. Using separate directories for each 44commit/board pair they could leave their build product around afterwards 45also. 46 47The intent behind building a single board for multiple commits, is to make 48use of incremental builds. Since each commit is built incrementally from 49the previous one, builds are faster. Reconfiguring for a different board 50removes all intermediate object files. 51 52Many threads can be working at once, but each has its own working directory. 53When a thread finishes a build, it puts the output files into a result 54directory. 55 56The base directory used by buildman is normally '../<branch>', i.e. 57a directory higher than the source repository and named after the branch 58being built. 59 60Within the base directory, we have one subdirectory for each commit. Within 61that is one subdirectory for each board. Within that is the build output for 62that commit/board combination. 63 64Buildman also create working directories for each thread, in a .bm-work/ 65subdirectory in the base dir. 66 67As an example, say we are building branch 'us-net' for boards 'sandbox' and 68'seaboard', and say that us-net has two commits. We will have directories 69like this: 70 71us-net/ base directory 72 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 73 sandbox/ 74 u-boot.bin 75 seaboard/ 76 u-boot.bin 77 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 78 sandbox/ 79 u-boot.bin 80 seaboard/ 81 u-boot.bin 82 .bm-work/ 83 00/ working directory for thread 0 (contains source checkout) 84 build/ build output 85 01/ working directory for thread 1 86 build/ build output 87 ... 88u-boot/ source directory 89 .git/ repository 90""" 91 92# Possible build outcomes 93OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 94 95# Translate a commit subject into a valid filename 96trans_valid_chars = string.maketrans("/: ", "---") 97 98 99class Builder: 100 """Class for building U-Boot for a particular commit. 101 102 Public members: (many should ->private) 103 active: True if the builder is active and has not been stopped 104 already_done: Number of builds already completed 105 base_dir: Base directory to use for builder 106 checkout: True to check out source, False to skip that step. 107 This is used for testing. 108 col: terminal.Color() object 109 count: Number of commits to build 110 do_make: Method to call to invoke Make 111 fail: Number of builds that failed due to error 112 force_build: Force building even if a build already exists 113 force_config_on_failure: If a commit fails for a board, disable 114 incremental building for the next commit we build for that 115 board, so that we will see all warnings/errors again. 116 force_build_failures: If a previously-built build (i.e. built on 117 a previous run of buildman) is marked as failed, rebuild it. 118 git_dir: Git directory containing source repository 119 last_line_len: Length of the last line we printed (used for erasing 120 it with new progress information) 121 num_jobs: Number of jobs to run at once (passed to make as -j) 122 num_threads: Number of builder threads to run 123 out_queue: Queue of results to process 124 re_make_err: Compiled regular expression for ignore_lines 125 queue: Queue of jobs to run 126 threads: List of active threads 127 toolchains: Toolchains object to use for building 128 upto: Current commit number we are building (0.count-1) 129 warned: Number of builds that produced at least one warning 130 force_reconfig: Reconfigure U-Boot on each comiit. This disables 131 incremental building, where buildman reconfigures on the first 132 commit for a baord, and then just does an incremental build for 133 the following commits. In fact buildman will reconfigure and 134 retry for any failing commits, so generally the only effect of 135 this option is to slow things down. 136 in_tree: Build U-Boot in-tree instead of specifying an output 137 directory separate from the source code. This option is really 138 only useful for testing in-tree builds. 139 140 Private members: 141 _base_board_dict: Last-summarised Dict of boards 142 _base_err_lines: Last-summarised list of errors 143 _build_period_us: Time taken for a single build (float object). 144 _complete_delay: Expected delay until completion (timedelta) 145 _next_delay_update: Next time we plan to display a progress update 146 (datatime) 147 _show_unknown: Show unknown boards (those not built) in summary 148 _timestamps: List of timestamps for the completion of the last 149 last _timestamp_count builds. Each is a datetime object. 150 _timestamp_count: Number of timestamps to keep in our list. 151 _working_dir: Base working directory containing all threads 152 """ 153 class Outcome: 154 """Records a build outcome for a single make invocation 155 156 Public Members: 157 rc: Outcome value (OUTCOME_...) 158 err_lines: List of error lines or [] if none 159 sizes: Dictionary of image size information, keyed by filename 160 - Each value is itself a dictionary containing 161 values for 'text', 'data' and 'bss', being the integer 162 size in bytes of each section. 163 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 164 value is itself a dictionary: 165 key: function name 166 value: Size of function in bytes 167 """ 168 def __init__(self, rc, err_lines, sizes, func_sizes): 169 self.rc = rc 170 self.err_lines = err_lines 171 self.sizes = sizes 172 self.func_sizes = func_sizes 173 174 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 175 gnu_make='make', checkout=True, show_unknown=True, step=1): 176 """Create a new Builder object 177 178 Args: 179 toolchains: Toolchains object to use for building 180 base_dir: Base directory to use for builder 181 git_dir: Git directory containing source repository 182 num_threads: Number of builder threads to run 183 num_jobs: Number of jobs to run at once (passed to make as -j) 184 gnu_make: the command name of GNU Make. 185 checkout: True to check out source, False to skip that step. 186 This is used for testing. 187 show_unknown: Show unknown boards (those not built) in summary 188 step: 1 to process every commit, n to process every nth commit 189 """ 190 self.toolchains = toolchains 191 self.base_dir = base_dir 192 self._working_dir = os.path.join(base_dir, '.bm-work') 193 self.threads = [] 194 self.active = True 195 self.do_make = self.Make 196 self.gnu_make = gnu_make 197 self.checkout = checkout 198 self.num_threads = num_threads 199 self.num_jobs = num_jobs 200 self.already_done = 0 201 self.force_build = False 202 self.git_dir = git_dir 203 self._show_unknown = show_unknown 204 self._timestamp_count = 10 205 self._build_period_us = None 206 self._complete_delay = None 207 self._next_delay_update = datetime.now() 208 self.force_config_on_failure = True 209 self.force_build_failures = False 210 self.force_reconfig = False 211 self._step = step 212 self.in_tree = False 213 self._error_lines = 0 214 215 self.col = terminal.Color() 216 217 self.queue = Queue.Queue() 218 self.out_queue = Queue.Queue() 219 for i in range(self.num_threads): 220 t = builderthread.BuilderThread(self, i) 221 t.setDaemon(True) 222 t.start() 223 self.threads.append(t) 224 225 self.last_line_len = 0 226 t = builderthread.ResultThread(self) 227 t.setDaemon(True) 228 t.start() 229 self.threads.append(t) 230 231 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 232 self.re_make_err = re.compile('|'.join(ignore_lines)) 233 234 def __del__(self): 235 """Get rid of all threads created by the builder""" 236 for t in self.threads: 237 del t 238 239 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 240 show_detail=False, show_bloat=False): 241 """Setup display options for the builder. 242 243 show_errors: True to show summarised error/warning info 244 show_sizes: Show size deltas 245 show_detail: Show detail for each board 246 show_bloat: Show detail for each function 247 """ 248 self._show_errors = show_errors 249 self._show_sizes = show_sizes 250 self._show_detail = show_detail 251 self._show_bloat = show_bloat 252 253 def _AddTimestamp(self): 254 """Add a new timestamp to the list and record the build period. 255 256 The build period is the length of time taken to perform a single 257 build (one board, one commit). 258 """ 259 now = datetime.now() 260 self._timestamps.append(now) 261 count = len(self._timestamps) 262 delta = self._timestamps[-1] - self._timestamps[0] 263 seconds = delta.total_seconds() 264 265 # If we have enough data, estimate build period (time taken for a 266 # single build) and therefore completion time. 267 if count > 1 and self._next_delay_update < now: 268 self._next_delay_update = now + timedelta(seconds=2) 269 if seconds > 0: 270 self._build_period = float(seconds) / count 271 todo = self.count - self.upto 272 self._complete_delay = timedelta(microseconds= 273 self._build_period * todo * 1000000) 274 # Round it 275 self._complete_delay -= timedelta( 276 microseconds=self._complete_delay.microseconds) 277 278 if seconds > 60: 279 self._timestamps.popleft() 280 count -= 1 281 282 def ClearLine(self, length): 283 """Clear any characters on the current line 284 285 Make way for a new line of length 'length', by outputting enough 286 spaces to clear out the old line. Then remember the new length for 287 next time. 288 289 Args: 290 length: Length of new line, in characters 291 """ 292 if length < self.last_line_len: 293 print ' ' * (self.last_line_len - length), 294 print '\r', 295 self.last_line_len = length 296 sys.stdout.flush() 297 298 def SelectCommit(self, commit, checkout=True): 299 """Checkout the selected commit for this build 300 """ 301 self.commit = commit 302 if checkout and self.checkout: 303 gitutil.Checkout(commit.hash) 304 305 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 306 """Run make 307 308 Args: 309 commit: Commit object that is being built 310 brd: Board object that is being built 311 stage: Stage that we are at (mrproper, config, build) 312 cwd: Directory where make should be run 313 args: Arguments to pass to make 314 kwargs: Arguments to pass to command.RunPipe() 315 """ 316 cmd = [self.gnu_make] + list(args) 317 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 318 cwd=cwd, raise_on_error=False, **kwargs) 319 return result 320 321 def ProcessResult(self, result): 322 """Process the result of a build, showing progress information 323 324 Args: 325 result: A CommandResult object, which indicates the result for 326 a single build 327 """ 328 col = terminal.Color() 329 if result: 330 target = result.brd.target 331 332 if result.return_code < 0: 333 self.active = False 334 command.StopAll() 335 return 336 337 self.upto += 1 338 if result.return_code != 0: 339 self.fail += 1 340 elif result.stderr: 341 self.warned += 1 342 if result.already_done: 343 self.already_done += 1 344 if self._verbose: 345 print '\r', 346 self.ClearLine(0) 347 boards_selected = {target : result.brd} 348 self.ResetResultSummary(boards_selected) 349 self.ProduceResultSummary(result.commit_upto, self.commits, 350 boards_selected) 351 else: 352 target = '(starting)' 353 354 # Display separate counts for ok, warned and fail 355 ok = self.upto - self.warned - self.fail 356 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 357 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 358 line += self.col.Color(self.col.RED, '%5d' % self.fail) 359 360 name = ' /%-5d ' % self.count 361 362 # Add our current completion time estimate 363 self._AddTimestamp() 364 if self._complete_delay: 365 name += '%s : ' % self._complete_delay 366 # When building all boards for a commit, we can print a commit 367 # progress message. 368 if result and result.commit_upto is None: 369 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 370 self.commit_count) 371 372 name += target 373 print line + name, 374 length = 14 + len(name) 375 self.ClearLine(length) 376 377 def _GetOutputDir(self, commit_upto): 378 """Get the name of the output directory for a commit number 379 380 The output directory is typically .../<branch>/<commit>. 381 382 Args: 383 commit_upto: Commit number to use (0..self.count-1) 384 """ 385 if self.commits: 386 commit = self.commits[commit_upto] 387 subject = commit.subject.translate(trans_valid_chars) 388 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 389 self.commit_count, commit.hash, subject[:20])) 390 else: 391 commit_dir = 'current' 392 output_dir = os.path.join(self.base_dir, commit_dir) 393 return output_dir 394 395 def GetBuildDir(self, commit_upto, target): 396 """Get the name of the build directory for a commit number 397 398 The build directory is typically .../<branch>/<commit>/<target>. 399 400 Args: 401 commit_upto: Commit number to use (0..self.count-1) 402 target: Target name 403 """ 404 output_dir = self._GetOutputDir(commit_upto) 405 return os.path.join(output_dir, target) 406 407 def GetDoneFile(self, commit_upto, target): 408 """Get the name of the done file for a commit number 409 410 Args: 411 commit_upto: Commit number to use (0..self.count-1) 412 target: Target name 413 """ 414 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 415 416 def GetSizesFile(self, commit_upto, target): 417 """Get the name of the sizes file for a commit number 418 419 Args: 420 commit_upto: Commit number to use (0..self.count-1) 421 target: Target name 422 """ 423 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 424 425 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 426 """Get the name of the funcsizes file for a commit number and ELF file 427 428 Args: 429 commit_upto: Commit number to use (0..self.count-1) 430 target: Target name 431 elf_fname: Filename of elf image 432 """ 433 return os.path.join(self.GetBuildDir(commit_upto, target), 434 '%s.sizes' % elf_fname.replace('/', '-')) 435 436 def GetObjdumpFile(self, commit_upto, target, elf_fname): 437 """Get the name of the objdump file for a commit number and ELF file 438 439 Args: 440 commit_upto: Commit number to use (0..self.count-1) 441 target: Target name 442 elf_fname: Filename of elf image 443 """ 444 return os.path.join(self.GetBuildDir(commit_upto, target), 445 '%s.objdump' % elf_fname.replace('/', '-')) 446 447 def GetErrFile(self, commit_upto, target): 448 """Get the name of the err file for a commit number 449 450 Args: 451 commit_upto: Commit number to use (0..self.count-1) 452 target: Target name 453 """ 454 output_dir = self.GetBuildDir(commit_upto, target) 455 return os.path.join(output_dir, 'err') 456 457 def FilterErrors(self, lines): 458 """Filter out errors in which we have no interest 459 460 We should probably use map(). 461 462 Args: 463 lines: List of error lines, each a string 464 Returns: 465 New list with only interesting lines included 466 """ 467 out_lines = [] 468 for line in lines: 469 if not self.re_make_err.search(line): 470 out_lines.append(line) 471 return out_lines 472 473 def ReadFuncSizes(self, fname, fd): 474 """Read function sizes from the output of 'nm' 475 476 Args: 477 fd: File containing data to read 478 fname: Filename we are reading from (just for errors) 479 480 Returns: 481 Dictionary containing size of each function in bytes, indexed by 482 function name. 483 """ 484 sym = {} 485 for line in fd.readlines(): 486 try: 487 size, type, name = line[:-1].split() 488 except: 489 print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) 490 continue 491 if type in 'tTdDbB': 492 # function names begin with '.' on 64-bit powerpc 493 if '.' in name[1:]: 494 name = 'static.' + name.split('.')[0] 495 sym[name] = sym.get(name, 0) + int(size, 16) 496 return sym 497 498 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 499 """Work out the outcome of a build. 500 501 Args: 502 commit_upto: Commit number to check (0..n-1) 503 target: Target board to check 504 read_func_sizes: True to read function size information 505 506 Returns: 507 Outcome object 508 """ 509 done_file = self.GetDoneFile(commit_upto, target) 510 sizes_file = self.GetSizesFile(commit_upto, target) 511 sizes = {} 512 func_sizes = {} 513 if os.path.exists(done_file): 514 with open(done_file, 'r') as fd: 515 return_code = int(fd.readline()) 516 err_lines = [] 517 err_file = self.GetErrFile(commit_upto, target) 518 if os.path.exists(err_file): 519 with open(err_file, 'r') as fd: 520 err_lines = self.FilterErrors(fd.readlines()) 521 522 # Decide whether the build was ok, failed or created warnings 523 if return_code: 524 rc = OUTCOME_ERROR 525 elif len(err_lines): 526 rc = OUTCOME_WARNING 527 else: 528 rc = OUTCOME_OK 529 530 # Convert size information to our simple format 531 if os.path.exists(sizes_file): 532 with open(sizes_file, 'r') as fd: 533 for line in fd.readlines(): 534 values = line.split() 535 rodata = 0 536 if len(values) > 6: 537 rodata = int(values[6], 16) 538 size_dict = { 539 'all' : int(values[0]) + int(values[1]) + 540 int(values[2]), 541 'text' : int(values[0]) - rodata, 542 'data' : int(values[1]), 543 'bss' : int(values[2]), 544 'rodata' : rodata, 545 } 546 sizes[values[5]] = size_dict 547 548 if read_func_sizes: 549 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 550 for fname in glob.glob(pattern): 551 with open(fname, 'r') as fd: 552 dict_name = os.path.basename(fname).replace('.sizes', 553 '') 554 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 555 556 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 557 558 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 559 560 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 561 """Calculate a summary of the results of building a commit. 562 563 Args: 564 board_selected: Dict containing boards to summarise 565 commit_upto: Commit number to summarize (0..self.count-1) 566 read_func_sizes: True to read function size information 567 568 Returns: 569 Tuple: 570 Dict containing boards which passed building this commit. 571 keyed by board.target 572 List containing a summary of error/warning lines 573 """ 574 board_dict = {} 575 err_lines_summary = [] 576 577 for board in boards_selected.itervalues(): 578 outcome = self.GetBuildOutcome(commit_upto, board.target, 579 read_func_sizes) 580 board_dict[board.target] = outcome 581 for err in outcome.err_lines: 582 if err and not err.rstrip() in err_lines_summary: 583 err_lines_summary.append(err.rstrip()) 584 return board_dict, err_lines_summary 585 586 def AddOutcome(self, board_dict, arch_list, changes, char, color): 587 """Add an output to our list of outcomes for each architecture 588 589 This simple function adds failing boards (changes) to the 590 relevant architecture string, so we can print the results out 591 sorted by architecture. 592 593 Args: 594 board_dict: Dict containing all boards 595 arch_list: Dict keyed by arch name. Value is a string containing 596 a list of board names which failed for that arch. 597 changes: List of boards to add to arch_list 598 color: terminal.Colour object 599 """ 600 done_arch = {} 601 for target in changes: 602 if target in board_dict: 603 arch = board_dict[target].arch 604 else: 605 arch = 'unknown' 606 str = self.col.Color(color, ' ' + target) 607 if not arch in done_arch: 608 str = self.col.Color(color, char) + ' ' + str 609 done_arch[arch] = True 610 if not arch in arch_list: 611 arch_list[arch] = str 612 else: 613 arch_list[arch] += str 614 615 616 def ColourNum(self, num): 617 color = self.col.RED if num > 0 else self.col.GREEN 618 if num == 0: 619 return '0' 620 return self.col.Color(color, str(num)) 621 622 def ResetResultSummary(self, board_selected): 623 """Reset the results summary ready for use. 624 625 Set up the base board list to be all those selected, and set the 626 error lines to empty. 627 628 Following this, calls to PrintResultSummary() will use this 629 information to work out what has changed. 630 631 Args: 632 board_selected: Dict containing boards to summarise, keyed by 633 board.target 634 """ 635 self._base_board_dict = {} 636 for board in board_selected: 637 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 638 self._base_err_lines = [] 639 640 def PrintFuncSizeDetail(self, fname, old, new): 641 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 642 delta, common = [], {} 643 644 for a in old: 645 if a in new: 646 common[a] = 1 647 648 for name in old: 649 if name not in common: 650 remove += 1 651 down += old[name] 652 delta.append([-old[name], name]) 653 654 for name in new: 655 if name not in common: 656 add += 1 657 up += new[name] 658 delta.append([new[name], name]) 659 660 for name in common: 661 diff = new.get(name, 0) - old.get(name, 0) 662 if diff > 0: 663 grow, up = grow + 1, up + diff 664 elif diff < 0: 665 shrink, down = shrink + 1, down - diff 666 delta.append([diff, name]) 667 668 delta.sort() 669 delta.reverse() 670 671 args = [add, -remove, grow, -shrink, up, -down, up - down] 672 if max(args) == 0: 673 return 674 args = [self.ColourNum(x) for x in args] 675 indent = ' ' * 15 676 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 677 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 678 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 679 'delta') 680 for diff, name in delta: 681 if diff: 682 color = self.col.RED if diff > 0 else self.col.GREEN 683 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 684 old.get(name, '-'), new.get(name,'-'), diff) 685 print self.col.Color(color, msg) 686 687 688 def PrintSizeDetail(self, target_list, show_bloat): 689 """Show details size information for each board 690 691 Args: 692 target_list: List of targets, each a dict containing: 693 'target': Target name 694 'total_diff': Total difference in bytes across all areas 695 <part_name>: Difference for that part 696 show_bloat: Show detail for each function 697 """ 698 targets_by_diff = sorted(target_list, reverse=True, 699 key=lambda x: x['_total_diff']) 700 for result in targets_by_diff: 701 printed_target = False 702 for name in sorted(result): 703 diff = result[name] 704 if name.startswith('_'): 705 continue 706 if diff != 0: 707 color = self.col.RED if diff > 0 else self.col.GREEN 708 msg = ' %s %+d' % (name, diff) 709 if not printed_target: 710 print '%10s %-15s:' % ('', result['_target']), 711 printed_target = True 712 print self.col.Color(color, msg), 713 if printed_target: 714 print 715 if show_bloat: 716 target = result['_target'] 717 outcome = result['_outcome'] 718 base_outcome = self._base_board_dict[target] 719 for fname in outcome.func_sizes: 720 self.PrintFuncSizeDetail(fname, 721 base_outcome.func_sizes[fname], 722 outcome.func_sizes[fname]) 723 724 725 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 726 show_bloat): 727 """Print a summary of image sizes broken down by section. 728 729 The summary takes the form of one line per architecture. The 730 line contains deltas for each of the sections (+ means the section 731 got bigger, - means smaller). The nunmbers are the average number 732 of bytes that a board in this section increased by. 733 734 For example: 735 powerpc: (622 boards) text -0.0 736 arm: (285 boards) text -0.0 737 nds32: (3 boards) text -8.0 738 739 Args: 740 board_selected: Dict containing boards to summarise, keyed by 741 board.target 742 board_dict: Dict containing boards for which we built this 743 commit, keyed by board.target. The value is an Outcome object. 744 show_detail: Show detail for each board 745 show_bloat: Show detail for each function 746 """ 747 arch_list = {} 748 arch_count = {} 749 750 # Calculate changes in size for different image parts 751 # The previous sizes are in Board.sizes, for each board 752 for target in board_dict: 753 if target not in board_selected: 754 continue 755 base_sizes = self._base_board_dict[target].sizes 756 outcome = board_dict[target] 757 sizes = outcome.sizes 758 759 # Loop through the list of images, creating a dict of size 760 # changes for each image/part. We end up with something like 761 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 762 # which means that U-Boot data increased by 5 bytes and SPL 763 # text decreased by 4. 764 err = {'_target' : target} 765 for image in sizes: 766 if image in base_sizes: 767 base_image = base_sizes[image] 768 # Loop through the text, data, bss parts 769 for part in sorted(sizes[image]): 770 diff = sizes[image][part] - base_image[part] 771 col = None 772 if diff: 773 if image == 'u-boot': 774 name = part 775 else: 776 name = image + ':' + part 777 err[name] = diff 778 arch = board_selected[target].arch 779 if not arch in arch_count: 780 arch_count[arch] = 1 781 else: 782 arch_count[arch] += 1 783 if not sizes: 784 pass # Only add to our list when we have some stats 785 elif not arch in arch_list: 786 arch_list[arch] = [err] 787 else: 788 arch_list[arch].append(err) 789 790 # We now have a list of image size changes sorted by arch 791 # Print out a summary of these 792 for arch, target_list in arch_list.iteritems(): 793 # Get total difference for each type 794 totals = {} 795 for result in target_list: 796 total = 0 797 for name, diff in result.iteritems(): 798 if name.startswith('_'): 799 continue 800 total += diff 801 if name in totals: 802 totals[name] += diff 803 else: 804 totals[name] = diff 805 result['_total_diff'] = total 806 result['_outcome'] = board_dict[result['_target']] 807 808 count = len(target_list) 809 printed_arch = False 810 for name in sorted(totals): 811 diff = totals[name] 812 if diff: 813 # Display the average difference in this name for this 814 # architecture 815 avg_diff = float(diff) / count 816 color = self.col.RED if avg_diff > 0 else self.col.GREEN 817 msg = ' %s %+1.1f' % (name, avg_diff) 818 if not printed_arch: 819 print '%10s: (for %d/%d boards)' % (arch, count, 820 arch_count[arch]), 821 printed_arch = True 822 print self.col.Color(color, msg), 823 824 if printed_arch: 825 print 826 if show_detail: 827 self.PrintSizeDetail(target_list, show_bloat) 828 829 830 def PrintResultSummary(self, board_selected, board_dict, err_lines, 831 show_sizes, show_detail, show_bloat): 832 """Compare results with the base results and display delta. 833 834 Only boards mentioned in board_selected will be considered. This 835 function is intended to be called repeatedly with the results of 836 each commit. It therefore shows a 'diff' between what it saw in 837 the last call and what it sees now. 838 839 Args: 840 board_selected: Dict containing boards to summarise, keyed by 841 board.target 842 board_dict: Dict containing boards for which we built this 843 commit, keyed by board.target. The value is an Outcome object. 844 err_lines: A list of errors for this commit, or [] if there is 845 none, or we don't want to print errors 846 show_sizes: Show image size deltas 847 show_detail: Show detail for each board 848 show_bloat: Show detail for each function 849 """ 850 better = [] # List of boards fixed since last commit 851 worse = [] # List of new broken boards since last commit 852 new = [] # List of boards that didn't exist last time 853 unknown = [] # List of boards that were not built 854 855 for target in board_dict: 856 if target not in board_selected: 857 continue 858 859 # If the board was built last time, add its outcome to a list 860 if target in self._base_board_dict: 861 base_outcome = self._base_board_dict[target].rc 862 outcome = board_dict[target] 863 if outcome.rc == OUTCOME_UNKNOWN: 864 unknown.append(target) 865 elif outcome.rc < base_outcome: 866 better.append(target) 867 elif outcome.rc > base_outcome: 868 worse.append(target) 869 else: 870 new.append(target) 871 872 # Get a list of errors that have appeared, and disappeared 873 better_err = [] 874 worse_err = [] 875 for line in err_lines: 876 if line not in self._base_err_lines: 877 worse_err.append('+' + line) 878 for line in self._base_err_lines: 879 if line not in err_lines: 880 better_err.append('-' + line) 881 882 # Display results by arch 883 if better or worse or unknown or new or worse_err or better_err: 884 arch_list = {} 885 self.AddOutcome(board_selected, arch_list, better, '', 886 self.col.GREEN) 887 self.AddOutcome(board_selected, arch_list, worse, '+', 888 self.col.RED) 889 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 890 if self._show_unknown: 891 self.AddOutcome(board_selected, arch_list, unknown, '?', 892 self.col.MAGENTA) 893 for arch, target_list in arch_list.iteritems(): 894 print '%10s: %s' % (arch, target_list) 895 self._error_lines += 1 896 if better_err: 897 print self.col.Color(self.col.GREEN, '\n'.join(better_err)) 898 self._error_lines += 1 899 if worse_err: 900 print self.col.Color(self.col.RED, '\n'.join(worse_err)) 901 self._error_lines += 1 902 903 if show_sizes: 904 self.PrintSizeSummary(board_selected, board_dict, show_detail, 905 show_bloat) 906 907 # Save our updated information for the next call to this function 908 self._base_board_dict = board_dict 909 self._base_err_lines = err_lines 910 911 # Get a list of boards that did not get built, if needed 912 not_built = [] 913 for board in board_selected: 914 if not board in board_dict: 915 not_built.append(board) 916 if not_built: 917 print "Boards not built (%d): %s" % (len(not_built), 918 ', '.join(not_built)) 919 920 def ProduceResultSummary(self, commit_upto, commits, board_selected): 921 board_dict, err_lines = self.GetResultSummary(board_selected, 922 commit_upto, read_func_sizes=self._show_bloat) 923 if commits: 924 msg = '%02d: %s' % (commit_upto + 1, 925 commits[commit_upto].subject) 926 print self.col.Color(self.col.BLUE, msg) 927 self.PrintResultSummary(board_selected, board_dict, 928 err_lines if self._show_errors else [], 929 self._show_sizes, self._show_detail, self._show_bloat) 930 931 def ShowSummary(self, commits, board_selected): 932 """Show a build summary for U-Boot for a given board list. 933 934 Reset the result summary, then repeatedly call GetResultSummary on 935 each commit's results, then display the differences we see. 936 937 Args: 938 commit: Commit objects to summarise 939 board_selected: Dict containing boards to summarise 940 """ 941 self.commit_count = len(commits) if commits else 1 942 self.commits = commits 943 self.ResetResultSummary(board_selected) 944 self._error_lines = 0 945 946 for commit_upto in range(0, self.commit_count, self._step): 947 self.ProduceResultSummary(commit_upto, commits, board_selected) 948 if not self._error_lines: 949 print self.col.Color(self.col.GREEN, '(no errors to report)') 950 951 952 def SetupBuild(self, board_selected, commits): 953 """Set up ready to start a build. 954 955 Args: 956 board_selected: Selected boards to build 957 commits: Selected commits to build 958 """ 959 # First work out how many commits we will build 960 count = (self.commit_count + self._step - 1) / self._step 961 self.count = len(board_selected) * count 962 self.upto = self.warned = self.fail = 0 963 self._timestamps = collections.deque() 964 965 def GetThreadDir(self, thread_num): 966 """Get the directory path to the working dir for a thread. 967 968 Args: 969 thread_num: Number of thread to check. 970 """ 971 return os.path.join(self._working_dir, '%02d' % thread_num) 972 973 def _PrepareThread(self, thread_num, setup_git): 974 """Prepare the working directory for a thread. 975 976 This clones or fetches the repo into the thread's work directory. 977 978 Args: 979 thread_num: Thread number (0, 1, ...) 980 setup_git: True to set up a git repo clone 981 """ 982 thread_dir = self.GetThreadDir(thread_num) 983 builderthread.Mkdir(thread_dir) 984 git_dir = os.path.join(thread_dir, '.git') 985 986 # Clone the repo if it doesn't already exist 987 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 988 # we have a private index but uses the origin repo's contents? 989 if setup_git and self.git_dir: 990 src_dir = os.path.abspath(self.git_dir) 991 if os.path.exists(git_dir): 992 gitutil.Fetch(git_dir, thread_dir) 993 else: 994 print 'Cloning repo for thread %d' % thread_num 995 gitutil.Clone(src_dir, thread_dir) 996 997 def _PrepareWorkingSpace(self, max_threads, setup_git): 998 """Prepare the working directory for use. 999 1000 Set up the git repo for each thread. 1001 1002 Args: 1003 max_threads: Maximum number of threads we expect to need. 1004 setup_git: True to set up a git repo clone 1005 """ 1006 builderthread.Mkdir(self._working_dir) 1007 for thread in range(max_threads): 1008 self._PrepareThread(thread, setup_git) 1009 1010 def _PrepareOutputSpace(self): 1011 """Get the output directories ready to receive files. 1012 1013 We delete any output directories which look like ones we need to 1014 create. Having left over directories is confusing when the user wants 1015 to check the output manually. 1016 """ 1017 dir_list = [] 1018 for commit_upto in range(self.commit_count): 1019 dir_list.append(self._GetOutputDir(commit_upto)) 1020 1021 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1022 if dirname not in dir_list: 1023 shutil.rmtree(dirname) 1024 1025 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1026 """Build all commits for a list of boards 1027 1028 Args: 1029 commits: List of commits to be build, each a Commit object 1030 boards_selected: Dict of selected boards, key is target name, 1031 value is Board object 1032 keep_outputs: True to save build output files 1033 verbose: Display build results as they are completed 1034 """ 1035 self.commit_count = len(commits) if commits else 1 1036 self.commits = commits 1037 self._verbose = verbose 1038 1039 self.ResetResultSummary(board_selected) 1040 builderthread.Mkdir(self.base_dir) 1041 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1042 commits is not None) 1043 self._PrepareOutputSpace() 1044 self.SetupBuild(board_selected, commits) 1045 self.ProcessResult(None) 1046 1047 # Create jobs to build all commits for each board 1048 for brd in board_selected.itervalues(): 1049 job = builderthread.BuilderJob() 1050 job.board = brd 1051 job.commits = commits 1052 job.keep_outputs = keep_outputs 1053 job.step = self._step 1054 self.queue.put(job) 1055 1056 # Wait until all jobs are started 1057 self.queue.join() 1058 1059 # Wait until we have processed all output 1060 self.out_queue.join() 1061 print 1062 self.ClearLine(0) 1063