1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100class Builder: 101 """Class for building U-Boot for a particular commit. 102 103 Public members: (many should ->private) 104 active: True if the builder is active and has not been stopped 105 already_done: Number of builds already completed 106 base_dir: Base directory to use for builder 107 checkout: True to check out source, False to skip that step. 108 This is used for testing. 109 col: terminal.Color() object 110 count: Number of commits to build 111 do_make: Method to call to invoke Make 112 fail: Number of builds that failed due to error 113 force_build: Force building even if a build already exists 114 force_config_on_failure: If a commit fails for a board, disable 115 incremental building for the next commit we build for that 116 board, so that we will see all warnings/errors again. 117 force_build_failures: If a previously-built build (i.e. built on 118 a previous run of buildman) is marked as failed, rebuild it. 119 git_dir: Git directory containing source repository 120 last_line_len: Length of the last line we printed (used for erasing 121 it with new progress information) 122 num_jobs: Number of jobs to run at once (passed to make as -j) 123 num_threads: Number of builder threads to run 124 out_queue: Queue of results to process 125 re_make_err: Compiled regular expression for ignore_lines 126 queue: Queue of jobs to run 127 threads: List of active threads 128 toolchains: Toolchains object to use for building 129 upto: Current commit number we are building (0.count-1) 130 warned: Number of builds that produced at least one warning 131 force_reconfig: Reconfigure U-Boot on each comiit. This disables 132 incremental building, where buildman reconfigures on the first 133 commit for a baord, and then just does an incremental build for 134 the following commits. In fact buildman will reconfigure and 135 retry for any failing commits, so generally the only effect of 136 this option is to slow things down. 137 in_tree: Build U-Boot in-tree instead of specifying an output 138 directory separate from the source code. This option is really 139 only useful for testing in-tree builds. 140 141 Private members: 142 _base_board_dict: Last-summarised Dict of boards 143 _base_err_lines: Last-summarised list of errors 144 _base_warn_lines: Last-summarised list of warnings 145 _build_period_us: Time taken for a single build (float object). 146 _complete_delay: Expected delay until completion (timedelta) 147 _next_delay_update: Next time we plan to display a progress update 148 (datatime) 149 _show_unknown: Show unknown boards (those not built) in summary 150 _timestamps: List of timestamps for the completion of the last 151 last _timestamp_count builds. Each is a datetime object. 152 _timestamp_count: Number of timestamps to keep in our list. 153 _working_dir: Base working directory containing all threads 154 """ 155 class Outcome: 156 """Records a build outcome for a single make invocation 157 158 Public Members: 159 rc: Outcome value (OUTCOME_...) 160 err_lines: List of error lines or [] if none 161 sizes: Dictionary of image size information, keyed by filename 162 - Each value is itself a dictionary containing 163 values for 'text', 'data' and 'bss', being the integer 164 size in bytes of each section. 165 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 166 value is itself a dictionary: 167 key: function name 168 value: Size of function in bytes 169 """ 170 def __init__(self, rc, err_lines, sizes, func_sizes): 171 self.rc = rc 172 self.err_lines = err_lines 173 self.sizes = sizes 174 self.func_sizes = func_sizes 175 176 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 177 gnu_make='make', checkout=True, show_unknown=True, step=1, 178 no_subdirs=False): 179 """Create a new Builder object 180 181 Args: 182 toolchains: Toolchains object to use for building 183 base_dir: Base directory to use for builder 184 git_dir: Git directory containing source repository 185 num_threads: Number of builder threads to run 186 num_jobs: Number of jobs to run at once (passed to make as -j) 187 gnu_make: the command name of GNU Make. 188 checkout: True to check out source, False to skip that step. 189 This is used for testing. 190 show_unknown: Show unknown boards (those not built) in summary 191 step: 1 to process every commit, n to process every nth commit 192 """ 193 self.toolchains = toolchains 194 self.base_dir = base_dir 195 self._working_dir = os.path.join(base_dir, '.bm-work') 196 self.threads = [] 197 self.active = True 198 self.do_make = self.Make 199 self.gnu_make = gnu_make 200 self.checkout = checkout 201 self.num_threads = num_threads 202 self.num_jobs = num_jobs 203 self.already_done = 0 204 self.force_build = False 205 self.git_dir = git_dir 206 self._show_unknown = show_unknown 207 self._timestamp_count = 10 208 self._build_period_us = None 209 self._complete_delay = None 210 self._next_delay_update = datetime.now() 211 self.force_config_on_failure = True 212 self.force_build_failures = False 213 self.force_reconfig = False 214 self._step = step 215 self.in_tree = False 216 self._error_lines = 0 217 self.no_subdirs = no_subdirs 218 219 self.col = terminal.Color() 220 221 self._re_function = re.compile('(.*): In function.*') 222 self._re_files = re.compile('In file included from.*') 223 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 224 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 225 226 self.queue = Queue.Queue() 227 self.out_queue = Queue.Queue() 228 for i in range(self.num_threads): 229 t = builderthread.BuilderThread(self, i) 230 t.setDaemon(True) 231 t.start() 232 self.threads.append(t) 233 234 self.last_line_len = 0 235 t = builderthread.ResultThread(self) 236 t.setDaemon(True) 237 t.start() 238 self.threads.append(t) 239 240 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 241 self.re_make_err = re.compile('|'.join(ignore_lines)) 242 243 def __del__(self): 244 """Get rid of all threads created by the builder""" 245 for t in self.threads: 246 del t 247 248 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 249 show_detail=False, show_bloat=False, 250 list_error_boards=False): 251 """Setup display options for the builder. 252 253 show_errors: True to show summarised error/warning info 254 show_sizes: Show size deltas 255 show_detail: Show detail for each board 256 show_bloat: Show detail for each function 257 list_error_boards: Show the boards which caused each error/warning 258 """ 259 self._show_errors = show_errors 260 self._show_sizes = show_sizes 261 self._show_detail = show_detail 262 self._show_bloat = show_bloat 263 self._list_error_boards = list_error_boards 264 265 def _AddTimestamp(self): 266 """Add a new timestamp to the list and record the build period. 267 268 The build period is the length of time taken to perform a single 269 build (one board, one commit). 270 """ 271 now = datetime.now() 272 self._timestamps.append(now) 273 count = len(self._timestamps) 274 delta = self._timestamps[-1] - self._timestamps[0] 275 seconds = delta.total_seconds() 276 277 # If we have enough data, estimate build period (time taken for a 278 # single build) and therefore completion time. 279 if count > 1 and self._next_delay_update < now: 280 self._next_delay_update = now + timedelta(seconds=2) 281 if seconds > 0: 282 self._build_period = float(seconds) / count 283 todo = self.count - self.upto 284 self._complete_delay = timedelta(microseconds= 285 self._build_period * todo * 1000000) 286 # Round it 287 self._complete_delay -= timedelta( 288 microseconds=self._complete_delay.microseconds) 289 290 if seconds > 60: 291 self._timestamps.popleft() 292 count -= 1 293 294 def ClearLine(self, length): 295 """Clear any characters on the current line 296 297 Make way for a new line of length 'length', by outputting enough 298 spaces to clear out the old line. Then remember the new length for 299 next time. 300 301 Args: 302 length: Length of new line, in characters 303 """ 304 if length < self.last_line_len: 305 Print(' ' * (self.last_line_len - length), newline=False) 306 Print('\r', newline=False) 307 self.last_line_len = length 308 sys.stdout.flush() 309 310 def SelectCommit(self, commit, checkout=True): 311 """Checkout the selected commit for this build 312 """ 313 self.commit = commit 314 if checkout and self.checkout: 315 gitutil.Checkout(commit.hash) 316 317 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 318 """Run make 319 320 Args: 321 commit: Commit object that is being built 322 brd: Board object that is being built 323 stage: Stage that we are at (mrproper, config, build) 324 cwd: Directory where make should be run 325 args: Arguments to pass to make 326 kwargs: Arguments to pass to command.RunPipe() 327 """ 328 cmd = [self.gnu_make] + list(args) 329 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 330 cwd=cwd, raise_on_error=False, **kwargs) 331 return result 332 333 def ProcessResult(self, result): 334 """Process the result of a build, showing progress information 335 336 Args: 337 result: A CommandResult object, which indicates the result for 338 a single build 339 """ 340 col = terminal.Color() 341 if result: 342 target = result.brd.target 343 344 if result.return_code < 0: 345 self.active = False 346 command.StopAll() 347 return 348 349 self.upto += 1 350 if result.return_code != 0: 351 self.fail += 1 352 elif result.stderr: 353 self.warned += 1 354 if result.already_done: 355 self.already_done += 1 356 if self._verbose: 357 Print('\r', newline=False) 358 self.ClearLine(0) 359 boards_selected = {target : result.brd} 360 self.ResetResultSummary(boards_selected) 361 self.ProduceResultSummary(result.commit_upto, self.commits, 362 boards_selected) 363 else: 364 target = '(starting)' 365 366 # Display separate counts for ok, warned and fail 367 ok = self.upto - self.warned - self.fail 368 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 369 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 370 line += self.col.Color(self.col.RED, '%5d' % self.fail) 371 372 name = ' /%-5d ' % self.count 373 374 # Add our current completion time estimate 375 self._AddTimestamp() 376 if self._complete_delay: 377 name += '%s : ' % self._complete_delay 378 # When building all boards for a commit, we can print a commit 379 # progress message. 380 if result and result.commit_upto is None: 381 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 382 self.commit_count) 383 384 name += target 385 Print(line + name, newline=False) 386 length = 14 + len(name) 387 self.ClearLine(length) 388 389 def _GetOutputDir(self, commit_upto): 390 """Get the name of the output directory for a commit number 391 392 The output directory is typically .../<branch>/<commit>. 393 394 Args: 395 commit_upto: Commit number to use (0..self.count-1) 396 """ 397 commit_dir = None 398 if self.commits: 399 commit = self.commits[commit_upto] 400 subject = commit.subject.translate(trans_valid_chars) 401 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 402 self.commit_count, commit.hash, subject[:20])) 403 elif not self.no_subdirs: 404 commit_dir = 'current' 405 if not commit_dir: 406 return self.base_dir 407 return os.path.join(self.base_dir, commit_dir) 408 409 def GetBuildDir(self, commit_upto, target): 410 """Get the name of the build directory for a commit number 411 412 The build directory is typically .../<branch>/<commit>/<target>. 413 414 Args: 415 commit_upto: Commit number to use (0..self.count-1) 416 target: Target name 417 """ 418 output_dir = self._GetOutputDir(commit_upto) 419 return os.path.join(output_dir, target) 420 421 def GetDoneFile(self, commit_upto, target): 422 """Get the name of the done file for a commit number 423 424 Args: 425 commit_upto: Commit number to use (0..self.count-1) 426 target: Target name 427 """ 428 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 429 430 def GetSizesFile(self, commit_upto, target): 431 """Get the name of the sizes file for a commit number 432 433 Args: 434 commit_upto: Commit number to use (0..self.count-1) 435 target: Target name 436 """ 437 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 438 439 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 440 """Get the name of the funcsizes file for a commit number and ELF file 441 442 Args: 443 commit_upto: Commit number to use (0..self.count-1) 444 target: Target name 445 elf_fname: Filename of elf image 446 """ 447 return os.path.join(self.GetBuildDir(commit_upto, target), 448 '%s.sizes' % elf_fname.replace('/', '-')) 449 450 def GetObjdumpFile(self, commit_upto, target, elf_fname): 451 """Get the name of the objdump file for a commit number and ELF file 452 453 Args: 454 commit_upto: Commit number to use (0..self.count-1) 455 target: Target name 456 elf_fname: Filename of elf image 457 """ 458 return os.path.join(self.GetBuildDir(commit_upto, target), 459 '%s.objdump' % elf_fname.replace('/', '-')) 460 461 def GetErrFile(self, commit_upto, target): 462 """Get the name of the err file for a commit number 463 464 Args: 465 commit_upto: Commit number to use (0..self.count-1) 466 target: Target name 467 """ 468 output_dir = self.GetBuildDir(commit_upto, target) 469 return os.path.join(output_dir, 'err') 470 471 def FilterErrors(self, lines): 472 """Filter out errors in which we have no interest 473 474 We should probably use map(). 475 476 Args: 477 lines: List of error lines, each a string 478 Returns: 479 New list with only interesting lines included 480 """ 481 out_lines = [] 482 for line in lines: 483 if not self.re_make_err.search(line): 484 out_lines.append(line) 485 return out_lines 486 487 def ReadFuncSizes(self, fname, fd): 488 """Read function sizes from the output of 'nm' 489 490 Args: 491 fd: File containing data to read 492 fname: Filename we are reading from (just for errors) 493 494 Returns: 495 Dictionary containing size of each function in bytes, indexed by 496 function name. 497 """ 498 sym = {} 499 for line in fd.readlines(): 500 try: 501 size, type, name = line[:-1].split() 502 except: 503 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 504 continue 505 if type in 'tTdDbB': 506 # function names begin with '.' on 64-bit powerpc 507 if '.' in name[1:]: 508 name = 'static.' + name.split('.')[0] 509 sym[name] = sym.get(name, 0) + int(size, 16) 510 return sym 511 512 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 513 """Work out the outcome of a build. 514 515 Args: 516 commit_upto: Commit number to check (0..n-1) 517 target: Target board to check 518 read_func_sizes: True to read function size information 519 520 Returns: 521 Outcome object 522 """ 523 done_file = self.GetDoneFile(commit_upto, target) 524 sizes_file = self.GetSizesFile(commit_upto, target) 525 sizes = {} 526 func_sizes = {} 527 if os.path.exists(done_file): 528 with open(done_file, 'r') as fd: 529 return_code = int(fd.readline()) 530 err_lines = [] 531 err_file = self.GetErrFile(commit_upto, target) 532 if os.path.exists(err_file): 533 with open(err_file, 'r') as fd: 534 err_lines = self.FilterErrors(fd.readlines()) 535 536 # Decide whether the build was ok, failed or created warnings 537 if return_code: 538 rc = OUTCOME_ERROR 539 elif len(err_lines): 540 rc = OUTCOME_WARNING 541 else: 542 rc = OUTCOME_OK 543 544 # Convert size information to our simple format 545 if os.path.exists(sizes_file): 546 with open(sizes_file, 'r') as fd: 547 for line in fd.readlines(): 548 values = line.split() 549 rodata = 0 550 if len(values) > 6: 551 rodata = int(values[6], 16) 552 size_dict = { 553 'all' : int(values[0]) + int(values[1]) + 554 int(values[2]), 555 'text' : int(values[0]) - rodata, 556 'data' : int(values[1]), 557 'bss' : int(values[2]), 558 'rodata' : rodata, 559 } 560 sizes[values[5]] = size_dict 561 562 if read_func_sizes: 563 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 564 for fname in glob.glob(pattern): 565 with open(fname, 'r') as fd: 566 dict_name = os.path.basename(fname).replace('.sizes', 567 '') 568 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 569 570 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 571 572 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 573 574 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 575 """Calculate a summary of the results of building a commit. 576 577 Args: 578 board_selected: Dict containing boards to summarise 579 commit_upto: Commit number to summarize (0..self.count-1) 580 read_func_sizes: True to read function size information 581 582 Returns: 583 Tuple: 584 Dict containing boards which passed building this commit. 585 keyed by board.target 586 List containing a summary of error lines 587 Dict keyed by error line, containing a list of the Board 588 objects with that error 589 List containing a summary of warning lines 590 Dict keyed by error line, containing a list of the Board 591 objects with that warning 592 """ 593 def AddLine(lines_summary, lines_boards, line, board): 594 line = line.rstrip() 595 if line in lines_boards: 596 lines_boards[line].append(board) 597 else: 598 lines_boards[line] = [board] 599 lines_summary.append(line) 600 601 board_dict = {} 602 err_lines_summary = [] 603 err_lines_boards = {} 604 warn_lines_summary = [] 605 warn_lines_boards = {} 606 607 for board in boards_selected.itervalues(): 608 outcome = self.GetBuildOutcome(commit_upto, board.target, 609 read_func_sizes) 610 board_dict[board.target] = outcome 611 last_func = None 612 last_was_warning = False 613 for line in outcome.err_lines: 614 if line: 615 if (self._re_function.match(line) or 616 self._re_files.match(line)): 617 last_func = line 618 else: 619 is_warning = self._re_warning.match(line) 620 is_note = self._re_note.match(line) 621 if is_warning or (last_was_warning and is_note): 622 if last_func: 623 AddLine(warn_lines_summary, warn_lines_boards, 624 last_func, board) 625 AddLine(warn_lines_summary, warn_lines_boards, 626 line, board) 627 else: 628 if last_func: 629 AddLine(err_lines_summary, err_lines_boards, 630 last_func, board) 631 AddLine(err_lines_summary, err_lines_boards, 632 line, board) 633 last_was_warning = is_warning 634 last_func = None 635 return (board_dict, err_lines_summary, err_lines_boards, 636 warn_lines_summary, warn_lines_boards) 637 638 def AddOutcome(self, board_dict, arch_list, changes, char, color): 639 """Add an output to our list of outcomes for each architecture 640 641 This simple function adds failing boards (changes) to the 642 relevant architecture string, so we can print the results out 643 sorted by architecture. 644 645 Args: 646 board_dict: Dict containing all boards 647 arch_list: Dict keyed by arch name. Value is a string containing 648 a list of board names which failed for that arch. 649 changes: List of boards to add to arch_list 650 color: terminal.Colour object 651 """ 652 done_arch = {} 653 for target in changes: 654 if target in board_dict: 655 arch = board_dict[target].arch 656 else: 657 arch = 'unknown' 658 str = self.col.Color(color, ' ' + target) 659 if not arch in done_arch: 660 str = self.col.Color(color, char) + ' ' + str 661 done_arch[arch] = True 662 if not arch in arch_list: 663 arch_list[arch] = str 664 else: 665 arch_list[arch] += str 666 667 668 def ColourNum(self, num): 669 color = self.col.RED if num > 0 else self.col.GREEN 670 if num == 0: 671 return '0' 672 return self.col.Color(color, str(num)) 673 674 def ResetResultSummary(self, board_selected): 675 """Reset the results summary ready for use. 676 677 Set up the base board list to be all those selected, and set the 678 error lines to empty. 679 680 Following this, calls to PrintResultSummary() will use this 681 information to work out what has changed. 682 683 Args: 684 board_selected: Dict containing boards to summarise, keyed by 685 board.target 686 """ 687 self._base_board_dict = {} 688 for board in board_selected: 689 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 690 self._base_err_lines = [] 691 self._base_warn_lines = [] 692 self._base_err_line_boards = {} 693 self._base_warn_line_boards = {} 694 695 def PrintFuncSizeDetail(self, fname, old, new): 696 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 697 delta, common = [], {} 698 699 for a in old: 700 if a in new: 701 common[a] = 1 702 703 for name in old: 704 if name not in common: 705 remove += 1 706 down += old[name] 707 delta.append([-old[name], name]) 708 709 for name in new: 710 if name not in common: 711 add += 1 712 up += new[name] 713 delta.append([new[name], name]) 714 715 for name in common: 716 diff = new.get(name, 0) - old.get(name, 0) 717 if diff > 0: 718 grow, up = grow + 1, up + diff 719 elif diff < 0: 720 shrink, down = shrink + 1, down - diff 721 delta.append([diff, name]) 722 723 delta.sort() 724 delta.reverse() 725 726 args = [add, -remove, grow, -shrink, up, -down, up - down] 727 if max(args) == 0: 728 return 729 args = [self.ColourNum(x) for x in args] 730 indent = ' ' * 15 731 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 732 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 733 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 734 'delta')) 735 for diff, name in delta: 736 if diff: 737 color = self.col.RED if diff > 0 else self.col.GREEN 738 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 739 old.get(name, '-'), new.get(name,'-'), diff) 740 Print(msg, colour=color) 741 742 743 def PrintSizeDetail(self, target_list, show_bloat): 744 """Show details size information for each board 745 746 Args: 747 target_list: List of targets, each a dict containing: 748 'target': Target name 749 'total_diff': Total difference in bytes across all areas 750 <part_name>: Difference for that part 751 show_bloat: Show detail for each function 752 """ 753 targets_by_diff = sorted(target_list, reverse=True, 754 key=lambda x: x['_total_diff']) 755 for result in targets_by_diff: 756 printed_target = False 757 for name in sorted(result): 758 diff = result[name] 759 if name.startswith('_'): 760 continue 761 if diff != 0: 762 color = self.col.RED if diff > 0 else self.col.GREEN 763 msg = ' %s %+d' % (name, diff) 764 if not printed_target: 765 Print('%10s %-15s:' % ('', result['_target']), 766 newline=False) 767 printed_target = True 768 Print(msg, colour=color, newline=False) 769 if printed_target: 770 Print() 771 if show_bloat: 772 target = result['_target'] 773 outcome = result['_outcome'] 774 base_outcome = self._base_board_dict[target] 775 for fname in outcome.func_sizes: 776 self.PrintFuncSizeDetail(fname, 777 base_outcome.func_sizes[fname], 778 outcome.func_sizes[fname]) 779 780 781 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 782 show_bloat): 783 """Print a summary of image sizes broken down by section. 784 785 The summary takes the form of one line per architecture. The 786 line contains deltas for each of the sections (+ means the section 787 got bigger, - means smaller). The nunmbers are the average number 788 of bytes that a board in this section increased by. 789 790 For example: 791 powerpc: (622 boards) text -0.0 792 arm: (285 boards) text -0.0 793 nds32: (3 boards) text -8.0 794 795 Args: 796 board_selected: Dict containing boards to summarise, keyed by 797 board.target 798 board_dict: Dict containing boards for which we built this 799 commit, keyed by board.target. The value is an Outcome object. 800 show_detail: Show detail for each board 801 show_bloat: Show detail for each function 802 """ 803 arch_list = {} 804 arch_count = {} 805 806 # Calculate changes in size for different image parts 807 # The previous sizes are in Board.sizes, for each board 808 for target in board_dict: 809 if target not in board_selected: 810 continue 811 base_sizes = self._base_board_dict[target].sizes 812 outcome = board_dict[target] 813 sizes = outcome.sizes 814 815 # Loop through the list of images, creating a dict of size 816 # changes for each image/part. We end up with something like 817 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 818 # which means that U-Boot data increased by 5 bytes and SPL 819 # text decreased by 4. 820 err = {'_target' : target} 821 for image in sizes: 822 if image in base_sizes: 823 base_image = base_sizes[image] 824 # Loop through the text, data, bss parts 825 for part in sorted(sizes[image]): 826 diff = sizes[image][part] - base_image[part] 827 col = None 828 if diff: 829 if image == 'u-boot': 830 name = part 831 else: 832 name = image + ':' + part 833 err[name] = diff 834 arch = board_selected[target].arch 835 if not arch in arch_count: 836 arch_count[arch] = 1 837 else: 838 arch_count[arch] += 1 839 if not sizes: 840 pass # Only add to our list when we have some stats 841 elif not arch in arch_list: 842 arch_list[arch] = [err] 843 else: 844 arch_list[arch].append(err) 845 846 # We now have a list of image size changes sorted by arch 847 # Print out a summary of these 848 for arch, target_list in arch_list.iteritems(): 849 # Get total difference for each type 850 totals = {} 851 for result in target_list: 852 total = 0 853 for name, diff in result.iteritems(): 854 if name.startswith('_'): 855 continue 856 total += diff 857 if name in totals: 858 totals[name] += diff 859 else: 860 totals[name] = diff 861 result['_total_diff'] = total 862 result['_outcome'] = board_dict[result['_target']] 863 864 count = len(target_list) 865 printed_arch = False 866 for name in sorted(totals): 867 diff = totals[name] 868 if diff: 869 # Display the average difference in this name for this 870 # architecture 871 avg_diff = float(diff) / count 872 color = self.col.RED if avg_diff > 0 else self.col.GREEN 873 msg = ' %s %+1.1f' % (name, avg_diff) 874 if not printed_arch: 875 Print('%10s: (for %d/%d boards)' % (arch, count, 876 arch_count[arch]), newline=False) 877 printed_arch = True 878 Print(msg, colour=color, newline=False) 879 880 if printed_arch: 881 Print() 882 if show_detail: 883 self.PrintSizeDetail(target_list, show_bloat) 884 885 886 def PrintResultSummary(self, board_selected, board_dict, err_lines, 887 err_line_boards, warn_lines, warn_line_boards, 888 show_sizes, show_detail, show_bloat): 889 """Compare results with the base results and display delta. 890 891 Only boards mentioned in board_selected will be considered. This 892 function is intended to be called repeatedly with the results of 893 each commit. It therefore shows a 'diff' between what it saw in 894 the last call and what it sees now. 895 896 Args: 897 board_selected: Dict containing boards to summarise, keyed by 898 board.target 899 board_dict: Dict containing boards for which we built this 900 commit, keyed by board.target. The value is an Outcome object. 901 err_lines: A list of errors for this commit, or [] if there is 902 none, or we don't want to print errors 903 err_line_boards: Dict keyed by error line, containing a list of 904 the Board objects with that error 905 warn_lines: A list of warnings for this commit, or [] if there is 906 none, or we don't want to print errors 907 warn_line_boards: Dict keyed by warning line, containing a list of 908 the Board objects with that warning 909 show_sizes: Show image size deltas 910 show_detail: Show detail for each board 911 show_bloat: Show detail for each function 912 """ 913 def _BoardList(line, line_boards): 914 """Helper function to get a line of boards containing a line 915 916 Args: 917 line: Error line to search for 918 Return: 919 String containing a list of boards with that error line, or 920 '' if the user has not requested such a list 921 """ 922 if self._list_error_boards: 923 names = [] 924 for board in line_boards[line]: 925 if not board.target in names: 926 names.append(board.target) 927 names_str = '(%s) ' % ','.join(names) 928 else: 929 names_str = '' 930 return names_str 931 932 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 933 char): 934 better_lines = [] 935 worse_lines = [] 936 for line in lines: 937 if line not in base_lines: 938 worse_lines.append(char + '+' + 939 _BoardList(line, line_boards) + line) 940 for line in base_lines: 941 if line not in lines: 942 better_lines.append(char + '-' + 943 _BoardList(line, base_line_boards) + line) 944 return better_lines, worse_lines 945 946 better = [] # List of boards fixed since last commit 947 worse = [] # List of new broken boards since last commit 948 new = [] # List of boards that didn't exist last time 949 unknown = [] # List of boards that were not built 950 951 for target in board_dict: 952 if target not in board_selected: 953 continue 954 955 # If the board was built last time, add its outcome to a list 956 if target in self._base_board_dict: 957 base_outcome = self._base_board_dict[target].rc 958 outcome = board_dict[target] 959 if outcome.rc == OUTCOME_UNKNOWN: 960 unknown.append(target) 961 elif outcome.rc < base_outcome: 962 better.append(target) 963 elif outcome.rc > base_outcome: 964 worse.append(target) 965 else: 966 new.append(target) 967 968 # Get a list of errors that have appeared, and disappeared 969 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 970 self._base_err_line_boards, err_lines, err_line_boards, '') 971 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 972 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 973 974 # Display results by arch 975 if (better or worse or unknown or new or worse_err or better_err 976 or worse_warn or better_warn): 977 arch_list = {} 978 self.AddOutcome(board_selected, arch_list, better, '', 979 self.col.GREEN) 980 self.AddOutcome(board_selected, arch_list, worse, '+', 981 self.col.RED) 982 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 983 if self._show_unknown: 984 self.AddOutcome(board_selected, arch_list, unknown, '?', 985 self.col.MAGENTA) 986 for arch, target_list in arch_list.iteritems(): 987 Print('%10s: %s' % (arch, target_list)) 988 self._error_lines += 1 989 if better_err: 990 Print('\n'.join(better_err), colour=self.col.GREEN) 991 self._error_lines += 1 992 if worse_err: 993 Print('\n'.join(worse_err), colour=self.col.RED) 994 self._error_lines += 1 995 if better_warn: 996 Print('\n'.join(better_warn), colour=self.col.CYAN) 997 self._error_lines += 1 998 if worse_warn: 999 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1000 self._error_lines += 1 1001 1002 if show_sizes: 1003 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1004 show_bloat) 1005 1006 # Save our updated information for the next call to this function 1007 self._base_board_dict = board_dict 1008 self._base_err_lines = err_lines 1009 self._base_warn_lines = warn_lines 1010 self._base_err_line_boards = err_line_boards 1011 self._base_warn_line_boards = warn_line_boards 1012 1013 # Get a list of boards that did not get built, if needed 1014 not_built = [] 1015 for board in board_selected: 1016 if not board in board_dict: 1017 not_built.append(board) 1018 if not_built: 1019 Print("Boards not built (%d): %s" % (len(not_built), 1020 ', '.join(not_built))) 1021 1022 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1023 (board_dict, err_lines, err_line_boards, warn_lines, 1024 warn_line_boards) = self.GetResultSummary( 1025 board_selected, commit_upto, 1026 read_func_sizes=self._show_bloat) 1027 if commits: 1028 msg = '%02d: %s' % (commit_upto + 1, 1029 commits[commit_upto].subject) 1030 Print(msg, colour=self.col.BLUE) 1031 self.PrintResultSummary(board_selected, board_dict, 1032 err_lines if self._show_errors else [], err_line_boards, 1033 warn_lines if self._show_errors else [], warn_line_boards, 1034 self._show_sizes, self._show_detail, self._show_bloat) 1035 1036 def ShowSummary(self, commits, board_selected): 1037 """Show a build summary for U-Boot for a given board list. 1038 1039 Reset the result summary, then repeatedly call GetResultSummary on 1040 each commit's results, then display the differences we see. 1041 1042 Args: 1043 commit: Commit objects to summarise 1044 board_selected: Dict containing boards to summarise 1045 """ 1046 self.commit_count = len(commits) if commits else 1 1047 self.commits = commits 1048 self.ResetResultSummary(board_selected) 1049 self._error_lines = 0 1050 1051 for commit_upto in range(0, self.commit_count, self._step): 1052 self.ProduceResultSummary(commit_upto, commits, board_selected) 1053 if not self._error_lines: 1054 Print('(no errors to report)', colour=self.col.GREEN) 1055 1056 1057 def SetupBuild(self, board_selected, commits): 1058 """Set up ready to start a build. 1059 1060 Args: 1061 board_selected: Selected boards to build 1062 commits: Selected commits to build 1063 """ 1064 # First work out how many commits we will build 1065 count = (self.commit_count + self._step - 1) / self._step 1066 self.count = len(board_selected) * count 1067 self.upto = self.warned = self.fail = 0 1068 self._timestamps = collections.deque() 1069 1070 def GetThreadDir(self, thread_num): 1071 """Get the directory path to the working dir for a thread. 1072 1073 Args: 1074 thread_num: Number of thread to check. 1075 """ 1076 return os.path.join(self._working_dir, '%02d' % thread_num) 1077 1078 def _PrepareThread(self, thread_num, setup_git): 1079 """Prepare the working directory for a thread. 1080 1081 This clones or fetches the repo into the thread's work directory. 1082 1083 Args: 1084 thread_num: Thread number (0, 1, ...) 1085 setup_git: True to set up a git repo clone 1086 """ 1087 thread_dir = self.GetThreadDir(thread_num) 1088 builderthread.Mkdir(thread_dir) 1089 git_dir = os.path.join(thread_dir, '.git') 1090 1091 # Clone the repo if it doesn't already exist 1092 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1093 # we have a private index but uses the origin repo's contents? 1094 if setup_git and self.git_dir: 1095 src_dir = os.path.abspath(self.git_dir) 1096 if os.path.exists(git_dir): 1097 gitutil.Fetch(git_dir, thread_dir) 1098 else: 1099 Print('Cloning repo for thread %d' % thread_num) 1100 gitutil.Clone(src_dir, thread_dir) 1101 1102 def _PrepareWorkingSpace(self, max_threads, setup_git): 1103 """Prepare the working directory for use. 1104 1105 Set up the git repo for each thread. 1106 1107 Args: 1108 max_threads: Maximum number of threads we expect to need. 1109 setup_git: True to set up a git repo clone 1110 """ 1111 builderthread.Mkdir(self._working_dir) 1112 for thread in range(max_threads): 1113 self._PrepareThread(thread, setup_git) 1114 1115 def _PrepareOutputSpace(self): 1116 """Get the output directories ready to receive files. 1117 1118 We delete any output directories which look like ones we need to 1119 create. Having left over directories is confusing when the user wants 1120 to check the output manually. 1121 """ 1122 if not self.commits: 1123 return 1124 dir_list = [] 1125 for commit_upto in range(self.commit_count): 1126 dir_list.append(self._GetOutputDir(commit_upto)) 1127 1128 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1129 if dirname not in dir_list: 1130 shutil.rmtree(dirname) 1131 1132 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1133 """Build all commits for a list of boards 1134 1135 Args: 1136 commits: List of commits to be build, each a Commit object 1137 boards_selected: Dict of selected boards, key is target name, 1138 value is Board object 1139 keep_outputs: True to save build output files 1140 verbose: Display build results as they are completed 1141 Returns: 1142 Tuple containing: 1143 - number of boards that failed to build 1144 - number of boards that issued warnings 1145 """ 1146 self.commit_count = len(commits) if commits else 1 1147 self.commits = commits 1148 self._verbose = verbose 1149 1150 self.ResetResultSummary(board_selected) 1151 builderthread.Mkdir(self.base_dir, parents = True) 1152 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1153 commits is not None) 1154 self._PrepareOutputSpace() 1155 self.SetupBuild(board_selected, commits) 1156 self.ProcessResult(None) 1157 1158 # Create jobs to build all commits for each board 1159 for brd in board_selected.itervalues(): 1160 job = builderthread.BuilderJob() 1161 job.board = brd 1162 job.commits = commits 1163 job.keep_outputs = keep_outputs 1164 job.step = self._step 1165 self.queue.put(job) 1166 1167 # Wait until all jobs are started 1168 self.queue.join() 1169 1170 # Wait until we have processed all output 1171 self.out_queue.join() 1172 Print() 1173 self.ClearLine(0) 1174 return (self.fail, self.warned) 1175