1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100class Builder: 101 """Class for building U-Boot for a particular commit. 102 103 Public members: (many should ->private) 104 active: True if the builder is active and has not been stopped 105 already_done: Number of builds already completed 106 base_dir: Base directory to use for builder 107 checkout: True to check out source, False to skip that step. 108 This is used for testing. 109 col: terminal.Color() object 110 count: Number of commits to build 111 do_make: Method to call to invoke Make 112 fail: Number of builds that failed due to error 113 force_build: Force building even if a build already exists 114 force_config_on_failure: If a commit fails for a board, disable 115 incremental building for the next commit we build for that 116 board, so that we will see all warnings/errors again. 117 force_build_failures: If a previously-built build (i.e. built on 118 a previous run of buildman) is marked as failed, rebuild it. 119 git_dir: Git directory containing source repository 120 last_line_len: Length of the last line we printed (used for erasing 121 it with new progress information) 122 num_jobs: Number of jobs to run at once (passed to make as -j) 123 num_threads: Number of builder threads to run 124 out_queue: Queue of results to process 125 re_make_err: Compiled regular expression for ignore_lines 126 queue: Queue of jobs to run 127 threads: List of active threads 128 toolchains: Toolchains object to use for building 129 upto: Current commit number we are building (0.count-1) 130 warned: Number of builds that produced at least one warning 131 force_reconfig: Reconfigure U-Boot on each comiit. This disables 132 incremental building, where buildman reconfigures on the first 133 commit for a baord, and then just does an incremental build for 134 the following commits. In fact buildman will reconfigure and 135 retry for any failing commits, so generally the only effect of 136 this option is to slow things down. 137 in_tree: Build U-Boot in-tree instead of specifying an output 138 directory separate from the source code. This option is really 139 only useful for testing in-tree builds. 140 141 Private members: 142 _base_board_dict: Last-summarised Dict of boards 143 _base_err_lines: Last-summarised list of errors 144 _base_warn_lines: Last-summarised list of warnings 145 _build_period_us: Time taken for a single build (float object). 146 _complete_delay: Expected delay until completion (timedelta) 147 _next_delay_update: Next time we plan to display a progress update 148 (datatime) 149 _show_unknown: Show unknown boards (those not built) in summary 150 _timestamps: List of timestamps for the completion of the last 151 last _timestamp_count builds. Each is a datetime object. 152 _timestamp_count: Number of timestamps to keep in our list. 153 _working_dir: Base working directory containing all threads 154 """ 155 class Outcome: 156 """Records a build outcome for a single make invocation 157 158 Public Members: 159 rc: Outcome value (OUTCOME_...) 160 err_lines: List of error lines or [] if none 161 sizes: Dictionary of image size information, keyed by filename 162 - Each value is itself a dictionary containing 163 values for 'text', 'data' and 'bss', being the integer 164 size in bytes of each section. 165 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 166 value is itself a dictionary: 167 key: function name 168 value: Size of function in bytes 169 """ 170 def __init__(self, rc, err_lines, sizes, func_sizes): 171 self.rc = rc 172 self.err_lines = err_lines 173 self.sizes = sizes 174 self.func_sizes = func_sizes 175 176 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 177 gnu_make='make', checkout=True, show_unknown=True, step=1): 178 """Create a new Builder object 179 180 Args: 181 toolchains: Toolchains object to use for building 182 base_dir: Base directory to use for builder 183 git_dir: Git directory containing source repository 184 num_threads: Number of builder threads to run 185 num_jobs: Number of jobs to run at once (passed to make as -j) 186 gnu_make: the command name of GNU Make. 187 checkout: True to check out source, False to skip that step. 188 This is used for testing. 189 show_unknown: Show unknown boards (those not built) in summary 190 step: 1 to process every commit, n to process every nth commit 191 """ 192 self.toolchains = toolchains 193 self.base_dir = base_dir 194 self._working_dir = os.path.join(base_dir, '.bm-work') 195 self.threads = [] 196 self.active = True 197 self.do_make = self.Make 198 self.gnu_make = gnu_make 199 self.checkout = checkout 200 self.num_threads = num_threads 201 self.num_jobs = num_jobs 202 self.already_done = 0 203 self.force_build = False 204 self.git_dir = git_dir 205 self._show_unknown = show_unknown 206 self._timestamp_count = 10 207 self._build_period_us = None 208 self._complete_delay = None 209 self._next_delay_update = datetime.now() 210 self.force_config_on_failure = True 211 self.force_build_failures = False 212 self.force_reconfig = False 213 self._step = step 214 self.in_tree = False 215 self._error_lines = 0 216 217 self.col = terminal.Color() 218 219 self._re_function = re.compile('(.*): In function.*') 220 self._re_files = re.compile('In file included from.*') 221 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 222 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 223 224 self.queue = Queue.Queue() 225 self.out_queue = Queue.Queue() 226 for i in range(self.num_threads): 227 t = builderthread.BuilderThread(self, i) 228 t.setDaemon(True) 229 t.start() 230 self.threads.append(t) 231 232 self.last_line_len = 0 233 t = builderthread.ResultThread(self) 234 t.setDaemon(True) 235 t.start() 236 self.threads.append(t) 237 238 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 239 self.re_make_err = re.compile('|'.join(ignore_lines)) 240 241 def __del__(self): 242 """Get rid of all threads created by the builder""" 243 for t in self.threads: 244 del t 245 246 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 247 show_detail=False, show_bloat=False, 248 list_error_boards=False): 249 """Setup display options for the builder. 250 251 show_errors: True to show summarised error/warning info 252 show_sizes: Show size deltas 253 show_detail: Show detail for each board 254 show_bloat: Show detail for each function 255 list_error_boards: Show the boards which caused each error/warning 256 """ 257 self._show_errors = show_errors 258 self._show_sizes = show_sizes 259 self._show_detail = show_detail 260 self._show_bloat = show_bloat 261 self._list_error_boards = list_error_boards 262 263 def _AddTimestamp(self): 264 """Add a new timestamp to the list and record the build period. 265 266 The build period is the length of time taken to perform a single 267 build (one board, one commit). 268 """ 269 now = datetime.now() 270 self._timestamps.append(now) 271 count = len(self._timestamps) 272 delta = self._timestamps[-1] - self._timestamps[0] 273 seconds = delta.total_seconds() 274 275 # If we have enough data, estimate build period (time taken for a 276 # single build) and therefore completion time. 277 if count > 1 and self._next_delay_update < now: 278 self._next_delay_update = now + timedelta(seconds=2) 279 if seconds > 0: 280 self._build_period = float(seconds) / count 281 todo = self.count - self.upto 282 self._complete_delay = timedelta(microseconds= 283 self._build_period * todo * 1000000) 284 # Round it 285 self._complete_delay -= timedelta( 286 microseconds=self._complete_delay.microseconds) 287 288 if seconds > 60: 289 self._timestamps.popleft() 290 count -= 1 291 292 def ClearLine(self, length): 293 """Clear any characters on the current line 294 295 Make way for a new line of length 'length', by outputting enough 296 spaces to clear out the old line. Then remember the new length for 297 next time. 298 299 Args: 300 length: Length of new line, in characters 301 """ 302 if length < self.last_line_len: 303 Print(' ' * (self.last_line_len - length), newline=False) 304 Print('\r', newline=False) 305 self.last_line_len = length 306 sys.stdout.flush() 307 308 def SelectCommit(self, commit, checkout=True): 309 """Checkout the selected commit for this build 310 """ 311 self.commit = commit 312 if checkout and self.checkout: 313 gitutil.Checkout(commit.hash) 314 315 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 316 """Run make 317 318 Args: 319 commit: Commit object that is being built 320 brd: Board object that is being built 321 stage: Stage that we are at (mrproper, config, build) 322 cwd: Directory where make should be run 323 args: Arguments to pass to make 324 kwargs: Arguments to pass to command.RunPipe() 325 """ 326 cmd = [self.gnu_make] + list(args) 327 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 328 cwd=cwd, raise_on_error=False, **kwargs) 329 return result 330 331 def ProcessResult(self, result): 332 """Process the result of a build, showing progress information 333 334 Args: 335 result: A CommandResult object, which indicates the result for 336 a single build 337 """ 338 col = terminal.Color() 339 if result: 340 target = result.brd.target 341 342 if result.return_code < 0: 343 self.active = False 344 command.StopAll() 345 return 346 347 self.upto += 1 348 if result.return_code != 0: 349 self.fail += 1 350 elif result.stderr: 351 self.warned += 1 352 if result.already_done: 353 self.already_done += 1 354 if self._verbose: 355 Print('\r', newline=False) 356 self.ClearLine(0) 357 boards_selected = {target : result.brd} 358 self.ResetResultSummary(boards_selected) 359 self.ProduceResultSummary(result.commit_upto, self.commits, 360 boards_selected) 361 else: 362 target = '(starting)' 363 364 # Display separate counts for ok, warned and fail 365 ok = self.upto - self.warned - self.fail 366 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 367 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 368 line += self.col.Color(self.col.RED, '%5d' % self.fail) 369 370 name = ' /%-5d ' % self.count 371 372 # Add our current completion time estimate 373 self._AddTimestamp() 374 if self._complete_delay: 375 name += '%s : ' % self._complete_delay 376 # When building all boards for a commit, we can print a commit 377 # progress message. 378 if result and result.commit_upto is None: 379 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 380 self.commit_count) 381 382 name += target 383 Print(line + name, newline=False) 384 length = 14 + len(name) 385 self.ClearLine(length) 386 387 def _GetOutputDir(self, commit_upto): 388 """Get the name of the output directory for a commit number 389 390 The output directory is typically .../<branch>/<commit>. 391 392 Args: 393 commit_upto: Commit number to use (0..self.count-1) 394 """ 395 if self.commits: 396 commit = self.commits[commit_upto] 397 subject = commit.subject.translate(trans_valid_chars) 398 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 399 self.commit_count, commit.hash, subject[:20])) 400 else: 401 commit_dir = 'current' 402 output_dir = os.path.join(self.base_dir, commit_dir) 403 return output_dir 404 405 def GetBuildDir(self, commit_upto, target): 406 """Get the name of the build directory for a commit number 407 408 The build directory is typically .../<branch>/<commit>/<target>. 409 410 Args: 411 commit_upto: Commit number to use (0..self.count-1) 412 target: Target name 413 """ 414 output_dir = self._GetOutputDir(commit_upto) 415 return os.path.join(output_dir, target) 416 417 def GetDoneFile(self, commit_upto, target): 418 """Get the name of the done file for a commit number 419 420 Args: 421 commit_upto: Commit number to use (0..self.count-1) 422 target: Target name 423 """ 424 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 425 426 def GetSizesFile(self, commit_upto, target): 427 """Get the name of the sizes file for a commit number 428 429 Args: 430 commit_upto: Commit number to use (0..self.count-1) 431 target: Target name 432 """ 433 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 434 435 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 436 """Get the name of the funcsizes file for a commit number and ELF file 437 438 Args: 439 commit_upto: Commit number to use (0..self.count-1) 440 target: Target name 441 elf_fname: Filename of elf image 442 """ 443 return os.path.join(self.GetBuildDir(commit_upto, target), 444 '%s.sizes' % elf_fname.replace('/', '-')) 445 446 def GetObjdumpFile(self, commit_upto, target, elf_fname): 447 """Get the name of the objdump file for a commit number and ELF file 448 449 Args: 450 commit_upto: Commit number to use (0..self.count-1) 451 target: Target name 452 elf_fname: Filename of elf image 453 """ 454 return os.path.join(self.GetBuildDir(commit_upto, target), 455 '%s.objdump' % elf_fname.replace('/', '-')) 456 457 def GetErrFile(self, commit_upto, target): 458 """Get the name of the err file for a commit number 459 460 Args: 461 commit_upto: Commit number to use (0..self.count-1) 462 target: Target name 463 """ 464 output_dir = self.GetBuildDir(commit_upto, target) 465 return os.path.join(output_dir, 'err') 466 467 def FilterErrors(self, lines): 468 """Filter out errors in which we have no interest 469 470 We should probably use map(). 471 472 Args: 473 lines: List of error lines, each a string 474 Returns: 475 New list with only interesting lines included 476 """ 477 out_lines = [] 478 for line in lines: 479 if not self.re_make_err.search(line): 480 out_lines.append(line) 481 return out_lines 482 483 def ReadFuncSizes(self, fname, fd): 484 """Read function sizes from the output of 'nm' 485 486 Args: 487 fd: File containing data to read 488 fname: Filename we are reading from (just for errors) 489 490 Returns: 491 Dictionary containing size of each function in bytes, indexed by 492 function name. 493 """ 494 sym = {} 495 for line in fd.readlines(): 496 try: 497 size, type, name = line[:-1].split() 498 except: 499 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 500 continue 501 if type in 'tTdDbB': 502 # function names begin with '.' on 64-bit powerpc 503 if '.' in name[1:]: 504 name = 'static.' + name.split('.')[0] 505 sym[name] = sym.get(name, 0) + int(size, 16) 506 return sym 507 508 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 509 """Work out the outcome of a build. 510 511 Args: 512 commit_upto: Commit number to check (0..n-1) 513 target: Target board to check 514 read_func_sizes: True to read function size information 515 516 Returns: 517 Outcome object 518 """ 519 done_file = self.GetDoneFile(commit_upto, target) 520 sizes_file = self.GetSizesFile(commit_upto, target) 521 sizes = {} 522 func_sizes = {} 523 if os.path.exists(done_file): 524 with open(done_file, 'r') as fd: 525 return_code = int(fd.readline()) 526 err_lines = [] 527 err_file = self.GetErrFile(commit_upto, target) 528 if os.path.exists(err_file): 529 with open(err_file, 'r') as fd: 530 err_lines = self.FilterErrors(fd.readlines()) 531 532 # Decide whether the build was ok, failed or created warnings 533 if return_code: 534 rc = OUTCOME_ERROR 535 elif len(err_lines): 536 rc = OUTCOME_WARNING 537 else: 538 rc = OUTCOME_OK 539 540 # Convert size information to our simple format 541 if os.path.exists(sizes_file): 542 with open(sizes_file, 'r') as fd: 543 for line in fd.readlines(): 544 values = line.split() 545 rodata = 0 546 if len(values) > 6: 547 rodata = int(values[6], 16) 548 size_dict = { 549 'all' : int(values[0]) + int(values[1]) + 550 int(values[2]), 551 'text' : int(values[0]) - rodata, 552 'data' : int(values[1]), 553 'bss' : int(values[2]), 554 'rodata' : rodata, 555 } 556 sizes[values[5]] = size_dict 557 558 if read_func_sizes: 559 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 560 for fname in glob.glob(pattern): 561 with open(fname, 'r') as fd: 562 dict_name = os.path.basename(fname).replace('.sizes', 563 '') 564 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 565 566 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 567 568 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 569 570 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 571 """Calculate a summary of the results of building a commit. 572 573 Args: 574 board_selected: Dict containing boards to summarise 575 commit_upto: Commit number to summarize (0..self.count-1) 576 read_func_sizes: True to read function size information 577 578 Returns: 579 Tuple: 580 Dict containing boards which passed building this commit. 581 keyed by board.target 582 List containing a summary of error lines 583 Dict keyed by error line, containing a list of the Board 584 objects with that error 585 List containing a summary of warning lines 586 Dict keyed by error line, containing a list of the Board 587 objects with that warning 588 """ 589 def AddLine(lines_summary, lines_boards, line, board): 590 line = line.rstrip() 591 if line in lines_boards: 592 lines_boards[line].append(board) 593 else: 594 lines_boards[line] = [board] 595 lines_summary.append(line) 596 597 board_dict = {} 598 err_lines_summary = [] 599 err_lines_boards = {} 600 warn_lines_summary = [] 601 warn_lines_boards = {} 602 603 for board in boards_selected.itervalues(): 604 outcome = self.GetBuildOutcome(commit_upto, board.target, 605 read_func_sizes) 606 board_dict[board.target] = outcome 607 last_func = None 608 last_was_warning = False 609 for line in outcome.err_lines: 610 if line: 611 if (self._re_function.match(line) or 612 self._re_files.match(line)): 613 last_func = line 614 else: 615 is_warning = self._re_warning.match(line) 616 is_note = self._re_note.match(line) 617 if is_warning or (last_was_warning and is_note): 618 if last_func: 619 AddLine(warn_lines_summary, warn_lines_boards, 620 last_func, board) 621 AddLine(warn_lines_summary, warn_lines_boards, 622 line, board) 623 else: 624 if last_func: 625 AddLine(err_lines_summary, err_lines_boards, 626 last_func, board) 627 AddLine(err_lines_summary, err_lines_boards, 628 line, board) 629 last_was_warning = is_warning 630 last_func = None 631 return (board_dict, err_lines_summary, err_lines_boards, 632 warn_lines_summary, warn_lines_boards) 633 634 def AddOutcome(self, board_dict, arch_list, changes, char, color): 635 """Add an output to our list of outcomes for each architecture 636 637 This simple function adds failing boards (changes) to the 638 relevant architecture string, so we can print the results out 639 sorted by architecture. 640 641 Args: 642 board_dict: Dict containing all boards 643 arch_list: Dict keyed by arch name. Value is a string containing 644 a list of board names which failed for that arch. 645 changes: List of boards to add to arch_list 646 color: terminal.Colour object 647 """ 648 done_arch = {} 649 for target in changes: 650 if target in board_dict: 651 arch = board_dict[target].arch 652 else: 653 arch = 'unknown' 654 str = self.col.Color(color, ' ' + target) 655 if not arch in done_arch: 656 str = self.col.Color(color, char) + ' ' + str 657 done_arch[arch] = True 658 if not arch in arch_list: 659 arch_list[arch] = str 660 else: 661 arch_list[arch] += str 662 663 664 def ColourNum(self, num): 665 color = self.col.RED if num > 0 else self.col.GREEN 666 if num == 0: 667 return '0' 668 return self.col.Color(color, str(num)) 669 670 def ResetResultSummary(self, board_selected): 671 """Reset the results summary ready for use. 672 673 Set up the base board list to be all those selected, and set the 674 error lines to empty. 675 676 Following this, calls to PrintResultSummary() will use this 677 information to work out what has changed. 678 679 Args: 680 board_selected: Dict containing boards to summarise, keyed by 681 board.target 682 """ 683 self._base_board_dict = {} 684 for board in board_selected: 685 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 686 self._base_err_lines = [] 687 self._base_warn_lines = [] 688 self._base_err_line_boards = {} 689 self._base_warn_line_boards = {} 690 691 def PrintFuncSizeDetail(self, fname, old, new): 692 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 693 delta, common = [], {} 694 695 for a in old: 696 if a in new: 697 common[a] = 1 698 699 for name in old: 700 if name not in common: 701 remove += 1 702 down += old[name] 703 delta.append([-old[name], name]) 704 705 for name in new: 706 if name not in common: 707 add += 1 708 up += new[name] 709 delta.append([new[name], name]) 710 711 for name in common: 712 diff = new.get(name, 0) - old.get(name, 0) 713 if diff > 0: 714 grow, up = grow + 1, up + diff 715 elif diff < 0: 716 shrink, down = shrink + 1, down - diff 717 delta.append([diff, name]) 718 719 delta.sort() 720 delta.reverse() 721 722 args = [add, -remove, grow, -shrink, up, -down, up - down] 723 if max(args) == 0: 724 return 725 args = [self.ColourNum(x) for x in args] 726 indent = ' ' * 15 727 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 728 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 729 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 730 'delta')) 731 for diff, name in delta: 732 if diff: 733 color = self.col.RED if diff > 0 else self.col.GREEN 734 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 735 old.get(name, '-'), new.get(name,'-'), diff) 736 Print(msg, colour=color) 737 738 739 def PrintSizeDetail(self, target_list, show_bloat): 740 """Show details size information for each board 741 742 Args: 743 target_list: List of targets, each a dict containing: 744 'target': Target name 745 'total_diff': Total difference in bytes across all areas 746 <part_name>: Difference for that part 747 show_bloat: Show detail for each function 748 """ 749 targets_by_diff = sorted(target_list, reverse=True, 750 key=lambda x: x['_total_diff']) 751 for result in targets_by_diff: 752 printed_target = False 753 for name in sorted(result): 754 diff = result[name] 755 if name.startswith('_'): 756 continue 757 if diff != 0: 758 color = self.col.RED if diff > 0 else self.col.GREEN 759 msg = ' %s %+d' % (name, diff) 760 if not printed_target: 761 Print('%10s %-15s:' % ('', result['_target']), 762 newline=False) 763 printed_target = True 764 Print(msg, colour=color, newline=False) 765 if printed_target: 766 Print() 767 if show_bloat: 768 target = result['_target'] 769 outcome = result['_outcome'] 770 base_outcome = self._base_board_dict[target] 771 for fname in outcome.func_sizes: 772 self.PrintFuncSizeDetail(fname, 773 base_outcome.func_sizes[fname], 774 outcome.func_sizes[fname]) 775 776 777 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 778 show_bloat): 779 """Print a summary of image sizes broken down by section. 780 781 The summary takes the form of one line per architecture. The 782 line contains deltas for each of the sections (+ means the section 783 got bigger, - means smaller). The nunmbers are the average number 784 of bytes that a board in this section increased by. 785 786 For example: 787 powerpc: (622 boards) text -0.0 788 arm: (285 boards) text -0.0 789 nds32: (3 boards) text -8.0 790 791 Args: 792 board_selected: Dict containing boards to summarise, keyed by 793 board.target 794 board_dict: Dict containing boards for which we built this 795 commit, keyed by board.target. The value is an Outcome object. 796 show_detail: Show detail for each board 797 show_bloat: Show detail for each function 798 """ 799 arch_list = {} 800 arch_count = {} 801 802 # Calculate changes in size for different image parts 803 # The previous sizes are in Board.sizes, for each board 804 for target in board_dict: 805 if target not in board_selected: 806 continue 807 base_sizes = self._base_board_dict[target].sizes 808 outcome = board_dict[target] 809 sizes = outcome.sizes 810 811 # Loop through the list of images, creating a dict of size 812 # changes for each image/part. We end up with something like 813 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 814 # which means that U-Boot data increased by 5 bytes and SPL 815 # text decreased by 4. 816 err = {'_target' : target} 817 for image in sizes: 818 if image in base_sizes: 819 base_image = base_sizes[image] 820 # Loop through the text, data, bss parts 821 for part in sorted(sizes[image]): 822 diff = sizes[image][part] - base_image[part] 823 col = None 824 if diff: 825 if image == 'u-boot': 826 name = part 827 else: 828 name = image + ':' + part 829 err[name] = diff 830 arch = board_selected[target].arch 831 if not arch in arch_count: 832 arch_count[arch] = 1 833 else: 834 arch_count[arch] += 1 835 if not sizes: 836 pass # Only add to our list when we have some stats 837 elif not arch in arch_list: 838 arch_list[arch] = [err] 839 else: 840 arch_list[arch].append(err) 841 842 # We now have a list of image size changes sorted by arch 843 # Print out a summary of these 844 for arch, target_list in arch_list.iteritems(): 845 # Get total difference for each type 846 totals = {} 847 for result in target_list: 848 total = 0 849 for name, diff in result.iteritems(): 850 if name.startswith('_'): 851 continue 852 total += diff 853 if name in totals: 854 totals[name] += diff 855 else: 856 totals[name] = diff 857 result['_total_diff'] = total 858 result['_outcome'] = board_dict[result['_target']] 859 860 count = len(target_list) 861 printed_arch = False 862 for name in sorted(totals): 863 diff = totals[name] 864 if diff: 865 # Display the average difference in this name for this 866 # architecture 867 avg_diff = float(diff) / count 868 color = self.col.RED if avg_diff > 0 else self.col.GREEN 869 msg = ' %s %+1.1f' % (name, avg_diff) 870 if not printed_arch: 871 Print('%10s: (for %d/%d boards)' % (arch, count, 872 arch_count[arch]), newline=False) 873 printed_arch = True 874 Print(msg, colour=color, newline=False) 875 876 if printed_arch: 877 Print() 878 if show_detail: 879 self.PrintSizeDetail(target_list, show_bloat) 880 881 882 def PrintResultSummary(self, board_selected, board_dict, err_lines, 883 err_line_boards, warn_lines, warn_line_boards, 884 show_sizes, show_detail, show_bloat): 885 """Compare results with the base results and display delta. 886 887 Only boards mentioned in board_selected will be considered. This 888 function is intended to be called repeatedly with the results of 889 each commit. It therefore shows a 'diff' between what it saw in 890 the last call and what it sees now. 891 892 Args: 893 board_selected: Dict containing boards to summarise, keyed by 894 board.target 895 board_dict: Dict containing boards for which we built this 896 commit, keyed by board.target. The value is an Outcome object. 897 err_lines: A list of errors for this commit, or [] if there is 898 none, or we don't want to print errors 899 err_line_boards: Dict keyed by error line, containing a list of 900 the Board objects with that error 901 warn_lines: A list of warnings for this commit, or [] if there is 902 none, or we don't want to print errors 903 warn_line_boards: Dict keyed by warning line, containing a list of 904 the Board objects with that warning 905 show_sizes: Show image size deltas 906 show_detail: Show detail for each board 907 show_bloat: Show detail for each function 908 """ 909 def _BoardList(line, line_boards): 910 """Helper function to get a line of boards containing a line 911 912 Args: 913 line: Error line to search for 914 Return: 915 String containing a list of boards with that error line, or 916 '' if the user has not requested such a list 917 """ 918 if self._list_error_boards: 919 names = [] 920 for board in line_boards[line]: 921 names.append(board.target) 922 names_str = '(%s) ' % ','.join(names) 923 else: 924 names_str = '' 925 return names_str 926 927 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 928 char): 929 better_lines = [] 930 worse_lines = [] 931 for line in lines: 932 if line not in base_lines: 933 worse_lines.append(char + '+' + 934 _BoardList(line, line_boards) + line) 935 for line in base_lines: 936 if line not in lines: 937 better_lines.append(char + '-' + 938 _BoardList(line, base_line_boards) + line) 939 return better_lines, worse_lines 940 941 better = [] # List of boards fixed since last commit 942 worse = [] # List of new broken boards since last commit 943 new = [] # List of boards that didn't exist last time 944 unknown = [] # List of boards that were not built 945 946 for target in board_dict: 947 if target not in board_selected: 948 continue 949 950 # If the board was built last time, add its outcome to a list 951 if target in self._base_board_dict: 952 base_outcome = self._base_board_dict[target].rc 953 outcome = board_dict[target] 954 if outcome.rc == OUTCOME_UNKNOWN: 955 unknown.append(target) 956 elif outcome.rc < base_outcome: 957 better.append(target) 958 elif outcome.rc > base_outcome: 959 worse.append(target) 960 else: 961 new.append(target) 962 963 # Get a list of errors that have appeared, and disappeared 964 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 965 self._base_err_line_boards, err_lines, err_line_boards, '') 966 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 967 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 968 969 # Display results by arch 970 if (better or worse or unknown or new or worse_err or better_err 971 or worse_warn or better_warn): 972 arch_list = {} 973 self.AddOutcome(board_selected, arch_list, better, '', 974 self.col.GREEN) 975 self.AddOutcome(board_selected, arch_list, worse, '+', 976 self.col.RED) 977 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 978 if self._show_unknown: 979 self.AddOutcome(board_selected, arch_list, unknown, '?', 980 self.col.MAGENTA) 981 for arch, target_list in arch_list.iteritems(): 982 Print('%10s: %s' % (arch, target_list)) 983 self._error_lines += 1 984 if better_err: 985 Print('\n'.join(better_err), colour=self.col.GREEN) 986 self._error_lines += 1 987 if worse_err: 988 Print('\n'.join(worse_err), colour=self.col.RED) 989 self._error_lines += 1 990 if better_warn: 991 Print('\n'.join(better_warn), colour=self.col.CYAN) 992 self._error_lines += 1 993 if worse_warn: 994 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 995 self._error_lines += 1 996 997 if show_sizes: 998 self.PrintSizeSummary(board_selected, board_dict, show_detail, 999 show_bloat) 1000 1001 # Save our updated information for the next call to this function 1002 self._base_board_dict = board_dict 1003 self._base_err_lines = err_lines 1004 self._base_warn_lines = warn_lines 1005 self._base_err_line_boards = err_line_boards 1006 self._base_warn_line_boards = warn_line_boards 1007 1008 # Get a list of boards that did not get built, if needed 1009 not_built = [] 1010 for board in board_selected: 1011 if not board in board_dict: 1012 not_built.append(board) 1013 if not_built: 1014 Print("Boards not built (%d): %s" % (len(not_built), 1015 ', '.join(not_built))) 1016 1017 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1018 (board_dict, err_lines, err_line_boards, warn_lines, 1019 warn_line_boards) = self.GetResultSummary( 1020 board_selected, commit_upto, 1021 read_func_sizes=self._show_bloat) 1022 if commits: 1023 msg = '%02d: %s' % (commit_upto + 1, 1024 commits[commit_upto].subject) 1025 Print(msg, colour=self.col.BLUE) 1026 self.PrintResultSummary(board_selected, board_dict, 1027 err_lines if self._show_errors else [], err_line_boards, 1028 warn_lines if self._show_errors else [], warn_line_boards, 1029 self._show_sizes, self._show_detail, self._show_bloat) 1030 1031 def ShowSummary(self, commits, board_selected): 1032 """Show a build summary for U-Boot for a given board list. 1033 1034 Reset the result summary, then repeatedly call GetResultSummary on 1035 each commit's results, then display the differences we see. 1036 1037 Args: 1038 commit: Commit objects to summarise 1039 board_selected: Dict containing boards to summarise 1040 """ 1041 self.commit_count = len(commits) if commits else 1 1042 self.commits = commits 1043 self.ResetResultSummary(board_selected) 1044 self._error_lines = 0 1045 1046 for commit_upto in range(0, self.commit_count, self._step): 1047 self.ProduceResultSummary(commit_upto, commits, board_selected) 1048 if not self._error_lines: 1049 Print('(no errors to report)', colour=self.col.GREEN) 1050 1051 1052 def SetupBuild(self, board_selected, commits): 1053 """Set up ready to start a build. 1054 1055 Args: 1056 board_selected: Selected boards to build 1057 commits: Selected commits to build 1058 """ 1059 # First work out how many commits we will build 1060 count = (self.commit_count + self._step - 1) / self._step 1061 self.count = len(board_selected) * count 1062 self.upto = self.warned = self.fail = 0 1063 self._timestamps = collections.deque() 1064 1065 def GetThreadDir(self, thread_num): 1066 """Get the directory path to the working dir for a thread. 1067 1068 Args: 1069 thread_num: Number of thread to check. 1070 """ 1071 return os.path.join(self._working_dir, '%02d' % thread_num) 1072 1073 def _PrepareThread(self, thread_num, setup_git): 1074 """Prepare the working directory for a thread. 1075 1076 This clones or fetches the repo into the thread's work directory. 1077 1078 Args: 1079 thread_num: Thread number (0, 1, ...) 1080 setup_git: True to set up a git repo clone 1081 """ 1082 thread_dir = self.GetThreadDir(thread_num) 1083 builderthread.Mkdir(thread_dir) 1084 git_dir = os.path.join(thread_dir, '.git') 1085 1086 # Clone the repo if it doesn't already exist 1087 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1088 # we have a private index but uses the origin repo's contents? 1089 if setup_git and self.git_dir: 1090 src_dir = os.path.abspath(self.git_dir) 1091 if os.path.exists(git_dir): 1092 gitutil.Fetch(git_dir, thread_dir) 1093 else: 1094 Print('Cloning repo for thread %d' % thread_num) 1095 gitutil.Clone(src_dir, thread_dir) 1096 1097 def _PrepareWorkingSpace(self, max_threads, setup_git): 1098 """Prepare the working directory for use. 1099 1100 Set up the git repo for each thread. 1101 1102 Args: 1103 max_threads: Maximum number of threads we expect to need. 1104 setup_git: True to set up a git repo clone 1105 """ 1106 builderthread.Mkdir(self._working_dir) 1107 for thread in range(max_threads): 1108 self._PrepareThread(thread, setup_git) 1109 1110 def _PrepareOutputSpace(self): 1111 """Get the output directories ready to receive files. 1112 1113 We delete any output directories which look like ones we need to 1114 create. Having left over directories is confusing when the user wants 1115 to check the output manually. 1116 """ 1117 dir_list = [] 1118 for commit_upto in range(self.commit_count): 1119 dir_list.append(self._GetOutputDir(commit_upto)) 1120 1121 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1122 if dirname not in dir_list: 1123 shutil.rmtree(dirname) 1124 1125 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1126 """Build all commits for a list of boards 1127 1128 Args: 1129 commits: List of commits to be build, each a Commit object 1130 boards_selected: Dict of selected boards, key is target name, 1131 value is Board object 1132 keep_outputs: True to save build output files 1133 verbose: Display build results as they are completed 1134 Returns: 1135 Tuple containing: 1136 - number of boards that failed to build 1137 - number of boards that issued warnings 1138 """ 1139 self.commit_count = len(commits) if commits else 1 1140 self.commits = commits 1141 self._verbose = verbose 1142 1143 self.ResetResultSummary(board_selected) 1144 builderthread.Mkdir(self.base_dir, parents = True) 1145 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1146 commits is not None) 1147 self._PrepareOutputSpace() 1148 self.SetupBuild(board_selected, commits) 1149 self.ProcessResult(None) 1150 1151 # Create jobs to build all commits for each board 1152 for brd in board_selected.itervalues(): 1153 job = builderthread.BuilderJob() 1154 job.board = brd 1155 job.commits = commits 1156 job.keep_outputs = keep_outputs 1157 job.step = self._step 1158 self.queue.put(job) 1159 1160 # Wait until all jobs are started 1161 self.queue.join() 1162 1163 # Wait until we have processed all output 1164 self.out_queue.join() 1165 Print() 1166 self.ClearLine(0) 1167 return (self.fail, self.warned) 1168