1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100class Builder: 101 """Class for building U-Boot for a particular commit. 102 103 Public members: (many should ->private) 104 active: True if the builder is active and has not been stopped 105 already_done: Number of builds already completed 106 base_dir: Base directory to use for builder 107 checkout: True to check out source, False to skip that step. 108 This is used for testing. 109 col: terminal.Color() object 110 count: Number of commits to build 111 do_make: Method to call to invoke Make 112 fail: Number of builds that failed due to error 113 force_build: Force building even if a build already exists 114 force_config_on_failure: If a commit fails for a board, disable 115 incremental building for the next commit we build for that 116 board, so that we will see all warnings/errors again. 117 force_build_failures: If a previously-built build (i.e. built on 118 a previous run of buildman) is marked as failed, rebuild it. 119 git_dir: Git directory containing source repository 120 last_line_len: Length of the last line we printed (used for erasing 121 it with new progress information) 122 num_jobs: Number of jobs to run at once (passed to make as -j) 123 num_threads: Number of builder threads to run 124 out_queue: Queue of results to process 125 re_make_err: Compiled regular expression for ignore_lines 126 queue: Queue of jobs to run 127 threads: List of active threads 128 toolchains: Toolchains object to use for building 129 upto: Current commit number we are building (0.count-1) 130 warned: Number of builds that produced at least one warning 131 force_reconfig: Reconfigure U-Boot on each comiit. This disables 132 incremental building, where buildman reconfigures on the first 133 commit for a baord, and then just does an incremental build for 134 the following commits. In fact buildman will reconfigure and 135 retry for any failing commits, so generally the only effect of 136 this option is to slow things down. 137 in_tree: Build U-Boot in-tree instead of specifying an output 138 directory separate from the source code. This option is really 139 only useful for testing in-tree builds. 140 141 Private members: 142 _base_board_dict: Last-summarised Dict of boards 143 _base_err_lines: Last-summarised list of errors 144 _base_warn_lines: Last-summarised list of warnings 145 _build_period_us: Time taken for a single build (float object). 146 _complete_delay: Expected delay until completion (timedelta) 147 _next_delay_update: Next time we plan to display a progress update 148 (datatime) 149 _show_unknown: Show unknown boards (those not built) in summary 150 _timestamps: List of timestamps for the completion of the last 151 last _timestamp_count builds. Each is a datetime object. 152 _timestamp_count: Number of timestamps to keep in our list. 153 _working_dir: Base working directory containing all threads 154 """ 155 class Outcome: 156 """Records a build outcome for a single make invocation 157 158 Public Members: 159 rc: Outcome value (OUTCOME_...) 160 err_lines: List of error lines or [] if none 161 sizes: Dictionary of image size information, keyed by filename 162 - Each value is itself a dictionary containing 163 values for 'text', 'data' and 'bss', being the integer 164 size in bytes of each section. 165 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 166 value is itself a dictionary: 167 key: function name 168 value: Size of function in bytes 169 """ 170 def __init__(self, rc, err_lines, sizes, func_sizes): 171 self.rc = rc 172 self.err_lines = err_lines 173 self.sizes = sizes 174 self.func_sizes = func_sizes 175 176 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 177 gnu_make='make', checkout=True, show_unknown=True, step=1, 178 no_subdirs=False, full_path=False, verbose_build=False): 179 """Create a new Builder object 180 181 Args: 182 toolchains: Toolchains object to use for building 183 base_dir: Base directory to use for builder 184 git_dir: Git directory containing source repository 185 num_threads: Number of builder threads to run 186 num_jobs: Number of jobs to run at once (passed to make as -j) 187 gnu_make: the command name of GNU Make. 188 checkout: True to check out source, False to skip that step. 189 This is used for testing. 190 show_unknown: Show unknown boards (those not built) in summary 191 step: 1 to process every commit, n to process every nth commit 192 no_subdirs: Don't create subdirectories when building current 193 source for a single board 194 full_path: Return the full path in CROSS_COMPILE and don't set 195 PATH 196 verbose_build: Run build with V=1 and don't use 'make -s' 197 """ 198 self.toolchains = toolchains 199 self.base_dir = base_dir 200 self._working_dir = os.path.join(base_dir, '.bm-work') 201 self.threads = [] 202 self.active = True 203 self.do_make = self.Make 204 self.gnu_make = gnu_make 205 self.checkout = checkout 206 self.num_threads = num_threads 207 self.num_jobs = num_jobs 208 self.already_done = 0 209 self.force_build = False 210 self.git_dir = git_dir 211 self._show_unknown = show_unknown 212 self._timestamp_count = 10 213 self._build_period_us = None 214 self._complete_delay = None 215 self._next_delay_update = datetime.now() 216 self.force_config_on_failure = True 217 self.force_build_failures = False 218 self.force_reconfig = False 219 self._step = step 220 self.in_tree = False 221 self._error_lines = 0 222 self.no_subdirs = no_subdirs 223 self.full_path = full_path 224 self.verbose_build = verbose_build 225 226 self.col = terminal.Color() 227 228 self._re_function = re.compile('(.*): In function.*') 229 self._re_files = re.compile('In file included from.*') 230 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 231 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 232 233 self.queue = Queue.Queue() 234 self.out_queue = Queue.Queue() 235 for i in range(self.num_threads): 236 t = builderthread.BuilderThread(self, i) 237 t.setDaemon(True) 238 t.start() 239 self.threads.append(t) 240 241 self.last_line_len = 0 242 t = builderthread.ResultThread(self) 243 t.setDaemon(True) 244 t.start() 245 self.threads.append(t) 246 247 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 248 self.re_make_err = re.compile('|'.join(ignore_lines)) 249 250 def __del__(self): 251 """Get rid of all threads created by the builder""" 252 for t in self.threads: 253 del t 254 255 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 256 show_detail=False, show_bloat=False, 257 list_error_boards=False): 258 """Setup display options for the builder. 259 260 show_errors: True to show summarised error/warning info 261 show_sizes: Show size deltas 262 show_detail: Show detail for each board 263 show_bloat: Show detail for each function 264 list_error_boards: Show the boards which caused each error/warning 265 """ 266 self._show_errors = show_errors 267 self._show_sizes = show_sizes 268 self._show_detail = show_detail 269 self._show_bloat = show_bloat 270 self._list_error_boards = list_error_boards 271 272 def _AddTimestamp(self): 273 """Add a new timestamp to the list and record the build period. 274 275 The build period is the length of time taken to perform a single 276 build (one board, one commit). 277 """ 278 now = datetime.now() 279 self._timestamps.append(now) 280 count = len(self._timestamps) 281 delta = self._timestamps[-1] - self._timestamps[0] 282 seconds = delta.total_seconds() 283 284 # If we have enough data, estimate build period (time taken for a 285 # single build) and therefore completion time. 286 if count > 1 and self._next_delay_update < now: 287 self._next_delay_update = now + timedelta(seconds=2) 288 if seconds > 0: 289 self._build_period = float(seconds) / count 290 todo = self.count - self.upto 291 self._complete_delay = timedelta(microseconds= 292 self._build_period * todo * 1000000) 293 # Round it 294 self._complete_delay -= timedelta( 295 microseconds=self._complete_delay.microseconds) 296 297 if seconds > 60: 298 self._timestamps.popleft() 299 count -= 1 300 301 def ClearLine(self, length): 302 """Clear any characters on the current line 303 304 Make way for a new line of length 'length', by outputting enough 305 spaces to clear out the old line. Then remember the new length for 306 next time. 307 308 Args: 309 length: Length of new line, in characters 310 """ 311 if length < self.last_line_len: 312 Print(' ' * (self.last_line_len - length), newline=False) 313 Print('\r', newline=False) 314 self.last_line_len = length 315 sys.stdout.flush() 316 317 def SelectCommit(self, commit, checkout=True): 318 """Checkout the selected commit for this build 319 """ 320 self.commit = commit 321 if checkout and self.checkout: 322 gitutil.Checkout(commit.hash) 323 324 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 325 """Run make 326 327 Args: 328 commit: Commit object that is being built 329 brd: Board object that is being built 330 stage: Stage that we are at (mrproper, config, build) 331 cwd: Directory where make should be run 332 args: Arguments to pass to make 333 kwargs: Arguments to pass to command.RunPipe() 334 """ 335 cmd = [self.gnu_make] + list(args) 336 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 337 cwd=cwd, raise_on_error=False, **kwargs) 338 return result 339 340 def ProcessResult(self, result): 341 """Process the result of a build, showing progress information 342 343 Args: 344 result: A CommandResult object, which indicates the result for 345 a single build 346 """ 347 col = terminal.Color() 348 if result: 349 target = result.brd.target 350 351 if result.return_code < 0: 352 self.active = False 353 command.StopAll() 354 return 355 356 self.upto += 1 357 if result.return_code != 0: 358 self.fail += 1 359 elif result.stderr: 360 self.warned += 1 361 if result.already_done: 362 self.already_done += 1 363 if self._verbose: 364 Print('\r', newline=False) 365 self.ClearLine(0) 366 boards_selected = {target : result.brd} 367 self.ResetResultSummary(boards_selected) 368 self.ProduceResultSummary(result.commit_upto, self.commits, 369 boards_selected) 370 else: 371 target = '(starting)' 372 373 # Display separate counts for ok, warned and fail 374 ok = self.upto - self.warned - self.fail 375 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 376 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 377 line += self.col.Color(self.col.RED, '%5d' % self.fail) 378 379 name = ' /%-5d ' % self.count 380 381 # Add our current completion time estimate 382 self._AddTimestamp() 383 if self._complete_delay: 384 name += '%s : ' % self._complete_delay 385 # When building all boards for a commit, we can print a commit 386 # progress message. 387 if result and result.commit_upto is None: 388 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 389 self.commit_count) 390 391 name += target 392 Print(line + name, newline=False) 393 length = 14 + len(name) 394 self.ClearLine(length) 395 396 def _GetOutputDir(self, commit_upto): 397 """Get the name of the output directory for a commit number 398 399 The output directory is typically .../<branch>/<commit>. 400 401 Args: 402 commit_upto: Commit number to use (0..self.count-1) 403 """ 404 commit_dir = None 405 if self.commits: 406 commit = self.commits[commit_upto] 407 subject = commit.subject.translate(trans_valid_chars) 408 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 409 self.commit_count, commit.hash, subject[:20])) 410 elif not self.no_subdirs: 411 commit_dir = 'current' 412 if not commit_dir: 413 return self.base_dir 414 return os.path.join(self.base_dir, commit_dir) 415 416 def GetBuildDir(self, commit_upto, target): 417 """Get the name of the build directory for a commit number 418 419 The build directory is typically .../<branch>/<commit>/<target>. 420 421 Args: 422 commit_upto: Commit number to use (0..self.count-1) 423 target: Target name 424 """ 425 output_dir = self._GetOutputDir(commit_upto) 426 return os.path.join(output_dir, target) 427 428 def GetDoneFile(self, commit_upto, target): 429 """Get the name of the done file for a commit number 430 431 Args: 432 commit_upto: Commit number to use (0..self.count-1) 433 target: Target name 434 """ 435 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 436 437 def GetSizesFile(self, commit_upto, target): 438 """Get the name of the sizes file for a commit number 439 440 Args: 441 commit_upto: Commit number to use (0..self.count-1) 442 target: Target name 443 """ 444 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 445 446 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 447 """Get the name of the funcsizes file for a commit number and ELF file 448 449 Args: 450 commit_upto: Commit number to use (0..self.count-1) 451 target: Target name 452 elf_fname: Filename of elf image 453 """ 454 return os.path.join(self.GetBuildDir(commit_upto, target), 455 '%s.sizes' % elf_fname.replace('/', '-')) 456 457 def GetObjdumpFile(self, commit_upto, target, elf_fname): 458 """Get the name of the objdump file for a commit number and ELF file 459 460 Args: 461 commit_upto: Commit number to use (0..self.count-1) 462 target: Target name 463 elf_fname: Filename of elf image 464 """ 465 return os.path.join(self.GetBuildDir(commit_upto, target), 466 '%s.objdump' % elf_fname.replace('/', '-')) 467 468 def GetErrFile(self, commit_upto, target): 469 """Get the name of the err file for a commit number 470 471 Args: 472 commit_upto: Commit number to use (0..self.count-1) 473 target: Target name 474 """ 475 output_dir = self.GetBuildDir(commit_upto, target) 476 return os.path.join(output_dir, 'err') 477 478 def FilterErrors(self, lines): 479 """Filter out errors in which we have no interest 480 481 We should probably use map(). 482 483 Args: 484 lines: List of error lines, each a string 485 Returns: 486 New list with only interesting lines included 487 """ 488 out_lines = [] 489 for line in lines: 490 if not self.re_make_err.search(line): 491 out_lines.append(line) 492 return out_lines 493 494 def ReadFuncSizes(self, fname, fd): 495 """Read function sizes from the output of 'nm' 496 497 Args: 498 fd: File containing data to read 499 fname: Filename we are reading from (just for errors) 500 501 Returns: 502 Dictionary containing size of each function in bytes, indexed by 503 function name. 504 """ 505 sym = {} 506 for line in fd.readlines(): 507 try: 508 size, type, name = line[:-1].split() 509 except: 510 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 511 continue 512 if type in 'tTdDbB': 513 # function names begin with '.' on 64-bit powerpc 514 if '.' in name[1:]: 515 name = 'static.' + name.split('.')[0] 516 sym[name] = sym.get(name, 0) + int(size, 16) 517 return sym 518 519 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 520 """Work out the outcome of a build. 521 522 Args: 523 commit_upto: Commit number to check (0..n-1) 524 target: Target board to check 525 read_func_sizes: True to read function size information 526 527 Returns: 528 Outcome object 529 """ 530 done_file = self.GetDoneFile(commit_upto, target) 531 sizes_file = self.GetSizesFile(commit_upto, target) 532 sizes = {} 533 func_sizes = {} 534 if os.path.exists(done_file): 535 with open(done_file, 'r') as fd: 536 return_code = int(fd.readline()) 537 err_lines = [] 538 err_file = self.GetErrFile(commit_upto, target) 539 if os.path.exists(err_file): 540 with open(err_file, 'r') as fd: 541 err_lines = self.FilterErrors(fd.readlines()) 542 543 # Decide whether the build was ok, failed or created warnings 544 if return_code: 545 rc = OUTCOME_ERROR 546 elif len(err_lines): 547 rc = OUTCOME_WARNING 548 else: 549 rc = OUTCOME_OK 550 551 # Convert size information to our simple format 552 if os.path.exists(sizes_file): 553 with open(sizes_file, 'r') as fd: 554 for line in fd.readlines(): 555 values = line.split() 556 rodata = 0 557 if len(values) > 6: 558 rodata = int(values[6], 16) 559 size_dict = { 560 'all' : int(values[0]) + int(values[1]) + 561 int(values[2]), 562 'text' : int(values[0]) - rodata, 563 'data' : int(values[1]), 564 'bss' : int(values[2]), 565 'rodata' : rodata, 566 } 567 sizes[values[5]] = size_dict 568 569 if read_func_sizes: 570 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 571 for fname in glob.glob(pattern): 572 with open(fname, 'r') as fd: 573 dict_name = os.path.basename(fname).replace('.sizes', 574 '') 575 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 576 577 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 578 579 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 580 581 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 582 """Calculate a summary of the results of building a commit. 583 584 Args: 585 board_selected: Dict containing boards to summarise 586 commit_upto: Commit number to summarize (0..self.count-1) 587 read_func_sizes: True to read function size information 588 589 Returns: 590 Tuple: 591 Dict containing boards which passed building this commit. 592 keyed by board.target 593 List containing a summary of error lines 594 Dict keyed by error line, containing a list of the Board 595 objects with that error 596 List containing a summary of warning lines 597 Dict keyed by error line, containing a list of the Board 598 objects with that warning 599 """ 600 def AddLine(lines_summary, lines_boards, line, board): 601 line = line.rstrip() 602 if line in lines_boards: 603 lines_boards[line].append(board) 604 else: 605 lines_boards[line] = [board] 606 lines_summary.append(line) 607 608 board_dict = {} 609 err_lines_summary = [] 610 err_lines_boards = {} 611 warn_lines_summary = [] 612 warn_lines_boards = {} 613 614 for board in boards_selected.itervalues(): 615 outcome = self.GetBuildOutcome(commit_upto, board.target, 616 read_func_sizes) 617 board_dict[board.target] = outcome 618 last_func = None 619 last_was_warning = False 620 for line in outcome.err_lines: 621 if line: 622 if (self._re_function.match(line) or 623 self._re_files.match(line)): 624 last_func = line 625 else: 626 is_warning = self._re_warning.match(line) 627 is_note = self._re_note.match(line) 628 if is_warning or (last_was_warning and is_note): 629 if last_func: 630 AddLine(warn_lines_summary, warn_lines_boards, 631 last_func, board) 632 AddLine(warn_lines_summary, warn_lines_boards, 633 line, board) 634 else: 635 if last_func: 636 AddLine(err_lines_summary, err_lines_boards, 637 last_func, board) 638 AddLine(err_lines_summary, err_lines_boards, 639 line, board) 640 last_was_warning = is_warning 641 last_func = None 642 return (board_dict, err_lines_summary, err_lines_boards, 643 warn_lines_summary, warn_lines_boards) 644 645 def AddOutcome(self, board_dict, arch_list, changes, char, color): 646 """Add an output to our list of outcomes for each architecture 647 648 This simple function adds failing boards (changes) to the 649 relevant architecture string, so we can print the results out 650 sorted by architecture. 651 652 Args: 653 board_dict: Dict containing all boards 654 arch_list: Dict keyed by arch name. Value is a string containing 655 a list of board names which failed for that arch. 656 changes: List of boards to add to arch_list 657 color: terminal.Colour object 658 """ 659 done_arch = {} 660 for target in changes: 661 if target in board_dict: 662 arch = board_dict[target].arch 663 else: 664 arch = 'unknown' 665 str = self.col.Color(color, ' ' + target) 666 if not arch in done_arch: 667 str = ' %s %s' % (self.col.Color(color, char), str) 668 done_arch[arch] = True 669 if not arch in arch_list: 670 arch_list[arch] = str 671 else: 672 arch_list[arch] += str 673 674 675 def ColourNum(self, num): 676 color = self.col.RED if num > 0 else self.col.GREEN 677 if num == 0: 678 return '0' 679 return self.col.Color(color, str(num)) 680 681 def ResetResultSummary(self, board_selected): 682 """Reset the results summary ready for use. 683 684 Set up the base board list to be all those selected, and set the 685 error lines to empty. 686 687 Following this, calls to PrintResultSummary() will use this 688 information to work out what has changed. 689 690 Args: 691 board_selected: Dict containing boards to summarise, keyed by 692 board.target 693 """ 694 self._base_board_dict = {} 695 for board in board_selected: 696 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 697 self._base_err_lines = [] 698 self._base_warn_lines = [] 699 self._base_err_line_boards = {} 700 self._base_warn_line_boards = {} 701 702 def PrintFuncSizeDetail(self, fname, old, new): 703 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 704 delta, common = [], {} 705 706 for a in old: 707 if a in new: 708 common[a] = 1 709 710 for name in old: 711 if name not in common: 712 remove += 1 713 down += old[name] 714 delta.append([-old[name], name]) 715 716 for name in new: 717 if name not in common: 718 add += 1 719 up += new[name] 720 delta.append([new[name], name]) 721 722 for name in common: 723 diff = new.get(name, 0) - old.get(name, 0) 724 if diff > 0: 725 grow, up = grow + 1, up + diff 726 elif diff < 0: 727 shrink, down = shrink + 1, down - diff 728 delta.append([diff, name]) 729 730 delta.sort() 731 delta.reverse() 732 733 args = [add, -remove, grow, -shrink, up, -down, up - down] 734 if max(args) == 0: 735 return 736 args = [self.ColourNum(x) for x in args] 737 indent = ' ' * 15 738 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 739 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 740 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 741 'delta')) 742 for diff, name in delta: 743 if diff: 744 color = self.col.RED if diff > 0 else self.col.GREEN 745 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 746 old.get(name, '-'), new.get(name,'-'), diff) 747 Print(msg, colour=color) 748 749 750 def PrintSizeDetail(self, target_list, show_bloat): 751 """Show details size information for each board 752 753 Args: 754 target_list: List of targets, each a dict containing: 755 'target': Target name 756 'total_diff': Total difference in bytes across all areas 757 <part_name>: Difference for that part 758 show_bloat: Show detail for each function 759 """ 760 targets_by_diff = sorted(target_list, reverse=True, 761 key=lambda x: x['_total_diff']) 762 for result in targets_by_diff: 763 printed_target = False 764 for name in sorted(result): 765 diff = result[name] 766 if name.startswith('_'): 767 continue 768 if diff != 0: 769 color = self.col.RED if diff > 0 else self.col.GREEN 770 msg = ' %s %+d' % (name, diff) 771 if not printed_target: 772 Print('%10s %-15s:' % ('', result['_target']), 773 newline=False) 774 printed_target = True 775 Print(msg, colour=color, newline=False) 776 if printed_target: 777 Print() 778 if show_bloat: 779 target = result['_target'] 780 outcome = result['_outcome'] 781 base_outcome = self._base_board_dict[target] 782 for fname in outcome.func_sizes: 783 self.PrintFuncSizeDetail(fname, 784 base_outcome.func_sizes[fname], 785 outcome.func_sizes[fname]) 786 787 788 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 789 show_bloat): 790 """Print a summary of image sizes broken down by section. 791 792 The summary takes the form of one line per architecture. The 793 line contains deltas for each of the sections (+ means the section 794 got bigger, - means smaller). The nunmbers are the average number 795 of bytes that a board in this section increased by. 796 797 For example: 798 powerpc: (622 boards) text -0.0 799 arm: (285 boards) text -0.0 800 nds32: (3 boards) text -8.0 801 802 Args: 803 board_selected: Dict containing boards to summarise, keyed by 804 board.target 805 board_dict: Dict containing boards for which we built this 806 commit, keyed by board.target. The value is an Outcome object. 807 show_detail: Show detail for each board 808 show_bloat: Show detail for each function 809 """ 810 arch_list = {} 811 arch_count = {} 812 813 # Calculate changes in size for different image parts 814 # The previous sizes are in Board.sizes, for each board 815 for target in board_dict: 816 if target not in board_selected: 817 continue 818 base_sizes = self._base_board_dict[target].sizes 819 outcome = board_dict[target] 820 sizes = outcome.sizes 821 822 # Loop through the list of images, creating a dict of size 823 # changes for each image/part. We end up with something like 824 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 825 # which means that U-Boot data increased by 5 bytes and SPL 826 # text decreased by 4. 827 err = {'_target' : target} 828 for image in sizes: 829 if image in base_sizes: 830 base_image = base_sizes[image] 831 # Loop through the text, data, bss parts 832 for part in sorted(sizes[image]): 833 diff = sizes[image][part] - base_image[part] 834 col = None 835 if diff: 836 if image == 'u-boot': 837 name = part 838 else: 839 name = image + ':' + part 840 err[name] = diff 841 arch = board_selected[target].arch 842 if not arch in arch_count: 843 arch_count[arch] = 1 844 else: 845 arch_count[arch] += 1 846 if not sizes: 847 pass # Only add to our list when we have some stats 848 elif not arch in arch_list: 849 arch_list[arch] = [err] 850 else: 851 arch_list[arch].append(err) 852 853 # We now have a list of image size changes sorted by arch 854 # Print out a summary of these 855 for arch, target_list in arch_list.iteritems(): 856 # Get total difference for each type 857 totals = {} 858 for result in target_list: 859 total = 0 860 for name, diff in result.iteritems(): 861 if name.startswith('_'): 862 continue 863 total += diff 864 if name in totals: 865 totals[name] += diff 866 else: 867 totals[name] = diff 868 result['_total_diff'] = total 869 result['_outcome'] = board_dict[result['_target']] 870 871 count = len(target_list) 872 printed_arch = False 873 for name in sorted(totals): 874 diff = totals[name] 875 if diff: 876 # Display the average difference in this name for this 877 # architecture 878 avg_diff = float(diff) / count 879 color = self.col.RED if avg_diff > 0 else self.col.GREEN 880 msg = ' %s %+1.1f' % (name, avg_diff) 881 if not printed_arch: 882 Print('%10s: (for %d/%d boards)' % (arch, count, 883 arch_count[arch]), newline=False) 884 printed_arch = True 885 Print(msg, colour=color, newline=False) 886 887 if printed_arch: 888 Print() 889 if show_detail: 890 self.PrintSizeDetail(target_list, show_bloat) 891 892 893 def PrintResultSummary(self, board_selected, board_dict, err_lines, 894 err_line_boards, warn_lines, warn_line_boards, 895 show_sizes, show_detail, show_bloat): 896 """Compare results with the base results and display delta. 897 898 Only boards mentioned in board_selected will be considered. This 899 function is intended to be called repeatedly with the results of 900 each commit. It therefore shows a 'diff' between what it saw in 901 the last call and what it sees now. 902 903 Args: 904 board_selected: Dict containing boards to summarise, keyed by 905 board.target 906 board_dict: Dict containing boards for which we built this 907 commit, keyed by board.target. The value is an Outcome object. 908 err_lines: A list of errors for this commit, or [] if there is 909 none, or we don't want to print errors 910 err_line_boards: Dict keyed by error line, containing a list of 911 the Board objects with that error 912 warn_lines: A list of warnings for this commit, or [] if there is 913 none, or we don't want to print errors 914 warn_line_boards: Dict keyed by warning line, containing a list of 915 the Board objects with that warning 916 show_sizes: Show image size deltas 917 show_detail: Show detail for each board 918 show_bloat: Show detail for each function 919 """ 920 def _BoardList(line, line_boards): 921 """Helper function to get a line of boards containing a line 922 923 Args: 924 line: Error line to search for 925 Return: 926 String containing a list of boards with that error line, or 927 '' if the user has not requested such a list 928 """ 929 if self._list_error_boards: 930 names = [] 931 for board in line_boards[line]: 932 if not board.target in names: 933 names.append(board.target) 934 names_str = '(%s) ' % ','.join(names) 935 else: 936 names_str = '' 937 return names_str 938 939 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 940 char): 941 better_lines = [] 942 worse_lines = [] 943 for line in lines: 944 if line not in base_lines: 945 worse_lines.append(char + '+' + 946 _BoardList(line, line_boards) + line) 947 for line in base_lines: 948 if line not in lines: 949 better_lines.append(char + '-' + 950 _BoardList(line, base_line_boards) + line) 951 return better_lines, worse_lines 952 953 better = [] # List of boards fixed since last commit 954 worse = [] # List of new broken boards since last commit 955 new = [] # List of boards that didn't exist last time 956 unknown = [] # List of boards that were not built 957 958 for target in board_dict: 959 if target not in board_selected: 960 continue 961 962 # If the board was built last time, add its outcome to a list 963 if target in self._base_board_dict: 964 base_outcome = self._base_board_dict[target].rc 965 outcome = board_dict[target] 966 if outcome.rc == OUTCOME_UNKNOWN: 967 unknown.append(target) 968 elif outcome.rc < base_outcome: 969 better.append(target) 970 elif outcome.rc > base_outcome: 971 worse.append(target) 972 else: 973 new.append(target) 974 975 # Get a list of errors that have appeared, and disappeared 976 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 977 self._base_err_line_boards, err_lines, err_line_boards, '') 978 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 979 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 980 981 # Display results by arch 982 if (better or worse or unknown or new or worse_err or better_err 983 or worse_warn or better_warn): 984 arch_list = {} 985 self.AddOutcome(board_selected, arch_list, better, '', 986 self.col.GREEN) 987 self.AddOutcome(board_selected, arch_list, worse, '+', 988 self.col.RED) 989 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 990 if self._show_unknown: 991 self.AddOutcome(board_selected, arch_list, unknown, '?', 992 self.col.MAGENTA) 993 for arch, target_list in arch_list.iteritems(): 994 Print('%10s: %s' % (arch, target_list)) 995 self._error_lines += 1 996 if better_err: 997 Print('\n'.join(better_err), colour=self.col.GREEN) 998 self._error_lines += 1 999 if worse_err: 1000 Print('\n'.join(worse_err), colour=self.col.RED) 1001 self._error_lines += 1 1002 if better_warn: 1003 Print('\n'.join(better_warn), colour=self.col.CYAN) 1004 self._error_lines += 1 1005 if worse_warn: 1006 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1007 self._error_lines += 1 1008 1009 if show_sizes: 1010 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1011 show_bloat) 1012 1013 # Save our updated information for the next call to this function 1014 self._base_board_dict = board_dict 1015 self._base_err_lines = err_lines 1016 self._base_warn_lines = warn_lines 1017 self._base_err_line_boards = err_line_boards 1018 self._base_warn_line_boards = warn_line_boards 1019 1020 # Get a list of boards that did not get built, if needed 1021 not_built = [] 1022 for board in board_selected: 1023 if not board in board_dict: 1024 not_built.append(board) 1025 if not_built: 1026 Print("Boards not built (%d): %s" % (len(not_built), 1027 ', '.join(not_built))) 1028 1029 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1030 (board_dict, err_lines, err_line_boards, warn_lines, 1031 warn_line_boards) = self.GetResultSummary( 1032 board_selected, commit_upto, 1033 read_func_sizes=self._show_bloat) 1034 if commits: 1035 msg = '%02d: %s' % (commit_upto + 1, 1036 commits[commit_upto].subject) 1037 Print(msg, colour=self.col.BLUE) 1038 self.PrintResultSummary(board_selected, board_dict, 1039 err_lines if self._show_errors else [], err_line_boards, 1040 warn_lines if self._show_errors else [], warn_line_boards, 1041 self._show_sizes, self._show_detail, self._show_bloat) 1042 1043 def ShowSummary(self, commits, board_selected): 1044 """Show a build summary for U-Boot for a given board list. 1045 1046 Reset the result summary, then repeatedly call GetResultSummary on 1047 each commit's results, then display the differences we see. 1048 1049 Args: 1050 commit: Commit objects to summarise 1051 board_selected: Dict containing boards to summarise 1052 """ 1053 self.commit_count = len(commits) if commits else 1 1054 self.commits = commits 1055 self.ResetResultSummary(board_selected) 1056 self._error_lines = 0 1057 1058 for commit_upto in range(0, self.commit_count, self._step): 1059 self.ProduceResultSummary(commit_upto, commits, board_selected) 1060 if not self._error_lines: 1061 Print('(no errors to report)', colour=self.col.GREEN) 1062 1063 1064 def SetupBuild(self, board_selected, commits): 1065 """Set up ready to start a build. 1066 1067 Args: 1068 board_selected: Selected boards to build 1069 commits: Selected commits to build 1070 """ 1071 # First work out how many commits we will build 1072 count = (self.commit_count + self._step - 1) / self._step 1073 self.count = len(board_selected) * count 1074 self.upto = self.warned = self.fail = 0 1075 self._timestamps = collections.deque() 1076 1077 def GetThreadDir(self, thread_num): 1078 """Get the directory path to the working dir for a thread. 1079 1080 Args: 1081 thread_num: Number of thread to check. 1082 """ 1083 return os.path.join(self._working_dir, '%02d' % thread_num) 1084 1085 def _PrepareThread(self, thread_num, setup_git): 1086 """Prepare the working directory for a thread. 1087 1088 This clones or fetches the repo into the thread's work directory. 1089 1090 Args: 1091 thread_num: Thread number (0, 1, ...) 1092 setup_git: True to set up a git repo clone 1093 """ 1094 thread_dir = self.GetThreadDir(thread_num) 1095 builderthread.Mkdir(thread_dir) 1096 git_dir = os.path.join(thread_dir, '.git') 1097 1098 # Clone the repo if it doesn't already exist 1099 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1100 # we have a private index but uses the origin repo's contents? 1101 if setup_git and self.git_dir: 1102 src_dir = os.path.abspath(self.git_dir) 1103 if os.path.exists(git_dir): 1104 gitutil.Fetch(git_dir, thread_dir) 1105 else: 1106 Print('Cloning repo for thread %d' % thread_num) 1107 gitutil.Clone(src_dir, thread_dir) 1108 1109 def _PrepareWorkingSpace(self, max_threads, setup_git): 1110 """Prepare the working directory for use. 1111 1112 Set up the git repo for each thread. 1113 1114 Args: 1115 max_threads: Maximum number of threads we expect to need. 1116 setup_git: True to set up a git repo clone 1117 """ 1118 builderthread.Mkdir(self._working_dir) 1119 for thread in range(max_threads): 1120 self._PrepareThread(thread, setup_git) 1121 1122 def _PrepareOutputSpace(self): 1123 """Get the output directories ready to receive files. 1124 1125 We delete any output directories which look like ones we need to 1126 create. Having left over directories is confusing when the user wants 1127 to check the output manually. 1128 """ 1129 if not self.commits: 1130 return 1131 dir_list = [] 1132 for commit_upto in range(self.commit_count): 1133 dir_list.append(self._GetOutputDir(commit_upto)) 1134 1135 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1136 if dirname not in dir_list: 1137 shutil.rmtree(dirname) 1138 1139 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1140 """Build all commits for a list of boards 1141 1142 Args: 1143 commits: List of commits to be build, each a Commit object 1144 boards_selected: Dict of selected boards, key is target name, 1145 value is Board object 1146 keep_outputs: True to save build output files 1147 verbose: Display build results as they are completed 1148 Returns: 1149 Tuple containing: 1150 - number of boards that failed to build 1151 - number of boards that issued warnings 1152 """ 1153 self.commit_count = len(commits) if commits else 1 1154 self.commits = commits 1155 self._verbose = verbose 1156 1157 self.ResetResultSummary(board_selected) 1158 builderthread.Mkdir(self.base_dir, parents = True) 1159 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1160 commits is not None) 1161 self._PrepareOutputSpace() 1162 self.SetupBuild(board_selected, commits) 1163 self.ProcessResult(None) 1164 1165 # Create jobs to build all commits for each board 1166 for brd in board_selected.itervalues(): 1167 job = builderthread.BuilderJob() 1168 job.board = brd 1169 job.commits = commits 1170 job.keep_outputs = keep_outputs 1171 job.step = self._step 1172 self.queue.put(job) 1173 1174 # Wait until all jobs are started 1175 self.queue.join() 1176 1177 # Wait until we have processed all output 1178 self.out_queue.join() 1179 Print() 1180 self.ClearLine(0) 1181 return (self.fail, self.warned) 1182