1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99 100class Builder: 101 """Class for building U-Boot for a particular commit. 102 103 Public members: (many should ->private) 104 active: True if the builder is active and has not been stopped 105 already_done: Number of builds already completed 106 base_dir: Base directory to use for builder 107 checkout: True to check out source, False to skip that step. 108 This is used for testing. 109 col: terminal.Color() object 110 count: Number of commits to build 111 do_make: Method to call to invoke Make 112 fail: Number of builds that failed due to error 113 force_build: Force building even if a build already exists 114 force_config_on_failure: If a commit fails for a board, disable 115 incremental building for the next commit we build for that 116 board, so that we will see all warnings/errors again. 117 force_build_failures: If a previously-built build (i.e. built on 118 a previous run of buildman) is marked as failed, rebuild it. 119 git_dir: Git directory containing source repository 120 last_line_len: Length of the last line we printed (used for erasing 121 it with new progress information) 122 num_jobs: Number of jobs to run at once (passed to make as -j) 123 num_threads: Number of builder threads to run 124 out_queue: Queue of results to process 125 re_make_err: Compiled regular expression for ignore_lines 126 queue: Queue of jobs to run 127 threads: List of active threads 128 toolchains: Toolchains object to use for building 129 upto: Current commit number we are building (0.count-1) 130 warned: Number of builds that produced at least one warning 131 force_reconfig: Reconfigure U-Boot on each comiit. This disables 132 incremental building, where buildman reconfigures on the first 133 commit for a baord, and then just does an incremental build for 134 the following commits. In fact buildman will reconfigure and 135 retry for any failing commits, so generally the only effect of 136 this option is to slow things down. 137 in_tree: Build U-Boot in-tree instead of specifying an output 138 directory separate from the source code. This option is really 139 only useful for testing in-tree builds. 140 141 Private members: 142 _base_board_dict: Last-summarised Dict of boards 143 _base_err_lines: Last-summarised list of errors 144 _base_warn_lines: Last-summarised list of warnings 145 _build_period_us: Time taken for a single build (float object). 146 _complete_delay: Expected delay until completion (timedelta) 147 _next_delay_update: Next time we plan to display a progress update 148 (datatime) 149 _show_unknown: Show unknown boards (those not built) in summary 150 _timestamps: List of timestamps for the completion of the last 151 last _timestamp_count builds. Each is a datetime object. 152 _timestamp_count: Number of timestamps to keep in our list. 153 _working_dir: Base working directory containing all threads 154 """ 155 class Outcome: 156 """Records a build outcome for a single make invocation 157 158 Public Members: 159 rc: Outcome value (OUTCOME_...) 160 err_lines: List of error lines or [] if none 161 sizes: Dictionary of image size information, keyed by filename 162 - Each value is itself a dictionary containing 163 values for 'text', 'data' and 'bss', being the integer 164 size in bytes of each section. 165 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 166 value is itself a dictionary: 167 key: function name 168 value: Size of function in bytes 169 """ 170 def __init__(self, rc, err_lines, sizes, func_sizes): 171 self.rc = rc 172 self.err_lines = err_lines 173 self.sizes = sizes 174 self.func_sizes = func_sizes 175 176 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 177 gnu_make='make', checkout=True, show_unknown=True, step=1, 178 no_subdirs=False, full_path=False): 179 """Create a new Builder object 180 181 Args: 182 toolchains: Toolchains object to use for building 183 base_dir: Base directory to use for builder 184 git_dir: Git directory containing source repository 185 num_threads: Number of builder threads to run 186 num_jobs: Number of jobs to run at once (passed to make as -j) 187 gnu_make: the command name of GNU Make. 188 checkout: True to check out source, False to skip that step. 189 This is used for testing. 190 show_unknown: Show unknown boards (those not built) in summary 191 step: 1 to process every commit, n to process every nth commit 192 no_subdirs: Don't create subdirectories when building current 193 source for a single board 194 full_path: Return the full path in CROSS_COMPILE and don't set 195 PATH 196 """ 197 self.toolchains = toolchains 198 self.base_dir = base_dir 199 self._working_dir = os.path.join(base_dir, '.bm-work') 200 self.threads = [] 201 self.active = True 202 self.do_make = self.Make 203 self.gnu_make = gnu_make 204 self.checkout = checkout 205 self.num_threads = num_threads 206 self.num_jobs = num_jobs 207 self.already_done = 0 208 self.force_build = False 209 self.git_dir = git_dir 210 self._show_unknown = show_unknown 211 self._timestamp_count = 10 212 self._build_period_us = None 213 self._complete_delay = None 214 self._next_delay_update = datetime.now() 215 self.force_config_on_failure = True 216 self.force_build_failures = False 217 self.force_reconfig = False 218 self._step = step 219 self.in_tree = False 220 self._error_lines = 0 221 self.no_subdirs = no_subdirs 222 self.full_path = full_path 223 224 self.col = terminal.Color() 225 226 self._re_function = re.compile('(.*): In function.*') 227 self._re_files = re.compile('In file included from.*') 228 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 229 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 230 231 self.queue = Queue.Queue() 232 self.out_queue = Queue.Queue() 233 for i in range(self.num_threads): 234 t = builderthread.BuilderThread(self, i) 235 t.setDaemon(True) 236 t.start() 237 self.threads.append(t) 238 239 self.last_line_len = 0 240 t = builderthread.ResultThread(self) 241 t.setDaemon(True) 242 t.start() 243 self.threads.append(t) 244 245 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 246 self.re_make_err = re.compile('|'.join(ignore_lines)) 247 248 def __del__(self): 249 """Get rid of all threads created by the builder""" 250 for t in self.threads: 251 del t 252 253 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 254 show_detail=False, show_bloat=False, 255 list_error_boards=False): 256 """Setup display options for the builder. 257 258 show_errors: True to show summarised error/warning info 259 show_sizes: Show size deltas 260 show_detail: Show detail for each board 261 show_bloat: Show detail for each function 262 list_error_boards: Show the boards which caused each error/warning 263 """ 264 self._show_errors = show_errors 265 self._show_sizes = show_sizes 266 self._show_detail = show_detail 267 self._show_bloat = show_bloat 268 self._list_error_boards = list_error_boards 269 270 def _AddTimestamp(self): 271 """Add a new timestamp to the list and record the build period. 272 273 The build period is the length of time taken to perform a single 274 build (one board, one commit). 275 """ 276 now = datetime.now() 277 self._timestamps.append(now) 278 count = len(self._timestamps) 279 delta = self._timestamps[-1] - self._timestamps[0] 280 seconds = delta.total_seconds() 281 282 # If we have enough data, estimate build period (time taken for a 283 # single build) and therefore completion time. 284 if count > 1 and self._next_delay_update < now: 285 self._next_delay_update = now + timedelta(seconds=2) 286 if seconds > 0: 287 self._build_period = float(seconds) / count 288 todo = self.count - self.upto 289 self._complete_delay = timedelta(microseconds= 290 self._build_period * todo * 1000000) 291 # Round it 292 self._complete_delay -= timedelta( 293 microseconds=self._complete_delay.microseconds) 294 295 if seconds > 60: 296 self._timestamps.popleft() 297 count -= 1 298 299 def ClearLine(self, length): 300 """Clear any characters on the current line 301 302 Make way for a new line of length 'length', by outputting enough 303 spaces to clear out the old line. Then remember the new length for 304 next time. 305 306 Args: 307 length: Length of new line, in characters 308 """ 309 if length < self.last_line_len: 310 Print(' ' * (self.last_line_len - length), newline=False) 311 Print('\r', newline=False) 312 self.last_line_len = length 313 sys.stdout.flush() 314 315 def SelectCommit(self, commit, checkout=True): 316 """Checkout the selected commit for this build 317 """ 318 self.commit = commit 319 if checkout and self.checkout: 320 gitutil.Checkout(commit.hash) 321 322 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 323 """Run make 324 325 Args: 326 commit: Commit object that is being built 327 brd: Board object that is being built 328 stage: Stage that we are at (mrproper, config, build) 329 cwd: Directory where make should be run 330 args: Arguments to pass to make 331 kwargs: Arguments to pass to command.RunPipe() 332 """ 333 cmd = [self.gnu_make] + list(args) 334 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 335 cwd=cwd, raise_on_error=False, **kwargs) 336 return result 337 338 def ProcessResult(self, result): 339 """Process the result of a build, showing progress information 340 341 Args: 342 result: A CommandResult object, which indicates the result for 343 a single build 344 """ 345 col = terminal.Color() 346 if result: 347 target = result.brd.target 348 349 if result.return_code < 0: 350 self.active = False 351 command.StopAll() 352 return 353 354 self.upto += 1 355 if result.return_code != 0: 356 self.fail += 1 357 elif result.stderr: 358 self.warned += 1 359 if result.already_done: 360 self.already_done += 1 361 if self._verbose: 362 Print('\r', newline=False) 363 self.ClearLine(0) 364 boards_selected = {target : result.brd} 365 self.ResetResultSummary(boards_selected) 366 self.ProduceResultSummary(result.commit_upto, self.commits, 367 boards_selected) 368 else: 369 target = '(starting)' 370 371 # Display separate counts for ok, warned and fail 372 ok = self.upto - self.warned - self.fail 373 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 374 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 375 line += self.col.Color(self.col.RED, '%5d' % self.fail) 376 377 name = ' /%-5d ' % self.count 378 379 # Add our current completion time estimate 380 self._AddTimestamp() 381 if self._complete_delay: 382 name += '%s : ' % self._complete_delay 383 # When building all boards for a commit, we can print a commit 384 # progress message. 385 if result and result.commit_upto is None: 386 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 387 self.commit_count) 388 389 name += target 390 Print(line + name, newline=False) 391 length = 14 + len(name) 392 self.ClearLine(length) 393 394 def _GetOutputDir(self, commit_upto): 395 """Get the name of the output directory for a commit number 396 397 The output directory is typically .../<branch>/<commit>. 398 399 Args: 400 commit_upto: Commit number to use (0..self.count-1) 401 """ 402 commit_dir = None 403 if self.commits: 404 commit = self.commits[commit_upto] 405 subject = commit.subject.translate(trans_valid_chars) 406 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 407 self.commit_count, commit.hash, subject[:20])) 408 elif not self.no_subdirs: 409 commit_dir = 'current' 410 if not commit_dir: 411 return self.base_dir 412 return os.path.join(self.base_dir, commit_dir) 413 414 def GetBuildDir(self, commit_upto, target): 415 """Get the name of the build directory for a commit number 416 417 The build directory is typically .../<branch>/<commit>/<target>. 418 419 Args: 420 commit_upto: Commit number to use (0..self.count-1) 421 target: Target name 422 """ 423 output_dir = self._GetOutputDir(commit_upto) 424 return os.path.join(output_dir, target) 425 426 def GetDoneFile(self, commit_upto, target): 427 """Get the name of the done file for a commit number 428 429 Args: 430 commit_upto: Commit number to use (0..self.count-1) 431 target: Target name 432 """ 433 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 434 435 def GetSizesFile(self, commit_upto, target): 436 """Get the name of the sizes file for a commit number 437 438 Args: 439 commit_upto: Commit number to use (0..self.count-1) 440 target: Target name 441 """ 442 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 443 444 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 445 """Get the name of the funcsizes file for a commit number and ELF file 446 447 Args: 448 commit_upto: Commit number to use (0..self.count-1) 449 target: Target name 450 elf_fname: Filename of elf image 451 """ 452 return os.path.join(self.GetBuildDir(commit_upto, target), 453 '%s.sizes' % elf_fname.replace('/', '-')) 454 455 def GetObjdumpFile(self, commit_upto, target, elf_fname): 456 """Get the name of the objdump file for a commit number and ELF file 457 458 Args: 459 commit_upto: Commit number to use (0..self.count-1) 460 target: Target name 461 elf_fname: Filename of elf image 462 """ 463 return os.path.join(self.GetBuildDir(commit_upto, target), 464 '%s.objdump' % elf_fname.replace('/', '-')) 465 466 def GetErrFile(self, commit_upto, target): 467 """Get the name of the err file for a commit number 468 469 Args: 470 commit_upto: Commit number to use (0..self.count-1) 471 target: Target name 472 """ 473 output_dir = self.GetBuildDir(commit_upto, target) 474 return os.path.join(output_dir, 'err') 475 476 def FilterErrors(self, lines): 477 """Filter out errors in which we have no interest 478 479 We should probably use map(). 480 481 Args: 482 lines: List of error lines, each a string 483 Returns: 484 New list with only interesting lines included 485 """ 486 out_lines = [] 487 for line in lines: 488 if not self.re_make_err.search(line): 489 out_lines.append(line) 490 return out_lines 491 492 def ReadFuncSizes(self, fname, fd): 493 """Read function sizes from the output of 'nm' 494 495 Args: 496 fd: File containing data to read 497 fname: Filename we are reading from (just for errors) 498 499 Returns: 500 Dictionary containing size of each function in bytes, indexed by 501 function name. 502 """ 503 sym = {} 504 for line in fd.readlines(): 505 try: 506 size, type, name = line[:-1].split() 507 except: 508 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 509 continue 510 if type in 'tTdDbB': 511 # function names begin with '.' on 64-bit powerpc 512 if '.' in name[1:]: 513 name = 'static.' + name.split('.')[0] 514 sym[name] = sym.get(name, 0) + int(size, 16) 515 return sym 516 517 def GetBuildOutcome(self, commit_upto, target, read_func_sizes): 518 """Work out the outcome of a build. 519 520 Args: 521 commit_upto: Commit number to check (0..n-1) 522 target: Target board to check 523 read_func_sizes: True to read function size information 524 525 Returns: 526 Outcome object 527 """ 528 done_file = self.GetDoneFile(commit_upto, target) 529 sizes_file = self.GetSizesFile(commit_upto, target) 530 sizes = {} 531 func_sizes = {} 532 if os.path.exists(done_file): 533 with open(done_file, 'r') as fd: 534 return_code = int(fd.readline()) 535 err_lines = [] 536 err_file = self.GetErrFile(commit_upto, target) 537 if os.path.exists(err_file): 538 with open(err_file, 'r') as fd: 539 err_lines = self.FilterErrors(fd.readlines()) 540 541 # Decide whether the build was ok, failed or created warnings 542 if return_code: 543 rc = OUTCOME_ERROR 544 elif len(err_lines): 545 rc = OUTCOME_WARNING 546 else: 547 rc = OUTCOME_OK 548 549 # Convert size information to our simple format 550 if os.path.exists(sizes_file): 551 with open(sizes_file, 'r') as fd: 552 for line in fd.readlines(): 553 values = line.split() 554 rodata = 0 555 if len(values) > 6: 556 rodata = int(values[6], 16) 557 size_dict = { 558 'all' : int(values[0]) + int(values[1]) + 559 int(values[2]), 560 'text' : int(values[0]) - rodata, 561 'data' : int(values[1]), 562 'bss' : int(values[2]), 563 'rodata' : rodata, 564 } 565 sizes[values[5]] = size_dict 566 567 if read_func_sizes: 568 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 569 for fname in glob.glob(pattern): 570 with open(fname, 'r') as fd: 571 dict_name = os.path.basename(fname).replace('.sizes', 572 '') 573 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 574 575 return Builder.Outcome(rc, err_lines, sizes, func_sizes) 576 577 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) 578 579 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): 580 """Calculate a summary of the results of building a commit. 581 582 Args: 583 board_selected: Dict containing boards to summarise 584 commit_upto: Commit number to summarize (0..self.count-1) 585 read_func_sizes: True to read function size information 586 587 Returns: 588 Tuple: 589 Dict containing boards which passed building this commit. 590 keyed by board.target 591 List containing a summary of error lines 592 Dict keyed by error line, containing a list of the Board 593 objects with that error 594 List containing a summary of warning lines 595 Dict keyed by error line, containing a list of the Board 596 objects with that warning 597 """ 598 def AddLine(lines_summary, lines_boards, line, board): 599 line = line.rstrip() 600 if line in lines_boards: 601 lines_boards[line].append(board) 602 else: 603 lines_boards[line] = [board] 604 lines_summary.append(line) 605 606 board_dict = {} 607 err_lines_summary = [] 608 err_lines_boards = {} 609 warn_lines_summary = [] 610 warn_lines_boards = {} 611 612 for board in boards_selected.itervalues(): 613 outcome = self.GetBuildOutcome(commit_upto, board.target, 614 read_func_sizes) 615 board_dict[board.target] = outcome 616 last_func = None 617 last_was_warning = False 618 for line in outcome.err_lines: 619 if line: 620 if (self._re_function.match(line) or 621 self._re_files.match(line)): 622 last_func = line 623 else: 624 is_warning = self._re_warning.match(line) 625 is_note = self._re_note.match(line) 626 if is_warning or (last_was_warning and is_note): 627 if last_func: 628 AddLine(warn_lines_summary, warn_lines_boards, 629 last_func, board) 630 AddLine(warn_lines_summary, warn_lines_boards, 631 line, board) 632 else: 633 if last_func: 634 AddLine(err_lines_summary, err_lines_boards, 635 last_func, board) 636 AddLine(err_lines_summary, err_lines_boards, 637 line, board) 638 last_was_warning = is_warning 639 last_func = None 640 return (board_dict, err_lines_summary, err_lines_boards, 641 warn_lines_summary, warn_lines_boards) 642 643 def AddOutcome(self, board_dict, arch_list, changes, char, color): 644 """Add an output to our list of outcomes for each architecture 645 646 This simple function adds failing boards (changes) to the 647 relevant architecture string, so we can print the results out 648 sorted by architecture. 649 650 Args: 651 board_dict: Dict containing all boards 652 arch_list: Dict keyed by arch name. Value is a string containing 653 a list of board names which failed for that arch. 654 changes: List of boards to add to arch_list 655 color: terminal.Colour object 656 """ 657 done_arch = {} 658 for target in changes: 659 if target in board_dict: 660 arch = board_dict[target].arch 661 else: 662 arch = 'unknown' 663 str = self.col.Color(color, ' ' + target) 664 if not arch in done_arch: 665 str = self.col.Color(color, char) + ' ' + str 666 done_arch[arch] = True 667 if not arch in arch_list: 668 arch_list[arch] = str 669 else: 670 arch_list[arch] += str 671 672 673 def ColourNum(self, num): 674 color = self.col.RED if num > 0 else self.col.GREEN 675 if num == 0: 676 return '0' 677 return self.col.Color(color, str(num)) 678 679 def ResetResultSummary(self, board_selected): 680 """Reset the results summary ready for use. 681 682 Set up the base board list to be all those selected, and set the 683 error lines to empty. 684 685 Following this, calls to PrintResultSummary() will use this 686 information to work out what has changed. 687 688 Args: 689 board_selected: Dict containing boards to summarise, keyed by 690 board.target 691 """ 692 self._base_board_dict = {} 693 for board in board_selected: 694 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) 695 self._base_err_lines = [] 696 self._base_warn_lines = [] 697 self._base_err_line_boards = {} 698 self._base_warn_line_boards = {} 699 700 def PrintFuncSizeDetail(self, fname, old, new): 701 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 702 delta, common = [], {} 703 704 for a in old: 705 if a in new: 706 common[a] = 1 707 708 for name in old: 709 if name not in common: 710 remove += 1 711 down += old[name] 712 delta.append([-old[name], name]) 713 714 for name in new: 715 if name not in common: 716 add += 1 717 up += new[name] 718 delta.append([new[name], name]) 719 720 for name in common: 721 diff = new.get(name, 0) - old.get(name, 0) 722 if diff > 0: 723 grow, up = grow + 1, up + diff 724 elif diff < 0: 725 shrink, down = shrink + 1, down - diff 726 delta.append([diff, name]) 727 728 delta.sort() 729 delta.reverse() 730 731 args = [add, -remove, grow, -shrink, up, -down, up - down] 732 if max(args) == 0: 733 return 734 args = [self.ColourNum(x) for x in args] 735 indent = ' ' * 15 736 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 737 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 738 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 739 'delta')) 740 for diff, name in delta: 741 if diff: 742 color = self.col.RED if diff > 0 else self.col.GREEN 743 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 744 old.get(name, '-'), new.get(name,'-'), diff) 745 Print(msg, colour=color) 746 747 748 def PrintSizeDetail(self, target_list, show_bloat): 749 """Show details size information for each board 750 751 Args: 752 target_list: List of targets, each a dict containing: 753 'target': Target name 754 'total_diff': Total difference in bytes across all areas 755 <part_name>: Difference for that part 756 show_bloat: Show detail for each function 757 """ 758 targets_by_diff = sorted(target_list, reverse=True, 759 key=lambda x: x['_total_diff']) 760 for result in targets_by_diff: 761 printed_target = False 762 for name in sorted(result): 763 diff = result[name] 764 if name.startswith('_'): 765 continue 766 if diff != 0: 767 color = self.col.RED if diff > 0 else self.col.GREEN 768 msg = ' %s %+d' % (name, diff) 769 if not printed_target: 770 Print('%10s %-15s:' % ('', result['_target']), 771 newline=False) 772 printed_target = True 773 Print(msg, colour=color, newline=False) 774 if printed_target: 775 Print() 776 if show_bloat: 777 target = result['_target'] 778 outcome = result['_outcome'] 779 base_outcome = self._base_board_dict[target] 780 for fname in outcome.func_sizes: 781 self.PrintFuncSizeDetail(fname, 782 base_outcome.func_sizes[fname], 783 outcome.func_sizes[fname]) 784 785 786 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 787 show_bloat): 788 """Print a summary of image sizes broken down by section. 789 790 The summary takes the form of one line per architecture. The 791 line contains deltas for each of the sections (+ means the section 792 got bigger, - means smaller). The nunmbers are the average number 793 of bytes that a board in this section increased by. 794 795 For example: 796 powerpc: (622 boards) text -0.0 797 arm: (285 boards) text -0.0 798 nds32: (3 boards) text -8.0 799 800 Args: 801 board_selected: Dict containing boards to summarise, keyed by 802 board.target 803 board_dict: Dict containing boards for which we built this 804 commit, keyed by board.target. The value is an Outcome object. 805 show_detail: Show detail for each board 806 show_bloat: Show detail for each function 807 """ 808 arch_list = {} 809 arch_count = {} 810 811 # Calculate changes in size for different image parts 812 # The previous sizes are in Board.sizes, for each board 813 for target in board_dict: 814 if target not in board_selected: 815 continue 816 base_sizes = self._base_board_dict[target].sizes 817 outcome = board_dict[target] 818 sizes = outcome.sizes 819 820 # Loop through the list of images, creating a dict of size 821 # changes for each image/part. We end up with something like 822 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 823 # which means that U-Boot data increased by 5 bytes and SPL 824 # text decreased by 4. 825 err = {'_target' : target} 826 for image in sizes: 827 if image in base_sizes: 828 base_image = base_sizes[image] 829 # Loop through the text, data, bss parts 830 for part in sorted(sizes[image]): 831 diff = sizes[image][part] - base_image[part] 832 col = None 833 if diff: 834 if image == 'u-boot': 835 name = part 836 else: 837 name = image + ':' + part 838 err[name] = diff 839 arch = board_selected[target].arch 840 if not arch in arch_count: 841 arch_count[arch] = 1 842 else: 843 arch_count[arch] += 1 844 if not sizes: 845 pass # Only add to our list when we have some stats 846 elif not arch in arch_list: 847 arch_list[arch] = [err] 848 else: 849 arch_list[arch].append(err) 850 851 # We now have a list of image size changes sorted by arch 852 # Print out a summary of these 853 for arch, target_list in arch_list.iteritems(): 854 # Get total difference for each type 855 totals = {} 856 for result in target_list: 857 total = 0 858 for name, diff in result.iteritems(): 859 if name.startswith('_'): 860 continue 861 total += diff 862 if name in totals: 863 totals[name] += diff 864 else: 865 totals[name] = diff 866 result['_total_diff'] = total 867 result['_outcome'] = board_dict[result['_target']] 868 869 count = len(target_list) 870 printed_arch = False 871 for name in sorted(totals): 872 diff = totals[name] 873 if diff: 874 # Display the average difference in this name for this 875 # architecture 876 avg_diff = float(diff) / count 877 color = self.col.RED if avg_diff > 0 else self.col.GREEN 878 msg = ' %s %+1.1f' % (name, avg_diff) 879 if not printed_arch: 880 Print('%10s: (for %d/%d boards)' % (arch, count, 881 arch_count[arch]), newline=False) 882 printed_arch = True 883 Print(msg, colour=color, newline=False) 884 885 if printed_arch: 886 Print() 887 if show_detail: 888 self.PrintSizeDetail(target_list, show_bloat) 889 890 891 def PrintResultSummary(self, board_selected, board_dict, err_lines, 892 err_line_boards, warn_lines, warn_line_boards, 893 show_sizes, show_detail, show_bloat): 894 """Compare results with the base results and display delta. 895 896 Only boards mentioned in board_selected will be considered. This 897 function is intended to be called repeatedly with the results of 898 each commit. It therefore shows a 'diff' between what it saw in 899 the last call and what it sees now. 900 901 Args: 902 board_selected: Dict containing boards to summarise, keyed by 903 board.target 904 board_dict: Dict containing boards for which we built this 905 commit, keyed by board.target. The value is an Outcome object. 906 err_lines: A list of errors for this commit, or [] if there is 907 none, or we don't want to print errors 908 err_line_boards: Dict keyed by error line, containing a list of 909 the Board objects with that error 910 warn_lines: A list of warnings for this commit, or [] if there is 911 none, or we don't want to print errors 912 warn_line_boards: Dict keyed by warning line, containing a list of 913 the Board objects with that warning 914 show_sizes: Show image size deltas 915 show_detail: Show detail for each board 916 show_bloat: Show detail for each function 917 """ 918 def _BoardList(line, line_boards): 919 """Helper function to get a line of boards containing a line 920 921 Args: 922 line: Error line to search for 923 Return: 924 String containing a list of boards with that error line, or 925 '' if the user has not requested such a list 926 """ 927 if self._list_error_boards: 928 names = [] 929 for board in line_boards[line]: 930 if not board.target in names: 931 names.append(board.target) 932 names_str = '(%s) ' % ','.join(names) 933 else: 934 names_str = '' 935 return names_str 936 937 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 938 char): 939 better_lines = [] 940 worse_lines = [] 941 for line in lines: 942 if line not in base_lines: 943 worse_lines.append(char + '+' + 944 _BoardList(line, line_boards) + line) 945 for line in base_lines: 946 if line not in lines: 947 better_lines.append(char + '-' + 948 _BoardList(line, base_line_boards) + line) 949 return better_lines, worse_lines 950 951 better = [] # List of boards fixed since last commit 952 worse = [] # List of new broken boards since last commit 953 new = [] # List of boards that didn't exist last time 954 unknown = [] # List of boards that were not built 955 956 for target in board_dict: 957 if target not in board_selected: 958 continue 959 960 # If the board was built last time, add its outcome to a list 961 if target in self._base_board_dict: 962 base_outcome = self._base_board_dict[target].rc 963 outcome = board_dict[target] 964 if outcome.rc == OUTCOME_UNKNOWN: 965 unknown.append(target) 966 elif outcome.rc < base_outcome: 967 better.append(target) 968 elif outcome.rc > base_outcome: 969 worse.append(target) 970 else: 971 new.append(target) 972 973 # Get a list of errors that have appeared, and disappeared 974 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 975 self._base_err_line_boards, err_lines, err_line_boards, '') 976 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 977 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 978 979 # Display results by arch 980 if (better or worse or unknown or new or worse_err or better_err 981 or worse_warn or better_warn): 982 arch_list = {} 983 self.AddOutcome(board_selected, arch_list, better, '', 984 self.col.GREEN) 985 self.AddOutcome(board_selected, arch_list, worse, '+', 986 self.col.RED) 987 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 988 if self._show_unknown: 989 self.AddOutcome(board_selected, arch_list, unknown, '?', 990 self.col.MAGENTA) 991 for arch, target_list in arch_list.iteritems(): 992 Print('%10s: %s' % (arch, target_list)) 993 self._error_lines += 1 994 if better_err: 995 Print('\n'.join(better_err), colour=self.col.GREEN) 996 self._error_lines += 1 997 if worse_err: 998 Print('\n'.join(worse_err), colour=self.col.RED) 999 self._error_lines += 1 1000 if better_warn: 1001 Print('\n'.join(better_warn), colour=self.col.CYAN) 1002 self._error_lines += 1 1003 if worse_warn: 1004 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1005 self._error_lines += 1 1006 1007 if show_sizes: 1008 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1009 show_bloat) 1010 1011 # Save our updated information for the next call to this function 1012 self._base_board_dict = board_dict 1013 self._base_err_lines = err_lines 1014 self._base_warn_lines = warn_lines 1015 self._base_err_line_boards = err_line_boards 1016 self._base_warn_line_boards = warn_line_boards 1017 1018 # Get a list of boards that did not get built, if needed 1019 not_built = [] 1020 for board in board_selected: 1021 if not board in board_dict: 1022 not_built.append(board) 1023 if not_built: 1024 Print("Boards not built (%d): %s" % (len(not_built), 1025 ', '.join(not_built))) 1026 1027 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1028 (board_dict, err_lines, err_line_boards, warn_lines, 1029 warn_line_boards) = self.GetResultSummary( 1030 board_selected, commit_upto, 1031 read_func_sizes=self._show_bloat) 1032 if commits: 1033 msg = '%02d: %s' % (commit_upto + 1, 1034 commits[commit_upto].subject) 1035 Print(msg, colour=self.col.BLUE) 1036 self.PrintResultSummary(board_selected, board_dict, 1037 err_lines if self._show_errors else [], err_line_boards, 1038 warn_lines if self._show_errors else [], warn_line_boards, 1039 self._show_sizes, self._show_detail, self._show_bloat) 1040 1041 def ShowSummary(self, commits, board_selected): 1042 """Show a build summary for U-Boot for a given board list. 1043 1044 Reset the result summary, then repeatedly call GetResultSummary on 1045 each commit's results, then display the differences we see. 1046 1047 Args: 1048 commit: Commit objects to summarise 1049 board_selected: Dict containing boards to summarise 1050 """ 1051 self.commit_count = len(commits) if commits else 1 1052 self.commits = commits 1053 self.ResetResultSummary(board_selected) 1054 self._error_lines = 0 1055 1056 for commit_upto in range(0, self.commit_count, self._step): 1057 self.ProduceResultSummary(commit_upto, commits, board_selected) 1058 if not self._error_lines: 1059 Print('(no errors to report)', colour=self.col.GREEN) 1060 1061 1062 def SetupBuild(self, board_selected, commits): 1063 """Set up ready to start a build. 1064 1065 Args: 1066 board_selected: Selected boards to build 1067 commits: Selected commits to build 1068 """ 1069 # First work out how many commits we will build 1070 count = (self.commit_count + self._step - 1) / self._step 1071 self.count = len(board_selected) * count 1072 self.upto = self.warned = self.fail = 0 1073 self._timestamps = collections.deque() 1074 1075 def GetThreadDir(self, thread_num): 1076 """Get the directory path to the working dir for a thread. 1077 1078 Args: 1079 thread_num: Number of thread to check. 1080 """ 1081 return os.path.join(self._working_dir, '%02d' % thread_num) 1082 1083 def _PrepareThread(self, thread_num, setup_git): 1084 """Prepare the working directory for a thread. 1085 1086 This clones or fetches the repo into the thread's work directory. 1087 1088 Args: 1089 thread_num: Thread number (0, 1, ...) 1090 setup_git: True to set up a git repo clone 1091 """ 1092 thread_dir = self.GetThreadDir(thread_num) 1093 builderthread.Mkdir(thread_dir) 1094 git_dir = os.path.join(thread_dir, '.git') 1095 1096 # Clone the repo if it doesn't already exist 1097 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1098 # we have a private index but uses the origin repo's contents? 1099 if setup_git and self.git_dir: 1100 src_dir = os.path.abspath(self.git_dir) 1101 if os.path.exists(git_dir): 1102 gitutil.Fetch(git_dir, thread_dir) 1103 else: 1104 Print('Cloning repo for thread %d' % thread_num) 1105 gitutil.Clone(src_dir, thread_dir) 1106 1107 def _PrepareWorkingSpace(self, max_threads, setup_git): 1108 """Prepare the working directory for use. 1109 1110 Set up the git repo for each thread. 1111 1112 Args: 1113 max_threads: Maximum number of threads we expect to need. 1114 setup_git: True to set up a git repo clone 1115 """ 1116 builderthread.Mkdir(self._working_dir) 1117 for thread in range(max_threads): 1118 self._PrepareThread(thread, setup_git) 1119 1120 def _PrepareOutputSpace(self): 1121 """Get the output directories ready to receive files. 1122 1123 We delete any output directories which look like ones we need to 1124 create. Having left over directories is confusing when the user wants 1125 to check the output manually. 1126 """ 1127 if not self.commits: 1128 return 1129 dir_list = [] 1130 for commit_upto in range(self.commit_count): 1131 dir_list.append(self._GetOutputDir(commit_upto)) 1132 1133 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1134 if dirname not in dir_list: 1135 shutil.rmtree(dirname) 1136 1137 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1138 """Build all commits for a list of boards 1139 1140 Args: 1141 commits: List of commits to be build, each a Commit object 1142 boards_selected: Dict of selected boards, key is target name, 1143 value is Board object 1144 keep_outputs: True to save build output files 1145 verbose: Display build results as they are completed 1146 Returns: 1147 Tuple containing: 1148 - number of boards that failed to build 1149 - number of boards that issued warnings 1150 """ 1151 self.commit_count = len(commits) if commits else 1 1152 self.commits = commits 1153 self._verbose = verbose 1154 1155 self.ResetResultSummary(board_selected) 1156 builderthread.Mkdir(self.base_dir, parents = True) 1157 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1158 commits is not None) 1159 self._PrepareOutputSpace() 1160 self.SetupBuild(board_selected, commits) 1161 self.ProcessResult(None) 1162 1163 # Create jobs to build all commits for each board 1164 for brd in board_selected.itervalues(): 1165 job = builderthread.BuilderJob() 1166 job.board = brd 1167 job.commits = commits 1168 job.keep_outputs = keep_outputs 1169 job.step = self._step 1170 self.queue.put(job) 1171 1172 # Wait until all jobs are started 1173 self.queue.join() 1174 1175 # Wait until we have processed all output 1176 self.out_queue.join() 1177 Print() 1178 self.ClearLine(0) 1179 return (self.fail, self.warned) 1180