1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99CONFIG_FILENAMES = [ 100 '.config', '.config-spl', '.config-tpl', 101 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 102 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 104] 105 106 107class Builder: 108 """Class for building U-Boot for a particular commit. 109 110 Public members: (many should ->private) 111 active: True if the builder is active and has not been stopped 112 already_done: Number of builds already completed 113 base_dir: Base directory to use for builder 114 checkout: True to check out source, False to skip that step. 115 This is used for testing. 116 col: terminal.Color() object 117 count: Number of commits to build 118 do_make: Method to call to invoke Make 119 fail: Number of builds that failed due to error 120 force_build: Force building even if a build already exists 121 force_config_on_failure: If a commit fails for a board, disable 122 incremental building for the next commit we build for that 123 board, so that we will see all warnings/errors again. 124 force_build_failures: If a previously-built build (i.e. built on 125 a previous run of buildman) is marked as failed, rebuild it. 126 git_dir: Git directory containing source repository 127 last_line_len: Length of the last line we printed (used for erasing 128 it with new progress information) 129 num_jobs: Number of jobs to run at once (passed to make as -j) 130 num_threads: Number of builder threads to run 131 out_queue: Queue of results to process 132 re_make_err: Compiled regular expression for ignore_lines 133 queue: Queue of jobs to run 134 threads: List of active threads 135 toolchains: Toolchains object to use for building 136 upto: Current commit number we are building (0.count-1) 137 warned: Number of builds that produced at least one warning 138 force_reconfig: Reconfigure U-Boot on each comiit. This disables 139 incremental building, where buildman reconfigures on the first 140 commit for a baord, and then just does an incremental build for 141 the following commits. In fact buildman will reconfigure and 142 retry for any failing commits, so generally the only effect of 143 this option is to slow things down. 144 in_tree: Build U-Boot in-tree instead of specifying an output 145 directory separate from the source code. This option is really 146 only useful for testing in-tree builds. 147 148 Private members: 149 _base_board_dict: Last-summarised Dict of boards 150 _base_err_lines: Last-summarised list of errors 151 _base_warn_lines: Last-summarised list of warnings 152 _build_period_us: Time taken for a single build (float object). 153 _complete_delay: Expected delay until completion (timedelta) 154 _next_delay_update: Next time we plan to display a progress update 155 (datatime) 156 _show_unknown: Show unknown boards (those not built) in summary 157 _timestamps: List of timestamps for the completion of the last 158 last _timestamp_count builds. Each is a datetime object. 159 _timestamp_count: Number of timestamps to keep in our list. 160 _working_dir: Base working directory containing all threads 161 """ 162 class Outcome: 163 """Records a build outcome for a single make invocation 164 165 Public Members: 166 rc: Outcome value (OUTCOME_...) 167 err_lines: List of error lines or [] if none 168 sizes: Dictionary of image size information, keyed by filename 169 - Each value is itself a dictionary containing 170 values for 'text', 'data' and 'bss', being the integer 171 size in bytes of each section. 172 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 173 value is itself a dictionary: 174 key: function name 175 value: Size of function in bytes 176 config: Dictionary keyed by filename - e.g. '.config'. Each 177 value is itself a dictionary: 178 key: config name 179 value: config value 180 """ 181 def __init__(self, rc, err_lines, sizes, func_sizes, config): 182 self.rc = rc 183 self.err_lines = err_lines 184 self.sizes = sizes 185 self.func_sizes = func_sizes 186 self.config = config 187 188 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 189 gnu_make='make', checkout=True, show_unknown=True, step=1, 190 no_subdirs=False, full_path=False, verbose_build=False): 191 """Create a new Builder object 192 193 Args: 194 toolchains: Toolchains object to use for building 195 base_dir: Base directory to use for builder 196 git_dir: Git directory containing source repository 197 num_threads: Number of builder threads to run 198 num_jobs: Number of jobs to run at once (passed to make as -j) 199 gnu_make: the command name of GNU Make. 200 checkout: True to check out source, False to skip that step. 201 This is used for testing. 202 show_unknown: Show unknown boards (those not built) in summary 203 step: 1 to process every commit, n to process every nth commit 204 no_subdirs: Don't create subdirectories when building current 205 source for a single board 206 full_path: Return the full path in CROSS_COMPILE and don't set 207 PATH 208 verbose_build: Run build with V=1 and don't use 'make -s' 209 """ 210 self.toolchains = toolchains 211 self.base_dir = base_dir 212 self._working_dir = os.path.join(base_dir, '.bm-work') 213 self.threads = [] 214 self.active = True 215 self.do_make = self.Make 216 self.gnu_make = gnu_make 217 self.checkout = checkout 218 self.num_threads = num_threads 219 self.num_jobs = num_jobs 220 self.already_done = 0 221 self.force_build = False 222 self.git_dir = git_dir 223 self._show_unknown = show_unknown 224 self._timestamp_count = 10 225 self._build_period_us = None 226 self._complete_delay = None 227 self._next_delay_update = datetime.now() 228 self.force_config_on_failure = True 229 self.force_build_failures = False 230 self.force_reconfig = False 231 self._step = step 232 self.in_tree = False 233 self._error_lines = 0 234 self.no_subdirs = no_subdirs 235 self.full_path = full_path 236 self.verbose_build = verbose_build 237 238 self.col = terminal.Color() 239 240 self._re_function = re.compile('(.*): In function.*') 241 self._re_files = re.compile('In file included from.*') 242 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 243 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 244 245 self.queue = Queue.Queue() 246 self.out_queue = Queue.Queue() 247 for i in range(self.num_threads): 248 t = builderthread.BuilderThread(self, i) 249 t.setDaemon(True) 250 t.start() 251 self.threads.append(t) 252 253 self.last_line_len = 0 254 t = builderthread.ResultThread(self) 255 t.setDaemon(True) 256 t.start() 257 self.threads.append(t) 258 259 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 260 self.re_make_err = re.compile('|'.join(ignore_lines)) 261 262 def __del__(self): 263 """Get rid of all threads created by the builder""" 264 for t in self.threads: 265 del t 266 267 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 268 show_detail=False, show_bloat=False, 269 list_error_boards=False, show_config=False): 270 """Setup display options for the builder. 271 272 show_errors: True to show summarised error/warning info 273 show_sizes: Show size deltas 274 show_detail: Show detail for each board 275 show_bloat: Show detail for each function 276 list_error_boards: Show the boards which caused each error/warning 277 show_config: Show config deltas 278 """ 279 self._show_errors = show_errors 280 self._show_sizes = show_sizes 281 self._show_detail = show_detail 282 self._show_bloat = show_bloat 283 self._list_error_boards = list_error_boards 284 self._show_config = show_config 285 286 def _AddTimestamp(self): 287 """Add a new timestamp to the list and record the build period. 288 289 The build period is the length of time taken to perform a single 290 build (one board, one commit). 291 """ 292 now = datetime.now() 293 self._timestamps.append(now) 294 count = len(self._timestamps) 295 delta = self._timestamps[-1] - self._timestamps[0] 296 seconds = delta.total_seconds() 297 298 # If we have enough data, estimate build period (time taken for a 299 # single build) and therefore completion time. 300 if count > 1 and self._next_delay_update < now: 301 self._next_delay_update = now + timedelta(seconds=2) 302 if seconds > 0: 303 self._build_period = float(seconds) / count 304 todo = self.count - self.upto 305 self._complete_delay = timedelta(microseconds= 306 self._build_period * todo * 1000000) 307 # Round it 308 self._complete_delay -= timedelta( 309 microseconds=self._complete_delay.microseconds) 310 311 if seconds > 60: 312 self._timestamps.popleft() 313 count -= 1 314 315 def ClearLine(self, length): 316 """Clear any characters on the current line 317 318 Make way for a new line of length 'length', by outputting enough 319 spaces to clear out the old line. Then remember the new length for 320 next time. 321 322 Args: 323 length: Length of new line, in characters 324 """ 325 if length < self.last_line_len: 326 Print(' ' * (self.last_line_len - length), newline=False) 327 Print('\r', newline=False) 328 self.last_line_len = length 329 sys.stdout.flush() 330 331 def SelectCommit(self, commit, checkout=True): 332 """Checkout the selected commit for this build 333 """ 334 self.commit = commit 335 if checkout and self.checkout: 336 gitutil.Checkout(commit.hash) 337 338 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 339 """Run make 340 341 Args: 342 commit: Commit object that is being built 343 brd: Board object that is being built 344 stage: Stage that we are at (mrproper, config, build) 345 cwd: Directory where make should be run 346 args: Arguments to pass to make 347 kwargs: Arguments to pass to command.RunPipe() 348 """ 349 cmd = [self.gnu_make] + list(args) 350 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 351 cwd=cwd, raise_on_error=False, **kwargs) 352 if self.verbose_build: 353 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 354 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 355 return result 356 357 def ProcessResult(self, result): 358 """Process the result of a build, showing progress information 359 360 Args: 361 result: A CommandResult object, which indicates the result for 362 a single build 363 """ 364 col = terminal.Color() 365 if result: 366 target = result.brd.target 367 368 if result.return_code < 0: 369 self.active = False 370 command.StopAll() 371 return 372 373 self.upto += 1 374 if result.return_code != 0: 375 self.fail += 1 376 elif result.stderr: 377 self.warned += 1 378 if result.already_done: 379 self.already_done += 1 380 if self._verbose: 381 Print('\r', newline=False) 382 self.ClearLine(0) 383 boards_selected = {target : result.brd} 384 self.ResetResultSummary(boards_selected) 385 self.ProduceResultSummary(result.commit_upto, self.commits, 386 boards_selected) 387 else: 388 target = '(starting)' 389 390 # Display separate counts for ok, warned and fail 391 ok = self.upto - self.warned - self.fail 392 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 393 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 394 line += self.col.Color(self.col.RED, '%5d' % self.fail) 395 396 name = ' /%-5d ' % self.count 397 398 # Add our current completion time estimate 399 self._AddTimestamp() 400 if self._complete_delay: 401 name += '%s : ' % self._complete_delay 402 # When building all boards for a commit, we can print a commit 403 # progress message. 404 if result and result.commit_upto is None: 405 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 406 self.commit_count) 407 408 name += target 409 Print(line + name, newline=False) 410 length = 14 + len(name) 411 self.ClearLine(length) 412 413 def _GetOutputDir(self, commit_upto): 414 """Get the name of the output directory for a commit number 415 416 The output directory is typically .../<branch>/<commit>. 417 418 Args: 419 commit_upto: Commit number to use (0..self.count-1) 420 """ 421 commit_dir = None 422 if self.commits: 423 commit = self.commits[commit_upto] 424 subject = commit.subject.translate(trans_valid_chars) 425 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 426 self.commit_count, commit.hash, subject[:20])) 427 elif not self.no_subdirs: 428 commit_dir = 'current' 429 if not commit_dir: 430 return self.base_dir 431 return os.path.join(self.base_dir, commit_dir) 432 433 def GetBuildDir(self, commit_upto, target): 434 """Get the name of the build directory for a commit number 435 436 The build directory is typically .../<branch>/<commit>/<target>. 437 438 Args: 439 commit_upto: Commit number to use (0..self.count-1) 440 target: Target name 441 """ 442 output_dir = self._GetOutputDir(commit_upto) 443 return os.path.join(output_dir, target) 444 445 def GetDoneFile(self, commit_upto, target): 446 """Get the name of the done file for a commit number 447 448 Args: 449 commit_upto: Commit number to use (0..self.count-1) 450 target: Target name 451 """ 452 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 453 454 def GetSizesFile(self, commit_upto, target): 455 """Get the name of the sizes file for a commit number 456 457 Args: 458 commit_upto: Commit number to use (0..self.count-1) 459 target: Target name 460 """ 461 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 462 463 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 464 """Get the name of the funcsizes file for a commit number and ELF file 465 466 Args: 467 commit_upto: Commit number to use (0..self.count-1) 468 target: Target name 469 elf_fname: Filename of elf image 470 """ 471 return os.path.join(self.GetBuildDir(commit_upto, target), 472 '%s.sizes' % elf_fname.replace('/', '-')) 473 474 def GetObjdumpFile(self, commit_upto, target, elf_fname): 475 """Get the name of the objdump file for a commit number and ELF file 476 477 Args: 478 commit_upto: Commit number to use (0..self.count-1) 479 target: Target name 480 elf_fname: Filename of elf image 481 """ 482 return os.path.join(self.GetBuildDir(commit_upto, target), 483 '%s.objdump' % elf_fname.replace('/', '-')) 484 485 def GetErrFile(self, commit_upto, target): 486 """Get the name of the err file for a commit number 487 488 Args: 489 commit_upto: Commit number to use (0..self.count-1) 490 target: Target name 491 """ 492 output_dir = self.GetBuildDir(commit_upto, target) 493 return os.path.join(output_dir, 'err') 494 495 def FilterErrors(self, lines): 496 """Filter out errors in which we have no interest 497 498 We should probably use map(). 499 500 Args: 501 lines: List of error lines, each a string 502 Returns: 503 New list with only interesting lines included 504 """ 505 out_lines = [] 506 for line in lines: 507 if not self.re_make_err.search(line): 508 out_lines.append(line) 509 return out_lines 510 511 def ReadFuncSizes(self, fname, fd): 512 """Read function sizes from the output of 'nm' 513 514 Args: 515 fd: File containing data to read 516 fname: Filename we are reading from (just for errors) 517 518 Returns: 519 Dictionary containing size of each function in bytes, indexed by 520 function name. 521 """ 522 sym = {} 523 for line in fd.readlines(): 524 try: 525 size, type, name = line[:-1].split() 526 except: 527 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 528 continue 529 if type in 'tTdDbB': 530 # function names begin with '.' on 64-bit powerpc 531 if '.' in name[1:]: 532 name = 'static.' + name.split('.')[0] 533 sym[name] = sym.get(name, 0) + int(size, 16) 534 return sym 535 536 def _ProcessConfig(self, fname): 537 """Read in a .config, autoconf.mk or autoconf.h file 538 539 This function handles all config file types. It ignores comments and 540 any #defines which don't start with CONFIG_. 541 542 Args: 543 fname: Filename to read 544 545 Returns: 546 Dictionary: 547 key: Config name (e.g. CONFIG_DM) 548 value: Config value (e.g. 1) 549 """ 550 config = {} 551 if os.path.exists(fname): 552 with open(fname) as fd: 553 for line in fd: 554 line = line.strip() 555 if line.startswith('#define'): 556 values = line[8:].split(' ', 1) 557 if len(values) > 1: 558 key, value = values 559 else: 560 key = values[0] 561 value = '' 562 if not key.startswith('CONFIG_'): 563 continue 564 elif not line or line[0] in ['#', '*', '/']: 565 continue 566 else: 567 key, value = line.split('=', 1) 568 config[key] = value 569 return config 570 571 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 572 read_config): 573 """Work out the outcome of a build. 574 575 Args: 576 commit_upto: Commit number to check (0..n-1) 577 target: Target board to check 578 read_func_sizes: True to read function size information 579 read_config: True to read .config and autoconf.h files 580 581 Returns: 582 Outcome object 583 """ 584 done_file = self.GetDoneFile(commit_upto, target) 585 sizes_file = self.GetSizesFile(commit_upto, target) 586 sizes = {} 587 func_sizes = {} 588 config = {} 589 if os.path.exists(done_file): 590 with open(done_file, 'r') as fd: 591 return_code = int(fd.readline()) 592 err_lines = [] 593 err_file = self.GetErrFile(commit_upto, target) 594 if os.path.exists(err_file): 595 with open(err_file, 'r') as fd: 596 err_lines = self.FilterErrors(fd.readlines()) 597 598 # Decide whether the build was ok, failed or created warnings 599 if return_code: 600 rc = OUTCOME_ERROR 601 elif len(err_lines): 602 rc = OUTCOME_WARNING 603 else: 604 rc = OUTCOME_OK 605 606 # Convert size information to our simple format 607 if os.path.exists(sizes_file): 608 with open(sizes_file, 'r') as fd: 609 for line in fd.readlines(): 610 values = line.split() 611 rodata = 0 612 if len(values) > 6: 613 rodata = int(values[6], 16) 614 size_dict = { 615 'all' : int(values[0]) + int(values[1]) + 616 int(values[2]), 617 'text' : int(values[0]) - rodata, 618 'data' : int(values[1]), 619 'bss' : int(values[2]), 620 'rodata' : rodata, 621 } 622 sizes[values[5]] = size_dict 623 624 if read_func_sizes: 625 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 626 for fname in glob.glob(pattern): 627 with open(fname, 'r') as fd: 628 dict_name = os.path.basename(fname).replace('.sizes', 629 '') 630 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 631 632 if read_config: 633 output_dir = self.GetBuildDir(commit_upto, target) 634 for name in CONFIG_FILENAMES: 635 fname = os.path.join(output_dir, name) 636 config[name] = self._ProcessConfig(fname) 637 638 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 639 640 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 641 642 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 643 read_config): 644 """Calculate a summary of the results of building a commit. 645 646 Args: 647 board_selected: Dict containing boards to summarise 648 commit_upto: Commit number to summarize (0..self.count-1) 649 read_func_sizes: True to read function size information 650 read_config: True to read .config and autoconf.h files 651 652 Returns: 653 Tuple: 654 Dict containing boards which passed building this commit. 655 keyed by board.target 656 List containing a summary of error lines 657 Dict keyed by error line, containing a list of the Board 658 objects with that error 659 List containing a summary of warning lines 660 Dict keyed by error line, containing a list of the Board 661 objects with that warning 662 Dictionary keyed by filename - e.g. '.config'. Each 663 value is itself a dictionary: 664 key: config name 665 value: config value 666 """ 667 def AddLine(lines_summary, lines_boards, line, board): 668 line = line.rstrip() 669 if line in lines_boards: 670 lines_boards[line].append(board) 671 else: 672 lines_boards[line] = [board] 673 lines_summary.append(line) 674 675 board_dict = {} 676 err_lines_summary = [] 677 err_lines_boards = {} 678 warn_lines_summary = [] 679 warn_lines_boards = {} 680 config = {} 681 for fname in CONFIG_FILENAMES: 682 config[fname] = {} 683 684 for board in boards_selected.itervalues(): 685 outcome = self.GetBuildOutcome(commit_upto, board.target, 686 read_func_sizes, read_config) 687 board_dict[board.target] = outcome 688 last_func = None 689 last_was_warning = False 690 for line in outcome.err_lines: 691 if line: 692 if (self._re_function.match(line) or 693 self._re_files.match(line)): 694 last_func = line 695 else: 696 is_warning = self._re_warning.match(line) 697 is_note = self._re_note.match(line) 698 if is_warning or (last_was_warning and is_note): 699 if last_func: 700 AddLine(warn_lines_summary, warn_lines_boards, 701 last_func, board) 702 AddLine(warn_lines_summary, warn_lines_boards, 703 line, board) 704 else: 705 if last_func: 706 AddLine(err_lines_summary, err_lines_boards, 707 last_func, board) 708 AddLine(err_lines_summary, err_lines_boards, 709 line, board) 710 last_was_warning = is_warning 711 last_func = None 712 for fname in CONFIG_FILENAMES: 713 config[fname] = {} 714 if outcome.config: 715 for key, value in outcome.config[fname].iteritems(): 716 config[fname][key] = value 717 718 return (board_dict, err_lines_summary, err_lines_boards, 719 warn_lines_summary, warn_lines_boards, config) 720 721 def AddOutcome(self, board_dict, arch_list, changes, char, color): 722 """Add an output to our list of outcomes for each architecture 723 724 This simple function adds failing boards (changes) to the 725 relevant architecture string, so we can print the results out 726 sorted by architecture. 727 728 Args: 729 board_dict: Dict containing all boards 730 arch_list: Dict keyed by arch name. Value is a string containing 731 a list of board names which failed for that arch. 732 changes: List of boards to add to arch_list 733 color: terminal.Colour object 734 """ 735 done_arch = {} 736 for target in changes: 737 if target in board_dict: 738 arch = board_dict[target].arch 739 else: 740 arch = 'unknown' 741 str = self.col.Color(color, ' ' + target) 742 if not arch in done_arch: 743 str = ' %s %s' % (self.col.Color(color, char), str) 744 done_arch[arch] = True 745 if not arch in arch_list: 746 arch_list[arch] = str 747 else: 748 arch_list[arch] += str 749 750 751 def ColourNum(self, num): 752 color = self.col.RED if num > 0 else self.col.GREEN 753 if num == 0: 754 return '0' 755 return self.col.Color(color, str(num)) 756 757 def ResetResultSummary(self, board_selected): 758 """Reset the results summary ready for use. 759 760 Set up the base board list to be all those selected, and set the 761 error lines to empty. 762 763 Following this, calls to PrintResultSummary() will use this 764 information to work out what has changed. 765 766 Args: 767 board_selected: Dict containing boards to summarise, keyed by 768 board.target 769 """ 770 self._base_board_dict = {} 771 for board in board_selected: 772 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 773 self._base_err_lines = [] 774 self._base_warn_lines = [] 775 self._base_err_line_boards = {} 776 self._base_warn_line_boards = {} 777 self._base_config = {} 778 for fname in CONFIG_FILENAMES: 779 self._base_config[fname] = {} 780 781 def PrintFuncSizeDetail(self, fname, old, new): 782 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 783 delta, common = [], {} 784 785 for a in old: 786 if a in new: 787 common[a] = 1 788 789 for name in old: 790 if name not in common: 791 remove += 1 792 down += old[name] 793 delta.append([-old[name], name]) 794 795 for name in new: 796 if name not in common: 797 add += 1 798 up += new[name] 799 delta.append([new[name], name]) 800 801 for name in common: 802 diff = new.get(name, 0) - old.get(name, 0) 803 if diff > 0: 804 grow, up = grow + 1, up + diff 805 elif diff < 0: 806 shrink, down = shrink + 1, down - diff 807 delta.append([diff, name]) 808 809 delta.sort() 810 delta.reverse() 811 812 args = [add, -remove, grow, -shrink, up, -down, up - down] 813 if max(args) == 0: 814 return 815 args = [self.ColourNum(x) for x in args] 816 indent = ' ' * 15 817 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 818 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 819 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 820 'delta')) 821 for diff, name in delta: 822 if diff: 823 color = self.col.RED if diff > 0 else self.col.GREEN 824 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 825 old.get(name, '-'), new.get(name,'-'), diff) 826 Print(msg, colour=color) 827 828 829 def PrintSizeDetail(self, target_list, show_bloat): 830 """Show details size information for each board 831 832 Args: 833 target_list: List of targets, each a dict containing: 834 'target': Target name 835 'total_diff': Total difference in bytes across all areas 836 <part_name>: Difference for that part 837 show_bloat: Show detail for each function 838 """ 839 targets_by_diff = sorted(target_list, reverse=True, 840 key=lambda x: x['_total_diff']) 841 for result in targets_by_diff: 842 printed_target = False 843 for name in sorted(result): 844 diff = result[name] 845 if name.startswith('_'): 846 continue 847 if diff != 0: 848 color = self.col.RED if diff > 0 else self.col.GREEN 849 msg = ' %s %+d' % (name, diff) 850 if not printed_target: 851 Print('%10s %-15s:' % ('', result['_target']), 852 newline=False) 853 printed_target = True 854 Print(msg, colour=color, newline=False) 855 if printed_target: 856 Print() 857 if show_bloat: 858 target = result['_target'] 859 outcome = result['_outcome'] 860 base_outcome = self._base_board_dict[target] 861 for fname in outcome.func_sizes: 862 self.PrintFuncSizeDetail(fname, 863 base_outcome.func_sizes[fname], 864 outcome.func_sizes[fname]) 865 866 867 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 868 show_bloat): 869 """Print a summary of image sizes broken down by section. 870 871 The summary takes the form of one line per architecture. The 872 line contains deltas for each of the sections (+ means the section 873 got bigger, - means smaller). The nunmbers are the average number 874 of bytes that a board in this section increased by. 875 876 For example: 877 powerpc: (622 boards) text -0.0 878 arm: (285 boards) text -0.0 879 nds32: (3 boards) text -8.0 880 881 Args: 882 board_selected: Dict containing boards to summarise, keyed by 883 board.target 884 board_dict: Dict containing boards for which we built this 885 commit, keyed by board.target. The value is an Outcome object. 886 show_detail: Show detail for each board 887 show_bloat: Show detail for each function 888 """ 889 arch_list = {} 890 arch_count = {} 891 892 # Calculate changes in size for different image parts 893 # The previous sizes are in Board.sizes, for each board 894 for target in board_dict: 895 if target not in board_selected: 896 continue 897 base_sizes = self._base_board_dict[target].sizes 898 outcome = board_dict[target] 899 sizes = outcome.sizes 900 901 # Loop through the list of images, creating a dict of size 902 # changes for each image/part. We end up with something like 903 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 904 # which means that U-Boot data increased by 5 bytes and SPL 905 # text decreased by 4. 906 err = {'_target' : target} 907 for image in sizes: 908 if image in base_sizes: 909 base_image = base_sizes[image] 910 # Loop through the text, data, bss parts 911 for part in sorted(sizes[image]): 912 diff = sizes[image][part] - base_image[part] 913 col = None 914 if diff: 915 if image == 'u-boot': 916 name = part 917 else: 918 name = image + ':' + part 919 err[name] = diff 920 arch = board_selected[target].arch 921 if not arch in arch_count: 922 arch_count[arch] = 1 923 else: 924 arch_count[arch] += 1 925 if not sizes: 926 pass # Only add to our list when we have some stats 927 elif not arch in arch_list: 928 arch_list[arch] = [err] 929 else: 930 arch_list[arch].append(err) 931 932 # We now have a list of image size changes sorted by arch 933 # Print out a summary of these 934 for arch, target_list in arch_list.iteritems(): 935 # Get total difference for each type 936 totals = {} 937 for result in target_list: 938 total = 0 939 for name, diff in result.iteritems(): 940 if name.startswith('_'): 941 continue 942 total += diff 943 if name in totals: 944 totals[name] += diff 945 else: 946 totals[name] = diff 947 result['_total_diff'] = total 948 result['_outcome'] = board_dict[result['_target']] 949 950 count = len(target_list) 951 printed_arch = False 952 for name in sorted(totals): 953 diff = totals[name] 954 if diff: 955 # Display the average difference in this name for this 956 # architecture 957 avg_diff = float(diff) / count 958 color = self.col.RED if avg_diff > 0 else self.col.GREEN 959 msg = ' %s %+1.1f' % (name, avg_diff) 960 if not printed_arch: 961 Print('%10s: (for %d/%d boards)' % (arch, count, 962 arch_count[arch]), newline=False) 963 printed_arch = True 964 Print(msg, colour=color, newline=False) 965 966 if printed_arch: 967 Print() 968 if show_detail: 969 self.PrintSizeDetail(target_list, show_bloat) 970 971 972 def PrintResultSummary(self, board_selected, board_dict, err_lines, 973 err_line_boards, warn_lines, warn_line_boards, 974 config, show_sizes, show_detail, show_bloat, 975 show_config): 976 """Compare results with the base results and display delta. 977 978 Only boards mentioned in board_selected will be considered. This 979 function is intended to be called repeatedly with the results of 980 each commit. It therefore shows a 'diff' between what it saw in 981 the last call and what it sees now. 982 983 Args: 984 board_selected: Dict containing boards to summarise, keyed by 985 board.target 986 board_dict: Dict containing boards for which we built this 987 commit, keyed by board.target. The value is an Outcome object. 988 err_lines: A list of errors for this commit, or [] if there is 989 none, or we don't want to print errors 990 err_line_boards: Dict keyed by error line, containing a list of 991 the Board objects with that error 992 warn_lines: A list of warnings for this commit, or [] if there is 993 none, or we don't want to print errors 994 warn_line_boards: Dict keyed by warning line, containing a list of 995 the Board objects with that warning 996 config: Dictionary keyed by filename - e.g. '.config'. Each 997 value is itself a dictionary: 998 key: config name 999 value: config value 1000 show_sizes: Show image size deltas 1001 show_detail: Show detail for each board 1002 show_bloat: Show detail for each function 1003 show_config: Show config changes 1004 """ 1005 def _BoardList(line, line_boards): 1006 """Helper function to get a line of boards containing a line 1007 1008 Args: 1009 line: Error line to search for 1010 Return: 1011 String containing a list of boards with that error line, or 1012 '' if the user has not requested such a list 1013 """ 1014 if self._list_error_boards: 1015 names = [] 1016 for board in line_boards[line]: 1017 if not board.target in names: 1018 names.append(board.target) 1019 names_str = '(%s) ' % ','.join(names) 1020 else: 1021 names_str = '' 1022 return names_str 1023 1024 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1025 char): 1026 better_lines = [] 1027 worse_lines = [] 1028 for line in lines: 1029 if line not in base_lines: 1030 worse_lines.append(char + '+' + 1031 _BoardList(line, line_boards) + line) 1032 for line in base_lines: 1033 if line not in lines: 1034 better_lines.append(char + '-' + 1035 _BoardList(line, base_line_boards) + line) 1036 return better_lines, worse_lines 1037 1038 def _CalcConfig(delta, name, config): 1039 """Calculate configuration changes 1040 1041 Args: 1042 delta: Type of the delta, e.g. '+' 1043 name: name of the file which changed (e.g. .config) 1044 config: configuration change dictionary 1045 key: config name 1046 value: config value 1047 Returns: 1048 String containing the configuration changes which can be 1049 printed 1050 """ 1051 out = '' 1052 for key in sorted(config.keys()): 1053 out += '%s=%s ' % (key, config[key]) 1054 return '%5s %s: %s' % (delta, name, out) 1055 1056 def _ShowConfig(name, config_plus, config_minus, config_change): 1057 """Show changes in configuration 1058 1059 Args: 1060 config_plus: configurations added, dictionary 1061 key: config name 1062 value: config value 1063 config_minus: configurations removed, dictionary 1064 key: config name 1065 value: config value 1066 config_change: configurations changed, dictionary 1067 key: config name 1068 value: config value 1069 """ 1070 if config_plus: 1071 Print(_CalcConfig('+', name, config_plus), 1072 colour=self.col.GREEN) 1073 if config_minus: 1074 Print(_CalcConfig('-', name, config_minus), 1075 colour=self.col.RED) 1076 if config_change: 1077 Print(_CalcConfig('+/-', name, config_change), 1078 colour=self.col.YELLOW) 1079 1080 better = [] # List of boards fixed since last commit 1081 worse = [] # List of new broken boards since last commit 1082 new = [] # List of boards that didn't exist last time 1083 unknown = [] # List of boards that were not built 1084 1085 for target in board_dict: 1086 if target not in board_selected: 1087 continue 1088 1089 # If the board was built last time, add its outcome to a list 1090 if target in self._base_board_dict: 1091 base_outcome = self._base_board_dict[target].rc 1092 outcome = board_dict[target] 1093 if outcome.rc == OUTCOME_UNKNOWN: 1094 unknown.append(target) 1095 elif outcome.rc < base_outcome: 1096 better.append(target) 1097 elif outcome.rc > base_outcome: 1098 worse.append(target) 1099 else: 1100 new.append(target) 1101 1102 # Get a list of errors that have appeared, and disappeared 1103 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1104 self._base_err_line_boards, err_lines, err_line_boards, '') 1105 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1106 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1107 1108 # Display results by arch 1109 if (better or worse or unknown or new or worse_err or better_err 1110 or worse_warn or better_warn): 1111 arch_list = {} 1112 self.AddOutcome(board_selected, arch_list, better, '', 1113 self.col.GREEN) 1114 self.AddOutcome(board_selected, arch_list, worse, '+', 1115 self.col.RED) 1116 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1117 if self._show_unknown: 1118 self.AddOutcome(board_selected, arch_list, unknown, '?', 1119 self.col.MAGENTA) 1120 for arch, target_list in arch_list.iteritems(): 1121 Print('%10s: %s' % (arch, target_list)) 1122 self._error_lines += 1 1123 if better_err: 1124 Print('\n'.join(better_err), colour=self.col.GREEN) 1125 self._error_lines += 1 1126 if worse_err: 1127 Print('\n'.join(worse_err), colour=self.col.RED) 1128 self._error_lines += 1 1129 if better_warn: 1130 Print('\n'.join(better_warn), colour=self.col.CYAN) 1131 self._error_lines += 1 1132 if worse_warn: 1133 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1134 self._error_lines += 1 1135 1136 if show_sizes: 1137 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1138 show_bloat) 1139 1140 if show_config: 1141 all_config_plus = {} 1142 all_config_minus = {} 1143 all_config_change = {} 1144 for name in CONFIG_FILENAMES: 1145 if not config[name]: 1146 continue 1147 config_plus = {} 1148 config_minus = {} 1149 config_change = {} 1150 base = self._base_config[name] 1151 for key, value in config[name].iteritems(): 1152 if key not in base: 1153 config_plus[key] = value 1154 all_config_plus[key] = value 1155 for key, value in base.iteritems(): 1156 if key not in config[name]: 1157 config_minus[key] = value 1158 all_config_minus[key] = value 1159 for key, value in base.iteritems(): 1160 new_value = base[key] 1161 if key in config[name] and value != new_value: 1162 desc = '%s -> %s' % (value, new_value) 1163 config_change[key] = desc 1164 all_config_change[key] = desc 1165 _ShowConfig(name, config_plus, config_minus, config_change) 1166 _ShowConfig('all', all_config_plus, all_config_minus, 1167 all_config_change) 1168 1169 # Save our updated information for the next call to this function 1170 self._base_board_dict = board_dict 1171 self._base_err_lines = err_lines 1172 self._base_warn_lines = warn_lines 1173 self._base_err_line_boards = err_line_boards 1174 self._base_warn_line_boards = warn_line_boards 1175 self._base_config = config 1176 1177 # Get a list of boards that did not get built, if needed 1178 not_built = [] 1179 for board in board_selected: 1180 if not board in board_dict: 1181 not_built.append(board) 1182 if not_built: 1183 Print("Boards not built (%d): %s" % (len(not_built), 1184 ', '.join(not_built))) 1185 1186 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1187 (board_dict, err_lines, err_line_boards, warn_lines, 1188 warn_line_boards, config) = self.GetResultSummary( 1189 board_selected, commit_upto, 1190 read_func_sizes=self._show_bloat, 1191 read_config=self._show_config) 1192 if commits: 1193 msg = '%02d: %s' % (commit_upto + 1, 1194 commits[commit_upto].subject) 1195 Print(msg, colour=self.col.BLUE) 1196 self.PrintResultSummary(board_selected, board_dict, 1197 err_lines if self._show_errors else [], err_line_boards, 1198 warn_lines if self._show_errors else [], warn_line_boards, 1199 config, self._show_sizes, self._show_detail, 1200 self._show_bloat, self._show_config) 1201 1202 def ShowSummary(self, commits, board_selected): 1203 """Show a build summary for U-Boot for a given board list. 1204 1205 Reset the result summary, then repeatedly call GetResultSummary on 1206 each commit's results, then display the differences we see. 1207 1208 Args: 1209 commit: Commit objects to summarise 1210 board_selected: Dict containing boards to summarise 1211 """ 1212 self.commit_count = len(commits) if commits else 1 1213 self.commits = commits 1214 self.ResetResultSummary(board_selected) 1215 self._error_lines = 0 1216 1217 for commit_upto in range(0, self.commit_count, self._step): 1218 self.ProduceResultSummary(commit_upto, commits, board_selected) 1219 if not self._error_lines: 1220 Print('(no errors to report)', colour=self.col.GREEN) 1221 1222 1223 def SetupBuild(self, board_selected, commits): 1224 """Set up ready to start a build. 1225 1226 Args: 1227 board_selected: Selected boards to build 1228 commits: Selected commits to build 1229 """ 1230 # First work out how many commits we will build 1231 count = (self.commit_count + self._step - 1) / self._step 1232 self.count = len(board_selected) * count 1233 self.upto = self.warned = self.fail = 0 1234 self._timestamps = collections.deque() 1235 1236 def GetThreadDir(self, thread_num): 1237 """Get the directory path to the working dir for a thread. 1238 1239 Args: 1240 thread_num: Number of thread to check. 1241 """ 1242 return os.path.join(self._working_dir, '%02d' % thread_num) 1243 1244 def _PrepareThread(self, thread_num, setup_git): 1245 """Prepare the working directory for a thread. 1246 1247 This clones or fetches the repo into the thread's work directory. 1248 1249 Args: 1250 thread_num: Thread number (0, 1, ...) 1251 setup_git: True to set up a git repo clone 1252 """ 1253 thread_dir = self.GetThreadDir(thread_num) 1254 builderthread.Mkdir(thread_dir) 1255 git_dir = os.path.join(thread_dir, '.git') 1256 1257 # Clone the repo if it doesn't already exist 1258 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1259 # we have a private index but uses the origin repo's contents? 1260 if setup_git and self.git_dir: 1261 src_dir = os.path.abspath(self.git_dir) 1262 if os.path.exists(git_dir): 1263 gitutil.Fetch(git_dir, thread_dir) 1264 else: 1265 Print('Cloning repo for thread %d' % thread_num) 1266 gitutil.Clone(src_dir, thread_dir) 1267 1268 def _PrepareWorkingSpace(self, max_threads, setup_git): 1269 """Prepare the working directory for use. 1270 1271 Set up the git repo for each thread. 1272 1273 Args: 1274 max_threads: Maximum number of threads we expect to need. 1275 setup_git: True to set up a git repo clone 1276 """ 1277 builderthread.Mkdir(self._working_dir) 1278 for thread in range(max_threads): 1279 self._PrepareThread(thread, setup_git) 1280 1281 def _PrepareOutputSpace(self): 1282 """Get the output directories ready to receive files. 1283 1284 We delete any output directories which look like ones we need to 1285 create. Having left over directories is confusing when the user wants 1286 to check the output manually. 1287 """ 1288 if not self.commits: 1289 return 1290 dir_list = [] 1291 for commit_upto in range(self.commit_count): 1292 dir_list.append(self._GetOutputDir(commit_upto)) 1293 1294 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1295 if dirname not in dir_list: 1296 shutil.rmtree(dirname) 1297 1298 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1299 """Build all commits for a list of boards 1300 1301 Args: 1302 commits: List of commits to be build, each a Commit object 1303 boards_selected: Dict of selected boards, key is target name, 1304 value is Board object 1305 keep_outputs: True to save build output files 1306 verbose: Display build results as they are completed 1307 Returns: 1308 Tuple containing: 1309 - number of boards that failed to build 1310 - number of boards that issued warnings 1311 """ 1312 self.commit_count = len(commits) if commits else 1 1313 self.commits = commits 1314 self._verbose = verbose 1315 1316 self.ResetResultSummary(board_selected) 1317 builderthread.Mkdir(self.base_dir, parents = True) 1318 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1319 commits is not None) 1320 self._PrepareOutputSpace() 1321 self.SetupBuild(board_selected, commits) 1322 self.ProcessResult(None) 1323 1324 # Create jobs to build all commits for each board 1325 for brd in board_selected.itervalues(): 1326 job = builderthread.BuilderJob() 1327 job.board = brd 1328 job.commits = commits 1329 job.keep_outputs = keep_outputs 1330 job.step = self._step 1331 self.queue.put(job) 1332 1333 # Wait until all jobs are started 1334 self.queue.join() 1335 1336 # Wait until we have processed all output 1337 self.out_queue.join() 1338 Print() 1339 self.ClearLine(0) 1340 return (self.fail, self.warned) 1341