1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99CONFIG_FILENAMES = [ 100 '.config', '.config-spl', '.config-tpl', 101 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 102 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 104] 105 106class Config: 107 """Holds information about configuration settings for a board.""" 108 def __init__(self, target): 109 self.target = target 110 self.config = {} 111 for fname in CONFIG_FILENAMES: 112 self.config[fname] = {} 113 114 def Add(self, fname, key, value): 115 self.config[fname][key] = value 116 117 def __hash__(self): 118 val = 0 119 for fname in self.config: 120 for key, value in self.config[fname].iteritems(): 121 print key, value 122 val = val ^ hash(key) & hash(value) 123 return val 124 125class Builder: 126 """Class for building U-Boot for a particular commit. 127 128 Public members: (many should ->private) 129 active: True if the builder is active and has not been stopped 130 already_done: Number of builds already completed 131 base_dir: Base directory to use for builder 132 checkout: True to check out source, False to skip that step. 133 This is used for testing. 134 col: terminal.Color() object 135 count: Number of commits to build 136 do_make: Method to call to invoke Make 137 fail: Number of builds that failed due to error 138 force_build: Force building even if a build already exists 139 force_config_on_failure: If a commit fails for a board, disable 140 incremental building for the next commit we build for that 141 board, so that we will see all warnings/errors again. 142 force_build_failures: If a previously-built build (i.e. built on 143 a previous run of buildman) is marked as failed, rebuild it. 144 git_dir: Git directory containing source repository 145 last_line_len: Length of the last line we printed (used for erasing 146 it with new progress information) 147 num_jobs: Number of jobs to run at once (passed to make as -j) 148 num_threads: Number of builder threads to run 149 out_queue: Queue of results to process 150 re_make_err: Compiled regular expression for ignore_lines 151 queue: Queue of jobs to run 152 threads: List of active threads 153 toolchains: Toolchains object to use for building 154 upto: Current commit number we are building (0.count-1) 155 warned: Number of builds that produced at least one warning 156 force_reconfig: Reconfigure U-Boot on each comiit. This disables 157 incremental building, where buildman reconfigures on the first 158 commit for a baord, and then just does an incremental build for 159 the following commits. In fact buildman will reconfigure and 160 retry for any failing commits, so generally the only effect of 161 this option is to slow things down. 162 in_tree: Build U-Boot in-tree instead of specifying an output 163 directory separate from the source code. This option is really 164 only useful for testing in-tree builds. 165 166 Private members: 167 _base_board_dict: Last-summarised Dict of boards 168 _base_err_lines: Last-summarised list of errors 169 _base_warn_lines: Last-summarised list of warnings 170 _build_period_us: Time taken for a single build (float object). 171 _complete_delay: Expected delay until completion (timedelta) 172 _next_delay_update: Next time we plan to display a progress update 173 (datatime) 174 _show_unknown: Show unknown boards (those not built) in summary 175 _timestamps: List of timestamps for the completion of the last 176 last _timestamp_count builds. Each is a datetime object. 177 _timestamp_count: Number of timestamps to keep in our list. 178 _working_dir: Base working directory containing all threads 179 """ 180 class Outcome: 181 """Records a build outcome for a single make invocation 182 183 Public Members: 184 rc: Outcome value (OUTCOME_...) 185 err_lines: List of error lines or [] if none 186 sizes: Dictionary of image size information, keyed by filename 187 - Each value is itself a dictionary containing 188 values for 'text', 'data' and 'bss', being the integer 189 size in bytes of each section. 190 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 191 value is itself a dictionary: 192 key: function name 193 value: Size of function in bytes 194 config: Dictionary keyed by filename - e.g. '.config'. Each 195 value is itself a dictionary: 196 key: config name 197 value: config value 198 """ 199 def __init__(self, rc, err_lines, sizes, func_sizes, config): 200 self.rc = rc 201 self.err_lines = err_lines 202 self.sizes = sizes 203 self.func_sizes = func_sizes 204 self.config = config 205 206 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 207 gnu_make='make', checkout=True, show_unknown=True, step=1, 208 no_subdirs=False, full_path=False, verbose_build=False, 209 incremental=False, per_board_out_dir=False): 210 """Create a new Builder object 211 212 Args: 213 toolchains: Toolchains object to use for building 214 base_dir: Base directory to use for builder 215 git_dir: Git directory containing source repository 216 num_threads: Number of builder threads to run 217 num_jobs: Number of jobs to run at once (passed to make as -j) 218 gnu_make: the command name of GNU Make. 219 checkout: True to check out source, False to skip that step. 220 This is used for testing. 221 show_unknown: Show unknown boards (those not built) in summary 222 step: 1 to process every commit, n to process every nth commit 223 no_subdirs: Don't create subdirectories when building current 224 source for a single board 225 full_path: Return the full path in CROSS_COMPILE and don't set 226 PATH 227 verbose_build: Run build with V=1 and don't use 'make -s' 228 incremental: Always perform incremental builds; don't run make 229 mrproper when configuring 230 per_board_out_dir: Build in a separate persistent directory per 231 board rather than a thread-specific directory 232 """ 233 self.toolchains = toolchains 234 self.base_dir = base_dir 235 self._working_dir = os.path.join(base_dir, '.bm-work') 236 self.threads = [] 237 self.active = True 238 self.do_make = self.Make 239 self.gnu_make = gnu_make 240 self.checkout = checkout 241 self.num_threads = num_threads 242 self.num_jobs = num_jobs 243 self.already_done = 0 244 self.force_build = False 245 self.git_dir = git_dir 246 self._show_unknown = show_unknown 247 self._timestamp_count = 10 248 self._build_period_us = None 249 self._complete_delay = None 250 self._next_delay_update = datetime.now() 251 self.force_config_on_failure = True 252 self.force_build_failures = False 253 self.force_reconfig = False 254 self._step = step 255 self.in_tree = False 256 self._error_lines = 0 257 self.no_subdirs = no_subdirs 258 self.full_path = full_path 259 self.verbose_build = verbose_build 260 261 self.col = terminal.Color() 262 263 self._re_function = re.compile('(.*): In function.*') 264 self._re_files = re.compile('In file included from.*') 265 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 266 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 267 268 self.queue = Queue.Queue() 269 self.out_queue = Queue.Queue() 270 for i in range(self.num_threads): 271 t = builderthread.BuilderThread(self, i, incremental, 272 per_board_out_dir) 273 t.setDaemon(True) 274 t.start() 275 self.threads.append(t) 276 277 self.last_line_len = 0 278 t = builderthread.ResultThread(self) 279 t.setDaemon(True) 280 t.start() 281 self.threads.append(t) 282 283 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 284 self.re_make_err = re.compile('|'.join(ignore_lines)) 285 286 def __del__(self): 287 """Get rid of all threads created by the builder""" 288 for t in self.threads: 289 del t 290 291 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 292 show_detail=False, show_bloat=False, 293 list_error_boards=False, show_config=False): 294 """Setup display options for the builder. 295 296 show_errors: True to show summarised error/warning info 297 show_sizes: Show size deltas 298 show_detail: Show detail for each board 299 show_bloat: Show detail for each function 300 list_error_boards: Show the boards which caused each error/warning 301 show_config: Show config deltas 302 """ 303 self._show_errors = show_errors 304 self._show_sizes = show_sizes 305 self._show_detail = show_detail 306 self._show_bloat = show_bloat 307 self._list_error_boards = list_error_boards 308 self._show_config = show_config 309 310 def _AddTimestamp(self): 311 """Add a new timestamp to the list and record the build period. 312 313 The build period is the length of time taken to perform a single 314 build (one board, one commit). 315 """ 316 now = datetime.now() 317 self._timestamps.append(now) 318 count = len(self._timestamps) 319 delta = self._timestamps[-1] - self._timestamps[0] 320 seconds = delta.total_seconds() 321 322 # If we have enough data, estimate build period (time taken for a 323 # single build) and therefore completion time. 324 if count > 1 and self._next_delay_update < now: 325 self._next_delay_update = now + timedelta(seconds=2) 326 if seconds > 0: 327 self._build_period = float(seconds) / count 328 todo = self.count - self.upto 329 self._complete_delay = timedelta(microseconds= 330 self._build_period * todo * 1000000) 331 # Round it 332 self._complete_delay -= timedelta( 333 microseconds=self._complete_delay.microseconds) 334 335 if seconds > 60: 336 self._timestamps.popleft() 337 count -= 1 338 339 def ClearLine(self, length): 340 """Clear any characters on the current line 341 342 Make way for a new line of length 'length', by outputting enough 343 spaces to clear out the old line. Then remember the new length for 344 next time. 345 346 Args: 347 length: Length of new line, in characters 348 """ 349 if length < self.last_line_len: 350 Print(' ' * (self.last_line_len - length), newline=False) 351 Print('\r', newline=False) 352 self.last_line_len = length 353 sys.stdout.flush() 354 355 def SelectCommit(self, commit, checkout=True): 356 """Checkout the selected commit for this build 357 """ 358 self.commit = commit 359 if checkout and self.checkout: 360 gitutil.Checkout(commit.hash) 361 362 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 363 """Run make 364 365 Args: 366 commit: Commit object that is being built 367 brd: Board object that is being built 368 stage: Stage that we are at (mrproper, config, build) 369 cwd: Directory where make should be run 370 args: Arguments to pass to make 371 kwargs: Arguments to pass to command.RunPipe() 372 """ 373 cmd = [self.gnu_make] + list(args) 374 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 375 cwd=cwd, raise_on_error=False, **kwargs) 376 if self.verbose_build: 377 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 378 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 379 return result 380 381 def ProcessResult(self, result): 382 """Process the result of a build, showing progress information 383 384 Args: 385 result: A CommandResult object, which indicates the result for 386 a single build 387 """ 388 col = terminal.Color() 389 if result: 390 target = result.brd.target 391 392 if result.return_code < 0: 393 self.active = False 394 command.StopAll() 395 return 396 397 self.upto += 1 398 if result.return_code != 0: 399 self.fail += 1 400 elif result.stderr: 401 self.warned += 1 402 if result.already_done: 403 self.already_done += 1 404 if self._verbose: 405 Print('\r', newline=False) 406 self.ClearLine(0) 407 boards_selected = {target : result.brd} 408 self.ResetResultSummary(boards_selected) 409 self.ProduceResultSummary(result.commit_upto, self.commits, 410 boards_selected) 411 else: 412 target = '(starting)' 413 414 # Display separate counts for ok, warned and fail 415 ok = self.upto - self.warned - self.fail 416 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 417 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 418 line += self.col.Color(self.col.RED, '%5d' % self.fail) 419 420 name = ' /%-5d ' % self.count 421 422 # Add our current completion time estimate 423 self._AddTimestamp() 424 if self._complete_delay: 425 name += '%s : ' % self._complete_delay 426 # When building all boards for a commit, we can print a commit 427 # progress message. 428 if result and result.commit_upto is None: 429 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 430 self.commit_count) 431 432 name += target 433 Print(line + name, newline=False) 434 length = 14 + len(name) 435 self.ClearLine(length) 436 437 def _GetOutputDir(self, commit_upto): 438 """Get the name of the output directory for a commit number 439 440 The output directory is typically .../<branch>/<commit>. 441 442 Args: 443 commit_upto: Commit number to use (0..self.count-1) 444 """ 445 commit_dir = None 446 if self.commits: 447 commit = self.commits[commit_upto] 448 subject = commit.subject.translate(trans_valid_chars) 449 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 450 self.commit_count, commit.hash, subject[:20])) 451 elif not self.no_subdirs: 452 commit_dir = 'current' 453 if not commit_dir: 454 return self.base_dir 455 return os.path.join(self.base_dir, commit_dir) 456 457 def GetBuildDir(self, commit_upto, target): 458 """Get the name of the build directory for a commit number 459 460 The build directory is typically .../<branch>/<commit>/<target>. 461 462 Args: 463 commit_upto: Commit number to use (0..self.count-1) 464 target: Target name 465 """ 466 output_dir = self._GetOutputDir(commit_upto) 467 return os.path.join(output_dir, target) 468 469 def GetDoneFile(self, commit_upto, target): 470 """Get the name of the done file for a commit number 471 472 Args: 473 commit_upto: Commit number to use (0..self.count-1) 474 target: Target name 475 """ 476 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 477 478 def GetSizesFile(self, commit_upto, target): 479 """Get the name of the sizes file for a commit number 480 481 Args: 482 commit_upto: Commit number to use (0..self.count-1) 483 target: Target name 484 """ 485 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 486 487 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 488 """Get the name of the funcsizes file for a commit number and ELF file 489 490 Args: 491 commit_upto: Commit number to use (0..self.count-1) 492 target: Target name 493 elf_fname: Filename of elf image 494 """ 495 return os.path.join(self.GetBuildDir(commit_upto, target), 496 '%s.sizes' % elf_fname.replace('/', '-')) 497 498 def GetObjdumpFile(self, commit_upto, target, elf_fname): 499 """Get the name of the objdump file for a commit number and ELF file 500 501 Args: 502 commit_upto: Commit number to use (0..self.count-1) 503 target: Target name 504 elf_fname: Filename of elf image 505 """ 506 return os.path.join(self.GetBuildDir(commit_upto, target), 507 '%s.objdump' % elf_fname.replace('/', '-')) 508 509 def GetErrFile(self, commit_upto, target): 510 """Get the name of the err file for a commit number 511 512 Args: 513 commit_upto: Commit number to use (0..self.count-1) 514 target: Target name 515 """ 516 output_dir = self.GetBuildDir(commit_upto, target) 517 return os.path.join(output_dir, 'err') 518 519 def FilterErrors(self, lines): 520 """Filter out errors in which we have no interest 521 522 We should probably use map(). 523 524 Args: 525 lines: List of error lines, each a string 526 Returns: 527 New list with only interesting lines included 528 """ 529 out_lines = [] 530 for line in lines: 531 if not self.re_make_err.search(line): 532 out_lines.append(line) 533 return out_lines 534 535 def ReadFuncSizes(self, fname, fd): 536 """Read function sizes from the output of 'nm' 537 538 Args: 539 fd: File containing data to read 540 fname: Filename we are reading from (just for errors) 541 542 Returns: 543 Dictionary containing size of each function in bytes, indexed by 544 function name. 545 """ 546 sym = {} 547 for line in fd.readlines(): 548 try: 549 size, type, name = line[:-1].split() 550 except: 551 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 552 continue 553 if type in 'tTdDbB': 554 # function names begin with '.' on 64-bit powerpc 555 if '.' in name[1:]: 556 name = 'static.' + name.split('.')[0] 557 sym[name] = sym.get(name, 0) + int(size, 16) 558 return sym 559 560 def _ProcessConfig(self, fname): 561 """Read in a .config, autoconf.mk or autoconf.h file 562 563 This function handles all config file types. It ignores comments and 564 any #defines which don't start with CONFIG_. 565 566 Args: 567 fname: Filename to read 568 569 Returns: 570 Dictionary: 571 key: Config name (e.g. CONFIG_DM) 572 value: Config value (e.g. 1) 573 """ 574 config = {} 575 if os.path.exists(fname): 576 with open(fname) as fd: 577 for line in fd: 578 line = line.strip() 579 if line.startswith('#define'): 580 values = line[8:].split(' ', 1) 581 if len(values) > 1: 582 key, value = values 583 else: 584 key = values[0] 585 value = '' 586 if not key.startswith('CONFIG_'): 587 continue 588 elif not line or line[0] in ['#', '*', '/']: 589 continue 590 else: 591 key, value = line.split('=', 1) 592 config[key] = value 593 return config 594 595 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 596 read_config): 597 """Work out the outcome of a build. 598 599 Args: 600 commit_upto: Commit number to check (0..n-1) 601 target: Target board to check 602 read_func_sizes: True to read function size information 603 read_config: True to read .config and autoconf.h files 604 605 Returns: 606 Outcome object 607 """ 608 done_file = self.GetDoneFile(commit_upto, target) 609 sizes_file = self.GetSizesFile(commit_upto, target) 610 sizes = {} 611 func_sizes = {} 612 config = {} 613 if os.path.exists(done_file): 614 with open(done_file, 'r') as fd: 615 return_code = int(fd.readline()) 616 err_lines = [] 617 err_file = self.GetErrFile(commit_upto, target) 618 if os.path.exists(err_file): 619 with open(err_file, 'r') as fd: 620 err_lines = self.FilterErrors(fd.readlines()) 621 622 # Decide whether the build was ok, failed or created warnings 623 if return_code: 624 rc = OUTCOME_ERROR 625 elif len(err_lines): 626 rc = OUTCOME_WARNING 627 else: 628 rc = OUTCOME_OK 629 630 # Convert size information to our simple format 631 if os.path.exists(sizes_file): 632 with open(sizes_file, 'r') as fd: 633 for line in fd.readlines(): 634 values = line.split() 635 rodata = 0 636 if len(values) > 6: 637 rodata = int(values[6], 16) 638 size_dict = { 639 'all' : int(values[0]) + int(values[1]) + 640 int(values[2]), 641 'text' : int(values[0]) - rodata, 642 'data' : int(values[1]), 643 'bss' : int(values[2]), 644 'rodata' : rodata, 645 } 646 sizes[values[5]] = size_dict 647 648 if read_func_sizes: 649 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 650 for fname in glob.glob(pattern): 651 with open(fname, 'r') as fd: 652 dict_name = os.path.basename(fname).replace('.sizes', 653 '') 654 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 655 656 if read_config: 657 output_dir = self.GetBuildDir(commit_upto, target) 658 for name in CONFIG_FILENAMES: 659 fname = os.path.join(output_dir, name) 660 config[name] = self._ProcessConfig(fname) 661 662 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 663 664 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 665 666 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 667 read_config): 668 """Calculate a summary of the results of building a commit. 669 670 Args: 671 board_selected: Dict containing boards to summarise 672 commit_upto: Commit number to summarize (0..self.count-1) 673 read_func_sizes: True to read function size information 674 read_config: True to read .config and autoconf.h files 675 676 Returns: 677 Tuple: 678 Dict containing boards which passed building this commit. 679 keyed by board.target 680 List containing a summary of error lines 681 Dict keyed by error line, containing a list of the Board 682 objects with that error 683 List containing a summary of warning lines 684 Dict keyed by error line, containing a list of the Board 685 objects with that warning 686 Dictionary keyed by board.target. Each value is a dictionary: 687 key: filename - e.g. '.config' 688 value is itself a dictionary: 689 key: config name 690 value: config value 691 """ 692 def AddLine(lines_summary, lines_boards, line, board): 693 line = line.rstrip() 694 if line in lines_boards: 695 lines_boards[line].append(board) 696 else: 697 lines_boards[line] = [board] 698 lines_summary.append(line) 699 700 board_dict = {} 701 err_lines_summary = [] 702 err_lines_boards = {} 703 warn_lines_summary = [] 704 warn_lines_boards = {} 705 config = {} 706 707 for board in boards_selected.itervalues(): 708 outcome = self.GetBuildOutcome(commit_upto, board.target, 709 read_func_sizes, read_config) 710 board_dict[board.target] = outcome 711 last_func = None 712 last_was_warning = False 713 for line in outcome.err_lines: 714 if line: 715 if (self._re_function.match(line) or 716 self._re_files.match(line)): 717 last_func = line 718 else: 719 is_warning = self._re_warning.match(line) 720 is_note = self._re_note.match(line) 721 if is_warning or (last_was_warning and is_note): 722 if last_func: 723 AddLine(warn_lines_summary, warn_lines_boards, 724 last_func, board) 725 AddLine(warn_lines_summary, warn_lines_boards, 726 line, board) 727 else: 728 if last_func: 729 AddLine(err_lines_summary, err_lines_boards, 730 last_func, board) 731 AddLine(err_lines_summary, err_lines_boards, 732 line, board) 733 last_was_warning = is_warning 734 last_func = None 735 tconfig = Config(board.target) 736 for fname in CONFIG_FILENAMES: 737 if outcome.config: 738 for key, value in outcome.config[fname].iteritems(): 739 tconfig.Add(fname, key, value) 740 config[board.target] = tconfig 741 742 return (board_dict, err_lines_summary, err_lines_boards, 743 warn_lines_summary, warn_lines_boards, config) 744 745 def AddOutcome(self, board_dict, arch_list, changes, char, color): 746 """Add an output to our list of outcomes for each architecture 747 748 This simple function adds failing boards (changes) to the 749 relevant architecture string, so we can print the results out 750 sorted by architecture. 751 752 Args: 753 board_dict: Dict containing all boards 754 arch_list: Dict keyed by arch name. Value is a string containing 755 a list of board names which failed for that arch. 756 changes: List of boards to add to arch_list 757 color: terminal.Colour object 758 """ 759 done_arch = {} 760 for target in changes: 761 if target in board_dict: 762 arch = board_dict[target].arch 763 else: 764 arch = 'unknown' 765 str = self.col.Color(color, ' ' + target) 766 if not arch in done_arch: 767 str = ' %s %s' % (self.col.Color(color, char), str) 768 done_arch[arch] = True 769 if not arch in arch_list: 770 arch_list[arch] = str 771 else: 772 arch_list[arch] += str 773 774 775 def ColourNum(self, num): 776 color = self.col.RED if num > 0 else self.col.GREEN 777 if num == 0: 778 return '0' 779 return self.col.Color(color, str(num)) 780 781 def ResetResultSummary(self, board_selected): 782 """Reset the results summary ready for use. 783 784 Set up the base board list to be all those selected, and set the 785 error lines to empty. 786 787 Following this, calls to PrintResultSummary() will use this 788 information to work out what has changed. 789 790 Args: 791 board_selected: Dict containing boards to summarise, keyed by 792 board.target 793 """ 794 self._base_board_dict = {} 795 for board in board_selected: 796 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 797 self._base_err_lines = [] 798 self._base_warn_lines = [] 799 self._base_err_line_boards = {} 800 self._base_warn_line_boards = {} 801 self._base_config = None 802 803 def PrintFuncSizeDetail(self, fname, old, new): 804 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 805 delta, common = [], {} 806 807 for a in old: 808 if a in new: 809 common[a] = 1 810 811 for name in old: 812 if name not in common: 813 remove += 1 814 down += old[name] 815 delta.append([-old[name], name]) 816 817 for name in new: 818 if name not in common: 819 add += 1 820 up += new[name] 821 delta.append([new[name], name]) 822 823 for name in common: 824 diff = new.get(name, 0) - old.get(name, 0) 825 if diff > 0: 826 grow, up = grow + 1, up + diff 827 elif diff < 0: 828 shrink, down = shrink + 1, down - diff 829 delta.append([diff, name]) 830 831 delta.sort() 832 delta.reverse() 833 834 args = [add, -remove, grow, -shrink, up, -down, up - down] 835 if max(args) == 0: 836 return 837 args = [self.ColourNum(x) for x in args] 838 indent = ' ' * 15 839 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 840 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 841 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 842 'delta')) 843 for diff, name in delta: 844 if diff: 845 color = self.col.RED if diff > 0 else self.col.GREEN 846 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 847 old.get(name, '-'), new.get(name,'-'), diff) 848 Print(msg, colour=color) 849 850 851 def PrintSizeDetail(self, target_list, show_bloat): 852 """Show details size information for each board 853 854 Args: 855 target_list: List of targets, each a dict containing: 856 'target': Target name 857 'total_diff': Total difference in bytes across all areas 858 <part_name>: Difference for that part 859 show_bloat: Show detail for each function 860 """ 861 targets_by_diff = sorted(target_list, reverse=True, 862 key=lambda x: x['_total_diff']) 863 for result in targets_by_diff: 864 printed_target = False 865 for name in sorted(result): 866 diff = result[name] 867 if name.startswith('_'): 868 continue 869 if diff != 0: 870 color = self.col.RED if diff > 0 else self.col.GREEN 871 msg = ' %s %+d' % (name, diff) 872 if not printed_target: 873 Print('%10s %-15s:' % ('', result['_target']), 874 newline=False) 875 printed_target = True 876 Print(msg, colour=color, newline=False) 877 if printed_target: 878 Print() 879 if show_bloat: 880 target = result['_target'] 881 outcome = result['_outcome'] 882 base_outcome = self._base_board_dict[target] 883 for fname in outcome.func_sizes: 884 self.PrintFuncSizeDetail(fname, 885 base_outcome.func_sizes[fname], 886 outcome.func_sizes[fname]) 887 888 889 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 890 show_bloat): 891 """Print a summary of image sizes broken down by section. 892 893 The summary takes the form of one line per architecture. The 894 line contains deltas for each of the sections (+ means the section 895 got bigger, - means smaller). The nunmbers are the average number 896 of bytes that a board in this section increased by. 897 898 For example: 899 powerpc: (622 boards) text -0.0 900 arm: (285 boards) text -0.0 901 nds32: (3 boards) text -8.0 902 903 Args: 904 board_selected: Dict containing boards to summarise, keyed by 905 board.target 906 board_dict: Dict containing boards for which we built this 907 commit, keyed by board.target. The value is an Outcome object. 908 show_detail: Show detail for each board 909 show_bloat: Show detail for each function 910 """ 911 arch_list = {} 912 arch_count = {} 913 914 # Calculate changes in size for different image parts 915 # The previous sizes are in Board.sizes, for each board 916 for target in board_dict: 917 if target not in board_selected: 918 continue 919 base_sizes = self._base_board_dict[target].sizes 920 outcome = board_dict[target] 921 sizes = outcome.sizes 922 923 # Loop through the list of images, creating a dict of size 924 # changes for each image/part. We end up with something like 925 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 926 # which means that U-Boot data increased by 5 bytes and SPL 927 # text decreased by 4. 928 err = {'_target' : target} 929 for image in sizes: 930 if image in base_sizes: 931 base_image = base_sizes[image] 932 # Loop through the text, data, bss parts 933 for part in sorted(sizes[image]): 934 diff = sizes[image][part] - base_image[part] 935 col = None 936 if diff: 937 if image == 'u-boot': 938 name = part 939 else: 940 name = image + ':' + part 941 err[name] = diff 942 arch = board_selected[target].arch 943 if not arch in arch_count: 944 arch_count[arch] = 1 945 else: 946 arch_count[arch] += 1 947 if not sizes: 948 pass # Only add to our list when we have some stats 949 elif not arch in arch_list: 950 arch_list[arch] = [err] 951 else: 952 arch_list[arch].append(err) 953 954 # We now have a list of image size changes sorted by arch 955 # Print out a summary of these 956 for arch, target_list in arch_list.iteritems(): 957 # Get total difference for each type 958 totals = {} 959 for result in target_list: 960 total = 0 961 for name, diff in result.iteritems(): 962 if name.startswith('_'): 963 continue 964 total += diff 965 if name in totals: 966 totals[name] += diff 967 else: 968 totals[name] = diff 969 result['_total_diff'] = total 970 result['_outcome'] = board_dict[result['_target']] 971 972 count = len(target_list) 973 printed_arch = False 974 for name in sorted(totals): 975 diff = totals[name] 976 if diff: 977 # Display the average difference in this name for this 978 # architecture 979 avg_diff = float(diff) / count 980 color = self.col.RED if avg_diff > 0 else self.col.GREEN 981 msg = ' %s %+1.1f' % (name, avg_diff) 982 if not printed_arch: 983 Print('%10s: (for %d/%d boards)' % (arch, count, 984 arch_count[arch]), newline=False) 985 printed_arch = True 986 Print(msg, colour=color, newline=False) 987 988 if printed_arch: 989 Print() 990 if show_detail: 991 self.PrintSizeDetail(target_list, show_bloat) 992 993 994 def PrintResultSummary(self, board_selected, board_dict, err_lines, 995 err_line_boards, warn_lines, warn_line_boards, 996 config, show_sizes, show_detail, show_bloat, 997 show_config): 998 """Compare results with the base results and display delta. 999 1000 Only boards mentioned in board_selected will be considered. This 1001 function is intended to be called repeatedly with the results of 1002 each commit. It therefore shows a 'diff' between what it saw in 1003 the last call and what it sees now. 1004 1005 Args: 1006 board_selected: Dict containing boards to summarise, keyed by 1007 board.target 1008 board_dict: Dict containing boards for which we built this 1009 commit, keyed by board.target. The value is an Outcome object. 1010 err_lines: A list of errors for this commit, or [] if there is 1011 none, or we don't want to print errors 1012 err_line_boards: Dict keyed by error line, containing a list of 1013 the Board objects with that error 1014 warn_lines: A list of warnings for this commit, or [] if there is 1015 none, or we don't want to print errors 1016 warn_line_boards: Dict keyed by warning line, containing a list of 1017 the Board objects with that warning 1018 config: Dictionary keyed by filename - e.g. '.config'. Each 1019 value is itself a dictionary: 1020 key: config name 1021 value: config value 1022 show_sizes: Show image size deltas 1023 show_detail: Show detail for each board 1024 show_bloat: Show detail for each function 1025 show_config: Show config changes 1026 """ 1027 def _BoardList(line, line_boards): 1028 """Helper function to get a line of boards containing a line 1029 1030 Args: 1031 line: Error line to search for 1032 Return: 1033 String containing a list of boards with that error line, or 1034 '' if the user has not requested such a list 1035 """ 1036 if self._list_error_boards: 1037 names = [] 1038 for board in line_boards[line]: 1039 if not board.target in names: 1040 names.append(board.target) 1041 names_str = '(%s) ' % ','.join(names) 1042 else: 1043 names_str = '' 1044 return names_str 1045 1046 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1047 char): 1048 better_lines = [] 1049 worse_lines = [] 1050 for line in lines: 1051 if line not in base_lines: 1052 worse_lines.append(char + '+' + 1053 _BoardList(line, line_boards) + line) 1054 for line in base_lines: 1055 if line not in lines: 1056 better_lines.append(char + '-' + 1057 _BoardList(line, base_line_boards) + line) 1058 return better_lines, worse_lines 1059 1060 def _CalcConfig(delta, name, config): 1061 """Calculate configuration changes 1062 1063 Args: 1064 delta: Type of the delta, e.g. '+' 1065 name: name of the file which changed (e.g. .config) 1066 config: configuration change dictionary 1067 key: config name 1068 value: config value 1069 Returns: 1070 String containing the configuration changes which can be 1071 printed 1072 """ 1073 out = '' 1074 for key in sorted(config.keys()): 1075 out += '%s=%s ' % (key, config[key]) 1076 return '%s %s: %s' % (delta, name, out) 1077 1078 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1079 """Add changes in configuration to a list 1080 1081 Args: 1082 lines: list to add to 1083 name: config file name 1084 config_plus: configurations added, dictionary 1085 key: config name 1086 value: config value 1087 config_minus: configurations removed, dictionary 1088 key: config name 1089 value: config value 1090 config_change: configurations changed, dictionary 1091 key: config name 1092 value: config value 1093 """ 1094 if config_plus: 1095 lines.append(_CalcConfig('+', name, config_plus)) 1096 if config_minus: 1097 lines.append(_CalcConfig('-', name, config_minus)) 1098 if config_change: 1099 lines.append(_CalcConfig('c', name, config_change)) 1100 1101 def _OutputConfigInfo(lines): 1102 for line in lines: 1103 if not line: 1104 continue 1105 if line[0] == '+': 1106 col = self.col.GREEN 1107 elif line[0] == '-': 1108 col = self.col.RED 1109 elif line[0] == 'c': 1110 col = self.col.YELLOW 1111 Print(' ' + line, newline=True, colour=col) 1112 1113 1114 better = [] # List of boards fixed since last commit 1115 worse = [] # List of new broken boards since last commit 1116 new = [] # List of boards that didn't exist last time 1117 unknown = [] # List of boards that were not built 1118 1119 for target in board_dict: 1120 if target not in board_selected: 1121 continue 1122 1123 # If the board was built last time, add its outcome to a list 1124 if target in self._base_board_dict: 1125 base_outcome = self._base_board_dict[target].rc 1126 outcome = board_dict[target] 1127 if outcome.rc == OUTCOME_UNKNOWN: 1128 unknown.append(target) 1129 elif outcome.rc < base_outcome: 1130 better.append(target) 1131 elif outcome.rc > base_outcome: 1132 worse.append(target) 1133 else: 1134 new.append(target) 1135 1136 # Get a list of errors that have appeared, and disappeared 1137 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1138 self._base_err_line_boards, err_lines, err_line_boards, '') 1139 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1140 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1141 1142 # Display results by arch 1143 if (better or worse or unknown or new or worse_err or better_err 1144 or worse_warn or better_warn): 1145 arch_list = {} 1146 self.AddOutcome(board_selected, arch_list, better, '', 1147 self.col.GREEN) 1148 self.AddOutcome(board_selected, arch_list, worse, '+', 1149 self.col.RED) 1150 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1151 if self._show_unknown: 1152 self.AddOutcome(board_selected, arch_list, unknown, '?', 1153 self.col.MAGENTA) 1154 for arch, target_list in arch_list.iteritems(): 1155 Print('%10s: %s' % (arch, target_list)) 1156 self._error_lines += 1 1157 if better_err: 1158 Print('\n'.join(better_err), colour=self.col.GREEN) 1159 self._error_lines += 1 1160 if worse_err: 1161 Print('\n'.join(worse_err), colour=self.col.RED) 1162 self._error_lines += 1 1163 if better_warn: 1164 Print('\n'.join(better_warn), colour=self.col.CYAN) 1165 self._error_lines += 1 1166 if worse_warn: 1167 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1168 self._error_lines += 1 1169 1170 if show_sizes: 1171 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1172 show_bloat) 1173 1174 if show_config and self._base_config: 1175 summary = {} 1176 arch_config_plus = {} 1177 arch_config_minus = {} 1178 arch_config_change = {} 1179 arch_list = [] 1180 1181 for target in board_dict: 1182 if target not in board_selected: 1183 continue 1184 arch = board_selected[target].arch 1185 if arch not in arch_list: 1186 arch_list.append(arch) 1187 1188 for arch in arch_list: 1189 arch_config_plus[arch] = {} 1190 arch_config_minus[arch] = {} 1191 arch_config_change[arch] = {} 1192 for name in CONFIG_FILENAMES: 1193 arch_config_plus[arch][name] = {} 1194 arch_config_minus[arch][name] = {} 1195 arch_config_change[arch][name] = {} 1196 1197 for target in board_dict: 1198 if target not in board_selected: 1199 continue 1200 1201 arch = board_selected[target].arch 1202 1203 all_config_plus = {} 1204 all_config_minus = {} 1205 all_config_change = {} 1206 tbase = self._base_config[target] 1207 tconfig = config[target] 1208 lines = [] 1209 for name in CONFIG_FILENAMES: 1210 if not tconfig.config[name]: 1211 continue 1212 config_plus = {} 1213 config_minus = {} 1214 config_change = {} 1215 base = tbase.config[name] 1216 for key, value in tconfig.config[name].iteritems(): 1217 if key not in base: 1218 config_plus[key] = value 1219 all_config_plus[key] = value 1220 for key, value in base.iteritems(): 1221 if key not in tconfig.config[name]: 1222 config_minus[key] = value 1223 all_config_minus[key] = value 1224 for key, value in base.iteritems(): 1225 new_value = tconfig.config.get(key) 1226 if new_value and value != new_value: 1227 desc = '%s -> %s' % (value, new_value) 1228 config_change[key] = desc 1229 all_config_change[key] = desc 1230 1231 arch_config_plus[arch][name].update(config_plus) 1232 arch_config_minus[arch][name].update(config_minus) 1233 arch_config_change[arch][name].update(config_change) 1234 1235 _AddConfig(lines, name, config_plus, config_minus, 1236 config_change) 1237 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1238 all_config_change) 1239 summary[target] = '\n'.join(lines) 1240 1241 lines_by_target = {} 1242 for target, lines in summary.iteritems(): 1243 if lines in lines_by_target: 1244 lines_by_target[lines].append(target) 1245 else: 1246 lines_by_target[lines] = [target] 1247 1248 for arch in arch_list: 1249 lines = [] 1250 all_plus = {} 1251 all_minus = {} 1252 all_change = {} 1253 for name in CONFIG_FILENAMES: 1254 all_plus.update(arch_config_plus[arch][name]) 1255 all_minus.update(arch_config_minus[arch][name]) 1256 all_change.update(arch_config_change[arch][name]) 1257 _AddConfig(lines, name, arch_config_plus[arch][name], 1258 arch_config_minus[arch][name], 1259 arch_config_change[arch][name]) 1260 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1261 #arch_summary[target] = '\n'.join(lines) 1262 if lines: 1263 Print('%s:' % arch) 1264 _OutputConfigInfo(lines) 1265 1266 for lines, targets in lines_by_target.iteritems(): 1267 if not lines: 1268 continue 1269 Print('%s :' % ' '.join(sorted(targets))) 1270 _OutputConfigInfo(lines.split('\n')) 1271 1272 1273 # Save our updated information for the next call to this function 1274 self._base_board_dict = board_dict 1275 self._base_err_lines = err_lines 1276 self._base_warn_lines = warn_lines 1277 self._base_err_line_boards = err_line_boards 1278 self._base_warn_line_boards = warn_line_boards 1279 self._base_config = config 1280 1281 # Get a list of boards that did not get built, if needed 1282 not_built = [] 1283 for board in board_selected: 1284 if not board in board_dict: 1285 not_built.append(board) 1286 if not_built: 1287 Print("Boards not built (%d): %s" % (len(not_built), 1288 ', '.join(not_built))) 1289 1290 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1291 (board_dict, err_lines, err_line_boards, warn_lines, 1292 warn_line_boards, config) = self.GetResultSummary( 1293 board_selected, commit_upto, 1294 read_func_sizes=self._show_bloat, 1295 read_config=self._show_config) 1296 if commits: 1297 msg = '%02d: %s' % (commit_upto + 1, 1298 commits[commit_upto].subject) 1299 Print(msg, colour=self.col.BLUE) 1300 self.PrintResultSummary(board_selected, board_dict, 1301 err_lines if self._show_errors else [], err_line_boards, 1302 warn_lines if self._show_errors else [], warn_line_boards, 1303 config, self._show_sizes, self._show_detail, 1304 self._show_bloat, self._show_config) 1305 1306 def ShowSummary(self, commits, board_selected): 1307 """Show a build summary for U-Boot for a given board list. 1308 1309 Reset the result summary, then repeatedly call GetResultSummary on 1310 each commit's results, then display the differences we see. 1311 1312 Args: 1313 commit: Commit objects to summarise 1314 board_selected: Dict containing boards to summarise 1315 """ 1316 self.commit_count = len(commits) if commits else 1 1317 self.commits = commits 1318 self.ResetResultSummary(board_selected) 1319 self._error_lines = 0 1320 1321 for commit_upto in range(0, self.commit_count, self._step): 1322 self.ProduceResultSummary(commit_upto, commits, board_selected) 1323 if not self._error_lines: 1324 Print('(no errors to report)', colour=self.col.GREEN) 1325 1326 1327 def SetupBuild(self, board_selected, commits): 1328 """Set up ready to start a build. 1329 1330 Args: 1331 board_selected: Selected boards to build 1332 commits: Selected commits to build 1333 """ 1334 # First work out how many commits we will build 1335 count = (self.commit_count + self._step - 1) / self._step 1336 self.count = len(board_selected) * count 1337 self.upto = self.warned = self.fail = 0 1338 self._timestamps = collections.deque() 1339 1340 def GetThreadDir(self, thread_num): 1341 """Get the directory path to the working dir for a thread. 1342 1343 Args: 1344 thread_num: Number of thread to check. 1345 """ 1346 return os.path.join(self._working_dir, '%02d' % thread_num) 1347 1348 def _PrepareThread(self, thread_num, setup_git): 1349 """Prepare the working directory for a thread. 1350 1351 This clones or fetches the repo into the thread's work directory. 1352 1353 Args: 1354 thread_num: Thread number (0, 1, ...) 1355 setup_git: True to set up a git repo clone 1356 """ 1357 thread_dir = self.GetThreadDir(thread_num) 1358 builderthread.Mkdir(thread_dir) 1359 git_dir = os.path.join(thread_dir, '.git') 1360 1361 # Clone the repo if it doesn't already exist 1362 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1363 # we have a private index but uses the origin repo's contents? 1364 if setup_git and self.git_dir: 1365 src_dir = os.path.abspath(self.git_dir) 1366 if os.path.exists(git_dir): 1367 gitutil.Fetch(git_dir, thread_dir) 1368 else: 1369 Print('Cloning repo for thread %d' % thread_num) 1370 gitutil.Clone(src_dir, thread_dir) 1371 1372 def _PrepareWorkingSpace(self, max_threads, setup_git): 1373 """Prepare the working directory for use. 1374 1375 Set up the git repo for each thread. 1376 1377 Args: 1378 max_threads: Maximum number of threads we expect to need. 1379 setup_git: True to set up a git repo clone 1380 """ 1381 builderthread.Mkdir(self._working_dir) 1382 for thread in range(max_threads): 1383 self._PrepareThread(thread, setup_git) 1384 1385 def _PrepareOutputSpace(self): 1386 """Get the output directories ready to receive files. 1387 1388 We delete any output directories which look like ones we need to 1389 create. Having left over directories is confusing when the user wants 1390 to check the output manually. 1391 """ 1392 if not self.commits: 1393 return 1394 dir_list = [] 1395 for commit_upto in range(self.commit_count): 1396 dir_list.append(self._GetOutputDir(commit_upto)) 1397 1398 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1399 if dirname not in dir_list: 1400 shutil.rmtree(dirname) 1401 1402 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1403 """Build all commits for a list of boards 1404 1405 Args: 1406 commits: List of commits to be build, each a Commit object 1407 boards_selected: Dict of selected boards, key is target name, 1408 value is Board object 1409 keep_outputs: True to save build output files 1410 verbose: Display build results as they are completed 1411 Returns: 1412 Tuple containing: 1413 - number of boards that failed to build 1414 - number of boards that issued warnings 1415 """ 1416 self.commit_count = len(commits) if commits else 1 1417 self.commits = commits 1418 self._verbose = verbose 1419 1420 self.ResetResultSummary(board_selected) 1421 builderthread.Mkdir(self.base_dir, parents = True) 1422 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1423 commits is not None) 1424 self._PrepareOutputSpace() 1425 self.SetupBuild(board_selected, commits) 1426 self.ProcessResult(None) 1427 1428 # Create jobs to build all commits for each board 1429 for brd in board_selected.itervalues(): 1430 job = builderthread.BuilderJob() 1431 job.board = brd 1432 job.commits = commits 1433 job.keep_outputs = keep_outputs 1434 job.step = self._step 1435 self.queue.put(job) 1436 1437 # Wait until all jobs are started 1438 self.queue.join() 1439 1440 # Wait until we have processed all output 1441 self.out_queue.join() 1442 Print() 1443 self.ClearLine(0) 1444 return (self.fail, self.warned) 1445