1# Copyright (c) 2013 The Chromium OS Authors. 2# 3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 4# 5# SPDX-License-Identifier: GPL-2.0+ 6# 7 8import collections 9from datetime import datetime, timedelta 10import glob 11import os 12import re 13import Queue 14import shutil 15import string 16import sys 17import time 18 19import builderthread 20import command 21import gitutil 22import terminal 23from terminal import Print 24import toolchain 25 26 27""" 28Theory of Operation 29 30Please see README for user documentation, and you should be familiar with 31that before trying to make sense of this. 32 33Buildman works by keeping the machine as busy as possible, building different 34commits for different boards on multiple CPUs at once. 35 36The source repo (self.git_dir) contains all the commits to be built. Each 37thread works on a single board at a time. It checks out the first commit, 38configures it for that board, then builds it. Then it checks out the next 39commit and builds it (typically without re-configuring). When it runs out 40of commits, it gets another job from the builder and starts again with that 41board. 42 43Clearly the builder threads could work either way - they could check out a 44commit and then built it for all boards. Using separate directories for each 45commit/board pair they could leave their build product around afterwards 46also. 47 48The intent behind building a single board for multiple commits, is to make 49use of incremental builds. Since each commit is built incrementally from 50the previous one, builds are faster. Reconfiguring for a different board 51removes all intermediate object files. 52 53Many threads can be working at once, but each has its own working directory. 54When a thread finishes a build, it puts the output files into a result 55directory. 56 57The base directory used by buildman is normally '../<branch>', i.e. 58a directory higher than the source repository and named after the branch 59being built. 60 61Within the base directory, we have one subdirectory for each commit. Within 62that is one subdirectory for each board. Within that is the build output for 63that commit/board combination. 64 65Buildman also create working directories for each thread, in a .bm-work/ 66subdirectory in the base dir. 67 68As an example, say we are building branch 'us-net' for boards 'sandbox' and 69'seaboard', and say that us-net has two commits. We will have directories 70like this: 71 72us-net/ base directory 73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 74 sandbox/ 75 u-boot.bin 76 seaboard/ 77 u-boot.bin 78 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 79 sandbox/ 80 u-boot.bin 81 seaboard/ 82 u-boot.bin 83 .bm-work/ 84 00/ working directory for thread 0 (contains source checkout) 85 build/ build output 86 01/ working directory for thread 1 87 build/ build output 88 ... 89u-boot/ source directory 90 .git/ repository 91""" 92 93# Possible build outcomes 94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 95 96# Translate a commit subject into a valid filename 97trans_valid_chars = string.maketrans("/: ", "---") 98 99CONFIG_FILENAMES = [ 100 '.config', '.config-spl', '.config-tpl', 101 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 102 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 104] 105 106class Config: 107 """Holds information about configuration settings for a board.""" 108 def __init__(self, target): 109 self.target = target 110 self.config = {} 111 for fname in CONFIG_FILENAMES: 112 self.config[fname] = {} 113 114 def Add(self, fname, key, value): 115 self.config[fname][key] = value 116 117 def __hash__(self): 118 val = 0 119 for fname in self.config: 120 for key, value in self.config[fname].iteritems(): 121 print key, value 122 val = val ^ hash(key) & hash(value) 123 return val 124 125class Builder: 126 """Class for building U-Boot for a particular commit. 127 128 Public members: (many should ->private) 129 active: True if the builder is active and has not been stopped 130 already_done: Number of builds already completed 131 base_dir: Base directory to use for builder 132 checkout: True to check out source, False to skip that step. 133 This is used for testing. 134 col: terminal.Color() object 135 count: Number of commits to build 136 do_make: Method to call to invoke Make 137 fail: Number of builds that failed due to error 138 force_build: Force building even if a build already exists 139 force_config_on_failure: If a commit fails for a board, disable 140 incremental building for the next commit we build for that 141 board, so that we will see all warnings/errors again. 142 force_build_failures: If a previously-built build (i.e. built on 143 a previous run of buildman) is marked as failed, rebuild it. 144 git_dir: Git directory containing source repository 145 last_line_len: Length of the last line we printed (used for erasing 146 it with new progress information) 147 num_jobs: Number of jobs to run at once (passed to make as -j) 148 num_threads: Number of builder threads to run 149 out_queue: Queue of results to process 150 re_make_err: Compiled regular expression for ignore_lines 151 queue: Queue of jobs to run 152 threads: List of active threads 153 toolchains: Toolchains object to use for building 154 upto: Current commit number we are building (0.count-1) 155 warned: Number of builds that produced at least one warning 156 force_reconfig: Reconfigure U-Boot on each comiit. This disables 157 incremental building, where buildman reconfigures on the first 158 commit for a baord, and then just does an incremental build for 159 the following commits. In fact buildman will reconfigure and 160 retry for any failing commits, so generally the only effect of 161 this option is to slow things down. 162 in_tree: Build U-Boot in-tree instead of specifying an output 163 directory separate from the source code. This option is really 164 only useful for testing in-tree builds. 165 166 Private members: 167 _base_board_dict: Last-summarised Dict of boards 168 _base_err_lines: Last-summarised list of errors 169 _base_warn_lines: Last-summarised list of warnings 170 _build_period_us: Time taken for a single build (float object). 171 _complete_delay: Expected delay until completion (timedelta) 172 _next_delay_update: Next time we plan to display a progress update 173 (datatime) 174 _show_unknown: Show unknown boards (those not built) in summary 175 _timestamps: List of timestamps for the completion of the last 176 last _timestamp_count builds. Each is a datetime object. 177 _timestamp_count: Number of timestamps to keep in our list. 178 _working_dir: Base working directory containing all threads 179 """ 180 class Outcome: 181 """Records a build outcome for a single make invocation 182 183 Public Members: 184 rc: Outcome value (OUTCOME_...) 185 err_lines: List of error lines or [] if none 186 sizes: Dictionary of image size information, keyed by filename 187 - Each value is itself a dictionary containing 188 values for 'text', 'data' and 'bss', being the integer 189 size in bytes of each section. 190 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 191 value is itself a dictionary: 192 key: function name 193 value: Size of function in bytes 194 config: Dictionary keyed by filename - e.g. '.config'. Each 195 value is itself a dictionary: 196 key: config name 197 value: config value 198 """ 199 def __init__(self, rc, err_lines, sizes, func_sizes, config): 200 self.rc = rc 201 self.err_lines = err_lines 202 self.sizes = sizes 203 self.func_sizes = func_sizes 204 self.config = config 205 206 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 207 gnu_make='make', checkout=True, show_unknown=True, step=1, 208 no_subdirs=False, full_path=False, verbose_build=False): 209 """Create a new Builder object 210 211 Args: 212 toolchains: Toolchains object to use for building 213 base_dir: Base directory to use for builder 214 git_dir: Git directory containing source repository 215 num_threads: Number of builder threads to run 216 num_jobs: Number of jobs to run at once (passed to make as -j) 217 gnu_make: the command name of GNU Make. 218 checkout: True to check out source, False to skip that step. 219 This is used for testing. 220 show_unknown: Show unknown boards (those not built) in summary 221 step: 1 to process every commit, n to process every nth commit 222 no_subdirs: Don't create subdirectories when building current 223 source for a single board 224 full_path: Return the full path in CROSS_COMPILE and don't set 225 PATH 226 verbose_build: Run build with V=1 and don't use 'make -s' 227 """ 228 self.toolchains = toolchains 229 self.base_dir = base_dir 230 self._working_dir = os.path.join(base_dir, '.bm-work') 231 self.threads = [] 232 self.active = True 233 self.do_make = self.Make 234 self.gnu_make = gnu_make 235 self.checkout = checkout 236 self.num_threads = num_threads 237 self.num_jobs = num_jobs 238 self.already_done = 0 239 self.force_build = False 240 self.git_dir = git_dir 241 self._show_unknown = show_unknown 242 self._timestamp_count = 10 243 self._build_period_us = None 244 self._complete_delay = None 245 self._next_delay_update = datetime.now() 246 self.force_config_on_failure = True 247 self.force_build_failures = False 248 self.force_reconfig = False 249 self._step = step 250 self.in_tree = False 251 self._error_lines = 0 252 self.no_subdirs = no_subdirs 253 self.full_path = full_path 254 self.verbose_build = verbose_build 255 256 self.col = terminal.Color() 257 258 self._re_function = re.compile('(.*): In function.*') 259 self._re_files = re.compile('In file included from.*') 260 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 261 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 262 263 self.queue = Queue.Queue() 264 self.out_queue = Queue.Queue() 265 for i in range(self.num_threads): 266 t = builderthread.BuilderThread(self, i) 267 t.setDaemon(True) 268 t.start() 269 self.threads.append(t) 270 271 self.last_line_len = 0 272 t = builderthread.ResultThread(self) 273 t.setDaemon(True) 274 t.start() 275 self.threads.append(t) 276 277 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 278 self.re_make_err = re.compile('|'.join(ignore_lines)) 279 280 def __del__(self): 281 """Get rid of all threads created by the builder""" 282 for t in self.threads: 283 del t 284 285 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 286 show_detail=False, show_bloat=False, 287 list_error_boards=False, show_config=False): 288 """Setup display options for the builder. 289 290 show_errors: True to show summarised error/warning info 291 show_sizes: Show size deltas 292 show_detail: Show detail for each board 293 show_bloat: Show detail for each function 294 list_error_boards: Show the boards which caused each error/warning 295 show_config: Show config deltas 296 """ 297 self._show_errors = show_errors 298 self._show_sizes = show_sizes 299 self._show_detail = show_detail 300 self._show_bloat = show_bloat 301 self._list_error_boards = list_error_boards 302 self._show_config = show_config 303 304 def _AddTimestamp(self): 305 """Add a new timestamp to the list and record the build period. 306 307 The build period is the length of time taken to perform a single 308 build (one board, one commit). 309 """ 310 now = datetime.now() 311 self._timestamps.append(now) 312 count = len(self._timestamps) 313 delta = self._timestamps[-1] - self._timestamps[0] 314 seconds = delta.total_seconds() 315 316 # If we have enough data, estimate build period (time taken for a 317 # single build) and therefore completion time. 318 if count > 1 and self._next_delay_update < now: 319 self._next_delay_update = now + timedelta(seconds=2) 320 if seconds > 0: 321 self._build_period = float(seconds) / count 322 todo = self.count - self.upto 323 self._complete_delay = timedelta(microseconds= 324 self._build_period * todo * 1000000) 325 # Round it 326 self._complete_delay -= timedelta( 327 microseconds=self._complete_delay.microseconds) 328 329 if seconds > 60: 330 self._timestamps.popleft() 331 count -= 1 332 333 def ClearLine(self, length): 334 """Clear any characters on the current line 335 336 Make way for a new line of length 'length', by outputting enough 337 spaces to clear out the old line. Then remember the new length for 338 next time. 339 340 Args: 341 length: Length of new line, in characters 342 """ 343 if length < self.last_line_len: 344 Print(' ' * (self.last_line_len - length), newline=False) 345 Print('\r', newline=False) 346 self.last_line_len = length 347 sys.stdout.flush() 348 349 def SelectCommit(self, commit, checkout=True): 350 """Checkout the selected commit for this build 351 """ 352 self.commit = commit 353 if checkout and self.checkout: 354 gitutil.Checkout(commit.hash) 355 356 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 357 """Run make 358 359 Args: 360 commit: Commit object that is being built 361 brd: Board object that is being built 362 stage: Stage that we are at (mrproper, config, build) 363 cwd: Directory where make should be run 364 args: Arguments to pass to make 365 kwargs: Arguments to pass to command.RunPipe() 366 """ 367 cmd = [self.gnu_make] + list(args) 368 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 369 cwd=cwd, raise_on_error=False, **kwargs) 370 if self.verbose_build: 371 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 372 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 373 return result 374 375 def ProcessResult(self, result): 376 """Process the result of a build, showing progress information 377 378 Args: 379 result: A CommandResult object, which indicates the result for 380 a single build 381 """ 382 col = terminal.Color() 383 if result: 384 target = result.brd.target 385 386 if result.return_code < 0: 387 self.active = False 388 command.StopAll() 389 return 390 391 self.upto += 1 392 if result.return_code != 0: 393 self.fail += 1 394 elif result.stderr: 395 self.warned += 1 396 if result.already_done: 397 self.already_done += 1 398 if self._verbose: 399 Print('\r', newline=False) 400 self.ClearLine(0) 401 boards_selected = {target : result.brd} 402 self.ResetResultSummary(boards_selected) 403 self.ProduceResultSummary(result.commit_upto, self.commits, 404 boards_selected) 405 else: 406 target = '(starting)' 407 408 # Display separate counts for ok, warned and fail 409 ok = self.upto - self.warned - self.fail 410 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 411 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 412 line += self.col.Color(self.col.RED, '%5d' % self.fail) 413 414 name = ' /%-5d ' % self.count 415 416 # Add our current completion time estimate 417 self._AddTimestamp() 418 if self._complete_delay: 419 name += '%s : ' % self._complete_delay 420 # When building all boards for a commit, we can print a commit 421 # progress message. 422 if result and result.commit_upto is None: 423 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 424 self.commit_count) 425 426 name += target 427 Print(line + name, newline=False) 428 length = 14 + len(name) 429 self.ClearLine(length) 430 431 def _GetOutputDir(self, commit_upto): 432 """Get the name of the output directory for a commit number 433 434 The output directory is typically .../<branch>/<commit>. 435 436 Args: 437 commit_upto: Commit number to use (0..self.count-1) 438 """ 439 commit_dir = None 440 if self.commits: 441 commit = self.commits[commit_upto] 442 subject = commit.subject.translate(trans_valid_chars) 443 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 444 self.commit_count, commit.hash, subject[:20])) 445 elif not self.no_subdirs: 446 commit_dir = 'current' 447 if not commit_dir: 448 return self.base_dir 449 return os.path.join(self.base_dir, commit_dir) 450 451 def GetBuildDir(self, commit_upto, target): 452 """Get the name of the build directory for a commit number 453 454 The build directory is typically .../<branch>/<commit>/<target>. 455 456 Args: 457 commit_upto: Commit number to use (0..self.count-1) 458 target: Target name 459 """ 460 output_dir = self._GetOutputDir(commit_upto) 461 return os.path.join(output_dir, target) 462 463 def GetDoneFile(self, commit_upto, target): 464 """Get the name of the done file for a commit number 465 466 Args: 467 commit_upto: Commit number to use (0..self.count-1) 468 target: Target name 469 """ 470 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 471 472 def GetSizesFile(self, commit_upto, target): 473 """Get the name of the sizes file for a commit number 474 475 Args: 476 commit_upto: Commit number to use (0..self.count-1) 477 target: Target name 478 """ 479 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 480 481 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 482 """Get the name of the funcsizes file for a commit number and ELF file 483 484 Args: 485 commit_upto: Commit number to use (0..self.count-1) 486 target: Target name 487 elf_fname: Filename of elf image 488 """ 489 return os.path.join(self.GetBuildDir(commit_upto, target), 490 '%s.sizes' % elf_fname.replace('/', '-')) 491 492 def GetObjdumpFile(self, commit_upto, target, elf_fname): 493 """Get the name of the objdump file for a commit number and ELF file 494 495 Args: 496 commit_upto: Commit number to use (0..self.count-1) 497 target: Target name 498 elf_fname: Filename of elf image 499 """ 500 return os.path.join(self.GetBuildDir(commit_upto, target), 501 '%s.objdump' % elf_fname.replace('/', '-')) 502 503 def GetErrFile(self, commit_upto, target): 504 """Get the name of the err file for a commit number 505 506 Args: 507 commit_upto: Commit number to use (0..self.count-1) 508 target: Target name 509 """ 510 output_dir = self.GetBuildDir(commit_upto, target) 511 return os.path.join(output_dir, 'err') 512 513 def FilterErrors(self, lines): 514 """Filter out errors in which we have no interest 515 516 We should probably use map(). 517 518 Args: 519 lines: List of error lines, each a string 520 Returns: 521 New list with only interesting lines included 522 """ 523 out_lines = [] 524 for line in lines: 525 if not self.re_make_err.search(line): 526 out_lines.append(line) 527 return out_lines 528 529 def ReadFuncSizes(self, fname, fd): 530 """Read function sizes from the output of 'nm' 531 532 Args: 533 fd: File containing data to read 534 fname: Filename we are reading from (just for errors) 535 536 Returns: 537 Dictionary containing size of each function in bytes, indexed by 538 function name. 539 """ 540 sym = {} 541 for line in fd.readlines(): 542 try: 543 size, type, name = line[:-1].split() 544 except: 545 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 546 continue 547 if type in 'tTdDbB': 548 # function names begin with '.' on 64-bit powerpc 549 if '.' in name[1:]: 550 name = 'static.' + name.split('.')[0] 551 sym[name] = sym.get(name, 0) + int(size, 16) 552 return sym 553 554 def _ProcessConfig(self, fname): 555 """Read in a .config, autoconf.mk or autoconf.h file 556 557 This function handles all config file types. It ignores comments and 558 any #defines which don't start with CONFIG_. 559 560 Args: 561 fname: Filename to read 562 563 Returns: 564 Dictionary: 565 key: Config name (e.g. CONFIG_DM) 566 value: Config value (e.g. 1) 567 """ 568 config = {} 569 if os.path.exists(fname): 570 with open(fname) as fd: 571 for line in fd: 572 line = line.strip() 573 if line.startswith('#define'): 574 values = line[8:].split(' ', 1) 575 if len(values) > 1: 576 key, value = values 577 else: 578 key = values[0] 579 value = '' 580 if not key.startswith('CONFIG_'): 581 continue 582 elif not line or line[0] in ['#', '*', '/']: 583 continue 584 else: 585 key, value = line.split('=', 1) 586 config[key] = value 587 return config 588 589 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 590 read_config): 591 """Work out the outcome of a build. 592 593 Args: 594 commit_upto: Commit number to check (0..n-1) 595 target: Target board to check 596 read_func_sizes: True to read function size information 597 read_config: True to read .config and autoconf.h files 598 599 Returns: 600 Outcome object 601 """ 602 done_file = self.GetDoneFile(commit_upto, target) 603 sizes_file = self.GetSizesFile(commit_upto, target) 604 sizes = {} 605 func_sizes = {} 606 config = {} 607 if os.path.exists(done_file): 608 with open(done_file, 'r') as fd: 609 return_code = int(fd.readline()) 610 err_lines = [] 611 err_file = self.GetErrFile(commit_upto, target) 612 if os.path.exists(err_file): 613 with open(err_file, 'r') as fd: 614 err_lines = self.FilterErrors(fd.readlines()) 615 616 # Decide whether the build was ok, failed or created warnings 617 if return_code: 618 rc = OUTCOME_ERROR 619 elif len(err_lines): 620 rc = OUTCOME_WARNING 621 else: 622 rc = OUTCOME_OK 623 624 # Convert size information to our simple format 625 if os.path.exists(sizes_file): 626 with open(sizes_file, 'r') as fd: 627 for line in fd.readlines(): 628 values = line.split() 629 rodata = 0 630 if len(values) > 6: 631 rodata = int(values[6], 16) 632 size_dict = { 633 'all' : int(values[0]) + int(values[1]) + 634 int(values[2]), 635 'text' : int(values[0]) - rodata, 636 'data' : int(values[1]), 637 'bss' : int(values[2]), 638 'rodata' : rodata, 639 } 640 sizes[values[5]] = size_dict 641 642 if read_func_sizes: 643 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 644 for fname in glob.glob(pattern): 645 with open(fname, 'r') as fd: 646 dict_name = os.path.basename(fname).replace('.sizes', 647 '') 648 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 649 650 if read_config: 651 output_dir = self.GetBuildDir(commit_upto, target) 652 for name in CONFIG_FILENAMES: 653 fname = os.path.join(output_dir, name) 654 config[name] = self._ProcessConfig(fname) 655 656 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) 657 658 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) 659 660 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 661 read_config): 662 """Calculate a summary of the results of building a commit. 663 664 Args: 665 board_selected: Dict containing boards to summarise 666 commit_upto: Commit number to summarize (0..self.count-1) 667 read_func_sizes: True to read function size information 668 read_config: True to read .config and autoconf.h files 669 670 Returns: 671 Tuple: 672 Dict containing boards which passed building this commit. 673 keyed by board.target 674 List containing a summary of error lines 675 Dict keyed by error line, containing a list of the Board 676 objects with that error 677 List containing a summary of warning lines 678 Dict keyed by error line, containing a list of the Board 679 objects with that warning 680 Dictionary keyed by board.target. Each value is a dictionary: 681 key: filename - e.g. '.config' 682 value is itself a dictionary: 683 key: config name 684 value: config value 685 """ 686 def AddLine(lines_summary, lines_boards, line, board): 687 line = line.rstrip() 688 if line in lines_boards: 689 lines_boards[line].append(board) 690 else: 691 lines_boards[line] = [board] 692 lines_summary.append(line) 693 694 board_dict = {} 695 err_lines_summary = [] 696 err_lines_boards = {} 697 warn_lines_summary = [] 698 warn_lines_boards = {} 699 config = {} 700 701 for board in boards_selected.itervalues(): 702 outcome = self.GetBuildOutcome(commit_upto, board.target, 703 read_func_sizes, read_config) 704 board_dict[board.target] = outcome 705 last_func = None 706 last_was_warning = False 707 for line in outcome.err_lines: 708 if line: 709 if (self._re_function.match(line) or 710 self._re_files.match(line)): 711 last_func = line 712 else: 713 is_warning = self._re_warning.match(line) 714 is_note = self._re_note.match(line) 715 if is_warning or (last_was_warning and is_note): 716 if last_func: 717 AddLine(warn_lines_summary, warn_lines_boards, 718 last_func, board) 719 AddLine(warn_lines_summary, warn_lines_boards, 720 line, board) 721 else: 722 if last_func: 723 AddLine(err_lines_summary, err_lines_boards, 724 last_func, board) 725 AddLine(err_lines_summary, err_lines_boards, 726 line, board) 727 last_was_warning = is_warning 728 last_func = None 729 tconfig = Config(board.target) 730 for fname in CONFIG_FILENAMES: 731 if outcome.config: 732 for key, value in outcome.config[fname].iteritems(): 733 tconfig.Add(fname, key, value) 734 config[board.target] = tconfig 735 736 return (board_dict, err_lines_summary, err_lines_boards, 737 warn_lines_summary, warn_lines_boards, config) 738 739 def AddOutcome(self, board_dict, arch_list, changes, char, color): 740 """Add an output to our list of outcomes for each architecture 741 742 This simple function adds failing boards (changes) to the 743 relevant architecture string, so we can print the results out 744 sorted by architecture. 745 746 Args: 747 board_dict: Dict containing all boards 748 arch_list: Dict keyed by arch name. Value is a string containing 749 a list of board names which failed for that arch. 750 changes: List of boards to add to arch_list 751 color: terminal.Colour object 752 """ 753 done_arch = {} 754 for target in changes: 755 if target in board_dict: 756 arch = board_dict[target].arch 757 else: 758 arch = 'unknown' 759 str = self.col.Color(color, ' ' + target) 760 if not arch in done_arch: 761 str = ' %s %s' % (self.col.Color(color, char), str) 762 done_arch[arch] = True 763 if not arch in arch_list: 764 arch_list[arch] = str 765 else: 766 arch_list[arch] += str 767 768 769 def ColourNum(self, num): 770 color = self.col.RED if num > 0 else self.col.GREEN 771 if num == 0: 772 return '0' 773 return self.col.Color(color, str(num)) 774 775 def ResetResultSummary(self, board_selected): 776 """Reset the results summary ready for use. 777 778 Set up the base board list to be all those selected, and set the 779 error lines to empty. 780 781 Following this, calls to PrintResultSummary() will use this 782 information to work out what has changed. 783 784 Args: 785 board_selected: Dict containing boards to summarise, keyed by 786 board.target 787 """ 788 self._base_board_dict = {} 789 for board in board_selected: 790 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) 791 self._base_err_lines = [] 792 self._base_warn_lines = [] 793 self._base_err_line_boards = {} 794 self._base_warn_line_boards = {} 795 self._base_config = None 796 797 def PrintFuncSizeDetail(self, fname, old, new): 798 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 799 delta, common = [], {} 800 801 for a in old: 802 if a in new: 803 common[a] = 1 804 805 for name in old: 806 if name not in common: 807 remove += 1 808 down += old[name] 809 delta.append([-old[name], name]) 810 811 for name in new: 812 if name not in common: 813 add += 1 814 up += new[name] 815 delta.append([new[name], name]) 816 817 for name in common: 818 diff = new.get(name, 0) - old.get(name, 0) 819 if diff > 0: 820 grow, up = grow + 1, up + diff 821 elif diff < 0: 822 shrink, down = shrink + 1, down - diff 823 delta.append([diff, name]) 824 825 delta.sort() 826 delta.reverse() 827 828 args = [add, -remove, grow, -shrink, up, -down, up - down] 829 if max(args) == 0: 830 return 831 args = [self.ColourNum(x) for x in args] 832 indent = ' ' * 15 833 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 834 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 835 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 836 'delta')) 837 for diff, name in delta: 838 if diff: 839 color = self.col.RED if diff > 0 else self.col.GREEN 840 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 841 old.get(name, '-'), new.get(name,'-'), diff) 842 Print(msg, colour=color) 843 844 845 def PrintSizeDetail(self, target_list, show_bloat): 846 """Show details size information for each board 847 848 Args: 849 target_list: List of targets, each a dict containing: 850 'target': Target name 851 'total_diff': Total difference in bytes across all areas 852 <part_name>: Difference for that part 853 show_bloat: Show detail for each function 854 """ 855 targets_by_diff = sorted(target_list, reverse=True, 856 key=lambda x: x['_total_diff']) 857 for result in targets_by_diff: 858 printed_target = False 859 for name in sorted(result): 860 diff = result[name] 861 if name.startswith('_'): 862 continue 863 if diff != 0: 864 color = self.col.RED if diff > 0 else self.col.GREEN 865 msg = ' %s %+d' % (name, diff) 866 if not printed_target: 867 Print('%10s %-15s:' % ('', result['_target']), 868 newline=False) 869 printed_target = True 870 Print(msg, colour=color, newline=False) 871 if printed_target: 872 Print() 873 if show_bloat: 874 target = result['_target'] 875 outcome = result['_outcome'] 876 base_outcome = self._base_board_dict[target] 877 for fname in outcome.func_sizes: 878 self.PrintFuncSizeDetail(fname, 879 base_outcome.func_sizes[fname], 880 outcome.func_sizes[fname]) 881 882 883 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 884 show_bloat): 885 """Print a summary of image sizes broken down by section. 886 887 The summary takes the form of one line per architecture. The 888 line contains deltas for each of the sections (+ means the section 889 got bigger, - means smaller). The nunmbers are the average number 890 of bytes that a board in this section increased by. 891 892 For example: 893 powerpc: (622 boards) text -0.0 894 arm: (285 boards) text -0.0 895 nds32: (3 boards) text -8.0 896 897 Args: 898 board_selected: Dict containing boards to summarise, keyed by 899 board.target 900 board_dict: Dict containing boards for which we built this 901 commit, keyed by board.target. The value is an Outcome object. 902 show_detail: Show detail for each board 903 show_bloat: Show detail for each function 904 """ 905 arch_list = {} 906 arch_count = {} 907 908 # Calculate changes in size for different image parts 909 # The previous sizes are in Board.sizes, for each board 910 for target in board_dict: 911 if target not in board_selected: 912 continue 913 base_sizes = self._base_board_dict[target].sizes 914 outcome = board_dict[target] 915 sizes = outcome.sizes 916 917 # Loop through the list of images, creating a dict of size 918 # changes for each image/part. We end up with something like 919 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 920 # which means that U-Boot data increased by 5 bytes and SPL 921 # text decreased by 4. 922 err = {'_target' : target} 923 for image in sizes: 924 if image in base_sizes: 925 base_image = base_sizes[image] 926 # Loop through the text, data, bss parts 927 for part in sorted(sizes[image]): 928 diff = sizes[image][part] - base_image[part] 929 col = None 930 if diff: 931 if image == 'u-boot': 932 name = part 933 else: 934 name = image + ':' + part 935 err[name] = diff 936 arch = board_selected[target].arch 937 if not arch in arch_count: 938 arch_count[arch] = 1 939 else: 940 arch_count[arch] += 1 941 if not sizes: 942 pass # Only add to our list when we have some stats 943 elif not arch in arch_list: 944 arch_list[arch] = [err] 945 else: 946 arch_list[arch].append(err) 947 948 # We now have a list of image size changes sorted by arch 949 # Print out a summary of these 950 for arch, target_list in arch_list.iteritems(): 951 # Get total difference for each type 952 totals = {} 953 for result in target_list: 954 total = 0 955 for name, diff in result.iteritems(): 956 if name.startswith('_'): 957 continue 958 total += diff 959 if name in totals: 960 totals[name] += diff 961 else: 962 totals[name] = diff 963 result['_total_diff'] = total 964 result['_outcome'] = board_dict[result['_target']] 965 966 count = len(target_list) 967 printed_arch = False 968 for name in sorted(totals): 969 diff = totals[name] 970 if diff: 971 # Display the average difference in this name for this 972 # architecture 973 avg_diff = float(diff) / count 974 color = self.col.RED if avg_diff > 0 else self.col.GREEN 975 msg = ' %s %+1.1f' % (name, avg_diff) 976 if not printed_arch: 977 Print('%10s: (for %d/%d boards)' % (arch, count, 978 arch_count[arch]), newline=False) 979 printed_arch = True 980 Print(msg, colour=color, newline=False) 981 982 if printed_arch: 983 Print() 984 if show_detail: 985 self.PrintSizeDetail(target_list, show_bloat) 986 987 988 def PrintResultSummary(self, board_selected, board_dict, err_lines, 989 err_line_boards, warn_lines, warn_line_boards, 990 config, show_sizes, show_detail, show_bloat, 991 show_config): 992 """Compare results with the base results and display delta. 993 994 Only boards mentioned in board_selected will be considered. This 995 function is intended to be called repeatedly with the results of 996 each commit. It therefore shows a 'diff' between what it saw in 997 the last call and what it sees now. 998 999 Args: 1000 board_selected: Dict containing boards to summarise, keyed by 1001 board.target 1002 board_dict: Dict containing boards for which we built this 1003 commit, keyed by board.target. The value is an Outcome object. 1004 err_lines: A list of errors for this commit, or [] if there is 1005 none, or we don't want to print errors 1006 err_line_boards: Dict keyed by error line, containing a list of 1007 the Board objects with that error 1008 warn_lines: A list of warnings for this commit, or [] if there is 1009 none, or we don't want to print errors 1010 warn_line_boards: Dict keyed by warning line, containing a list of 1011 the Board objects with that warning 1012 config: Dictionary keyed by filename - e.g. '.config'. Each 1013 value is itself a dictionary: 1014 key: config name 1015 value: config value 1016 show_sizes: Show image size deltas 1017 show_detail: Show detail for each board 1018 show_bloat: Show detail for each function 1019 show_config: Show config changes 1020 """ 1021 def _BoardList(line, line_boards): 1022 """Helper function to get a line of boards containing a line 1023 1024 Args: 1025 line: Error line to search for 1026 Return: 1027 String containing a list of boards with that error line, or 1028 '' if the user has not requested such a list 1029 """ 1030 if self._list_error_boards: 1031 names = [] 1032 for board in line_boards[line]: 1033 if not board.target in names: 1034 names.append(board.target) 1035 names_str = '(%s) ' % ','.join(names) 1036 else: 1037 names_str = '' 1038 return names_str 1039 1040 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1041 char): 1042 better_lines = [] 1043 worse_lines = [] 1044 for line in lines: 1045 if line not in base_lines: 1046 worse_lines.append(char + '+' + 1047 _BoardList(line, line_boards) + line) 1048 for line in base_lines: 1049 if line not in lines: 1050 better_lines.append(char + '-' + 1051 _BoardList(line, base_line_boards) + line) 1052 return better_lines, worse_lines 1053 1054 def _CalcConfig(delta, name, config): 1055 """Calculate configuration changes 1056 1057 Args: 1058 delta: Type of the delta, e.g. '+' 1059 name: name of the file which changed (e.g. .config) 1060 config: configuration change dictionary 1061 key: config name 1062 value: config value 1063 Returns: 1064 String containing the configuration changes which can be 1065 printed 1066 """ 1067 out = '' 1068 for key in sorted(config.keys()): 1069 out += '%s=%s ' % (key, config[key]) 1070 return '%s %s: %s' % (delta, name, out) 1071 1072 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1073 """Add changes in configuration to a list 1074 1075 Args: 1076 lines: list to add to 1077 name: config file name 1078 config_plus: configurations added, dictionary 1079 key: config name 1080 value: config value 1081 config_minus: configurations removed, dictionary 1082 key: config name 1083 value: config value 1084 config_change: configurations changed, dictionary 1085 key: config name 1086 value: config value 1087 """ 1088 if config_plus: 1089 lines.append(_CalcConfig('+', name, config_plus)) 1090 if config_minus: 1091 lines.append(_CalcConfig('-', name, config_minus)) 1092 if config_change: 1093 lines.append(_CalcConfig('c', name, config_change)) 1094 1095 def _OutputConfigInfo(lines): 1096 for line in lines: 1097 if not line: 1098 continue 1099 if line[0] == '+': 1100 col = self.col.GREEN 1101 elif line[0] == '-': 1102 col = self.col.RED 1103 elif line[0] == 'c': 1104 col = self.col.YELLOW 1105 Print(' ' + line, newline=True, colour=col) 1106 1107 1108 better = [] # List of boards fixed since last commit 1109 worse = [] # List of new broken boards since last commit 1110 new = [] # List of boards that didn't exist last time 1111 unknown = [] # List of boards that were not built 1112 1113 for target in board_dict: 1114 if target not in board_selected: 1115 continue 1116 1117 # If the board was built last time, add its outcome to a list 1118 if target in self._base_board_dict: 1119 base_outcome = self._base_board_dict[target].rc 1120 outcome = board_dict[target] 1121 if outcome.rc == OUTCOME_UNKNOWN: 1122 unknown.append(target) 1123 elif outcome.rc < base_outcome: 1124 better.append(target) 1125 elif outcome.rc > base_outcome: 1126 worse.append(target) 1127 else: 1128 new.append(target) 1129 1130 # Get a list of errors that have appeared, and disappeared 1131 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1132 self._base_err_line_boards, err_lines, err_line_boards, '') 1133 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1134 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1135 1136 # Display results by arch 1137 if (better or worse or unknown or new or worse_err or better_err 1138 or worse_warn or better_warn): 1139 arch_list = {} 1140 self.AddOutcome(board_selected, arch_list, better, '', 1141 self.col.GREEN) 1142 self.AddOutcome(board_selected, arch_list, worse, '+', 1143 self.col.RED) 1144 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) 1145 if self._show_unknown: 1146 self.AddOutcome(board_selected, arch_list, unknown, '?', 1147 self.col.MAGENTA) 1148 for arch, target_list in arch_list.iteritems(): 1149 Print('%10s: %s' % (arch, target_list)) 1150 self._error_lines += 1 1151 if better_err: 1152 Print('\n'.join(better_err), colour=self.col.GREEN) 1153 self._error_lines += 1 1154 if worse_err: 1155 Print('\n'.join(worse_err), colour=self.col.RED) 1156 self._error_lines += 1 1157 if better_warn: 1158 Print('\n'.join(better_warn), colour=self.col.CYAN) 1159 self._error_lines += 1 1160 if worse_warn: 1161 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1162 self._error_lines += 1 1163 1164 if show_sizes: 1165 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1166 show_bloat) 1167 1168 if show_config and self._base_config: 1169 summary = {} 1170 arch_config_plus = {} 1171 arch_config_minus = {} 1172 arch_config_change = {} 1173 arch_list = [] 1174 1175 for target in board_dict: 1176 if target not in board_selected: 1177 continue 1178 arch = board_selected[target].arch 1179 if arch not in arch_list: 1180 arch_list.append(arch) 1181 1182 for arch in arch_list: 1183 arch_config_plus[arch] = {} 1184 arch_config_minus[arch] = {} 1185 arch_config_change[arch] = {} 1186 for name in CONFIG_FILENAMES: 1187 arch_config_plus[arch][name] = {} 1188 arch_config_minus[arch][name] = {} 1189 arch_config_change[arch][name] = {} 1190 1191 for target in board_dict: 1192 if target not in board_selected: 1193 continue 1194 1195 arch = board_selected[target].arch 1196 1197 all_config_plus = {} 1198 all_config_minus = {} 1199 all_config_change = {} 1200 tbase = self._base_config[target] 1201 tconfig = config[target] 1202 lines = [] 1203 for name in CONFIG_FILENAMES: 1204 if not tconfig.config[name]: 1205 continue 1206 config_plus = {} 1207 config_minus = {} 1208 config_change = {} 1209 base = tbase.config[name] 1210 for key, value in tconfig.config[name].iteritems(): 1211 if key not in base: 1212 config_plus[key] = value 1213 all_config_plus[key] = value 1214 for key, value in base.iteritems(): 1215 if key not in tconfig.config[name]: 1216 config_minus[key] = value 1217 all_config_minus[key] = value 1218 for key, value in base.iteritems(): 1219 new_value = tconfig.config.get(key) 1220 if new_value and value != new_value: 1221 desc = '%s -> %s' % (value, new_value) 1222 config_change[key] = desc 1223 all_config_change[key] = desc 1224 1225 arch_config_plus[arch][name].update(config_plus) 1226 arch_config_minus[arch][name].update(config_minus) 1227 arch_config_change[arch][name].update(config_change) 1228 1229 _AddConfig(lines, name, config_plus, config_minus, 1230 config_change) 1231 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1232 all_config_change) 1233 summary[target] = '\n'.join(lines) 1234 1235 lines_by_target = {} 1236 for target, lines in summary.iteritems(): 1237 if lines in lines_by_target: 1238 lines_by_target[lines].append(target) 1239 else: 1240 lines_by_target[lines] = [target] 1241 1242 for arch in arch_list: 1243 lines = [] 1244 all_plus = {} 1245 all_minus = {} 1246 all_change = {} 1247 for name in CONFIG_FILENAMES: 1248 all_plus.update(arch_config_plus[arch][name]) 1249 all_minus.update(arch_config_minus[arch][name]) 1250 all_change.update(arch_config_change[arch][name]) 1251 _AddConfig(lines, name, arch_config_plus[arch][name], 1252 arch_config_minus[arch][name], 1253 arch_config_change[arch][name]) 1254 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1255 #arch_summary[target] = '\n'.join(lines) 1256 if lines: 1257 Print('%s:' % arch) 1258 _OutputConfigInfo(lines) 1259 1260 for lines, targets in lines_by_target.iteritems(): 1261 if not lines: 1262 continue 1263 Print('%s :' % ' '.join(sorted(targets))) 1264 _OutputConfigInfo(lines.split('\n')) 1265 1266 1267 # Save our updated information for the next call to this function 1268 self._base_board_dict = board_dict 1269 self._base_err_lines = err_lines 1270 self._base_warn_lines = warn_lines 1271 self._base_err_line_boards = err_line_boards 1272 self._base_warn_line_boards = warn_line_boards 1273 self._base_config = config 1274 1275 # Get a list of boards that did not get built, if needed 1276 not_built = [] 1277 for board in board_selected: 1278 if not board in board_dict: 1279 not_built.append(board) 1280 if not_built: 1281 Print("Boards not built (%d): %s" % (len(not_built), 1282 ', '.join(not_built))) 1283 1284 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1285 (board_dict, err_lines, err_line_boards, warn_lines, 1286 warn_line_boards, config) = self.GetResultSummary( 1287 board_selected, commit_upto, 1288 read_func_sizes=self._show_bloat, 1289 read_config=self._show_config) 1290 if commits: 1291 msg = '%02d: %s' % (commit_upto + 1, 1292 commits[commit_upto].subject) 1293 Print(msg, colour=self.col.BLUE) 1294 self.PrintResultSummary(board_selected, board_dict, 1295 err_lines if self._show_errors else [], err_line_boards, 1296 warn_lines if self._show_errors else [], warn_line_boards, 1297 config, self._show_sizes, self._show_detail, 1298 self._show_bloat, self._show_config) 1299 1300 def ShowSummary(self, commits, board_selected): 1301 """Show a build summary for U-Boot for a given board list. 1302 1303 Reset the result summary, then repeatedly call GetResultSummary on 1304 each commit's results, then display the differences we see. 1305 1306 Args: 1307 commit: Commit objects to summarise 1308 board_selected: Dict containing boards to summarise 1309 """ 1310 self.commit_count = len(commits) if commits else 1 1311 self.commits = commits 1312 self.ResetResultSummary(board_selected) 1313 self._error_lines = 0 1314 1315 for commit_upto in range(0, self.commit_count, self._step): 1316 self.ProduceResultSummary(commit_upto, commits, board_selected) 1317 if not self._error_lines: 1318 Print('(no errors to report)', colour=self.col.GREEN) 1319 1320 1321 def SetupBuild(self, board_selected, commits): 1322 """Set up ready to start a build. 1323 1324 Args: 1325 board_selected: Selected boards to build 1326 commits: Selected commits to build 1327 """ 1328 # First work out how many commits we will build 1329 count = (self.commit_count + self._step - 1) / self._step 1330 self.count = len(board_selected) * count 1331 self.upto = self.warned = self.fail = 0 1332 self._timestamps = collections.deque() 1333 1334 def GetThreadDir(self, thread_num): 1335 """Get the directory path to the working dir for a thread. 1336 1337 Args: 1338 thread_num: Number of thread to check. 1339 """ 1340 return os.path.join(self._working_dir, '%02d' % thread_num) 1341 1342 def _PrepareThread(self, thread_num, setup_git): 1343 """Prepare the working directory for a thread. 1344 1345 This clones or fetches the repo into the thread's work directory. 1346 1347 Args: 1348 thread_num: Thread number (0, 1, ...) 1349 setup_git: True to set up a git repo clone 1350 """ 1351 thread_dir = self.GetThreadDir(thread_num) 1352 builderthread.Mkdir(thread_dir) 1353 git_dir = os.path.join(thread_dir, '.git') 1354 1355 # Clone the repo if it doesn't already exist 1356 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1357 # we have a private index but uses the origin repo's contents? 1358 if setup_git and self.git_dir: 1359 src_dir = os.path.abspath(self.git_dir) 1360 if os.path.exists(git_dir): 1361 gitutil.Fetch(git_dir, thread_dir) 1362 else: 1363 Print('Cloning repo for thread %d' % thread_num) 1364 gitutil.Clone(src_dir, thread_dir) 1365 1366 def _PrepareWorkingSpace(self, max_threads, setup_git): 1367 """Prepare the working directory for use. 1368 1369 Set up the git repo for each thread. 1370 1371 Args: 1372 max_threads: Maximum number of threads we expect to need. 1373 setup_git: True to set up a git repo clone 1374 """ 1375 builderthread.Mkdir(self._working_dir) 1376 for thread in range(max_threads): 1377 self._PrepareThread(thread, setup_git) 1378 1379 def _PrepareOutputSpace(self): 1380 """Get the output directories ready to receive files. 1381 1382 We delete any output directories which look like ones we need to 1383 create. Having left over directories is confusing when the user wants 1384 to check the output manually. 1385 """ 1386 if not self.commits: 1387 return 1388 dir_list = [] 1389 for commit_upto in range(self.commit_count): 1390 dir_list.append(self._GetOutputDir(commit_upto)) 1391 1392 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1393 if dirname not in dir_list: 1394 shutil.rmtree(dirname) 1395 1396 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1397 """Build all commits for a list of boards 1398 1399 Args: 1400 commits: List of commits to be build, each a Commit object 1401 boards_selected: Dict of selected boards, key is target name, 1402 value is Board object 1403 keep_outputs: True to save build output files 1404 verbose: Display build results as they are completed 1405 Returns: 1406 Tuple containing: 1407 - number of boards that failed to build 1408 - number of boards that issued warnings 1409 """ 1410 self.commit_count = len(commits) if commits else 1 1411 self.commits = commits 1412 self._verbose = verbose 1413 1414 self.ResetResultSummary(board_selected) 1415 builderthread.Mkdir(self.base_dir, parents = True) 1416 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1417 commits is not None) 1418 self._PrepareOutputSpace() 1419 self.SetupBuild(board_selected, commits) 1420 self.ProcessResult(None) 1421 1422 # Create jobs to build all commits for each board 1423 for brd in board_selected.itervalues(): 1424 job = builderthread.BuilderJob() 1425 job.board = brd 1426 job.commits = commits 1427 job.keep_outputs = keep_outputs 1428 job.step = self._step 1429 self.queue.put(job) 1430 1431 # Wait until all jobs are started 1432 self.queue.join() 1433 1434 # Wait until we have processed all output 1435 self.out_queue.join() 1436 Print() 1437 self.ClearLine(0) 1438 return (self.fail, self.warned) 1439