1# SPDX-License-Identifier: GPL-2.0+ 2# Copyright (c) 2013 The Chromium OS Authors. 3# 4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> 5# 6 7import collections 8from datetime import datetime, timedelta 9import glob 10import os 11import re 12import Queue 13import shutil 14import signal 15import string 16import sys 17import threading 18import time 19 20import builderthread 21import command 22import gitutil 23import terminal 24from terminal import Print 25import toolchain 26 27 28""" 29Theory of Operation 30 31Please see README for user documentation, and you should be familiar with 32that before trying to make sense of this. 33 34Buildman works by keeping the machine as busy as possible, building different 35commits for different boards on multiple CPUs at once. 36 37The source repo (self.git_dir) contains all the commits to be built. Each 38thread works on a single board at a time. It checks out the first commit, 39configures it for that board, then builds it. Then it checks out the next 40commit and builds it (typically without re-configuring). When it runs out 41of commits, it gets another job from the builder and starts again with that 42board. 43 44Clearly the builder threads could work either way - they could check out a 45commit and then built it for all boards. Using separate directories for each 46commit/board pair they could leave their build product around afterwards 47also. 48 49The intent behind building a single board for multiple commits, is to make 50use of incremental builds. Since each commit is built incrementally from 51the previous one, builds are faster. Reconfiguring for a different board 52removes all intermediate object files. 53 54Many threads can be working at once, but each has its own working directory. 55When a thread finishes a build, it puts the output files into a result 56directory. 57 58The base directory used by buildman is normally '../<branch>', i.e. 59a directory higher than the source repository and named after the branch 60being built. 61 62Within the base directory, we have one subdirectory for each commit. Within 63that is one subdirectory for each board. Within that is the build output for 64that commit/board combination. 65 66Buildman also create working directories for each thread, in a .bm-work/ 67subdirectory in the base dir. 68 69As an example, say we are building branch 'us-net' for boards 'sandbox' and 70'seaboard', and say that us-net has two commits. We will have directories 71like this: 72 73us-net/ base directory 74 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ 75 sandbox/ 76 u-boot.bin 77 seaboard/ 78 u-boot.bin 79 02_of_02_g4ed4ebc_net--Check-tftp-comp/ 80 sandbox/ 81 u-boot.bin 82 seaboard/ 83 u-boot.bin 84 .bm-work/ 85 00/ working directory for thread 0 (contains source checkout) 86 build/ build output 87 01/ working directory for thread 1 88 build/ build output 89 ... 90u-boot/ source directory 91 .git/ repository 92""" 93 94# Possible build outcomes 95OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) 96 97# Translate a commit subject into a valid filename (and handle unicode) 98trans_valid_chars = string.maketrans('/: ', '---') 99trans_valid_chars = trans_valid_chars.decode('latin-1') 100 101BASE_CONFIG_FILENAMES = [ 102 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' 103] 104 105EXTRA_CONFIG_FILENAMES = [ 106 '.config', '.config-spl', '.config-tpl', 107 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', 108 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', 109] 110 111class Config: 112 """Holds information about configuration settings for a board.""" 113 def __init__(self, config_filename, target): 114 self.target = target 115 self.config = {} 116 for fname in config_filename: 117 self.config[fname] = {} 118 119 def Add(self, fname, key, value): 120 self.config[fname][key] = value 121 122 def __hash__(self): 123 val = 0 124 for fname in self.config: 125 for key, value in self.config[fname].iteritems(): 126 print key, value 127 val = val ^ hash(key) & hash(value) 128 return val 129 130class Environment: 131 """Holds information about environment variables for a board.""" 132 def __init__(self, target): 133 self.target = target 134 self.environment = {} 135 136 def Add(self, key, value): 137 self.environment[key] = value 138 139class Builder: 140 """Class for building U-Boot for a particular commit. 141 142 Public members: (many should ->private) 143 already_done: Number of builds already completed 144 base_dir: Base directory to use for builder 145 checkout: True to check out source, False to skip that step. 146 This is used for testing. 147 col: terminal.Color() object 148 count: Number of commits to build 149 do_make: Method to call to invoke Make 150 fail: Number of builds that failed due to error 151 force_build: Force building even if a build already exists 152 force_config_on_failure: If a commit fails for a board, disable 153 incremental building for the next commit we build for that 154 board, so that we will see all warnings/errors again. 155 force_build_failures: If a previously-built build (i.e. built on 156 a previous run of buildman) is marked as failed, rebuild it. 157 git_dir: Git directory containing source repository 158 last_line_len: Length of the last line we printed (used for erasing 159 it with new progress information) 160 num_jobs: Number of jobs to run at once (passed to make as -j) 161 num_threads: Number of builder threads to run 162 out_queue: Queue of results to process 163 re_make_err: Compiled regular expression for ignore_lines 164 queue: Queue of jobs to run 165 threads: List of active threads 166 toolchains: Toolchains object to use for building 167 upto: Current commit number we are building (0.count-1) 168 warned: Number of builds that produced at least one warning 169 force_reconfig: Reconfigure U-Boot on each comiit. This disables 170 incremental building, where buildman reconfigures on the first 171 commit for a baord, and then just does an incremental build for 172 the following commits. In fact buildman will reconfigure and 173 retry for any failing commits, so generally the only effect of 174 this option is to slow things down. 175 in_tree: Build U-Boot in-tree instead of specifying an output 176 directory separate from the source code. This option is really 177 only useful for testing in-tree builds. 178 179 Private members: 180 _base_board_dict: Last-summarised Dict of boards 181 _base_err_lines: Last-summarised list of errors 182 _base_warn_lines: Last-summarised list of warnings 183 _build_period_us: Time taken for a single build (float object). 184 _complete_delay: Expected delay until completion (timedelta) 185 _next_delay_update: Next time we plan to display a progress update 186 (datatime) 187 _show_unknown: Show unknown boards (those not built) in summary 188 _timestamps: List of timestamps for the completion of the last 189 last _timestamp_count builds. Each is a datetime object. 190 _timestamp_count: Number of timestamps to keep in our list. 191 _working_dir: Base working directory containing all threads 192 """ 193 class Outcome: 194 """Records a build outcome for a single make invocation 195 196 Public Members: 197 rc: Outcome value (OUTCOME_...) 198 err_lines: List of error lines or [] if none 199 sizes: Dictionary of image size information, keyed by filename 200 - Each value is itself a dictionary containing 201 values for 'text', 'data' and 'bss', being the integer 202 size in bytes of each section. 203 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each 204 value is itself a dictionary: 205 key: function name 206 value: Size of function in bytes 207 config: Dictionary keyed by filename - e.g. '.config'. Each 208 value is itself a dictionary: 209 key: config name 210 value: config value 211 environment: Dictionary keyed by environment variable, Each 212 value is the value of environment variable. 213 """ 214 def __init__(self, rc, err_lines, sizes, func_sizes, config, 215 environment): 216 self.rc = rc 217 self.err_lines = err_lines 218 self.sizes = sizes 219 self.func_sizes = func_sizes 220 self.config = config 221 self.environment = environment 222 223 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, 224 gnu_make='make', checkout=True, show_unknown=True, step=1, 225 no_subdirs=False, full_path=False, verbose_build=False, 226 incremental=False, per_board_out_dir=False, 227 config_only=False, squash_config_y=False, 228 warnings_as_errors=False): 229 """Create a new Builder object 230 231 Args: 232 toolchains: Toolchains object to use for building 233 base_dir: Base directory to use for builder 234 git_dir: Git directory containing source repository 235 num_threads: Number of builder threads to run 236 num_jobs: Number of jobs to run at once (passed to make as -j) 237 gnu_make: the command name of GNU Make. 238 checkout: True to check out source, False to skip that step. 239 This is used for testing. 240 show_unknown: Show unknown boards (those not built) in summary 241 step: 1 to process every commit, n to process every nth commit 242 no_subdirs: Don't create subdirectories when building current 243 source for a single board 244 full_path: Return the full path in CROSS_COMPILE and don't set 245 PATH 246 verbose_build: Run build with V=1 and don't use 'make -s' 247 incremental: Always perform incremental builds; don't run make 248 mrproper when configuring 249 per_board_out_dir: Build in a separate persistent directory per 250 board rather than a thread-specific directory 251 config_only: Only configure each build, don't build it 252 squash_config_y: Convert CONFIG options with the value 'y' to '1' 253 warnings_as_errors: Treat all compiler warnings as errors 254 """ 255 self.toolchains = toolchains 256 self.base_dir = base_dir 257 self._working_dir = os.path.join(base_dir, '.bm-work') 258 self.threads = [] 259 self.do_make = self.Make 260 self.gnu_make = gnu_make 261 self.checkout = checkout 262 self.num_threads = num_threads 263 self.num_jobs = num_jobs 264 self.already_done = 0 265 self.force_build = False 266 self.git_dir = git_dir 267 self._show_unknown = show_unknown 268 self._timestamp_count = 10 269 self._build_period_us = None 270 self._complete_delay = None 271 self._next_delay_update = datetime.now() 272 self.force_config_on_failure = True 273 self.force_build_failures = False 274 self.force_reconfig = False 275 self._step = step 276 self.in_tree = False 277 self._error_lines = 0 278 self.no_subdirs = no_subdirs 279 self.full_path = full_path 280 self.verbose_build = verbose_build 281 self.config_only = config_only 282 self.squash_config_y = squash_config_y 283 self.config_filenames = BASE_CONFIG_FILENAMES 284 if not self.squash_config_y: 285 self.config_filenames += EXTRA_CONFIG_FILENAMES 286 287 self.warnings_as_errors = warnings_as_errors 288 self.col = terminal.Color() 289 290 self._re_function = re.compile('(.*): In function.*') 291 self._re_files = re.compile('In file included from.*') 292 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') 293 self._re_dtb_warning = re.compile('(.*): Warning .*') 294 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') 295 296 self.queue = Queue.Queue() 297 self.out_queue = Queue.Queue() 298 for i in range(self.num_threads): 299 t = builderthread.BuilderThread(self, i, incremental, 300 per_board_out_dir) 301 t.setDaemon(True) 302 t.start() 303 self.threads.append(t) 304 305 self.last_line_len = 0 306 t = builderthread.ResultThread(self) 307 t.setDaemon(True) 308 t.start() 309 self.threads.append(t) 310 311 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] 312 self.re_make_err = re.compile('|'.join(ignore_lines)) 313 314 # Handle existing graceful with SIGINT / Ctrl-C 315 signal.signal(signal.SIGINT, self.signal_handler) 316 317 def __del__(self): 318 """Get rid of all threads created by the builder""" 319 for t in self.threads: 320 del t 321 322 def signal_handler(self, signal, frame): 323 sys.exit(1) 324 325 def SetDisplayOptions(self, show_errors=False, show_sizes=False, 326 show_detail=False, show_bloat=False, 327 list_error_boards=False, show_config=False, 328 show_environment=False): 329 """Setup display options for the builder. 330 331 show_errors: True to show summarised error/warning info 332 show_sizes: Show size deltas 333 show_detail: Show detail for each board 334 show_bloat: Show detail for each function 335 list_error_boards: Show the boards which caused each error/warning 336 show_config: Show config deltas 337 show_environment: Show environment deltas 338 """ 339 self._show_errors = show_errors 340 self._show_sizes = show_sizes 341 self._show_detail = show_detail 342 self._show_bloat = show_bloat 343 self._list_error_boards = list_error_boards 344 self._show_config = show_config 345 self._show_environment = show_environment 346 347 def _AddTimestamp(self): 348 """Add a new timestamp to the list and record the build period. 349 350 The build period is the length of time taken to perform a single 351 build (one board, one commit). 352 """ 353 now = datetime.now() 354 self._timestamps.append(now) 355 count = len(self._timestamps) 356 delta = self._timestamps[-1] - self._timestamps[0] 357 seconds = delta.total_seconds() 358 359 # If we have enough data, estimate build period (time taken for a 360 # single build) and therefore completion time. 361 if count > 1 and self._next_delay_update < now: 362 self._next_delay_update = now + timedelta(seconds=2) 363 if seconds > 0: 364 self._build_period = float(seconds) / count 365 todo = self.count - self.upto 366 self._complete_delay = timedelta(microseconds= 367 self._build_period * todo * 1000000) 368 # Round it 369 self._complete_delay -= timedelta( 370 microseconds=self._complete_delay.microseconds) 371 372 if seconds > 60: 373 self._timestamps.popleft() 374 count -= 1 375 376 def ClearLine(self, length): 377 """Clear any characters on the current line 378 379 Make way for a new line of length 'length', by outputting enough 380 spaces to clear out the old line. Then remember the new length for 381 next time. 382 383 Args: 384 length: Length of new line, in characters 385 """ 386 if length < self.last_line_len: 387 Print(' ' * (self.last_line_len - length), newline=False) 388 Print('\r', newline=False) 389 self.last_line_len = length 390 sys.stdout.flush() 391 392 def SelectCommit(self, commit, checkout=True): 393 """Checkout the selected commit for this build 394 """ 395 self.commit = commit 396 if checkout and self.checkout: 397 gitutil.Checkout(commit.hash) 398 399 def Make(self, commit, brd, stage, cwd, *args, **kwargs): 400 """Run make 401 402 Args: 403 commit: Commit object that is being built 404 brd: Board object that is being built 405 stage: Stage that we are at (mrproper, config, build) 406 cwd: Directory where make should be run 407 args: Arguments to pass to make 408 kwargs: Arguments to pass to command.RunPipe() 409 """ 410 cmd = [self.gnu_make] + list(args) 411 result = command.RunPipe([cmd], capture=True, capture_stderr=True, 412 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs) 413 if self.verbose_build: 414 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout 415 result.combined = '%s\n' % (' '.join(cmd)) + result.combined 416 return result 417 418 def ProcessResult(self, result): 419 """Process the result of a build, showing progress information 420 421 Args: 422 result: A CommandResult object, which indicates the result for 423 a single build 424 """ 425 col = terminal.Color() 426 if result: 427 target = result.brd.target 428 429 self.upto += 1 430 if result.return_code != 0: 431 self.fail += 1 432 elif result.stderr: 433 self.warned += 1 434 if result.already_done: 435 self.already_done += 1 436 if self._verbose: 437 Print('\r', newline=False) 438 self.ClearLine(0) 439 boards_selected = {target : result.brd} 440 self.ResetResultSummary(boards_selected) 441 self.ProduceResultSummary(result.commit_upto, self.commits, 442 boards_selected) 443 else: 444 target = '(starting)' 445 446 # Display separate counts for ok, warned and fail 447 ok = self.upto - self.warned - self.fail 448 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) 449 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) 450 line += self.col.Color(self.col.RED, '%5d' % self.fail) 451 452 name = ' /%-5d ' % self.count 453 454 # Add our current completion time estimate 455 self._AddTimestamp() 456 if self._complete_delay: 457 name += '%s : ' % self._complete_delay 458 # When building all boards for a commit, we can print a commit 459 # progress message. 460 if result and result.commit_upto is None: 461 name += 'commit %2d/%-3d' % (self.commit_upto + 1, 462 self.commit_count) 463 464 name += target 465 Print(line + name, newline=False) 466 length = 16 + len(name) 467 self.ClearLine(length) 468 469 def _GetOutputDir(self, commit_upto): 470 """Get the name of the output directory for a commit number 471 472 The output directory is typically .../<branch>/<commit>. 473 474 Args: 475 commit_upto: Commit number to use (0..self.count-1) 476 """ 477 commit_dir = None 478 if self.commits: 479 commit = self.commits[commit_upto] 480 subject = commit.subject.translate(trans_valid_chars) 481 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, 482 self.commit_count, commit.hash, subject[:20])) 483 elif not self.no_subdirs: 484 commit_dir = 'current' 485 if not commit_dir: 486 return self.base_dir 487 return os.path.join(self.base_dir, commit_dir) 488 489 def GetBuildDir(self, commit_upto, target): 490 """Get the name of the build directory for a commit number 491 492 The build directory is typically .../<branch>/<commit>/<target>. 493 494 Args: 495 commit_upto: Commit number to use (0..self.count-1) 496 target: Target name 497 """ 498 output_dir = self._GetOutputDir(commit_upto) 499 return os.path.join(output_dir, target) 500 501 def GetDoneFile(self, commit_upto, target): 502 """Get the name of the done file for a commit number 503 504 Args: 505 commit_upto: Commit number to use (0..self.count-1) 506 target: Target name 507 """ 508 return os.path.join(self.GetBuildDir(commit_upto, target), 'done') 509 510 def GetSizesFile(self, commit_upto, target): 511 """Get the name of the sizes file for a commit number 512 513 Args: 514 commit_upto: Commit number to use (0..self.count-1) 515 target: Target name 516 """ 517 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') 518 519 def GetFuncSizesFile(self, commit_upto, target, elf_fname): 520 """Get the name of the funcsizes file for a commit number and ELF file 521 522 Args: 523 commit_upto: Commit number to use (0..self.count-1) 524 target: Target name 525 elf_fname: Filename of elf image 526 """ 527 return os.path.join(self.GetBuildDir(commit_upto, target), 528 '%s.sizes' % elf_fname.replace('/', '-')) 529 530 def GetObjdumpFile(self, commit_upto, target, elf_fname): 531 """Get the name of the objdump file for a commit number and ELF file 532 533 Args: 534 commit_upto: Commit number to use (0..self.count-1) 535 target: Target name 536 elf_fname: Filename of elf image 537 """ 538 return os.path.join(self.GetBuildDir(commit_upto, target), 539 '%s.objdump' % elf_fname.replace('/', '-')) 540 541 def GetErrFile(self, commit_upto, target): 542 """Get the name of the err file for a commit number 543 544 Args: 545 commit_upto: Commit number to use (0..self.count-1) 546 target: Target name 547 """ 548 output_dir = self.GetBuildDir(commit_upto, target) 549 return os.path.join(output_dir, 'err') 550 551 def FilterErrors(self, lines): 552 """Filter out errors in which we have no interest 553 554 We should probably use map(). 555 556 Args: 557 lines: List of error lines, each a string 558 Returns: 559 New list with only interesting lines included 560 """ 561 out_lines = [] 562 for line in lines: 563 if not self.re_make_err.search(line): 564 out_lines.append(line) 565 return out_lines 566 567 def ReadFuncSizes(self, fname, fd): 568 """Read function sizes from the output of 'nm' 569 570 Args: 571 fd: File containing data to read 572 fname: Filename we are reading from (just for errors) 573 574 Returns: 575 Dictionary containing size of each function in bytes, indexed by 576 function name. 577 """ 578 sym = {} 579 for line in fd.readlines(): 580 try: 581 size, type, name = line[:-1].split() 582 except: 583 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) 584 continue 585 if type in 'tTdDbB': 586 # function names begin with '.' on 64-bit powerpc 587 if '.' in name[1:]: 588 name = 'static.' + name.split('.')[0] 589 sym[name] = sym.get(name, 0) + int(size, 16) 590 return sym 591 592 def _ProcessConfig(self, fname): 593 """Read in a .config, autoconf.mk or autoconf.h file 594 595 This function handles all config file types. It ignores comments and 596 any #defines which don't start with CONFIG_. 597 598 Args: 599 fname: Filename to read 600 601 Returns: 602 Dictionary: 603 key: Config name (e.g. CONFIG_DM) 604 value: Config value (e.g. 1) 605 """ 606 config = {} 607 if os.path.exists(fname): 608 with open(fname) as fd: 609 for line in fd: 610 line = line.strip() 611 if line.startswith('#define'): 612 values = line[8:].split(' ', 1) 613 if len(values) > 1: 614 key, value = values 615 else: 616 key = values[0] 617 value = '1' if self.squash_config_y else '' 618 if not key.startswith('CONFIG_'): 619 continue 620 elif not line or line[0] in ['#', '*', '/']: 621 continue 622 else: 623 key, value = line.split('=', 1) 624 if self.squash_config_y and value == 'y': 625 value = '1' 626 config[key] = value 627 return config 628 629 def _ProcessEnvironment(self, fname): 630 """Read in a uboot.env file 631 632 This function reads in environment variables from a file. 633 634 Args: 635 fname: Filename to read 636 637 Returns: 638 Dictionary: 639 key: environment variable (e.g. bootlimit) 640 value: value of environment variable (e.g. 1) 641 """ 642 environment = {} 643 if os.path.exists(fname): 644 with open(fname) as fd: 645 for line in fd.read().split('\0'): 646 try: 647 key, value = line.split('=', 1) 648 environment[key] = value 649 except ValueError: 650 # ignore lines we can't parse 651 pass 652 return environment 653 654 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, 655 read_config, read_environment): 656 """Work out the outcome of a build. 657 658 Args: 659 commit_upto: Commit number to check (0..n-1) 660 target: Target board to check 661 read_func_sizes: True to read function size information 662 read_config: True to read .config and autoconf.h files 663 read_environment: True to read uboot.env files 664 665 Returns: 666 Outcome object 667 """ 668 done_file = self.GetDoneFile(commit_upto, target) 669 sizes_file = self.GetSizesFile(commit_upto, target) 670 sizes = {} 671 func_sizes = {} 672 config = {} 673 environment = {} 674 if os.path.exists(done_file): 675 with open(done_file, 'r') as fd: 676 return_code = int(fd.readline()) 677 err_lines = [] 678 err_file = self.GetErrFile(commit_upto, target) 679 if os.path.exists(err_file): 680 with open(err_file, 'r') as fd: 681 err_lines = self.FilterErrors(fd.readlines()) 682 683 # Decide whether the build was ok, failed or created warnings 684 if return_code: 685 rc = OUTCOME_ERROR 686 elif len(err_lines): 687 rc = OUTCOME_WARNING 688 else: 689 rc = OUTCOME_OK 690 691 # Convert size information to our simple format 692 if os.path.exists(sizes_file): 693 with open(sizes_file, 'r') as fd: 694 for line in fd.readlines(): 695 values = line.split() 696 rodata = 0 697 if len(values) > 6: 698 rodata = int(values[6], 16) 699 size_dict = { 700 'all' : int(values[0]) + int(values[1]) + 701 int(values[2]), 702 'text' : int(values[0]) - rodata, 703 'data' : int(values[1]), 704 'bss' : int(values[2]), 705 'rodata' : rodata, 706 } 707 sizes[values[5]] = size_dict 708 709 if read_func_sizes: 710 pattern = self.GetFuncSizesFile(commit_upto, target, '*') 711 for fname in glob.glob(pattern): 712 with open(fname, 'r') as fd: 713 dict_name = os.path.basename(fname).replace('.sizes', 714 '') 715 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) 716 717 if read_config: 718 output_dir = self.GetBuildDir(commit_upto, target) 719 for name in self.config_filenames: 720 fname = os.path.join(output_dir, name) 721 config[name] = self._ProcessConfig(fname) 722 723 if read_environment: 724 output_dir = self.GetBuildDir(commit_upto, target) 725 fname = os.path.join(output_dir, 'uboot.env') 726 environment = self._ProcessEnvironment(fname) 727 728 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config, 729 environment) 730 731 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {}) 732 733 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, 734 read_config, read_environment): 735 """Calculate a summary of the results of building a commit. 736 737 Args: 738 board_selected: Dict containing boards to summarise 739 commit_upto: Commit number to summarize (0..self.count-1) 740 read_func_sizes: True to read function size information 741 read_config: True to read .config and autoconf.h files 742 read_environment: True to read uboot.env files 743 744 Returns: 745 Tuple: 746 Dict containing boards which passed building this commit. 747 keyed by board.target 748 List containing a summary of error lines 749 Dict keyed by error line, containing a list of the Board 750 objects with that error 751 List containing a summary of warning lines 752 Dict keyed by error line, containing a list of the Board 753 objects with that warning 754 Dictionary keyed by board.target. Each value is a dictionary: 755 key: filename - e.g. '.config' 756 value is itself a dictionary: 757 key: config name 758 value: config value 759 Dictionary keyed by board.target. Each value is a dictionary: 760 key: environment variable 761 value: value of environment variable 762 """ 763 def AddLine(lines_summary, lines_boards, line, board): 764 line = line.rstrip() 765 if line in lines_boards: 766 lines_boards[line].append(board) 767 else: 768 lines_boards[line] = [board] 769 lines_summary.append(line) 770 771 board_dict = {} 772 err_lines_summary = [] 773 err_lines_boards = {} 774 warn_lines_summary = [] 775 warn_lines_boards = {} 776 config = {} 777 environment = {} 778 779 for board in boards_selected.itervalues(): 780 outcome = self.GetBuildOutcome(commit_upto, board.target, 781 read_func_sizes, read_config, 782 read_environment) 783 board_dict[board.target] = outcome 784 last_func = None 785 last_was_warning = False 786 for line in outcome.err_lines: 787 if line: 788 if (self._re_function.match(line) or 789 self._re_files.match(line)): 790 last_func = line 791 else: 792 is_warning = (self._re_warning.match(line) or 793 self._re_dtb_warning.match(line)) 794 is_note = self._re_note.match(line) 795 if is_warning or (last_was_warning and is_note): 796 if last_func: 797 AddLine(warn_lines_summary, warn_lines_boards, 798 last_func, board) 799 AddLine(warn_lines_summary, warn_lines_boards, 800 line, board) 801 else: 802 if last_func: 803 AddLine(err_lines_summary, err_lines_boards, 804 last_func, board) 805 AddLine(err_lines_summary, err_lines_boards, 806 line, board) 807 last_was_warning = is_warning 808 last_func = None 809 tconfig = Config(self.config_filenames, board.target) 810 for fname in self.config_filenames: 811 if outcome.config: 812 for key, value in outcome.config[fname].iteritems(): 813 tconfig.Add(fname, key, value) 814 config[board.target] = tconfig 815 816 tenvironment = Environment(board.target) 817 if outcome.environment: 818 for key, value in outcome.environment.iteritems(): 819 tenvironment.Add(key, value) 820 environment[board.target] = tenvironment 821 822 return (board_dict, err_lines_summary, err_lines_boards, 823 warn_lines_summary, warn_lines_boards, config, environment) 824 825 def AddOutcome(self, board_dict, arch_list, changes, char, color): 826 """Add an output to our list of outcomes for each architecture 827 828 This simple function adds failing boards (changes) to the 829 relevant architecture string, so we can print the results out 830 sorted by architecture. 831 832 Args: 833 board_dict: Dict containing all boards 834 arch_list: Dict keyed by arch name. Value is a string containing 835 a list of board names which failed for that arch. 836 changes: List of boards to add to arch_list 837 color: terminal.Colour object 838 """ 839 done_arch = {} 840 for target in changes: 841 if target in board_dict: 842 arch = board_dict[target].arch 843 else: 844 arch = 'unknown' 845 str = self.col.Color(color, ' ' + target) 846 if not arch in done_arch: 847 str = ' %s %s' % (self.col.Color(color, char), str) 848 done_arch[arch] = True 849 if not arch in arch_list: 850 arch_list[arch] = str 851 else: 852 arch_list[arch] += str 853 854 855 def ColourNum(self, num): 856 color = self.col.RED if num > 0 else self.col.GREEN 857 if num == 0: 858 return '0' 859 return self.col.Color(color, str(num)) 860 861 def ResetResultSummary(self, board_selected): 862 """Reset the results summary ready for use. 863 864 Set up the base board list to be all those selected, and set the 865 error lines to empty. 866 867 Following this, calls to PrintResultSummary() will use this 868 information to work out what has changed. 869 870 Args: 871 board_selected: Dict containing boards to summarise, keyed by 872 board.target 873 """ 874 self._base_board_dict = {} 875 for board in board_selected: 876 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}, 877 {}) 878 self._base_err_lines = [] 879 self._base_warn_lines = [] 880 self._base_err_line_boards = {} 881 self._base_warn_line_boards = {} 882 self._base_config = None 883 self._base_environment = None 884 885 def PrintFuncSizeDetail(self, fname, old, new): 886 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 887 delta, common = [], {} 888 889 for a in old: 890 if a in new: 891 common[a] = 1 892 893 for name in old: 894 if name not in common: 895 remove += 1 896 down += old[name] 897 delta.append([-old[name], name]) 898 899 for name in new: 900 if name not in common: 901 add += 1 902 up += new[name] 903 delta.append([new[name], name]) 904 905 for name in common: 906 diff = new.get(name, 0) - old.get(name, 0) 907 if diff > 0: 908 grow, up = grow + 1, up + diff 909 elif diff < 0: 910 shrink, down = shrink + 1, down - diff 911 delta.append([diff, name]) 912 913 delta.sort() 914 delta.reverse() 915 916 args = [add, -remove, grow, -shrink, up, -down, up - down] 917 if max(args) == 0 and min(args) == 0: 918 return 919 args = [self.ColourNum(x) for x in args] 920 indent = ' ' * 15 921 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % 922 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) 923 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', 924 'delta')) 925 for diff, name in delta: 926 if diff: 927 color = self.col.RED if diff > 0 else self.col.GREEN 928 msg = '%s %-38s %7s %7s %+7d' % (indent, name, 929 old.get(name, '-'), new.get(name,'-'), diff) 930 Print(msg, colour=color) 931 932 933 def PrintSizeDetail(self, target_list, show_bloat): 934 """Show details size information for each board 935 936 Args: 937 target_list: List of targets, each a dict containing: 938 'target': Target name 939 'total_diff': Total difference in bytes across all areas 940 <part_name>: Difference for that part 941 show_bloat: Show detail for each function 942 """ 943 targets_by_diff = sorted(target_list, reverse=True, 944 key=lambda x: x['_total_diff']) 945 for result in targets_by_diff: 946 printed_target = False 947 for name in sorted(result): 948 diff = result[name] 949 if name.startswith('_'): 950 continue 951 if diff != 0: 952 color = self.col.RED if diff > 0 else self.col.GREEN 953 msg = ' %s %+d' % (name, diff) 954 if not printed_target: 955 Print('%10s %-15s:' % ('', result['_target']), 956 newline=False) 957 printed_target = True 958 Print(msg, colour=color, newline=False) 959 if printed_target: 960 Print() 961 if show_bloat: 962 target = result['_target'] 963 outcome = result['_outcome'] 964 base_outcome = self._base_board_dict[target] 965 for fname in outcome.func_sizes: 966 self.PrintFuncSizeDetail(fname, 967 base_outcome.func_sizes[fname], 968 outcome.func_sizes[fname]) 969 970 971 def PrintSizeSummary(self, board_selected, board_dict, show_detail, 972 show_bloat): 973 """Print a summary of image sizes broken down by section. 974 975 The summary takes the form of one line per architecture. The 976 line contains deltas for each of the sections (+ means the section 977 got bigger, - means smaller). The nunmbers are the average number 978 of bytes that a board in this section increased by. 979 980 For example: 981 powerpc: (622 boards) text -0.0 982 arm: (285 boards) text -0.0 983 nds32: (3 boards) text -8.0 984 985 Args: 986 board_selected: Dict containing boards to summarise, keyed by 987 board.target 988 board_dict: Dict containing boards for which we built this 989 commit, keyed by board.target. The value is an Outcome object. 990 show_detail: Show detail for each board 991 show_bloat: Show detail for each function 992 """ 993 arch_list = {} 994 arch_count = {} 995 996 # Calculate changes in size for different image parts 997 # The previous sizes are in Board.sizes, for each board 998 for target in board_dict: 999 if target not in board_selected: 1000 continue 1001 base_sizes = self._base_board_dict[target].sizes 1002 outcome = board_dict[target] 1003 sizes = outcome.sizes 1004 1005 # Loop through the list of images, creating a dict of size 1006 # changes for each image/part. We end up with something like 1007 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} 1008 # which means that U-Boot data increased by 5 bytes and SPL 1009 # text decreased by 4. 1010 err = {'_target' : target} 1011 for image in sizes: 1012 if image in base_sizes: 1013 base_image = base_sizes[image] 1014 # Loop through the text, data, bss parts 1015 for part in sorted(sizes[image]): 1016 diff = sizes[image][part] - base_image[part] 1017 col = None 1018 if diff: 1019 if image == 'u-boot': 1020 name = part 1021 else: 1022 name = image + ':' + part 1023 err[name] = diff 1024 arch = board_selected[target].arch 1025 if not arch in arch_count: 1026 arch_count[arch] = 1 1027 else: 1028 arch_count[arch] += 1 1029 if not sizes: 1030 pass # Only add to our list when we have some stats 1031 elif not arch in arch_list: 1032 arch_list[arch] = [err] 1033 else: 1034 arch_list[arch].append(err) 1035 1036 # We now have a list of image size changes sorted by arch 1037 # Print out a summary of these 1038 for arch, target_list in arch_list.iteritems(): 1039 # Get total difference for each type 1040 totals = {} 1041 for result in target_list: 1042 total = 0 1043 for name, diff in result.iteritems(): 1044 if name.startswith('_'): 1045 continue 1046 total += diff 1047 if name in totals: 1048 totals[name] += diff 1049 else: 1050 totals[name] = diff 1051 result['_total_diff'] = total 1052 result['_outcome'] = board_dict[result['_target']] 1053 1054 count = len(target_list) 1055 printed_arch = False 1056 for name in sorted(totals): 1057 diff = totals[name] 1058 if diff: 1059 # Display the average difference in this name for this 1060 # architecture 1061 avg_diff = float(diff) / count 1062 color = self.col.RED if avg_diff > 0 else self.col.GREEN 1063 msg = ' %s %+1.1f' % (name, avg_diff) 1064 if not printed_arch: 1065 Print('%10s: (for %d/%d boards)' % (arch, count, 1066 arch_count[arch]), newline=False) 1067 printed_arch = True 1068 Print(msg, colour=color, newline=False) 1069 1070 if printed_arch: 1071 Print() 1072 if show_detail: 1073 self.PrintSizeDetail(target_list, show_bloat) 1074 1075 1076 def PrintResultSummary(self, board_selected, board_dict, err_lines, 1077 err_line_boards, warn_lines, warn_line_boards, 1078 config, environment, show_sizes, show_detail, 1079 show_bloat, show_config, show_environment): 1080 """Compare results with the base results and display delta. 1081 1082 Only boards mentioned in board_selected will be considered. This 1083 function is intended to be called repeatedly with the results of 1084 each commit. It therefore shows a 'diff' between what it saw in 1085 the last call and what it sees now. 1086 1087 Args: 1088 board_selected: Dict containing boards to summarise, keyed by 1089 board.target 1090 board_dict: Dict containing boards for which we built this 1091 commit, keyed by board.target. The value is an Outcome object. 1092 err_lines: A list of errors for this commit, or [] if there is 1093 none, or we don't want to print errors 1094 err_line_boards: Dict keyed by error line, containing a list of 1095 the Board objects with that error 1096 warn_lines: A list of warnings for this commit, or [] if there is 1097 none, or we don't want to print errors 1098 warn_line_boards: Dict keyed by warning line, containing a list of 1099 the Board objects with that warning 1100 config: Dictionary keyed by filename - e.g. '.config'. Each 1101 value is itself a dictionary: 1102 key: config name 1103 value: config value 1104 environment: Dictionary keyed by environment variable, Each 1105 value is the value of environment variable. 1106 show_sizes: Show image size deltas 1107 show_detail: Show detail for each board 1108 show_bloat: Show detail for each function 1109 show_config: Show config changes 1110 show_environment: Show environment changes 1111 """ 1112 def _BoardList(line, line_boards): 1113 """Helper function to get a line of boards containing a line 1114 1115 Args: 1116 line: Error line to search for 1117 Return: 1118 String containing a list of boards with that error line, or 1119 '' if the user has not requested such a list 1120 """ 1121 if self._list_error_boards: 1122 names = [] 1123 for board in line_boards[line]: 1124 if not board.target in names: 1125 names.append(board.target) 1126 names_str = '(%s) ' % ','.join(names) 1127 else: 1128 names_str = '' 1129 return names_str 1130 1131 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, 1132 char): 1133 better_lines = [] 1134 worse_lines = [] 1135 for line in lines: 1136 if line not in base_lines: 1137 worse_lines.append(char + '+' + 1138 _BoardList(line, line_boards) + line) 1139 for line in base_lines: 1140 if line not in lines: 1141 better_lines.append(char + '-' + 1142 _BoardList(line, base_line_boards) + line) 1143 return better_lines, worse_lines 1144 1145 def _CalcConfig(delta, name, config): 1146 """Calculate configuration changes 1147 1148 Args: 1149 delta: Type of the delta, e.g. '+' 1150 name: name of the file which changed (e.g. .config) 1151 config: configuration change dictionary 1152 key: config name 1153 value: config value 1154 Returns: 1155 String containing the configuration changes which can be 1156 printed 1157 """ 1158 out = '' 1159 for key in sorted(config.keys()): 1160 out += '%s=%s ' % (key, config[key]) 1161 return '%s %s: %s' % (delta, name, out) 1162 1163 def _AddConfig(lines, name, config_plus, config_minus, config_change): 1164 """Add changes in configuration to a list 1165 1166 Args: 1167 lines: list to add to 1168 name: config file name 1169 config_plus: configurations added, dictionary 1170 key: config name 1171 value: config value 1172 config_minus: configurations removed, dictionary 1173 key: config name 1174 value: config value 1175 config_change: configurations changed, dictionary 1176 key: config name 1177 value: config value 1178 """ 1179 if config_plus: 1180 lines.append(_CalcConfig('+', name, config_plus)) 1181 if config_minus: 1182 lines.append(_CalcConfig('-', name, config_minus)) 1183 if config_change: 1184 lines.append(_CalcConfig('c', name, config_change)) 1185 1186 def _OutputConfigInfo(lines): 1187 for line in lines: 1188 if not line: 1189 continue 1190 if line[0] == '+': 1191 col = self.col.GREEN 1192 elif line[0] == '-': 1193 col = self.col.RED 1194 elif line[0] == 'c': 1195 col = self.col.YELLOW 1196 Print(' ' + line, newline=True, colour=col) 1197 1198 1199 ok_boards = [] # List of boards fixed since last commit 1200 warn_boards = [] # List of boards with warnings since last commit 1201 err_boards = [] # List of new broken boards since last commit 1202 new_boards = [] # List of boards that didn't exist last time 1203 unknown_boards = [] # List of boards that were not built 1204 1205 for target in board_dict: 1206 if target not in board_selected: 1207 continue 1208 1209 # If the board was built last time, add its outcome to a list 1210 if target in self._base_board_dict: 1211 base_outcome = self._base_board_dict[target].rc 1212 outcome = board_dict[target] 1213 if outcome.rc == OUTCOME_UNKNOWN: 1214 unknown_boards.append(target) 1215 elif outcome.rc < base_outcome: 1216 if outcome.rc == OUTCOME_WARNING: 1217 warn_boards.append(target) 1218 else: 1219 ok_boards.append(target) 1220 elif outcome.rc > base_outcome: 1221 if outcome.rc == OUTCOME_WARNING: 1222 warn_boards.append(target) 1223 else: 1224 err_boards.append(target) 1225 else: 1226 new_boards.append(target) 1227 1228 # Get a list of errors that have appeared, and disappeared 1229 better_err, worse_err = _CalcErrorDelta(self._base_err_lines, 1230 self._base_err_line_boards, err_lines, err_line_boards, '') 1231 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, 1232 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') 1233 1234 # Display results by arch 1235 if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards, 1236 worse_err, better_err, worse_warn, better_warn)): 1237 arch_list = {} 1238 self.AddOutcome(board_selected, arch_list, ok_boards, '', 1239 self.col.GREEN) 1240 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+', 1241 self.col.YELLOW) 1242 self.AddOutcome(board_selected, arch_list, err_boards, '+', 1243 self.col.RED) 1244 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE) 1245 if self._show_unknown: 1246 self.AddOutcome(board_selected, arch_list, unknown_boards, '?', 1247 self.col.MAGENTA) 1248 for arch, target_list in arch_list.iteritems(): 1249 Print('%10s: %s' % (arch, target_list)) 1250 self._error_lines += 1 1251 if better_err: 1252 Print('\n'.join(better_err), colour=self.col.GREEN) 1253 self._error_lines += 1 1254 if worse_err: 1255 Print('\n'.join(worse_err), colour=self.col.RED) 1256 self._error_lines += 1 1257 if better_warn: 1258 Print('\n'.join(better_warn), colour=self.col.CYAN) 1259 self._error_lines += 1 1260 if worse_warn: 1261 Print('\n'.join(worse_warn), colour=self.col.MAGENTA) 1262 self._error_lines += 1 1263 1264 if show_sizes: 1265 self.PrintSizeSummary(board_selected, board_dict, show_detail, 1266 show_bloat) 1267 1268 if show_environment and self._base_environment: 1269 lines = [] 1270 1271 for target in board_dict: 1272 if target not in board_selected: 1273 continue 1274 1275 tbase = self._base_environment[target] 1276 tenvironment = environment[target] 1277 environment_plus = {} 1278 environment_minus = {} 1279 environment_change = {} 1280 base = tbase.environment 1281 for key, value in tenvironment.environment.iteritems(): 1282 if key not in base: 1283 environment_plus[key] = value 1284 for key, value in base.iteritems(): 1285 if key not in tenvironment.environment: 1286 environment_minus[key] = value 1287 for key, value in base.iteritems(): 1288 new_value = tenvironment.environment.get(key) 1289 if new_value and value != new_value: 1290 desc = '%s -> %s' % (value, new_value) 1291 environment_change[key] = desc 1292 1293 _AddConfig(lines, target, environment_plus, environment_minus, 1294 environment_change) 1295 1296 _OutputConfigInfo(lines) 1297 1298 if show_config and self._base_config: 1299 summary = {} 1300 arch_config_plus = {} 1301 arch_config_minus = {} 1302 arch_config_change = {} 1303 arch_list = [] 1304 1305 for target in board_dict: 1306 if target not in board_selected: 1307 continue 1308 arch = board_selected[target].arch 1309 if arch not in arch_list: 1310 arch_list.append(arch) 1311 1312 for arch in arch_list: 1313 arch_config_plus[arch] = {} 1314 arch_config_minus[arch] = {} 1315 arch_config_change[arch] = {} 1316 for name in self.config_filenames: 1317 arch_config_plus[arch][name] = {} 1318 arch_config_minus[arch][name] = {} 1319 arch_config_change[arch][name] = {} 1320 1321 for target in board_dict: 1322 if target not in board_selected: 1323 continue 1324 1325 arch = board_selected[target].arch 1326 1327 all_config_plus = {} 1328 all_config_minus = {} 1329 all_config_change = {} 1330 tbase = self._base_config[target] 1331 tconfig = config[target] 1332 lines = [] 1333 for name in self.config_filenames: 1334 if not tconfig.config[name]: 1335 continue 1336 config_plus = {} 1337 config_minus = {} 1338 config_change = {} 1339 base = tbase.config[name] 1340 for key, value in tconfig.config[name].iteritems(): 1341 if key not in base: 1342 config_plus[key] = value 1343 all_config_plus[key] = value 1344 for key, value in base.iteritems(): 1345 if key not in tconfig.config[name]: 1346 config_minus[key] = value 1347 all_config_minus[key] = value 1348 for key, value in base.iteritems(): 1349 new_value = tconfig.config.get(key) 1350 if new_value and value != new_value: 1351 desc = '%s -> %s' % (value, new_value) 1352 config_change[key] = desc 1353 all_config_change[key] = desc 1354 1355 arch_config_plus[arch][name].update(config_plus) 1356 arch_config_minus[arch][name].update(config_minus) 1357 arch_config_change[arch][name].update(config_change) 1358 1359 _AddConfig(lines, name, config_plus, config_minus, 1360 config_change) 1361 _AddConfig(lines, 'all', all_config_plus, all_config_minus, 1362 all_config_change) 1363 summary[target] = '\n'.join(lines) 1364 1365 lines_by_target = {} 1366 for target, lines in summary.iteritems(): 1367 if lines in lines_by_target: 1368 lines_by_target[lines].append(target) 1369 else: 1370 lines_by_target[lines] = [target] 1371 1372 for arch in arch_list: 1373 lines = [] 1374 all_plus = {} 1375 all_minus = {} 1376 all_change = {} 1377 for name in self.config_filenames: 1378 all_plus.update(arch_config_plus[arch][name]) 1379 all_minus.update(arch_config_minus[arch][name]) 1380 all_change.update(arch_config_change[arch][name]) 1381 _AddConfig(lines, name, arch_config_plus[arch][name], 1382 arch_config_minus[arch][name], 1383 arch_config_change[arch][name]) 1384 _AddConfig(lines, 'all', all_plus, all_minus, all_change) 1385 #arch_summary[target] = '\n'.join(lines) 1386 if lines: 1387 Print('%s:' % arch) 1388 _OutputConfigInfo(lines) 1389 1390 for lines, targets in lines_by_target.iteritems(): 1391 if not lines: 1392 continue 1393 Print('%s :' % ' '.join(sorted(targets))) 1394 _OutputConfigInfo(lines.split('\n')) 1395 1396 1397 # Save our updated information for the next call to this function 1398 self._base_board_dict = board_dict 1399 self._base_err_lines = err_lines 1400 self._base_warn_lines = warn_lines 1401 self._base_err_line_boards = err_line_boards 1402 self._base_warn_line_boards = warn_line_boards 1403 self._base_config = config 1404 self._base_environment = environment 1405 1406 # Get a list of boards that did not get built, if needed 1407 not_built = [] 1408 for board in board_selected: 1409 if not board in board_dict: 1410 not_built.append(board) 1411 if not_built: 1412 Print("Boards not built (%d): %s" % (len(not_built), 1413 ', '.join(not_built))) 1414 1415 def ProduceResultSummary(self, commit_upto, commits, board_selected): 1416 (board_dict, err_lines, err_line_boards, warn_lines, 1417 warn_line_boards, config, environment) = self.GetResultSummary( 1418 board_selected, commit_upto, 1419 read_func_sizes=self._show_bloat, 1420 read_config=self._show_config, 1421 read_environment=self._show_environment) 1422 if commits: 1423 msg = '%02d: %s' % (commit_upto + 1, 1424 commits[commit_upto].subject) 1425 Print(msg, colour=self.col.BLUE) 1426 self.PrintResultSummary(board_selected, board_dict, 1427 err_lines if self._show_errors else [], err_line_boards, 1428 warn_lines if self._show_errors else [], warn_line_boards, 1429 config, environment, self._show_sizes, self._show_detail, 1430 self._show_bloat, self._show_config, self._show_environment) 1431 1432 def ShowSummary(self, commits, board_selected): 1433 """Show a build summary for U-Boot for a given board list. 1434 1435 Reset the result summary, then repeatedly call GetResultSummary on 1436 each commit's results, then display the differences we see. 1437 1438 Args: 1439 commit: Commit objects to summarise 1440 board_selected: Dict containing boards to summarise 1441 """ 1442 self.commit_count = len(commits) if commits else 1 1443 self.commits = commits 1444 self.ResetResultSummary(board_selected) 1445 self._error_lines = 0 1446 1447 for commit_upto in range(0, self.commit_count, self._step): 1448 self.ProduceResultSummary(commit_upto, commits, board_selected) 1449 if not self._error_lines: 1450 Print('(no errors to report)', colour=self.col.GREEN) 1451 1452 1453 def SetupBuild(self, board_selected, commits): 1454 """Set up ready to start a build. 1455 1456 Args: 1457 board_selected: Selected boards to build 1458 commits: Selected commits to build 1459 """ 1460 # First work out how many commits we will build 1461 count = (self.commit_count + self._step - 1) / self._step 1462 self.count = len(board_selected) * count 1463 self.upto = self.warned = self.fail = 0 1464 self._timestamps = collections.deque() 1465 1466 def GetThreadDir(self, thread_num): 1467 """Get the directory path to the working dir for a thread. 1468 1469 Args: 1470 thread_num: Number of thread to check. 1471 """ 1472 return os.path.join(self._working_dir, '%02d' % thread_num) 1473 1474 def _PrepareThread(self, thread_num, setup_git): 1475 """Prepare the working directory for a thread. 1476 1477 This clones or fetches the repo into the thread's work directory. 1478 1479 Args: 1480 thread_num: Thread number (0, 1, ...) 1481 setup_git: True to set up a git repo clone 1482 """ 1483 thread_dir = self.GetThreadDir(thread_num) 1484 builderthread.Mkdir(thread_dir) 1485 git_dir = os.path.join(thread_dir, '.git') 1486 1487 # Clone the repo if it doesn't already exist 1488 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so 1489 # we have a private index but uses the origin repo's contents? 1490 if setup_git and self.git_dir: 1491 src_dir = os.path.abspath(self.git_dir) 1492 if os.path.exists(git_dir): 1493 gitutil.Fetch(git_dir, thread_dir) 1494 else: 1495 Print('\rCloning repo for thread %d' % thread_num, 1496 newline=False) 1497 gitutil.Clone(src_dir, thread_dir) 1498 Print('\r%s\r' % (' ' * 30), newline=False) 1499 1500 def _PrepareWorkingSpace(self, max_threads, setup_git): 1501 """Prepare the working directory for use. 1502 1503 Set up the git repo for each thread. 1504 1505 Args: 1506 max_threads: Maximum number of threads we expect to need. 1507 setup_git: True to set up a git repo clone 1508 """ 1509 builderthread.Mkdir(self._working_dir) 1510 for thread in range(max_threads): 1511 self._PrepareThread(thread, setup_git) 1512 1513 def _PrepareOutputSpace(self): 1514 """Get the output directories ready to receive files. 1515 1516 We delete any output directories which look like ones we need to 1517 create. Having left over directories is confusing when the user wants 1518 to check the output manually. 1519 """ 1520 if not self.commits: 1521 return 1522 dir_list = [] 1523 for commit_upto in range(self.commit_count): 1524 dir_list.append(self._GetOutputDir(commit_upto)) 1525 1526 to_remove = [] 1527 for dirname in glob.glob(os.path.join(self.base_dir, '*')): 1528 if dirname not in dir_list: 1529 to_remove.append(dirname) 1530 if to_remove: 1531 Print('Removing %d old build directories' % len(to_remove), 1532 newline=False) 1533 for dirname in to_remove: 1534 shutil.rmtree(dirname) 1535 1536 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): 1537 """Build all commits for a list of boards 1538 1539 Args: 1540 commits: List of commits to be build, each a Commit object 1541 boards_selected: Dict of selected boards, key is target name, 1542 value is Board object 1543 keep_outputs: True to save build output files 1544 verbose: Display build results as they are completed 1545 Returns: 1546 Tuple containing: 1547 - number of boards that failed to build 1548 - number of boards that issued warnings 1549 """ 1550 self.commit_count = len(commits) if commits else 1 1551 self.commits = commits 1552 self._verbose = verbose 1553 1554 self.ResetResultSummary(board_selected) 1555 builderthread.Mkdir(self.base_dir, parents = True) 1556 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), 1557 commits is not None) 1558 self._PrepareOutputSpace() 1559 Print('\rStarting build...', newline=False) 1560 self.SetupBuild(board_selected, commits) 1561 self.ProcessResult(None) 1562 1563 # Create jobs to build all commits for each board 1564 for brd in board_selected.itervalues(): 1565 job = builderthread.BuilderJob() 1566 job.board = brd 1567 job.commits = commits 1568 job.keep_outputs = keep_outputs 1569 job.step = self._step 1570 self.queue.put(job) 1571 1572 term = threading.Thread(target=self.queue.join) 1573 term.setDaemon(True) 1574 term.start() 1575 while term.isAlive(): 1576 term.join(100) 1577 1578 # Wait until we have processed all output 1579 self.out_queue.join() 1580 Print() 1581 self.ClearLine(0) 1582 return (self.fail, self.warned) 1583