Lines Matching +full:self +full:- +full:working
1 # SPDX-License-Identifier: GPL-2.0+
4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
37 The source repo (self.git_dir) contains all the commits to be built. Each
40 commit and builds it (typically without re-configuring). When it runs out
44 Clearly the builder threads could work either way - they could check out a
54 Many threads can be working at once, but each has its own working directory.
66 Buildman also create working directories for each thread, in a .bm-work/
69 As an example, say we are building branch 'us-net' for boards 'sandbox' and
70 'seaboard', and say that us-net has two commits. We will have directories
73 us-net/ base directory
74 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
76 u-boot.bin
78 u-boot.bin
79 02_of_02_g4ed4ebc_net--Check-tftp-comp/
81 u-boot.bin
83 u-boot.bin
84 .bm-work/
85 00/ working directory for thread 0 (contains source checkout)
87 01/ working directory for thread 1
90 u-boot/ source directory
98 trans_valid_chars = string.maketrans('/: ', '---')
99 trans_valid_chars = trans_valid_chars.decode('latin-1')
102 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
106 '.config', '.config-spl', '.config-tpl',
107 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
108 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
113 def __init__(self, config_filename, target): argument
114 self.target = target
115 self.config = {}
117 self.config[fname] = {}
119 def Add(self, fname, key, value): argument
120 self.config[fname][key] = value
122 def __hash__(self): argument
124 for fname in self.config:
125 for key, value in self.config[fname].iteritems():
132 def __init__(self, target): argument
133 self.target = target
134 self.environment = {}
136 def Add(self, key, value): argument
137 self.environment[key] = value
140 """Class for building U-Boot for a particular commit.
142 Public members: (many should ->private)
155 force_build_failures: If a previously-built build (i.e. built on
160 num_jobs: Number of jobs to run at once (passed to make as -j)
167 upto: Current commit number we are building (0.count-1)
169 force_reconfig: Reconfigure U-Boot on each comiit. This disables
175 in_tree: Build U-Boot in-tree instead of specifying an output
177 only useful for testing in-tree builds.
180 _base_board_dict: Last-summarised Dict of boards
181 _base_err_lines: Last-summarised list of errors
182 _base_warn_lines: Last-summarised list of warnings
191 _working_dir: Base working directory containing all threads
200 - Each value is itself a dictionary containing
203 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
207 config: Dictionary keyed by filename - e.g. '.config'. Each
214 def __init__(self, rc, err_lines, sizes, func_sizes, config, argument
216 self.rc = rc
217 self.err_lines = err_lines
218 self.sizes = sizes
219 self.func_sizes = func_sizes
220 self.config = config
221 self.environment = environment
223 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, argument
236 num_jobs: Number of jobs to run at once (passed to make as -j)
246 verbose_build: Run build with V=1 and don't use 'make -s'
250 board rather than a thread-specific directory
255 self.toolchains = toolchains
256 self.base_dir = base_dir
257 self._working_dir = os.path.join(base_dir, '.bm-work')
258 self.threads = []
259 self.do_make = self.Make
260 self.gnu_make = gnu_make
261 self.checkout = checkout
262 self.num_threads = num_threads
263 self.num_jobs = num_jobs
264 self.already_done = 0
265 self.force_build = False
266 self.git_dir = git_dir
267 self._show_unknown = show_unknown
268 self._timestamp_count = 10
269 self._build_period_us = None
270 self._complete_delay = None
271 self._next_delay_update = datetime.now()
272 self.force_config_on_failure = True
273 self.force_build_failures = False
274 self.force_reconfig = False
275 self._step = step
276 self.in_tree = False
277 self._error_lines = 0
278 self.no_subdirs = no_subdirs
279 self.full_path = full_path
280 self.verbose_build = verbose_build
281 self.config_only = config_only
282 self.squash_config_y = squash_config_y
283 self.config_filenames = BASE_CONFIG_FILENAMES
284 if not self.squash_config_y:
285 self.config_filenames += EXTRA_CONFIG_FILENAMES
287 self.warnings_as_errors = warnings_as_errors
288 self.col = terminal.Color()
290 self._re_function = re.compile('(.*): In function.*')
291 self._re_files = re.compile('In file included from.*')
292 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
293 self._re_dtb_warning = re.compile('(.*): Warning .*')
294 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
296 self.queue = Queue.Queue()
297 self.out_queue = Queue.Queue()
298 for i in range(self.num_threads):
299 t = builderthread.BuilderThread(self, i, incremental,
303 self.threads.append(t)
305 self.last_line_len = 0
306 t = builderthread.ResultThread(self)
309 self.threads.append(t)
312 self.re_make_err = re.compile('|'.join(ignore_lines))
314 # Handle existing graceful with SIGINT / Ctrl-C
315 signal.signal(signal.SIGINT, self.signal_handler)
317 def __del__(self): argument
319 for t in self.threads:
322 def signal_handler(self, signal, frame): argument
325 def SetDisplayOptions(self, show_errors=False, show_sizes=False, argument
339 self._show_errors = show_errors
340 self._show_sizes = show_sizes
341 self._show_detail = show_detail
342 self._show_bloat = show_bloat
343 self._list_error_boards = list_error_boards
344 self._show_config = show_config
345 self._show_environment = show_environment
347 def _AddTimestamp(self): argument
354 self._timestamps.append(now)
355 count = len(self._timestamps)
356 delta = self._timestamps[-1] - self._timestamps[0]
361 if count > 1 and self._next_delay_update < now:
362 self._next_delay_update = now + timedelta(seconds=2)
364 self._build_period = float(seconds) / count
365 todo = self.count - self.upto
366 self._complete_delay = timedelta(microseconds=
367 self._build_period * todo * 1000000)
369 self._complete_delay -= timedelta(
370 microseconds=self._complete_delay.microseconds)
373 self._timestamps.popleft()
374 count -= 1
376 def ClearLine(self, length): argument
386 if length < self.last_line_len:
387 Print(' ' * (self.last_line_len - length), newline=False)
389 self.last_line_len = length
392 def SelectCommit(self, commit, checkout=True): argument
395 self.commit = commit
396 if checkout and self.checkout:
399 def Make(self, commit, brd, stage, cwd, *args, **kwargs): argument
410 cmd = [self.gnu_make] + list(args)
413 if self.verbose_build:
418 def ProcessResult(self, result): argument
429 self.upto += 1
431 self.fail += 1
433 self.warned += 1
435 self.already_done += 1
436 if self._verbose:
438 self.ClearLine(0)
440 self.ResetResultSummary(boards_selected)
441 self.ProduceResultSummary(result.commit_upto, self.commits,
447 ok = self.upto - self.warned - self.fail
448 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
449 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
450 line += self.col.Color(self.col.RED, '%5d' % self.fail)
452 name = ' /%-5d ' % self.count
455 self._AddTimestamp()
456 if self._complete_delay:
457 name += '%s : ' % self._complete_delay
461 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
462 self.commit_count)
467 self.ClearLine(length)
469 def _GetOutputDir(self, commit_upto): argument
475 commit_upto: Commit number to use (0..self.count-1)
478 if self.commits:
479 commit = self.commits[commit_upto]
482 self.commit_count, commit.hash, subject[:20]))
483 elif not self.no_subdirs:
486 return self.base_dir
487 return os.path.join(self.base_dir, commit_dir)
489 def GetBuildDir(self, commit_upto, target): argument
495 commit_upto: Commit number to use (0..self.count-1)
498 output_dir = self._GetOutputDir(commit_upto)
501 def GetDoneFile(self, commit_upto, target): argument
505 commit_upto: Commit number to use (0..self.count-1)
508 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
510 def GetSizesFile(self, commit_upto, target): argument
514 commit_upto: Commit number to use (0..self.count-1)
517 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
519 def GetFuncSizesFile(self, commit_upto, target, elf_fname): argument
523 commit_upto: Commit number to use (0..self.count-1)
527 return os.path.join(self.GetBuildDir(commit_upto, target),
528 '%s.sizes' % elf_fname.replace('/', '-'))
530 def GetObjdumpFile(self, commit_upto, target, elf_fname): argument
534 commit_upto: Commit number to use (0..self.count-1)
538 return os.path.join(self.GetBuildDir(commit_upto, target),
539 '%s.objdump' % elf_fname.replace('/', '-'))
541 def GetErrFile(self, commit_upto, target): argument
545 commit_upto: Commit number to use (0..self.count-1)
548 output_dir = self.GetBuildDir(commit_upto, target)
551 def FilterErrors(self, lines): argument
563 if not self.re_make_err.search(line):
567 def ReadFuncSizes(self, fname, fd): argument
581 size, type, name = line[:-1].split()
583 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
586 # function names begin with '.' on 64-bit powerpc
592 def _ProcessConfig(self, fname): argument
617 value = '1' if self.squash_config_y else ''
624 if self.squash_config_y and value == 'y':
629 def _ProcessEnvironment(self, fname): argument
654 def GetBuildOutcome(self, commit_upto, target, read_func_sizes, argument
659 commit_upto: Commit number to check (0..n-1)
668 done_file = self.GetDoneFile(commit_upto, target)
669 sizes_file = self.GetSizesFile(commit_upto, target)
678 err_file = self.GetErrFile(commit_upto, target)
681 err_lines = self.FilterErrors(fd.readlines())
702 'text' : int(values[0]) - rodata,
710 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
715 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
718 output_dir = self.GetBuildDir(commit_upto, target)
719 for name in self.config_filenames:
721 config[name] = self._ProcessConfig(fname)
724 output_dir = self.GetBuildDir(commit_upto, target)
726 environment = self._ProcessEnvironment(fname)
733 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, argument
739 commit_upto: Commit number to summarize (0..self.count-1)
755 key: filename - e.g. '.config'
780 outcome = self.GetBuildOutcome(commit_upto, board.target,
788 if (self._re_function.match(line) or
789 self._re_files.match(line)):
792 is_warning = (self._re_warning.match(line) or
793 self._re_dtb_warning.match(line))
794 is_note = self._re_note.match(line)
809 tconfig = Config(self.config_filenames, board.target)
810 for fname in self.config_filenames:
825 def AddOutcome(self, board_dict, arch_list, changes, char, color): argument
845 str = self.col.Color(color, ' ' + target)
847 str = ' %s %s' % (self.col.Color(color, char), str)
855 def ColourNum(self, num): argument
856 color = self.col.RED if num > 0 else self.col.GREEN
859 return self.col.Color(color, str(num))
861 def ResetResultSummary(self, board_selected): argument
874 self._base_board_dict = {}
876 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
878 self._base_err_lines = []
879 self._base_warn_lines = []
880 self._base_err_line_boards = {}
881 self._base_warn_line_boards = {}
882 self._base_config = None
883 self._base_environment = None
885 def PrintFuncSizeDetail(self, fname, old, new): argument
897 delta.append([-old[name], name])
906 diff = new.get(name, 0) - old.get(name, 0)
910 shrink, down = shrink + 1, down - diff
916 args = [add, -remove, grow, -shrink, up, -down, up - down]
919 args = [self.ColourNum(x) for x in args]
922 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
923 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
927 color = self.col.RED if diff > 0 else self.col.GREEN
928 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
929 old.get(name, '-'), new.get(name,'-'), diff)
933 def PrintSizeDetail(self, target_list, show_bloat): argument
952 color = self.col.RED if diff > 0 else self.col.GREEN
955 Print('%10s %-15s:' % ('', result['_target']),
964 base_outcome = self._base_board_dict[target]
966 self.PrintFuncSizeDetail(fname,
971 def PrintSizeSummary(self, board_selected, board_dict, show_detail, argument
977 got bigger, - means smaller). The nunmbers are the average number
981 powerpc: (622 boards) text -0.0
982 arm: (285 boards) text -0.0
983 nds32: (3 boards) text -8.0
1001 base_sizes = self._base_board_dict[target].sizes
1007 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1008 # which means that U-Boot data increased by 5 bytes and SPL
1016 diff = sizes[image][part] - base_image[part]
1019 if image == 'u-boot':
1062 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1073 self.PrintSizeDetail(target_list, show_bloat)
1076 def PrintResultSummary(self, board_selected, board_dict, err_lines, argument
1100 config: Dictionary keyed by filename - e.g. '.config'. Each
1121 if self._list_error_boards:
1141 better_lines.append(char + '-' +
1182 lines.append(_CalcConfig('-', name, config_minus))
1191 col = self.col.GREEN
1192 elif line[0] == '-':
1193 col = self.col.RED
1195 col = self.col.YELLOW
1210 if target in self._base_board_dict:
1211 base_outcome = self._base_board_dict[target].rc
1229 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1230 self._base_err_line_boards, err_lines, err_line_boards, '')
1231 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1232 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1238 self.AddOutcome(board_selected, arch_list, ok_boards, '',
1239 self.col.GREEN)
1240 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1241 self.col.YELLOW)
1242 self.AddOutcome(board_selected, arch_list, err_boards, '+',
1243 self.col.RED)
1244 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1245 if self._show_unknown:
1246 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1247 self.col.MAGENTA)
1250 self._error_lines += 1
1252 Print('\n'.join(better_err), colour=self.col.GREEN)
1253 self._error_lines += 1
1255 Print('\n'.join(worse_err), colour=self.col.RED)
1256 self._error_lines += 1
1258 Print('\n'.join(better_warn), colour=self.col.CYAN)
1259 self._error_lines += 1
1261 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
1262 self._error_lines += 1
1265 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1268 if show_environment and self._base_environment:
1275 tbase = self._base_environment[target]
1290 desc = '%s -> %s' % (value, new_value)
1298 if show_config and self._base_config:
1316 for name in self.config_filenames:
1330 tbase = self._base_config[target]
1333 for name in self.config_filenames:
1351 desc = '%s -> %s' % (value, new_value)
1377 for name in self.config_filenames:
1398 self._base_board_dict = board_dict
1399 self._base_err_lines = err_lines
1400 self._base_warn_lines = warn_lines
1401 self._base_err_line_boards = err_line_boards
1402 self._base_warn_line_boards = warn_line_boards
1403 self._base_config = config
1404 self._base_environment = environment
1415 def ProduceResultSummary(self, commit_upto, commits, board_selected): argument
1417 warn_line_boards, config, environment) = self.GetResultSummary(
1419 read_func_sizes=self._show_bloat,
1420 read_config=self._show_config,
1421 read_environment=self._show_environment)
1425 Print(msg, colour=self.col.BLUE)
1426 self.PrintResultSummary(board_selected, board_dict,
1427 err_lines if self._show_errors else [], err_line_boards,
1428 warn_lines if self._show_errors else [], warn_line_boards,
1429 config, environment, self._show_sizes, self._show_detail,
1430 self._show_bloat, self._show_config, self._show_environment)
1432 def ShowSummary(self, commits, board_selected): argument
1433 """Show a build summary for U-Boot for a given board list.
1442 self.commit_count = len(commits) if commits else 1
1443 self.commits = commits
1444 self.ResetResultSummary(board_selected)
1445 self._error_lines = 0
1447 for commit_upto in range(0, self.commit_count, self._step):
1448 self.ProduceResultSummary(commit_upto, commits, board_selected)
1449 if not self._error_lines:
1450 Print('(no errors to report)', colour=self.col.GREEN)
1453 def SetupBuild(self, board_selected, commits): argument
1461 count = (self.commit_count + self._step - 1) / self._step
1462 self.count = len(board_selected) * count
1463 self.upto = self.warned = self.fail = 0
1464 self._timestamps = collections.deque()
1466 def GetThreadDir(self, thread_num): argument
1467 """Get the directory path to the working dir for a thread.
1472 return os.path.join(self._working_dir, '%02d' % thread_num)
1474 def _PrepareThread(self, thread_num, setup_git): argument
1475 """Prepare the working directory for a thread.
1483 thread_dir = self.GetThreadDir(thread_num)
1490 if setup_git and self.git_dir:
1491 src_dir = os.path.abspath(self.git_dir)
1500 def _PrepareWorkingSpace(self, max_threads, setup_git): argument
1501 """Prepare the working directory for use.
1509 builderthread.Mkdir(self._working_dir)
1511 self._PrepareThread(thread, setup_git)
1513 def _PrepareOutputSpace(self): argument
1520 if not self.commits:
1523 for commit_upto in range(self.commit_count):
1524 dir_list.append(self._GetOutputDir(commit_upto))
1527 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1536 def BuildBoards(self, commits, board_selected, keep_outputs, verbose): argument
1547 - number of boards that failed to build
1548 - number of boards that issued warnings
1550 self.commit_count = len(commits) if commits else 1
1551 self.commits = commits
1552 self._verbose = verbose
1554 self.ResetResultSummary(board_selected)
1555 builderthread.Mkdir(self.base_dir, parents = True)
1556 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1558 self._PrepareOutputSpace()
1560 self.SetupBuild(board_selected, commits)
1561 self.ProcessResult(None)
1569 job.step = self._step
1570 self.queue.put(job)
1572 term = threading.Thread(target=self.queue.join)
1579 self.out_queue.join()
1581 self.ClearLine(0)
1582 return (self.fail, self.warned)