1""" 2BitBake progress handling code 3""" 4 5# Copyright (C) 2016 Intel Corporation 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import re 11import time 12import inspect 13import bb.event 14import bb.build 15from bb.build import StdoutNoopContextManager 16 17 18# from https://stackoverflow.com/a/14693789/221061 19ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') 20 21 22def filter_color(string): 23 """ 24 Filter ANSI escape codes out of |string|, return new string 25 """ 26 return ANSI_ESCAPE_REGEX.sub('', string) 27 28 29def filter_color_n(string): 30 """ 31 Filter ANSI escape codes out of |string|, returns tuple of 32 (new string, # of ANSI codes removed) 33 """ 34 return ANSI_ESCAPE_REGEX.subn('', string) 35 36 37class ProgressHandler: 38 """ 39 Base class that can pretend to be a file object well enough to be 40 used to build objects to intercept console output and determine the 41 progress of some operation. 42 """ 43 def __init__(self, d, outfile=None): 44 self._progress = 0 45 self._data = d 46 self._lastevent = 0 47 if outfile: 48 self._outfile = outfile 49 else: 50 self._outfile = StdoutNoopContextManager() 51 52 def __enter__(self): 53 self._outfile.__enter__() 54 return self 55 56 def __exit__(self, *excinfo): 57 self._outfile.__exit__(*excinfo) 58 59 def _fire_progress(self, taskprogress, rate=None): 60 """Internal function to fire the progress event""" 61 bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data) 62 63 def write(self, string): 64 self._outfile.write(string) 65 66 def flush(self): 67 self._outfile.flush() 68 69 def update(self, progress, rate=None): 70 ts = time.time() 71 if progress > 100: 72 progress = 100 73 if progress != self._progress or self._lastevent + 1 < ts: 74 self._fire_progress(progress, rate) 75 self._lastevent = ts 76 self._progress = progress 77 78 79class LineFilterProgressHandler(ProgressHandler): 80 """ 81 A ProgressHandler variant that provides the ability to filter out 82 the lines if they contain progress information. Additionally, it 83 filters out anything before the last line feed on a line. This can 84 be used to keep the logs clean of output that we've only enabled for 85 getting progress, assuming that that can be done on a per-line 86 basis. 87 """ 88 def __init__(self, d, outfile=None): 89 self._linebuffer = '' 90 super().__init__(d, outfile) 91 92 def write(self, string): 93 self._linebuffer += string 94 while True: 95 breakpos = self._linebuffer.find('\n') + 1 96 if breakpos == 0: 97 # for the case when the line with progress ends with only '\r' 98 breakpos = self._linebuffer.find('\r') + 1 99 if breakpos == 0: 100 break 101 line = self._linebuffer[:breakpos] 102 self._linebuffer = self._linebuffer[breakpos:] 103 # Drop any line feeds and anything that precedes them 104 lbreakpos = line.rfind('\r') + 1 105 if lbreakpos and lbreakpos != breakpos: 106 line = line[lbreakpos:] 107 if self.writeline(filter_color(line)): 108 super().write(line) 109 110 def writeline(self, line): 111 return True 112 113 114class BasicProgressHandler(ProgressHandler): 115 def __init__(self, d, regex=r'(\d+)%', outfile=None): 116 super().__init__(d, outfile) 117 self._regex = re.compile(regex) 118 # Send an initial progress event so the bar gets shown 119 self._fire_progress(0) 120 121 def write(self, string): 122 percs = self._regex.findall(filter_color(string)) 123 if percs: 124 progress = int(percs[-1]) 125 self.update(progress) 126 super().write(string) 127 128 129class OutOfProgressHandler(ProgressHandler): 130 def __init__(self, d, regex, outfile=None): 131 super().__init__(d, outfile) 132 self._regex = re.compile(regex) 133 # Send an initial progress event so the bar gets shown 134 self._fire_progress(0) 135 136 def write(self, string): 137 nums = self._regex.findall(filter_color(string)) 138 if nums: 139 progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100 140 self.update(progress) 141 super().write(string) 142 143 144class MultiStageProgressReporter: 145 """ 146 Class which allows reporting progress without the caller 147 having to know where they are in the overall sequence. Useful 148 for tasks made up of python code spread across multiple 149 classes / functions - the progress reporter object can 150 be passed around or stored at the object level and calls 151 to next_stage() and update() made wherever needed. 152 """ 153 def __init__(self, d, stage_weights, debug=False): 154 """ 155 Initialise the progress reporter. 156 157 Parameters: 158 * d: the datastore (needed for firing the events) 159 * stage_weights: a list of weight values, one for each stage. 160 The value is scaled internally so you only need to specify 161 values relative to other values in the list, so if there 162 are two stages and the first takes 2s and the second takes 163 10s you would specify [2, 10] (or [1, 5], it doesn't matter). 164 * debug: specify True (and ensure you call finish() at the end) 165 in order to show a printout of the calculated stage weights 166 based on timing each stage. Use this to determine what the 167 weights should be when you're not sure. 168 """ 169 self._data = d 170 total = sum(stage_weights) 171 self._stage_weights = [float(x)/total for x in stage_weights] 172 self._stage = -1 173 self._base_progress = 0 174 # Send an initial progress event so the bar gets shown 175 self._fire_progress(0) 176 self._debug = debug 177 self._finished = False 178 if self._debug: 179 self._last_time = time.time() 180 self._stage_times = [] 181 self._stage_total = None 182 self._callers = [] 183 184 def __enter__(self): 185 return self 186 187 def __exit__(self, *excinfo): 188 pass 189 190 def _fire_progress(self, taskprogress): 191 bb.event.fire(bb.build.TaskProgress(taskprogress), self._data) 192 193 def next_stage(self, stage_total=None): 194 """ 195 Move to the next stage. 196 Parameters: 197 * stage_total: optional total for progress within the stage, 198 see update() for details 199 NOTE: you need to call this before the first stage. 200 """ 201 self._stage += 1 202 self._stage_total = stage_total 203 if self._stage == 0: 204 # First stage 205 if self._debug: 206 self._last_time = time.time() 207 else: 208 if self._stage < len(self._stage_weights): 209 self._base_progress = sum(self._stage_weights[:self._stage]) * 100 210 if self._debug: 211 currtime = time.time() 212 self._stage_times.append(currtime - self._last_time) 213 self._last_time = currtime 214 self._callers.append(inspect.getouterframes(inspect.currentframe())[1]) 215 elif not self._debug: 216 bb.warn('ProgressReporter: current stage beyond declared number of stages') 217 self._base_progress = 100 218 self._fire_progress(self._base_progress) 219 220 def update(self, stage_progress): 221 """ 222 Update progress within the current stage. 223 Parameters: 224 * stage_progress: progress value within the stage. If stage_total 225 was specified when next_stage() was last called, then this 226 value is considered to be out of stage_total, otherwise it should 227 be a percentage value from 0 to 100. 228 """ 229 progress = None 230 if self._stage_total: 231 stage_progress = (float(stage_progress) / self._stage_total) * 100 232 if self._stage < 0: 233 bb.warn('ProgressReporter: update called before first call to next_stage()') 234 elif self._stage < len(self._stage_weights): 235 progress = self._base_progress + (stage_progress * self._stage_weights[self._stage]) 236 else: 237 progress = self._base_progress 238 if progress: 239 if progress > 100: 240 progress = 100 241 self._fire_progress(progress) 242 243 def finish(self): 244 if self._finished: 245 return 246 self._finished = True 247 if self._debug: 248 import math 249 self._stage_times.append(time.time() - self._last_time) 250 mintime = max(min(self._stage_times), 0.01) 251 self._callers.append(None) 252 stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times] 253 bb.warn('Stage weights: %s' % stage_weights) 254 out = [] 255 for stage_weight, caller in zip(stage_weights, self._callers): 256 if caller: 257 out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight)) 258 else: 259 out.append('Up to finish: %d' % stage_weight) 260 bb.warn('Stage times:\n %s' % '\n '.join(out)) 261 262 263class MultiStageProcessProgressReporter(MultiStageProgressReporter): 264 """ 265 Version of MultiStageProgressReporter intended for use with 266 standalone processes (such as preparing the runqueue) 267 """ 268 def __init__(self, d, processname, stage_weights, debug=False): 269 self._processname = processname 270 self._started = False 271 super().__init__(d, stage_weights, debug) 272 273 def start(self): 274 if not self._started: 275 bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data) 276 self._started = True 277 278 def _fire_progress(self, taskprogress): 279 if taskprogress == 0: 280 self.start() 281 return 282 bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data) 283 284 def finish(self): 285 MultiStageProgressReporter.finish(self) 286 bb.event.fire(bb.event.ProcessFinished(self._processname), self._data) 287 288 289class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter): 290 """ 291 MultiStageProcessProgressReporter that takes the calls and does nothing 292 with them (to avoid a bunch of "if progress_reporter:" checks) 293 """ 294 def __init__(self): 295 super().__init__(None, []) 296 297 def _fire_progress(self, taskprogress, rate=None): 298 pass 299 300 def start(self): 301 pass 302 303 def next_stage(self, stage_total=None): 304 pass 305 306 def update(self, stage_progress): 307 pass 308 309 def finish(self): 310 pass 311