1eb8dc403SDave Cobbley""" 2eb8dc403SDave CobbleyBitBake progress handling code 3eb8dc403SDave Cobbley""" 4eb8dc403SDave Cobbley 5eb8dc403SDave Cobbley# Copyright (C) 2016 Intel Corporation 6eb8dc403SDave Cobbley# 7c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 8eb8dc403SDave Cobbley# 9eb8dc403SDave Cobbley 10eb8dc403SDave Cobbleyimport re 11eb8dc403SDave Cobbleyimport time 12eb8dc403SDave Cobbleyimport inspect 13eb8dc403SDave Cobbleyimport bb.event 14eb8dc403SDave Cobbleyimport bb.build 1515ae2509SBrad Bishopfrom bb.build import StdoutNoopContextManager 16eb8dc403SDave Cobbley 17635e0e46SAndrew Geissler 18635e0e46SAndrew Geissler# from https://stackoverflow.com/a/14693789/221061 19635e0e46SAndrew GeisslerANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') 20635e0e46SAndrew Geissler 21635e0e46SAndrew Geissler 22635e0e46SAndrew Geisslerdef filter_color(string): 23635e0e46SAndrew Geissler """ 24635e0e46SAndrew Geissler Filter ANSI escape codes out of |string|, return new string 25635e0e46SAndrew Geissler """ 26635e0e46SAndrew Geissler return ANSI_ESCAPE_REGEX.sub('', string) 27635e0e46SAndrew Geissler 28635e0e46SAndrew Geissler 29635e0e46SAndrew Geisslerdef filter_color_n(string): 30635e0e46SAndrew Geissler """ 31635e0e46SAndrew Geissler Filter ANSI escape codes out of |string|, returns tuple of 32635e0e46SAndrew Geissler (new string, # of ANSI codes removed) 33635e0e46SAndrew Geissler """ 34635e0e46SAndrew Geissler return ANSI_ESCAPE_REGEX.subn('', string) 35635e0e46SAndrew Geissler 36635e0e46SAndrew Geissler 37635e0e46SAndrew Geisslerclass ProgressHandler: 38eb8dc403SDave Cobbley """ 39eb8dc403SDave Cobbley Base class that can pretend to be a file object well enough to be 40eb8dc403SDave Cobbley used to build objects to intercept console output and determine the 41eb8dc403SDave Cobbley progress of some operation. 42eb8dc403SDave Cobbley """ 43eb8dc403SDave Cobbley def __init__(self, d, outfile=None): 44eb8dc403SDave Cobbley self._progress = 0 45eb8dc403SDave Cobbley self._data = d 46eb8dc403SDave Cobbley self._lastevent = 0 47eb8dc403SDave Cobbley if outfile: 48eb8dc403SDave Cobbley self._outfile = outfile 49eb8dc403SDave Cobbley else: 5015ae2509SBrad Bishop self._outfile = StdoutNoopContextManager() 5115ae2509SBrad Bishop 5215ae2509SBrad Bishop def __enter__(self): 5315ae2509SBrad Bishop self._outfile.__enter__() 5415ae2509SBrad Bishop return self 5515ae2509SBrad Bishop 5615ae2509SBrad Bishop def __exit__(self, *excinfo): 5715ae2509SBrad Bishop self._outfile.__exit__(*excinfo) 58eb8dc403SDave Cobbley 59eb8dc403SDave Cobbley def _fire_progress(self, taskprogress, rate=None): 60eb8dc403SDave Cobbley """Internal function to fire the progress event""" 61eb8dc403SDave Cobbley bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data) 62eb8dc403SDave Cobbley 63eb8dc403SDave Cobbley def write(self, string): 64eb8dc403SDave Cobbley self._outfile.write(string) 65eb8dc403SDave Cobbley 66eb8dc403SDave Cobbley def flush(self): 67eb8dc403SDave Cobbley self._outfile.flush() 68eb8dc403SDave Cobbley 69eb8dc403SDave Cobbley def update(self, progress, rate=None): 70eb8dc403SDave Cobbley ts = time.time() 71eb8dc403SDave Cobbley if progress > 100: 72eb8dc403SDave Cobbley progress = 100 73eb8dc403SDave Cobbley if progress != self._progress or self._lastevent + 1 < ts: 74eb8dc403SDave Cobbley self._fire_progress(progress, rate) 75eb8dc403SDave Cobbley self._lastevent = ts 76eb8dc403SDave Cobbley self._progress = progress 77eb8dc403SDave Cobbley 78635e0e46SAndrew Geissler 79eb8dc403SDave Cobbleyclass LineFilterProgressHandler(ProgressHandler): 80eb8dc403SDave Cobbley """ 81eb8dc403SDave Cobbley A ProgressHandler variant that provides the ability to filter out 82eb8dc403SDave Cobbley the lines if they contain progress information. Additionally, it 83eb8dc403SDave Cobbley filters out anything before the last line feed on a line. This can 84eb8dc403SDave Cobbley be used to keep the logs clean of output that we've only enabled for 85eb8dc403SDave Cobbley getting progress, assuming that that can be done on a per-line 86eb8dc403SDave Cobbley basis. 87eb8dc403SDave Cobbley """ 88eb8dc403SDave Cobbley def __init__(self, d, outfile=None): 89eb8dc403SDave Cobbley self._linebuffer = '' 90635e0e46SAndrew Geissler super().__init__(d, outfile) 91eb8dc403SDave Cobbley 92eb8dc403SDave Cobbley def write(self, string): 93eb8dc403SDave Cobbley self._linebuffer += string 94eb8dc403SDave Cobbley while True: 95eb8dc403SDave Cobbley breakpos = self._linebuffer.find('\n') + 1 96eb8dc403SDave Cobbley if breakpos == 0: 97c926e17cSAndrew Geissler # for the case when the line with progress ends with only '\r' 98c926e17cSAndrew Geissler breakpos = self._linebuffer.find('\r') + 1 99c926e17cSAndrew Geissler if breakpos == 0: 100eb8dc403SDave Cobbley break 101eb8dc403SDave Cobbley line = self._linebuffer[:breakpos] 102eb8dc403SDave Cobbley self._linebuffer = self._linebuffer[breakpos:] 103eb8dc403SDave Cobbley # Drop any line feeds and anything that precedes them 104eb8dc403SDave Cobbley lbreakpos = line.rfind('\r') + 1 105c926e17cSAndrew Geissler if lbreakpos and lbreakpos != breakpos: 106eb8dc403SDave Cobbley line = line[lbreakpos:] 107635e0e46SAndrew Geissler if self.writeline(filter_color(line)): 108635e0e46SAndrew Geissler super().write(line) 109eb8dc403SDave Cobbley 110eb8dc403SDave Cobbley def writeline(self, line): 111eb8dc403SDave Cobbley return True 112eb8dc403SDave Cobbley 113635e0e46SAndrew Geissler 114eb8dc403SDave Cobbleyclass BasicProgressHandler(ProgressHandler): 115eb8dc403SDave Cobbley def __init__(self, d, regex=r'(\d+)%', outfile=None): 116635e0e46SAndrew Geissler super().__init__(d, outfile) 117eb8dc403SDave Cobbley self._regex = re.compile(regex) 118eb8dc403SDave Cobbley # Send an initial progress event so the bar gets shown 119eb8dc403SDave Cobbley self._fire_progress(0) 120eb8dc403SDave Cobbley 121eb8dc403SDave Cobbley def write(self, string): 122635e0e46SAndrew Geissler percs = self._regex.findall(filter_color(string)) 123eb8dc403SDave Cobbley if percs: 124eb8dc403SDave Cobbley progress = int(percs[-1]) 125eb8dc403SDave Cobbley self.update(progress) 126635e0e46SAndrew Geissler super().write(string) 127635e0e46SAndrew Geissler 128eb8dc403SDave Cobbley 129eb8dc403SDave Cobbleyclass OutOfProgressHandler(ProgressHandler): 130eb8dc403SDave Cobbley def __init__(self, d, regex, outfile=None): 131635e0e46SAndrew Geissler super().__init__(d, outfile) 132eb8dc403SDave Cobbley self._regex = re.compile(regex) 133eb8dc403SDave Cobbley # Send an initial progress event so the bar gets shown 134eb8dc403SDave Cobbley self._fire_progress(0) 135eb8dc403SDave Cobbley 136eb8dc403SDave Cobbley def write(self, string): 137635e0e46SAndrew Geissler nums = self._regex.findall(filter_color(string)) 138eb8dc403SDave Cobbley if nums: 139eb8dc403SDave Cobbley progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100 140eb8dc403SDave Cobbley self.update(progress) 141635e0e46SAndrew Geissler super().write(string) 142eb8dc403SDave Cobbley 143635e0e46SAndrew Geissler 144635e0e46SAndrew Geisslerclass MultiStageProgressReporter: 145eb8dc403SDave Cobbley """ 146eb8dc403SDave Cobbley Class which allows reporting progress without the caller 147eb8dc403SDave Cobbley having to know where they are in the overall sequence. Useful 148eb8dc403SDave Cobbley for tasks made up of python code spread across multiple 149eb8dc403SDave Cobbley classes / functions - the progress reporter object can 150eb8dc403SDave Cobbley be passed around or stored at the object level and calls 151*7e0e3c0cSAndrew Geissler to next_stage() and update() made wherever needed. 152eb8dc403SDave Cobbley """ 153eb8dc403SDave Cobbley def __init__(self, d, stage_weights, debug=False): 154eb8dc403SDave Cobbley """ 155eb8dc403SDave Cobbley Initialise the progress reporter. 156eb8dc403SDave Cobbley 157eb8dc403SDave Cobbley Parameters: 158eb8dc403SDave Cobbley * d: the datastore (needed for firing the events) 159eb8dc403SDave Cobbley * stage_weights: a list of weight values, one for each stage. 160eb8dc403SDave Cobbley The value is scaled internally so you only need to specify 161eb8dc403SDave Cobbley values relative to other values in the list, so if there 162eb8dc403SDave Cobbley are two stages and the first takes 2s and the second takes 163eb8dc403SDave Cobbley 10s you would specify [2, 10] (or [1, 5], it doesn't matter). 164eb8dc403SDave Cobbley * debug: specify True (and ensure you call finish() at the end) 165eb8dc403SDave Cobbley in order to show a printout of the calculated stage weights 166eb8dc403SDave Cobbley based on timing each stage. Use this to determine what the 167eb8dc403SDave Cobbley weights should be when you're not sure. 168eb8dc403SDave Cobbley """ 169eb8dc403SDave Cobbley self._data = d 170eb8dc403SDave Cobbley total = sum(stage_weights) 171eb8dc403SDave Cobbley self._stage_weights = [float(x)/total for x in stage_weights] 172eb8dc403SDave Cobbley self._stage = -1 173eb8dc403SDave Cobbley self._base_progress = 0 174eb8dc403SDave Cobbley # Send an initial progress event so the bar gets shown 175eb8dc403SDave Cobbley self._fire_progress(0) 176eb8dc403SDave Cobbley self._debug = debug 177eb8dc403SDave Cobbley self._finished = False 178eb8dc403SDave Cobbley if self._debug: 179eb8dc403SDave Cobbley self._last_time = time.time() 180eb8dc403SDave Cobbley self._stage_times = [] 181eb8dc403SDave Cobbley self._stage_total = None 182eb8dc403SDave Cobbley self._callers = [] 183eb8dc403SDave Cobbley 18415ae2509SBrad Bishop def __enter__(self): 18515ae2509SBrad Bishop return self 18615ae2509SBrad Bishop 18715ae2509SBrad Bishop def __exit__(self, *excinfo): 18815ae2509SBrad Bishop pass 18915ae2509SBrad Bishop 190eb8dc403SDave Cobbley def _fire_progress(self, taskprogress): 191eb8dc403SDave Cobbley bb.event.fire(bb.build.TaskProgress(taskprogress), self._data) 192eb8dc403SDave Cobbley 193eb8dc403SDave Cobbley def next_stage(self, stage_total=None): 194eb8dc403SDave Cobbley """ 195eb8dc403SDave Cobbley Move to the next stage. 196eb8dc403SDave Cobbley Parameters: 197eb8dc403SDave Cobbley * stage_total: optional total for progress within the stage, 198eb8dc403SDave Cobbley see update() for details 199eb8dc403SDave Cobbley NOTE: you need to call this before the first stage. 200eb8dc403SDave Cobbley """ 201eb8dc403SDave Cobbley self._stage += 1 202eb8dc403SDave Cobbley self._stage_total = stage_total 203eb8dc403SDave Cobbley if self._stage == 0: 204eb8dc403SDave Cobbley # First stage 205eb8dc403SDave Cobbley if self._debug: 206eb8dc403SDave Cobbley self._last_time = time.time() 207eb8dc403SDave Cobbley else: 208eb8dc403SDave Cobbley if self._stage < len(self._stage_weights): 209eb8dc403SDave Cobbley self._base_progress = sum(self._stage_weights[:self._stage]) * 100 210eb8dc403SDave Cobbley if self._debug: 211eb8dc403SDave Cobbley currtime = time.time() 212eb8dc403SDave Cobbley self._stage_times.append(currtime - self._last_time) 213eb8dc403SDave Cobbley self._last_time = currtime 214eb8dc403SDave Cobbley self._callers.append(inspect.getouterframes(inspect.currentframe())[1]) 215eb8dc403SDave Cobbley elif not self._debug: 216eb8dc403SDave Cobbley bb.warn('ProgressReporter: current stage beyond declared number of stages') 217eb8dc403SDave Cobbley self._base_progress = 100 218eb8dc403SDave Cobbley self._fire_progress(self._base_progress) 219eb8dc403SDave Cobbley 220eb8dc403SDave Cobbley def update(self, stage_progress): 221eb8dc403SDave Cobbley """ 222eb8dc403SDave Cobbley Update progress within the current stage. 223eb8dc403SDave Cobbley Parameters: 224eb8dc403SDave Cobbley * stage_progress: progress value within the stage. If stage_total 225eb8dc403SDave Cobbley was specified when next_stage() was last called, then this 226eb8dc403SDave Cobbley value is considered to be out of stage_total, otherwise it should 227eb8dc403SDave Cobbley be a percentage value from 0 to 100. 228eb8dc403SDave Cobbley """ 229635e0e46SAndrew Geissler progress = None 230eb8dc403SDave Cobbley if self._stage_total: 231eb8dc403SDave Cobbley stage_progress = (float(stage_progress) / self._stage_total) * 100 232eb8dc403SDave Cobbley if self._stage < 0: 233eb8dc403SDave Cobbley bb.warn('ProgressReporter: update called before first call to next_stage()') 234eb8dc403SDave Cobbley elif self._stage < len(self._stage_weights): 235eb8dc403SDave Cobbley progress = self._base_progress + (stage_progress * self._stage_weights[self._stage]) 236eb8dc403SDave Cobbley else: 237eb8dc403SDave Cobbley progress = self._base_progress 238635e0e46SAndrew Geissler if progress: 239eb8dc403SDave Cobbley if progress > 100: 240eb8dc403SDave Cobbley progress = 100 241eb8dc403SDave Cobbley self._fire_progress(progress) 242eb8dc403SDave Cobbley 243eb8dc403SDave Cobbley def finish(self): 244eb8dc403SDave Cobbley if self._finished: 245eb8dc403SDave Cobbley return 246eb8dc403SDave Cobbley self._finished = True 247eb8dc403SDave Cobbley if self._debug: 248eb8dc403SDave Cobbley import math 249eb8dc403SDave Cobbley self._stage_times.append(time.time() - self._last_time) 250eb8dc403SDave Cobbley mintime = max(min(self._stage_times), 0.01) 251eb8dc403SDave Cobbley self._callers.append(None) 252eb8dc403SDave Cobbley stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times] 253eb8dc403SDave Cobbley bb.warn('Stage weights: %s' % stage_weights) 254eb8dc403SDave Cobbley out = [] 255eb8dc403SDave Cobbley for stage_weight, caller in zip(stage_weights, self._callers): 256eb8dc403SDave Cobbley if caller: 257eb8dc403SDave Cobbley out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight)) 258eb8dc403SDave Cobbley else: 259eb8dc403SDave Cobbley out.append('Up to finish: %d' % stage_weight) 260eb8dc403SDave Cobbley bb.warn('Stage times:\n %s' % '\n '.join(out)) 261eb8dc403SDave Cobbley 262635e0e46SAndrew Geissler 263eb8dc403SDave Cobbleyclass MultiStageProcessProgressReporter(MultiStageProgressReporter): 264eb8dc403SDave Cobbley """ 265eb8dc403SDave Cobbley Version of MultiStageProgressReporter intended for use with 266eb8dc403SDave Cobbley standalone processes (such as preparing the runqueue) 267eb8dc403SDave Cobbley """ 268eb8dc403SDave Cobbley def __init__(self, d, processname, stage_weights, debug=False): 269eb8dc403SDave Cobbley self._processname = processname 270eb8dc403SDave Cobbley self._started = False 271635e0e46SAndrew Geissler super().__init__(d, stage_weights, debug) 272eb8dc403SDave Cobbley 273eb8dc403SDave Cobbley def start(self): 274eb8dc403SDave Cobbley if not self._started: 275eb8dc403SDave Cobbley bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data) 276eb8dc403SDave Cobbley self._started = True 277eb8dc403SDave Cobbley 278eb8dc403SDave Cobbley def _fire_progress(self, taskprogress): 279eb8dc403SDave Cobbley if taskprogress == 0: 280eb8dc403SDave Cobbley self.start() 281eb8dc403SDave Cobbley return 282eb8dc403SDave Cobbley bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data) 283eb8dc403SDave Cobbley 284eb8dc403SDave Cobbley def finish(self): 285eb8dc403SDave Cobbley MultiStageProgressReporter.finish(self) 286eb8dc403SDave Cobbley bb.event.fire(bb.event.ProcessFinished(self._processname), self._data) 287eb8dc403SDave Cobbley 288635e0e46SAndrew Geissler 289eb8dc403SDave Cobbleyclass DummyMultiStageProcessProgressReporter(MultiStageProgressReporter): 290eb8dc403SDave Cobbley """ 291eb8dc403SDave Cobbley MultiStageProcessProgressReporter that takes the calls and does nothing 292eb8dc403SDave Cobbley with them (to avoid a bunch of "if progress_reporter:" checks) 293eb8dc403SDave Cobbley """ 294eb8dc403SDave Cobbley def __init__(self): 295635e0e46SAndrew Geissler super().__init__(None, []) 296eb8dc403SDave Cobbley 297eb8dc403SDave Cobbley def _fire_progress(self, taskprogress, rate=None): 298eb8dc403SDave Cobbley pass 299eb8dc403SDave Cobbley 300eb8dc403SDave Cobbley def start(self): 301eb8dc403SDave Cobbley pass 302eb8dc403SDave Cobbley 303eb8dc403SDave Cobbley def next_stage(self, stage_total=None): 304eb8dc403SDave Cobbley pass 305eb8dc403SDave Cobbley 306eb8dc403SDave Cobbley def update(self, stage_progress): 307eb8dc403SDave Cobbley pass 308eb8dc403SDave Cobbley 309eb8dc403SDave Cobbley def finish(self): 310eb8dc403SDave Cobbley pass 311