1eb8dc403SDave Cobbley""" 2eb8dc403SDave CobbleyBitBake 'msg' implementation 3eb8dc403SDave Cobbley 4eb8dc403SDave CobbleyMessage handling infrastructure for bitbake 5eb8dc403SDave Cobbley 6eb8dc403SDave Cobbley""" 7eb8dc403SDave Cobbley 8eb8dc403SDave Cobbley# Copyright (C) 2006 Richard Purdie 9eb8dc403SDave Cobbley# 10c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 11eb8dc403SDave Cobbley# 12eb8dc403SDave Cobbley 13eb8dc403SDave Cobbleyimport sys 14eb8dc403SDave Cobbleyimport copy 15eb8dc403SDave Cobbleyimport logging 1682c905dcSAndrew Geisslerimport logging.config 17c9f7865aSAndrew Geisslerimport os 18eb8dc403SDave Cobbleyfrom itertools import groupby 19eb8dc403SDave Cobbleyimport bb 20eb8dc403SDave Cobbleyimport bb.event 21eb8dc403SDave Cobbley 22eb8dc403SDave Cobbleyclass BBLogFormatter(logging.Formatter): 23eb8dc403SDave Cobbley """Formatter which ensures that our 'plain' messages (logging.INFO + 1) are used as is""" 24eb8dc403SDave Cobbley 25eb8dc403SDave Cobbley DEBUG3 = logging.DEBUG - 2 26eb8dc403SDave Cobbley DEBUG2 = logging.DEBUG - 1 27eb8dc403SDave Cobbley DEBUG = logging.DEBUG 28eb8dc403SDave Cobbley VERBOSE = logging.INFO - 1 29eb8dc403SDave Cobbley NOTE = logging.INFO 30eb8dc403SDave Cobbley PLAIN = logging.INFO + 1 311a4b7ee2SBrad Bishop VERBNOTE = logging.INFO + 2 32eb8dc403SDave Cobbley ERROR = logging.ERROR 337e0e3c0cSAndrew Geissler ERRORONCE = logging.ERROR - 1 34eb8dc403SDave Cobbley WARNING = logging.WARNING 357e0e3c0cSAndrew Geissler WARNONCE = logging.WARNING - 1 36eb8dc403SDave Cobbley CRITICAL = logging.CRITICAL 37eb8dc403SDave Cobbley 38eb8dc403SDave Cobbley levelnames = { 39eb8dc403SDave Cobbley DEBUG3 : 'DEBUG', 40eb8dc403SDave Cobbley DEBUG2 : 'DEBUG', 41eb8dc403SDave Cobbley DEBUG : 'DEBUG', 42eb8dc403SDave Cobbley VERBOSE: 'NOTE', 43eb8dc403SDave Cobbley NOTE : 'NOTE', 44eb8dc403SDave Cobbley PLAIN : '', 451a4b7ee2SBrad Bishop VERBNOTE: 'NOTE', 46eb8dc403SDave Cobbley WARNING : 'WARNING', 477e0e3c0cSAndrew Geissler WARNONCE : 'WARNING', 48eb8dc403SDave Cobbley ERROR : 'ERROR', 497e0e3c0cSAndrew Geissler ERRORONCE : 'ERROR', 50eb8dc403SDave Cobbley CRITICAL: 'ERROR', 51eb8dc403SDave Cobbley } 52eb8dc403SDave Cobbley 53eb8dc403SDave Cobbley color_enabled = False 54eb8dc403SDave Cobbley BASECOLOR, BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(29,38)) 55eb8dc403SDave Cobbley 56eb8dc403SDave Cobbley COLORS = { 57eb8dc403SDave Cobbley DEBUG3 : CYAN, 58eb8dc403SDave Cobbley DEBUG2 : CYAN, 59eb8dc403SDave Cobbley DEBUG : CYAN, 60eb8dc403SDave Cobbley VERBOSE : BASECOLOR, 61eb8dc403SDave Cobbley NOTE : BASECOLOR, 62eb8dc403SDave Cobbley PLAIN : BASECOLOR, 631a4b7ee2SBrad Bishop VERBNOTE: BASECOLOR, 64eb8dc403SDave Cobbley WARNING : YELLOW, 657e0e3c0cSAndrew Geissler WARNONCE : YELLOW, 66eb8dc403SDave Cobbley ERROR : RED, 677e0e3c0cSAndrew Geissler ERRORONCE : RED, 68eb8dc403SDave Cobbley CRITICAL: RED, 69eb8dc403SDave Cobbley } 70eb8dc403SDave Cobbley 71eb8dc403SDave Cobbley BLD = '\033[1;%dm' 72eb8dc403SDave Cobbley STD = '\033[%dm' 73eb8dc403SDave Cobbley RST = '\033[0m' 74eb8dc403SDave Cobbley 75eb8dc403SDave Cobbley def getLevelName(self, levelno): 76eb8dc403SDave Cobbley try: 77eb8dc403SDave Cobbley return self.levelnames[levelno] 78eb8dc403SDave Cobbley except KeyError: 79eb8dc403SDave Cobbley self.levelnames[levelno] = value = 'Level %d' % levelno 80eb8dc403SDave Cobbley return value 81eb8dc403SDave Cobbley 82eb8dc403SDave Cobbley def format(self, record): 83eb8dc403SDave Cobbley record.levelname = self.getLevelName(record.levelno) 84eb8dc403SDave Cobbley if record.levelno == self.PLAIN: 85eb8dc403SDave Cobbley msg = record.getMessage() 86eb8dc403SDave Cobbley else: 87eb8dc403SDave Cobbley if self.color_enabled: 88eb8dc403SDave Cobbley record = self.colorize(record) 89eb8dc403SDave Cobbley msg = logging.Formatter.format(self, record) 90eb8dc403SDave Cobbley if hasattr(record, 'bb_exc_formatted'): 91eb8dc403SDave Cobbley msg += '\n' + ''.join(record.bb_exc_formatted) 92eb8dc403SDave Cobbley elif hasattr(record, 'bb_exc_info'): 93eb8dc403SDave Cobbley etype, value, tb = record.bb_exc_info 94eb8dc403SDave Cobbley formatted = bb.exceptions.format_exception(etype, value, tb, limit=5) 95eb8dc403SDave Cobbley msg += '\n' + ''.join(formatted) 96eb8dc403SDave Cobbley return msg 97eb8dc403SDave Cobbley 98eb8dc403SDave Cobbley def colorize(self, record): 99eb8dc403SDave Cobbley color = self.COLORS[record.levelno] 100eb8dc403SDave Cobbley if self.color_enabled and color is not None: 101eb8dc403SDave Cobbley record = copy.copy(record) 102eb8dc403SDave Cobbley record.levelname = "".join([self.BLD % color, record.levelname, self.RST]) 103eb8dc403SDave Cobbley record.msg = "".join([self.STD % color, record.msg, self.RST]) 104eb8dc403SDave Cobbley return record 105eb8dc403SDave Cobbley 106eb8dc403SDave Cobbley def enable_color(self): 107eb8dc403SDave Cobbley self.color_enabled = True 108eb8dc403SDave Cobbley 10982c905dcSAndrew Geissler def __repr__(self): 11082c905dcSAndrew Geissler return "%s fmt='%s' color=%s" % (self.__class__.__name__, self._fmt, "True" if self.color_enabled else "False") 11182c905dcSAndrew Geissler 112eb8dc403SDave Cobbleyclass BBLogFilter(object): 113eb8dc403SDave Cobbley def __init__(self, handler, level, debug_domains): 114eb8dc403SDave Cobbley self.stdlevel = level 115eb8dc403SDave Cobbley self.debug_domains = debug_domains 116eb8dc403SDave Cobbley loglevel = level 117eb8dc403SDave Cobbley for domain in debug_domains: 118eb8dc403SDave Cobbley if debug_domains[domain] < loglevel: 119eb8dc403SDave Cobbley loglevel = debug_domains[domain] 120eb8dc403SDave Cobbley handler.setLevel(loglevel) 121eb8dc403SDave Cobbley handler.addFilter(self) 122eb8dc403SDave Cobbley 123eb8dc403SDave Cobbley def filter(self, record): 124eb8dc403SDave Cobbley if record.levelno >= self.stdlevel: 125eb8dc403SDave Cobbley return True 126eb8dc403SDave Cobbley if record.name in self.debug_domains and record.levelno >= self.debug_domains[record.name]: 127eb8dc403SDave Cobbley return True 128eb8dc403SDave Cobbley return False 129eb8dc403SDave Cobbley 1307e0e3c0cSAndrew Geisslerclass LogFilterShowOnce(logging.Filter): 1317e0e3c0cSAndrew Geissler def __init__(self): 1327e0e3c0cSAndrew Geissler self.seen_warnings = set() 1337e0e3c0cSAndrew Geissler self.seen_errors = set() 1347e0e3c0cSAndrew Geissler 1357e0e3c0cSAndrew Geissler def filter(self, record): 1367e0e3c0cSAndrew Geissler if record.levelno == bb.msg.BBLogFormatter.WARNONCE: 1377e0e3c0cSAndrew Geissler if record.msg in self.seen_warnings: 1387e0e3c0cSAndrew Geissler return False 1397e0e3c0cSAndrew Geissler self.seen_warnings.add(record.msg) 1407e0e3c0cSAndrew Geissler if record.levelno == bb.msg.BBLogFormatter.ERRORONCE: 1417e0e3c0cSAndrew Geissler if record.msg in self.seen_errors: 1427e0e3c0cSAndrew Geissler return False 1437e0e3c0cSAndrew Geissler self.seen_errors.add(record.msg) 1447e0e3c0cSAndrew Geissler return True 1457e0e3c0cSAndrew Geissler 14682c905dcSAndrew Geisslerclass LogFilterGEQLevel(logging.Filter): 14782c905dcSAndrew Geissler def __init__(self, level): 14882c905dcSAndrew Geissler self.strlevel = str(level) 14982c905dcSAndrew Geissler self.level = stringToLevel(level) 150eb8dc403SDave Cobbley 15182c905dcSAndrew Geissler def __repr__(self): 15282c905dcSAndrew Geissler return "%s level >= %s (%d)" % (self.__class__.__name__, self.strlevel, self.level) 15382c905dcSAndrew Geissler 154eb8dc403SDave Cobbley def filter(self, record): 15582c905dcSAndrew Geissler return (record.levelno >= self.level) 15682c905dcSAndrew Geissler 15782c905dcSAndrew Geisslerclass LogFilterLTLevel(logging.Filter): 15882c905dcSAndrew Geissler def __init__(self, level): 15982c905dcSAndrew Geissler self.strlevel = str(level) 16082c905dcSAndrew Geissler self.level = stringToLevel(level) 16182c905dcSAndrew Geissler 16282c905dcSAndrew Geissler def __repr__(self): 16382c905dcSAndrew Geissler return "%s level < %s (%d)" % (self.__class__.__name__, self.strlevel, self.level) 16482c905dcSAndrew Geissler 16582c905dcSAndrew Geissler def filter(self, record): 16682c905dcSAndrew Geissler return (record.levelno < self.level) 167eb8dc403SDave Cobbley 168eb8dc403SDave Cobbley# Message control functions 169eb8dc403SDave Cobbley# 170eb8dc403SDave Cobbley 17182c905dcSAndrew GeisslerloggerDefaultLogLevel = BBLogFormatter.NOTE 17282c905dcSAndrew GeisslerloggerDefaultDomains = {} 173eb8dc403SDave Cobbley 174eb8dc403SDave Cobbleydef init_msgconfig(verbose, debug, debug_domains=None): 175eb8dc403SDave Cobbley """ 176eb8dc403SDave Cobbley Set default verbosity and debug levels config the logger 177eb8dc403SDave Cobbley """ 178eb8dc403SDave Cobbley if debug: 17982c905dcSAndrew Geissler bb.msg.loggerDefaultLogLevel = BBLogFormatter.DEBUG - debug + 1 180eb8dc403SDave Cobbley elif verbose: 18182c905dcSAndrew Geissler bb.msg.loggerDefaultLogLevel = BBLogFormatter.VERBOSE 182eb8dc403SDave Cobbley else: 18382c905dcSAndrew Geissler bb.msg.loggerDefaultLogLevel = BBLogFormatter.NOTE 184eb8dc403SDave Cobbley 18582c905dcSAndrew Geissler bb.msg.loggerDefaultDomains = {} 18682c905dcSAndrew Geissler if debug_domains: 18782c905dcSAndrew Geissler for (domainarg, iterator) in groupby(debug_domains): 188eb8dc403SDave Cobbley dlevel = len(tuple(iterator)) 18982c905dcSAndrew Geissler bb.msg.loggerDefaultDomains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1 19082c905dcSAndrew Geissler 19182c905dcSAndrew Geisslerdef constructLogOptions(): 19282c905dcSAndrew Geissler return loggerDefaultLogLevel, loggerDefaultDomains 193eb8dc403SDave Cobbley 194eb8dc403SDave Cobbleydef addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None): 195eb8dc403SDave Cobbley level, debug_domains = constructLogOptions() 196eb8dc403SDave Cobbley 197eb8dc403SDave Cobbley if forcelevel is not None: 198eb8dc403SDave Cobbley level = forcelevel 199eb8dc403SDave Cobbley 200eb8dc403SDave Cobbley cls(handler, level, debug_domains) 201eb8dc403SDave Cobbley 20282c905dcSAndrew Geisslerdef stringToLevel(level): 20382c905dcSAndrew Geissler try: 20482c905dcSAndrew Geissler return int(level) 20582c905dcSAndrew Geissler except ValueError: 20682c905dcSAndrew Geissler pass 20782c905dcSAndrew Geissler 20882c905dcSAndrew Geissler try: 20982c905dcSAndrew Geissler return getattr(logging, level) 21082c905dcSAndrew Geissler except AttributeError: 21182c905dcSAndrew Geissler pass 21282c905dcSAndrew Geissler 21382c905dcSAndrew Geissler return getattr(BBLogFormatter, level) 21482c905dcSAndrew Geissler 215eb8dc403SDave Cobbley# 216eb8dc403SDave Cobbley# Message handling functions 217eb8dc403SDave Cobbley# 218eb8dc403SDave Cobbley 219eb8dc403SDave Cobbleydef fatal(msgdomain, msg): 220eb8dc403SDave Cobbley if msgdomain: 221eb8dc403SDave Cobbley logger = logging.getLogger("BitBake.%s" % msgdomain) 222eb8dc403SDave Cobbley else: 223eb8dc403SDave Cobbley logger = logging.getLogger("BitBake") 224eb8dc403SDave Cobbley logger.critical(msg) 225eb8dc403SDave Cobbley sys.exit(1) 226eb8dc403SDave Cobbley 227eb8dc403SDave Cobbleydef logger_create(name, output=sys.stderr, level=logging.INFO, preserve_handlers=False, color='auto'): 228eb8dc403SDave Cobbley """Standalone logger creation function""" 229eb8dc403SDave Cobbley logger = logging.getLogger(name) 230eb8dc403SDave Cobbley console = logging.StreamHandler(output) 2317e0e3c0cSAndrew Geissler console.addFilter(bb.msg.LogFilterShowOnce()) 232eb8dc403SDave Cobbley format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 233*03514f19SPatrick Williams if color == 'always' or (color == 'auto' and output.isatty() and os.environ.get('NO_COLOR', '') == ''): 234eb8dc403SDave Cobbley format.enable_color() 235eb8dc403SDave Cobbley console.setFormatter(format) 236eb8dc403SDave Cobbley if preserve_handlers: 237eb8dc403SDave Cobbley logger.addHandler(console) 238eb8dc403SDave Cobbley else: 239eb8dc403SDave Cobbley logger.handlers = [console] 240eb8dc403SDave Cobbley logger.setLevel(level) 241eb8dc403SDave Cobbley return logger 242eb8dc403SDave Cobbley 243eb8dc403SDave Cobbleydef has_console_handler(logger): 244eb8dc403SDave Cobbley for handler in logger.handlers: 245eb8dc403SDave Cobbley if isinstance(handler, logging.StreamHandler): 246eb8dc403SDave Cobbley if handler.stream in [sys.stderr, sys.stdout]: 247eb8dc403SDave Cobbley return True 248eb8dc403SDave Cobbley return False 24982c905dcSAndrew Geissler 25082c905dcSAndrew Geisslerdef mergeLoggingConfig(logconfig, userconfig): 25182c905dcSAndrew Geissler logconfig = copy.deepcopy(logconfig) 25282c905dcSAndrew Geissler userconfig = copy.deepcopy(userconfig) 25382c905dcSAndrew Geissler 25482c905dcSAndrew Geissler # Merge config with the default config 25582c905dcSAndrew Geissler if userconfig.get('version') != logconfig['version']: 25682c905dcSAndrew Geissler raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version'))) 25782c905dcSAndrew Geissler 25882c905dcSAndrew Geissler # Set some defaults to make merging easier 25982c905dcSAndrew Geissler userconfig.setdefault("loggers", {}) 26082c905dcSAndrew Geissler 26182c905dcSAndrew Geissler # If a handler, formatter, or filter is defined in the user 26282c905dcSAndrew Geissler # config, it will replace an existing one in the default config 26382c905dcSAndrew Geissler for k in ("handlers", "formatters", "filters"): 26482c905dcSAndrew Geissler logconfig.setdefault(k, {}).update(userconfig.get(k, {})) 26582c905dcSAndrew Geissler 26682c905dcSAndrew Geissler seen_loggers = set() 26782c905dcSAndrew Geissler for name, l in logconfig["loggers"].items(): 26882c905dcSAndrew Geissler # If the merge option is set, merge the handlers and 26982c905dcSAndrew Geissler # filters. Otherwise, if it is False, this logger won't get 27082c905dcSAndrew Geissler # add to the set of seen loggers and will replace the 27182c905dcSAndrew Geissler # existing one 27282c905dcSAndrew Geissler if l.get('bitbake_merge', True): 27382c905dcSAndrew Geissler ulogger = userconfig["loggers"].setdefault(name, {}) 27482c905dcSAndrew Geissler ulogger.setdefault("handlers", []) 27582c905dcSAndrew Geissler ulogger.setdefault("filters", []) 27682c905dcSAndrew Geissler 27782c905dcSAndrew Geissler # Merge lists 27882c905dcSAndrew Geissler l.setdefault("handlers", []).extend(ulogger["handlers"]) 27982c905dcSAndrew Geissler l.setdefault("filters", []).extend(ulogger["filters"]) 28082c905dcSAndrew Geissler 28182c905dcSAndrew Geissler # Replace other properties if present 28282c905dcSAndrew Geissler if "level" in ulogger: 28382c905dcSAndrew Geissler l["level"] = ulogger["level"] 28482c905dcSAndrew Geissler 28582c905dcSAndrew Geissler if "propagate" in ulogger: 28682c905dcSAndrew Geissler l["propagate"] = ulogger["propagate"] 28782c905dcSAndrew Geissler 28882c905dcSAndrew Geissler seen_loggers.add(name) 28982c905dcSAndrew Geissler 29082c905dcSAndrew Geissler # Add all loggers present in the user config, but not any that 29182c905dcSAndrew Geissler # have already been processed 29282c905dcSAndrew Geissler for name in set(userconfig["loggers"].keys()) - seen_loggers: 29382c905dcSAndrew Geissler logconfig["loggers"][name] = userconfig["loggers"][name] 29482c905dcSAndrew Geissler 29582c905dcSAndrew Geissler return logconfig 29682c905dcSAndrew Geissler 29782c905dcSAndrew Geisslerdef setLoggingConfig(defaultconfig, userconfigfile=None): 29882c905dcSAndrew Geissler logconfig = copy.deepcopy(defaultconfig) 29982c905dcSAndrew Geissler 30082c905dcSAndrew Geissler if userconfigfile: 301475cb72dSAndrew Geissler with open(os.path.normpath(userconfigfile), 'r') as f: 30282c905dcSAndrew Geissler if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'): 30382c905dcSAndrew Geissler import yaml 30409209eecSAndrew Geissler userconfig = yaml.safe_load(f) 30582c905dcSAndrew Geissler elif userconfigfile.endswith('.json') or userconfigfile.endswith('.cfg'): 30682c905dcSAndrew Geissler import json 30782c905dcSAndrew Geissler userconfig = json.load(f) 30882c905dcSAndrew Geissler else: 30982c905dcSAndrew Geissler raise BaseException("Unrecognized file format: %s" % userconfigfile) 31082c905dcSAndrew Geissler 31182c905dcSAndrew Geissler if userconfig.get('bitbake_merge', True): 31282c905dcSAndrew Geissler logconfig = mergeLoggingConfig(logconfig, userconfig) 31382c905dcSAndrew Geissler else: 31482c905dcSAndrew Geissler # Replace the entire default config 31582c905dcSAndrew Geissler logconfig = userconfig 31682c905dcSAndrew Geissler 31782c905dcSAndrew Geissler # Convert all level parameters to integers in case users want to use the 31882c905dcSAndrew Geissler # bitbake defined level names 3197e0e3c0cSAndrew Geissler for name, h in logconfig["handlers"].items(): 32082c905dcSAndrew Geissler if "level" in h: 32182c905dcSAndrew Geissler h["level"] = bb.msg.stringToLevel(h["level"]) 32282c905dcSAndrew Geissler 3237e0e3c0cSAndrew Geissler # Every handler needs its own instance of the once filter. 3247e0e3c0cSAndrew Geissler once_filter_name = name + ".showonceFilter" 3257e0e3c0cSAndrew Geissler logconfig.setdefault("filters", {})[once_filter_name] = { 3267e0e3c0cSAndrew Geissler "()": "bb.msg.LogFilterShowOnce", 3277e0e3c0cSAndrew Geissler } 3287e0e3c0cSAndrew Geissler h.setdefault("filters", []).append(once_filter_name) 3297e0e3c0cSAndrew Geissler 33082c905dcSAndrew Geissler for l in logconfig["loggers"].values(): 33182c905dcSAndrew Geissler if "level" in l: 33282c905dcSAndrew Geissler l["level"] = bb.msg.stringToLevel(l["level"]) 33382c905dcSAndrew Geissler 33482c905dcSAndrew Geissler conf = logging.config.dictConfigClass(logconfig) 33582c905dcSAndrew Geissler conf.configure() 33682c905dcSAndrew Geissler 33782c905dcSAndrew Geissler # The user may have specified logging domains they want at a higher debug 33882c905dcSAndrew Geissler # level than the standard. 33982c905dcSAndrew Geissler for name, l in logconfig["loggers"].items(): 34082c905dcSAndrew Geissler if not name.startswith("BitBake."): 34182c905dcSAndrew Geissler continue 34282c905dcSAndrew Geissler 34382c905dcSAndrew Geissler if not "level" in l: 34482c905dcSAndrew Geissler continue 34582c905dcSAndrew Geissler 34682c905dcSAndrew Geissler curlevel = bb.msg.loggerDefaultDomains.get(name) 34782c905dcSAndrew Geissler # Note: level parameter should already be a int because of conversion 34882c905dcSAndrew Geissler # above 34982c905dcSAndrew Geissler newlevel = int(l["level"]) 35082c905dcSAndrew Geissler if curlevel is None or newlevel < curlevel: 35182c905dcSAndrew Geissler bb.msg.loggerDefaultDomains[name] = newlevel 35282c905dcSAndrew Geissler 35382c905dcSAndrew Geissler # TODO: I don't think that setting the global log level should be necessary 35482c905dcSAndrew Geissler #if newlevel < bb.msg.loggerDefaultLogLevel: 35582c905dcSAndrew Geissler # bb.msg.loggerDefaultLogLevel = newlevel 35682c905dcSAndrew Geissler 35782c905dcSAndrew Geissler return conf 358