1""" 2BitBake Smart Dictionary Implementation 3 4Functions for interacting with the data structure used by the 5BitBake build tools. 6 7""" 8 9# Copyright (C) 2003, 2004 Chris Larson 10# Copyright (C) 2004, 2005 Seb Frankengul 11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther 12# Copyright (C) 2005 Uli Luckas 13# Copyright (C) 2005 ROAD GmbH 14# 15# SPDX-License-Identifier: GPL-2.0-only 16# 17# Based on functions from the base bb module, Copyright 2003 Holger Schurig 18 19import builtins 20import copy 21import re 22import sys 23from collections.abc import MutableMapping 24import logging 25import hashlib 26import bb, bb.codeparser 27from bb import utils 28from bb.COW import COWDictBase 29 30logger = logging.getLogger("BitBake.Data") 31 32__setvar_keyword__ = [":append", ":prepend", ":remove"] 33__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') 34__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") 35__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") 36__whitespace_split__ = re.compile(r'(\s)') 37__override_regexp__ = re.compile(r'[a-z0-9]+') 38 39bitbake_renamed_vars = { 40 "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", 41 "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", 42 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", 43 "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", 44 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", 45 "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", 46 "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", 47 "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", 48 "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", 49} 50 51def infer_caller_details(loginfo, parent = False, varval = True): 52 """Save the caller the trouble of specifying everything.""" 53 # Save effort. 54 if 'ignore' in loginfo and loginfo['ignore']: 55 return 56 # If nothing was provided, mark this as possibly unneeded. 57 if not loginfo: 58 loginfo['ignore'] = True 59 return 60 # Infer caller's likely values for variable (var) and value (value), 61 # to reduce clutter in the rest of the code. 62 above = None 63 def set_above(): 64 try: 65 raise Exception 66 except Exception: 67 tb = sys.exc_info()[2] 68 if parent: 69 return tb.tb_frame.f_back.f_back.f_back 70 else: 71 return tb.tb_frame.f_back.f_back 72 73 if varval and ('variable' not in loginfo or 'detail' not in loginfo): 74 if not above: 75 above = set_above() 76 lcls = above.f_locals.items() 77 for k, v in lcls: 78 if k == 'value' and 'detail' not in loginfo: 79 loginfo['detail'] = v 80 if k == 'var' and 'variable' not in loginfo: 81 loginfo['variable'] = v 82 # Infer file/line/function from traceback 83 # Don't use traceback.extract_stack() since it fills the line contents which 84 # we don't need and that hits stat syscalls 85 if 'file' not in loginfo: 86 if not above: 87 above = set_above() 88 f = above.f_back 89 line = f.f_lineno 90 file = f.f_code.co_filename 91 func = f.f_code.co_name 92 loginfo['file'] = file 93 loginfo['line'] = line 94 if func not in loginfo: 95 loginfo['func'] = func 96 97class VariableParse: 98 def __init__(self, varname, d, unexpanded_value = None, val = None): 99 self.varname = varname 100 self.d = d 101 self.value = val 102 self.unexpanded_value = unexpanded_value 103 104 self.references = set() 105 self.execs = set() 106 self.contains = {} 107 108 def var_sub(self, match): 109 key = match.group()[2:-1] 110 if self.varname and key: 111 if self.varname == key: 112 raise Exception("variable %s references itself!" % self.varname) 113 var = self.d.getVarFlag(key, "_content") 114 self.references.add(key) 115 if var is not None: 116 return var 117 else: 118 return match.group() 119 120 def python_sub(self, match): 121 if isinstance(match, str): 122 code = match 123 else: 124 code = match.group()[3:-1] 125 126 # Do not run code that contains one or more unexpanded variables 127 # instead return the code with the characters we removed put back 128 if __expand_var_regexp__.findall(code): 129 return "${@" + code + "}" 130 131 if self.varname: 132 varname = 'Var <%s>' % self.varname 133 else: 134 varname = '<expansion>' 135 codeobj = compile(code.strip(), varname, "eval") 136 137 parser = bb.codeparser.PythonParser(self.varname, logger) 138 parser.parse_python(code) 139 if self.varname: 140 vardeps = self.d.getVarFlag(self.varname, "vardeps") 141 if vardeps is None: 142 parser.log.flush() 143 else: 144 parser.log.flush() 145 self.references |= parser.references 146 self.execs |= parser.execs 147 148 for k in parser.contains: 149 if k not in self.contains: 150 self.contains[k] = parser.contains[k].copy() 151 else: 152 self.contains[k].update(parser.contains[k]) 153 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 154 return str(value) 155 156class DataContext(dict): 157 excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe']) 158 159 def __init__(self, metadata, **kwargs): 160 self.metadata = metadata 161 dict.__init__(self, **kwargs) 162 self['d'] = metadata 163 self.context = set(bb.utils.get_context()) 164 165 def __missing__(self, key): 166 if key in self.excluded or key in self.context: 167 raise KeyError(key) 168 169 value = self.metadata.getVar(key) 170 if value is None: 171 raise KeyError(key) 172 else: 173 return value 174 175class ExpansionError(Exception): 176 def __init__(self, varname, expression, exception): 177 self.expression = expression 178 self.variablename = varname 179 self.exception = exception 180 self.varlist = [varname or expression or ""] 181 if varname: 182 if expression: 183 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 184 else: 185 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) 186 else: 187 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 188 Exception.__init__(self, self.msg) 189 self.args = (varname, expression, exception) 190 191 def addVar(self, varname): 192 if varname: 193 self.varlist.append(varname) 194 195 def __str__(self): 196 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) 197 return self.msg + chain 198 199class IncludeHistory(object): 200 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 201 self.parent = parent 202 self.filename = filename 203 self.children = [] 204 self.current = self 205 206 def copy(self): 207 new = IncludeHistory(self.parent, self.filename) 208 for c in self.children: 209 new.children.append(c) 210 return new 211 212 def include(self, filename): 213 newfile = IncludeHistory(self.current, filename) 214 self.current.children.append(newfile) 215 self.current = newfile 216 return self 217 218 def __enter__(self): 219 pass 220 221 def __exit__(self, a, b, c): 222 if self.current.parent: 223 self.current = self.current.parent 224 else: 225 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) 226 return False 227 228 def emit(self, o, level = 0): 229 """Emit an include history file, and its children.""" 230 if level: 231 spaces = " " * (level - 1) 232 o.write("# %s%s" % (spaces, self.filename)) 233 if len(self.children) > 0: 234 o.write(" includes:") 235 else: 236 o.write("#\n# INCLUDE HISTORY:\n#") 237 level = level + 1 238 for child in self.children: 239 o.write("\n") 240 child.emit(o, level) 241 242class VariableHistory(object): 243 def __init__(self, dataroot): 244 self.dataroot = dataroot 245 self.variables = COWDictBase.copy() 246 247 def copy(self): 248 new = VariableHistory(self.dataroot) 249 new.variables = self.variables.copy() 250 return new 251 252 def __getstate__(self): 253 vardict = {} 254 for k, v in self.variables.iteritems(): 255 vardict[k] = v 256 return {'dataroot': self.dataroot, 257 'variables': vardict} 258 259 def __setstate__(self, state): 260 self.dataroot = state['dataroot'] 261 self.variables = COWDictBase.copy() 262 for k, v in state['variables'].items(): 263 self.variables[k] = v 264 265 def record(self, *kwonly, **loginfo): 266 if not self.dataroot._tracking: 267 return 268 if len(kwonly) > 0: 269 raise TypeError 270 infer_caller_details(loginfo, parent = True) 271 if 'ignore' in loginfo and loginfo['ignore']: 272 return 273 if 'op' not in loginfo or not loginfo['op']: 274 loginfo['op'] = 'set' 275 if 'detail' in loginfo: 276 loginfo['detail'] = str(loginfo['detail']) 277 if 'variable' not in loginfo or 'file' not in loginfo: 278 raise ValueError("record() missing variable or file.") 279 var = loginfo['variable'] 280 281 if var not in self.variables: 282 self.variables[var] = [] 283 if not isinstance(self.variables[var], list): 284 return 285 if 'nodups' in loginfo and loginfo in self.variables[var]: 286 return 287 self.variables[var].append(loginfo.copy()) 288 289 def rename_variable_hist(self, oldvar, newvar): 290 if not self.dataroot._tracking: 291 return 292 if oldvar not in self.variables: 293 return 294 if newvar not in self.variables: 295 self.variables[newvar] = [] 296 for i in self.variables[oldvar]: 297 self.variables[newvar].append(i.copy()) 298 299 def variable(self, var): 300 varhistory = [] 301 if var in self.variables: 302 varhistory.extend(self.variables[var]) 303 return varhistory 304 305 def emit(self, var, oval, val, o, d): 306 history = self.variable(var) 307 308 # Append override history 309 if var in d.overridedata: 310 for (r, override) in d.overridedata[var]: 311 for event in self.variable(r): 312 loginfo = event.copy() 313 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): 314 continue 315 loginfo['variable'] = var 316 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 317 history.append(loginfo) 318 319 commentVal = re.sub('\n', '\n#', str(oval)) 320 if history: 321 if len(history) == 1: 322 o.write("#\n# $%s\n" % var) 323 else: 324 o.write("#\n# $%s [%d operations]\n" % (var, len(history))) 325 for event in history: 326 # o.write("# %s\n" % str(event)) 327 if 'func' in event: 328 # If we have a function listed, this is internal 329 # code, not an operation in a config file, and the 330 # full path is distracting. 331 event['file'] = re.sub('.*/', '', event['file']) 332 display_func = ' [%s]' % event['func'] 333 else: 334 display_func = '' 335 if 'flag' in event: 336 flag = '[%s] ' % (event['flag']) 337 else: 338 flag = '' 339 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) 340 if len(history) > 1: 341 o.write("# pre-expansion value:\n") 342 o.write('# "%s"\n' % (commentVal)) 343 else: 344 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) 345 o.write('# "%s"\n' % (commentVal)) 346 347 def get_variable_files(self, var): 348 """Get the files where operations are made on a variable""" 349 var_history = self.variable(var) 350 files = [] 351 for event in var_history: 352 files.append(event['file']) 353 return files 354 355 def get_variable_lines(self, var, f): 356 """Get the line where a operation is made on a variable in file f""" 357 var_history = self.variable(var) 358 lines = [] 359 for event in var_history: 360 if f== event['file']: 361 line = event['line'] 362 lines.append(line) 363 return lines 364 365 def get_variable_refs(self, var): 366 """Return a dict of file/line references""" 367 var_history = self.variable(var) 368 refs = {} 369 for event in var_history: 370 if event['file'] not in refs: 371 refs[event['file']] = [] 372 refs[event['file']].append(event['line']) 373 return refs 374 375 def get_variable_items_files(self, var): 376 """ 377 Use variable history to map items added to a list variable and 378 the files in which they were added. 379 """ 380 d = self.dataroot 381 history = self.variable(var) 382 finalitems = (d.getVar(var) or '').split() 383 filemap = {} 384 isset = False 385 for event in history: 386 if 'flag' in event: 387 continue 388 if event['op'] == ':remove': 389 continue 390 if isset and event['op'] == 'set?': 391 continue 392 isset = True 393 items = d.expand(event['detail']).split() 394 for item in items: 395 # This is a little crude but is belt-and-braces to avoid us 396 # having to handle every possible operation type specifically 397 if item in finalitems and not item in filemap: 398 filemap[item] = event['file'] 399 return filemap 400 401 def del_var_history(self, var, f=None, line=None): 402 """If file f and line are not given, the entire history of var is deleted""" 403 if var in self.variables: 404 if f and line: 405 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] 406 else: 407 self.variables[var] = [] 408 409def _print_rename_error(var, loginfo, renamedvars, fullvar=None): 410 info = "" 411 if "file" in loginfo: 412 info = " file: %s" % loginfo["file"] 413 if "line" in loginfo: 414 info += " line: %s" % loginfo["line"] 415 if fullvar and fullvar != var: 416 info += " referenced as: %s" % fullvar 417 if info: 418 info = " (%s)" % info.strip() 419 renameinfo = renamedvars[var] 420 if " " in renameinfo: 421 # A space signals a string to display instead of a rename 422 bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) 423 else: 424 bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) 425 426class DataSmart(MutableMapping): 427 def __init__(self): 428 self.dict = {} 429 430 self.inchistory = IncludeHistory() 431 self.varhistory = VariableHistory(self) 432 self._tracking = False 433 self._var_renames = {} 434 self._var_renames.update(bitbake_renamed_vars) 435 436 self.expand_cache = {} 437 438 # cookie monster tribute 439 # Need to be careful about writes to overridedata as 440 # its only a shallow copy, could influence other data store 441 # copies! 442 self.overridedata = {} 443 self.overrides = None 444 self.overridevars = set(["OVERRIDES", "FILE"]) 445 self.inoverride = False 446 447 def enableTracking(self): 448 self._tracking = True 449 450 def disableTracking(self): 451 self._tracking = False 452 453 def expandWithRefs(self, s, varname): 454 455 if not isinstance(s, str): # sanity check 456 return VariableParse(varname, self, s, s) 457 458 varparse = VariableParse(varname, self, s) 459 460 while s.find('${') != -1: 461 olds = s 462 try: 463 s = __expand_var_regexp__.sub(varparse.var_sub, s) 464 try: 465 s = __expand_python_regexp__.sub(varparse.python_sub, s) 466 except SyntaxError as e: 467 # Likely unmatched brackets, just don't expand the expression 468 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): 469 raise 470 if s == olds: 471 break 472 except ExpansionError as e: 473 e.addVar(varname) 474 raise 475 except bb.parse.SkipRecipe: 476 raise 477 except bb.BBHandledException: 478 raise 479 except Exception as exc: 480 tb = sys.exc_info()[2] 481 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 482 483 varparse.value = s 484 485 return varparse 486 487 def expand(self, s, varname = None): 488 return self.expandWithRefs(s, varname).value 489 490 def need_overrides(self): 491 if self.overrides is not None: 492 return 493 if self.inoverride: 494 return 495 overrride_stack = [] 496 for count in range(5): 497 self.inoverride = True 498 # Can end up here recursively so setup dummy values 499 self.overrides = [] 500 self.overridesset = set() 501 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 502 overrride_stack.append(self.overrides) 503 self.overridesset = set(self.overrides) 504 self.inoverride = False 505 self.expand_cache = {} 506 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or [] 507 if newoverrides == self.overrides: 508 break 509 self.overrides = newoverrides 510 self.overridesset = set(self.overrides) 511 else: 512 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) 513 514 def initVar(self, var): 515 self.expand_cache = {} 516 if not var in self.dict: 517 self.dict[var] = {} 518 519 def _findVar(self, var): 520 dest = self.dict 521 while dest: 522 if var in dest: 523 return dest[var] 524 525 if "_data" not in dest: 526 break 527 dest = dest["_data"] 528 return None 529 530 def _makeShadowCopy(self, var): 531 if var in self.dict: 532 return 533 534 local_var = self._findVar(var) 535 536 if local_var: 537 self.dict[var] = copy.copy(local_var) 538 else: 539 self.initVar(var) 540 541 def hasOverrides(self, var): 542 return var in self.overridedata 543 544 def setVar(self, var, value, **loginfo): 545 #print("var=" + str(var) + " val=" + str(value)) 546 547 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): 548 info = "%s" % var 549 if "file" in loginfo: 550 info += " file: %s" % loginfo["file"] 551 if "line" in loginfo: 552 info += " line: %s" % loginfo["line"] 553 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) 554 555 shortvar = var.split(":", 1)[0] 556 if shortvar in self._var_renames: 557 _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) 558 # Mark that we have seen a renamed variable 559 self.setVar("_FAILPARSINGERRORHANDLED", True) 560 561 self.expand_cache = {} 562 parsing=False 563 if 'parsing' in loginfo: 564 parsing=True 565 566 if 'op' not in loginfo: 567 loginfo['op'] = "set" 568 569 match = __setvar_regexp__.match(var) 570 if match and match.group("keyword") in __setvar_keyword__: 571 base = match.group('base') 572 keyword = match.group("keyword") 573 override = match.group('add') 574 l = self.getVarFlag(base, keyword, False) or [] 575 l.append([value, override]) 576 self.setVarFlag(base, keyword, l, ignore=True) 577 # And cause that to be recorded: 578 loginfo['detail'] = value 579 loginfo['variable'] = base 580 if override: 581 loginfo['op'] = '%s[%s]' % (keyword, override) 582 else: 583 loginfo['op'] = keyword 584 self.varhistory.record(**loginfo) 585 # todo make sure keyword is not __doc__ or __module__ 586 # pay the cookie monster 587 588 # more cookies for the cookie monster 589 if ':' in var: 590 self._setvar_update_overrides(base, **loginfo) 591 592 if base in self.overridevars: 593 self._setvar_update_overridevars(var, value) 594 return 595 596 if not var in self.dict: 597 self._makeShadowCopy(var) 598 599 if not parsing: 600 if ":append" in self.dict[var]: 601 del self.dict[var][":append"] 602 if ":prepend" in self.dict[var]: 603 del self.dict[var][":prepend"] 604 if ":remove" in self.dict[var]: 605 del self.dict[var][":remove"] 606 if var in self.overridedata: 607 active = [] 608 self.need_overrides() 609 for (r, o) in self.overridedata[var]: 610 if o in self.overridesset: 611 active.append(r) 612 elif ":" in o: 613 if set(o.split(":")).issubset(self.overridesset): 614 active.append(r) 615 for a in active: 616 self.delVar(a) 617 del self.overridedata[var] 618 619 # more cookies for the cookie monster 620 if ':' in var: 621 self._setvar_update_overrides(var, **loginfo) 622 623 # setting var 624 self.dict[var]["_content"] = value 625 self.varhistory.record(**loginfo) 626 627 if var in self.overridevars: 628 self._setvar_update_overridevars(var, value) 629 630 def _setvar_update_overridevars(self, var, value): 631 vardata = self.expandWithRefs(value, var) 632 new = vardata.references 633 new.update(vardata.contains.keys()) 634 while not new.issubset(self.overridevars): 635 nextnew = set() 636 self.overridevars.update(new) 637 for i in new: 638 vardata = self.expandWithRefs(self.getVar(i), i) 639 nextnew.update(vardata.references) 640 nextnew.update(vardata.contains.keys()) 641 new = nextnew 642 self.overrides = None 643 644 def _setvar_update_overrides(self, var, **loginfo): 645 # aka pay the cookie monster 646 override = var[var.rfind(':')+1:] 647 shortvar = var[:var.rfind(':')] 648 while override and __override_regexp__.match(override): 649 if shortvar not in self.overridedata: 650 self.overridedata[shortvar] = [] 651 if [var, override] not in self.overridedata[shortvar]: 652 # Force CoW by recreating the list first 653 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 654 self.overridedata[shortvar].append([var, override]) 655 override = None 656 if ":" in shortvar: 657 override = var[shortvar.rfind(':')+1:] 658 shortvar = var[:shortvar.rfind(':')] 659 if len(shortvar) == 0: 660 override = None 661 662 def getVar(self, var, expand=True, noweakdefault=False, parsing=False): 663 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) 664 665 def renameVar(self, key, newkey, **loginfo): 666 """ 667 Rename the variable key to newkey 668 """ 669 if key == newkey: 670 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) 671 return 672 673 val = self.getVar(key, 0, parsing=True) 674 if val is not None: 675 self.varhistory.rename_variable_hist(key, newkey) 676 loginfo['variable'] = newkey 677 loginfo['op'] = 'rename from %s' % key 678 loginfo['detail'] = val 679 self.varhistory.record(**loginfo) 680 self.setVar(newkey, val, ignore=True, parsing=True) 681 682 srcflags = self.getVarFlags(key, False, True) or {} 683 for i in srcflags: 684 if i not in (__setvar_keyword__): 685 continue 686 src = srcflags[i] 687 688 dest = self.getVarFlag(newkey, i, False) or [] 689 dest.extend(src) 690 self.setVarFlag(newkey, i, dest, ignore=True) 691 692 if key in self.overridedata: 693 self.overridedata[newkey] = [] 694 for (v, o) in self.overridedata[key]: 695 self.overridedata[newkey].append([v.replace(key, newkey), o]) 696 self.renameVar(v, v.replace(key, newkey)) 697 698 if ':' in newkey and val is None: 699 self._setvar_update_overrides(newkey, **loginfo) 700 701 loginfo['variable'] = key 702 loginfo['op'] = 'rename (to)' 703 loginfo['detail'] = newkey 704 self.varhistory.record(**loginfo) 705 self.delVar(key, ignore=True) 706 707 def appendVar(self, var, value, **loginfo): 708 loginfo['op'] = 'append' 709 self.varhistory.record(**loginfo) 710 self.setVar(var + ":append", value, ignore=True, parsing=True) 711 712 def prependVar(self, var, value, **loginfo): 713 loginfo['op'] = 'prepend' 714 self.varhistory.record(**loginfo) 715 self.setVar(var + ":prepend", value, ignore=True, parsing=True) 716 717 def delVar(self, var, **loginfo): 718 self.expand_cache = {} 719 720 loginfo['detail'] = "" 721 loginfo['op'] = 'del' 722 self.varhistory.record(**loginfo) 723 self.dict[var] = {} 724 if var in self.overridedata: 725 del self.overridedata[var] 726 if ':' in var: 727 override = var[var.rfind(':')+1:] 728 shortvar = var[:var.rfind(':')] 729 while override and __override_regexp__.match(override): 730 try: 731 if shortvar in self.overridedata: 732 # Force CoW by recreating the list first 733 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 734 self.overridedata[shortvar].remove([var, override]) 735 except ValueError as e: 736 pass 737 override = None 738 if ":" in shortvar: 739 override = var[shortvar.rfind(':')+1:] 740 shortvar = var[:shortvar.rfind(':')] 741 if len(shortvar) == 0: 742 override = None 743 744 def setVarFlag(self, var, flag, value, **loginfo): 745 self.expand_cache = {} 746 747 if var == "BB_RENAMED_VARIABLES": 748 self._var_renames[flag] = value 749 750 if var in self._var_renames: 751 _print_rename_error(var, loginfo, self._var_renames) 752 # Mark that we have seen a renamed variable 753 self.setVar("_FAILPARSINGERRORHANDLED", True) 754 755 if 'op' not in loginfo: 756 loginfo['op'] = "set" 757 loginfo['flag'] = flag 758 self.varhistory.record(**loginfo) 759 if not var in self.dict: 760 self._makeShadowCopy(var) 761 self.dict[var][flag] = value 762 763 if flag == "_defaultval" and ':' in var: 764 self._setvar_update_overrides(var, **loginfo) 765 if flag == "_defaultval" and var in self.overridevars: 766 self._setvar_update_overridevars(var, value) 767 768 if flag == "unexport" or flag == "export": 769 if not "__exportlist" in self.dict: 770 self._makeShadowCopy("__exportlist") 771 if not "_content" in self.dict["__exportlist"]: 772 self.dict["__exportlist"]["_content"] = set() 773 self.dict["__exportlist"]["_content"].add(var) 774 775 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): 776 if flag == "_content": 777 cachename = var 778 else: 779 if not flag: 780 bb.warn("Calling getVarFlag with flag unset is invalid") 781 return None 782 cachename = var + "[" + flag + "]" 783 784 if not expand and retparser and cachename in self.expand_cache: 785 return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] 786 787 if expand and cachename in self.expand_cache: 788 return self.expand_cache[cachename].value 789 790 local_var = self._findVar(var) 791 value = None 792 removes = set() 793 if flag == "_content" and not parsing: 794 overridedata = self.overridedata.get(var, None) 795 if flag == "_content" and not parsing and overridedata is not None: 796 match = False 797 active = {} 798 self.need_overrides() 799 for (r, o) in overridedata: 800 # FIXME What about double overrides both with "_" in the name? 801 if o in self.overridesset: 802 active[o] = r 803 elif ":" in o: 804 if set(o.split(":")).issubset(self.overridesset): 805 active[o] = r 806 807 mod = True 808 while mod: 809 mod = False 810 for o in self.overrides: 811 for a in active.copy(): 812 if a.endswith(":" + o): 813 t = active[a] 814 del active[a] 815 active[a.replace(":" + o, "")] = t 816 mod = True 817 elif a == o: 818 match = active[a] 819 del active[a] 820 if match: 821 value, subparser = self.getVarFlag(match, "_content", False, retparser=True) 822 if hasattr(subparser, "removes"): 823 # We have to carry the removes from the overridden variable to apply at the 824 # end of processing 825 removes = subparser.removes 826 827 if local_var is not None and value is None: 828 if flag in local_var: 829 value = copy.copy(local_var[flag]) 830 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 831 value = copy.copy(local_var["_defaultval"]) 832 833 834 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: 835 self.need_overrides() 836 for (r, o) in local_var[":append"]: 837 match = True 838 if o: 839 for o2 in o.split(":"): 840 if not o2 in self.overrides: 841 match = False 842 if match: 843 if value is None: 844 value = "" 845 value = value + r 846 847 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: 848 self.need_overrides() 849 for (r, o) in local_var[":prepend"]: 850 851 match = True 852 if o: 853 for o2 in o.split(":"): 854 if not o2 in self.overrides: 855 match = False 856 if match: 857 if value is None: 858 value = "" 859 value = r + value 860 861 parser = None 862 if expand or retparser: 863 parser = self.expandWithRefs(value, cachename) 864 if expand: 865 value = parser.value 866 867 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: 868 self.need_overrides() 869 for (r, o) in local_var[":remove"]: 870 match = True 871 if o: 872 for o2 in o.split(":"): 873 if not o2 in self.overrides: 874 match = False 875 if match: 876 removes.add(r) 877 878 if value and flag == "_content" and not parsing: 879 if removes and parser: 880 expanded_removes = {} 881 for r in removes: 882 expanded_removes[r] = self.expand(r).split() 883 884 parser.removes = set() 885 val = [] 886 for v in __whitespace_split__.split(parser.value): 887 skip = False 888 for r in removes: 889 if v in expanded_removes[r]: 890 parser.removes.add(r) 891 skip = True 892 if skip: 893 continue 894 val.append(v) 895 parser.value = "".join(val) 896 if expand: 897 value = parser.value 898 899 if parser: 900 self.expand_cache[cachename] = parser 901 902 if retparser: 903 return value, parser 904 905 return value 906 907 def delVarFlag(self, var, flag, **loginfo): 908 self.expand_cache = {} 909 910 local_var = self._findVar(var) 911 if not local_var: 912 return 913 if not var in self.dict: 914 self._makeShadowCopy(var) 915 916 if var in self.dict and flag in self.dict[var]: 917 loginfo['detail'] = "" 918 loginfo['op'] = 'delFlag' 919 loginfo['flag'] = flag 920 self.varhistory.record(**loginfo) 921 922 del self.dict[var][flag] 923 924 def appendVarFlag(self, var, flag, value, **loginfo): 925 loginfo['op'] = 'append' 926 loginfo['flag'] = flag 927 self.varhistory.record(**loginfo) 928 newvalue = (self.getVarFlag(var, flag, False) or "") + value 929 self.setVarFlag(var, flag, newvalue, ignore=True) 930 931 def prependVarFlag(self, var, flag, value, **loginfo): 932 loginfo['op'] = 'prepend' 933 loginfo['flag'] = flag 934 self.varhistory.record(**loginfo) 935 newvalue = value + (self.getVarFlag(var, flag, False) or "") 936 self.setVarFlag(var, flag, newvalue, ignore=True) 937 938 def setVarFlags(self, var, flags, **loginfo): 939 self.expand_cache = {} 940 infer_caller_details(loginfo) 941 if not var in self.dict: 942 self._makeShadowCopy(var) 943 944 for i in flags: 945 if i == "_content": 946 continue 947 loginfo['flag'] = i 948 loginfo['detail'] = flags[i] 949 self.varhistory.record(**loginfo) 950 self.dict[var][i] = flags[i] 951 952 def getVarFlags(self, var, expand = False, internalflags=False): 953 local_var = self._findVar(var) 954 flags = {} 955 956 if local_var: 957 for i in local_var: 958 if i.startswith(("_", ":")) and not internalflags: 959 continue 960 flags[i] = local_var[i] 961 if expand and i in expand: 962 flags[i] = self.expand(flags[i], var + "[" + i + "]") 963 if len(flags) == 0: 964 return None 965 return flags 966 967 968 def delVarFlags(self, var, **loginfo): 969 self.expand_cache = {} 970 if not var in self.dict: 971 self._makeShadowCopy(var) 972 973 if var in self.dict: 974 content = None 975 976 loginfo['op'] = 'delete flags' 977 self.varhistory.record(**loginfo) 978 979 # try to save the content 980 if "_content" in self.dict[var]: 981 content = self.dict[var]["_content"] 982 self.dict[var] = {} 983 self.dict[var]["_content"] = content 984 else: 985 del self.dict[var] 986 987 def createCopy(self): 988 """ 989 Create a copy of self by setting _data to self 990 """ 991 # we really want this to be a DataSmart... 992 data = DataSmart() 993 data.dict["_data"] = self.dict 994 data.varhistory = self.varhistory.copy() 995 data.varhistory.dataroot = data 996 data.inchistory = self.inchistory.copy() 997 998 data._tracking = self._tracking 999 data._var_renames = self._var_renames 1000 1001 data.overrides = None 1002 data.overridevars = copy.copy(self.overridevars) 1003 # Should really be a deepcopy but has heavy overhead. 1004 # Instead, we're careful with writes. 1005 data.overridedata = copy.copy(self.overridedata) 1006 1007 return data 1008 1009 def expandVarref(self, variable, parents=False): 1010 """Find all references to variable in the data and expand it 1011 in place, optionally descending to parent datastores.""" 1012 1013 if parents: 1014 keys = iter(self) 1015 else: 1016 keys = self.localkeys() 1017 1018 ref = '${%s}' % variable 1019 value = self.getVar(variable, False) 1020 for key in keys: 1021 referrervalue = self.getVar(key, False) 1022 if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: 1023 self.setVar(key, referrervalue.replace(ref, value)) 1024 1025 def localkeys(self): 1026 for key in self.dict: 1027 if key not in ['_data']: 1028 yield key 1029 1030 def __iter__(self): 1031 deleted = set() 1032 overrides = set() 1033 def keylist(d): 1034 klist = set() 1035 for key in d: 1036 if key in ["_data"]: 1037 continue 1038 if key in deleted: 1039 continue 1040 if key in overrides: 1041 continue 1042 if not d[key]: 1043 deleted.add(key) 1044 continue 1045 klist.add(key) 1046 1047 if "_data" in d: 1048 klist |= keylist(d["_data"]) 1049 1050 return klist 1051 1052 self.need_overrides() 1053 for var in self.overridedata: 1054 for (r, o) in self.overridedata[var]: 1055 if o in self.overridesset: 1056 overrides.add(var) 1057 elif ":" in o: 1058 if set(o.split(":")).issubset(self.overridesset): 1059 overrides.add(var) 1060 1061 for k in keylist(self.dict): 1062 yield k 1063 1064 for k in overrides: 1065 yield k 1066 1067 def __len__(self): 1068 return len(frozenset(iter(self))) 1069 1070 def __getitem__(self, item): 1071 value = self.getVar(item, False) 1072 if value is None: 1073 raise KeyError(item) 1074 else: 1075 return value 1076 1077 def __setitem__(self, var, value): 1078 self.setVar(var, value) 1079 1080 def __delitem__(self, var): 1081 self.delVar(var) 1082 1083 def get_hash(self): 1084 data = {} 1085 d = self.createCopy() 1086 bb.data.expandKeys(d) 1087 1088 config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) 1089 keys = set(key for key in iter(d) if not key.startswith("__")) 1090 for key in keys: 1091 if key in config_ignore_vars: 1092 continue 1093 1094 value = d.getVar(key, False) or "" 1095 if type(value) is type(self): 1096 data.update({key:value.get_hash()}) 1097 else: 1098 data.update({key:value}) 1099 1100 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) 1101 if not varflags: 1102 continue 1103 for f in varflags: 1104 if f == "_content": 1105 continue 1106 data.update({'%s[%s]' % (key, f):varflags[f]}) 1107 1108 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: 1109 bb_list = d.getVar(key, False) or [] 1110 data.update({key:str(bb_list)}) 1111 1112 if key == "__BBANONFUNCS": 1113 for i in bb_list: 1114 value = d.getVar(i, False) or "" 1115 data.update({i:value}) 1116 1117 data_str = str([(k, data[k]) for k in sorted(data.keys())]) 1118 return hashlib.sha256(data_str.encode("utf-8")).hexdigest() 1119