1""" 2BitBake Smart Dictionary Implementation 3 4Functions for interacting with the data structure used by the 5BitBake build tools. 6 7""" 8 9# Copyright (C) 2003, 2004 Chris Larson 10# Copyright (C) 2004, 2005 Seb Frankengul 11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther 12# Copyright (C) 2005 Uli Luckas 13# Copyright (C) 2005 ROAD GmbH 14# 15# SPDX-License-Identifier: GPL-2.0-only 16# 17# Based on functions from the base bb module, Copyright 2003 Holger Schurig 18 19import copy, re, sys, traceback 20from collections.abc import MutableMapping 21import logging 22import hashlib 23import bb, bb.codeparser 24from bb import utils 25from bb.COW import COWDictBase 26 27logger = logging.getLogger("BitBake.Data") 28 29__setvar_keyword__ = [":append", ":prepend", ":remove"] 30__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') 31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") 32__expand_python_regexp__ = re.compile(r"\${@.+?}") 33__whitespace_split__ = re.compile(r'(\s)') 34__override_regexp__ = re.compile(r'[a-z0-9]+') 35 36bitbake_renamed_vars = { 37 "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", 38 "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", 39 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", 40 "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", 41 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", 42 "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", 43 "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", 44 "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", 45 "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", 46} 47 48def infer_caller_details(loginfo, parent = False, varval = True): 49 """Save the caller the trouble of specifying everything.""" 50 # Save effort. 51 if 'ignore' in loginfo and loginfo['ignore']: 52 return 53 # If nothing was provided, mark this as possibly unneeded. 54 if not loginfo: 55 loginfo['ignore'] = True 56 return 57 # Infer caller's likely values for variable (var) and value (value), 58 # to reduce clutter in the rest of the code. 59 above = None 60 def set_above(): 61 try: 62 raise Exception 63 except Exception: 64 tb = sys.exc_info()[2] 65 if parent: 66 return tb.tb_frame.f_back.f_back.f_back 67 else: 68 return tb.tb_frame.f_back.f_back 69 70 if varval and ('variable' not in loginfo or 'detail' not in loginfo): 71 if not above: 72 above = set_above() 73 lcls = above.f_locals.items() 74 for k, v in lcls: 75 if k == 'value' and 'detail' not in loginfo: 76 loginfo['detail'] = v 77 if k == 'var' and 'variable' not in loginfo: 78 loginfo['variable'] = v 79 # Infer file/line/function from traceback 80 # Don't use traceback.extract_stack() since it fills the line contents which 81 # we don't need and that hits stat syscalls 82 if 'file' not in loginfo: 83 if not above: 84 above = set_above() 85 f = above.f_back 86 line = f.f_lineno 87 file = f.f_code.co_filename 88 func = f.f_code.co_name 89 loginfo['file'] = file 90 loginfo['line'] = line 91 if func not in loginfo: 92 loginfo['func'] = func 93 94class VariableParse: 95 def __init__(self, varname, d, val = None): 96 self.varname = varname 97 self.d = d 98 self.value = val 99 100 self.references = set() 101 self.execs = set() 102 self.contains = {} 103 104 def var_sub(self, match): 105 key = match.group()[2:-1] 106 if self.varname and key: 107 if self.varname == key: 108 raise Exception("variable %s references itself!" % self.varname) 109 var = self.d.getVarFlag(key, "_content") 110 self.references.add(key) 111 if var is not None: 112 return var 113 else: 114 return match.group() 115 116 def python_sub(self, match): 117 if isinstance(match, str): 118 code = match 119 else: 120 code = match.group()[3:-1] 121 122 if self.varname: 123 varname = 'Var <%s>' % self.varname 124 else: 125 varname = '<expansion>' 126 codeobj = compile(code.strip(), varname, "eval") 127 128 parser = bb.codeparser.PythonParser(self.varname, logger) 129 parser.parse_python(code) 130 if self.varname: 131 vardeps = self.d.getVarFlag(self.varname, "vardeps") 132 if vardeps is None: 133 parser.log.flush() 134 else: 135 parser.log.flush() 136 self.references |= parser.references 137 self.execs |= parser.execs 138 139 for k in parser.contains: 140 if k not in self.contains: 141 self.contains[k] = parser.contains[k].copy() 142 else: 143 self.contains[k].update(parser.contains[k]) 144 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 145 return str(value) 146 147 148class DataContext(dict): 149 def __init__(self, metadata, **kwargs): 150 self.metadata = metadata 151 dict.__init__(self, **kwargs) 152 self['d'] = metadata 153 154 def __missing__(self, key): 155 # Skip commonly accessed invalid variables 156 if key in ['bb', 'oe', 'int', 'bool', 'time', 'str', 'os']: 157 raise KeyError(key) 158 value = self.metadata.getVar(key) 159 if value is None or self.metadata.getVarFlag(key, 'func', False): 160 raise KeyError(key) 161 else: 162 return value 163 164class ExpansionError(Exception): 165 def __init__(self, varname, expression, exception): 166 self.expression = expression 167 self.variablename = varname 168 self.exception = exception 169 self.varlist = [varname or expression or ""] 170 if varname: 171 if expression: 172 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 173 else: 174 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) 175 else: 176 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 177 Exception.__init__(self, self.msg) 178 self.args = (varname, expression, exception) 179 180 def addVar(self, varname): 181 if varname: 182 self.varlist.append(varname) 183 184 def __str__(self): 185 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) 186 return self.msg + chain 187 188class IncludeHistory(object): 189 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 190 self.parent = parent 191 self.filename = filename 192 self.children = [] 193 self.current = self 194 195 def copy(self): 196 new = IncludeHistory(self.parent, self.filename) 197 for c in self.children: 198 new.children.append(c) 199 return new 200 201 def include(self, filename): 202 newfile = IncludeHistory(self.current, filename) 203 self.current.children.append(newfile) 204 self.current = newfile 205 return self 206 207 def __enter__(self): 208 pass 209 210 def __exit__(self, a, b, c): 211 if self.current.parent: 212 self.current = self.current.parent 213 else: 214 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) 215 return False 216 217 def emit(self, o, level = 0): 218 """Emit an include history file, and its children.""" 219 if level: 220 spaces = " " * (level - 1) 221 o.write("# %s%s" % (spaces, self.filename)) 222 if len(self.children) > 0: 223 o.write(" includes:") 224 else: 225 o.write("#\n# INCLUDE HISTORY:\n#") 226 level = level + 1 227 for child in self.children: 228 o.write("\n") 229 child.emit(o, level) 230 231class VariableHistory(object): 232 def __init__(self, dataroot): 233 self.dataroot = dataroot 234 self.variables = COWDictBase.copy() 235 236 def copy(self): 237 new = VariableHistory(self.dataroot) 238 new.variables = self.variables.copy() 239 return new 240 241 def __getstate__(self): 242 vardict = {} 243 for k, v in self.variables.iteritems(): 244 vardict[k] = v 245 return {'dataroot': self.dataroot, 246 'variables': vardict} 247 248 def __setstate__(self, state): 249 self.dataroot = state['dataroot'] 250 self.variables = COWDictBase.copy() 251 for k, v in state['variables'].items(): 252 self.variables[k] = v 253 254 def record(self, *kwonly, **loginfo): 255 if not self.dataroot._tracking: 256 return 257 if len(kwonly) > 0: 258 raise TypeError 259 infer_caller_details(loginfo, parent = True) 260 if 'ignore' in loginfo and loginfo['ignore']: 261 return 262 if 'op' not in loginfo or not loginfo['op']: 263 loginfo['op'] = 'set' 264 if 'detail' in loginfo: 265 loginfo['detail'] = str(loginfo['detail']) 266 if 'variable' not in loginfo or 'file' not in loginfo: 267 raise ValueError("record() missing variable or file.") 268 var = loginfo['variable'] 269 270 if var not in self.variables: 271 self.variables[var] = [] 272 if not isinstance(self.variables[var], list): 273 return 274 if 'nodups' in loginfo and loginfo in self.variables[var]: 275 return 276 self.variables[var].append(loginfo.copy()) 277 278 def rename_variable_hist(self, oldvar, newvar): 279 if not self.dataroot._tracking: 280 return 281 if oldvar not in self.variables: 282 return 283 if newvar not in self.variables: 284 self.variables[newvar] = [] 285 for i in self.variables[oldvar]: 286 self.variables[newvar].append(i.copy()) 287 288 def variable(self, var): 289 varhistory = [] 290 if var in self.variables: 291 varhistory.extend(self.variables[var]) 292 return varhistory 293 294 def emit(self, var, oval, val, o, d): 295 history = self.variable(var) 296 297 # Append override history 298 if var in d.overridedata: 299 for (r, override) in d.overridedata[var]: 300 for event in self.variable(r): 301 loginfo = event.copy() 302 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): 303 continue 304 loginfo['variable'] = var 305 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 306 history.append(loginfo) 307 308 commentVal = re.sub('\n', '\n#', str(oval)) 309 if history: 310 if len(history) == 1: 311 o.write("#\n# $%s\n" % var) 312 else: 313 o.write("#\n# $%s [%d operations]\n" % (var, len(history))) 314 for event in history: 315 # o.write("# %s\n" % str(event)) 316 if 'func' in event: 317 # If we have a function listed, this is internal 318 # code, not an operation in a config file, and the 319 # full path is distracting. 320 event['file'] = re.sub('.*/', '', event['file']) 321 display_func = ' [%s]' % event['func'] 322 else: 323 display_func = '' 324 if 'flag' in event: 325 flag = '[%s] ' % (event['flag']) 326 else: 327 flag = '' 328 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) 329 if len(history) > 1: 330 o.write("# pre-expansion value:\n") 331 o.write('# "%s"\n' % (commentVal)) 332 else: 333 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) 334 o.write('# "%s"\n' % (commentVal)) 335 336 def get_variable_files(self, var): 337 """Get the files where operations are made on a variable""" 338 var_history = self.variable(var) 339 files = [] 340 for event in var_history: 341 files.append(event['file']) 342 return files 343 344 def get_variable_lines(self, var, f): 345 """Get the line where a operation is made on a variable in file f""" 346 var_history = self.variable(var) 347 lines = [] 348 for event in var_history: 349 if f== event['file']: 350 line = event['line'] 351 lines.append(line) 352 return lines 353 354 def get_variable_refs(self, var): 355 """Return a dict of file/line references""" 356 var_history = self.variable(var) 357 refs = {} 358 for event in var_history: 359 if event['file'] not in refs: 360 refs[event['file']] = [] 361 refs[event['file']].append(event['line']) 362 return refs 363 364 def get_variable_items_files(self, var): 365 """ 366 Use variable history to map items added to a list variable and 367 the files in which they were added. 368 """ 369 d = self.dataroot 370 history = self.variable(var) 371 finalitems = (d.getVar(var) or '').split() 372 filemap = {} 373 isset = False 374 for event in history: 375 if 'flag' in event: 376 continue 377 if event['op'] == ':remove': 378 continue 379 if isset and event['op'] == 'set?': 380 continue 381 isset = True 382 items = d.expand(event['detail']).split() 383 for item in items: 384 # This is a little crude but is belt-and-braces to avoid us 385 # having to handle every possible operation type specifically 386 if item in finalitems and not item in filemap: 387 filemap[item] = event['file'] 388 return filemap 389 390 def del_var_history(self, var, f=None, line=None): 391 """If file f and line are not given, the entire history of var is deleted""" 392 if var in self.variables: 393 if f and line: 394 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] 395 else: 396 self.variables[var] = [] 397 398def _print_rename_error(var, loginfo, renamedvars, fullvar=None): 399 info = "" 400 if "file" in loginfo: 401 info = " file: %s" % loginfo["file"] 402 if "line" in loginfo: 403 info += " line: %s" % loginfo["line"] 404 if fullvar and fullvar != var: 405 info += " referenced as: %s" % fullvar 406 if info: 407 info = " (%s)" % info.strip() 408 renameinfo = renamedvars[var] 409 if " " in renameinfo: 410 # A space signals a string to display instead of a rename 411 bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) 412 else: 413 bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) 414 415class DataSmart(MutableMapping): 416 def __init__(self): 417 self.dict = {} 418 419 self.inchistory = IncludeHistory() 420 self.varhistory = VariableHistory(self) 421 self._tracking = False 422 self._var_renames = {} 423 self._var_renames.update(bitbake_renamed_vars) 424 425 self.expand_cache = {} 426 427 # cookie monster tribute 428 # Need to be careful about writes to overridedata as 429 # its only a shallow copy, could influence other data store 430 # copies! 431 self.overridedata = {} 432 self.overrides = None 433 self.overridevars = set(["OVERRIDES", "FILE"]) 434 self.inoverride = False 435 436 def enableTracking(self): 437 self._tracking = True 438 439 def disableTracking(self): 440 self._tracking = False 441 442 def expandWithRefs(self, s, varname): 443 444 if not isinstance(s, str): # sanity check 445 return VariableParse(varname, self, s) 446 447 varparse = VariableParse(varname, self) 448 449 while s.find('${') != -1: 450 olds = s 451 try: 452 s = __expand_var_regexp__.sub(varparse.var_sub, s) 453 try: 454 s = __expand_python_regexp__.sub(varparse.python_sub, s) 455 except SyntaxError as e: 456 # Likely unmatched brackets, just don't expand the expression 457 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): 458 raise 459 if s == olds: 460 break 461 except ExpansionError as e: 462 e.addVar(varname) 463 raise 464 except bb.parse.SkipRecipe: 465 raise 466 except bb.BBHandledException: 467 raise 468 except Exception as exc: 469 tb = sys.exc_info()[2] 470 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 471 472 varparse.value = s 473 474 return varparse 475 476 def expand(self, s, varname = None): 477 return self.expandWithRefs(s, varname).value 478 479 def finalize(self, parent = False): 480 return 481 482 def internal_finalize(self, parent = False): 483 """Performs final steps upon the datastore, including application of overrides""" 484 self.overrides = None 485 486 def need_overrides(self): 487 if self.overrides is not None: 488 return 489 if self.inoverride: 490 return 491 for count in range(5): 492 self.inoverride = True 493 # Can end up here recursively so setup dummy values 494 self.overrides = [] 495 self.overridesset = set() 496 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 497 self.overridesset = set(self.overrides) 498 self.inoverride = False 499 self.expand_cache = {} 500 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or [] 501 if newoverrides == self.overrides: 502 break 503 self.overrides = newoverrides 504 self.overridesset = set(self.overrides) 505 else: 506 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") 507 508 def initVar(self, var): 509 self.expand_cache = {} 510 if not var in self.dict: 511 self.dict[var] = {} 512 513 def _findVar(self, var): 514 dest = self.dict 515 while dest: 516 if var in dest: 517 return dest[var], self.overridedata.get(var, None) 518 519 if "_data" not in dest: 520 break 521 dest = dest["_data"] 522 return None, self.overridedata.get(var, None) 523 524 def _makeShadowCopy(self, var): 525 if var in self.dict: 526 return 527 528 local_var, _ = self._findVar(var) 529 530 if local_var: 531 self.dict[var] = copy.copy(local_var) 532 else: 533 self.initVar(var) 534 535 def hasOverrides(self, var): 536 return var in self.overridedata 537 538 def setVar(self, var, value, **loginfo): 539 #print("var=" + str(var) + " val=" + str(value)) 540 541 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): 542 info = "%s" % var 543 if "file" in loginfo: 544 info += " file: %s" % loginfo["file"] 545 if "line" in loginfo: 546 info += " line: %s" % loginfo["line"] 547 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) 548 549 shortvar = var.split(":", 1)[0] 550 if shortvar in self._var_renames: 551 _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) 552 # Mark that we have seen a renamed variable 553 self.setVar("_FAILPARSINGERRORHANDLED", True) 554 555 self.expand_cache = {} 556 parsing=False 557 if 'parsing' in loginfo: 558 parsing=True 559 560 if 'op' not in loginfo: 561 loginfo['op'] = "set" 562 563 match = __setvar_regexp__.match(var) 564 if match and match.group("keyword") in __setvar_keyword__: 565 base = match.group('base') 566 keyword = match.group("keyword") 567 override = match.group('add') 568 l = self.getVarFlag(base, keyword, False) or [] 569 l.append([value, override]) 570 self.setVarFlag(base, keyword, l, ignore=True) 571 # And cause that to be recorded: 572 loginfo['detail'] = value 573 loginfo['variable'] = base 574 if override: 575 loginfo['op'] = '%s[%s]' % (keyword, override) 576 else: 577 loginfo['op'] = keyword 578 self.varhistory.record(**loginfo) 579 # todo make sure keyword is not __doc__ or __module__ 580 # pay the cookie monster 581 582 # more cookies for the cookie monster 583 if ':' in var: 584 self._setvar_update_overrides(base, **loginfo) 585 586 if base in self.overridevars: 587 self._setvar_update_overridevars(var, value) 588 return 589 590 if not var in self.dict: 591 self._makeShadowCopy(var) 592 593 if not parsing: 594 if ":append" in self.dict[var]: 595 del self.dict[var][":append"] 596 if ":prepend" in self.dict[var]: 597 del self.dict[var][":prepend"] 598 if ":remove" in self.dict[var]: 599 del self.dict[var][":remove"] 600 if var in self.overridedata: 601 active = [] 602 self.need_overrides() 603 for (r, o) in self.overridedata[var]: 604 if o in self.overridesset: 605 active.append(r) 606 elif ":" in o: 607 if set(o.split(":")).issubset(self.overridesset): 608 active.append(r) 609 for a in active: 610 self.delVar(a) 611 del self.overridedata[var] 612 613 # more cookies for the cookie monster 614 if ':' in var: 615 self._setvar_update_overrides(var, **loginfo) 616 617 # setting var 618 self.dict[var]["_content"] = value 619 self.varhistory.record(**loginfo) 620 621 if var in self.overridevars: 622 self._setvar_update_overridevars(var, value) 623 624 def _setvar_update_overridevars(self, var, value): 625 vardata = self.expandWithRefs(value, var) 626 new = vardata.references 627 new.update(vardata.contains.keys()) 628 while not new.issubset(self.overridevars): 629 nextnew = set() 630 self.overridevars.update(new) 631 for i in new: 632 vardata = self.expandWithRefs(self.getVar(i), i) 633 nextnew.update(vardata.references) 634 nextnew.update(vardata.contains.keys()) 635 new = nextnew 636 self.internal_finalize(True) 637 638 def _setvar_update_overrides(self, var, **loginfo): 639 # aka pay the cookie monster 640 override = var[var.rfind(':')+1:] 641 shortvar = var[:var.rfind(':')] 642 while override and __override_regexp__.match(override): 643 if shortvar not in self.overridedata: 644 self.overridedata[shortvar] = [] 645 if [var, override] not in self.overridedata[shortvar]: 646 # Force CoW by recreating the list first 647 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 648 self.overridedata[shortvar].append([var, override]) 649 override = None 650 if ":" in shortvar: 651 override = var[shortvar.rfind(':')+1:] 652 shortvar = var[:shortvar.rfind(':')] 653 if len(shortvar) == 0: 654 override = None 655 656 def getVar(self, var, expand=True, noweakdefault=False, parsing=False): 657 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) 658 659 def renameVar(self, key, newkey, **loginfo): 660 """ 661 Rename the variable key to newkey 662 """ 663 if key == newkey: 664 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) 665 return 666 667 val = self.getVar(key, 0, parsing=True) 668 if val is not None: 669 self.varhistory.rename_variable_hist(key, newkey) 670 loginfo['variable'] = newkey 671 loginfo['op'] = 'rename from %s' % key 672 loginfo['detail'] = val 673 self.varhistory.record(**loginfo) 674 self.setVar(newkey, val, ignore=True, parsing=True) 675 676 srcflags = self.getVarFlags(key, False, True) or {} 677 for i in srcflags: 678 if i not in (__setvar_keyword__): 679 continue 680 src = srcflags[i] 681 682 dest = self.getVarFlag(newkey, i, False) or [] 683 dest.extend(src) 684 self.setVarFlag(newkey, i, dest, ignore=True) 685 686 if key in self.overridedata: 687 self.overridedata[newkey] = [] 688 for (v, o) in self.overridedata[key]: 689 self.overridedata[newkey].append([v.replace(key, newkey), o]) 690 self.renameVar(v, v.replace(key, newkey)) 691 692 if ':' in newkey and val is None: 693 self._setvar_update_overrides(newkey, **loginfo) 694 695 loginfo['variable'] = key 696 loginfo['op'] = 'rename (to)' 697 loginfo['detail'] = newkey 698 self.varhistory.record(**loginfo) 699 self.delVar(key, ignore=True) 700 701 def appendVar(self, var, value, **loginfo): 702 loginfo['op'] = 'append' 703 self.varhistory.record(**loginfo) 704 self.setVar(var + ":append", value, ignore=True, parsing=True) 705 706 def prependVar(self, var, value, **loginfo): 707 loginfo['op'] = 'prepend' 708 self.varhistory.record(**loginfo) 709 self.setVar(var + ":prepend", value, ignore=True, parsing=True) 710 711 def delVar(self, var, **loginfo): 712 self.expand_cache = {} 713 714 loginfo['detail'] = "" 715 loginfo['op'] = 'del' 716 self.varhistory.record(**loginfo) 717 self.dict[var] = {} 718 if var in self.overridedata: 719 del self.overridedata[var] 720 if ':' in var: 721 override = var[var.rfind(':')+1:] 722 shortvar = var[:var.rfind(':')] 723 while override and override.islower(): 724 try: 725 if shortvar in self.overridedata: 726 # Force CoW by recreating the list first 727 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 728 self.overridedata[shortvar].remove([var, override]) 729 except ValueError as e: 730 pass 731 override = None 732 if ":" in shortvar: 733 override = var[shortvar.rfind(':')+1:] 734 shortvar = var[:shortvar.rfind(':')] 735 if len(shortvar) == 0: 736 override = None 737 738 def setVarFlag(self, var, flag, value, **loginfo): 739 self.expand_cache = {} 740 741 if var == "BB_RENAMED_VARIABLES": 742 self._var_renames[flag] = value 743 744 if var in self._var_renames: 745 _print_rename_error(var, loginfo, self._var_renames) 746 # Mark that we have seen a renamed variable 747 self.setVar("_FAILPARSINGERRORHANDLED", True) 748 749 if 'op' not in loginfo: 750 loginfo['op'] = "set" 751 loginfo['flag'] = flag 752 self.varhistory.record(**loginfo) 753 if not var in self.dict: 754 self._makeShadowCopy(var) 755 self.dict[var][flag] = value 756 757 if flag == "_defaultval" and ':' in var: 758 self._setvar_update_overrides(var, **loginfo) 759 if flag == "_defaultval" and var in self.overridevars: 760 self._setvar_update_overridevars(var, value) 761 762 if flag == "unexport" or flag == "export": 763 if not "__exportlist" in self.dict: 764 self._makeShadowCopy("__exportlist") 765 if not "_content" in self.dict["__exportlist"]: 766 self.dict["__exportlist"]["_content"] = set() 767 self.dict["__exportlist"]["_content"].add(var) 768 769 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): 770 if flag == "_content": 771 cachename = var 772 else: 773 if not flag: 774 bb.warn("Calling getVarFlag with flag unset is invalid") 775 return None 776 cachename = var + "[" + flag + "]" 777 778 if expand and cachename in self.expand_cache: 779 return self.expand_cache[cachename].value 780 781 local_var, overridedata = self._findVar(var) 782 value = None 783 removes = set() 784 if flag == "_content" and overridedata is not None and not parsing: 785 match = False 786 active = {} 787 self.need_overrides() 788 for (r, o) in overridedata: 789 # FIXME What about double overrides both with "_" in the name? 790 if o in self.overridesset: 791 active[o] = r 792 elif ":" in o: 793 if set(o.split(":")).issubset(self.overridesset): 794 active[o] = r 795 796 mod = True 797 while mod: 798 mod = False 799 for o in self.overrides: 800 for a in active.copy(): 801 if a.endswith(":" + o): 802 t = active[a] 803 del active[a] 804 active[a.replace(":" + o, "")] = t 805 mod = True 806 elif a == o: 807 match = active[a] 808 del active[a] 809 if match: 810 value, subparser = self.getVarFlag(match, "_content", False, retparser=True) 811 if hasattr(subparser, "removes"): 812 # We have to carry the removes from the overridden variable to apply at the 813 # end of processing 814 removes = subparser.removes 815 816 if local_var is not None and value is None: 817 if flag in local_var: 818 value = copy.copy(local_var[flag]) 819 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 820 value = copy.copy(local_var["_defaultval"]) 821 822 823 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: 824 self.need_overrides() 825 for (r, o) in local_var[":append"]: 826 match = True 827 if o: 828 for o2 in o.split(":"): 829 if not o2 in self.overrides: 830 match = False 831 if match: 832 if value is None: 833 value = "" 834 value = value + r 835 836 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: 837 self.need_overrides() 838 for (r, o) in local_var[":prepend"]: 839 840 match = True 841 if o: 842 for o2 in o.split(":"): 843 if not o2 in self.overrides: 844 match = False 845 if match: 846 if value is None: 847 value = "" 848 value = r + value 849 850 parser = None 851 if expand or retparser: 852 parser = self.expandWithRefs(value, cachename) 853 if expand: 854 value = parser.value 855 856 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: 857 self.need_overrides() 858 for (r, o) in local_var[":remove"]: 859 match = True 860 if o: 861 for o2 in o.split(":"): 862 if not o2 in self.overrides: 863 match = False 864 if match: 865 removes.add(r) 866 867 if value and flag == "_content" and not parsing: 868 if removes and parser: 869 expanded_removes = {} 870 for r in removes: 871 expanded_removes[r] = self.expand(r).split() 872 873 parser.removes = set() 874 val = [] 875 for v in __whitespace_split__.split(parser.value): 876 skip = False 877 for r in removes: 878 if v in expanded_removes[r]: 879 parser.removes.add(r) 880 skip = True 881 if skip: 882 continue 883 val.append(v) 884 parser.value = "".join(val) 885 if expand: 886 value = parser.value 887 888 if parser: 889 self.expand_cache[cachename] = parser 890 891 if retparser: 892 return value, parser 893 894 return value 895 896 def delVarFlag(self, var, flag, **loginfo): 897 self.expand_cache = {} 898 899 local_var, _ = self._findVar(var) 900 if not local_var: 901 return 902 if not var in self.dict: 903 self._makeShadowCopy(var) 904 905 if var in self.dict and flag in self.dict[var]: 906 loginfo['detail'] = "" 907 loginfo['op'] = 'delFlag' 908 loginfo['flag'] = flag 909 self.varhistory.record(**loginfo) 910 911 del self.dict[var][flag] 912 913 def appendVarFlag(self, var, flag, value, **loginfo): 914 loginfo['op'] = 'append' 915 loginfo['flag'] = flag 916 self.varhistory.record(**loginfo) 917 newvalue = (self.getVarFlag(var, flag, False) or "") + value 918 self.setVarFlag(var, flag, newvalue, ignore=True) 919 920 def prependVarFlag(self, var, flag, value, **loginfo): 921 loginfo['op'] = 'prepend' 922 loginfo['flag'] = flag 923 self.varhistory.record(**loginfo) 924 newvalue = value + (self.getVarFlag(var, flag, False) or "") 925 self.setVarFlag(var, flag, newvalue, ignore=True) 926 927 def setVarFlags(self, var, flags, **loginfo): 928 self.expand_cache = {} 929 infer_caller_details(loginfo) 930 if not var in self.dict: 931 self._makeShadowCopy(var) 932 933 for i in flags: 934 if i == "_content": 935 continue 936 loginfo['flag'] = i 937 loginfo['detail'] = flags[i] 938 self.varhistory.record(**loginfo) 939 self.dict[var][i] = flags[i] 940 941 def getVarFlags(self, var, expand = False, internalflags=False): 942 local_var, _ = self._findVar(var) 943 flags = {} 944 945 if local_var: 946 for i in local_var: 947 if i.startswith(("_", ":")) and not internalflags: 948 continue 949 flags[i] = local_var[i] 950 if expand and i in expand: 951 flags[i] = self.expand(flags[i], var + "[" + i + "]") 952 if len(flags) == 0: 953 return None 954 return flags 955 956 957 def delVarFlags(self, var, **loginfo): 958 self.expand_cache = {} 959 if not var in self.dict: 960 self._makeShadowCopy(var) 961 962 if var in self.dict: 963 content = None 964 965 loginfo['op'] = 'delete flags' 966 self.varhistory.record(**loginfo) 967 968 # try to save the content 969 if "_content" in self.dict[var]: 970 content = self.dict[var]["_content"] 971 self.dict[var] = {} 972 self.dict[var]["_content"] = content 973 else: 974 del self.dict[var] 975 976 def createCopy(self): 977 """ 978 Create a copy of self by setting _data to self 979 """ 980 # we really want this to be a DataSmart... 981 data = DataSmart() 982 data.dict["_data"] = self.dict 983 data.varhistory = self.varhistory.copy() 984 data.varhistory.dataroot = data 985 data.inchistory = self.inchistory.copy() 986 987 data._tracking = self._tracking 988 data._var_renames = self._var_renames 989 990 data.overrides = None 991 data.overridevars = copy.copy(self.overridevars) 992 # Should really be a deepcopy but has heavy overhead. 993 # Instead, we're careful with writes. 994 data.overridedata = copy.copy(self.overridedata) 995 996 return data 997 998 def expandVarref(self, variable, parents=False): 999 """Find all references to variable in the data and expand it 1000 in place, optionally descending to parent datastores.""" 1001 1002 if parents: 1003 keys = iter(self) 1004 else: 1005 keys = self.localkeys() 1006 1007 ref = '${%s}' % variable 1008 value = self.getVar(variable, False) 1009 for key in keys: 1010 referrervalue = self.getVar(key, False) 1011 if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: 1012 self.setVar(key, referrervalue.replace(ref, value)) 1013 1014 def localkeys(self): 1015 for key in self.dict: 1016 if key not in ['_data']: 1017 yield key 1018 1019 def __iter__(self): 1020 deleted = set() 1021 overrides = set() 1022 def keylist(d): 1023 klist = set() 1024 for key in d: 1025 if key in ["_data"]: 1026 continue 1027 if key in deleted: 1028 continue 1029 if key in overrides: 1030 continue 1031 if not d[key]: 1032 deleted.add(key) 1033 continue 1034 klist.add(key) 1035 1036 if "_data" in d: 1037 klist |= keylist(d["_data"]) 1038 1039 return klist 1040 1041 self.need_overrides() 1042 for var in self.overridedata: 1043 for (r, o) in self.overridedata[var]: 1044 if o in self.overridesset: 1045 overrides.add(var) 1046 elif ":" in o: 1047 if set(o.split(":")).issubset(self.overridesset): 1048 overrides.add(var) 1049 1050 for k in keylist(self.dict): 1051 yield k 1052 1053 for k in overrides: 1054 yield k 1055 1056 def __len__(self): 1057 return len(frozenset(iter(self))) 1058 1059 def __getitem__(self, item): 1060 value = self.getVar(item, False) 1061 if value is None: 1062 raise KeyError(item) 1063 else: 1064 return value 1065 1066 def __setitem__(self, var, value): 1067 self.setVar(var, value) 1068 1069 def __delitem__(self, var): 1070 self.delVar(var) 1071 1072 def get_hash(self): 1073 data = {} 1074 d = self.createCopy() 1075 bb.data.expandKeys(d) 1076 1077 config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) 1078 keys = set(key for key in iter(d) if not key.startswith("__")) 1079 for key in keys: 1080 if key in config_ignore_vars: 1081 continue 1082 1083 value = d.getVar(key, False) or "" 1084 if type(value) is type(self): 1085 data.update({key:value.get_hash()}) 1086 else: 1087 data.update({key:value}) 1088 1089 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) 1090 if not varflags: 1091 continue 1092 for f in varflags: 1093 if f == "_content": 1094 continue 1095 data.update({'%s[%s]' % (key, f):varflags[f]}) 1096 1097 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: 1098 bb_list = d.getVar(key, False) or [] 1099 data.update({key:str(bb_list)}) 1100 1101 if key == "__BBANONFUNCS": 1102 for i in bb_list: 1103 value = d.getVar(i, False) or "" 1104 data.update({i:value}) 1105 1106 data_str = str([(k, data[k]) for k in sorted(data.keys())]) 1107 return hashlib.sha256(data_str.encode("utf-8")).hexdigest() 1108