1""" 2BitBake Smart Dictionary Implementation 3 4Functions for interacting with the data structure used by the 5BitBake build tools. 6 7""" 8 9# Copyright (C) 2003, 2004 Chris Larson 10# Copyright (C) 2004, 2005 Seb Frankengul 11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther 12# Copyright (C) 2005 Uli Luckas 13# Copyright (C) 2005 ROAD GmbH 14# 15# SPDX-License-Identifier: GPL-2.0-only 16# 17# Based on functions from the base bb module, Copyright 2003 Holger Schurig 18 19import copy, re, sys, traceback 20from collections import MutableMapping 21import logging 22import hashlib 23import bb, bb.codeparser 24from bb import utils 25from bb.COW import COWDictBase 26 27logger = logging.getLogger("BitBake.Data") 28 29__setvar_keyword__ = [":append", ":prepend", ":remove"] 30__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') 31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") 32__expand_python_regexp__ = re.compile(r"\${@.+?}") 33__whitespace_split__ = re.compile(r'(\s)') 34__override_regexp__ = re.compile(r'[a-z0-9]+') 35 36def infer_caller_details(loginfo, parent = False, varval = True): 37 """Save the caller the trouble of specifying everything.""" 38 # Save effort. 39 if 'ignore' in loginfo and loginfo['ignore']: 40 return 41 # If nothing was provided, mark this as possibly unneeded. 42 if not loginfo: 43 loginfo['ignore'] = True 44 return 45 # Infer caller's likely values for variable (var) and value (value), 46 # to reduce clutter in the rest of the code. 47 above = None 48 def set_above(): 49 try: 50 raise Exception 51 except Exception: 52 tb = sys.exc_info()[2] 53 if parent: 54 return tb.tb_frame.f_back.f_back.f_back 55 else: 56 return tb.tb_frame.f_back.f_back 57 58 if varval and ('variable' not in loginfo or 'detail' not in loginfo): 59 if not above: 60 above = set_above() 61 lcls = above.f_locals.items() 62 for k, v in lcls: 63 if k == 'value' and 'detail' not in loginfo: 64 loginfo['detail'] = v 65 if k == 'var' and 'variable' not in loginfo: 66 loginfo['variable'] = v 67 # Infer file/line/function from traceback 68 # Don't use traceback.extract_stack() since it fills the line contents which 69 # we don't need and that hits stat syscalls 70 if 'file' not in loginfo: 71 if not above: 72 above = set_above() 73 f = above.f_back 74 line = f.f_lineno 75 file = f.f_code.co_filename 76 func = f.f_code.co_name 77 loginfo['file'] = file 78 loginfo['line'] = line 79 if func not in loginfo: 80 loginfo['func'] = func 81 82class VariableParse: 83 def __init__(self, varname, d, val = None): 84 self.varname = varname 85 self.d = d 86 self.value = val 87 88 self.references = set() 89 self.execs = set() 90 self.contains = {} 91 92 def var_sub(self, match): 93 key = match.group()[2:-1] 94 if self.varname and key: 95 if self.varname == key: 96 raise Exception("variable %s references itself!" % self.varname) 97 var = self.d.getVarFlag(key, "_content") 98 self.references.add(key) 99 if var is not None: 100 return var 101 else: 102 return match.group() 103 104 def python_sub(self, match): 105 if isinstance(match, str): 106 code = match 107 else: 108 code = match.group()[3:-1] 109 110 if self.varname: 111 varname = 'Var <%s>' % self.varname 112 else: 113 varname = '<expansion>' 114 codeobj = compile(code.strip(), varname, "eval") 115 116 parser = bb.codeparser.PythonParser(self.varname, logger) 117 parser.parse_python(code) 118 if self.varname: 119 vardeps = self.d.getVarFlag(self.varname, "vardeps") 120 if vardeps is None: 121 parser.log.flush() 122 else: 123 parser.log.flush() 124 self.references |= parser.references 125 self.execs |= parser.execs 126 127 for k in parser.contains: 128 if k not in self.contains: 129 self.contains[k] = parser.contains[k].copy() 130 else: 131 self.contains[k].update(parser.contains[k]) 132 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 133 return str(value) 134 135 136class DataContext(dict): 137 def __init__(self, metadata, **kwargs): 138 self.metadata = metadata 139 dict.__init__(self, **kwargs) 140 self['d'] = metadata 141 142 def __missing__(self, key): 143 value = self.metadata.getVar(key) 144 if value is None or self.metadata.getVarFlag(key, 'func', False): 145 raise KeyError(key) 146 else: 147 return value 148 149class ExpansionError(Exception): 150 def __init__(self, varname, expression, exception): 151 self.expression = expression 152 self.variablename = varname 153 self.exception = exception 154 if varname: 155 if expression: 156 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 157 else: 158 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) 159 else: 160 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 161 Exception.__init__(self, self.msg) 162 self.args = (varname, expression, exception) 163 def __str__(self): 164 return self.msg 165 166class IncludeHistory(object): 167 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 168 self.parent = parent 169 self.filename = filename 170 self.children = [] 171 self.current = self 172 173 def copy(self): 174 new = IncludeHistory(self.parent, self.filename) 175 for c in self.children: 176 new.children.append(c) 177 return new 178 179 def include(self, filename): 180 newfile = IncludeHistory(self.current, filename) 181 self.current.children.append(newfile) 182 self.current = newfile 183 return self 184 185 def __enter__(self): 186 pass 187 188 def __exit__(self, a, b, c): 189 if self.current.parent: 190 self.current = self.current.parent 191 else: 192 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) 193 return False 194 195 def emit(self, o, level = 0): 196 """Emit an include history file, and its children.""" 197 if level: 198 spaces = " " * (level - 1) 199 o.write("# %s%s" % (spaces, self.filename)) 200 if len(self.children) > 0: 201 o.write(" includes:") 202 else: 203 o.write("#\n# INCLUDE HISTORY:\n#") 204 level = level + 1 205 for child in self.children: 206 o.write("\n") 207 child.emit(o, level) 208 209class VariableHistory(object): 210 def __init__(self, dataroot): 211 self.dataroot = dataroot 212 self.variables = COWDictBase.copy() 213 214 def copy(self): 215 new = VariableHistory(self.dataroot) 216 new.variables = self.variables.copy() 217 return new 218 219 def __getstate__(self): 220 vardict = {} 221 for k, v in self.variables.iteritems(): 222 vardict[k] = v 223 return {'dataroot': self.dataroot, 224 'variables': vardict} 225 226 def __setstate__(self, state): 227 self.dataroot = state['dataroot'] 228 self.variables = COWDictBase.copy() 229 for k, v in state['variables'].items(): 230 self.variables[k] = v 231 232 def record(self, *kwonly, **loginfo): 233 if not self.dataroot._tracking: 234 return 235 if len(kwonly) > 0: 236 raise TypeError 237 infer_caller_details(loginfo, parent = True) 238 if 'ignore' in loginfo and loginfo['ignore']: 239 return 240 if 'op' not in loginfo or not loginfo['op']: 241 loginfo['op'] = 'set' 242 if 'detail' in loginfo: 243 loginfo['detail'] = str(loginfo['detail']) 244 if 'variable' not in loginfo or 'file' not in loginfo: 245 raise ValueError("record() missing variable or file.") 246 var = loginfo['variable'] 247 248 if var not in self.variables: 249 self.variables[var] = [] 250 if not isinstance(self.variables[var], list): 251 return 252 if 'nodups' in loginfo and loginfo in self.variables[var]: 253 return 254 self.variables[var].append(loginfo.copy()) 255 256 def rename_variable_hist(self, oldvar, newvar): 257 if not self.dataroot._tracking: 258 return 259 if oldvar not in self.variables: 260 return 261 if newvar not in self.variables: 262 self.variables[newvar] = [] 263 for i in self.variables[oldvar]: 264 self.variables[newvar].append(i.copy()) 265 266 def variable(self, var): 267 varhistory = [] 268 if var in self.variables: 269 varhistory.extend(self.variables[var]) 270 return varhistory 271 272 def emit(self, var, oval, val, o, d): 273 history = self.variable(var) 274 275 # Append override history 276 if var in d.overridedata: 277 for (r, override) in d.overridedata[var]: 278 for event in self.variable(r): 279 loginfo = event.copy() 280 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): 281 continue 282 loginfo['variable'] = var 283 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 284 history.append(loginfo) 285 286 commentVal = re.sub('\n', '\n#', str(oval)) 287 if history: 288 if len(history) == 1: 289 o.write("#\n# $%s\n" % var) 290 else: 291 o.write("#\n# $%s [%d operations]\n" % (var, len(history))) 292 for event in history: 293 # o.write("# %s\n" % str(event)) 294 if 'func' in event: 295 # If we have a function listed, this is internal 296 # code, not an operation in a config file, and the 297 # full path is distracting. 298 event['file'] = re.sub('.*/', '', event['file']) 299 display_func = ' [%s]' % event['func'] 300 else: 301 display_func = '' 302 if 'flag' in event: 303 flag = '[%s] ' % (event['flag']) 304 else: 305 flag = '' 306 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) 307 if len(history) > 1: 308 o.write("# pre-expansion value:\n") 309 o.write('# "%s"\n' % (commentVal)) 310 else: 311 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) 312 o.write('# "%s"\n' % (commentVal)) 313 314 def get_variable_files(self, var): 315 """Get the files where operations are made on a variable""" 316 var_history = self.variable(var) 317 files = [] 318 for event in var_history: 319 files.append(event['file']) 320 return files 321 322 def get_variable_lines(self, var, f): 323 """Get the line where a operation is made on a variable in file f""" 324 var_history = self.variable(var) 325 lines = [] 326 for event in var_history: 327 if f== event['file']: 328 line = event['line'] 329 lines.append(line) 330 return lines 331 332 def get_variable_items_files(self, var): 333 """ 334 Use variable history to map items added to a list variable and 335 the files in which they were added. 336 """ 337 d = self.dataroot 338 history = self.variable(var) 339 finalitems = (d.getVar(var) or '').split() 340 filemap = {} 341 isset = False 342 for event in history: 343 if 'flag' in event: 344 continue 345 if event['op'] == ':remove': 346 continue 347 if isset and event['op'] == 'set?': 348 continue 349 isset = True 350 items = d.expand(event['detail']).split() 351 for item in items: 352 # This is a little crude but is belt-and-braces to avoid us 353 # having to handle every possible operation type specifically 354 if item in finalitems and not item in filemap: 355 filemap[item] = event['file'] 356 return filemap 357 358 def del_var_history(self, var, f=None, line=None): 359 """If file f and line are not given, the entire history of var is deleted""" 360 if var in self.variables: 361 if f and line: 362 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] 363 else: 364 self.variables[var] = [] 365 366class DataSmart(MutableMapping): 367 def __init__(self): 368 self.dict = {} 369 370 self.inchistory = IncludeHistory() 371 self.varhistory = VariableHistory(self) 372 self._tracking = False 373 374 self.expand_cache = {} 375 376 # cookie monster tribute 377 # Need to be careful about writes to overridedata as 378 # its only a shallow copy, could influence other data store 379 # copies! 380 self.overridedata = {} 381 self.overrides = None 382 self.overridevars = set(["OVERRIDES", "FILE"]) 383 self.inoverride = False 384 385 def enableTracking(self): 386 self._tracking = True 387 388 def disableTracking(self): 389 self._tracking = False 390 391 def expandWithRefs(self, s, varname): 392 393 if not isinstance(s, str): # sanity check 394 return VariableParse(varname, self, s) 395 396 varparse = VariableParse(varname, self) 397 398 while s.find('${') != -1: 399 olds = s 400 try: 401 s = __expand_var_regexp__.sub(varparse.var_sub, s) 402 try: 403 s = __expand_python_regexp__.sub(varparse.python_sub, s) 404 except SyntaxError as e: 405 # Likely unmatched brackets, just don't expand the expression 406 if e.msg != "EOL while scanning string literal": 407 raise 408 if s == olds: 409 break 410 except ExpansionError: 411 raise 412 except bb.parse.SkipRecipe: 413 raise 414 except Exception as exc: 415 tb = sys.exc_info()[2] 416 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 417 418 varparse.value = s 419 420 return varparse 421 422 def expand(self, s, varname = None): 423 return self.expandWithRefs(s, varname).value 424 425 def finalize(self, parent = False): 426 return 427 428 def internal_finalize(self, parent = False): 429 """Performs final steps upon the datastore, including application of overrides""" 430 self.overrides = None 431 432 def need_overrides(self): 433 if self.overrides is not None: 434 return 435 if self.inoverride: 436 return 437 for count in range(5): 438 self.inoverride = True 439 # Can end up here recursively so setup dummy values 440 self.overrides = [] 441 self.overridesset = set() 442 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 443 self.overridesset = set(self.overrides) 444 self.inoverride = False 445 self.expand_cache = {} 446 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or [] 447 if newoverrides == self.overrides: 448 break 449 self.overrides = newoverrides 450 self.overridesset = set(self.overrides) 451 else: 452 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") 453 454 def initVar(self, var): 455 self.expand_cache = {} 456 if not var in self.dict: 457 self.dict[var] = {} 458 459 def _findVar(self, var): 460 dest = self.dict 461 while dest: 462 if var in dest: 463 return dest[var], self.overridedata.get(var, None) 464 465 if "_data" not in dest: 466 break 467 dest = dest["_data"] 468 return None, self.overridedata.get(var, None) 469 470 def _makeShadowCopy(self, var): 471 if var in self.dict: 472 return 473 474 local_var, _ = self._findVar(var) 475 476 if local_var: 477 self.dict[var] = copy.copy(local_var) 478 else: 479 self.initVar(var) 480 481 482 def setVar(self, var, value, **loginfo): 483 #print("var=" + str(var) + " val=" + str(value)) 484 485 if "_append" in var or "_prepend" in var or "_remove" in var: 486 info = "%s" % var 487 if "filename" in loginfo: 488 info += " file: %s" % loginfo[filename] 489 if "lineno" in loginfo: 490 info += " line: %s" % loginfo[lineno] 491 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) 492 493 self.expand_cache = {} 494 parsing=False 495 if 'parsing' in loginfo: 496 parsing=True 497 498 if 'op' not in loginfo: 499 loginfo['op'] = "set" 500 501 match = __setvar_regexp__.match(var) 502 if match and match.group("keyword") in __setvar_keyword__: 503 base = match.group('base') 504 keyword = match.group("keyword") 505 override = match.group('add') 506 l = self.getVarFlag(base, keyword, False) or [] 507 l.append([value, override]) 508 self.setVarFlag(base, keyword, l, ignore=True) 509 # And cause that to be recorded: 510 loginfo['detail'] = value 511 loginfo['variable'] = base 512 if override: 513 loginfo['op'] = '%s[%s]' % (keyword, override) 514 else: 515 loginfo['op'] = keyword 516 self.varhistory.record(**loginfo) 517 # todo make sure keyword is not __doc__ or __module__ 518 # pay the cookie monster 519 520 # more cookies for the cookie monster 521 if ':' in var: 522 self._setvar_update_overrides(base, **loginfo) 523 524 if base in self.overridevars: 525 self._setvar_update_overridevars(var, value) 526 return 527 528 if not var in self.dict: 529 self._makeShadowCopy(var) 530 531 if not parsing: 532 if ":append" in self.dict[var]: 533 del self.dict[var][":append"] 534 if ":prepend" in self.dict[var]: 535 del self.dict[var][":prepend"] 536 if ":remove" in self.dict[var]: 537 del self.dict[var][":remove"] 538 if var in self.overridedata: 539 active = [] 540 self.need_overrides() 541 for (r, o) in self.overridedata[var]: 542 if o in self.overridesset: 543 active.append(r) 544 elif ":" in o: 545 if set(o.split(":")).issubset(self.overridesset): 546 active.append(r) 547 for a in active: 548 self.delVar(a) 549 del self.overridedata[var] 550 551 # more cookies for the cookie monster 552 if ':' in var: 553 self._setvar_update_overrides(var, **loginfo) 554 555 # setting var 556 self.dict[var]["_content"] = value 557 self.varhistory.record(**loginfo) 558 559 if var in self.overridevars: 560 self._setvar_update_overridevars(var, value) 561 562 def _setvar_update_overridevars(self, var, value): 563 vardata = self.expandWithRefs(value, var) 564 new = vardata.references 565 new.update(vardata.contains.keys()) 566 while not new.issubset(self.overridevars): 567 nextnew = set() 568 self.overridevars.update(new) 569 for i in new: 570 vardata = self.expandWithRefs(self.getVar(i), i) 571 nextnew.update(vardata.references) 572 nextnew.update(vardata.contains.keys()) 573 new = nextnew 574 self.internal_finalize(True) 575 576 def _setvar_update_overrides(self, var, **loginfo): 577 # aka pay the cookie monster 578 override = var[var.rfind(':')+1:] 579 shortvar = var[:var.rfind(':')] 580 while override and __override_regexp__.match(override): 581 if shortvar not in self.overridedata: 582 self.overridedata[shortvar] = [] 583 if [var, override] not in self.overridedata[shortvar]: 584 # Force CoW by recreating the list first 585 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 586 self.overridedata[shortvar].append([var, override]) 587 override = None 588 if ":" in shortvar: 589 override = var[shortvar.rfind(':')+1:] 590 shortvar = var[:shortvar.rfind(':')] 591 if len(shortvar) == 0: 592 override = None 593 594 def getVar(self, var, expand=True, noweakdefault=False, parsing=False): 595 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) 596 597 def renameVar(self, key, newkey, **loginfo): 598 """ 599 Rename the variable key to newkey 600 """ 601 if key == newkey: 602 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) 603 return 604 605 val = self.getVar(key, 0, parsing=True) 606 if val is not None: 607 self.varhistory.rename_variable_hist(key, newkey) 608 loginfo['variable'] = newkey 609 loginfo['op'] = 'rename from %s' % key 610 loginfo['detail'] = val 611 self.varhistory.record(**loginfo) 612 self.setVar(newkey, val, ignore=True, parsing=True) 613 614 for i in (__setvar_keyword__): 615 src = self.getVarFlag(key, i, False) 616 if src is None: 617 continue 618 619 dest = self.getVarFlag(newkey, i, False) or [] 620 dest.extend(src) 621 self.setVarFlag(newkey, i, dest, ignore=True) 622 623 if key in self.overridedata: 624 self.overridedata[newkey] = [] 625 for (v, o) in self.overridedata[key]: 626 self.overridedata[newkey].append([v.replace(key, newkey), o]) 627 self.renameVar(v, v.replace(key, newkey)) 628 629 if ':' in newkey and val is None: 630 self._setvar_update_overrides(newkey, **loginfo) 631 632 loginfo['variable'] = key 633 loginfo['op'] = 'rename (to)' 634 loginfo['detail'] = newkey 635 self.varhistory.record(**loginfo) 636 self.delVar(key, ignore=True) 637 638 def appendVar(self, var, value, **loginfo): 639 loginfo['op'] = 'append' 640 self.varhistory.record(**loginfo) 641 self.setVar(var + ":append", value, ignore=True, parsing=True) 642 643 def prependVar(self, var, value, **loginfo): 644 loginfo['op'] = 'prepend' 645 self.varhistory.record(**loginfo) 646 self.setVar(var + ":prepend", value, ignore=True, parsing=True) 647 648 def delVar(self, var, **loginfo): 649 self.expand_cache = {} 650 651 loginfo['detail'] = "" 652 loginfo['op'] = 'del' 653 self.varhistory.record(**loginfo) 654 self.dict[var] = {} 655 if var in self.overridedata: 656 del self.overridedata[var] 657 if ':' in var: 658 override = var[var.rfind(':')+1:] 659 shortvar = var[:var.rfind(':')] 660 while override and override.islower(): 661 try: 662 if shortvar in self.overridedata: 663 # Force CoW by recreating the list first 664 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 665 self.overridedata[shortvar].remove([var, override]) 666 except ValueError as e: 667 pass 668 override = None 669 if ":" in shortvar: 670 override = var[shortvar.rfind(':')+1:] 671 shortvar = var[:shortvar.rfind(':')] 672 if len(shortvar) == 0: 673 override = None 674 675 def setVarFlag(self, var, flag, value, **loginfo): 676 self.expand_cache = {} 677 678 if 'op' not in loginfo: 679 loginfo['op'] = "set" 680 loginfo['flag'] = flag 681 self.varhistory.record(**loginfo) 682 if not var in self.dict: 683 self._makeShadowCopy(var) 684 self.dict[var][flag] = value 685 686 if flag == "_defaultval" and ':' in var: 687 self._setvar_update_overrides(var, **loginfo) 688 if flag == "_defaultval" and var in self.overridevars: 689 self._setvar_update_overridevars(var, value) 690 691 if flag == "unexport" or flag == "export": 692 if not "__exportlist" in self.dict: 693 self._makeShadowCopy("__exportlist") 694 if not "_content" in self.dict["__exportlist"]: 695 self.dict["__exportlist"]["_content"] = set() 696 self.dict["__exportlist"]["_content"].add(var) 697 698 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): 699 if flag == "_content": 700 cachename = var 701 else: 702 if not flag: 703 bb.warn("Calling getVarFlag with flag unset is invalid") 704 return None 705 cachename = var + "[" + flag + "]" 706 707 if expand and cachename in self.expand_cache: 708 return self.expand_cache[cachename].value 709 710 local_var, overridedata = self._findVar(var) 711 value = None 712 removes = set() 713 if flag == "_content" and overridedata is not None and not parsing: 714 match = False 715 active = {} 716 self.need_overrides() 717 for (r, o) in overridedata: 718 # FIXME What about double overrides both with "_" in the name? 719 if o in self.overridesset: 720 active[o] = r 721 elif ":" in o: 722 if set(o.split(":")).issubset(self.overridesset): 723 active[o] = r 724 725 mod = True 726 while mod: 727 mod = False 728 for o in self.overrides: 729 for a in active.copy(): 730 if a.endswith(":" + o): 731 t = active[a] 732 del active[a] 733 active[a.replace(":" + o, "")] = t 734 mod = True 735 elif a == o: 736 match = active[a] 737 del active[a] 738 if match: 739 value, subparser = self.getVarFlag(match, "_content", False, retparser=True) 740 if hasattr(subparser, "removes"): 741 # We have to carry the removes from the overridden variable to apply at the 742 # end of processing 743 removes = subparser.removes 744 745 if local_var is not None and value is None: 746 if flag in local_var: 747 value = copy.copy(local_var[flag]) 748 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 749 value = copy.copy(local_var["_defaultval"]) 750 751 752 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: 753 self.need_overrides() 754 for (r, o) in local_var[":append"]: 755 match = True 756 if o: 757 for o2 in o.split(":"): 758 if not o2 in self.overrides: 759 match = False 760 if match: 761 if value is None: 762 value = "" 763 value = value + r 764 765 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: 766 self.need_overrides() 767 for (r, o) in local_var[":prepend"]: 768 769 match = True 770 if o: 771 for o2 in o.split(":"): 772 if not o2 in self.overrides: 773 match = False 774 if match: 775 if value is None: 776 value = "" 777 value = r + value 778 779 parser = None 780 if expand or retparser: 781 parser = self.expandWithRefs(value, cachename) 782 if expand: 783 value = parser.value 784 785 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: 786 self.need_overrides() 787 for (r, o) in local_var[":remove"]: 788 match = True 789 if o: 790 for o2 in o.split(":"): 791 if not o2 in self.overrides: 792 match = False 793 if match: 794 removes.add(r) 795 796 if value and flag == "_content" and not parsing: 797 if removes and parser: 798 expanded_removes = {} 799 for r in removes: 800 expanded_removes[r] = self.expand(r).split() 801 802 parser.removes = set() 803 val = "" 804 for v in __whitespace_split__.split(parser.value): 805 skip = False 806 for r in removes: 807 if v in expanded_removes[r]: 808 parser.removes.add(r) 809 skip = True 810 if skip: 811 continue 812 val = val + v 813 parser.value = val 814 if expand: 815 value = parser.value 816 817 if parser: 818 self.expand_cache[cachename] = parser 819 820 if retparser: 821 return value, parser 822 823 return value 824 825 def delVarFlag(self, var, flag, **loginfo): 826 self.expand_cache = {} 827 828 local_var, _ = self._findVar(var) 829 if not local_var: 830 return 831 if not var in self.dict: 832 self._makeShadowCopy(var) 833 834 if var in self.dict and flag in self.dict[var]: 835 loginfo['detail'] = "" 836 loginfo['op'] = 'delFlag' 837 loginfo['flag'] = flag 838 self.varhistory.record(**loginfo) 839 840 del self.dict[var][flag] 841 842 def appendVarFlag(self, var, flag, value, **loginfo): 843 loginfo['op'] = 'append' 844 loginfo['flag'] = flag 845 self.varhistory.record(**loginfo) 846 newvalue = (self.getVarFlag(var, flag, False) or "") + value 847 self.setVarFlag(var, flag, newvalue, ignore=True) 848 849 def prependVarFlag(self, var, flag, value, **loginfo): 850 loginfo['op'] = 'prepend' 851 loginfo['flag'] = flag 852 self.varhistory.record(**loginfo) 853 newvalue = value + (self.getVarFlag(var, flag, False) or "") 854 self.setVarFlag(var, flag, newvalue, ignore=True) 855 856 def setVarFlags(self, var, flags, **loginfo): 857 self.expand_cache = {} 858 infer_caller_details(loginfo) 859 if not var in self.dict: 860 self._makeShadowCopy(var) 861 862 for i in flags: 863 if i == "_content": 864 continue 865 loginfo['flag'] = i 866 loginfo['detail'] = flags[i] 867 self.varhistory.record(**loginfo) 868 self.dict[var][i] = flags[i] 869 870 def getVarFlags(self, var, expand = False, internalflags=False): 871 local_var, _ = self._findVar(var) 872 flags = {} 873 874 if local_var: 875 for i in local_var: 876 if i.startswith(("_", ":")) and not internalflags: 877 continue 878 flags[i] = local_var[i] 879 if expand and i in expand: 880 flags[i] = self.expand(flags[i], var + "[" + i + "]") 881 if len(flags) == 0: 882 return None 883 return flags 884 885 886 def delVarFlags(self, var, **loginfo): 887 self.expand_cache = {} 888 if not var in self.dict: 889 self._makeShadowCopy(var) 890 891 if var in self.dict: 892 content = None 893 894 loginfo['op'] = 'delete flags' 895 self.varhistory.record(**loginfo) 896 897 # try to save the content 898 if "_content" in self.dict[var]: 899 content = self.dict[var]["_content"] 900 self.dict[var] = {} 901 self.dict[var]["_content"] = content 902 else: 903 del self.dict[var] 904 905 def createCopy(self): 906 """ 907 Create a copy of self by setting _data to self 908 """ 909 # we really want this to be a DataSmart... 910 data = DataSmart() 911 data.dict["_data"] = self.dict 912 data.varhistory = self.varhistory.copy() 913 data.varhistory.dataroot = data 914 data.inchistory = self.inchistory.copy() 915 916 data._tracking = self._tracking 917 918 data.overrides = None 919 data.overridevars = copy.copy(self.overridevars) 920 # Should really be a deepcopy but has heavy overhead. 921 # Instead, we're careful with writes. 922 data.overridedata = copy.copy(self.overridedata) 923 924 return data 925 926 def expandVarref(self, variable, parents=False): 927 """Find all references to variable in the data and expand it 928 in place, optionally descending to parent datastores.""" 929 930 if parents: 931 keys = iter(self) 932 else: 933 keys = self.localkeys() 934 935 ref = '${%s}' % variable 936 value = self.getVar(variable, False) 937 for key in keys: 938 referrervalue = self.getVar(key, False) 939 if referrervalue and ref in referrervalue: 940 self.setVar(key, referrervalue.replace(ref, value)) 941 942 def localkeys(self): 943 for key in self.dict: 944 if key not in ['_data']: 945 yield key 946 947 def __iter__(self): 948 deleted = set() 949 overrides = set() 950 def keylist(d): 951 klist = set() 952 for key in d: 953 if key in ["_data"]: 954 continue 955 if key in deleted: 956 continue 957 if key in overrides: 958 continue 959 if not d[key]: 960 deleted.add(key) 961 continue 962 klist.add(key) 963 964 if "_data" in d: 965 klist |= keylist(d["_data"]) 966 967 return klist 968 969 self.need_overrides() 970 for var in self.overridedata: 971 for (r, o) in self.overridedata[var]: 972 if o in self.overridesset: 973 overrides.add(var) 974 elif ":" in o: 975 if set(o.split(":")).issubset(self.overridesset): 976 overrides.add(var) 977 978 for k in keylist(self.dict): 979 yield k 980 981 for k in overrides: 982 yield k 983 984 def __len__(self): 985 return len(frozenset(iter(self))) 986 987 def __getitem__(self, item): 988 value = self.getVar(item, False) 989 if value is None: 990 raise KeyError(item) 991 else: 992 return value 993 994 def __setitem__(self, var, value): 995 self.setVar(var, value) 996 997 def __delitem__(self, var): 998 self.delVar(var) 999 1000 def get_hash(self): 1001 data = {} 1002 d = self.createCopy() 1003 bb.data.expandKeys(d) 1004 1005 config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split()) 1006 keys = set(key for key in iter(d) if not key.startswith("__")) 1007 for key in keys: 1008 if key in config_whitelist: 1009 continue 1010 1011 value = d.getVar(key, False) or "" 1012 if type(value) is type(self): 1013 data.update({key:value.get_hash()}) 1014 else: 1015 data.update({key:value}) 1016 1017 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) 1018 if not varflags: 1019 continue 1020 for f in varflags: 1021 if f == "_content": 1022 continue 1023 data.update({'%s[%s]' % (key, f):varflags[f]}) 1024 1025 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: 1026 bb_list = d.getVar(key, False) or [] 1027 data.update({key:str(bb_list)}) 1028 1029 if key == "__BBANONFUNCS": 1030 for i in bb_list: 1031 value = d.getVar(i, False) or "" 1032 data.update({i:value}) 1033 1034 data_str = str([(k, data[k]) for k in sorted(data.keys())]) 1035 return hashlib.sha256(data_str.encode("utf-8")).hexdigest() 1036