1""" 2BitBake Smart Dictionary Implementation 3 4Functions for interacting with the data structure used by the 5BitBake build tools. 6 7""" 8 9# Copyright (C) 2003, 2004 Chris Larson 10# Copyright (C) 2004, 2005 Seb Frankengul 11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther 12# Copyright (C) 2005 Uli Luckas 13# Copyright (C) 2005 ROAD GmbH 14# 15# SPDX-License-Identifier: GPL-2.0-only 16# 17# Based on functions from the base bb module, Copyright 2003 Holger Schurig 18 19import builtins 20import copy 21import re 22import sys 23from collections.abc import MutableMapping 24import logging 25import hashlib 26import bb, bb.codeparser 27from bb import utils 28from bb.COW import COWDictBase 29 30logger = logging.getLogger("BitBake.Data") 31 32__setvar_keyword__ = [":append", ":prepend", ":remove"] 33__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') 34__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") 35__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") 36__whitespace_split__ = re.compile(r'(\s)') 37__override_regexp__ = re.compile(r'[a-z0-9]+') 38 39bitbake_renamed_vars = { 40 "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", 41 "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", 42 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", 43 "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", 44 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", 45 "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", 46 "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", 47 "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", 48 "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", 49} 50 51def infer_caller_details(loginfo, parent = False, varval = True): 52 """Save the caller the trouble of specifying everything.""" 53 # Save effort. 54 if 'ignore' in loginfo and loginfo['ignore']: 55 return 56 # If nothing was provided, mark this as possibly unneeded. 57 if not loginfo: 58 loginfo['ignore'] = True 59 return 60 # Infer caller's likely values for variable (var) and value (value), 61 # to reduce clutter in the rest of the code. 62 above = None 63 def set_above(): 64 try: 65 raise Exception 66 except Exception: 67 tb = sys.exc_info()[2] 68 if parent: 69 return tb.tb_frame.f_back.f_back.f_back 70 else: 71 return tb.tb_frame.f_back.f_back 72 73 if varval and ('variable' not in loginfo or 'detail' not in loginfo): 74 if not above: 75 above = set_above() 76 lcls = above.f_locals.items() 77 for k, v in lcls: 78 if k == 'value' and 'detail' not in loginfo: 79 loginfo['detail'] = v 80 if k == 'var' and 'variable' not in loginfo: 81 loginfo['variable'] = v 82 # Infer file/line/function from traceback 83 # Don't use traceback.extract_stack() since it fills the line contents which 84 # we don't need and that hits stat syscalls 85 if 'file' not in loginfo: 86 if not above: 87 above = set_above() 88 f = above.f_back 89 line = f.f_lineno 90 file = f.f_code.co_filename 91 func = f.f_code.co_name 92 loginfo['file'] = file 93 loginfo['line'] = line 94 if func not in loginfo: 95 loginfo['func'] = func 96 97class VariableParse: 98 def __init__(self, varname, d, unexpanded_value = None, val = None): 99 self.varname = varname 100 self.d = d 101 self.value = val 102 self.unexpanded_value = unexpanded_value 103 104 self.references = set() 105 self.execs = set() 106 self.contains = {} 107 108 def var_sub(self, match): 109 key = match.group()[2:-1] 110 if self.varname and key: 111 if self.varname == key: 112 raise Exception("variable %s references itself!" % self.varname) 113 var = self.d.getVarFlag(key, "_content") 114 self.references.add(key) 115 if var is not None: 116 return var 117 else: 118 return match.group() 119 120 def python_sub(self, match): 121 if isinstance(match, str): 122 code = match 123 else: 124 code = match.group()[3:-1] 125 126 # Do not run code that contains one or more unexpanded variables 127 # instead return the code with the characters we removed put back 128 if __expand_var_regexp__.findall(code): 129 return "${@" + code + "}" 130 131 if self.varname: 132 varname = 'Var <%s>' % self.varname 133 else: 134 varname = '<expansion>' 135 codeobj = compile(code.strip(), varname, "eval") 136 137 parser = bb.codeparser.PythonParser(self.varname, logger) 138 parser.parse_python(code) 139 if self.varname: 140 vardeps = self.d.getVarFlag(self.varname, "vardeps") 141 if vardeps is None: 142 parser.log.flush() 143 else: 144 parser.log.flush() 145 self.references |= parser.references 146 self.execs |= parser.execs 147 148 for k in parser.contains: 149 if k not in self.contains: 150 self.contains[k] = parser.contains[k].copy() 151 else: 152 self.contains[k].update(parser.contains[k]) 153 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 154 return str(value) 155 156class DataContext(dict): 157 excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe']) 158 159 def __init__(self, metadata, **kwargs): 160 self.metadata = metadata 161 dict.__init__(self, **kwargs) 162 self['d'] = metadata 163 self.context = set(bb.utils.get_context()) 164 165 def __missing__(self, key): 166 if key in self.excluded or key in self.context: 167 raise KeyError(key) 168 169 value = self.metadata.getVar(key) 170 if value is None: 171 raise KeyError(key) 172 else: 173 return value 174 175class ExpansionError(Exception): 176 def __init__(self, varname, expression, exception): 177 self.expression = expression 178 self.variablename = varname 179 self.exception = exception 180 self.varlist = [varname or expression or ""] 181 if varname: 182 if expression: 183 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 184 else: 185 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) 186 else: 187 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 188 Exception.__init__(self, self.msg) 189 self.args = (varname, expression, exception) 190 191 def addVar(self, varname): 192 if varname: 193 self.varlist.append(varname) 194 195 def __str__(self): 196 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) 197 return self.msg + chain 198 199class IncludeHistory(object): 200 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 201 self.parent = parent 202 self.filename = filename 203 self.children = [] 204 self.current = self 205 206 def copy(self): 207 new = IncludeHistory(self.parent, self.filename) 208 for c in self.children: 209 new.children.append(c) 210 return new 211 212 def include(self, filename): 213 newfile = IncludeHistory(self.current, filename) 214 self.current.children.append(newfile) 215 self.current = newfile 216 return self 217 218 def __enter__(self): 219 pass 220 221 def __exit__(self, a, b, c): 222 if self.current.parent: 223 self.current = self.current.parent 224 else: 225 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) 226 return False 227 228 def emit(self, o, level = 0): 229 """Emit an include history file, and its children.""" 230 if level: 231 spaces = " " * (level - 1) 232 o.write("# %s%s" % (spaces, self.filename)) 233 if len(self.children) > 0: 234 o.write(" includes:") 235 else: 236 o.write("#\n# INCLUDE HISTORY:\n#") 237 level = level + 1 238 for child in self.children: 239 o.write("\n") 240 child.emit(o, level) 241 242class VariableHistory(object): 243 def __init__(self, dataroot): 244 self.dataroot = dataroot 245 self.variables = COWDictBase.copy() 246 247 def copy(self): 248 new = VariableHistory(self.dataroot) 249 new.variables = self.variables.copy() 250 return new 251 252 def __getstate__(self): 253 vardict = {} 254 for k, v in self.variables.iteritems(): 255 vardict[k] = v 256 return {'dataroot': self.dataroot, 257 'variables': vardict} 258 259 def __setstate__(self, state): 260 self.dataroot = state['dataroot'] 261 self.variables = COWDictBase.copy() 262 for k, v in state['variables'].items(): 263 self.variables[k] = v 264 265 def record(self, *kwonly, **loginfo): 266 if not self.dataroot._tracking: 267 return 268 if len(kwonly) > 0: 269 raise TypeError 270 infer_caller_details(loginfo, parent = True) 271 if 'ignore' in loginfo and loginfo['ignore']: 272 return 273 if 'op' not in loginfo or not loginfo['op']: 274 loginfo['op'] = 'set' 275 if 'variable' not in loginfo or 'file' not in loginfo: 276 raise ValueError("record() missing variable or file.") 277 var = loginfo['variable'] 278 if var not in self.variables: 279 self.variables[var] = [] 280 if not isinstance(self.variables[var], list): 281 return 282 if 'nodups' in loginfo and loginfo in self.variables[var]: 283 return 284 self.variables[var].append(loginfo.copy()) 285 286 def rename_variable_hist(self, oldvar, newvar): 287 if not self.dataroot._tracking: 288 return 289 if oldvar not in self.variables: 290 return 291 if newvar not in self.variables: 292 self.variables[newvar] = [] 293 for i in self.variables[oldvar]: 294 self.variables[newvar].append(i.copy()) 295 296 def variable(self, var): 297 varhistory = [] 298 if var in self.variables: 299 varhistory.extend(self.variables[var]) 300 return varhistory 301 302 def emit(self, var, oval, val, o, d): 303 history = self.variable(var) 304 305 # Append override history 306 if var in d.overridedata: 307 for (r, override) in d.overridedata[var]: 308 for event in self.variable(r): 309 loginfo = event.copy() 310 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): 311 continue 312 loginfo['variable'] = var 313 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 314 history.append(loginfo) 315 316 commentVal = re.sub('\n', '\n#', str(oval)) 317 if history: 318 if len(history) == 1: 319 o.write("#\n# $%s\n" % var) 320 else: 321 o.write("#\n# $%s [%d operations]\n" % (var, len(history))) 322 for event in history: 323 # o.write("# %s\n" % str(event)) 324 if 'func' in event: 325 # If we have a function listed, this is internal 326 # code, not an operation in a config file, and the 327 # full path is distracting. 328 event['file'] = re.sub('.*/', '', event['file']) 329 display_func = ' [%s]' % event['func'] 330 else: 331 display_func = '' 332 if 'flag' in event: 333 flag = '[%s] ' % (event['flag']) 334 else: 335 flag = '' 336 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \ 337 (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', str(event['detail'])))) 338 if len(history) > 1: 339 o.write("# pre-expansion value:\n") 340 o.write('# "%s"\n' % (commentVal)) 341 else: 342 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) 343 o.write('# "%s"\n' % (commentVal)) 344 345 def get_variable_files(self, var): 346 """Get the files where operations are made on a variable""" 347 var_history = self.variable(var) 348 files = [] 349 for event in var_history: 350 files.append(event['file']) 351 return files 352 353 def get_variable_lines(self, var, f): 354 """Get the line where a operation is made on a variable in file f""" 355 var_history = self.variable(var) 356 lines = [] 357 for event in var_history: 358 if f== event['file']: 359 line = event['line'] 360 lines.append(line) 361 return lines 362 363 def get_variable_refs(self, var): 364 """Return a dict of file/line references""" 365 var_history = self.variable(var) 366 refs = {} 367 for event in var_history: 368 if event['file'] not in refs: 369 refs[event['file']] = [] 370 refs[event['file']].append(event['line']) 371 return refs 372 373 def get_variable_items_files(self, var): 374 """ 375 Use variable history to map items added to a list variable and 376 the files in which they were added. 377 """ 378 d = self.dataroot 379 history = self.variable(var) 380 finalitems = (d.getVar(var) or '').split() 381 filemap = {} 382 isset = False 383 for event in history: 384 if 'flag' in event: 385 continue 386 if event['op'] == ':remove': 387 continue 388 if isset and event['op'] == 'set?': 389 continue 390 isset = True 391 items = d.expand(str(event['detail'])).split() 392 for item in items: 393 # This is a little crude but is belt-and-braces to avoid us 394 # having to handle every possible operation type specifically 395 if item in finalitems and not item in filemap: 396 filemap[item] = event['file'] 397 return filemap 398 399 def del_var_history(self, var, f=None, line=None): 400 """If file f and line are not given, the entire history of var is deleted""" 401 if var in self.variables: 402 if f and line: 403 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] 404 else: 405 self.variables[var] = [] 406 407def _print_rename_error(var, loginfo, renamedvars, fullvar=None): 408 info = "" 409 if "file" in loginfo: 410 info = " file: %s" % loginfo["file"] 411 if "line" in loginfo: 412 info += " line: %s" % loginfo["line"] 413 if fullvar and fullvar != var: 414 info += " referenced as: %s" % fullvar 415 if info: 416 info = " (%s)" % info.strip() 417 renameinfo = renamedvars[var] 418 if " " in renameinfo: 419 # A space signals a string to display instead of a rename 420 bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) 421 else: 422 bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) 423 424class DataSmart(MutableMapping): 425 def __init__(self): 426 self.dict = {} 427 428 self.inchistory = IncludeHistory() 429 self.varhistory = VariableHistory(self) 430 self._tracking = False 431 self._var_renames = {} 432 self._var_renames.update(bitbake_renamed_vars) 433 434 self.expand_cache = {} 435 436 # cookie monster tribute 437 # Need to be careful about writes to overridedata as 438 # its only a shallow copy, could influence other data store 439 # copies! 440 self.overridedata = {} 441 self.overrides = None 442 self.overridevars = set(["OVERRIDES", "FILE"]) 443 self.inoverride = False 444 445 def enableTracking(self): 446 self._tracking = True 447 448 def disableTracking(self): 449 self._tracking = False 450 451 def expandWithRefs(self, s, varname): 452 453 if not isinstance(s, str): # sanity check 454 return VariableParse(varname, self, s, s) 455 456 varparse = VariableParse(varname, self, s) 457 458 while s.find('${') != -1: 459 olds = s 460 try: 461 s = __expand_var_regexp__.sub(varparse.var_sub, s) 462 try: 463 s = __expand_python_regexp__.sub(varparse.python_sub, s) 464 except SyntaxError as e: 465 # Likely unmatched brackets, just don't expand the expression 466 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): 467 raise 468 if s == olds: 469 break 470 except ExpansionError as e: 471 e.addVar(varname) 472 raise 473 except bb.parse.SkipRecipe: 474 raise 475 except bb.BBHandledException: 476 raise 477 except Exception as exc: 478 tb = sys.exc_info()[2] 479 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 480 481 varparse.value = s 482 483 return varparse 484 485 def expand(self, s, varname = None): 486 return self.expandWithRefs(s, varname).value 487 488 def need_overrides(self): 489 if self.overrides is not None: 490 return 491 if self.inoverride: 492 return 493 overrride_stack = [] 494 for count in range(5): 495 self.inoverride = True 496 # Can end up here recursively so setup dummy values 497 self.overrides = [] 498 self.overridesset = set() 499 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 500 overrride_stack.append(self.overrides) 501 self.overridesset = set(self.overrides) 502 self.inoverride = False 503 self.expand_cache = {} 504 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or [] 505 if newoverrides == self.overrides: 506 break 507 self.overrides = newoverrides 508 self.overridesset = set(self.overrides) 509 else: 510 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) 511 512 def initVar(self, var): 513 self.expand_cache = {} 514 if not var in self.dict: 515 self.dict[var] = {} 516 517 def _findVar(self, var): 518 dest = self.dict 519 while dest: 520 if var in dest: 521 return dest[var] 522 523 if "_data" not in dest: 524 break 525 dest = dest["_data"] 526 return None 527 528 def _makeShadowCopy(self, var): 529 if var in self.dict: 530 return 531 532 local_var = self._findVar(var) 533 534 if local_var: 535 self.dict[var] = copy.copy(local_var) 536 else: 537 self.initVar(var) 538 539 def hasOverrides(self, var): 540 return var in self.overridedata 541 542 def setVar(self, var, value, **loginfo): 543 #print("var=" + str(var) + " val=" + str(value)) 544 545 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): 546 info = "%s" % var 547 if "file" in loginfo: 548 info += " file: %s" % loginfo["file"] 549 if "line" in loginfo: 550 info += " line: %s" % loginfo["line"] 551 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) 552 553 shortvar = var.split(":", 1)[0] 554 if shortvar in self._var_renames: 555 _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) 556 # Mark that we have seen a renamed variable 557 self.setVar("_FAILPARSINGERRORHANDLED", True) 558 559 self.expand_cache = {} 560 parsing=False 561 if 'parsing' in loginfo: 562 parsing=True 563 564 if 'op' not in loginfo: 565 loginfo['op'] = "set" 566 567 match = __setvar_regexp__.match(var) 568 if match and match.group("keyword") in __setvar_keyword__: 569 base = match.group('base') 570 keyword = match.group("keyword") 571 override = match.group('add') 572 l = self.getVarFlag(base, keyword, False) or [] 573 l.append([value, override]) 574 self.setVarFlag(base, keyword, l, ignore=True) 575 # And cause that to be recorded: 576 loginfo['detail'] = value 577 loginfo['variable'] = base 578 if override: 579 loginfo['op'] = '%s[%s]' % (keyword, override) 580 else: 581 loginfo['op'] = keyword 582 self.varhistory.record(**loginfo) 583 # todo make sure keyword is not __doc__ or __module__ 584 # pay the cookie monster 585 586 # more cookies for the cookie monster 587 if ':' in var: 588 self._setvar_update_overrides(base, **loginfo) 589 590 if base in self.overridevars: 591 self._setvar_update_overridevars(var, value) 592 return 593 594 if not var in self.dict: 595 self._makeShadowCopy(var) 596 597 if not parsing: 598 if ":append" in self.dict[var]: 599 del self.dict[var][":append"] 600 if ":prepend" in self.dict[var]: 601 del self.dict[var][":prepend"] 602 if ":remove" in self.dict[var]: 603 del self.dict[var][":remove"] 604 if var in self.overridedata: 605 active = [] 606 self.need_overrides() 607 for (r, o) in self.overridedata[var]: 608 if o in self.overridesset: 609 active.append(r) 610 elif ":" in o: 611 if set(o.split(":")).issubset(self.overridesset): 612 active.append(r) 613 for a in active: 614 self.delVar(a) 615 del self.overridedata[var] 616 617 # more cookies for the cookie monster 618 if ':' in var: 619 self._setvar_update_overrides(var, **loginfo) 620 621 # setting var 622 self.dict[var]["_content"] = value 623 self.varhistory.record(**loginfo) 624 625 if var in self.overridevars: 626 self._setvar_update_overridevars(var, value) 627 628 def _setvar_update_overridevars(self, var, value): 629 vardata = self.expandWithRefs(value, var) 630 new = vardata.references 631 new.update(vardata.contains.keys()) 632 while not new.issubset(self.overridevars): 633 nextnew = set() 634 self.overridevars.update(new) 635 for i in new: 636 vardata = self.expandWithRefs(self.getVar(i), i) 637 nextnew.update(vardata.references) 638 nextnew.update(vardata.contains.keys()) 639 new = nextnew 640 self.overrides = None 641 642 def _setvar_update_overrides(self, var, **loginfo): 643 # aka pay the cookie monster 644 override = var[var.rfind(':')+1:] 645 shortvar = var[:var.rfind(':')] 646 while override and __override_regexp__.match(override): 647 if shortvar not in self.overridedata: 648 self.overridedata[shortvar] = [] 649 if [var, override] not in self.overridedata[shortvar]: 650 # Force CoW by recreating the list first 651 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 652 self.overridedata[shortvar].append([var, override]) 653 override = None 654 if ":" in shortvar: 655 override = var[shortvar.rfind(':')+1:] 656 shortvar = var[:shortvar.rfind(':')] 657 if len(shortvar) == 0: 658 override = None 659 660 def getVar(self, var, expand=True, noweakdefault=False, parsing=False): 661 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) 662 663 def renameVar(self, key, newkey, **loginfo): 664 """ 665 Rename the variable key to newkey 666 """ 667 if key == newkey: 668 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) 669 return 670 671 val = self.getVar(key, 0, parsing=True) 672 if val is not None: 673 self.varhistory.rename_variable_hist(key, newkey) 674 loginfo['variable'] = newkey 675 loginfo['op'] = 'rename from %s' % key 676 loginfo['detail'] = val 677 self.varhistory.record(**loginfo) 678 self.setVar(newkey, val, ignore=True, parsing=True) 679 680 srcflags = self.getVarFlags(key, False, True) or {} 681 for i in srcflags: 682 if i not in (__setvar_keyword__): 683 continue 684 src = srcflags[i] 685 686 dest = self.getVarFlag(newkey, i, False) or [] 687 dest.extend(src) 688 self.setVarFlag(newkey, i, dest, ignore=True) 689 690 if key in self.overridedata: 691 self.overridedata[newkey] = [] 692 for (v, o) in self.overridedata[key]: 693 self.overridedata[newkey].append([v.replace(key, newkey), o]) 694 self.renameVar(v, v.replace(key, newkey)) 695 696 if ':' in newkey and val is None: 697 self._setvar_update_overrides(newkey, **loginfo) 698 699 loginfo['variable'] = key 700 loginfo['op'] = 'rename (to)' 701 loginfo['detail'] = newkey 702 self.varhistory.record(**loginfo) 703 self.delVar(key, ignore=True) 704 705 def appendVar(self, var, value, **loginfo): 706 loginfo['op'] = 'append' 707 self.varhistory.record(**loginfo) 708 self.setVar(var + ":append", value, ignore=True, parsing=True) 709 710 def prependVar(self, var, value, **loginfo): 711 loginfo['op'] = 'prepend' 712 self.varhistory.record(**loginfo) 713 self.setVar(var + ":prepend", value, ignore=True, parsing=True) 714 715 def delVar(self, var, **loginfo): 716 self.expand_cache = {} 717 718 loginfo['detail'] = "" 719 loginfo['op'] = 'del' 720 self.varhistory.record(**loginfo) 721 self.dict[var] = {} 722 if var in self.overridedata: 723 del self.overridedata[var] 724 if ':' in var: 725 override = var[var.rfind(':')+1:] 726 shortvar = var[:var.rfind(':')] 727 while override and __override_regexp__.match(override): 728 try: 729 if shortvar in self.overridedata: 730 # Force CoW by recreating the list first 731 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 732 self.overridedata[shortvar].remove([var, override]) 733 except ValueError as e: 734 pass 735 override = None 736 if ":" in shortvar: 737 override = var[shortvar.rfind(':')+1:] 738 shortvar = var[:shortvar.rfind(':')] 739 if len(shortvar) == 0: 740 override = None 741 742 def setVarFlag(self, var, flag, value, **loginfo): 743 self.expand_cache = {} 744 745 if var == "BB_RENAMED_VARIABLES": 746 self._var_renames[flag] = value 747 748 if var in self._var_renames: 749 _print_rename_error(var, loginfo, self._var_renames) 750 # Mark that we have seen a renamed variable 751 self.setVar("_FAILPARSINGERRORHANDLED", True) 752 753 if 'op' not in loginfo: 754 loginfo['op'] = "set" 755 loginfo['flag'] = flag 756 self.varhistory.record(**loginfo) 757 if not var in self.dict: 758 self._makeShadowCopy(var) 759 self.dict[var][flag] = value 760 761 if flag == "_defaultval" and ':' in var: 762 self._setvar_update_overrides(var, **loginfo) 763 if flag == "_defaultval" and var in self.overridevars: 764 self._setvar_update_overridevars(var, value) 765 766 if flag == "unexport" or flag == "export": 767 if not "__exportlist" in self.dict: 768 self._makeShadowCopy("__exportlist") 769 if not "_content" in self.dict["__exportlist"]: 770 self.dict["__exportlist"]["_content"] = set() 771 self.dict["__exportlist"]["_content"].add(var) 772 773 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): 774 if flag == "_content": 775 cachename = var 776 else: 777 if not flag: 778 bb.warn("Calling getVarFlag with flag unset is invalid") 779 return None 780 cachename = var + "[" + flag + "]" 781 782 if not expand and retparser and cachename in self.expand_cache: 783 return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] 784 785 if expand and cachename in self.expand_cache: 786 return self.expand_cache[cachename].value 787 788 local_var = self._findVar(var) 789 value = None 790 removes = set() 791 if flag == "_content" and not parsing: 792 overridedata = self.overridedata.get(var, None) 793 if flag == "_content" and not parsing and overridedata is not None: 794 match = False 795 active = {} 796 self.need_overrides() 797 for (r, o) in overridedata: 798 # FIXME What about double overrides both with "_" in the name? 799 if o in self.overridesset: 800 active[o] = r 801 elif ":" in o: 802 if set(o.split(":")).issubset(self.overridesset): 803 active[o] = r 804 805 mod = True 806 while mod: 807 mod = False 808 for o in self.overrides: 809 for a in active.copy(): 810 if a.endswith(":" + o): 811 t = active[a] 812 del active[a] 813 active[a.replace(":" + o, "")] = t 814 mod = True 815 elif a == o: 816 match = active[a] 817 del active[a] 818 if match: 819 value, subparser = self.getVarFlag(match, "_content", False, retparser=True) 820 if hasattr(subparser, "removes"): 821 # We have to carry the removes from the overridden variable to apply at the 822 # end of processing 823 removes = subparser.removes 824 825 if local_var is not None and value is None: 826 if flag in local_var: 827 value = copy.copy(local_var[flag]) 828 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 829 value = copy.copy(local_var["_defaultval"]) 830 831 832 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: 833 self.need_overrides() 834 for (r, o) in local_var[":append"]: 835 match = True 836 if o: 837 for o2 in o.split(":"): 838 if not o2 in self.overrides: 839 match = False 840 if match: 841 if value is None: 842 value = "" 843 value = value + r 844 845 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: 846 self.need_overrides() 847 for (r, o) in local_var[":prepend"]: 848 849 match = True 850 if o: 851 for o2 in o.split(":"): 852 if not o2 in self.overrides: 853 match = False 854 if match: 855 if value is None: 856 value = "" 857 value = r + value 858 859 parser = None 860 if expand or retparser: 861 parser = self.expandWithRefs(value, cachename) 862 if expand: 863 value = parser.value 864 865 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: 866 self.need_overrides() 867 for (r, o) in local_var[":remove"]: 868 match = True 869 if o: 870 for o2 in o.split(":"): 871 if not o2 in self.overrides: 872 match = False 873 if match: 874 removes.add(r) 875 876 if value and flag == "_content" and not parsing: 877 if removes and parser: 878 expanded_removes = {} 879 for r in removes: 880 expanded_removes[r] = self.expand(r).split() 881 882 parser.removes = set() 883 val = [] 884 for v in __whitespace_split__.split(parser.value): 885 skip = False 886 for r in removes: 887 if v in expanded_removes[r]: 888 parser.removes.add(r) 889 skip = True 890 if skip: 891 continue 892 val.append(v) 893 parser.value = "".join(val) 894 if expand: 895 value = parser.value 896 897 if parser: 898 self.expand_cache[cachename] = parser 899 900 if retparser: 901 return value, parser 902 903 return value 904 905 def delVarFlag(self, var, flag, **loginfo): 906 self.expand_cache = {} 907 908 local_var = self._findVar(var) 909 if not local_var: 910 return 911 if not var in self.dict: 912 self._makeShadowCopy(var) 913 914 if var in self.dict and flag in self.dict[var]: 915 loginfo['detail'] = "" 916 loginfo['op'] = 'delFlag' 917 loginfo['flag'] = flag 918 self.varhistory.record(**loginfo) 919 920 del self.dict[var][flag] 921 922 def appendVarFlag(self, var, flag, value, **loginfo): 923 loginfo['op'] = 'append' 924 loginfo['flag'] = flag 925 self.varhistory.record(**loginfo) 926 newvalue = (self.getVarFlag(var, flag, False) or "") + value 927 self.setVarFlag(var, flag, newvalue, ignore=True) 928 929 def prependVarFlag(self, var, flag, value, **loginfo): 930 loginfo['op'] = 'prepend' 931 loginfo['flag'] = flag 932 self.varhistory.record(**loginfo) 933 newvalue = value + (self.getVarFlag(var, flag, False) or "") 934 self.setVarFlag(var, flag, newvalue, ignore=True) 935 936 def setVarFlags(self, var, flags, **loginfo): 937 self.expand_cache = {} 938 infer_caller_details(loginfo) 939 if not var in self.dict: 940 self._makeShadowCopy(var) 941 942 for i in flags: 943 if i == "_content": 944 continue 945 loginfo['flag'] = i 946 loginfo['detail'] = flags[i] 947 self.varhistory.record(**loginfo) 948 self.dict[var][i] = flags[i] 949 950 def getVarFlags(self, var, expand = False, internalflags=False): 951 local_var = self._findVar(var) 952 flags = {} 953 954 if local_var: 955 for i in local_var: 956 if i.startswith(("_", ":")) and not internalflags: 957 continue 958 flags[i] = local_var[i] 959 if expand and i in expand: 960 flags[i] = self.expand(flags[i], var + "[" + i + "]") 961 if len(flags) == 0: 962 return None 963 return flags 964 965 966 def delVarFlags(self, var, **loginfo): 967 self.expand_cache = {} 968 if not var in self.dict: 969 self._makeShadowCopy(var) 970 971 if var in self.dict: 972 content = None 973 974 loginfo['op'] = 'delete flags' 975 self.varhistory.record(**loginfo) 976 977 # try to save the content 978 if "_content" in self.dict[var]: 979 content = self.dict[var]["_content"] 980 self.dict[var] = {} 981 self.dict[var]["_content"] = content 982 else: 983 del self.dict[var] 984 985 def createCopy(self): 986 """ 987 Create a copy of self by setting _data to self 988 """ 989 # we really want this to be a DataSmart... 990 data = DataSmart() 991 data.dict["_data"] = self.dict 992 data.varhistory = self.varhistory.copy() 993 data.varhistory.dataroot = data 994 data.inchistory = self.inchistory.copy() 995 996 data._tracking = self._tracking 997 data._var_renames = self._var_renames 998 999 data.overrides = None 1000 data.overridevars = copy.copy(self.overridevars) 1001 # Should really be a deepcopy but has heavy overhead. 1002 # Instead, we're careful with writes. 1003 data.overridedata = copy.copy(self.overridedata) 1004 1005 return data 1006 1007 def expandVarref(self, variable, parents=False): 1008 """Find all references to variable in the data and expand it 1009 in place, optionally descending to parent datastores.""" 1010 1011 if parents: 1012 keys = iter(self) 1013 else: 1014 keys = self.localkeys() 1015 1016 ref = '${%s}' % variable 1017 value = self.getVar(variable, False) 1018 for key in keys: 1019 referrervalue = self.getVar(key, False) 1020 if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: 1021 self.setVar(key, referrervalue.replace(ref, value)) 1022 1023 def localkeys(self): 1024 for key in self.dict: 1025 if key not in ['_data']: 1026 yield key 1027 1028 def __iter__(self): 1029 deleted = set() 1030 overrides = set() 1031 def keylist(d): 1032 klist = set() 1033 for key in d: 1034 if key in ["_data"]: 1035 continue 1036 if key in deleted: 1037 continue 1038 if key in overrides: 1039 continue 1040 if not d[key]: 1041 deleted.add(key) 1042 continue 1043 klist.add(key) 1044 1045 if "_data" in d: 1046 klist |= keylist(d["_data"]) 1047 1048 return klist 1049 1050 self.need_overrides() 1051 for var in self.overridedata: 1052 for (r, o) in self.overridedata[var]: 1053 if o in self.overridesset: 1054 overrides.add(var) 1055 elif ":" in o: 1056 if set(o.split(":")).issubset(self.overridesset): 1057 overrides.add(var) 1058 1059 for k in keylist(self.dict): 1060 yield k 1061 1062 for k in overrides: 1063 yield k 1064 1065 def __len__(self): 1066 return len(frozenset(iter(self))) 1067 1068 def __getitem__(self, item): 1069 value = self.getVar(item, False) 1070 if value is None: 1071 raise KeyError(item) 1072 else: 1073 return value 1074 1075 def __setitem__(self, var, value): 1076 self.setVar(var, value) 1077 1078 def __delitem__(self, var): 1079 self.delVar(var) 1080 1081 def get_hash(self): 1082 data = {} 1083 d = self.createCopy() 1084 bb.data.expandKeys(d) 1085 1086 config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) 1087 keys = set(key for key in iter(d) if not key.startswith("__")) 1088 for key in keys: 1089 if key in config_ignore_vars: 1090 continue 1091 1092 value = d.getVar(key, False) or "" 1093 if type(value) is type(self): 1094 data.update({key:value.get_hash()}) 1095 else: 1096 data.update({key:value}) 1097 1098 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) 1099 if not varflags: 1100 continue 1101 for f in varflags: 1102 if f == "_content": 1103 continue 1104 data.update({'%s[%s]' % (key, f):varflags[f]}) 1105 1106 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: 1107 bb_list = d.getVar(key, False) or [] 1108 data.update({key:str(bb_list)}) 1109 1110 if key == "__BBANONFUNCS": 1111 for i in bb_list: 1112 value = d.getVar(i, False) or "" 1113 data.update({i:value}) 1114 1115 data_str = str([(k, data[k]) for k in sorted(data.keys())]) 1116 return hashlib.sha256(data_str.encode("utf-8")).hexdigest() 1117