1""" 2BitBake Smart Dictionary Implementation 3 4Functions for interacting with the data structure used by the 5BitBake build tools. 6 7""" 8 9# Copyright (C) 2003, 2004 Chris Larson 10# Copyright (C) 2004, 2005 Seb Frankengul 11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther 12# Copyright (C) 2005 Uli Luckas 13# Copyright (C) 2005 ROAD GmbH 14# 15# SPDX-License-Identifier: GPL-2.0-only 16# 17# Based on functions from the base bb module, Copyright 2003 Holger Schurig 18 19import copy, re, sys, traceback 20from collections.abc import MutableMapping 21import logging 22import hashlib 23import bb, bb.codeparser 24from bb import utils 25from bb.COW import COWDictBase 26 27logger = logging.getLogger("BitBake.Data") 28 29__setvar_keyword__ = [":append", ":prepend", ":remove"] 30__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') 31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") 32__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") 33__whitespace_split__ = re.compile(r'(\s)') 34__override_regexp__ = re.compile(r'[a-z0-9]+') 35 36bitbake_renamed_vars = { 37 "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", 38 "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", 39 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", 40 "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", 41 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", 42 "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", 43 "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", 44 "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", 45 "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", 46} 47 48def infer_caller_details(loginfo, parent = False, varval = True): 49 """Save the caller the trouble of specifying everything.""" 50 # Save effort. 51 if 'ignore' in loginfo and loginfo['ignore']: 52 return 53 # If nothing was provided, mark this as possibly unneeded. 54 if not loginfo: 55 loginfo['ignore'] = True 56 return 57 # Infer caller's likely values for variable (var) and value (value), 58 # to reduce clutter in the rest of the code. 59 above = None 60 def set_above(): 61 try: 62 raise Exception 63 except Exception: 64 tb = sys.exc_info()[2] 65 if parent: 66 return tb.tb_frame.f_back.f_back.f_back 67 else: 68 return tb.tb_frame.f_back.f_back 69 70 if varval and ('variable' not in loginfo or 'detail' not in loginfo): 71 if not above: 72 above = set_above() 73 lcls = above.f_locals.items() 74 for k, v in lcls: 75 if k == 'value' and 'detail' not in loginfo: 76 loginfo['detail'] = v 77 if k == 'var' and 'variable' not in loginfo: 78 loginfo['variable'] = v 79 # Infer file/line/function from traceback 80 # Don't use traceback.extract_stack() since it fills the line contents which 81 # we don't need and that hits stat syscalls 82 if 'file' not in loginfo: 83 if not above: 84 above = set_above() 85 f = above.f_back 86 line = f.f_lineno 87 file = f.f_code.co_filename 88 func = f.f_code.co_name 89 loginfo['file'] = file 90 loginfo['line'] = line 91 if func not in loginfo: 92 loginfo['func'] = func 93 94class VariableParse: 95 def __init__(self, varname, d, unexpanded_value = None, val = None): 96 self.varname = varname 97 self.d = d 98 self.value = val 99 self.unexpanded_value = unexpanded_value 100 101 self.references = set() 102 self.execs = set() 103 self.contains = {} 104 105 def var_sub(self, match): 106 key = match.group()[2:-1] 107 if self.varname and key: 108 if self.varname == key: 109 raise Exception("variable %s references itself!" % self.varname) 110 var = self.d.getVarFlag(key, "_content") 111 self.references.add(key) 112 if var is not None: 113 return var 114 else: 115 return match.group() 116 117 def python_sub(self, match): 118 if isinstance(match, str): 119 code = match 120 else: 121 code = match.group()[3:-1] 122 123 # Do not run code that contains one or more unexpanded variables 124 # instead return the code with the characters we removed put back 125 if __expand_var_regexp__.findall(code): 126 return "${@" + code + "}" 127 128 if self.varname: 129 varname = 'Var <%s>' % self.varname 130 else: 131 varname = '<expansion>' 132 codeobj = compile(code.strip(), varname, "eval") 133 134 parser = bb.codeparser.PythonParser(self.varname, logger) 135 parser.parse_python(code) 136 if self.varname: 137 vardeps = self.d.getVarFlag(self.varname, "vardeps") 138 if vardeps is None: 139 parser.log.flush() 140 else: 141 parser.log.flush() 142 self.references |= parser.references 143 self.execs |= parser.execs 144 145 for k in parser.contains: 146 if k not in self.contains: 147 self.contains[k] = parser.contains[k].copy() 148 else: 149 self.contains[k].update(parser.contains[k]) 150 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 151 return str(value) 152 153 154class DataContext(dict): 155 def __init__(self, metadata, **kwargs): 156 self.metadata = metadata 157 dict.__init__(self, **kwargs) 158 self['d'] = metadata 159 160 def __missing__(self, key): 161 # Skip commonly accessed invalid variables 162 if key in ['bb', 'oe', 'int', 'bool', 'time', 'str', 'os']: 163 raise KeyError(key) 164 value = self.metadata.getVar(key) 165 if value is None or self.metadata.getVarFlag(key, 'func', False): 166 raise KeyError(key) 167 else: 168 return value 169 170class ExpansionError(Exception): 171 def __init__(self, varname, expression, exception): 172 self.expression = expression 173 self.variablename = varname 174 self.exception = exception 175 self.varlist = [varname or expression or ""] 176 if varname: 177 if expression: 178 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 179 else: 180 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) 181 else: 182 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 183 Exception.__init__(self, self.msg) 184 self.args = (varname, expression, exception) 185 186 def addVar(self, varname): 187 if varname: 188 self.varlist.append(varname) 189 190 def __str__(self): 191 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) 192 return self.msg + chain 193 194class IncludeHistory(object): 195 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 196 self.parent = parent 197 self.filename = filename 198 self.children = [] 199 self.current = self 200 201 def copy(self): 202 new = IncludeHistory(self.parent, self.filename) 203 for c in self.children: 204 new.children.append(c) 205 return new 206 207 def include(self, filename): 208 newfile = IncludeHistory(self.current, filename) 209 self.current.children.append(newfile) 210 self.current = newfile 211 return self 212 213 def __enter__(self): 214 pass 215 216 def __exit__(self, a, b, c): 217 if self.current.parent: 218 self.current = self.current.parent 219 else: 220 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) 221 return False 222 223 def emit(self, o, level = 0): 224 """Emit an include history file, and its children.""" 225 if level: 226 spaces = " " * (level - 1) 227 o.write("# %s%s" % (spaces, self.filename)) 228 if len(self.children) > 0: 229 o.write(" includes:") 230 else: 231 o.write("#\n# INCLUDE HISTORY:\n#") 232 level = level + 1 233 for child in self.children: 234 o.write("\n") 235 child.emit(o, level) 236 237class VariableHistory(object): 238 def __init__(self, dataroot): 239 self.dataroot = dataroot 240 self.variables = COWDictBase.copy() 241 242 def copy(self): 243 new = VariableHistory(self.dataroot) 244 new.variables = self.variables.copy() 245 return new 246 247 def __getstate__(self): 248 vardict = {} 249 for k, v in self.variables.iteritems(): 250 vardict[k] = v 251 return {'dataroot': self.dataroot, 252 'variables': vardict} 253 254 def __setstate__(self, state): 255 self.dataroot = state['dataroot'] 256 self.variables = COWDictBase.copy() 257 for k, v in state['variables'].items(): 258 self.variables[k] = v 259 260 def record(self, *kwonly, **loginfo): 261 if not self.dataroot._tracking: 262 return 263 if len(kwonly) > 0: 264 raise TypeError 265 infer_caller_details(loginfo, parent = True) 266 if 'ignore' in loginfo and loginfo['ignore']: 267 return 268 if 'op' not in loginfo or not loginfo['op']: 269 loginfo['op'] = 'set' 270 if 'detail' in loginfo: 271 loginfo['detail'] = str(loginfo['detail']) 272 if 'variable' not in loginfo or 'file' not in loginfo: 273 raise ValueError("record() missing variable or file.") 274 var = loginfo['variable'] 275 276 if var not in self.variables: 277 self.variables[var] = [] 278 if not isinstance(self.variables[var], list): 279 return 280 if 'nodups' in loginfo and loginfo in self.variables[var]: 281 return 282 self.variables[var].append(loginfo.copy()) 283 284 def rename_variable_hist(self, oldvar, newvar): 285 if not self.dataroot._tracking: 286 return 287 if oldvar not in self.variables: 288 return 289 if newvar not in self.variables: 290 self.variables[newvar] = [] 291 for i in self.variables[oldvar]: 292 self.variables[newvar].append(i.copy()) 293 294 def variable(self, var): 295 varhistory = [] 296 if var in self.variables: 297 varhistory.extend(self.variables[var]) 298 return varhistory 299 300 def emit(self, var, oval, val, o, d): 301 history = self.variable(var) 302 303 # Append override history 304 if var in d.overridedata: 305 for (r, override) in d.overridedata[var]: 306 for event in self.variable(r): 307 loginfo = event.copy() 308 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): 309 continue 310 loginfo['variable'] = var 311 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 312 history.append(loginfo) 313 314 commentVal = re.sub('\n', '\n#', str(oval)) 315 if history: 316 if len(history) == 1: 317 o.write("#\n# $%s\n" % var) 318 else: 319 o.write("#\n# $%s [%d operations]\n" % (var, len(history))) 320 for event in history: 321 # o.write("# %s\n" % str(event)) 322 if 'func' in event: 323 # If we have a function listed, this is internal 324 # code, not an operation in a config file, and the 325 # full path is distracting. 326 event['file'] = re.sub('.*/', '', event['file']) 327 display_func = ' [%s]' % event['func'] 328 else: 329 display_func = '' 330 if 'flag' in event: 331 flag = '[%s] ' % (event['flag']) 332 else: 333 flag = '' 334 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) 335 if len(history) > 1: 336 o.write("# pre-expansion value:\n") 337 o.write('# "%s"\n' % (commentVal)) 338 else: 339 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) 340 o.write('# "%s"\n' % (commentVal)) 341 342 def get_variable_files(self, var): 343 """Get the files where operations are made on a variable""" 344 var_history = self.variable(var) 345 files = [] 346 for event in var_history: 347 files.append(event['file']) 348 return files 349 350 def get_variable_lines(self, var, f): 351 """Get the line where a operation is made on a variable in file f""" 352 var_history = self.variable(var) 353 lines = [] 354 for event in var_history: 355 if f== event['file']: 356 line = event['line'] 357 lines.append(line) 358 return lines 359 360 def get_variable_refs(self, var): 361 """Return a dict of file/line references""" 362 var_history = self.variable(var) 363 refs = {} 364 for event in var_history: 365 if event['file'] not in refs: 366 refs[event['file']] = [] 367 refs[event['file']].append(event['line']) 368 return refs 369 370 def get_variable_items_files(self, var): 371 """ 372 Use variable history to map items added to a list variable and 373 the files in which they were added. 374 """ 375 d = self.dataroot 376 history = self.variable(var) 377 finalitems = (d.getVar(var) or '').split() 378 filemap = {} 379 isset = False 380 for event in history: 381 if 'flag' in event: 382 continue 383 if event['op'] == ':remove': 384 continue 385 if isset and event['op'] == 'set?': 386 continue 387 isset = True 388 items = d.expand(event['detail']).split() 389 for item in items: 390 # This is a little crude but is belt-and-braces to avoid us 391 # having to handle every possible operation type specifically 392 if item in finalitems and not item in filemap: 393 filemap[item] = event['file'] 394 return filemap 395 396 def del_var_history(self, var, f=None, line=None): 397 """If file f and line are not given, the entire history of var is deleted""" 398 if var in self.variables: 399 if f and line: 400 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] 401 else: 402 self.variables[var] = [] 403 404def _print_rename_error(var, loginfo, renamedvars, fullvar=None): 405 info = "" 406 if "file" in loginfo: 407 info = " file: %s" % loginfo["file"] 408 if "line" in loginfo: 409 info += " line: %s" % loginfo["line"] 410 if fullvar and fullvar != var: 411 info += " referenced as: %s" % fullvar 412 if info: 413 info = " (%s)" % info.strip() 414 renameinfo = renamedvars[var] 415 if " " in renameinfo: 416 # A space signals a string to display instead of a rename 417 bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) 418 else: 419 bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) 420 421class DataSmart(MutableMapping): 422 def __init__(self): 423 self.dict = {} 424 425 self.inchistory = IncludeHistory() 426 self.varhistory = VariableHistory(self) 427 self._tracking = False 428 self._var_renames = {} 429 self._var_renames.update(bitbake_renamed_vars) 430 431 self.expand_cache = {} 432 433 # cookie monster tribute 434 # Need to be careful about writes to overridedata as 435 # its only a shallow copy, could influence other data store 436 # copies! 437 self.overridedata = {} 438 self.overrides = None 439 self.overridevars = set(["OVERRIDES", "FILE"]) 440 self.inoverride = False 441 442 def enableTracking(self): 443 self._tracking = True 444 445 def disableTracking(self): 446 self._tracking = False 447 448 def expandWithRefs(self, s, varname): 449 450 if not isinstance(s, str): # sanity check 451 return VariableParse(varname, self, s, s) 452 453 varparse = VariableParse(varname, self, s) 454 455 while s.find('${') != -1: 456 olds = s 457 try: 458 s = __expand_var_regexp__.sub(varparse.var_sub, s) 459 try: 460 s = __expand_python_regexp__.sub(varparse.python_sub, s) 461 except SyntaxError as e: 462 # Likely unmatched brackets, just don't expand the expression 463 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): 464 raise 465 if s == olds: 466 break 467 except ExpansionError as e: 468 e.addVar(varname) 469 raise 470 except bb.parse.SkipRecipe: 471 raise 472 except bb.BBHandledException: 473 raise 474 except Exception as exc: 475 tb = sys.exc_info()[2] 476 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 477 478 varparse.value = s 479 480 return varparse 481 482 def expand(self, s, varname = None): 483 return self.expandWithRefs(s, varname).value 484 485 def need_overrides(self): 486 if self.overrides is not None: 487 return 488 if self.inoverride: 489 return 490 overrride_stack = [] 491 for count in range(5): 492 self.inoverride = True 493 # Can end up here recursively so setup dummy values 494 self.overrides = [] 495 self.overridesset = set() 496 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 497 overrride_stack.append(self.overrides) 498 self.overridesset = set(self.overrides) 499 self.inoverride = False 500 self.expand_cache = {} 501 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or [] 502 if newoverrides == self.overrides: 503 break 504 self.overrides = newoverrides 505 self.overridesset = set(self.overrides) 506 else: 507 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) 508 509 def initVar(self, var): 510 self.expand_cache = {} 511 if not var in self.dict: 512 self.dict[var] = {} 513 514 def _findVar(self, var): 515 dest = self.dict 516 while dest: 517 if var in dest: 518 return dest[var] 519 520 if "_data" not in dest: 521 break 522 dest = dest["_data"] 523 return None 524 525 def _makeShadowCopy(self, var): 526 if var in self.dict: 527 return 528 529 local_var = self._findVar(var) 530 531 if local_var: 532 self.dict[var] = copy.copy(local_var) 533 else: 534 self.initVar(var) 535 536 def hasOverrides(self, var): 537 return var in self.overridedata 538 539 def setVar(self, var, value, **loginfo): 540 #print("var=" + str(var) + " val=" + str(value)) 541 542 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): 543 info = "%s" % var 544 if "file" in loginfo: 545 info += " file: %s" % loginfo["file"] 546 if "line" in loginfo: 547 info += " line: %s" % loginfo["line"] 548 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) 549 550 shortvar = var.split(":", 1)[0] 551 if shortvar in self._var_renames: 552 _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) 553 # Mark that we have seen a renamed variable 554 self.setVar("_FAILPARSINGERRORHANDLED", True) 555 556 self.expand_cache = {} 557 parsing=False 558 if 'parsing' in loginfo: 559 parsing=True 560 561 if 'op' not in loginfo: 562 loginfo['op'] = "set" 563 564 match = __setvar_regexp__.match(var) 565 if match and match.group("keyword") in __setvar_keyword__: 566 base = match.group('base') 567 keyword = match.group("keyword") 568 override = match.group('add') 569 l = self.getVarFlag(base, keyword, False) or [] 570 l.append([value, override]) 571 self.setVarFlag(base, keyword, l, ignore=True) 572 # And cause that to be recorded: 573 loginfo['detail'] = value 574 loginfo['variable'] = base 575 if override: 576 loginfo['op'] = '%s[%s]' % (keyword, override) 577 else: 578 loginfo['op'] = keyword 579 self.varhistory.record(**loginfo) 580 # todo make sure keyword is not __doc__ or __module__ 581 # pay the cookie monster 582 583 # more cookies for the cookie monster 584 if ':' in var: 585 self._setvar_update_overrides(base, **loginfo) 586 587 if base in self.overridevars: 588 self._setvar_update_overridevars(var, value) 589 return 590 591 if not var in self.dict: 592 self._makeShadowCopy(var) 593 594 if not parsing: 595 if ":append" in self.dict[var]: 596 del self.dict[var][":append"] 597 if ":prepend" in self.dict[var]: 598 del self.dict[var][":prepend"] 599 if ":remove" in self.dict[var]: 600 del self.dict[var][":remove"] 601 if var in self.overridedata: 602 active = [] 603 self.need_overrides() 604 for (r, o) in self.overridedata[var]: 605 if o in self.overridesset: 606 active.append(r) 607 elif ":" in o: 608 if set(o.split(":")).issubset(self.overridesset): 609 active.append(r) 610 for a in active: 611 self.delVar(a) 612 del self.overridedata[var] 613 614 # more cookies for the cookie monster 615 if ':' in var: 616 self._setvar_update_overrides(var, **loginfo) 617 618 # setting var 619 self.dict[var]["_content"] = value 620 self.varhistory.record(**loginfo) 621 622 if var in self.overridevars: 623 self._setvar_update_overridevars(var, value) 624 625 def _setvar_update_overridevars(self, var, value): 626 vardata = self.expandWithRefs(value, var) 627 new = vardata.references 628 new.update(vardata.contains.keys()) 629 while not new.issubset(self.overridevars): 630 nextnew = set() 631 self.overridevars.update(new) 632 for i in new: 633 vardata = self.expandWithRefs(self.getVar(i), i) 634 nextnew.update(vardata.references) 635 nextnew.update(vardata.contains.keys()) 636 new = nextnew 637 self.overrides = None 638 639 def _setvar_update_overrides(self, var, **loginfo): 640 # aka pay the cookie monster 641 override = var[var.rfind(':')+1:] 642 shortvar = var[:var.rfind(':')] 643 while override and __override_regexp__.match(override): 644 if shortvar not in self.overridedata: 645 self.overridedata[shortvar] = [] 646 if [var, override] not in self.overridedata[shortvar]: 647 # Force CoW by recreating the list first 648 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 649 self.overridedata[shortvar].append([var, override]) 650 override = None 651 if ":" in shortvar: 652 override = var[shortvar.rfind(':')+1:] 653 shortvar = var[:shortvar.rfind(':')] 654 if len(shortvar) == 0: 655 override = None 656 657 def getVar(self, var, expand=True, noweakdefault=False, parsing=False): 658 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) 659 660 def renameVar(self, key, newkey, **loginfo): 661 """ 662 Rename the variable key to newkey 663 """ 664 if key == newkey: 665 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) 666 return 667 668 val = self.getVar(key, 0, parsing=True) 669 if val is not None: 670 self.varhistory.rename_variable_hist(key, newkey) 671 loginfo['variable'] = newkey 672 loginfo['op'] = 'rename from %s' % key 673 loginfo['detail'] = val 674 self.varhistory.record(**loginfo) 675 self.setVar(newkey, val, ignore=True, parsing=True) 676 677 srcflags = self.getVarFlags(key, False, True) or {} 678 for i in srcflags: 679 if i not in (__setvar_keyword__): 680 continue 681 src = srcflags[i] 682 683 dest = self.getVarFlag(newkey, i, False) or [] 684 dest.extend(src) 685 self.setVarFlag(newkey, i, dest, ignore=True) 686 687 if key in self.overridedata: 688 self.overridedata[newkey] = [] 689 for (v, o) in self.overridedata[key]: 690 self.overridedata[newkey].append([v.replace(key, newkey), o]) 691 self.renameVar(v, v.replace(key, newkey)) 692 693 if ':' in newkey and val is None: 694 self._setvar_update_overrides(newkey, **loginfo) 695 696 loginfo['variable'] = key 697 loginfo['op'] = 'rename (to)' 698 loginfo['detail'] = newkey 699 self.varhistory.record(**loginfo) 700 self.delVar(key, ignore=True) 701 702 def appendVar(self, var, value, **loginfo): 703 loginfo['op'] = 'append' 704 self.varhistory.record(**loginfo) 705 self.setVar(var + ":append", value, ignore=True, parsing=True) 706 707 def prependVar(self, var, value, **loginfo): 708 loginfo['op'] = 'prepend' 709 self.varhistory.record(**loginfo) 710 self.setVar(var + ":prepend", value, ignore=True, parsing=True) 711 712 def delVar(self, var, **loginfo): 713 self.expand_cache = {} 714 715 loginfo['detail'] = "" 716 loginfo['op'] = 'del' 717 self.varhistory.record(**loginfo) 718 self.dict[var] = {} 719 if var in self.overridedata: 720 del self.overridedata[var] 721 if ':' in var: 722 override = var[var.rfind(':')+1:] 723 shortvar = var[:var.rfind(':')] 724 while override and __override_regexp__.match(override): 725 try: 726 if shortvar in self.overridedata: 727 # Force CoW by recreating the list first 728 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 729 self.overridedata[shortvar].remove([var, override]) 730 except ValueError as e: 731 pass 732 override = None 733 if ":" in shortvar: 734 override = var[shortvar.rfind(':')+1:] 735 shortvar = var[:shortvar.rfind(':')] 736 if len(shortvar) == 0: 737 override = None 738 739 def setVarFlag(self, var, flag, value, **loginfo): 740 self.expand_cache = {} 741 742 if var == "BB_RENAMED_VARIABLES": 743 self._var_renames[flag] = value 744 745 if var in self._var_renames: 746 _print_rename_error(var, loginfo, self._var_renames) 747 # Mark that we have seen a renamed variable 748 self.setVar("_FAILPARSINGERRORHANDLED", True) 749 750 if 'op' not in loginfo: 751 loginfo['op'] = "set" 752 loginfo['flag'] = flag 753 self.varhistory.record(**loginfo) 754 if not var in self.dict: 755 self._makeShadowCopy(var) 756 self.dict[var][flag] = value 757 758 if flag == "_defaultval" and ':' in var: 759 self._setvar_update_overrides(var, **loginfo) 760 if flag == "_defaultval" and var in self.overridevars: 761 self._setvar_update_overridevars(var, value) 762 763 if flag == "unexport" or flag == "export": 764 if not "__exportlist" in self.dict: 765 self._makeShadowCopy("__exportlist") 766 if not "_content" in self.dict["__exportlist"]: 767 self.dict["__exportlist"]["_content"] = set() 768 self.dict["__exportlist"]["_content"].add(var) 769 770 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): 771 if flag == "_content": 772 cachename = var 773 else: 774 if not flag: 775 bb.warn("Calling getVarFlag with flag unset is invalid") 776 return None 777 cachename = var + "[" + flag + "]" 778 779 if not expand and retparser and cachename in self.expand_cache: 780 return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] 781 782 if expand and cachename in self.expand_cache: 783 return self.expand_cache[cachename].value 784 785 local_var = self._findVar(var) 786 value = None 787 removes = set() 788 if flag == "_content" and not parsing: 789 overridedata = self.overridedata.get(var, None) 790 if flag == "_content" and not parsing and overridedata is not None: 791 match = False 792 active = {} 793 self.need_overrides() 794 for (r, o) in overridedata: 795 # FIXME What about double overrides both with "_" in the name? 796 if o in self.overridesset: 797 active[o] = r 798 elif ":" in o: 799 if set(o.split(":")).issubset(self.overridesset): 800 active[o] = r 801 802 mod = True 803 while mod: 804 mod = False 805 for o in self.overrides: 806 for a in active.copy(): 807 if a.endswith(":" + o): 808 t = active[a] 809 del active[a] 810 active[a.replace(":" + o, "")] = t 811 mod = True 812 elif a == o: 813 match = active[a] 814 del active[a] 815 if match: 816 value, subparser = self.getVarFlag(match, "_content", False, retparser=True) 817 if hasattr(subparser, "removes"): 818 # We have to carry the removes from the overridden variable to apply at the 819 # end of processing 820 removes = subparser.removes 821 822 if local_var is not None and value is None: 823 if flag in local_var: 824 value = copy.copy(local_var[flag]) 825 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 826 value = copy.copy(local_var["_defaultval"]) 827 828 829 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: 830 self.need_overrides() 831 for (r, o) in local_var[":append"]: 832 match = True 833 if o: 834 for o2 in o.split(":"): 835 if not o2 in self.overrides: 836 match = False 837 if match: 838 if value is None: 839 value = "" 840 value = value + r 841 842 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: 843 self.need_overrides() 844 for (r, o) in local_var[":prepend"]: 845 846 match = True 847 if o: 848 for o2 in o.split(":"): 849 if not o2 in self.overrides: 850 match = False 851 if match: 852 if value is None: 853 value = "" 854 value = r + value 855 856 parser = None 857 if expand or retparser: 858 parser = self.expandWithRefs(value, cachename) 859 if expand: 860 value = parser.value 861 862 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: 863 self.need_overrides() 864 for (r, o) in local_var[":remove"]: 865 match = True 866 if o: 867 for o2 in o.split(":"): 868 if not o2 in self.overrides: 869 match = False 870 if match: 871 removes.add(r) 872 873 if value and flag == "_content" and not parsing: 874 if removes and parser: 875 expanded_removes = {} 876 for r in removes: 877 expanded_removes[r] = self.expand(r).split() 878 879 parser.removes = set() 880 val = [] 881 for v in __whitespace_split__.split(parser.value): 882 skip = False 883 for r in removes: 884 if v in expanded_removes[r]: 885 parser.removes.add(r) 886 skip = True 887 if skip: 888 continue 889 val.append(v) 890 parser.value = "".join(val) 891 if expand: 892 value = parser.value 893 894 if parser: 895 self.expand_cache[cachename] = parser 896 897 if retparser: 898 return value, parser 899 900 return value 901 902 def delVarFlag(self, var, flag, **loginfo): 903 self.expand_cache = {} 904 905 local_var = self._findVar(var) 906 if not local_var: 907 return 908 if not var in self.dict: 909 self._makeShadowCopy(var) 910 911 if var in self.dict and flag in self.dict[var]: 912 loginfo['detail'] = "" 913 loginfo['op'] = 'delFlag' 914 loginfo['flag'] = flag 915 self.varhistory.record(**loginfo) 916 917 del self.dict[var][flag] 918 919 def appendVarFlag(self, var, flag, value, **loginfo): 920 loginfo['op'] = 'append' 921 loginfo['flag'] = flag 922 self.varhistory.record(**loginfo) 923 newvalue = (self.getVarFlag(var, flag, False) or "") + value 924 self.setVarFlag(var, flag, newvalue, ignore=True) 925 926 def prependVarFlag(self, var, flag, value, **loginfo): 927 loginfo['op'] = 'prepend' 928 loginfo['flag'] = flag 929 self.varhistory.record(**loginfo) 930 newvalue = value + (self.getVarFlag(var, flag, False) or "") 931 self.setVarFlag(var, flag, newvalue, ignore=True) 932 933 def setVarFlags(self, var, flags, **loginfo): 934 self.expand_cache = {} 935 infer_caller_details(loginfo) 936 if not var in self.dict: 937 self._makeShadowCopy(var) 938 939 for i in flags: 940 if i == "_content": 941 continue 942 loginfo['flag'] = i 943 loginfo['detail'] = flags[i] 944 self.varhistory.record(**loginfo) 945 self.dict[var][i] = flags[i] 946 947 def getVarFlags(self, var, expand = False, internalflags=False): 948 local_var = self._findVar(var) 949 flags = {} 950 951 if local_var: 952 for i in local_var: 953 if i.startswith(("_", ":")) and not internalflags: 954 continue 955 flags[i] = local_var[i] 956 if expand and i in expand: 957 flags[i] = self.expand(flags[i], var + "[" + i + "]") 958 if len(flags) == 0: 959 return None 960 return flags 961 962 963 def delVarFlags(self, var, **loginfo): 964 self.expand_cache = {} 965 if not var in self.dict: 966 self._makeShadowCopy(var) 967 968 if var in self.dict: 969 content = None 970 971 loginfo['op'] = 'delete flags' 972 self.varhistory.record(**loginfo) 973 974 # try to save the content 975 if "_content" in self.dict[var]: 976 content = self.dict[var]["_content"] 977 self.dict[var] = {} 978 self.dict[var]["_content"] = content 979 else: 980 del self.dict[var] 981 982 def createCopy(self): 983 """ 984 Create a copy of self by setting _data to self 985 """ 986 # we really want this to be a DataSmart... 987 data = DataSmart() 988 data.dict["_data"] = self.dict 989 data.varhistory = self.varhistory.copy() 990 data.varhistory.dataroot = data 991 data.inchistory = self.inchistory.copy() 992 993 data._tracking = self._tracking 994 data._var_renames = self._var_renames 995 996 data.overrides = None 997 data.overridevars = copy.copy(self.overridevars) 998 # Should really be a deepcopy but has heavy overhead. 999 # Instead, we're careful with writes. 1000 data.overridedata = copy.copy(self.overridedata) 1001 1002 return data 1003 1004 def expandVarref(self, variable, parents=False): 1005 """Find all references to variable in the data and expand it 1006 in place, optionally descending to parent datastores.""" 1007 1008 if parents: 1009 keys = iter(self) 1010 else: 1011 keys = self.localkeys() 1012 1013 ref = '${%s}' % variable 1014 value = self.getVar(variable, False) 1015 for key in keys: 1016 referrervalue = self.getVar(key, False) 1017 if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: 1018 self.setVar(key, referrervalue.replace(ref, value)) 1019 1020 def localkeys(self): 1021 for key in self.dict: 1022 if key not in ['_data']: 1023 yield key 1024 1025 def __iter__(self): 1026 deleted = set() 1027 overrides = set() 1028 def keylist(d): 1029 klist = set() 1030 for key in d: 1031 if key in ["_data"]: 1032 continue 1033 if key in deleted: 1034 continue 1035 if key in overrides: 1036 continue 1037 if not d[key]: 1038 deleted.add(key) 1039 continue 1040 klist.add(key) 1041 1042 if "_data" in d: 1043 klist |= keylist(d["_data"]) 1044 1045 return klist 1046 1047 self.need_overrides() 1048 for var in self.overridedata: 1049 for (r, o) in self.overridedata[var]: 1050 if o in self.overridesset: 1051 overrides.add(var) 1052 elif ":" in o: 1053 if set(o.split(":")).issubset(self.overridesset): 1054 overrides.add(var) 1055 1056 for k in keylist(self.dict): 1057 yield k 1058 1059 for k in overrides: 1060 yield k 1061 1062 def __len__(self): 1063 return len(frozenset(iter(self))) 1064 1065 def __getitem__(self, item): 1066 value = self.getVar(item, False) 1067 if value is None: 1068 raise KeyError(item) 1069 else: 1070 return value 1071 1072 def __setitem__(self, var, value): 1073 self.setVar(var, value) 1074 1075 def __delitem__(self, var): 1076 self.delVar(var) 1077 1078 def get_hash(self): 1079 data = {} 1080 d = self.createCopy() 1081 bb.data.expandKeys(d) 1082 1083 config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) 1084 keys = set(key for key in iter(d) if not key.startswith("__")) 1085 for key in keys: 1086 if key in config_ignore_vars: 1087 continue 1088 1089 value = d.getVar(key, False) or "" 1090 if type(value) is type(self): 1091 data.update({key:value.get_hash()}) 1092 else: 1093 data.update({key:value}) 1094 1095 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) 1096 if not varflags: 1097 continue 1098 for f in varflags: 1099 if f == "_content": 1100 continue 1101 data.update({'%s[%s]' % (key, f):varflags[f]}) 1102 1103 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: 1104 bb_list = d.getVar(key, False) or [] 1105 data.update({key:str(bb_list)}) 1106 1107 if key == "__BBANONFUNCS": 1108 for i in bb_list: 1109 value = d.getVar(i, False) or "" 1110 data.update({i:value}) 1111 1112 data_str = str([(k, data[k]) for k in sorted(data.keys())]) 1113 return hashlib.sha256(data_str.encode("utf-8")).hexdigest() 1114