1 """ 2 BitBake Smart Dictionary Implementation 3 4 Functions for interacting with the data structure used by the 5 BitBake build tools. 6 7 """ 8 9 # Copyright (C) 2003, 2004 Chris Larson 10 # Copyright (C) 2004, 2005 Seb Frankengul 11 # Copyright (C) 2005, 2006 Holger Hans Peter Freyther 12 # Copyright (C) 2005 Uli Luckas 13 # Copyright (C) 2005 ROAD GmbH 14 # 15 # SPDX-License-Identifier: GPL-2.0-only 16 # 17 # Based on functions from the base bb module, Copyright 2003 Holger Schurig 18 19 import builtins 20 import copy 21 import re 22 import sys 23 from collections.abc import MutableMapping 24 import logging 25 import hashlib 26 import bb, bb.codeparser 27 from bb import utils 28 from bb.COW import COWDictBase 29 30 logger = logging.getLogger("BitBake.Data") 31 32 __setvar_keyword__ = [":append", ":prepend", ":remove"] 33 __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') 34 __expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") 35 __expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") 36 __whitespace_split__ = re.compile(r'(\s)') 37 __override_regexp__ = re.compile(r'[a-z0-9]+') 38 39 bitbake_renamed_vars = { 40 "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", 41 "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", 42 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", 43 "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", 44 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", 45 "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", 46 "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", 47 "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", 48 "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", 49 } 50 51 def infer_caller_details(loginfo, parent = False, varval = True): 52 """Save the caller the trouble of specifying everything.""" 53 # Save effort. 54 if 'ignore' in loginfo and loginfo['ignore']: 55 return 56 # If nothing was provided, mark this as possibly unneeded. 57 if not loginfo: 58 loginfo['ignore'] = True 59 return 60 # Infer caller's likely values for variable (var) and value (value), 61 # to reduce clutter in the rest of the code. 62 above = None 63 def set_above(): 64 try: 65 raise Exception 66 except Exception: 67 tb = sys.exc_info()[2] 68 if parent: 69 return tb.tb_frame.f_back.f_back.f_back 70 else: 71 return tb.tb_frame.f_back.f_back 72 73 if varval and ('variable' not in loginfo or 'detail' not in loginfo): 74 if not above: 75 above = set_above() 76 lcls = above.f_locals.items() 77 for k, v in lcls: 78 if k == 'value' and 'detail' not in loginfo: 79 loginfo['detail'] = v 80 if k == 'var' and 'variable' not in loginfo: 81 loginfo['variable'] = v 82 # Infer file/line/function from traceback 83 # Don't use traceback.extract_stack() since it fills the line contents which 84 # we don't need and that hits stat syscalls 85 if 'file' not in loginfo: 86 if not above: 87 above = set_above() 88 f = above.f_back 89 line = f.f_lineno 90 file = f.f_code.co_filename 91 func = f.f_code.co_name 92 loginfo['file'] = file 93 loginfo['line'] = line 94 if func not in loginfo: 95 loginfo['func'] = func 96 97 class VariableParse: 98 def __init__(self, varname, d, unexpanded_value = None, val = None): 99 self.varname = varname 100 self.d = d 101 self.value = val 102 self.unexpanded_value = unexpanded_value 103 104 self.references = set() 105 self.execs = set() 106 self.contains = {} 107 108 def var_sub(self, match): 109 key = match.group()[2:-1] 110 if self.varname and key: 111 if self.varname == key: 112 raise Exception("variable %s references itself!" % self.varname) 113 var = self.d.getVarFlag(key, "_content") 114 self.references.add(key) 115 if var is not None: 116 return var 117 else: 118 return match.group() 119 120 def python_sub(self, match): 121 if isinstance(match, str): 122 code = match 123 else: 124 code = match.group()[3:-1] 125 126 # Do not run code that contains one or more unexpanded variables 127 # instead return the code with the characters we removed put back 128 if __expand_var_regexp__.findall(code): 129 return "${@" + code + "}" 130 131 if self.varname: 132 varname = 'Var <%s>' % self.varname 133 else: 134 varname = '<expansion>' 135 codeobj = compile(code.strip(), varname, "eval") 136 137 parser = bb.codeparser.PythonParser(self.varname, logger) 138 parser.parse_python(code) 139 if self.varname: 140 vardeps = self.d.getVarFlag(self.varname, "vardeps") 141 if vardeps is None: 142 parser.log.flush() 143 else: 144 parser.log.flush() 145 self.references |= parser.references 146 self.execs |= parser.execs 147 148 for k in parser.contains: 149 if k not in self.contains: 150 self.contains[k] = parser.contains[k].copy() 151 else: 152 self.contains[k].update(parser.contains[k]) 153 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 154 return str(value) 155 156 class DataContext(dict): 157 excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe']) 158 159 def __init__(self, metadata, **kwargs): 160 self.metadata = metadata 161 dict.__init__(self, **kwargs) 162 self['d'] = metadata 163 self.context = set(bb.utils.get_context()) 164 165 def __missing__(self, key): 166 if key in self.excluded or key in self.context: 167 raise KeyError(key) 168 169 value = self.metadata.getVar(key) 170 if value is None: 171 raise KeyError(key) 172 else: 173 return value 174 175 class ExpansionError(Exception): 176 def __init__(self, varname, expression, exception): 177 self.expression = expression 178 self.variablename = varname 179 self.exception = exception 180 self.varlist = [varname or expression or ""] 181 if varname: 182 if expression: 183 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 184 else: 185 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) 186 else: 187 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 188 Exception.__init__(self, self.msg) 189 self.args = (varname, expression, exception) 190 191 def addVar(self, varname): 192 if varname: 193 self.varlist.append(varname) 194 195 def __str__(self): 196 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) 197 return self.msg + chain 198 199 class IncludeHistory(object): 200 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 201 self.parent = parent 202 self.filename = filename 203 self.children = [] 204 self.current = self 205 206 def copy(self): 207 new = IncludeHistory(self.parent, self.filename) 208 for c in self.children: 209 new.children.append(c) 210 return new 211 212 def include(self, filename): 213 newfile = IncludeHistory(self.current, filename) 214 self.current.children.append(newfile) 215 self.current = newfile 216 return self 217 218 def __enter__(self): 219 pass 220 221 def __exit__(self, a, b, c): 222 if self.current.parent: 223 self.current = self.current.parent 224 else: 225 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) 226 return False 227 228 def emit(self, o, level = 0): 229 """Emit an include history file, and its children.""" 230 if level: 231 spaces = " " * (level - 1) 232 o.write("# %s%s" % (spaces, self.filename)) 233 if len(self.children) > 0: 234 o.write(" includes:") 235 else: 236 o.write("#\n# INCLUDE HISTORY:\n#") 237 level = level + 1 238 for child in self.children: 239 o.write("\n") 240 child.emit(o, level) 241 242 class VariableHistory(object): 243 def __init__(self, dataroot): 244 self.dataroot = dataroot 245 self.variables = COWDictBase.copy() 246 247 def copy(self): 248 new = VariableHistory(self.dataroot) 249 new.variables = self.variables.copy() 250 return new 251 252 def __getstate__(self): 253 vardict = {} 254 for k, v in self.variables.iteritems(): 255 vardict[k] = v 256 return {'dataroot': self.dataroot, 257 'variables': vardict} 258 259 def __setstate__(self, state): 260 self.dataroot = state['dataroot'] 261 self.variables = COWDictBase.copy() 262 for k, v in state['variables'].items(): 263 self.variables[k] = v 264 265 def record(self, *kwonly, **loginfo): 266 if not self.dataroot._tracking: 267 return 268 if len(kwonly) > 0: 269 raise TypeError 270 infer_caller_details(loginfo, parent = True) 271 if 'ignore' in loginfo and loginfo['ignore']: 272 return 273 if 'op' not in loginfo or not loginfo['op']: 274 loginfo['op'] = 'set' 275 if 'variable' not in loginfo or 'file' not in loginfo: 276 raise ValueError("record() missing variable or file.") 277 var = loginfo['variable'] 278 if var not in self.variables: 279 self.variables[var] = [] 280 if not isinstance(self.variables[var], list): 281 return 282 if 'nodups' in loginfo and loginfo in self.variables[var]: 283 return 284 self.variables[var].append(loginfo.copy()) 285 286 def rename_variable_hist(self, oldvar, newvar): 287 if not self.dataroot._tracking: 288 return 289 if oldvar not in self.variables: 290 return 291 if newvar not in self.variables: 292 self.variables[newvar] = [] 293 for i in self.variables[oldvar]: 294 self.variables[newvar].append(i.copy()) 295 296 def variable(self, var): 297 varhistory = [] 298 if var in self.variables: 299 varhistory.extend(self.variables[var]) 300 return varhistory 301 302 def emit(self, var, oval, val, o, d): 303 history = self.variable(var) 304 305 # Append override history 306 if var in d.overridedata: 307 for (r, override) in d.overridedata[var]: 308 for event in self.variable(r): 309 loginfo = event.copy() 310 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): 311 continue 312 loginfo['variable'] = var 313 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 314 history.append(loginfo) 315 316 commentVal = re.sub('\n', '\n#', str(oval)) 317 if history: 318 if len(history) == 1: 319 o.write("#\n# $%s\n" % var) 320 else: 321 o.write("#\n# $%s [%d operations]\n" % (var, len(history))) 322 for event in history: 323 # o.write("# %s\n" % str(event)) 324 if 'func' in event: 325 # If we have a function listed, this is internal 326 # code, not an operation in a config file, and the 327 # full path is distracting. 328 event['file'] = re.sub('.*/', '', event['file']) 329 display_func = ' [%s]' % event['func'] 330 else: 331 display_func = '' 332 if 'flag' in event: 333 flag = '[%s] ' % (event['flag']) 334 else: 335 flag = '' 336 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \ 337 (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', str(event['detail'])))) 338 if len(history) > 1: 339 o.write("# pre-expansion value:\n") 340 o.write('# "%s"\n' % (commentVal)) 341 else: 342 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) 343 o.write('# "%s"\n' % (commentVal)) 344 345 def get_variable_files(self, var): 346 """Get the files where operations are made on a variable""" 347 var_history = self.variable(var) 348 files = [] 349 for event in var_history: 350 files.append(event['file']) 351 return files 352 353 def get_variable_lines(self, var, f): 354 """Get the line where a operation is made on a variable in file f""" 355 var_history = self.variable(var) 356 lines = [] 357 for event in var_history: 358 if f== event['file']: 359 line = event['line'] 360 lines.append(line) 361 return lines 362 363 def get_variable_refs(self, var): 364 """Return a dict of file/line references""" 365 var_history = self.variable(var) 366 refs = {} 367 for event in var_history: 368 if event['file'] not in refs: 369 refs[event['file']] = [] 370 refs[event['file']].append(event['line']) 371 return refs 372 373 def get_variable_items_files(self, var): 374 """ 375 Use variable history to map items added to a list variable and 376 the files in which they were added. 377 """ 378 d = self.dataroot 379 history = self.variable(var) 380 finalitems = (d.getVar(var) or '').split() 381 filemap = {} 382 isset = False 383 for event in history: 384 if 'flag' in event: 385 continue 386 if event['op'] == ':remove': 387 continue 388 if isset and event['op'] == 'set?': 389 continue 390 isset = True 391 items = d.expand(str(event['detail'])).split() 392 for item in items: 393 # This is a little crude but is belt-and-braces to avoid us 394 # having to handle every possible operation type specifically 395 if item in finalitems and not item in filemap: 396 filemap[item] = event['file'] 397 return filemap 398 399 def del_var_history(self, var, f=None, line=None): 400 """If file f and line are not given, the entire history of var is deleted""" 401 if var in self.variables: 402 if f and line: 403 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] 404 else: 405 self.variables[var] = [] 406 407 def _print_rename_error(var, loginfo, renamedvars, fullvar=None): 408 info = "" 409 if "file" in loginfo: 410 info = " file: %s" % loginfo["file"] 411 if "line" in loginfo: 412 info += " line: %s" % loginfo["line"] 413 if fullvar and fullvar != var: 414 info += " referenced as: %s" % fullvar 415 if info: 416 info = " (%s)" % info.strip() 417 renameinfo = renamedvars[var] 418 if " " in renameinfo: 419 # A space signals a string to display instead of a rename 420 bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) 421 else: 422 bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) 423 424 class DataSmart(MutableMapping): 425 def __init__(self): 426 self.dict = {} 427 428 self.inchistory = IncludeHistory() 429 self.varhistory = VariableHistory(self) 430 self._tracking = False 431 self._var_renames = {} 432 self._var_renames.update(bitbake_renamed_vars) 433 434 self.expand_cache = {} 435 436 # cookie monster tribute 437 # Need to be careful about writes to overridedata as 438 # its only a shallow copy, could influence other data store 439 # copies! 440 self.overridedata = {} 441 self.overrides = None 442 self.overridevars = set(["OVERRIDES", "FILE"]) 443 self.inoverride = False 444 445 def enableTracking(self): 446 self._tracking = True 447 448 def disableTracking(self): 449 self._tracking = False 450 451 def expandWithRefs(self, s, varname): 452 453 if not isinstance(s, str): # sanity check 454 return VariableParse(varname, self, s, s) 455 456 varparse = VariableParse(varname, self, s) 457 458 while s.find('${') != -1: 459 olds = s 460 try: 461 s = __expand_var_regexp__.sub(varparse.var_sub, s) 462 try: 463 s = __expand_python_regexp__.sub(varparse.python_sub, s) 464 except SyntaxError as e: 465 # Likely unmatched brackets, just don't expand the expression 466 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): 467 raise 468 if s == olds: 469 break 470 except ExpansionError as e: 471 e.addVar(varname) 472 raise 473 except bb.parse.SkipRecipe: 474 raise 475 except bb.BBHandledException: 476 raise 477 except Exception as exc: 478 tb = sys.exc_info()[2] 479 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 480 481 varparse.value = s 482 483 return varparse 484 485 def expand(self, s, varname = None): 486 return self.expandWithRefs(s, varname).value 487 488 def need_overrides(self): 489 if self.overrides is not None: 490 return 491 if self.inoverride: 492 return 493 overrride_stack = [] 494 for count in range(5): 495 self.inoverride = True 496 # Can end up here recursively so setup dummy values 497 self.overrides = [] 498 self.overridesset = set() 499 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 500 overrride_stack.append(self.overrides) 501 self.overridesset = set(self.overrides) 502 self.inoverride = False 503 self.expand_cache = {} 504 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or [] 505 if newoverrides == self.overrides: 506 break 507 self.overrides = newoverrides 508 self.overridesset = set(self.overrides) 509 else: 510 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) 511 512 def initVar(self, var): 513 self.expand_cache = {} 514 if not var in self.dict: 515 self.dict[var] = {} 516 517 def _findVar(self, var): 518 dest = self.dict 519 while dest: 520 if var in dest: 521 return dest[var] 522 523 if "_data" not in dest: 524 break 525 dest = dest["_data"] 526 return None 527 528 def _makeShadowCopy(self, var): 529 if var in self.dict: 530 return 531 532 local_var = self._findVar(var) 533 534 if local_var: 535 self.dict[var] = copy.copy(local_var) 536 else: 537 self.initVar(var) 538 539 def hasOverrides(self, var): 540 return var in self.overridedata 541 542 def setVar(self, var, value, **loginfo): 543 #print("var=" + str(var) + " val=" + str(value)) 544 545 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): 546 info = "%s" % var 547 if "file" in loginfo: 548 info += " file: %s" % loginfo["file"] 549 if "line" in loginfo: 550 info += " line: %s" % loginfo["line"] 551 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) 552 553 shortvar = var.split(":", 1)[0] 554 if shortvar in self._var_renames: 555 _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) 556 # Mark that we have seen a renamed variable 557 self.setVar("_FAILPARSINGERRORHANDLED", True) 558 559 self.expand_cache = {} 560 parsing=False 561 if 'parsing' in loginfo: 562 parsing=True 563 564 if 'op' not in loginfo: 565 loginfo['op'] = "set" 566 567 match = __setvar_regexp__.match(var) 568 if match and match.group("keyword") in __setvar_keyword__: 569 base = match.group('base') 570 keyword = match.group("keyword") 571 override = match.group('add') 572 l = self.getVarFlag(base, keyword, False) or [] 573 l.append([value, override]) 574 self.setVarFlag(base, keyword, l, ignore=True) 575 # And cause that to be recorded: 576 loginfo['detail'] = value 577 loginfo['variable'] = base 578 if override: 579 loginfo['op'] = '%s[%s]' % (keyword, override) 580 else: 581 loginfo['op'] = keyword 582 self.varhistory.record(**loginfo) 583 # todo make sure keyword is not __doc__ or __module__ 584 # pay the cookie monster 585 586 # more cookies for the cookie monster 587 if ':' in var: 588 self._setvar_update_overrides(base, **loginfo) 589 590 if base in self.overridevars: 591 self._setvar_update_overridevars(var, value) 592 return 593 594 if not var in self.dict: 595 self._makeShadowCopy(var) 596 597 if not parsing: 598 if ":append" in self.dict[var]: 599 del self.dict[var][":append"] 600 if ":prepend" in self.dict[var]: 601 del self.dict[var][":prepend"] 602 if ":remove" in self.dict[var]: 603 del self.dict[var][":remove"] 604 if var in self.overridedata: 605 active = [] 606 self.need_overrides() 607 for (r, o) in self.overridedata[var]: 608 if o in self.overridesset: 609 active.append(r) 610 elif ":" in o: 611 if set(o.split(":")).issubset(self.overridesset): 612 active.append(r) 613 for a in active: 614 self.delVar(a) 615 del self.overridedata[var] 616 617 # more cookies for the cookie monster 618 if ':' in var: 619 self._setvar_update_overrides(var, **loginfo) 620 621 # setting var 622 self.dict[var]["_content"] = value 623 self.varhistory.record(**loginfo) 624 625 if var in self.overridevars: 626 self._setvar_update_overridevars(var, value) 627 628 def _setvar_update_overridevars(self, var, value): 629 vardata = self.expandWithRefs(value, var) 630 new = vardata.references 631 new.update(vardata.contains.keys()) 632 while not new.issubset(self.overridevars): 633 nextnew = set() 634 self.overridevars.update(new) 635 for i in new: 636 vardata = self.expandWithRefs(self.getVar(i), i) 637 nextnew.update(vardata.references) 638 nextnew.update(vardata.contains.keys()) 639 new = nextnew 640 self.overrides = None 641 642 def _setvar_update_overrides(self, var, **loginfo): 643 # aka pay the cookie monster 644 override = var[var.rfind(':')+1:] 645 shortvar = var[:var.rfind(':')] 646 while override and __override_regexp__.match(override): 647 if shortvar not in self.overridedata: 648 self.overridedata[shortvar] = [] 649 if [var, override] not in self.overridedata[shortvar]: 650 # Force CoW by recreating the list first 651 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 652 self.overridedata[shortvar].append([var, override]) 653 override = None 654 if ":" in shortvar: 655 override = var[shortvar.rfind(':')+1:] 656 shortvar = var[:shortvar.rfind(':')] 657 if len(shortvar) == 0: 658 override = None 659 660 def getVar(self, var, expand=True, noweakdefault=False, parsing=False): 661 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) 662 663 def renameVar(self, key, newkey, **loginfo): 664 """ 665 Rename the variable key to newkey 666 """ 667 if key == newkey: 668 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) 669 return 670 671 val = self.getVar(key, 0, parsing=True) 672 if val is not None: 673 self.varhistory.rename_variable_hist(key, newkey) 674 loginfo['variable'] = newkey 675 loginfo['op'] = 'rename from %s' % key 676 loginfo['detail'] = val 677 self.varhistory.record(**loginfo) 678 self.setVar(newkey, val, ignore=True, parsing=True) 679 680 srcflags = self.getVarFlags(key, False, True) or {} 681 for i in srcflags: 682 if i not in (__setvar_keyword__): 683 continue 684 src = srcflags[i] 685 686 dest = self.getVarFlag(newkey, i, False) or [] 687 dest.extend(src) 688 self.setVarFlag(newkey, i, dest, ignore=True) 689 690 if key in self.overridedata: 691 self.overridedata[newkey] = [] 692 for (v, o) in self.overridedata[key]: 693 self.overridedata[newkey].append([v.replace(key, newkey), o]) 694 self.renameVar(v, v.replace(key, newkey)) 695 696 if ':' in newkey and val is None: 697 self._setvar_update_overrides(newkey, **loginfo) 698 699 loginfo['variable'] = key 700 loginfo['op'] = 'rename (to)' 701 loginfo['detail'] = newkey 702 self.varhistory.record(**loginfo) 703 self.delVar(key, ignore=True) 704 705 def appendVar(self, var, value, **loginfo): 706 loginfo['op'] = 'append' 707 self.varhistory.record(**loginfo) 708 self.setVar(var + ":append", value, ignore=True, parsing=True) 709 710 def prependVar(self, var, value, **loginfo): 711 loginfo['op'] = 'prepend' 712 self.varhistory.record(**loginfo) 713 self.setVar(var + ":prepend", value, ignore=True, parsing=True) 714 715 def delVar(self, var, **loginfo): 716 self.expand_cache = {} 717 718 loginfo['detail'] = "" 719 loginfo['op'] = 'del' 720 self.varhistory.record(**loginfo) 721 self.dict[var] = {} 722 if var in self.overridedata: 723 del self.overridedata[var] 724 if ':' in var: 725 override = var[var.rfind(':')+1:] 726 shortvar = var[:var.rfind(':')] 727 while override and __override_regexp__.match(override): 728 try: 729 if shortvar in self.overridedata: 730 # Force CoW by recreating the list first 731 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 732 self.overridedata[shortvar].remove([var, override]) 733 except ValueError as e: 734 pass 735 override = None 736 if ":" in shortvar: 737 override = var[shortvar.rfind(':')+1:] 738 shortvar = var[:shortvar.rfind(':')] 739 if len(shortvar) == 0: 740 override = None 741 742 def setVarFlag(self, var, flag, value, **loginfo): 743 self.expand_cache = {} 744 745 if var == "BB_RENAMED_VARIABLES": 746 self._var_renames[flag] = value 747 748 if var in self._var_renames: 749 _print_rename_error(var, loginfo, self._var_renames) 750 # Mark that we have seen a renamed variable 751 self.setVar("_FAILPARSINGERRORHANDLED", True) 752 753 if 'op' not in loginfo: 754 loginfo['op'] = "set" 755 loginfo['flag'] = flag 756 self.varhistory.record(**loginfo) 757 if not var in self.dict: 758 self._makeShadowCopy(var) 759 self.dict[var][flag] = value 760 761 if flag == "_defaultval" and ':' in var: 762 self._setvar_update_overrides(var, **loginfo) 763 if flag == "_defaultval" and var in self.overridevars: 764 self._setvar_update_overridevars(var, value) 765 766 if flag == "unexport" or flag == "export": 767 if not "__exportlist" in self.dict: 768 self._makeShadowCopy("__exportlist") 769 if not "_content" in self.dict["__exportlist"]: 770 self.dict["__exportlist"]["_content"] = set() 771 self.dict["__exportlist"]["_content"].add(var) 772 773 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): 774 if flag == "_content": 775 cachename = var 776 else: 777 if not flag: 778 bb.warn("Calling getVarFlag with flag unset is invalid") 779 return None 780 cachename = var + "[" + flag + "]" 781 782 if not expand and retparser and cachename in self.expand_cache: 783 return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] 784 785 if expand and cachename in self.expand_cache: 786 return self.expand_cache[cachename].value 787 788 local_var = self._findVar(var) 789 value = None 790 removes = set() 791 if flag == "_content" and not parsing: 792 overridedata = self.overridedata.get(var, None) 793 if flag == "_content" and not parsing and overridedata is not None: 794 match = False 795 active = {} 796 self.need_overrides() 797 for (r, o) in overridedata: 798 # FIXME What about double overrides both with "_" in the name? 799 if o in self.overridesset: 800 active[o] = r 801 elif ":" in o: 802 if set(o.split(":")).issubset(self.overridesset): 803 active[o] = r 804 805 mod = True 806 while mod: 807 mod = False 808 for o in self.overrides: 809 for a in active.copy(): 810 if a.endswith(":" + o): 811 t = active[a] 812 del active[a] 813 active[a.replace(":" + o, "")] = t 814 mod = True 815 elif a == o: 816 match = active[a] 817 del active[a] 818 if match: 819 value, subparser = self.getVarFlag(match, "_content", False, retparser=True) 820 if hasattr(subparser, "removes"): 821 # We have to carry the removes from the overridden variable to apply at the 822 # end of processing 823 removes = subparser.removes 824 825 if local_var is not None and value is None: 826 if flag in local_var: 827 value = copy.copy(local_var[flag]) 828 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 829 value = copy.copy(local_var["_defaultval"]) 830 elif "_defaultval_flag_"+flag in local_var and not noweakdefault: 831 value = copy.copy(local_var["_defaultval_flag_"+flag]) 832 833 834 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: 835 self.need_overrides() 836 for (r, o) in local_var[":append"]: 837 match = True 838 if o: 839 for o2 in o.split(":"): 840 if not o2 in self.overrides: 841 match = False 842 if match: 843 if value is None: 844 value = "" 845 value = value + r 846 847 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: 848 self.need_overrides() 849 for (r, o) in local_var[":prepend"]: 850 851 match = True 852 if o: 853 for o2 in o.split(":"): 854 if not o2 in self.overrides: 855 match = False 856 if match: 857 if value is None: 858 value = "" 859 value = r + value 860 861 parser = None 862 if expand or retparser: 863 parser = self.expandWithRefs(value, cachename) 864 if expand: 865 value = parser.value 866 867 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: 868 self.need_overrides() 869 for (r, o) in local_var[":remove"]: 870 match = True 871 if o: 872 for o2 in o.split(":"): 873 if not o2 in self.overrides: 874 match = False 875 if match: 876 removes.add(r) 877 878 if value and flag == "_content" and not parsing: 879 if removes and parser: 880 expanded_removes = {} 881 for r in removes: 882 expanded_removes[r] = self.expand(r).split() 883 884 parser.removes = set() 885 val = [] 886 for v in __whitespace_split__.split(parser.value): 887 skip = False 888 for r in removes: 889 if v in expanded_removes[r]: 890 parser.removes.add(r) 891 skip = True 892 if skip: 893 continue 894 val.append(v) 895 parser.value = "".join(val) 896 if expand: 897 value = parser.value 898 899 if parser: 900 self.expand_cache[cachename] = parser 901 902 if retparser: 903 return value, parser 904 905 return value 906 907 def delVarFlag(self, var, flag, **loginfo): 908 self.expand_cache = {} 909 910 local_var = self._findVar(var) 911 if not local_var: 912 return 913 if not var in self.dict: 914 self._makeShadowCopy(var) 915 916 if var in self.dict and flag in self.dict[var]: 917 loginfo['detail'] = "" 918 loginfo['op'] = 'delFlag' 919 loginfo['flag'] = flag 920 self.varhistory.record(**loginfo) 921 922 del self.dict[var][flag] 923 924 def appendVarFlag(self, var, flag, value, **loginfo): 925 loginfo['op'] = 'append' 926 loginfo['flag'] = flag 927 self.varhistory.record(**loginfo) 928 newvalue = (self.getVarFlag(var, flag, False) or "") + value 929 self.setVarFlag(var, flag, newvalue, ignore=True) 930 931 def prependVarFlag(self, var, flag, value, **loginfo): 932 loginfo['op'] = 'prepend' 933 loginfo['flag'] = flag 934 self.varhistory.record(**loginfo) 935 newvalue = value + (self.getVarFlag(var, flag, False) or "") 936 self.setVarFlag(var, flag, newvalue, ignore=True) 937 938 def setVarFlags(self, var, flags, **loginfo): 939 self.expand_cache = {} 940 infer_caller_details(loginfo) 941 if not var in self.dict: 942 self._makeShadowCopy(var) 943 944 for i in flags: 945 if i == "_content": 946 continue 947 loginfo['flag'] = i 948 loginfo['detail'] = flags[i] 949 self.varhistory.record(**loginfo) 950 self.dict[var][i] = flags[i] 951 952 def getVarFlags(self, var, expand = False, internalflags=False): 953 local_var = self._findVar(var) 954 flags = {} 955 956 if local_var: 957 for i in local_var: 958 if i.startswith(("_", ":")) and not internalflags: 959 continue 960 flags[i] = local_var[i] 961 if expand and i in expand: 962 flags[i] = self.expand(flags[i], var + "[" + i + "]") 963 if len(flags) == 0: 964 return None 965 return flags 966 967 968 def delVarFlags(self, var, **loginfo): 969 self.expand_cache = {} 970 if not var in self.dict: 971 self._makeShadowCopy(var) 972 973 if var in self.dict: 974 content = None 975 976 loginfo['op'] = 'delete flags' 977 self.varhistory.record(**loginfo) 978 979 # try to save the content 980 if "_content" in self.dict[var]: 981 content = self.dict[var]["_content"] 982 self.dict[var] = {} 983 self.dict[var]["_content"] = content 984 else: 985 del self.dict[var] 986 987 def createCopy(self): 988 """ 989 Create a copy of self by setting _data to self 990 """ 991 # we really want this to be a DataSmart... 992 data = DataSmart() 993 data.dict["_data"] = self.dict 994 data.varhistory = self.varhistory.copy() 995 data.varhistory.dataroot = data 996 data.inchistory = self.inchistory.copy() 997 998 data._tracking = self._tracking 999 data._var_renames = self._var_renames 1000 1001 data.overrides = None 1002 data.overridevars = copy.copy(self.overridevars) 1003 # Should really be a deepcopy but has heavy overhead. 1004 # Instead, we're careful with writes. 1005 data.overridedata = copy.copy(self.overridedata) 1006 1007 return data 1008 1009 def expandVarref(self, variable, parents=False): 1010 """Find all references to variable in the data and expand it 1011 in place, optionally descending to parent datastores.""" 1012 1013 if parents: 1014 keys = iter(self) 1015 else: 1016 keys = self.localkeys() 1017 1018 ref = '${%s}' % variable 1019 value = self.getVar(variable, False) 1020 for key in keys: 1021 referrervalue = self.getVar(key, False) 1022 if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: 1023 self.setVar(key, referrervalue.replace(ref, value)) 1024 1025 def localkeys(self): 1026 for key in self.dict: 1027 if key not in ['_data']: 1028 yield key 1029 1030 def __iter__(self): 1031 deleted = set() 1032 overrides = set() 1033 def keylist(d): 1034 klist = set() 1035 for key in d: 1036 if key in ["_data"]: 1037 continue 1038 if key in deleted: 1039 continue 1040 if key in overrides: 1041 continue 1042 if not d[key]: 1043 deleted.add(key) 1044 continue 1045 klist.add(key) 1046 1047 if "_data" in d: 1048 klist |= keylist(d["_data"]) 1049 1050 return klist 1051 1052 self.need_overrides() 1053 for var in self.overridedata: 1054 for (r, o) in self.overridedata[var]: 1055 if o in self.overridesset: 1056 overrides.add(var) 1057 elif ":" in o: 1058 if set(o.split(":")).issubset(self.overridesset): 1059 overrides.add(var) 1060 1061 for k in keylist(self.dict): 1062 yield k 1063 1064 for k in overrides: 1065 yield k 1066 1067 def __len__(self): 1068 return len(frozenset(iter(self))) 1069 1070 def __getitem__(self, item): 1071 value = self.getVar(item, False) 1072 if value is None: 1073 raise KeyError(item) 1074 else: 1075 return value 1076 1077 def __setitem__(self, var, value): 1078 self.setVar(var, value) 1079 1080 def __delitem__(self, var): 1081 self.delVar(var) 1082 1083 def get_hash(self): 1084 data = {} 1085 d = self.createCopy() 1086 bb.data.expandKeys(d) 1087 1088 config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) 1089 keys = set(key for key in iter(d) if not key.startswith("__")) 1090 for key in keys: 1091 if key in config_ignore_vars: 1092 continue 1093 1094 value = d.getVar(key, False) or "" 1095 if type(value) is type(self): 1096 data.update({key:value.get_hash()}) 1097 else: 1098 data.update({key:value}) 1099 1100 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) 1101 if not varflags: 1102 continue 1103 for f in varflags: 1104 if f == "_content": 1105 continue 1106 data.update({'%s[%s]' % (key, f):varflags[f]}) 1107 1108 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: 1109 bb_list = d.getVar(key, False) or [] 1110 data.update({key:str(bb_list)}) 1111 1112 if key == "__BBANONFUNCS": 1113 for i in bb_list: 1114 value = d.getVar(i, False) or "" 1115 data.update({i:value}) 1116 1117 data_str = str([(k, data[k]) for k in sorted(data.keys())]) 1118 return hashlib.sha256(data_str.encode("utf-8")).hexdigest() 1119