1""" 2BitBake Smart Dictionary Implementation 3 4Functions for interacting with the data structure used by the 5BitBake build tools. 6 7""" 8 9# Copyright (C) 2003, 2004 Chris Larson 10# Copyright (C) 2004, 2005 Seb Frankengul 11# Copyright (C) 2005, 2006 Holger Hans Peter Freyther 12# Copyright (C) 2005 Uli Luckas 13# Copyright (C) 2005 ROAD GmbH 14# 15# SPDX-License-Identifier: GPL-2.0-only 16# 17# Based on functions from the base bb module, Copyright 2003 Holger Schurig 18 19import builtins 20import copy 21import re 22import sys 23from collections.abc import MutableMapping 24import logging 25import hashlib 26import bb, bb.codeparser 27from bb import utils 28from bb.COW import COWDictBase 29 30logger = logging.getLogger("BitBake.Data") 31 32__setvar_keyword__ = [":append", ":prepend", ":remove"] 33__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') 34__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+}") 35__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") 36__whitespace_split__ = re.compile(r'(\s)') 37__override_regexp__ = re.compile(r'[a-z0-9]+') 38 39bitbake_renamed_vars = { 40 "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", 41 "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", 42 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", 43 "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", 44 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", 45 "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", 46 "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", 47 "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", 48 "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", 49} 50 51def infer_caller_details(loginfo, parent = False, varval = True): 52 """Save the caller the trouble of specifying everything.""" 53 # Save effort. 54 if 'ignore' in loginfo and loginfo['ignore']: 55 return 56 # If nothing was provided, mark this as possibly unneeded. 57 if not loginfo: 58 loginfo['ignore'] = True 59 return 60 # Infer caller's likely values for variable (var) and value (value), 61 # to reduce clutter in the rest of the code. 62 above = None 63 def set_above(): 64 try: 65 raise Exception 66 except Exception: 67 tb = sys.exc_info()[2] 68 if parent: 69 return tb.tb_frame.f_back.f_back.f_back 70 else: 71 return tb.tb_frame.f_back.f_back 72 73 if varval and ('variable' not in loginfo or 'detail' not in loginfo): 74 if not above: 75 above = set_above() 76 lcls = above.f_locals.items() 77 for k, v in lcls: 78 if k == 'value' and 'detail' not in loginfo: 79 loginfo['detail'] = v 80 if k == 'var' and 'variable' not in loginfo: 81 loginfo['variable'] = v 82 # Infer file/line/function from traceback 83 # Don't use traceback.extract_stack() since it fills the line contents which 84 # we don't need and that hits stat syscalls 85 if 'file' not in loginfo: 86 if not above: 87 above = set_above() 88 f = above.f_back 89 line = f.f_lineno 90 file = f.f_code.co_filename 91 func = f.f_code.co_name 92 loginfo['file'] = file 93 loginfo['line'] = line 94 if func not in loginfo: 95 loginfo['func'] = func 96 97class VariableParse: 98 def __init__(self, varname, d, unexpanded_value = None, val = None): 99 self.varname = varname 100 self.d = d 101 self.value = val 102 self.unexpanded_value = unexpanded_value 103 104 self.references = set() 105 self.execs = set() 106 self.contains = {} 107 108 def var_sub(self, match): 109 key = match.group()[2:-1] 110 if self.varname and key: 111 if self.varname == key: 112 raise Exception("variable %s references itself!" % self.varname) 113 var = self.d.getVarFlag(key, "_content") 114 self.references.add(key) 115 if var is not None: 116 return var 117 else: 118 return match.group() 119 120 def python_sub(self, match): 121 if isinstance(match, str): 122 code = match 123 else: 124 code = match.group()[3:-1] 125 126 # Do not run code that contains one or more unexpanded variables 127 # instead return the code with the characters we removed put back 128 if __expand_var_regexp__.findall(code): 129 return "${@" + code + "}" 130 131 if self.varname: 132 varname = 'Var <%s>' % self.varname 133 else: 134 varname = '<expansion>' 135 codeobj = compile(code.strip(), varname, "eval") 136 137 parser = bb.codeparser.PythonParser(self.varname, logger) 138 parser.parse_python(code) 139 if self.varname: 140 vardeps = self.d.getVarFlag(self.varname, "vardeps") 141 if vardeps is None: 142 parser.log.flush() 143 else: 144 parser.log.flush() 145 self.references |= parser.references 146 self.execs |= parser.execs 147 148 for k in parser.contains: 149 if k not in self.contains: 150 self.contains[k] = parser.contains[k].copy() 151 else: 152 self.contains[k].update(parser.contains[k]) 153 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 154 return str(value) 155 156class DataContext(dict): 157 excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe']) 158 159 def __init__(self, metadata, **kwargs): 160 self.metadata = metadata 161 dict.__init__(self, **kwargs) 162 self['d'] = metadata 163 self.context = set(bb.utils.get_context()) 164 165 def __missing__(self, key): 166 if key in self.excluded or key in self.context: 167 raise KeyError(key) 168 169 value = self.metadata.getVar(key) 170 if value is None: 171 raise KeyError(key) 172 else: 173 return value 174 175class ExpansionError(Exception): 176 def __init__(self, varname, expression, exception): 177 self.expression = expression 178 self.variablename = varname 179 self.exception = exception 180 self.varlist = [varname or expression or ""] 181 if varname: 182 if expression: 183 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 184 else: 185 self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) 186 else: 187 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 188 Exception.__init__(self, self.msg) 189 self.args = (varname, expression, exception) 190 191 def addVar(self, varname): 192 if varname: 193 self.varlist.append(varname) 194 195 def __str__(self): 196 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) 197 return self.msg + chain 198 199class IncludeHistory(object): 200 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 201 self.parent = parent 202 self.filename = filename 203 self.children = [] 204 self.current = self 205 206 def copy(self): 207 new = IncludeHistory(self.parent, self.filename) 208 for c in self.children: 209 new.children.append(c) 210 return new 211 212 def include(self, filename): 213 newfile = IncludeHistory(self.current, filename) 214 self.current.children.append(newfile) 215 self.current = newfile 216 return self 217 218 def __enter__(self): 219 pass 220 221 def __exit__(self, a, b, c): 222 if self.current.parent: 223 self.current = self.current.parent 224 else: 225 bb.warn("Include log: Tried to finish '%s' at top level." % self.filename) 226 return False 227 228 def emit(self, o, level = 0): 229 """Emit an include history file, and its children.""" 230 if level: 231 spaces = " " * (level - 1) 232 o.write("# %s%s" % (spaces, self.filename)) 233 if len(self.children) > 0: 234 o.write(" includes:") 235 else: 236 o.write("#\n# INCLUDE HISTORY:\n#") 237 level = level + 1 238 for child in self.children: 239 o.write("\n") 240 child.emit(o, level) 241 242class VariableHistory(object): 243 def __init__(self, dataroot): 244 self.dataroot = dataroot 245 self.variables = COWDictBase.copy() 246 247 def copy(self): 248 new = VariableHistory(self.dataroot) 249 new.variables = self.variables.copy() 250 return new 251 252 def __getstate__(self): 253 vardict = {} 254 for k, v in self.variables.iteritems(): 255 vardict[k] = v 256 return {'dataroot': self.dataroot, 257 'variables': vardict} 258 259 def __setstate__(self, state): 260 self.dataroot = state['dataroot'] 261 self.variables = COWDictBase.copy() 262 for k, v in state['variables'].items(): 263 self.variables[k] = v 264 265 def record(self, *kwonly, **loginfo): 266 if not self.dataroot._tracking: 267 return 268 if len(kwonly) > 0: 269 raise TypeError 270 infer_caller_details(loginfo, parent = True) 271 if 'ignore' in loginfo and loginfo['ignore']: 272 return 273 if 'op' not in loginfo or not loginfo['op']: 274 loginfo['op'] = 'set' 275 if 'variable' not in loginfo or 'file' not in loginfo: 276 raise ValueError("record() missing variable or file.") 277 var = loginfo['variable'] 278 if var not in self.variables: 279 self.variables[var] = [] 280 if not isinstance(self.variables[var], list): 281 return 282 if 'nodups' in loginfo and loginfo in self.variables[var]: 283 return 284 self.variables[var].append(loginfo.copy()) 285 286 def rename_variable_hist(self, oldvar, newvar): 287 if not self.dataroot._tracking: 288 return 289 if oldvar not in self.variables: 290 return 291 if newvar not in self.variables: 292 self.variables[newvar] = [] 293 for i in self.variables[oldvar]: 294 self.variables[newvar].append(i.copy()) 295 296 def variable(self, var): 297 varhistory = [] 298 if var in self.variables: 299 varhistory.extend(self.variables[var]) 300 return varhistory 301 302 def emit(self, var, oval, val, o, d): 303 history = self.variable(var) 304 305 # Append override history 306 if var in d.overridedata: 307 for (r, override) in d.overridedata[var]: 308 for event in self.variable(r): 309 loginfo = event.copy() 310 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): 311 continue 312 loginfo['variable'] = var 313 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 314 history.append(loginfo) 315 316 commentVal = re.sub('\n', '\n#', str(oval)) 317 if history: 318 if len(history) == 1: 319 o.write("#\n# $%s\n" % var) 320 else: 321 o.write("#\n# $%s [%d operations]\n" % (var, len(history))) 322 for event in history: 323 # o.write("# %s\n" % str(event)) 324 if 'func' in event: 325 # If we have a function listed, this is internal 326 # code, not an operation in a config file, and the 327 # full path is distracting. 328 event['file'] = re.sub('.*/', '', event['file']) 329 display_func = ' [%s]' % event['func'] 330 else: 331 display_func = '' 332 if 'flag' in event: 333 flag = '[%s] ' % (event['flag']) 334 else: 335 flag = '' 336 o.write("# %s %s:%s%s\n# %s\"%s\"\n" % \ 337 (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', str(event['detail'])))) 338 if len(history) > 1: 339 o.write("# pre-expansion value:\n") 340 o.write('# "%s"\n' % (commentVal)) 341 else: 342 o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) 343 o.write('# "%s"\n' % (commentVal)) 344 345 def get_variable_files(self, var): 346 """Get the files where operations are made on a variable""" 347 var_history = self.variable(var) 348 files = [] 349 for event in var_history: 350 files.append(event['file']) 351 return files 352 353 def get_variable_lines(self, var, f): 354 """Get the line where a operation is made on a variable in file f""" 355 var_history = self.variable(var) 356 lines = [] 357 for event in var_history: 358 if f== event['file']: 359 line = event['line'] 360 lines.append(line) 361 return lines 362 363 def get_variable_refs(self, var): 364 """Return a dict of file/line references""" 365 var_history = self.variable(var) 366 refs = {} 367 for event in var_history: 368 if event['file'] not in refs: 369 refs[event['file']] = [] 370 refs[event['file']].append(event['line']) 371 return refs 372 373 def get_variable_items_files(self, var): 374 """ 375 Use variable history to map items added to a list variable and 376 the files in which they were added. 377 """ 378 d = self.dataroot 379 history = self.variable(var) 380 finalitems = (d.getVar(var) or '').split() 381 filemap = {} 382 isset = False 383 for event in history: 384 if 'flag' in event: 385 continue 386 if event['op'] == ':remove': 387 continue 388 if isset and event['op'] == 'set?': 389 continue 390 isset = True 391 items = d.expand(str(event['detail'])).split() 392 for item in items: 393 # This is a little crude but is belt-and-braces to avoid us 394 # having to handle every possible operation type specifically 395 if item in finalitems and not item in filemap: 396 filemap[item] = event['file'] 397 return filemap 398 399 def del_var_history(self, var, f=None, line=None): 400 """If file f and line are not given, the entire history of var is deleted""" 401 if var in self.variables: 402 if f and line: 403 self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] 404 else: 405 self.variables[var] = [] 406 407def _print_rename_error(var, loginfo, renamedvars, fullvar=None): 408 info = "" 409 if "file" in loginfo: 410 info = " file: %s" % loginfo["file"] 411 if "line" in loginfo: 412 info += " line: %s" % loginfo["line"] 413 if fullvar and fullvar != var: 414 info += " referenced as: %s" % fullvar 415 if info: 416 info = " (%s)" % info.strip() 417 renameinfo = renamedvars[var] 418 if " " in renameinfo: 419 # A space signals a string to display instead of a rename 420 bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) 421 else: 422 bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) 423 424class DataSmart(MutableMapping): 425 def __init__(self): 426 self.dict = {} 427 428 self.inchistory = IncludeHistory() 429 self.varhistory = VariableHistory(self) 430 self._tracking = False 431 self._var_renames = {} 432 self._var_renames.update(bitbake_renamed_vars) 433 434 self.expand_cache = {} 435 436 # cookie monster tribute 437 # Need to be careful about writes to overridedata as 438 # its only a shallow copy, could influence other data store 439 # copies! 440 self.overridedata = {} 441 self.overrides = None 442 self.overridevars = set(["OVERRIDES", "FILE"]) 443 self.inoverride = False 444 445 def enableTracking(self): 446 self._tracking = True 447 448 def disableTracking(self): 449 self._tracking = False 450 451 def expandWithRefs(self, s, varname): 452 453 if not isinstance(s, str): # sanity check 454 return VariableParse(varname, self, s, s) 455 456 varparse = VariableParse(varname, self, s) 457 458 while s.find('${') != -1: 459 olds = s 460 try: 461 s = __expand_var_regexp__.sub(varparse.var_sub, s) 462 try: 463 s = __expand_python_regexp__.sub(varparse.python_sub, s) 464 except SyntaxError as e: 465 # Likely unmatched brackets, just don't expand the expression 466 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): 467 raise 468 if s == olds: 469 break 470 except ExpansionError as e: 471 e.addVar(varname) 472 raise 473 except bb.parse.SkipRecipe: 474 raise 475 except bb.BBHandledException: 476 raise 477 except Exception as exc: 478 tb = sys.exc_info()[2] 479 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 480 481 varparse.value = s 482 483 return varparse 484 485 def expand(self, s, varname = None): 486 return self.expandWithRefs(s, varname).value 487 488 def need_overrides(self): 489 if self.overrides is not None: 490 return 491 if self.inoverride: 492 return 493 overrride_stack = [] 494 for count in range(5): 495 self.inoverride = True 496 # Can end up here recursively so setup dummy values 497 self.overrides = [] 498 self.overridesset = set() 499 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 500 overrride_stack.append(self.overrides) 501 self.overridesset = set(self.overrides) 502 self.inoverride = False 503 self.expand_cache = {} 504 newoverrides = (self.getVar("OVERRIDES") or "").split(":") or [] 505 if newoverrides == self.overrides: 506 break 507 self.overrides = newoverrides 508 self.overridesset = set(self.overrides) 509 else: 510 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) 511 512 def initVar(self, var): 513 self.expand_cache = {} 514 if not var in self.dict: 515 self.dict[var] = {} 516 517 def _findVar(self, var): 518 dest = self.dict 519 while dest: 520 if var in dest: 521 return dest[var] 522 523 if "_data" not in dest: 524 break 525 dest = dest["_data"] 526 return None 527 528 def _makeShadowCopy(self, var): 529 if var in self.dict: 530 return 531 532 local_var = self._findVar(var) 533 534 if local_var: 535 self.dict[var] = copy.copy(local_var) 536 else: 537 self.initVar(var) 538 539 def hasOverrides(self, var): 540 return var in self.overridedata 541 542 def setVar(self, var, value, **loginfo): 543 #print("var=" + str(var) + " val=" + str(value)) 544 545 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): 546 info = "%s" % var 547 if "file" in loginfo: 548 info += " file: %s" % loginfo["file"] 549 if "line" in loginfo: 550 info += " line: %s" % loginfo["line"] 551 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) 552 553 shortvar = var.split(":", 1)[0] 554 if shortvar in self._var_renames: 555 _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) 556 # Mark that we have seen a renamed variable 557 self.setVar("_FAILPARSINGERRORHANDLED", True) 558 559 self.expand_cache = {} 560 parsing=False 561 if 'parsing' in loginfo: 562 parsing=True 563 564 if 'op' not in loginfo: 565 loginfo['op'] = "set" 566 567 match = __setvar_regexp__.match(var) 568 if match and match.group("keyword") in __setvar_keyword__: 569 base = match.group('base') 570 keyword = match.group("keyword") 571 override = match.group('add') 572 l = self.getVarFlag(base, keyword, False) or [] 573 l.append([value, override]) 574 self.setVarFlag(base, keyword, l, ignore=True) 575 # And cause that to be recorded: 576 loginfo['detail'] = value 577 loginfo['variable'] = base 578 if override: 579 loginfo['op'] = '%s[%s]' % (keyword, override) 580 else: 581 loginfo['op'] = keyword 582 self.varhistory.record(**loginfo) 583 # pay the cookie monster 584 585 # more cookies for the cookie monster 586 self._setvar_update_overrides(base, **loginfo) 587 588 if base in self.overridevars: 589 self._setvar_update_overridevars(var, value) 590 return 591 592 if not var in self.dict: 593 self._makeShadowCopy(var) 594 595 if not parsing: 596 if ":append" in self.dict[var]: 597 del self.dict[var][":append"] 598 if ":prepend" in self.dict[var]: 599 del self.dict[var][":prepend"] 600 if ":remove" in self.dict[var]: 601 del self.dict[var][":remove"] 602 if var in self.overridedata: 603 active = [] 604 self.need_overrides() 605 for (r, o) in self.overridedata[var]: 606 if o in self.overridesset: 607 active.append(r) 608 elif ":" in o: 609 if set(o.split(":")).issubset(self.overridesset): 610 active.append(r) 611 for a in active: 612 self.delVar(a) 613 del self.overridedata[var] 614 615 # more cookies for the cookie monster 616 if ':' in var: 617 self._setvar_update_overrides(var, **loginfo) 618 619 # setting var 620 self.dict[var]["_content"] = value 621 self.varhistory.record(**loginfo) 622 623 if var in self.overridevars: 624 self._setvar_update_overridevars(var, value) 625 626 def _setvar_update_overridevars(self, var, value): 627 vardata = self.expandWithRefs(value, var) 628 new = vardata.references 629 new.update(vardata.contains.keys()) 630 while not new.issubset(self.overridevars): 631 nextnew = set() 632 self.overridevars.update(new) 633 for i in new: 634 vardata = self.expandWithRefs(self.getVar(i), i) 635 nextnew.update(vardata.references) 636 nextnew.update(vardata.contains.keys()) 637 new = nextnew 638 self.overrides = None 639 self.expand_cache = {} 640 641 def _setvar_update_overrides(self, var, **loginfo): 642 # aka pay the cookie monster 643 override = var[var.rfind(':')+1:] 644 shortvar = var[:var.rfind(':')] 645 while override and __override_regexp__.match(override): 646 if shortvar not in self.overridedata: 647 self.overridedata[shortvar] = [] 648 if [var, override] not in self.overridedata[shortvar]: 649 # Force CoW by recreating the list first 650 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 651 self.overridedata[shortvar].append([var, override]) 652 override = None 653 if ":" in shortvar: 654 override = var[shortvar.rfind(':')+1:] 655 shortvar = var[:shortvar.rfind(':')] 656 if len(shortvar) == 0: 657 override = None 658 659 def getVar(self, var, expand=True, noweakdefault=False, parsing=False): 660 return self.getVarFlag(var, "_content", expand, noweakdefault, parsing) 661 662 def renameVar(self, key, newkey, **loginfo): 663 """ 664 Rename the variable key to newkey 665 """ 666 if key == newkey: 667 bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) 668 return 669 670 val = self.getVar(key, 0, parsing=True) 671 if val is not None: 672 self.varhistory.rename_variable_hist(key, newkey) 673 loginfo['variable'] = newkey 674 loginfo['op'] = 'rename from %s' % key 675 loginfo['detail'] = val 676 self.varhistory.record(**loginfo) 677 self.setVar(newkey, val, ignore=True, parsing=True) 678 679 srcflags = self.getVarFlags(key, False, True) or {} 680 for i in srcflags: 681 if i not in (__setvar_keyword__): 682 continue 683 src = srcflags[i] 684 685 dest = self.getVarFlag(newkey, i, False) or [] 686 dest.extend(src) 687 self.setVarFlag(newkey, i, dest, ignore=True) 688 689 if key in self.overridedata: 690 self.overridedata[newkey] = [] 691 for (v, o) in self.overridedata[key]: 692 self.overridedata[newkey].append([v.replace(key, newkey), o]) 693 self.renameVar(v, v.replace(key, newkey)) 694 695 if ':' in newkey and val is None: 696 self._setvar_update_overrides(newkey, **loginfo) 697 698 loginfo['variable'] = key 699 loginfo['op'] = 'rename (to)' 700 loginfo['detail'] = newkey 701 self.varhistory.record(**loginfo) 702 self.delVar(key, ignore=True) 703 704 def appendVar(self, var, value, **loginfo): 705 loginfo['op'] = 'append' 706 self.varhistory.record(**loginfo) 707 self.setVar(var + ":append", value, ignore=True, parsing=True) 708 709 def prependVar(self, var, value, **loginfo): 710 loginfo['op'] = 'prepend' 711 self.varhistory.record(**loginfo) 712 self.setVar(var + ":prepend", value, ignore=True, parsing=True) 713 714 def delVar(self, var, **loginfo): 715 self.expand_cache = {} 716 717 loginfo['detail'] = "" 718 loginfo['op'] = 'del' 719 self.varhistory.record(**loginfo) 720 self.dict[var] = {} 721 if var in self.overridedata: 722 del self.overridedata[var] 723 if ':' in var: 724 override = var[var.rfind(':')+1:] 725 shortvar = var[:var.rfind(':')] 726 while override and __override_regexp__.match(override): 727 try: 728 if shortvar in self.overridedata: 729 # Force CoW by recreating the list first 730 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 731 self.overridedata[shortvar].remove([var, override]) 732 except ValueError as e: 733 pass 734 override = None 735 if ":" in shortvar: 736 override = var[shortvar.rfind(':')+1:] 737 shortvar = var[:shortvar.rfind(':')] 738 if len(shortvar) == 0: 739 override = None 740 741 def setVarFlag(self, var, flag, value, **loginfo): 742 self.expand_cache = {} 743 744 if var == "BB_RENAMED_VARIABLES": 745 self._var_renames[flag] = value 746 747 if var in self._var_renames: 748 _print_rename_error(var, loginfo, self._var_renames) 749 # Mark that we have seen a renamed variable 750 self.setVar("_FAILPARSINGERRORHANDLED", True) 751 752 if 'op' not in loginfo: 753 loginfo['op'] = "set" 754 loginfo['flag'] = flag 755 self.varhistory.record(**loginfo) 756 if not var in self.dict: 757 self._makeShadowCopy(var) 758 self.dict[var][flag] = value 759 760 if flag == "_defaultval" and ':' in var: 761 self._setvar_update_overrides(var, **loginfo) 762 if flag == "_defaultval" and var in self.overridevars: 763 self._setvar_update_overridevars(var, value) 764 765 if flag == "unexport" or flag == "export": 766 if not "__exportlist" in self.dict: 767 self._makeShadowCopy("__exportlist") 768 if not "_content" in self.dict["__exportlist"]: 769 self.dict["__exportlist"]["_content"] = set() 770 self.dict["__exportlist"]["_content"].add(var) 771 772 def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False): 773 if flag == "_content": 774 cachename = var 775 else: 776 if not flag: 777 bb.warn("Calling getVarFlag with flag unset is invalid") 778 return None 779 cachename = var + "[" + flag + "]" 780 781 if not expand and retparser and cachename in self.expand_cache: 782 return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] 783 784 if expand and cachename in self.expand_cache: 785 return self.expand_cache[cachename].value 786 787 local_var = self._findVar(var) 788 value = None 789 removes = set() 790 if flag == "_content" and not parsing: 791 overridedata = self.overridedata.get(var, None) 792 if flag == "_content" and not parsing and overridedata is not None: 793 match = False 794 active = {} 795 self.need_overrides() 796 for (r, o) in overridedata: 797 # FIXME What about double overrides both with "_" in the name? 798 if o in self.overridesset: 799 active[o] = r 800 elif ":" in o: 801 if set(o.split(":")).issubset(self.overridesset): 802 active[o] = r 803 804 mod = True 805 while mod: 806 mod = False 807 for o in self.overrides: 808 for a in active.copy(): 809 if a.endswith(":" + o): 810 t = active[a] 811 del active[a] 812 active[a.replace(":" + o, "")] = t 813 mod = True 814 elif a == o: 815 match = active[a] 816 del active[a] 817 if match: 818 value, subparser = self.getVarFlag(match, "_content", False, retparser=True) 819 if hasattr(subparser, "removes"): 820 # We have to carry the removes from the overridden variable to apply at the 821 # end of processing 822 removes = subparser.removes 823 824 if local_var is not None and value is None: 825 if flag in local_var: 826 value = copy.copy(local_var[flag]) 827 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 828 value = copy.copy(local_var["_defaultval"]) 829 elif "_defaultval_flag_"+flag in local_var and not noweakdefault: 830 value = copy.copy(local_var["_defaultval_flag_"+flag]) 831 832 833 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: 834 self.need_overrides() 835 for (r, o) in local_var[":append"]: 836 match = True 837 if o: 838 for o2 in o.split(":"): 839 if not o2 in self.overrides: 840 match = False 841 if match: 842 if value is None: 843 value = "" 844 value = value + r 845 846 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: 847 self.need_overrides() 848 for (r, o) in local_var[":prepend"]: 849 850 match = True 851 if o: 852 for o2 in o.split(":"): 853 if not o2 in self.overrides: 854 match = False 855 if match: 856 if value is None: 857 value = "" 858 value = r + value 859 860 parser = None 861 if expand or retparser: 862 parser = self.expandWithRefs(value, cachename) 863 if expand: 864 value = parser.value 865 866 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: 867 self.need_overrides() 868 for (r, o) in local_var[":remove"]: 869 match = True 870 if o: 871 for o2 in o.split(":"): 872 if not o2 in self.overrides: 873 match = False 874 if match: 875 removes.add(r) 876 877 if value and flag == "_content" and not parsing: 878 if removes and parser: 879 expanded_removes = {} 880 for r in removes: 881 expanded_removes[r] = self.expand(r).split() 882 883 parser.removes = set() 884 val = [] 885 for v in __whitespace_split__.split(parser.value): 886 skip = False 887 for r in removes: 888 if v in expanded_removes[r]: 889 parser.removes.add(r) 890 skip = True 891 if skip: 892 continue 893 val.append(v) 894 parser.value = "".join(val) 895 if expand: 896 value = parser.value 897 898 if parser: 899 self.expand_cache[cachename] = parser 900 901 if retparser: 902 return value, parser 903 904 return value 905 906 def delVarFlag(self, var, flag, **loginfo): 907 self.expand_cache = {} 908 909 local_var = self._findVar(var) 910 if not local_var: 911 return 912 if not var in self.dict: 913 self._makeShadowCopy(var) 914 915 if var in self.dict and flag in self.dict[var]: 916 loginfo['detail'] = "" 917 loginfo['op'] = 'delFlag' 918 loginfo['flag'] = flag 919 self.varhistory.record(**loginfo) 920 921 del self.dict[var][flag] 922 if ("_defaultval_flag_" + flag) in self.dict[var]: 923 del self.dict[var]["_defaultval_flag_" + flag] 924 925 def appendVarFlag(self, var, flag, value, **loginfo): 926 loginfo['op'] = 'append' 927 loginfo['flag'] = flag 928 self.varhistory.record(**loginfo) 929 newvalue = (self.getVarFlag(var, flag, False) or "") + value 930 self.setVarFlag(var, flag, newvalue, ignore=True) 931 932 def prependVarFlag(self, var, flag, value, **loginfo): 933 loginfo['op'] = 'prepend' 934 loginfo['flag'] = flag 935 self.varhistory.record(**loginfo) 936 newvalue = value + (self.getVarFlag(var, flag, False) or "") 937 self.setVarFlag(var, flag, newvalue, ignore=True) 938 939 def setVarFlags(self, var, flags, **loginfo): 940 self.expand_cache = {} 941 infer_caller_details(loginfo) 942 if not var in self.dict: 943 self._makeShadowCopy(var) 944 945 for i in flags: 946 if i == "_content": 947 continue 948 loginfo['flag'] = i 949 loginfo['detail'] = flags[i] 950 self.varhistory.record(**loginfo) 951 self.dict[var][i] = flags[i] 952 953 def getVarFlags(self, var, expand = False, internalflags=False): 954 local_var = self._findVar(var) 955 flags = {} 956 957 if local_var: 958 for i, val in local_var.items(): 959 if i.startswith("_defaultval_flag_") and not internalflags: 960 i = i[len("_defaultval_flag_"):] 961 if i not in local_var: 962 flags[i] = val 963 elif i.startswith(("_", ":")) and not internalflags: 964 continue 965 else: 966 flags[i] = val 967 968 if expand and i in expand: 969 flags[i] = self.expand(flags[i], var + "[" + i + "]") 970 if len(flags) == 0: 971 return None 972 return flags 973 974 def delVarFlags(self, var, **loginfo): 975 self.expand_cache = {} 976 if not var in self.dict: 977 self._makeShadowCopy(var) 978 979 if var in self.dict: 980 content = None 981 982 loginfo['op'] = 'delete flags' 983 self.varhistory.record(**loginfo) 984 985 # try to save the content 986 if "_content" in self.dict[var]: 987 content = self.dict[var]["_content"] 988 self.dict[var] = {} 989 self.dict[var]["_content"] = content 990 else: 991 del self.dict[var] 992 993 def createCopy(self): 994 """ 995 Create a copy of self by setting _data to self 996 """ 997 # we really want this to be a DataSmart... 998 data = DataSmart() 999 data.dict["_data"] = self.dict 1000 data.varhistory = self.varhistory.copy() 1001 data.varhistory.dataroot = data 1002 data.inchistory = self.inchistory.copy() 1003 1004 data._tracking = self._tracking 1005 data._var_renames = self._var_renames 1006 1007 data.overrides = None 1008 data.overridevars = copy.copy(self.overridevars) 1009 # Should really be a deepcopy but has heavy overhead. 1010 # Instead, we're careful with writes. 1011 data.overridedata = copy.copy(self.overridedata) 1012 1013 return data 1014 1015 def expandVarref(self, variable, parents=False): 1016 """Find all references to variable in the data and expand it 1017 in place, optionally descending to parent datastores.""" 1018 1019 if parents: 1020 keys = iter(self) 1021 else: 1022 keys = self.localkeys() 1023 1024 ref = '${%s}' % variable 1025 value = self.getVar(variable, False) 1026 for key in keys: 1027 referrervalue = self.getVar(key, False) 1028 if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: 1029 self.setVar(key, referrervalue.replace(ref, value)) 1030 1031 def localkeys(self): 1032 for key in self.dict: 1033 if key not in ['_data']: 1034 yield key 1035 1036 def __iter__(self): 1037 deleted = set() 1038 overrides = set() 1039 def keylist(d): 1040 klist = set() 1041 for key in d: 1042 if key in ["_data"]: 1043 continue 1044 if key in deleted: 1045 continue 1046 if key in overrides: 1047 continue 1048 if not d[key]: 1049 deleted.add(key) 1050 continue 1051 klist.add(key) 1052 1053 if "_data" in d: 1054 klist |= keylist(d["_data"]) 1055 1056 return klist 1057 1058 self.need_overrides() 1059 for var in self.overridedata: 1060 for (r, o) in self.overridedata[var]: 1061 if o in self.overridesset: 1062 overrides.add(var) 1063 elif ":" in o: 1064 if set(o.split(":")).issubset(self.overridesset): 1065 overrides.add(var) 1066 1067 for k in keylist(self.dict): 1068 yield k 1069 1070 for k in overrides: 1071 yield k 1072 1073 def __len__(self): 1074 return len(frozenset(iter(self))) 1075 1076 def __getitem__(self, item): 1077 value = self.getVar(item, False) 1078 if value is None: 1079 raise KeyError(item) 1080 else: 1081 return value 1082 1083 def __setitem__(self, var, value): 1084 self.setVar(var, value) 1085 1086 def __delitem__(self, var): 1087 self.delVar(var) 1088 1089 def get_hash(self): 1090 data = {} 1091 d = self.createCopy() 1092 bb.data.expandKeys(d) 1093 1094 config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) 1095 keys = set(key for key in iter(d) if not key.startswith("__")) 1096 for key in keys: 1097 if key in config_ignore_vars: 1098 continue 1099 1100 value = d.getVar(key, False) or "" 1101 if type(value) is type(self): 1102 data.update({key:value.get_hash()}) 1103 else: 1104 data.update({key:value}) 1105 1106 varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"]) 1107 if not varflags: 1108 continue 1109 for f in varflags: 1110 if f == "_content": 1111 continue 1112 data.update({'%s[%s]' % (key, f):varflags[f]}) 1113 1114 for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: 1115 bb_list = d.getVar(key, False) or [] 1116 data.update({key:str(bb_list)}) 1117 1118 if key == "__BBANONFUNCS": 1119 for i in bb_list: 1120 value = d.getVar(i, False) or "" 1121 data.update({i:value}) 1122 1123 moddeps = bb.codeparser.modulecode_deps 1124 for dep in sorted(moddeps): 1125 # Ignore visitor code, sort sets 1126 data.update({'moddep[%s]' % dep : [sorted(moddeps[dep][0]), sorted(moddeps[dep][1]), sorted(moddeps[dep][2]), sorted(moddeps[dep][3]), moddeps[dep][4]]}) 1127 1128 data_str = str([(k, data[k]) for k in sorted(data.keys())]) 1129 return hashlib.sha256(data_str.encode("utf-8")).hexdigest() 1130