1"""
2BitBake Smart Dictionary Implementation
3
4Functions for interacting with the data structure used by the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004  Chris Larson
10# Copyright (C) 2004, 2005  Seb Frankengul
11# Copyright (C) 2005, 2006  Holger Hans Peter Freyther
12# Copyright (C) 2005        Uli Luckas
13# Copyright (C) 2005        ROAD GmbH
14#
15# SPDX-License-Identifier: GPL-2.0-only
16#
17# Based on functions from the base bb module, Copyright 2003 Holger Schurig
18
19import copy, re, sys, traceback
20from collections.abc import MutableMapping
21import logging
22import hashlib
23import bb, bb.codeparser
24from bb   import utils
25from bb.COW  import COWDictBase
26
27logger = logging.getLogger("BitBake.Data")
28
29__setvar_keyword__ = [":append", ":prepend", ":remove"]
30__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
32__expand_python_regexp__ = re.compile(r"\${@.+?}")
33__whitespace_split__ = re.compile(r'(\s)')
34__override_regexp__ = re.compile(r'[a-z0-9]+')
35
36bitbake_renamed_vars = {
37    "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH",
38    "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS",
39    "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS",
40    "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS",
41    "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS",
42    "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS",
43    "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED",
44    "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed",
45    "BB_STAMP_POLICY": "is a deprecated variable and support has been removed",
46}
47
48def infer_caller_details(loginfo, parent = False, varval = True):
49    """Save the caller the trouble of specifying everything."""
50    # Save effort.
51    if 'ignore' in loginfo and loginfo['ignore']:
52        return
53    # If nothing was provided, mark this as possibly unneeded.
54    if not loginfo:
55        loginfo['ignore'] = True
56        return
57    # Infer caller's likely values for variable (var) and value (value),
58    # to reduce clutter in the rest of the code.
59    above = None
60    def set_above():
61        try:
62            raise Exception
63        except Exception:
64            tb = sys.exc_info()[2]
65            if parent:
66                return tb.tb_frame.f_back.f_back.f_back
67            else:
68                return tb.tb_frame.f_back.f_back
69
70    if varval and ('variable' not in loginfo or 'detail' not in loginfo):
71        if not above:
72            above = set_above()
73        lcls = above.f_locals.items()
74        for k, v in lcls:
75            if k == 'value' and 'detail' not in loginfo:
76                loginfo['detail'] = v
77            if k == 'var' and 'variable' not in loginfo:
78                loginfo['variable'] = v
79    # Infer file/line/function from traceback
80    # Don't use traceback.extract_stack() since it fills the line contents which
81    # we don't need and that hits stat syscalls
82    if 'file' not in loginfo:
83        if not above:
84            above = set_above()
85        f = above.f_back
86        line = f.f_lineno
87        file = f.f_code.co_filename
88        func = f.f_code.co_name
89        loginfo['file'] = file
90        loginfo['line'] = line
91        if func not in loginfo:
92            loginfo['func'] = func
93
94class VariableParse:
95    def __init__(self, varname, d, val = None):
96        self.varname = varname
97        self.d = d
98        self.value = val
99
100        self.references = set()
101        self.execs = set()
102        self.contains = {}
103
104    def var_sub(self, match):
105            key = match.group()[2:-1]
106            if self.varname and key:
107                if self.varname == key:
108                    raise Exception("variable %s references itself!" % self.varname)
109            var = self.d.getVarFlag(key, "_content")
110            self.references.add(key)
111            if var is not None:
112                return var
113            else:
114                return match.group()
115
116    def python_sub(self, match):
117            if isinstance(match, str):
118                code = match
119            else:
120                code = match.group()[3:-1]
121
122            if self.varname:
123                varname = 'Var <%s>' % self.varname
124            else:
125                varname = '<expansion>'
126            codeobj = compile(code.strip(), varname, "eval")
127
128            parser = bb.codeparser.PythonParser(self.varname, logger)
129            parser.parse_python(code)
130            if self.varname:
131                vardeps = self.d.getVarFlag(self.varname, "vardeps")
132                if vardeps is None:
133                    parser.log.flush()
134            else:
135                parser.log.flush()
136            self.references |= parser.references
137            self.execs |= parser.execs
138
139            for k in parser.contains:
140                if k not in self.contains:
141                    self.contains[k] = parser.contains[k].copy()
142                else:
143                    self.contains[k].update(parser.contains[k])
144            value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
145            return str(value)
146
147
148class DataContext(dict):
149    def __init__(self, metadata, **kwargs):
150        self.metadata = metadata
151        dict.__init__(self, **kwargs)
152        self['d'] = metadata
153
154    def __missing__(self, key):
155        # Skip commonly accessed invalid variables
156        if key in ['bb', 'oe', 'int', 'bool', 'time', 'str', 'os']:
157            raise KeyError(key)
158        value = self.metadata.getVar(key)
159        if value is None or self.metadata.getVarFlag(key, 'func', False):
160            raise KeyError(key)
161        else:
162            return value
163
164class ExpansionError(Exception):
165    def __init__(self, varname, expression, exception):
166        self.expression = expression
167        self.variablename = varname
168        self.exception = exception
169        self.varlist = [varname or expression or ""]
170        if varname:
171            if expression:
172                self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
173            else:
174                self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
175        else:
176            self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
177        Exception.__init__(self, self.msg)
178        self.args = (varname, expression, exception)
179
180    def addVar(self, varname):
181        if varname:
182            self.varlist.append(varname)
183
184    def __str__(self):
185        chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist)
186        return self.msg + chain
187
188class IncludeHistory(object):
189    def __init__(self, parent = None, filename = '[TOP LEVEL]'):
190        self.parent = parent
191        self.filename = filename
192        self.children = []
193        self.current = self
194
195    def copy(self):
196        new = IncludeHistory(self.parent, self.filename)
197        for c in self.children:
198            new.children.append(c)
199        return new
200
201    def include(self, filename):
202        newfile = IncludeHistory(self.current, filename)
203        self.current.children.append(newfile)
204        self.current = newfile
205        return self
206
207    def __enter__(self):
208        pass
209
210    def __exit__(self, a, b, c):
211        if self.current.parent:
212            self.current = self.current.parent
213        else:
214            bb.warn("Include log: Tried to finish '%s' at top level." % self.filename)
215        return False
216
217    def emit(self, o, level = 0):
218        """Emit an include history file, and its children."""
219        if level:
220            spaces = "  " * (level - 1)
221            o.write("# %s%s" % (spaces, self.filename))
222            if len(self.children) > 0:
223                o.write(" includes:")
224        else:
225            o.write("#\n# INCLUDE HISTORY:\n#")
226        level = level + 1
227        for child in self.children:
228            o.write("\n")
229            child.emit(o, level)
230
231class VariableHistory(object):
232    def __init__(self, dataroot):
233        self.dataroot = dataroot
234        self.variables = COWDictBase.copy()
235
236    def copy(self):
237        new = VariableHistory(self.dataroot)
238        new.variables = self.variables.copy()
239        return new
240
241    def __getstate__(self):
242        vardict = {}
243        for k, v in self.variables.iteritems():
244            vardict[k] = v
245        return {'dataroot': self.dataroot,
246                'variables': vardict}
247
248    def __setstate__(self, state):
249        self.dataroot = state['dataroot']
250        self.variables = COWDictBase.copy()
251        for k, v in state['variables'].items():
252            self.variables[k] = v
253
254    def record(self, *kwonly, **loginfo):
255        if not self.dataroot._tracking:
256            return
257        if len(kwonly) > 0:
258            raise TypeError
259        infer_caller_details(loginfo, parent = True)
260        if 'ignore' in loginfo and loginfo['ignore']:
261            return
262        if 'op' not in loginfo or not loginfo['op']:
263            loginfo['op'] = 'set'
264        if 'detail' in loginfo:
265            loginfo['detail'] = str(loginfo['detail'])
266        if 'variable' not in loginfo or 'file' not in loginfo:
267            raise ValueError("record() missing variable or file.")
268        var = loginfo['variable']
269
270        if var not in self.variables:
271            self.variables[var] = []
272        if not isinstance(self.variables[var], list):
273            return
274        if 'nodups' in loginfo and loginfo in self.variables[var]:
275            return
276        self.variables[var].append(loginfo.copy())
277
278    def rename_variable_hist(self, oldvar, newvar):
279        if not self.dataroot._tracking:
280            return
281        if oldvar not in self.variables:
282            return
283        if newvar not in self.variables:
284            self.variables[newvar] = []
285        for i in self.variables[oldvar]:
286            self.variables[newvar].append(i.copy())
287
288    def variable(self, var):
289        varhistory = []
290        if var in self.variables:
291            varhistory.extend(self.variables[var])
292        return varhistory
293
294    def emit(self, var, oval, val, o, d):
295        history = self.variable(var)
296
297        # Append override history
298        if var in d.overridedata:
299            for (r, override) in d.overridedata[var]:
300                for event in self.variable(r):
301                    loginfo = event.copy()
302                    if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")):
303                        continue
304                    loginfo['variable'] = var
305                    loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
306                    history.append(loginfo)
307
308        commentVal = re.sub('\n', '\n#', str(oval))
309        if history:
310            if len(history) == 1:
311                o.write("#\n# $%s\n" % var)
312            else:
313                o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
314            for event in history:
315                # o.write("# %s\n" % str(event))
316                if 'func' in event:
317                    # If we have a function listed, this is internal
318                    # code, not an operation in a config file, and the
319                    # full path is distracting.
320                    event['file'] = re.sub('.*/', '', event['file'])
321                    display_func = ' [%s]' % event['func']
322                else:
323                    display_func = ''
324                if 'flag' in event:
325                    flag = '[%s] ' % (event['flag'])
326                else:
327                    flag = ''
328                o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', event['detail'])))
329            if len(history) > 1:
330                o.write("# pre-expansion value:\n")
331                o.write('#   "%s"\n' % (commentVal))
332        else:
333            o.write("#\n# $%s\n#   [no history recorded]\n#\n" % var)
334            o.write('#   "%s"\n' % (commentVal))
335
336    def get_variable_files(self, var):
337        """Get the files where operations are made on a variable"""
338        var_history = self.variable(var)
339        files = []
340        for event in var_history:
341            files.append(event['file'])
342        return files
343
344    def get_variable_lines(self, var, f):
345        """Get the line where a operation is made on a variable in file f"""
346        var_history = self.variable(var)
347        lines = []
348        for event in var_history:
349            if f== event['file']:
350                line = event['line']
351                lines.append(line)
352        return lines
353
354    def get_variable_refs(self, var):
355        """Return a dict of file/line references"""
356        var_history = self.variable(var)
357        refs = {}
358        for event in var_history:
359            if event['file'] not in refs:
360                refs[event['file']] = []
361            refs[event['file']].append(event['line'])
362        return refs
363
364    def get_variable_items_files(self, var):
365        """
366        Use variable history to map items added to a list variable and
367        the files in which they were added.
368        """
369        d = self.dataroot
370        history = self.variable(var)
371        finalitems = (d.getVar(var) or '').split()
372        filemap = {}
373        isset = False
374        for event in history:
375            if 'flag' in event:
376                continue
377            if event['op'] == ':remove':
378                continue
379            if isset and event['op'] == 'set?':
380                continue
381            isset = True
382            items = d.expand(event['detail']).split()
383            for item in items:
384                # This is a little crude but is belt-and-braces to avoid us
385                # having to handle every possible operation type specifically
386                if item in finalitems and not item in filemap:
387                    filemap[item] = event['file']
388        return filemap
389
390    def del_var_history(self, var, f=None, line=None):
391        """If file f and line are not given, the entire history of var is deleted"""
392        if var in self.variables:
393            if f and line:
394                self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
395            else:
396                self.variables[var] = []
397
398def _print_rename_error(var, loginfo, renamedvars, fullvar=None):
399    info = ""
400    if "file" in loginfo:
401        info = " file: %s" % loginfo["file"]
402    if "line" in loginfo:
403        info += " line: %s" % loginfo["line"]
404    if fullvar and fullvar != var:
405        info += " referenced as: %s" % fullvar
406    if info:
407        info = " (%s)" % info.strip()
408    renameinfo = renamedvars[var]
409    if " " in renameinfo:
410        # A space signals a string to display instead of a rename
411        bb.erroronce('Variable %s %s%s' % (var, renameinfo, info))
412    else:
413        bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info))
414
415class DataSmart(MutableMapping):
416    def __init__(self):
417        self.dict = {}
418
419        self.inchistory = IncludeHistory()
420        self.varhistory = VariableHistory(self)
421        self._tracking = False
422        self._var_renames = {}
423        self._var_renames.update(bitbake_renamed_vars)
424
425        self.expand_cache = {}
426
427        # cookie monster tribute
428        # Need to be careful about writes to overridedata as
429        # its only a shallow copy, could influence other data store
430        # copies!
431        self.overridedata = {}
432        self.overrides = None
433        self.overridevars = set(["OVERRIDES", "FILE"])
434        self.inoverride = False
435
436    def enableTracking(self):
437        self._tracking = True
438
439    def disableTracking(self):
440        self._tracking = False
441
442    def expandWithRefs(self, s, varname):
443
444        if not isinstance(s, str): # sanity check
445            return VariableParse(varname, self, s)
446
447        varparse = VariableParse(varname, self)
448
449        while s.find('${') != -1:
450            olds = s
451            try:
452                s = __expand_var_regexp__.sub(varparse.var_sub, s)
453                try:
454                    s = __expand_python_regexp__.sub(varparse.python_sub, s)
455                except SyntaxError as e:
456                    # Likely unmatched brackets, just don't expand the expression
457                    if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
458                        raise
459                if s == olds:
460                    break
461            except ExpansionError as e:
462                e.addVar(varname)
463                raise
464            except bb.parse.SkipRecipe:
465                raise
466            except bb.BBHandledException:
467                raise
468            except Exception as exc:
469                tb = sys.exc_info()[2]
470                raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
471
472        varparse.value = s
473
474        return varparse
475
476    def expand(self, s, varname = None):
477        return self.expandWithRefs(s, varname).value
478
479    def finalize(self, parent = False):
480        return
481
482    def internal_finalize(self, parent = False):
483        """Performs final steps upon the datastore, including application of overrides"""
484        self.overrides = None
485
486    def need_overrides(self):
487        if self.overrides is not None:
488            return
489        if self.inoverride:
490            return
491        for count in range(5):
492            self.inoverride = True
493            # Can end up here recursively so setup dummy values
494            self.overrides = []
495            self.overridesset = set()
496            self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
497            self.overridesset = set(self.overrides)
498            self.inoverride = False
499            self.expand_cache = {}
500            newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
501            if newoverrides == self.overrides:
502                break
503            self.overrides = newoverrides
504            self.overridesset = set(self.overrides)
505        else:
506            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
507
508    def initVar(self, var):
509        self.expand_cache = {}
510        if not var in self.dict:
511            self.dict[var] = {}
512
513    def _findVar(self, var):
514        dest = self.dict
515        while dest:
516            if var in dest:
517                return dest[var], self.overridedata.get(var, None)
518
519            if "_data" not in dest:
520                break
521            dest = dest["_data"]
522        return None, self.overridedata.get(var, None)
523
524    def _makeShadowCopy(self, var):
525        if var in self.dict:
526            return
527
528        local_var, _ = self._findVar(var)
529
530        if local_var:
531            self.dict[var] = copy.copy(local_var)
532        else:
533            self.initVar(var)
534
535    def hasOverrides(self, var):
536        return var in self.overridedata
537
538    def setVar(self, var, value, **loginfo):
539        #print("var=" + str(var) + "  val=" + str(value))
540
541        if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
542            info = "%s" % var
543            if "file" in loginfo:
544                info += " file: %s" % loginfo["file"]
545            if "line" in loginfo:
546                info += " line: %s" % loginfo["line"]
547            bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
548
549        shortvar = var.split(":", 1)[0]
550        if shortvar in self._var_renames:
551            _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var)
552            # Mark that we have seen a renamed variable
553            self.setVar("_FAILPARSINGERRORHANDLED", True)
554
555        self.expand_cache = {}
556        parsing=False
557        if 'parsing' in loginfo:
558            parsing=True
559
560        if 'op' not in loginfo:
561            loginfo['op'] = "set"
562
563        match  = __setvar_regexp__.match(var)
564        if match and match.group("keyword") in __setvar_keyword__:
565            base = match.group('base')
566            keyword = match.group("keyword")
567            override = match.group('add')
568            l = self.getVarFlag(base, keyword, False) or []
569            l.append([value, override])
570            self.setVarFlag(base, keyword, l, ignore=True)
571            # And cause that to be recorded:
572            loginfo['detail'] = value
573            loginfo['variable'] = base
574            if override:
575                loginfo['op'] = '%s[%s]' % (keyword, override)
576            else:
577                loginfo['op'] = keyword
578            self.varhistory.record(**loginfo)
579            # todo make sure keyword is not __doc__ or __module__
580            # pay the cookie monster
581
582            # more cookies for the cookie monster
583            if ':' in var:
584                self._setvar_update_overrides(base, **loginfo)
585
586            if base in self.overridevars:
587                self._setvar_update_overridevars(var, value)
588            return
589
590        if not var in self.dict:
591            self._makeShadowCopy(var)
592
593        if not parsing:
594            if ":append" in self.dict[var]:
595                del self.dict[var][":append"]
596            if ":prepend" in self.dict[var]:
597                del self.dict[var][":prepend"]
598            if ":remove" in self.dict[var]:
599                del self.dict[var][":remove"]
600            if var in self.overridedata:
601                active = []
602                self.need_overrides()
603                for (r, o) in self.overridedata[var]:
604                    if o in self.overridesset:
605                        active.append(r)
606                    elif ":" in o:
607                        if set(o.split(":")).issubset(self.overridesset):
608                            active.append(r)
609                for a in active:
610                    self.delVar(a)
611                del self.overridedata[var]
612
613        # more cookies for the cookie monster
614        if ':' in var:
615            self._setvar_update_overrides(var, **loginfo)
616
617        # setting var
618        self.dict[var]["_content"] = value
619        self.varhistory.record(**loginfo)
620
621        if var in self.overridevars:
622            self._setvar_update_overridevars(var, value)
623
624    def _setvar_update_overridevars(self, var, value):
625        vardata = self.expandWithRefs(value, var)
626        new = vardata.references
627        new.update(vardata.contains.keys())
628        while not new.issubset(self.overridevars):
629            nextnew = set()
630            self.overridevars.update(new)
631            for i in new:
632                vardata = self.expandWithRefs(self.getVar(i), i)
633                nextnew.update(vardata.references)
634                nextnew.update(vardata.contains.keys())
635            new = nextnew
636        self.internal_finalize(True)
637
638    def _setvar_update_overrides(self, var, **loginfo):
639        # aka pay the cookie monster
640        override = var[var.rfind(':')+1:]
641        shortvar = var[:var.rfind(':')]
642        while override and __override_regexp__.match(override):
643            if shortvar not in self.overridedata:
644                self.overridedata[shortvar] = []
645            if [var, override] not in self.overridedata[shortvar]:
646                # Force CoW by recreating the list first
647                self.overridedata[shortvar] = list(self.overridedata[shortvar])
648                self.overridedata[shortvar].append([var, override])
649            override = None
650            if ":" in shortvar:
651                override = var[shortvar.rfind(':')+1:]
652                shortvar = var[:shortvar.rfind(':')]
653                if len(shortvar) == 0:
654                    override = None
655
656    def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
657        return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
658
659    def renameVar(self, key, newkey, **loginfo):
660        """
661        Rename the variable key to newkey
662        """
663        if key == newkey:
664            bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
665            return
666
667        val = self.getVar(key, 0, parsing=True)
668        if val is not None:
669            self.varhistory.rename_variable_hist(key, newkey)
670            loginfo['variable'] = newkey
671            loginfo['op'] = 'rename from %s' % key
672            loginfo['detail'] = val
673            self.varhistory.record(**loginfo)
674            self.setVar(newkey, val, ignore=True, parsing=True)
675
676        srcflags = self.getVarFlags(key, False, True) or {}
677        for i in srcflags:
678            if i not in (__setvar_keyword__):
679                continue
680            src = srcflags[i]
681
682            dest = self.getVarFlag(newkey, i, False) or []
683            dest.extend(src)
684            self.setVarFlag(newkey, i, dest, ignore=True)
685
686        if key in self.overridedata:
687            self.overridedata[newkey] = []
688            for (v, o) in self.overridedata[key]:
689                self.overridedata[newkey].append([v.replace(key, newkey), o])
690                self.renameVar(v, v.replace(key, newkey))
691
692        if ':' in newkey and val is None:
693            self._setvar_update_overrides(newkey, **loginfo)
694
695        loginfo['variable'] = key
696        loginfo['op'] = 'rename (to)'
697        loginfo['detail'] = newkey
698        self.varhistory.record(**loginfo)
699        self.delVar(key, ignore=True)
700
701    def appendVar(self, var, value, **loginfo):
702        loginfo['op'] = 'append'
703        self.varhistory.record(**loginfo)
704        self.setVar(var + ":append", value, ignore=True, parsing=True)
705
706    def prependVar(self, var, value, **loginfo):
707        loginfo['op'] = 'prepend'
708        self.varhistory.record(**loginfo)
709        self.setVar(var + ":prepend", value, ignore=True, parsing=True)
710
711    def delVar(self, var, **loginfo):
712        self.expand_cache = {}
713
714        loginfo['detail'] = ""
715        loginfo['op'] = 'del'
716        self.varhistory.record(**loginfo)
717        self.dict[var] = {}
718        if var in self.overridedata:
719            del self.overridedata[var]
720        if ':' in var:
721            override = var[var.rfind(':')+1:]
722            shortvar = var[:var.rfind(':')]
723            while override and override.islower():
724                try:
725                    if shortvar in self.overridedata:
726                        # Force CoW by recreating the list first
727                        self.overridedata[shortvar] = list(self.overridedata[shortvar])
728                        self.overridedata[shortvar].remove([var, override])
729                except ValueError as e:
730                    pass
731                override = None
732                if ":" in shortvar:
733                    override = var[shortvar.rfind(':')+1:]
734                    shortvar = var[:shortvar.rfind(':')]
735                    if len(shortvar) == 0:
736                         override = None
737
738    def setVarFlag(self, var, flag, value, **loginfo):
739        self.expand_cache = {}
740
741        if var == "BB_RENAMED_VARIABLES":
742            self._var_renames[flag] = value
743
744        if var in self._var_renames:
745            _print_rename_error(var, loginfo, self._var_renames)
746            # Mark that we have seen a renamed variable
747            self.setVar("_FAILPARSINGERRORHANDLED", True)
748
749        if 'op' not in loginfo:
750            loginfo['op'] = "set"
751        loginfo['flag'] = flag
752        self.varhistory.record(**loginfo)
753        if not var in self.dict:
754            self._makeShadowCopy(var)
755        self.dict[var][flag] = value
756
757        if flag == "_defaultval" and ':' in var:
758            self._setvar_update_overrides(var, **loginfo)
759        if flag == "_defaultval" and var in self.overridevars:
760            self._setvar_update_overridevars(var, value)
761
762        if flag == "unexport" or flag == "export":
763            if not "__exportlist" in self.dict:
764                self._makeShadowCopy("__exportlist")
765            if not "_content" in self.dict["__exportlist"]:
766                self.dict["__exportlist"]["_content"] = set()
767            self.dict["__exportlist"]["_content"].add(var)
768
769    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
770        if flag == "_content":
771            cachename = var
772        else:
773            if not flag:
774                bb.warn("Calling getVarFlag with flag unset is invalid")
775                return None
776            cachename = var + "[" + flag + "]"
777
778        if expand and cachename in self.expand_cache:
779            return self.expand_cache[cachename].value
780
781        local_var, overridedata = self._findVar(var)
782        value = None
783        removes = set()
784        if flag == "_content" and overridedata is not None and not parsing:
785            match = False
786            active = {}
787            self.need_overrides()
788            for (r, o) in overridedata:
789                # FIXME What about double overrides both with "_" in the name?
790                if o in self.overridesset:
791                    active[o] = r
792                elif ":" in o:
793                    if set(o.split(":")).issubset(self.overridesset):
794                        active[o] = r
795
796            mod = True
797            while mod:
798                mod = False
799                for o in self.overrides:
800                    for a in active.copy():
801                        if a.endswith(":" + o):
802                            t = active[a]
803                            del active[a]
804                            active[a.replace(":" + o, "")] = t
805                            mod = True
806                        elif a == o:
807                            match = active[a]
808                            del active[a]
809            if match:
810                value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
811                if hasattr(subparser, "removes"):
812                    # We have to carry the removes from the overridden variable to apply at the
813                    # end of processing
814                    removes = subparser.removes
815
816        if local_var is not None and value is None:
817            if flag in local_var:
818                value = copy.copy(local_var[flag])
819            elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
820                value = copy.copy(local_var["_defaultval"])
821
822
823        if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
824            self.need_overrides()
825            for (r, o) in local_var[":append"]:
826                match = True
827                if o:
828                    for o2 in o.split(":"):
829                        if not o2 in self.overrides:
830                            match = False
831                if match:
832                    if value is None:
833                        value = ""
834                    value = value + r
835
836        if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing:
837            self.need_overrides()
838            for (r, o) in local_var[":prepend"]:
839
840                match = True
841                if o:
842                    for o2 in o.split(":"):
843                        if not o2 in self.overrides:
844                            match = False
845                if match:
846                    if value is None:
847                        value = ""
848                    value = r + value
849
850        parser = None
851        if expand or retparser:
852            parser = self.expandWithRefs(value, cachename)
853        if expand:
854            value = parser.value
855
856        if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing:
857            self.need_overrides()
858            for (r, o) in local_var[":remove"]:
859                match = True
860                if o:
861                    for o2 in o.split(":"):
862                        if not o2 in self.overrides:
863                            match = False
864                if match:
865                    removes.add(r)
866
867        if value and flag == "_content" and not parsing:
868            if removes and parser:
869                expanded_removes = {}
870                for r in removes:
871                    expanded_removes[r] = self.expand(r).split()
872
873                parser.removes = set()
874                val = []
875                for v in __whitespace_split__.split(parser.value):
876                    skip = False
877                    for r in removes:
878                        if v in expanded_removes[r]:
879                            parser.removes.add(r)
880                            skip = True
881                    if skip:
882                        continue
883                    val.append(v)
884                parser.value = "".join(val)
885                if expand:
886                    value = parser.value
887
888        if parser:
889            self.expand_cache[cachename] = parser
890
891        if retparser:
892            return value, parser
893
894        return value
895
896    def delVarFlag(self, var, flag, **loginfo):
897        self.expand_cache = {}
898
899        local_var, _ = self._findVar(var)
900        if not local_var:
901            return
902        if not var in self.dict:
903            self._makeShadowCopy(var)
904
905        if var in self.dict and flag in self.dict[var]:
906            loginfo['detail'] = ""
907            loginfo['op'] = 'delFlag'
908            loginfo['flag'] = flag
909            self.varhistory.record(**loginfo)
910
911            del self.dict[var][flag]
912
913    def appendVarFlag(self, var, flag, value, **loginfo):
914        loginfo['op'] = 'append'
915        loginfo['flag'] = flag
916        self.varhistory.record(**loginfo)
917        newvalue = (self.getVarFlag(var, flag, False) or "") + value
918        self.setVarFlag(var, flag, newvalue, ignore=True)
919
920    def prependVarFlag(self, var, flag, value, **loginfo):
921        loginfo['op'] = 'prepend'
922        loginfo['flag'] = flag
923        self.varhistory.record(**loginfo)
924        newvalue = value + (self.getVarFlag(var, flag, False) or "")
925        self.setVarFlag(var, flag, newvalue, ignore=True)
926
927    def setVarFlags(self, var, flags, **loginfo):
928        self.expand_cache = {}
929        infer_caller_details(loginfo)
930        if not var in self.dict:
931            self._makeShadowCopy(var)
932
933        for i in flags:
934            if i == "_content":
935                continue
936            loginfo['flag'] = i
937            loginfo['detail'] = flags[i]
938            self.varhistory.record(**loginfo)
939            self.dict[var][i] = flags[i]
940
941    def getVarFlags(self, var, expand = False, internalflags=False):
942        local_var, _ = self._findVar(var)
943        flags = {}
944
945        if local_var:
946            for i in local_var:
947                if i.startswith(("_", ":")) and not internalflags:
948                    continue
949                flags[i] = local_var[i]
950                if expand and i in expand:
951                    flags[i] = self.expand(flags[i], var + "[" + i + "]")
952        if len(flags) == 0:
953            return None
954        return flags
955
956
957    def delVarFlags(self, var, **loginfo):
958        self.expand_cache = {}
959        if not var in self.dict:
960            self._makeShadowCopy(var)
961
962        if var in self.dict:
963            content = None
964
965            loginfo['op'] = 'delete flags'
966            self.varhistory.record(**loginfo)
967
968            # try to save the content
969            if "_content" in self.dict[var]:
970                content  = self.dict[var]["_content"]
971                self.dict[var]            = {}
972                self.dict[var]["_content"] = content
973            else:
974                del self.dict[var]
975
976    def createCopy(self):
977        """
978        Create a copy of self by setting _data to self
979        """
980        # we really want this to be a DataSmart...
981        data = DataSmart()
982        data.dict["_data"] = self.dict
983        data.varhistory = self.varhistory.copy()
984        data.varhistory.dataroot = data
985        data.inchistory = self.inchistory.copy()
986
987        data._tracking = self._tracking
988        data._var_renames = self._var_renames
989
990        data.overrides = None
991        data.overridevars = copy.copy(self.overridevars)
992        # Should really be a deepcopy but has heavy overhead.
993        # Instead, we're careful with writes.
994        data.overridedata = copy.copy(self.overridedata)
995
996        return data
997
998    def expandVarref(self, variable, parents=False):
999        """Find all references to variable in the data and expand it
1000           in place, optionally descending to parent datastores."""
1001
1002        if parents:
1003            keys = iter(self)
1004        else:
1005            keys = self.localkeys()
1006
1007        ref = '${%s}' % variable
1008        value = self.getVar(variable, False)
1009        for key in keys:
1010            referrervalue = self.getVar(key, False)
1011            if referrervalue and isinstance(referrervalue, str) and ref in referrervalue:
1012                self.setVar(key, referrervalue.replace(ref, value))
1013
1014    def localkeys(self):
1015        for key in self.dict:
1016            if key not in ['_data']:
1017                yield key
1018
1019    def __iter__(self):
1020        deleted = set()
1021        overrides = set()
1022        def keylist(d):
1023            klist = set()
1024            for key in d:
1025                if key in ["_data"]:
1026                    continue
1027                if key in deleted:
1028                    continue
1029                if key in overrides:
1030                    continue
1031                if not d[key]:
1032                    deleted.add(key)
1033                    continue
1034                klist.add(key)
1035
1036            if "_data" in d:
1037                klist |= keylist(d["_data"])
1038
1039            return klist
1040
1041        self.need_overrides()
1042        for var in self.overridedata:
1043            for (r, o) in self.overridedata[var]:
1044                if o in self.overridesset:
1045                    overrides.add(var)
1046                elif ":" in o:
1047                    if set(o.split(":")).issubset(self.overridesset):
1048                        overrides.add(var)
1049
1050        for k in keylist(self.dict):
1051             yield k
1052
1053        for k in overrides:
1054             yield k
1055
1056    def __len__(self):
1057        return len(frozenset(iter(self)))
1058
1059    def __getitem__(self, item):
1060        value = self.getVar(item, False)
1061        if value is None:
1062            raise KeyError(item)
1063        else:
1064            return value
1065
1066    def __setitem__(self, var, value):
1067        self.setVar(var, value)
1068
1069    def __delitem__(self, var):
1070        self.delVar(var)
1071
1072    def get_hash(self):
1073        data = {}
1074        d = self.createCopy()
1075        bb.data.expandKeys(d)
1076
1077        config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split())
1078        keys = set(key for key in iter(d) if not key.startswith("__"))
1079        for key in keys:
1080            if key in config_ignore_vars:
1081                continue
1082
1083            value = d.getVar(key, False) or ""
1084            if type(value) is type(self):
1085                data.update({key:value.get_hash()})
1086            else:
1087                data.update({key:value})
1088
1089            varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"])
1090            if not varflags:
1091                continue
1092            for f in varflags:
1093                if f == "_content":
1094                    continue
1095                data.update({'%s[%s]' % (key, f):varflags[f]})
1096
1097        for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
1098            bb_list = d.getVar(key, False) or []
1099            data.update({key:str(bb_list)})
1100
1101            if key == "__BBANONFUNCS":
1102                for i in bb_list:
1103                    value = d.getVar(i, False) or ""
1104                    data.update({i:value})
1105
1106        data_str = str([(k, data[k]) for k in sorted(data.keys())])
1107        return hashlib.sha256(data_str.encode("utf-8")).hexdigest()
1108