1"""
2BitBake Smart Dictionary Implementation
3
4Functions for interacting with the data structure used by the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004  Chris Larson
10# Copyright (C) 2004, 2005  Seb Frankengul
11# Copyright (C) 2005, 2006  Holger Hans Peter Freyther
12# Copyright (C) 2005        Uli Luckas
13# Copyright (C) 2005        ROAD GmbH
14#
15# SPDX-License-Identifier: GPL-2.0-only
16#
17# Based on functions from the base bb module, Copyright 2003 Holger Schurig
18
19import copy, re, sys, traceback
20from collections.abc import MutableMapping
21import logging
22import hashlib
23import bb, bb.codeparser
24from bb   import utils
25from bb.COW  import COWDictBase
26
27logger = logging.getLogger("BitBake.Data")
28
29__setvar_keyword__ = [":append", ":prepend", ":remove"]
30__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
32__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}")
33__whitespace_split__ = re.compile(r'(\s)')
34__override_regexp__ = re.compile(r'[a-z0-9]+')
35
36bitbake_renamed_vars = {
37    "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH",
38    "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS",
39    "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS",
40    "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS",
41    "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS",
42    "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS",
43    "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED",
44    "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed",
45    "BB_STAMP_POLICY": "is a deprecated variable and support has been removed",
46}
47
48def infer_caller_details(loginfo, parent = False, varval = True):
49    """Save the caller the trouble of specifying everything."""
50    # Save effort.
51    if 'ignore' in loginfo and loginfo['ignore']:
52        return
53    # If nothing was provided, mark this as possibly unneeded.
54    if not loginfo:
55        loginfo['ignore'] = True
56        return
57    # Infer caller's likely values for variable (var) and value (value),
58    # to reduce clutter in the rest of the code.
59    above = None
60    def set_above():
61        try:
62            raise Exception
63        except Exception:
64            tb = sys.exc_info()[2]
65            if parent:
66                return tb.tb_frame.f_back.f_back.f_back
67            else:
68                return tb.tb_frame.f_back.f_back
69
70    if varval and ('variable' not in loginfo or 'detail' not in loginfo):
71        if not above:
72            above = set_above()
73        lcls = above.f_locals.items()
74        for k, v in lcls:
75            if k == 'value' and 'detail' not in loginfo:
76                loginfo['detail'] = v
77            if k == 'var' and 'variable' not in loginfo:
78                loginfo['variable'] = v
79    # Infer file/line/function from traceback
80    # Don't use traceback.extract_stack() since it fills the line contents which
81    # we don't need and that hits stat syscalls
82    if 'file' not in loginfo:
83        if not above:
84            above = set_above()
85        f = above.f_back
86        line = f.f_lineno
87        file = f.f_code.co_filename
88        func = f.f_code.co_name
89        loginfo['file'] = file
90        loginfo['line'] = line
91        if func not in loginfo:
92            loginfo['func'] = func
93
94class VariableParse:
95    def __init__(self, varname, d, unexpanded_value = None, val = None):
96        self.varname = varname
97        self.d = d
98        self.value = val
99        self.unexpanded_value = unexpanded_value
100
101        self.references = set()
102        self.execs = set()
103        self.contains = {}
104
105    def var_sub(self, match):
106            key = match.group()[2:-1]
107            if self.varname and key:
108                if self.varname == key:
109                    raise Exception("variable %s references itself!" % self.varname)
110            var = self.d.getVarFlag(key, "_content")
111            self.references.add(key)
112            if var is not None:
113                return var
114            else:
115                return match.group()
116
117    def python_sub(self, match):
118            if isinstance(match, str):
119                code = match
120            else:
121                code = match.group()[3:-1]
122
123            # Do not run code that contains one or more unexpanded variables
124            # instead return the code with the characters we removed put back
125            if __expand_var_regexp__.findall(code):
126                return "${@" + code + "}"
127
128            if self.varname:
129                varname = 'Var <%s>' % self.varname
130            else:
131                varname = '<expansion>'
132            codeobj = compile(code.strip(), varname, "eval")
133
134            parser = bb.codeparser.PythonParser(self.varname, logger)
135            parser.parse_python(code)
136            if self.varname:
137                vardeps = self.d.getVarFlag(self.varname, "vardeps")
138                if vardeps is None:
139                    parser.log.flush()
140            else:
141                parser.log.flush()
142            self.references |= parser.references
143            self.execs |= parser.execs
144
145            for k in parser.contains:
146                if k not in self.contains:
147                    self.contains[k] = parser.contains[k].copy()
148                else:
149                    self.contains[k].update(parser.contains[k])
150            value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
151            return str(value)
152
153
154class DataContext(dict):
155    def __init__(self, metadata, **kwargs):
156        self.metadata = metadata
157        dict.__init__(self, **kwargs)
158        self['d'] = metadata
159
160    def __missing__(self, key):
161        # Skip commonly accessed invalid variables
162        if key in ['bb', 'oe', 'int', 'bool', 'time', 'str', 'os']:
163            raise KeyError(key)
164        value = self.metadata.getVar(key)
165        if value is None or self.metadata.getVarFlag(key, 'func', False):
166            raise KeyError(key)
167        else:
168            return value
169
170class ExpansionError(Exception):
171    def __init__(self, varname, expression, exception):
172        self.expression = expression
173        self.variablename = varname
174        self.exception = exception
175        self.varlist = [varname or expression or ""]
176        if varname:
177            if expression:
178                self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
179            else:
180                self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
181        else:
182            self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
183        Exception.__init__(self, self.msg)
184        self.args = (varname, expression, exception)
185
186    def addVar(self, varname):
187        if varname:
188            self.varlist.append(varname)
189
190    def __str__(self):
191        chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist)
192        return self.msg + chain
193
194class IncludeHistory(object):
195    def __init__(self, parent = None, filename = '[TOP LEVEL]'):
196        self.parent = parent
197        self.filename = filename
198        self.children = []
199        self.current = self
200
201    def copy(self):
202        new = IncludeHistory(self.parent, self.filename)
203        for c in self.children:
204            new.children.append(c)
205        return new
206
207    def include(self, filename):
208        newfile = IncludeHistory(self.current, filename)
209        self.current.children.append(newfile)
210        self.current = newfile
211        return self
212
213    def __enter__(self):
214        pass
215
216    def __exit__(self, a, b, c):
217        if self.current.parent:
218            self.current = self.current.parent
219        else:
220            bb.warn("Include log: Tried to finish '%s' at top level." % self.filename)
221        return False
222
223    def emit(self, o, level = 0):
224        """Emit an include history file, and its children."""
225        if level:
226            spaces = "  " * (level - 1)
227            o.write("# %s%s" % (spaces, self.filename))
228            if len(self.children) > 0:
229                o.write(" includes:")
230        else:
231            o.write("#\n# INCLUDE HISTORY:\n#")
232        level = level + 1
233        for child in self.children:
234            o.write("\n")
235            child.emit(o, level)
236
237class VariableHistory(object):
238    def __init__(self, dataroot):
239        self.dataroot = dataroot
240        self.variables = COWDictBase.copy()
241
242    def copy(self):
243        new = VariableHistory(self.dataroot)
244        new.variables = self.variables.copy()
245        return new
246
247    def __getstate__(self):
248        vardict = {}
249        for k, v in self.variables.iteritems():
250            vardict[k] = v
251        return {'dataroot': self.dataroot,
252                'variables': vardict}
253
254    def __setstate__(self, state):
255        self.dataroot = state['dataroot']
256        self.variables = COWDictBase.copy()
257        for k, v in state['variables'].items():
258            self.variables[k] = v
259
260    def record(self, *kwonly, **loginfo):
261        if not self.dataroot._tracking:
262            return
263        if len(kwonly) > 0:
264            raise TypeError
265        infer_caller_details(loginfo, parent = True)
266        if 'ignore' in loginfo and loginfo['ignore']:
267            return
268        if 'op' not in loginfo or not loginfo['op']:
269            loginfo['op'] = 'set'
270        if 'detail' in loginfo:
271            loginfo['detail'] = str(loginfo['detail'])
272        if 'variable' not in loginfo or 'file' not in loginfo:
273            raise ValueError("record() missing variable or file.")
274        var = loginfo['variable']
275
276        if var not in self.variables:
277            self.variables[var] = []
278        if not isinstance(self.variables[var], list):
279            return
280        if 'nodups' in loginfo and loginfo in self.variables[var]:
281            return
282        self.variables[var].append(loginfo.copy())
283
284    def rename_variable_hist(self, oldvar, newvar):
285        if not self.dataroot._tracking:
286            return
287        if oldvar not in self.variables:
288            return
289        if newvar not in self.variables:
290            self.variables[newvar] = []
291        for i in self.variables[oldvar]:
292            self.variables[newvar].append(i.copy())
293
294    def variable(self, var):
295        varhistory = []
296        if var in self.variables:
297            varhistory.extend(self.variables[var])
298        return varhistory
299
300    def emit(self, var, oval, val, o, d):
301        history = self.variable(var)
302
303        # Append override history
304        if var in d.overridedata:
305            for (r, override) in d.overridedata[var]:
306                for event in self.variable(r):
307                    loginfo = event.copy()
308                    if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")):
309                        continue
310                    loginfo['variable'] = var
311                    loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
312                    history.append(loginfo)
313
314        commentVal = re.sub('\n', '\n#', str(oval))
315        if history:
316            if len(history) == 1:
317                o.write("#\n# $%s\n" % var)
318            else:
319                o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
320            for event in history:
321                # o.write("# %s\n" % str(event))
322                if 'func' in event:
323                    # If we have a function listed, this is internal
324                    # code, not an operation in a config file, and the
325                    # full path is distracting.
326                    event['file'] = re.sub('.*/', '', event['file'])
327                    display_func = ' [%s]' % event['func']
328                else:
329                    display_func = ''
330                if 'flag' in event:
331                    flag = '[%s] ' % (event['flag'])
332                else:
333                    flag = ''
334                o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', event['detail'])))
335            if len(history) > 1:
336                o.write("# pre-expansion value:\n")
337                o.write('#   "%s"\n' % (commentVal))
338        else:
339            o.write("#\n# $%s\n#   [no history recorded]\n#\n" % var)
340            o.write('#   "%s"\n' % (commentVal))
341
342    def get_variable_files(self, var):
343        """Get the files where operations are made on a variable"""
344        var_history = self.variable(var)
345        files = []
346        for event in var_history:
347            files.append(event['file'])
348        return files
349
350    def get_variable_lines(self, var, f):
351        """Get the line where a operation is made on a variable in file f"""
352        var_history = self.variable(var)
353        lines = []
354        for event in var_history:
355            if f== event['file']:
356                line = event['line']
357                lines.append(line)
358        return lines
359
360    def get_variable_refs(self, var):
361        """Return a dict of file/line references"""
362        var_history = self.variable(var)
363        refs = {}
364        for event in var_history:
365            if event['file'] not in refs:
366                refs[event['file']] = []
367            refs[event['file']].append(event['line'])
368        return refs
369
370    def get_variable_items_files(self, var):
371        """
372        Use variable history to map items added to a list variable and
373        the files in which they were added.
374        """
375        d = self.dataroot
376        history = self.variable(var)
377        finalitems = (d.getVar(var) or '').split()
378        filemap = {}
379        isset = False
380        for event in history:
381            if 'flag' in event:
382                continue
383            if event['op'] == ':remove':
384                continue
385            if isset and event['op'] == 'set?':
386                continue
387            isset = True
388            items = d.expand(event['detail']).split()
389            for item in items:
390                # This is a little crude but is belt-and-braces to avoid us
391                # having to handle every possible operation type specifically
392                if item in finalitems and not item in filemap:
393                    filemap[item] = event['file']
394        return filemap
395
396    def del_var_history(self, var, f=None, line=None):
397        """If file f and line are not given, the entire history of var is deleted"""
398        if var in self.variables:
399            if f and line:
400                self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
401            else:
402                self.variables[var] = []
403
404def _print_rename_error(var, loginfo, renamedvars, fullvar=None):
405    info = ""
406    if "file" in loginfo:
407        info = " file: %s" % loginfo["file"]
408    if "line" in loginfo:
409        info += " line: %s" % loginfo["line"]
410    if fullvar and fullvar != var:
411        info += " referenced as: %s" % fullvar
412    if info:
413        info = " (%s)" % info.strip()
414    renameinfo = renamedvars[var]
415    if " " in renameinfo:
416        # A space signals a string to display instead of a rename
417        bb.erroronce('Variable %s %s%s' % (var, renameinfo, info))
418    else:
419        bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info))
420
421class DataSmart(MutableMapping):
422    def __init__(self):
423        self.dict = {}
424
425        self.inchistory = IncludeHistory()
426        self.varhistory = VariableHistory(self)
427        self._tracking = False
428        self._var_renames = {}
429        self._var_renames.update(bitbake_renamed_vars)
430
431        self.expand_cache = {}
432
433        # cookie monster tribute
434        # Need to be careful about writes to overridedata as
435        # its only a shallow copy, could influence other data store
436        # copies!
437        self.overridedata = {}
438        self.overrides = None
439        self.overridevars = set(["OVERRIDES", "FILE"])
440        self.inoverride = False
441
442    def enableTracking(self):
443        self._tracking = True
444
445    def disableTracking(self):
446        self._tracking = False
447
448    def expandWithRefs(self, s, varname):
449
450        if not isinstance(s, str): # sanity check
451            return VariableParse(varname, self, s, s)
452
453        varparse = VariableParse(varname, self, s)
454
455        while s.find('${') != -1:
456            olds = s
457            try:
458                s = __expand_var_regexp__.sub(varparse.var_sub, s)
459                try:
460                    s = __expand_python_regexp__.sub(varparse.python_sub, s)
461                except SyntaxError as e:
462                    # Likely unmatched brackets, just don't expand the expression
463                    if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
464                        raise
465                if s == olds:
466                    break
467            except ExpansionError as e:
468                e.addVar(varname)
469                raise
470            except bb.parse.SkipRecipe:
471                raise
472            except bb.BBHandledException:
473                raise
474            except Exception as exc:
475                tb = sys.exc_info()[2]
476                raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
477
478        varparse.value = s
479
480        return varparse
481
482    def expand(self, s, varname = None):
483        return self.expandWithRefs(s, varname).value
484
485    def need_overrides(self):
486        if self.overrides is not None:
487            return
488        if self.inoverride:
489            return
490        overrride_stack = []
491        for count in range(5):
492            self.inoverride = True
493            # Can end up here recursively so setup dummy values
494            self.overrides = []
495            self.overridesset = set()
496            self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
497            overrride_stack.append(self.overrides)
498            self.overridesset = set(self.overrides)
499            self.inoverride = False
500            self.expand_cache = {}
501            newoverrides = (self.getVar("OVERRIDES") or "").split(":") or []
502            if newoverrides == self.overrides:
503                break
504            self.overrides = newoverrides
505            self.overridesset = set(self.overrides)
506        else:
507            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack))
508
509    def initVar(self, var):
510        self.expand_cache = {}
511        if not var in self.dict:
512            self.dict[var] = {}
513
514    def _findVar(self, var):
515        dest = self.dict
516        while dest:
517            if var in dest:
518                return dest[var]
519
520            if "_data" not in dest:
521                break
522            dest = dest["_data"]
523        return None
524
525    def _makeShadowCopy(self, var):
526        if var in self.dict:
527            return
528
529        local_var = self._findVar(var)
530
531        if local_var:
532            self.dict[var] = copy.copy(local_var)
533        else:
534            self.initVar(var)
535
536    def hasOverrides(self, var):
537        return var in self.overridedata
538
539    def setVar(self, var, value, **loginfo):
540        #print("var=" + str(var) + "  val=" + str(value))
541
542        if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
543            info = "%s" % var
544            if "file" in loginfo:
545                info += " file: %s" % loginfo["file"]
546            if "line" in loginfo:
547                info += " line: %s" % loginfo["line"]
548            bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
549
550        shortvar = var.split(":", 1)[0]
551        if shortvar in self._var_renames:
552            _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var)
553            # Mark that we have seen a renamed variable
554            self.setVar("_FAILPARSINGERRORHANDLED", True)
555
556        self.expand_cache = {}
557        parsing=False
558        if 'parsing' in loginfo:
559            parsing=True
560
561        if 'op' not in loginfo:
562            loginfo['op'] = "set"
563
564        match  = __setvar_regexp__.match(var)
565        if match and match.group("keyword") in __setvar_keyword__:
566            base = match.group('base')
567            keyword = match.group("keyword")
568            override = match.group('add')
569            l = self.getVarFlag(base, keyword, False) or []
570            l.append([value, override])
571            self.setVarFlag(base, keyword, l, ignore=True)
572            # And cause that to be recorded:
573            loginfo['detail'] = value
574            loginfo['variable'] = base
575            if override:
576                loginfo['op'] = '%s[%s]' % (keyword, override)
577            else:
578                loginfo['op'] = keyword
579            self.varhistory.record(**loginfo)
580            # todo make sure keyword is not __doc__ or __module__
581            # pay the cookie monster
582
583            # more cookies for the cookie monster
584            if ':' in var:
585                self._setvar_update_overrides(base, **loginfo)
586
587            if base in self.overridevars:
588                self._setvar_update_overridevars(var, value)
589            return
590
591        if not var in self.dict:
592            self._makeShadowCopy(var)
593
594        if not parsing:
595            if ":append" in self.dict[var]:
596                del self.dict[var][":append"]
597            if ":prepend" in self.dict[var]:
598                del self.dict[var][":prepend"]
599            if ":remove" in self.dict[var]:
600                del self.dict[var][":remove"]
601            if var in self.overridedata:
602                active = []
603                self.need_overrides()
604                for (r, o) in self.overridedata[var]:
605                    if o in self.overridesset:
606                        active.append(r)
607                    elif ":" in o:
608                        if set(o.split(":")).issubset(self.overridesset):
609                            active.append(r)
610                for a in active:
611                    self.delVar(a)
612                del self.overridedata[var]
613
614        # more cookies for the cookie monster
615        if ':' in var:
616            self._setvar_update_overrides(var, **loginfo)
617
618        # setting var
619        self.dict[var]["_content"] = value
620        self.varhistory.record(**loginfo)
621
622        if var in self.overridevars:
623            self._setvar_update_overridevars(var, value)
624
625    def _setvar_update_overridevars(self, var, value):
626        vardata = self.expandWithRefs(value, var)
627        new = vardata.references
628        new.update(vardata.contains.keys())
629        while not new.issubset(self.overridevars):
630            nextnew = set()
631            self.overridevars.update(new)
632            for i in new:
633                vardata = self.expandWithRefs(self.getVar(i), i)
634                nextnew.update(vardata.references)
635                nextnew.update(vardata.contains.keys())
636            new = nextnew
637        self.overrides = None
638
639    def _setvar_update_overrides(self, var, **loginfo):
640        # aka pay the cookie monster
641        override = var[var.rfind(':')+1:]
642        shortvar = var[:var.rfind(':')]
643        while override and __override_regexp__.match(override):
644            if shortvar not in self.overridedata:
645                self.overridedata[shortvar] = []
646            if [var, override] not in self.overridedata[shortvar]:
647                # Force CoW by recreating the list first
648                self.overridedata[shortvar] = list(self.overridedata[shortvar])
649                self.overridedata[shortvar].append([var, override])
650            override = None
651            if ":" in shortvar:
652                override = var[shortvar.rfind(':')+1:]
653                shortvar = var[:shortvar.rfind(':')]
654                if len(shortvar) == 0:
655                    override = None
656
657    def getVar(self, var, expand=True, noweakdefault=False, parsing=False):
658        return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
659
660    def renameVar(self, key, newkey, **loginfo):
661        """
662        Rename the variable key to newkey
663        """
664        if key == newkey:
665            bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
666            return
667
668        val = self.getVar(key, 0, parsing=True)
669        if val is not None:
670            self.varhistory.rename_variable_hist(key, newkey)
671            loginfo['variable'] = newkey
672            loginfo['op'] = 'rename from %s' % key
673            loginfo['detail'] = val
674            self.varhistory.record(**loginfo)
675            self.setVar(newkey, val, ignore=True, parsing=True)
676
677        srcflags = self.getVarFlags(key, False, True) or {}
678        for i in srcflags:
679            if i not in (__setvar_keyword__):
680                continue
681            src = srcflags[i]
682
683            dest = self.getVarFlag(newkey, i, False) or []
684            dest.extend(src)
685            self.setVarFlag(newkey, i, dest, ignore=True)
686
687        if key in self.overridedata:
688            self.overridedata[newkey] = []
689            for (v, o) in self.overridedata[key]:
690                self.overridedata[newkey].append([v.replace(key, newkey), o])
691                self.renameVar(v, v.replace(key, newkey))
692
693        if ':' in newkey and val is None:
694            self._setvar_update_overrides(newkey, **loginfo)
695
696        loginfo['variable'] = key
697        loginfo['op'] = 'rename (to)'
698        loginfo['detail'] = newkey
699        self.varhistory.record(**loginfo)
700        self.delVar(key, ignore=True)
701
702    def appendVar(self, var, value, **loginfo):
703        loginfo['op'] = 'append'
704        self.varhistory.record(**loginfo)
705        self.setVar(var + ":append", value, ignore=True, parsing=True)
706
707    def prependVar(self, var, value, **loginfo):
708        loginfo['op'] = 'prepend'
709        self.varhistory.record(**loginfo)
710        self.setVar(var + ":prepend", value, ignore=True, parsing=True)
711
712    def delVar(self, var, **loginfo):
713        self.expand_cache = {}
714
715        loginfo['detail'] = ""
716        loginfo['op'] = 'del'
717        self.varhistory.record(**loginfo)
718        self.dict[var] = {}
719        if var in self.overridedata:
720            del self.overridedata[var]
721        if ':' in var:
722            override = var[var.rfind(':')+1:]
723            shortvar = var[:var.rfind(':')]
724            while override and __override_regexp__.match(override):
725                try:
726                    if shortvar in self.overridedata:
727                        # Force CoW by recreating the list first
728                        self.overridedata[shortvar] = list(self.overridedata[shortvar])
729                        self.overridedata[shortvar].remove([var, override])
730                except ValueError as e:
731                    pass
732                override = None
733                if ":" in shortvar:
734                    override = var[shortvar.rfind(':')+1:]
735                    shortvar = var[:shortvar.rfind(':')]
736                    if len(shortvar) == 0:
737                         override = None
738
739    def setVarFlag(self, var, flag, value, **loginfo):
740        self.expand_cache = {}
741
742        if var == "BB_RENAMED_VARIABLES":
743            self._var_renames[flag] = value
744
745        if var in self._var_renames:
746            _print_rename_error(var, loginfo, self._var_renames)
747            # Mark that we have seen a renamed variable
748            self.setVar("_FAILPARSINGERRORHANDLED", True)
749
750        if 'op' not in loginfo:
751            loginfo['op'] = "set"
752        loginfo['flag'] = flag
753        self.varhistory.record(**loginfo)
754        if not var in self.dict:
755            self._makeShadowCopy(var)
756        self.dict[var][flag] = value
757
758        if flag == "_defaultval" and ':' in var:
759            self._setvar_update_overrides(var, **loginfo)
760        if flag == "_defaultval" and var in self.overridevars:
761            self._setvar_update_overridevars(var, value)
762
763        if flag == "unexport" or flag == "export":
764            if not "__exportlist" in self.dict:
765                self._makeShadowCopy("__exportlist")
766            if not "_content" in self.dict["__exportlist"]:
767                self.dict["__exportlist"]["_content"] = set()
768            self.dict["__exportlist"]["_content"].add(var)
769
770    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
771        if flag == "_content":
772            cachename = var
773        else:
774            if not flag:
775                bb.warn("Calling getVarFlag with flag unset is invalid")
776                return None
777            cachename = var + "[" + flag + "]"
778
779        if not expand and retparser and cachename in self.expand_cache:
780            return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename]
781
782        if expand and cachename in self.expand_cache:
783            return self.expand_cache[cachename].value
784
785        local_var = self._findVar(var)
786        value = None
787        removes = set()
788        if flag == "_content" and not parsing:
789            overridedata = self.overridedata.get(var, None)
790        if flag == "_content" and not parsing and overridedata is not None:
791            match = False
792            active = {}
793            self.need_overrides()
794            for (r, o) in overridedata:
795                # FIXME What about double overrides both with "_" in the name?
796                if o in self.overridesset:
797                    active[o] = r
798                elif ":" in o:
799                    if set(o.split(":")).issubset(self.overridesset):
800                        active[o] = r
801
802            mod = True
803            while mod:
804                mod = False
805                for o in self.overrides:
806                    for a in active.copy():
807                        if a.endswith(":" + o):
808                            t = active[a]
809                            del active[a]
810                            active[a.replace(":" + o, "")] = t
811                            mod = True
812                        elif a == o:
813                            match = active[a]
814                            del active[a]
815            if match:
816                value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
817                if hasattr(subparser, "removes"):
818                    # We have to carry the removes from the overridden variable to apply at the
819                    # end of processing
820                    removes = subparser.removes
821
822        if local_var is not None and value is None:
823            if flag in local_var:
824                value = copy.copy(local_var[flag])
825            elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
826                value = copy.copy(local_var["_defaultval"])
827
828
829        if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
830            self.need_overrides()
831            for (r, o) in local_var[":append"]:
832                match = True
833                if o:
834                    for o2 in o.split(":"):
835                        if not o2 in self.overrides:
836                            match = False
837                if match:
838                    if value is None:
839                        value = ""
840                    value = value + r
841
842        if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing:
843            self.need_overrides()
844            for (r, o) in local_var[":prepend"]:
845
846                match = True
847                if o:
848                    for o2 in o.split(":"):
849                        if not o2 in self.overrides:
850                            match = False
851                if match:
852                    if value is None:
853                        value = ""
854                    value = r + value
855
856        parser = None
857        if expand or retparser:
858            parser = self.expandWithRefs(value, cachename)
859        if expand:
860            value = parser.value
861
862        if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing:
863            self.need_overrides()
864            for (r, o) in local_var[":remove"]:
865                match = True
866                if o:
867                    for o2 in o.split(":"):
868                        if not o2 in self.overrides:
869                            match = False
870                if match:
871                    removes.add(r)
872
873        if value and flag == "_content" and not parsing:
874            if removes and parser:
875                expanded_removes = {}
876                for r in removes:
877                    expanded_removes[r] = self.expand(r).split()
878
879                parser.removes = set()
880                val = []
881                for v in __whitespace_split__.split(parser.value):
882                    skip = False
883                    for r in removes:
884                        if v in expanded_removes[r]:
885                            parser.removes.add(r)
886                            skip = True
887                    if skip:
888                        continue
889                    val.append(v)
890                parser.value = "".join(val)
891                if expand:
892                    value = parser.value
893
894        if parser:
895            self.expand_cache[cachename] = parser
896
897        if retparser:
898            return value, parser
899
900        return value
901
902    def delVarFlag(self, var, flag, **loginfo):
903        self.expand_cache = {}
904
905        local_var = self._findVar(var)
906        if not local_var:
907            return
908        if not var in self.dict:
909            self._makeShadowCopy(var)
910
911        if var in self.dict and flag in self.dict[var]:
912            loginfo['detail'] = ""
913            loginfo['op'] = 'delFlag'
914            loginfo['flag'] = flag
915            self.varhistory.record(**loginfo)
916
917            del self.dict[var][flag]
918
919    def appendVarFlag(self, var, flag, value, **loginfo):
920        loginfo['op'] = 'append'
921        loginfo['flag'] = flag
922        self.varhistory.record(**loginfo)
923        newvalue = (self.getVarFlag(var, flag, False) or "") + value
924        self.setVarFlag(var, flag, newvalue, ignore=True)
925
926    def prependVarFlag(self, var, flag, value, **loginfo):
927        loginfo['op'] = 'prepend'
928        loginfo['flag'] = flag
929        self.varhistory.record(**loginfo)
930        newvalue = value + (self.getVarFlag(var, flag, False) or "")
931        self.setVarFlag(var, flag, newvalue, ignore=True)
932
933    def setVarFlags(self, var, flags, **loginfo):
934        self.expand_cache = {}
935        infer_caller_details(loginfo)
936        if not var in self.dict:
937            self._makeShadowCopy(var)
938
939        for i in flags:
940            if i == "_content":
941                continue
942            loginfo['flag'] = i
943            loginfo['detail'] = flags[i]
944            self.varhistory.record(**loginfo)
945            self.dict[var][i] = flags[i]
946
947    def getVarFlags(self, var, expand = False, internalflags=False):
948        local_var = self._findVar(var)
949        flags = {}
950
951        if local_var:
952            for i in local_var:
953                if i.startswith(("_", ":")) and not internalflags:
954                    continue
955                flags[i] = local_var[i]
956                if expand and i in expand:
957                    flags[i] = self.expand(flags[i], var + "[" + i + "]")
958        if len(flags) == 0:
959            return None
960        return flags
961
962
963    def delVarFlags(self, var, **loginfo):
964        self.expand_cache = {}
965        if not var in self.dict:
966            self._makeShadowCopy(var)
967
968        if var in self.dict:
969            content = None
970
971            loginfo['op'] = 'delete flags'
972            self.varhistory.record(**loginfo)
973
974            # try to save the content
975            if "_content" in self.dict[var]:
976                content  = self.dict[var]["_content"]
977                self.dict[var]            = {}
978                self.dict[var]["_content"] = content
979            else:
980                del self.dict[var]
981
982    def createCopy(self):
983        """
984        Create a copy of self by setting _data to self
985        """
986        # we really want this to be a DataSmart...
987        data = DataSmart()
988        data.dict["_data"] = self.dict
989        data.varhistory = self.varhistory.copy()
990        data.varhistory.dataroot = data
991        data.inchistory = self.inchistory.copy()
992
993        data._tracking = self._tracking
994        data._var_renames = self._var_renames
995
996        data.overrides = None
997        data.overridevars = copy.copy(self.overridevars)
998        # Should really be a deepcopy but has heavy overhead.
999        # Instead, we're careful with writes.
1000        data.overridedata = copy.copy(self.overridedata)
1001
1002        return data
1003
1004    def expandVarref(self, variable, parents=False):
1005        """Find all references to variable in the data and expand it
1006           in place, optionally descending to parent datastores."""
1007
1008        if parents:
1009            keys = iter(self)
1010        else:
1011            keys = self.localkeys()
1012
1013        ref = '${%s}' % variable
1014        value = self.getVar(variable, False)
1015        for key in keys:
1016            referrervalue = self.getVar(key, False)
1017            if referrervalue and isinstance(referrervalue, str) and ref in referrervalue:
1018                self.setVar(key, referrervalue.replace(ref, value))
1019
1020    def localkeys(self):
1021        for key in self.dict:
1022            if key not in ['_data']:
1023                yield key
1024
1025    def __iter__(self):
1026        deleted = set()
1027        overrides = set()
1028        def keylist(d):
1029            klist = set()
1030            for key in d:
1031                if key in ["_data"]:
1032                    continue
1033                if key in deleted:
1034                    continue
1035                if key in overrides:
1036                    continue
1037                if not d[key]:
1038                    deleted.add(key)
1039                    continue
1040                klist.add(key)
1041
1042            if "_data" in d:
1043                klist |= keylist(d["_data"])
1044
1045            return klist
1046
1047        self.need_overrides()
1048        for var in self.overridedata:
1049            for (r, o) in self.overridedata[var]:
1050                if o in self.overridesset:
1051                    overrides.add(var)
1052                elif ":" in o:
1053                    if set(o.split(":")).issubset(self.overridesset):
1054                        overrides.add(var)
1055
1056        for k in keylist(self.dict):
1057             yield k
1058
1059        for k in overrides:
1060             yield k
1061
1062    def __len__(self):
1063        return len(frozenset(iter(self)))
1064
1065    def __getitem__(self, item):
1066        value = self.getVar(item, False)
1067        if value is None:
1068            raise KeyError(item)
1069        else:
1070            return value
1071
1072    def __setitem__(self, var, value):
1073        self.setVar(var, value)
1074
1075    def __delitem__(self, var):
1076        self.delVar(var)
1077
1078    def get_hash(self):
1079        data = {}
1080        d = self.createCopy()
1081        bb.data.expandKeys(d)
1082
1083        config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split())
1084        keys = set(key for key in iter(d) if not key.startswith("__"))
1085        for key in keys:
1086            if key in config_ignore_vars:
1087                continue
1088
1089            value = d.getVar(key, False) or ""
1090            if type(value) is type(self):
1091                data.update({key:value.get_hash()})
1092            else:
1093                data.update({key:value})
1094
1095            varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"])
1096            if not varflags:
1097                continue
1098            for f in varflags:
1099                if f == "_content":
1100                    continue
1101                data.update({'%s[%s]' % (key, f):varflags[f]})
1102
1103        for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
1104            bb_list = d.getVar(key, False) or []
1105            data.update({key:str(bb_list)})
1106
1107            if key == "__BBANONFUNCS":
1108                for i in bb_list:
1109                    value = d.getVar(i, False) or ""
1110                    data.update({i:value})
1111
1112        data_str = str([(k, data[k]) for k in sorted(data.keys())])
1113        return hashlib.sha256(data_str.encode("utf-8")).hexdigest()
1114