xref: /openbmc/openbmc/poky/bitbake/lib/bb/parse/ast.py (revision 56b44a98) 
1"""
2 AbstractSyntaxTree classes for the Bitbake language
3"""
4
5# Copyright (C) 2003, 2004 Chris Larson
6# Copyright (C) 2003, 2004 Phil Blundell
7# Copyright (C) 2009 Holger Hans Peter Freyther
8#
9# SPDX-License-Identifier: GPL-2.0-only
10#
11
12import sys
13import bb
14from bb import methodpool
15from bb.parse import logger
16
17class StatementGroup(list):
18    def eval(self, data):
19        for statement in self:
20            statement.eval(data)
21
22class AstNode(object):
23    def __init__(self, filename, lineno):
24        self.filename = filename
25        self.lineno = lineno
26
27class IncludeNode(AstNode):
28    def __init__(self, filename, lineno, what_file, force):
29        AstNode.__init__(self, filename, lineno)
30        self.what_file = what_file
31        self.force = force
32
33    def eval(self, data):
34        """
35        Include the file and evaluate the statements
36        """
37        s = data.expand(self.what_file)
38        logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s)
39
40        # TODO: Cache those includes... maybe not here though
41        if self.force:
42            bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, "include required")
43        else:
44            bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False)
45
46class ExportNode(AstNode):
47    def __init__(self, filename, lineno, var):
48        AstNode.__init__(self, filename, lineno)
49        self.var = var
50
51    def eval(self, data):
52        data.setVarFlag(self.var, "export", 1, op = 'exported')
53
54class UnsetNode(AstNode):
55    def __init__(self, filename, lineno, var):
56        AstNode.__init__(self, filename, lineno)
57        self.var = var
58
59    def eval(self, data):
60        loginfo = {
61            'variable': self.var,
62            'file': self.filename,
63            'line': self.lineno,
64        }
65        data.delVar(self.var,**loginfo)
66
67class UnsetFlagNode(AstNode):
68    def __init__(self, filename, lineno, var, flag):
69        AstNode.__init__(self, filename, lineno)
70        self.var = var
71        self.flag = flag
72
73    def eval(self, data):
74        loginfo = {
75            'variable': self.var,
76            'file': self.filename,
77            'line': self.lineno,
78        }
79        data.delVarFlag(self.var, self.flag, **loginfo)
80
81class DataNode(AstNode):
82    """
83    Various data related updates. For the sake of sanity
84    we have one class doing all this. This means that all
85    this need to be re-evaluated... we might be able to do
86    that faster with multiple classes.
87    """
88    def __init__(self, filename, lineno, groupd):
89        AstNode.__init__(self, filename, lineno)
90        self.groupd = groupd
91
92    def getFunc(self, key, data):
93        if 'flag' in self.groupd and self.groupd['flag'] is not None:
94            return data.getVarFlag(key, self.groupd['flag'], expand=False, noweakdefault=True)
95        else:
96            return data.getVar(key, False, noweakdefault=True, parsing=True)
97
98    def eval(self, data):
99        groupd = self.groupd
100        key = groupd["var"]
101        loginfo = {
102            'variable': key,
103            'file': self.filename,
104            'line': self.lineno,
105        }
106        if "exp" in groupd and groupd["exp"] is not None:
107            data.setVarFlag(key, "export", 1, op = 'exported', **loginfo)
108
109        op = "set"
110        if "ques" in groupd and groupd["ques"] is not None:
111            val = self.getFunc(key, data)
112            op = "set?"
113            if val is None:
114                val = groupd["value"]
115        elif "colon" in groupd and groupd["colon"] is not None:
116            e = data.createCopy()
117            op = "immediate"
118            val = e.expand(groupd["value"], key + "[:=]")
119        elif "append" in groupd and groupd["append"] is not None:
120            op = "append"
121            val = "%s %s" % ((self.getFunc(key, data) or ""), groupd["value"])
122        elif "prepend" in groupd and groupd["prepend"] is not None:
123            op = "prepend"
124            val = "%s %s" % (groupd["value"], (self.getFunc(key, data) or ""))
125        elif "postdot" in groupd and groupd["postdot"] is not None:
126            op = "postdot"
127            val = "%s%s" % ((self.getFunc(key, data) or ""), groupd["value"])
128        elif "predot" in groupd and groupd["predot"] is not None:
129            op = "predot"
130            val = "%s%s" % (groupd["value"], (self.getFunc(key, data) or ""))
131        else:
132            val = groupd["value"]
133
134        if ":append" in key or ":remove" in key or ":prepend" in key:
135            if op in ["append", "prepend", "postdot", "predot", "ques"]:
136                bb.warn(key + " " + groupd[op] + " is not a recommended operator combination, please replace it.")
137
138        flag = None
139        if 'flag' in groupd and groupd['flag'] is not None:
140            flag = groupd['flag']
141        elif groupd["lazyques"]:
142            flag = "_defaultval"
143
144        loginfo['op'] = op
145        loginfo['detail'] = groupd["value"]
146
147        if flag:
148            data.setVarFlag(key, flag, val, **loginfo)
149        else:
150            data.setVar(key, val, parsing=True, **loginfo)
151
152class MethodNode(AstNode):
153    tr_tbl = str.maketrans('/.+-@%&~', '________')
154
155    def __init__(self, filename, lineno, func_name, body, python, fakeroot):
156        AstNode.__init__(self, filename, lineno)
157        self.func_name = func_name
158        self.body = body
159        self.python = python
160        self.fakeroot = fakeroot
161
162    def eval(self, data):
163        text = '\n'.join(self.body)
164        funcname = self.func_name
165        if self.func_name == "__anonymous":
166            funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl)))
167            self.python = True
168            text = "def %s(d):\n" % (funcname) + text
169            bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body) - 1)
170            anonfuncs = data.getVar('__BBANONFUNCS', False) or []
171            anonfuncs.append(funcname)
172            data.setVar('__BBANONFUNCS', anonfuncs)
173        if data.getVar(funcname, False):
174            # clean up old version of this piece of metadata, as its
175            # flags could cause problems
176            data.delVarFlag(funcname, 'python')
177            data.delVarFlag(funcname, 'fakeroot')
178        if self.python:
179            data.setVarFlag(funcname, "python", "1")
180        if self.fakeroot:
181            data.setVarFlag(funcname, "fakeroot", "1")
182        data.setVarFlag(funcname, "func", 1)
183        data.setVar(funcname, text, parsing=True)
184        data.setVarFlag(funcname, 'filename', self.filename)
185        data.setVarFlag(funcname, 'lineno', str(self.lineno - len(self.body)))
186
187class PythonMethodNode(AstNode):
188    def __init__(self, filename, lineno, function, modulename, body):
189        AstNode.__init__(self, filename, lineno)
190        self.function = function
191        self.modulename = modulename
192        self.body = body
193
194    def eval(self, data):
195        # Note we will add root to parsedmethods after having parse
196        # 'this' file. This means we will not parse methods from
197        # bb classes twice
198        text = '\n'.join(self.body)
199        bb.methodpool.insert_method(self.modulename, text, self.filename, self.lineno - len(self.body) - 1)
200        data.setVarFlag(self.function, "func", 1)
201        data.setVarFlag(self.function, "python", 1)
202        data.setVar(self.function, text, parsing=True)
203        data.setVarFlag(self.function, 'filename', self.filename)
204        data.setVarFlag(self.function, 'lineno', str(self.lineno - len(self.body) - 1))
205
206class ExportFuncsNode(AstNode):
207    def __init__(self, filename, lineno, fns, classname):
208        AstNode.__init__(self, filename, lineno)
209        self.n = fns.split()
210        self.classname = classname
211
212    def eval(self, data):
213
214        sentinel = "    # Export function set\n"
215        for func in self.n:
216            calledfunc = self.classname + "_" + func
217
218            basevar = data.getVar(func, False)
219            if basevar and sentinel not in basevar:
220                continue
221
222            if data.getVar(func, False):
223                data.setVarFlag(func, 'python', None)
224                data.setVarFlag(func, 'func', None)
225
226            for flag in [ "func", "python" ]:
227                if data.getVarFlag(calledfunc, flag, False):
228                    data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False))
229            for flag in ["dirs", "cleandirs", "fakeroot"]:
230                if data.getVarFlag(func, flag, False):
231                    data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False))
232            data.setVarFlag(func, "filename", "autogenerated")
233            data.setVarFlag(func, "lineno", 1)
234
235            if data.getVarFlag(calledfunc, "python", False):
236                data.setVar(func, sentinel + "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
237            else:
238                if "-" in self.classname:
239                   bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc))
240                data.setVar(func, sentinel + "    " + calledfunc + "\n", parsing=True)
241
242class AddTaskNode(AstNode):
243    def __init__(self, filename, lineno, func, before, after):
244        AstNode.__init__(self, filename, lineno)
245        self.func = func
246        self.before = before
247        self.after = after
248
249    def eval(self, data):
250        bb.build.addtask(self.func, self.before, self.after, data)
251
252class DelTaskNode(AstNode):
253    def __init__(self, filename, lineno, tasks):
254        AstNode.__init__(self, filename, lineno)
255        self.tasks = tasks
256
257    def eval(self, data):
258        tasks = data.expand(self.tasks).split()
259        for task in tasks:
260            bb.build.deltask(task, data)
261
262class BBHandlerNode(AstNode):
263    def __init__(self, filename, lineno, fns):
264        AstNode.__init__(self, filename, lineno)
265        self.hs = fns.split()
266
267    def eval(self, data):
268        bbhands = data.getVar('__BBHANDLERS', False) or []
269        for h in self.hs:
270            bbhands.append(h)
271            data.setVarFlag(h, "handler", 1)
272        data.setVar('__BBHANDLERS', bbhands)
273
274class PyLibNode(AstNode):
275    def __init__(self, filename, lineno, libdir, namespace):
276        AstNode.__init__(self, filename, lineno)
277        self.libdir = libdir
278        self.namespace = namespace
279
280    def eval(self, data):
281        global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split()
282        for m in global_mods:
283            if m not in bb.utils._context:
284                bb.utils._context[m] = __import__(m)
285
286        libdir = data.expand(self.libdir)
287        if libdir not in sys.path:
288            sys.path.append(libdir)
289        try:
290            bb.utils._context[self.namespace] = __import__(self.namespace)
291            toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", [])
292            for i in toimport:
293                bb.utils._context[self.namespace] = __import__(self.namespace + "." + i)
294                mod = getattr(bb.utils._context[self.namespace], i)
295                fn = getattr(mod, "__file__")
296                funcs = {}
297                for f in dir(mod):
298                    if f.startswith("_"):
299                        continue
300                    fcall = getattr(mod, f)
301                    if not callable(fcall):
302                        continue
303                    funcs[f] = fcall
304                bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i))
305
306        except AttributeError as e:
307            bb.error("Error importing OE modules: %s" % str(e))
308
309class InheritNode(AstNode):
310    def __init__(self, filename, lineno, classes):
311        AstNode.__init__(self, filename, lineno)
312        self.classes = classes
313
314    def eval(self, data):
315        bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data)
316
317class InheritDeferredNode(AstNode):
318    def __init__(self, filename, lineno, classes):
319        AstNode.__init__(self, filename, lineno)
320        self.inherit = (classes, filename, lineno)
321
322    def eval(self, data):
323        inherits = data.getVar('__BBDEFINHERITS', False) or []
324        inherits.append(self.inherit)
325        data.setVar('__BBDEFINHERITS', inherits)
326
327def handleInclude(statements, filename, lineno, m, force):
328    statements.append(IncludeNode(filename, lineno, m.group(1), force))
329
330def handleExport(statements, filename, lineno, m):
331    statements.append(ExportNode(filename, lineno, m.group(1)))
332
333def handleUnset(statements, filename, lineno, m):
334    statements.append(UnsetNode(filename, lineno, m.group(1)))
335
336def handleUnsetFlag(statements, filename, lineno, m):
337    statements.append(UnsetFlagNode(filename, lineno, m.group(1), m.group(2)))
338
339def handleData(statements, filename, lineno, groupd):
340    statements.append(DataNode(filename, lineno, groupd))
341
342def handleMethod(statements, filename, lineno, func_name, body, python, fakeroot):
343    statements.append(MethodNode(filename, lineno, func_name, body, python, fakeroot))
344
345def handlePythonMethod(statements, filename, lineno, funcname, modulename, body):
346    statements.append(PythonMethodNode(filename, lineno, funcname, modulename, body))
347
348def handleExportFuncs(statements, filename, lineno, m, classname):
349    statements.append(ExportFuncsNode(filename, lineno, m.group(1), classname))
350
351def handleAddTask(statements, filename, lineno, m):
352    func = m.group("func")
353    before = m.group("before")
354    after = m.group("after")
355    if func is None:
356        return
357
358    statements.append(AddTaskNode(filename, lineno, func, before, after))
359
360def handleDelTask(statements, filename, lineno, m):
361    func = m.group(1)
362    if func is None:
363        return
364
365    statements.append(DelTaskNode(filename, lineno, func))
366
367def handleBBHandlers(statements, filename, lineno, m):
368    statements.append(BBHandlerNode(filename, lineno, m.group(1)))
369
370def handlePyLib(statements, filename, lineno, m):
371    statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2)))
372
373def handleInherit(statements, filename, lineno, m):
374    classes = m.group(1)
375    statements.append(InheritNode(filename, lineno, classes))
376
377def handleInheritDeferred(statements, filename, lineno, m):
378    classes = m.group(1)
379    statements.append(InheritDeferredNode(filename, lineno, classes))
380
381def runAnonFuncs(d):
382    code = []
383    for funcname in d.getVar("__BBANONFUNCS", False) or []:
384        code.append("%s(d)" % funcname)
385    bb.utils.better_exec("\n".join(code), {"d": d})
386
387def finalize(fn, d, variant = None):
388    saved_handlers = bb.event.get_handlers().copy()
389    try:
390        # Found renamed variables. Exit immediately
391        if d.getVar("_FAILPARSINGERRORHANDLED", False) == True:
392            raise bb.BBHandledException()
393
394        for var in d.getVar('__BBHANDLERS', False) or []:
395            # try to add the handler
396            handlerfn = d.getVarFlag(var, "filename", False)
397            if not handlerfn:
398                bb.fatal("Undefined event handler function '%s'" % var)
399            handlerln = int(d.getVarFlag(var, "lineno", False))
400            bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data=d)
401
402        bb.event.fire(bb.event.RecipePreFinalise(fn), d)
403
404        bb.data.expandKeys(d)
405
406        bb.event.fire(bb.event.RecipePostKeyExpansion(fn), d)
407
408        runAnonFuncs(d)
409
410        tasklist = d.getVar('__BBTASKS', False) or []
411        bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
412        bb.build.add_tasks(tasklist, d)
413
414        bb.parse.siggen.finalise(fn, d, variant)
415
416        d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
417
418        if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"):
419            bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.")
420
421        bb.event.fire(bb.event.RecipeParsed(fn), d)
422    finally:
423        bb.event.set_handlers(saved_handlers)
424
425def _create_variants(datastores, names, function, onlyfinalise):
426    def create_variant(name, orig_d, arg = None):
427        if onlyfinalise and name not in onlyfinalise:
428            return
429        new_d = bb.data.createCopy(orig_d)
430        function(arg or name, new_d)
431        datastores[name] = new_d
432
433    for variant in list(datastores.keys()):
434        for name in names:
435            if not variant:
436                # Based on main recipe
437                create_variant(name, datastores[""])
438            else:
439                create_variant("%s-%s" % (variant, name), datastores[variant], name)
440
441def multi_finalize(fn, d):
442    appends = (d.getVar("__BBAPPEND") or "").split()
443    for append in appends:
444        logger.debug("Appending .bbappend file %s to %s", append, fn)
445        bb.parse.BBHandler.handle(append, d, True)
446
447    while True:
448        inherits = d.getVar('__BBDEFINHERITS', False) or []
449        if not inherits:
450            break
451        inherit, filename, lineno = inherits.pop(0)
452        d.setVar('__BBDEFINHERITS', inherits)
453        bb.parse.BBHandler.inherit(inherit, filename, lineno, d, deferred=True)
454
455    onlyfinalise = d.getVar("__ONLYFINALISE", False)
456
457    safe_d = d
458    d = bb.data.createCopy(safe_d)
459    try:
460        finalize(fn, d)
461    except bb.parse.SkipRecipe as e:
462        d.setVar("__SKIPPED", e.args[0])
463    datastores = {"": safe_d}
464
465    extended = d.getVar("BBCLASSEXTEND") or ""
466    if extended:
467        # the following is to support bbextends with arguments, for e.g. multilib
468        # an example is as follows:
469        #   BBCLASSEXTEND = "multilib:lib32"
470        # it will create foo-lib32, inheriting multilib.bbclass and set
471        # BBEXTENDCURR to "multilib" and BBEXTENDVARIANT to "lib32"
472        extendedmap = {}
473        variantmap = {}
474
475        for ext in extended.split():
476            eext = ext.split(':', 2)
477            if len(eext) > 1:
478                extendedmap[ext] = eext[0]
479                variantmap[ext] = eext[1]
480            else:
481                extendedmap[ext] = ext
482
483        pn = d.getVar("PN")
484        def extendfunc(name, d):
485            if name != extendedmap[name]:
486                d.setVar("BBEXTENDCURR", extendedmap[name])
487                d.setVar("BBEXTENDVARIANT", variantmap[name])
488            else:
489                d.setVar("PN", "%s-%s" % (pn, name))
490            bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d)
491
492        safe_d.setVar("BBCLASSEXTEND", extended)
493        _create_variants(datastores, extendedmap.keys(), extendfunc, onlyfinalise)
494
495    for variant in datastores.keys():
496        if variant:
497            try:
498                if not onlyfinalise or variant in onlyfinalise:
499                    finalize(fn, datastores[variant], variant)
500            except bb.parse.SkipRecipe as e:
501                datastores[variant].setVar("__SKIPPED", e.args[0])
502
503    datastores[""] = d
504    return datastores
505