1""" 2 AbstractSyntaxTree classes for the Bitbake language 3""" 4 5# Copyright (C) 2003, 2004 Chris Larson 6# Copyright (C) 2003, 2004 Phil Blundell 7# Copyright (C) 2009 Holger Hans Peter Freyther 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12import sys 13import bb 14from bb import methodpool 15from bb.parse import logger 16 17class StatementGroup(list): 18 def eval(self, data): 19 for statement in self: 20 statement.eval(data) 21 22class AstNode(object): 23 def __init__(self, filename, lineno): 24 self.filename = filename 25 self.lineno = lineno 26 27class IncludeNode(AstNode): 28 def __init__(self, filename, lineno, what_file, force): 29 AstNode.__init__(self, filename, lineno) 30 self.what_file = what_file 31 self.force = force 32 33 def eval(self, data): 34 """ 35 Include the file and evaluate the statements 36 """ 37 s = data.expand(self.what_file) 38 logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s) 39 40 # TODO: Cache those includes... maybe not here though 41 if self.force: 42 bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, "include required") 43 else: 44 bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False) 45 46class ExportNode(AstNode): 47 def __init__(self, filename, lineno, var): 48 AstNode.__init__(self, filename, lineno) 49 self.var = var 50 51 def eval(self, data): 52 data.setVarFlag(self.var, "export", 1, op = 'exported') 53 54class UnsetNode(AstNode): 55 def __init__(self, filename, lineno, var): 56 AstNode.__init__(self, filename, lineno) 57 self.var = var 58 59 def eval(self, data): 60 loginfo = { 61 'variable': self.var, 62 'file': self.filename, 63 'line': self.lineno, 64 } 65 data.delVar(self.var,**loginfo) 66 67class UnsetFlagNode(AstNode): 68 def __init__(self, filename, lineno, var, flag): 69 AstNode.__init__(self, filename, lineno) 70 self.var = var 71 self.flag = flag 72 73 def eval(self, data): 74 loginfo = { 75 'variable': self.var, 76 'file': self.filename, 77 'line': self.lineno, 78 } 79 data.delVarFlag(self.var, self.flag, **loginfo) 80 81class DataNode(AstNode): 82 """ 83 Various data related updates. For the sake of sanity 84 we have one class doing all this. This means that all 85 this need to be re-evaluated... we might be able to do 86 that faster with multiple classes. 87 """ 88 def __init__(self, filename, lineno, groupd): 89 AstNode.__init__(self, filename, lineno) 90 self.groupd = groupd 91 92 def getFunc(self, key, data): 93 if 'flag' in self.groupd and self.groupd['flag'] is not None: 94 return data.getVarFlag(key, self.groupd['flag'], expand=False, noweakdefault=True) 95 else: 96 return data.getVar(key, False, noweakdefault=True, parsing=True) 97 98 def eval(self, data): 99 groupd = self.groupd 100 key = groupd["var"] 101 loginfo = { 102 'variable': key, 103 'file': self.filename, 104 'line': self.lineno, 105 } 106 if "exp" in groupd and groupd["exp"] is not None: 107 data.setVarFlag(key, "export", 1, op = 'exported', **loginfo) 108 109 op = "set" 110 if "ques" in groupd and groupd["ques"] is not None: 111 val = self.getFunc(key, data) 112 op = "set?" 113 if val is None: 114 val = groupd["value"] 115 elif "colon" in groupd and groupd["colon"] is not None: 116 e = data.createCopy() 117 op = "immediate" 118 val = e.expand(groupd["value"], key + "[:=]") 119 elif "append" in groupd and groupd["append"] is not None: 120 op = "append" 121 val = "%s %s" % ((self.getFunc(key, data) or ""), groupd["value"]) 122 elif "prepend" in groupd and groupd["prepend"] is not None: 123 op = "prepend" 124 val = "%s %s" % (groupd["value"], (self.getFunc(key, data) or "")) 125 elif "postdot" in groupd and groupd["postdot"] is not None: 126 op = "postdot" 127 val = "%s%s" % ((self.getFunc(key, data) or ""), groupd["value"]) 128 elif "predot" in groupd and groupd["predot"] is not None: 129 op = "predot" 130 val = "%s%s" % (groupd["value"], (self.getFunc(key, data) or "")) 131 else: 132 val = groupd["value"] 133 134 if ":append" in key or ":remove" in key or ":prepend" in key: 135 if op in ["append", "prepend", "postdot", "predot", "ques"]: 136 bb.warn(key + " " + groupd[op] + " is not a recommended operator combination, please replace it.") 137 138 flag = None 139 if 'flag' in groupd and groupd['flag'] is not None: 140 flag = groupd['flag'] 141 elif groupd["lazyques"]: 142 flag = "_defaultval" 143 144 loginfo['op'] = op 145 loginfo['detail'] = groupd["value"] 146 147 if flag: 148 data.setVarFlag(key, flag, val, **loginfo) 149 else: 150 data.setVar(key, val, parsing=True, **loginfo) 151 152class MethodNode(AstNode): 153 tr_tbl = str.maketrans('/.+-@%&~', '________') 154 155 def __init__(self, filename, lineno, func_name, body, python, fakeroot): 156 AstNode.__init__(self, filename, lineno) 157 self.func_name = func_name 158 self.body = body 159 self.python = python 160 self.fakeroot = fakeroot 161 162 def eval(self, data): 163 text = '\n'.join(self.body) 164 funcname = self.func_name 165 if self.func_name == "__anonymous": 166 funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl))) 167 self.python = True 168 text = "def %s(d):\n" % (funcname) + text 169 bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body) - 1) 170 anonfuncs = data.getVar('__BBANONFUNCS', False) or [] 171 anonfuncs.append(funcname) 172 data.setVar('__BBANONFUNCS', anonfuncs) 173 if data.getVar(funcname, False): 174 # clean up old version of this piece of metadata, as its 175 # flags could cause problems 176 data.delVarFlag(funcname, 'python') 177 data.delVarFlag(funcname, 'fakeroot') 178 if self.python: 179 data.setVarFlag(funcname, "python", "1") 180 if self.fakeroot: 181 data.setVarFlag(funcname, "fakeroot", "1") 182 data.setVarFlag(funcname, "func", 1) 183 data.setVar(funcname, text, parsing=True) 184 data.setVarFlag(funcname, 'filename', self.filename) 185 data.setVarFlag(funcname, 'lineno', str(self.lineno - len(self.body))) 186 187class PythonMethodNode(AstNode): 188 def __init__(self, filename, lineno, function, modulename, body): 189 AstNode.__init__(self, filename, lineno) 190 self.function = function 191 self.modulename = modulename 192 self.body = body 193 194 def eval(self, data): 195 # Note we will add root to parsedmethods after having parse 196 # 'this' file. This means we will not parse methods from 197 # bb classes twice 198 text = '\n'.join(self.body) 199 bb.methodpool.insert_method(self.modulename, text, self.filename, self.lineno - len(self.body) - 1) 200 data.setVarFlag(self.function, "func", 1) 201 data.setVarFlag(self.function, "python", 1) 202 data.setVar(self.function, text, parsing=True) 203 data.setVarFlag(self.function, 'filename', self.filename) 204 data.setVarFlag(self.function, 'lineno', str(self.lineno - len(self.body) - 1)) 205 206class ExportFuncsNode(AstNode): 207 def __init__(self, filename, lineno, fns, classname): 208 AstNode.__init__(self, filename, lineno) 209 self.n = fns.split() 210 self.classname = classname 211 212 def eval(self, data): 213 214 for func in self.n: 215 calledfunc = self.classname + "_" + func 216 217 if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False): 218 continue 219 220 if data.getVar(func, False): 221 data.setVarFlag(func, 'python', None) 222 data.setVarFlag(func, 'func', None) 223 224 for flag in [ "func", "python" ]: 225 if data.getVarFlag(calledfunc, flag, False): 226 data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False)) 227 for flag in ["dirs", "cleandirs", "fakeroot"]: 228 if data.getVarFlag(func, flag, False): 229 data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False)) 230 data.setVarFlag(func, "filename", "autogenerated") 231 data.setVarFlag(func, "lineno", 1) 232 233 if data.getVarFlag(calledfunc, "python", False): 234 data.setVar(func, " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True) 235 else: 236 if "-" in self.classname: 237 bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc)) 238 data.setVar(func, " " + calledfunc + "\n", parsing=True) 239 data.setVarFlag(func, 'export_func', '1') 240 241class AddTaskNode(AstNode): 242 def __init__(self, filename, lineno, func, before, after): 243 AstNode.__init__(self, filename, lineno) 244 self.func = func 245 self.before = before 246 self.after = after 247 248 def eval(self, data): 249 bb.build.addtask(self.func, self.before, self.after, data) 250 251class DelTaskNode(AstNode): 252 def __init__(self, filename, lineno, tasks): 253 AstNode.__init__(self, filename, lineno) 254 self.tasks = tasks 255 256 def eval(self, data): 257 tasks = data.expand(self.tasks).split() 258 for task in tasks: 259 bb.build.deltask(task, data) 260 261class BBHandlerNode(AstNode): 262 def __init__(self, filename, lineno, fns): 263 AstNode.__init__(self, filename, lineno) 264 self.hs = fns.split() 265 266 def eval(self, data): 267 bbhands = data.getVar('__BBHANDLERS', False) or [] 268 for h in self.hs: 269 bbhands.append(h) 270 data.setVarFlag(h, "handler", 1) 271 data.setVar('__BBHANDLERS', bbhands) 272 273class PyLibNode(AstNode): 274 def __init__(self, filename, lineno, libdir, namespace): 275 AstNode.__init__(self, filename, lineno) 276 self.libdir = libdir 277 self.namespace = namespace 278 279 def eval(self, data): 280 global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split() 281 for m in global_mods: 282 if m not in bb.utils._context: 283 bb.utils._context[m] = __import__(m) 284 285 libdir = data.expand(self.libdir) 286 if libdir not in sys.path: 287 sys.path.append(libdir) 288 try: 289 bb.utils._context[self.namespace] = __import__(self.namespace) 290 toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) 291 for i in toimport: 292 bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) 293 mod = getattr(bb.utils._context[self.namespace], i) 294 fn = getattr(mod, "__file__") 295 funcs = {} 296 for f in dir(mod): 297 if f.startswith("_"): 298 continue 299 fcall = getattr(mod, f) 300 if not callable(fcall): 301 continue 302 funcs[f] = fcall 303 bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i)) 304 305 except AttributeError as e: 306 bb.error("Error importing OE modules: %s" % str(e)) 307 308class InheritNode(AstNode): 309 def __init__(self, filename, lineno, classes): 310 AstNode.__init__(self, filename, lineno) 311 self.classes = classes 312 313 def eval(self, data): 314 bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data) 315 316def handleInclude(statements, filename, lineno, m, force): 317 statements.append(IncludeNode(filename, lineno, m.group(1), force)) 318 319def handleExport(statements, filename, lineno, m): 320 statements.append(ExportNode(filename, lineno, m.group(1))) 321 322def handleUnset(statements, filename, lineno, m): 323 statements.append(UnsetNode(filename, lineno, m.group(1))) 324 325def handleUnsetFlag(statements, filename, lineno, m): 326 statements.append(UnsetFlagNode(filename, lineno, m.group(1), m.group(2))) 327 328def handleData(statements, filename, lineno, groupd): 329 statements.append(DataNode(filename, lineno, groupd)) 330 331def handleMethod(statements, filename, lineno, func_name, body, python, fakeroot): 332 statements.append(MethodNode(filename, lineno, func_name, body, python, fakeroot)) 333 334def handlePythonMethod(statements, filename, lineno, funcname, modulename, body): 335 statements.append(PythonMethodNode(filename, lineno, funcname, modulename, body)) 336 337def handleExportFuncs(statements, filename, lineno, m, classname): 338 statements.append(ExportFuncsNode(filename, lineno, m.group(1), classname)) 339 340def handleAddTask(statements, filename, lineno, m): 341 func = m.group("func") 342 before = m.group("before") 343 after = m.group("after") 344 if func is None: 345 return 346 347 statements.append(AddTaskNode(filename, lineno, func, before, after)) 348 349def handleDelTask(statements, filename, lineno, m): 350 func = m.group(1) 351 if func is None: 352 return 353 354 statements.append(DelTaskNode(filename, lineno, func)) 355 356def handleBBHandlers(statements, filename, lineno, m): 357 statements.append(BBHandlerNode(filename, lineno, m.group(1))) 358 359def handlePyLib(statements, filename, lineno, m): 360 statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2))) 361 362def handleInherit(statements, filename, lineno, m): 363 classes = m.group(1) 364 statements.append(InheritNode(filename, lineno, classes)) 365 366def runAnonFuncs(d): 367 code = [] 368 for funcname in d.getVar("__BBANONFUNCS", False) or []: 369 code.append("%s(d)" % funcname) 370 bb.utils.better_exec("\n".join(code), {"d": d}) 371 372def finalize(fn, d, variant = None): 373 saved_handlers = bb.event.get_handlers().copy() 374 try: 375 # Found renamed variables. Exit immediately 376 if d.getVar("_FAILPARSINGERRORHANDLED", False) == True: 377 raise bb.BBHandledException() 378 379 for var in d.getVar('__BBHANDLERS', False) or []: 380 # try to add the handler 381 handlerfn = d.getVarFlag(var, "filename", False) 382 if not handlerfn: 383 bb.fatal("Undefined event handler function '%s'" % var) 384 handlerln = int(d.getVarFlag(var, "lineno", False)) 385 bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data=d) 386 387 bb.event.fire(bb.event.RecipePreFinalise(fn), d) 388 389 bb.data.expandKeys(d) 390 391 bb.event.fire(bb.event.RecipePostKeyExpansion(fn), d) 392 393 runAnonFuncs(d) 394 395 tasklist = d.getVar('__BBTASKS', False) or [] 396 bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d) 397 bb.build.add_tasks(tasklist, d) 398 399 bb.parse.siggen.finalise(fn, d, variant) 400 401 d.setVar('BBINCLUDED', bb.parse.get_file_depends(d)) 402 403 if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"): 404 bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.") 405 406 bb.event.fire(bb.event.RecipeParsed(fn), d) 407 finally: 408 bb.event.set_handlers(saved_handlers) 409 410def _create_variants(datastores, names, function, onlyfinalise): 411 def create_variant(name, orig_d, arg = None): 412 if onlyfinalise and name not in onlyfinalise: 413 return 414 new_d = bb.data.createCopy(orig_d) 415 function(arg or name, new_d) 416 datastores[name] = new_d 417 418 for variant in list(datastores.keys()): 419 for name in names: 420 if not variant: 421 # Based on main recipe 422 create_variant(name, datastores[""]) 423 else: 424 create_variant("%s-%s" % (variant, name), datastores[variant], name) 425 426def multi_finalize(fn, d): 427 appends = (d.getVar("__BBAPPEND") or "").split() 428 for append in appends: 429 logger.debug("Appending .bbappend file %s to %s", append, fn) 430 bb.parse.BBHandler.handle(append, d, True) 431 432 onlyfinalise = d.getVar("__ONLYFINALISE", False) 433 434 safe_d = d 435 d = bb.data.createCopy(safe_d) 436 try: 437 finalize(fn, d) 438 except bb.parse.SkipRecipe as e: 439 d.setVar("__SKIPPED", e.args[0]) 440 datastores = {"": safe_d} 441 442 extended = d.getVar("BBCLASSEXTEND") or "" 443 if extended: 444 # the following is to support bbextends with arguments, for e.g. multilib 445 # an example is as follows: 446 # BBCLASSEXTEND = "multilib:lib32" 447 # it will create foo-lib32, inheriting multilib.bbclass and set 448 # BBEXTENDCURR to "multilib" and BBEXTENDVARIANT to "lib32" 449 extendedmap = {} 450 variantmap = {} 451 452 for ext in extended.split(): 453 eext = ext.split(':', 2) 454 if len(eext) > 1: 455 extendedmap[ext] = eext[0] 456 variantmap[ext] = eext[1] 457 else: 458 extendedmap[ext] = ext 459 460 pn = d.getVar("PN") 461 def extendfunc(name, d): 462 if name != extendedmap[name]: 463 d.setVar("BBEXTENDCURR", extendedmap[name]) 464 d.setVar("BBEXTENDVARIANT", variantmap[name]) 465 else: 466 d.setVar("PN", "%s-%s" % (pn, name)) 467 bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d) 468 469 safe_d.setVar("BBCLASSEXTEND", extended) 470 _create_variants(datastores, extendedmap.keys(), extendfunc, onlyfinalise) 471 472 for variant in datastores.keys(): 473 if variant: 474 try: 475 if not onlyfinalise or variant in onlyfinalise: 476 finalize(fn, datastores[variant], variant) 477 except bb.parse.SkipRecipe as e: 478 datastores[variant].setVar("__SKIPPED", e.args[0]) 479 480 datastores[""] = d 481 return datastores 482