1""" 2 AbstractSyntaxTree classes for the Bitbake language 3""" 4 5# Copyright (C) 2003, 2004 Chris Larson 6# Copyright (C) 2003, 2004 Phil Blundell 7# Copyright (C) 2009 Holger Hans Peter Freyther 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12import sys 13import bb 14from bb import methodpool 15from bb.parse import logger 16 17class StatementGroup(list): 18 def eval(self, data): 19 for statement in self: 20 statement.eval(data) 21 22class AstNode(object): 23 def __init__(self, filename, lineno): 24 self.filename = filename 25 self.lineno = lineno 26 27class IncludeNode(AstNode): 28 def __init__(self, filename, lineno, what_file, force): 29 AstNode.__init__(self, filename, lineno) 30 self.what_file = what_file 31 self.force = force 32 33 def eval(self, data): 34 """ 35 Include the file and evaluate the statements 36 """ 37 s = data.expand(self.what_file) 38 logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s) 39 40 # TODO: Cache those includes... maybe not here though 41 if self.force: 42 bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, "include required") 43 else: 44 bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False) 45 46class ExportNode(AstNode): 47 def __init__(self, filename, lineno, var): 48 AstNode.__init__(self, filename, lineno) 49 self.var = var 50 51 def eval(self, data): 52 data.setVarFlag(self.var, "export", 1, op = 'exported') 53 54class UnsetNode(AstNode): 55 def __init__(self, filename, lineno, var): 56 AstNode.__init__(self, filename, lineno) 57 self.var = var 58 59 def eval(self, data): 60 loginfo = { 61 'variable': self.var, 62 'file': self.filename, 63 'line': self.lineno, 64 } 65 data.delVar(self.var,**loginfo) 66 67class UnsetFlagNode(AstNode): 68 def __init__(self, filename, lineno, var, flag): 69 AstNode.__init__(self, filename, lineno) 70 self.var = var 71 self.flag = flag 72 73 def eval(self, data): 74 loginfo = { 75 'variable': self.var, 76 'file': self.filename, 77 'line': self.lineno, 78 } 79 data.delVarFlag(self.var, self.flag, **loginfo) 80 81class DataNode(AstNode): 82 """ 83 Various data related updates. For the sake of sanity 84 we have one class doing all this. This means that all 85 this need to be re-evaluated... we might be able to do 86 that faster with multiple classes. 87 """ 88 def __init__(self, filename, lineno, groupd): 89 AstNode.__init__(self, filename, lineno) 90 self.groupd = groupd 91 92 def getFunc(self, key, data): 93 if 'flag' in self.groupd and self.groupd['flag'] is not None: 94 return data.getVarFlag(key, self.groupd['flag'], expand=False, noweakdefault=True) 95 else: 96 return data.getVar(key, False, noweakdefault=True, parsing=True) 97 98 def eval(self, data): 99 groupd = self.groupd 100 key = groupd["var"] 101 loginfo = { 102 'variable': key, 103 'file': self.filename, 104 'line': self.lineno, 105 } 106 if "exp" in groupd and groupd["exp"] is not None: 107 data.setVarFlag(key, "export", 1, op = 'exported', **loginfo) 108 109 op = "set" 110 if "ques" in groupd and groupd["ques"] is not None: 111 val = self.getFunc(key, data) 112 op = "set?" 113 if val is None: 114 val = groupd["value"] 115 elif "colon" in groupd and groupd["colon"] is not None: 116 e = data.createCopy() 117 op = "immediate" 118 val = e.expand(groupd["value"], key + "[:=]") 119 elif "append" in groupd and groupd["append"] is not None: 120 op = "append" 121 val = "%s %s" % ((self.getFunc(key, data) or ""), groupd["value"]) 122 elif "prepend" in groupd and groupd["prepend"] is not None: 123 op = "prepend" 124 val = "%s %s" % (groupd["value"], (self.getFunc(key, data) or "")) 125 elif "postdot" in groupd and groupd["postdot"] is not None: 126 op = "postdot" 127 val = "%s%s" % ((self.getFunc(key, data) or ""), groupd["value"]) 128 elif "predot" in groupd and groupd["predot"] is not None: 129 op = "predot" 130 val = "%s%s" % (groupd["value"], (self.getFunc(key, data) or "")) 131 else: 132 val = groupd["value"] 133 134 if ":append" in key or ":remove" in key or ":prepend" in key: 135 if op in ["append", "prepend", "postdot", "predot", "ques"]: 136 bb.warn(key + " " + groupd[op] + " is not a recommended operator combination, please replace it.") 137 138 flag = None 139 if 'flag' in groupd and groupd['flag'] is not None: 140 flag = groupd['flag'] 141 elif groupd["lazyques"]: 142 flag = "_defaultval" 143 144 loginfo['op'] = op 145 loginfo['detail'] = groupd["value"] 146 147 if flag: 148 data.setVarFlag(key, flag, val, **loginfo) 149 else: 150 data.setVar(key, val, parsing=True, **loginfo) 151 152class MethodNode(AstNode): 153 tr_tbl = str.maketrans('/.+-@%&~', '________') 154 155 def __init__(self, filename, lineno, func_name, body, python, fakeroot): 156 AstNode.__init__(self, filename, lineno) 157 self.func_name = func_name 158 self.body = body 159 self.python = python 160 self.fakeroot = fakeroot 161 162 def eval(self, data): 163 text = '\n'.join(self.body) 164 funcname = self.func_name 165 if self.func_name == "__anonymous": 166 funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl))) 167 self.python = True 168 text = "def %s(d):\n" % (funcname) + text 169 bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body) - 1) 170 anonfuncs = data.getVar('__BBANONFUNCS', False) or [] 171 anonfuncs.append(funcname) 172 data.setVar('__BBANONFUNCS', anonfuncs) 173 if data.getVar(funcname, False): 174 # clean up old version of this piece of metadata, as its 175 # flags could cause problems 176 data.delVarFlag(funcname, 'python') 177 data.delVarFlag(funcname, 'fakeroot') 178 if self.python: 179 data.setVarFlag(funcname, "python", "1") 180 if self.fakeroot: 181 data.setVarFlag(funcname, "fakeroot", "1") 182 data.setVarFlag(funcname, "func", 1) 183 data.setVar(funcname, text, parsing=True) 184 data.setVarFlag(funcname, 'filename', self.filename) 185 data.setVarFlag(funcname, 'lineno', str(self.lineno - len(self.body))) 186 187class PythonMethodNode(AstNode): 188 def __init__(self, filename, lineno, function, modulename, body): 189 AstNode.__init__(self, filename, lineno) 190 self.function = function 191 self.modulename = modulename 192 self.body = body 193 194 def eval(self, data): 195 # Note we will add root to parsedmethods after having parse 196 # 'this' file. This means we will not parse methods from 197 # bb classes twice 198 text = '\n'.join(self.body) 199 bb.methodpool.insert_method(self.modulename, text, self.filename, self.lineno - len(self.body) - 1) 200 data.setVarFlag(self.function, "func", 1) 201 data.setVarFlag(self.function, "python", 1) 202 data.setVar(self.function, text, parsing=True) 203 data.setVarFlag(self.function, 'filename', self.filename) 204 data.setVarFlag(self.function, 'lineno', str(self.lineno - len(self.body) - 1)) 205 206class ExportFuncsNode(AstNode): 207 def __init__(self, filename, lineno, fns, classname): 208 AstNode.__init__(self, filename, lineno) 209 self.n = fns.split() 210 self.classname = classname 211 212 def eval(self, data): 213 214 sentinel = " # Export function set\n" 215 for func in self.n: 216 calledfunc = self.classname + "_" + func 217 218 basevar = data.getVar(func, False) 219 if basevar and sentinel not in basevar: 220 continue 221 222 if data.getVar(func, False): 223 data.setVarFlag(func, 'python', None) 224 data.setVarFlag(func, 'func', None) 225 226 for flag in [ "func", "python" ]: 227 if data.getVarFlag(calledfunc, flag, False): 228 data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False)) 229 for flag in ["dirs", "cleandirs", "fakeroot"]: 230 if data.getVarFlag(func, flag, False): 231 data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False)) 232 data.setVarFlag(func, "filename", "autogenerated") 233 data.setVarFlag(func, "lineno", 1) 234 235 if data.getVarFlag(calledfunc, "python", False): 236 data.setVar(func, sentinel + " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True) 237 else: 238 if "-" in self.classname: 239 bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc)) 240 data.setVar(func, sentinel + " " + calledfunc + "\n", parsing=True) 241 242class AddTaskNode(AstNode): 243 def __init__(self, filename, lineno, tasks, before, after): 244 AstNode.__init__(self, filename, lineno) 245 self.tasks = tasks 246 self.before = before 247 self.after = after 248 249 def eval(self, data): 250 tasks = self.tasks.split() 251 for task in tasks: 252 bb.build.addtask(task, self.before, self.after, data) 253 254class DelTaskNode(AstNode): 255 def __init__(self, filename, lineno, tasks): 256 AstNode.__init__(self, filename, lineno) 257 self.tasks = tasks 258 259 def eval(self, data): 260 tasks = data.expand(self.tasks).split() 261 for task in tasks: 262 bb.build.deltask(task, data) 263 264class BBHandlerNode(AstNode): 265 def __init__(self, filename, lineno, fns): 266 AstNode.__init__(self, filename, lineno) 267 self.hs = fns.split() 268 269 def eval(self, data): 270 bbhands = data.getVar('__BBHANDLERS', False) or [] 271 for h in self.hs: 272 bbhands.append(h) 273 data.setVarFlag(h, "handler", 1) 274 data.setVar('__BBHANDLERS', bbhands) 275 276class PyLibNode(AstNode): 277 def __init__(self, filename, lineno, libdir, namespace): 278 AstNode.__init__(self, filename, lineno) 279 self.libdir = libdir 280 self.namespace = namespace 281 282 def eval(self, data): 283 global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split() 284 for m in global_mods: 285 if m not in bb.utils._context: 286 bb.utils._context[m] = __import__(m) 287 288 libdir = data.expand(self.libdir) 289 if libdir not in sys.path: 290 sys.path.append(libdir) 291 try: 292 bb.utils._context[self.namespace] = __import__(self.namespace) 293 toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) 294 for i in toimport: 295 bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) 296 mod = getattr(bb.utils._context[self.namespace], i) 297 fn = getattr(mod, "__file__") 298 funcs = {} 299 for f in dir(mod): 300 if f.startswith("_"): 301 continue 302 fcall = getattr(mod, f) 303 if not callable(fcall): 304 continue 305 funcs[f] = fcall 306 bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i)) 307 308 except AttributeError as e: 309 bb.error("Error importing OE modules: %s" % str(e)) 310 311class InheritNode(AstNode): 312 def __init__(self, filename, lineno, classes): 313 AstNode.__init__(self, filename, lineno) 314 self.classes = classes 315 316 def eval(self, data): 317 bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data) 318 319class InheritDeferredNode(AstNode): 320 def __init__(self, filename, lineno, classes): 321 AstNode.__init__(self, filename, lineno) 322 self.inherit = (classes, filename, lineno) 323 324 def eval(self, data): 325 inherits = data.getVar('__BBDEFINHERITS', False) or [] 326 inherits.append(self.inherit) 327 data.setVar('__BBDEFINHERITS', inherits) 328 329class AddFragmentsNode(AstNode): 330 def __init__(self, filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable): 331 AstNode.__init__(self, filename, lineno) 332 self.fragments_path_prefix = fragments_path_prefix 333 self.fragments_variable = fragments_variable 334 self.flagged_variables_list_variable = flagged_variables_list_variable 335 336 def eval(self, data): 337 # No need to use mark_dependency since we would only match a fragment 338 # from a specific layer and there can only be a single layer with a 339 # given namespace. 340 def find_fragment(layers, layerid, full_fragment_name): 341 for layerpath in layers.split(): 342 candidate_fragment_path = os.path.join(layerpath, full_fragment_name) 343 if os.path.exists(candidate_fragment_path) and bb.utils.get_file_layer(candidate_fragment_path, data) == layerid: 344 return candidate_fragment_path 345 return None 346 347 fragments = data.getVar(self.fragments_variable) 348 layers = data.getVar('BBLAYERS') 349 flagged_variables = data.getVar(self.flagged_variables_list_variable).split() 350 351 if not fragments: 352 return 353 for f in fragments.split(): 354 layerid, fragment_name = f.split('/', 1) 355 full_fragment_name = data.expand("{}/{}.conf".format(self.fragments_path_prefix, fragment_name)) 356 fragment_path = find_fragment(layers, layerid, full_fragment_name) 357 if fragment_path: 358 bb.parse.ConfHandler.include(self.filename, fragment_path, self.lineno, data, "include fragment") 359 for flagged_var in flagged_variables: 360 val = data.getVar(flagged_var) 361 data.setVarFlag(flagged_var, f, val) 362 data.setVar(flagged_var, None) 363 else: 364 bb.error("Could not find fragment {} in enabled layers: {}".format(f, layers)) 365 366def handleInclude(statements, filename, lineno, m, force): 367 statements.append(IncludeNode(filename, lineno, m.group(1), force)) 368 369def handleExport(statements, filename, lineno, m): 370 statements.append(ExportNode(filename, lineno, m.group(1))) 371 372def handleUnset(statements, filename, lineno, m): 373 statements.append(UnsetNode(filename, lineno, m.group(1))) 374 375def handleUnsetFlag(statements, filename, lineno, m): 376 statements.append(UnsetFlagNode(filename, lineno, m.group(1), m.group(2))) 377 378def handleData(statements, filename, lineno, groupd): 379 statements.append(DataNode(filename, lineno, groupd)) 380 381def handleMethod(statements, filename, lineno, func_name, body, python, fakeroot): 382 statements.append(MethodNode(filename, lineno, func_name, body, python, fakeroot)) 383 384def handlePythonMethod(statements, filename, lineno, funcname, modulename, body): 385 statements.append(PythonMethodNode(filename, lineno, funcname, modulename, body)) 386 387def handleExportFuncs(statements, filename, lineno, m, classname): 388 statements.append(ExportFuncsNode(filename, lineno, m.group(1), classname)) 389 390def handleAddTask(statements, filename, lineno, tasks, before, after): 391 statements.append(AddTaskNode(filename, lineno, tasks, before, after)) 392 393def handleDelTask(statements, filename, lineno, tasks): 394 statements.append(DelTaskNode(filename, lineno, tasks)) 395 396def handleBBHandlers(statements, filename, lineno, m): 397 statements.append(BBHandlerNode(filename, lineno, m.group(1))) 398 399def handlePyLib(statements, filename, lineno, m): 400 statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2))) 401 402def handleInherit(statements, filename, lineno, m): 403 classes = m.group(1) 404 statements.append(InheritNode(filename, lineno, classes)) 405 406def handleInheritDeferred(statements, filename, lineno, m): 407 classes = m.group(1) 408 statements.append(InheritDeferredNode(filename, lineno, classes)) 409 410def handleAddFragments(statements, filename, lineno, m): 411 fragments_path_prefix = m.group(1) 412 fragments_variable = m.group(2) 413 flagged_variables_list_variable = m.group(3) 414 statements.append(AddFragmentsNode(filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable)) 415 416def runAnonFuncs(d): 417 code = [] 418 for funcname in d.getVar("__BBANONFUNCS", False) or []: 419 code.append("%s(d)" % funcname) 420 bb.utils.better_exec("\n".join(code), {"d": d}) 421 422def finalize(fn, d, variant = None): 423 saved_handlers = bb.event.get_handlers().copy() 424 try: 425 # Found renamed variables. Exit immediately 426 if d.getVar("_FAILPARSINGERRORHANDLED", False) == True: 427 raise bb.BBHandledException() 428 429 for var in d.getVar('__BBHANDLERS', False) or []: 430 # try to add the handler 431 handlerfn = d.getVarFlag(var, "filename", False) 432 if not handlerfn: 433 bb.fatal("Undefined event handler function '%s'" % var) 434 handlerln = int(d.getVarFlag(var, "lineno", False)) 435 bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data=d) 436 437 bb.event.fire(bb.event.RecipePreFinalise(fn), d) 438 439 bb.data.expandKeys(d) 440 441 bb.event.fire(bb.event.RecipePostKeyExpansion(fn), d) 442 443 runAnonFuncs(d) 444 445 tasklist = d.getVar('__BBTASKS', False) or [] 446 bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d) 447 bb.build.add_tasks(tasklist, d) 448 449 bb.parse.siggen.finalise(fn, d, variant) 450 451 d.setVar('BBINCLUDED', bb.parse.get_file_depends(d)) 452 453 if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"): 454 bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.") 455 456 bb.event.fire(bb.event.RecipeParsed(fn), d) 457 finally: 458 bb.event.set_handlers(saved_handlers) 459 460def _create_variants(datastores, names, function, onlyfinalise): 461 def create_variant(name, orig_d, arg = None): 462 if onlyfinalise and name not in onlyfinalise: 463 return 464 new_d = bb.data.createCopy(orig_d) 465 function(arg or name, new_d) 466 datastores[name] = new_d 467 468 for variant in list(datastores.keys()): 469 for name in names: 470 if not variant: 471 # Based on main recipe 472 create_variant(name, datastores[""]) 473 else: 474 create_variant("%s-%s" % (variant, name), datastores[variant], name) 475 476def multi_finalize(fn, d): 477 appends = (d.getVar("__BBAPPEND") or "").split() 478 for append in appends: 479 logger.debug("Appending .bbappend file %s to %s", append, fn) 480 bb.parse.BBHandler.handle(append, d, True) 481 482 while True: 483 inherits = d.getVar('__BBDEFINHERITS', False) or [] 484 if not inherits: 485 break 486 inherit, filename, lineno = inherits.pop(0) 487 d.setVar('__BBDEFINHERITS', inherits) 488 bb.parse.BBHandler.inherit(inherit, filename, lineno, d, deferred=True) 489 490 onlyfinalise = d.getVar("__ONLYFINALISE", False) 491 492 safe_d = d 493 d = bb.data.createCopy(safe_d) 494 try: 495 finalize(fn, d) 496 except bb.parse.SkipRecipe as e: 497 d.setVar("__SKIPPED", e.args[0]) 498 datastores = {"": safe_d} 499 500 extended = d.getVar("BBCLASSEXTEND") or "" 501 if extended: 502 # the following is to support bbextends with arguments, for e.g. multilib 503 # an example is as follows: 504 # BBCLASSEXTEND = "multilib:lib32" 505 # it will create foo-lib32, inheriting multilib.bbclass and set 506 # BBEXTENDCURR to "multilib" and BBEXTENDVARIANT to "lib32" 507 extendedmap = {} 508 variantmap = {} 509 510 for ext in extended.split(): 511 eext = ext.split(':', 2) 512 if len(eext) > 1: 513 extendedmap[ext] = eext[0] 514 variantmap[ext] = eext[1] 515 else: 516 extendedmap[ext] = ext 517 518 pn = d.getVar("PN") 519 def extendfunc(name, d): 520 if name != extendedmap[name]: 521 d.setVar("BBEXTENDCURR", extendedmap[name]) 522 d.setVar("BBEXTENDVARIANT", variantmap[name]) 523 else: 524 d.setVar("PN", "%s-%s" % (pn, name)) 525 bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d) 526 527 safe_d.setVar("BBCLASSEXTEND", extended) 528 _create_variants(datastores, extendedmap.keys(), extendfunc, onlyfinalise) 529 530 for variant in datastores.keys(): 531 if variant: 532 try: 533 if not onlyfinalise or variant in onlyfinalise: 534 finalize(fn, datastores[variant], variant) 535 except bb.parse.SkipRecipe as e: 536 datastores[variant].setVar("__SKIPPED", e.args[0]) 537 538 datastores[""] = d 539 return datastores 540