1""" 2 AbstractSyntaxTree classes for the Bitbake language 3""" 4 5# Copyright (C) 2003, 2004 Chris Larson 6# Copyright (C) 2003, 2004 Phil Blundell 7# Copyright (C) 2009 Holger Hans Peter Freyther 8# 9# SPDX-License-Identifier: GPL-2.0-only 10# 11 12import sys 13import bb 14from bb import methodpool 15from bb.parse import logger 16 17class StatementGroup(list): 18 def eval(self, data): 19 for statement in self: 20 statement.eval(data) 21 22class AstNode(object): 23 def __init__(self, filename, lineno): 24 self.filename = filename 25 self.lineno = lineno 26 27class IncludeNode(AstNode): 28 def __init__(self, filename, lineno, what_file, force): 29 AstNode.__init__(self, filename, lineno) 30 self.what_file = what_file 31 self.force = force 32 33 def eval(self, data): 34 """ 35 Include the file and evaluate the statements 36 """ 37 s = data.expand(self.what_file) 38 logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s) 39 40 # TODO: Cache those includes... maybe not here though 41 if self.force: 42 bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, "include required") 43 else: 44 bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False) 45 46class IncludeAllNode(AstNode): 47 def __init__(self, filename, lineno, what_file): 48 AstNode.__init__(self, filename, lineno) 49 self.what_file = what_file 50 51 def eval(self, data): 52 """ 53 Include the file and evaluate the statements 54 """ 55 s = data.expand(self.what_file) 56 logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s) 57 58 for path in data.getVar("BBPATH").split(":"): 59 bb.parse.ConfHandler.include(self.filename, os.path.join(path, s), self.lineno, data, False) 60 61class ExportNode(AstNode): 62 def __init__(self, filename, lineno, var): 63 AstNode.__init__(self, filename, lineno) 64 self.var = var 65 66 def eval(self, data): 67 data.setVarFlag(self.var, "export", 1, op = 'exported') 68 69class UnsetNode(AstNode): 70 def __init__(self, filename, lineno, var): 71 AstNode.__init__(self, filename, lineno) 72 self.var = var 73 74 def eval(self, data): 75 loginfo = { 76 'variable': self.var, 77 'file': self.filename, 78 'line': self.lineno, 79 } 80 data.delVar(self.var,**loginfo) 81 82class UnsetFlagNode(AstNode): 83 def __init__(self, filename, lineno, var, flag): 84 AstNode.__init__(self, filename, lineno) 85 self.var = var 86 self.flag = flag 87 88 def eval(self, data): 89 loginfo = { 90 'variable': self.var, 91 'file': self.filename, 92 'line': self.lineno, 93 } 94 data.delVarFlag(self.var, self.flag, **loginfo) 95 96class DataNode(AstNode): 97 """ 98 Various data related updates. For the sake of sanity 99 we have one class doing all this. This means that all 100 this need to be re-evaluated... we might be able to do 101 that faster with multiple classes. 102 """ 103 def __init__(self, filename, lineno, groupd): 104 AstNode.__init__(self, filename, lineno) 105 self.groupd = groupd 106 107 def getFunc(self, key, data): 108 if 'flag' in self.groupd and self.groupd['flag'] is not None: 109 return data.getVarFlag(key, self.groupd['flag'], expand=False, noweakdefault=True) 110 else: 111 return data.getVar(key, False, noweakdefault=True, parsing=True) 112 113 def eval(self, data): 114 groupd = self.groupd 115 key = groupd["var"] 116 loginfo = { 117 'variable': key, 118 'file': self.filename, 119 'line': self.lineno, 120 } 121 if "exp" in groupd and groupd["exp"] is not None: 122 data.setVarFlag(key, "export", 1, op = 'exported', **loginfo) 123 124 op = "set" 125 if "ques" in groupd and groupd["ques"] is not None: 126 val = self.getFunc(key, data) 127 op = "set?" 128 if val is None: 129 val = groupd["value"] 130 elif "colon" in groupd and groupd["colon"] is not None: 131 e = data.createCopy() 132 op = "immediate" 133 val = e.expand(groupd["value"], key + "[:=]") 134 elif "append" in groupd and groupd["append"] is not None: 135 op = "append" 136 val = "%s %s" % ((self.getFunc(key, data) or ""), groupd["value"]) 137 elif "prepend" in groupd and groupd["prepend"] is not None: 138 op = "prepend" 139 val = "%s %s" % (groupd["value"], (self.getFunc(key, data) or "")) 140 elif "postdot" in groupd and groupd["postdot"] is not None: 141 op = "postdot" 142 val = "%s%s" % ((self.getFunc(key, data) or ""), groupd["value"]) 143 elif "predot" in groupd and groupd["predot"] is not None: 144 op = "predot" 145 val = "%s%s" % (groupd["value"], (self.getFunc(key, data) or "")) 146 else: 147 val = groupd["value"] 148 149 if ":append" in key or ":remove" in key or ":prepend" in key: 150 if op in ["append", "prepend", "postdot", "predot", "ques"]: 151 bb.warn(key + " " + groupd[op] + " is not a recommended operator combination, please replace it.") 152 153 flag = None 154 if 'flag' in groupd and groupd['flag'] is not None: 155 if groupd["lazyques"]: 156 flag = "_defaultval_flag_"+groupd['flag'] 157 else: 158 flag = groupd['flag'] 159 elif groupd["lazyques"]: 160 flag = "_defaultval" 161 162 loginfo['op'] = op 163 loginfo['detail'] = groupd["value"] 164 165 if flag: 166 data.setVarFlag(key, flag, val, **loginfo) 167 else: 168 data.setVar(key, val, parsing=True, **loginfo) 169 170class MethodNode(AstNode): 171 tr_tbl = str.maketrans('/.+-@%&~', '________') 172 173 def __init__(self, filename, lineno, func_name, body, python, fakeroot): 174 AstNode.__init__(self, filename, lineno) 175 self.func_name = func_name 176 self.body = body 177 self.python = python 178 self.fakeroot = fakeroot 179 180 def eval(self, data): 181 text = '\n'.join(self.body) 182 funcname = self.func_name 183 if self.func_name == "__anonymous": 184 funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl))) 185 self.python = True 186 text = "def %s(d):\n" % (funcname) + text 187 bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body) - 1) 188 anonfuncs = data.getVar('__BBANONFUNCS', False) or [] 189 anonfuncs.append(funcname) 190 data.setVar('__BBANONFUNCS', anonfuncs) 191 if data.getVar(funcname, False): 192 # clean up old version of this piece of metadata, as its 193 # flags could cause problems 194 data.delVarFlag(funcname, 'python') 195 data.delVarFlag(funcname, 'fakeroot') 196 if self.python: 197 data.setVarFlag(funcname, "python", "1") 198 if self.fakeroot: 199 data.setVarFlag(funcname, "fakeroot", "1") 200 data.setVarFlag(funcname, "func", 1) 201 data.setVar(funcname, text, parsing=True) 202 data.setVarFlag(funcname, 'filename', self.filename) 203 data.setVarFlag(funcname, 'lineno', str(self.lineno - len(self.body))) 204 205class PythonMethodNode(AstNode): 206 def __init__(self, filename, lineno, function, modulename, body): 207 AstNode.__init__(self, filename, lineno) 208 self.function = function 209 self.modulename = modulename 210 self.body = body 211 212 def eval(self, data): 213 # Note we will add root to parsedmethods after having parse 214 # 'this' file. This means we will not parse methods from 215 # bb classes twice 216 text = '\n'.join(self.body) 217 bb.methodpool.insert_method(self.modulename, text, self.filename, self.lineno - len(self.body) - 1) 218 data.setVarFlag(self.function, "func", 1) 219 data.setVarFlag(self.function, "python", 1) 220 data.setVar(self.function, text, parsing=True) 221 data.setVarFlag(self.function, 'filename', self.filename) 222 data.setVarFlag(self.function, 'lineno', str(self.lineno - len(self.body) - 1)) 223 224class ExportFuncsNode(AstNode): 225 def __init__(self, filename, lineno, fns, classname): 226 AstNode.__init__(self, filename, lineno) 227 self.n = fns.split() 228 self.classname = classname 229 230 def eval(self, data): 231 232 sentinel = " # Export function set\n" 233 for func in self.n: 234 calledfunc = self.classname + "_" + func 235 236 basevar = data.getVar(func, False) 237 if basevar and sentinel not in basevar: 238 continue 239 240 if data.getVar(func, False): 241 data.setVarFlag(func, 'python', None) 242 data.setVarFlag(func, 'func', None) 243 244 for flag in [ "func", "python" ]: 245 if data.getVarFlag(calledfunc, flag, False): 246 data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False)) 247 for flag in ["dirs", "cleandirs", "fakeroot"]: 248 if data.getVarFlag(func, flag, False): 249 data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False)) 250 data.setVarFlag(func, "filename", "autogenerated") 251 data.setVarFlag(func, "lineno", 1) 252 253 if data.getVarFlag(calledfunc, "python", False): 254 data.setVar(func, sentinel + " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True) 255 else: 256 if "-" in self.classname: 257 bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc)) 258 data.setVar(func, sentinel + " " + calledfunc + "\n", parsing=True) 259 260class AddTaskNode(AstNode): 261 def __init__(self, filename, lineno, tasks, before, after): 262 AstNode.__init__(self, filename, lineno) 263 self.tasks = tasks 264 self.before = before 265 self.after = after 266 267 def eval(self, data): 268 tasks = self.tasks.split() 269 for task in tasks: 270 bb.build.addtask(task, self.before, self.after, data) 271 272class DelTaskNode(AstNode): 273 def __init__(self, filename, lineno, tasks): 274 AstNode.__init__(self, filename, lineno) 275 self.tasks = tasks 276 277 def eval(self, data): 278 tasks = data.expand(self.tasks).split() 279 for task in tasks: 280 bb.build.deltask(task, data) 281 282class BBHandlerNode(AstNode): 283 def __init__(self, filename, lineno, fns): 284 AstNode.__init__(self, filename, lineno) 285 self.hs = fns.split() 286 287 def eval(self, data): 288 bbhands = data.getVar('__BBHANDLERS', False) or [] 289 for h in self.hs: 290 bbhands.append(h) 291 data.setVarFlag(h, "handler", 1) 292 data.setVar('__BBHANDLERS', bbhands) 293 294class PyLibNode(AstNode): 295 def __init__(self, filename, lineno, libdir, namespace): 296 AstNode.__init__(self, filename, lineno) 297 self.libdir = libdir 298 self.namespace = namespace 299 300 def eval(self, data): 301 global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split() 302 for m in global_mods: 303 if m not in bb.utils._context: 304 bb.utils._context[m] = __import__(m) 305 306 libdir = data.expand(self.libdir) 307 if libdir not in sys.path: 308 sys.path.append(libdir) 309 try: 310 bb.utils._context[self.namespace] = __import__(self.namespace) 311 toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) 312 for i in toimport: 313 bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) 314 mod = getattr(bb.utils._context[self.namespace], i) 315 fn = getattr(mod, "__file__") 316 funcs = {} 317 for f in dir(mod): 318 if f.startswith("_"): 319 continue 320 fcall = getattr(mod, f) 321 if not callable(fcall): 322 continue 323 funcs[f] = fcall 324 bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i)) 325 326 except AttributeError as e: 327 bb.error("Error importing OE modules: %s" % str(e)) 328 329class InheritNode(AstNode): 330 def __init__(self, filename, lineno, classes): 331 AstNode.__init__(self, filename, lineno) 332 self.classes = classes 333 334 def eval(self, data): 335 bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data) 336 337class InheritDeferredNode(AstNode): 338 def __init__(self, filename, lineno, classes): 339 AstNode.__init__(self, filename, lineno) 340 self.inherit = (classes, filename, lineno) 341 342 def eval(self, data): 343 inherits = data.getVar('__BBDEFINHERITS', False) or [] 344 inherits.append(self.inherit) 345 data.setVar('__BBDEFINHERITS', inherits) 346 347class AddFragmentsNode(AstNode): 348 def __init__(self, filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable): 349 AstNode.__init__(self, filename, lineno) 350 self.fragments_path_prefix = fragments_path_prefix 351 self.fragments_variable = fragments_variable 352 self.flagged_variables_list_variable = flagged_variables_list_variable 353 354 def eval(self, data): 355 # No need to use mark_dependency since we would only match a fragment 356 # from a specific layer and there can only be a single layer with a 357 # given namespace. 358 def find_fragment(layers, layerid, full_fragment_name): 359 for layerpath in layers.split(): 360 candidate_fragment_path = os.path.join(layerpath, full_fragment_name) 361 if os.path.exists(candidate_fragment_path) and bb.utils.get_file_layer(candidate_fragment_path, data) == layerid: 362 return candidate_fragment_path 363 return None 364 365 fragments = data.getVar(self.fragments_variable) 366 layers = data.getVar('BBLAYERS') 367 flagged_variables = data.getVar(self.flagged_variables_list_variable).split() 368 369 if not fragments: 370 return 371 for f in fragments.split(): 372 layerid, fragment_name = f.split('/', 1) 373 full_fragment_name = data.expand("{}/{}.conf".format(self.fragments_path_prefix, fragment_name)) 374 fragment_path = find_fragment(layers, layerid, full_fragment_name) 375 if fragment_path: 376 bb.parse.ConfHandler.include(self.filename, fragment_path, self.lineno, data, "include fragment") 377 for flagged_var in flagged_variables: 378 val = data.getVar(flagged_var) 379 data.setVarFlag(flagged_var, f, val) 380 data.setVar(flagged_var, None) 381 else: 382 bb.error("Could not find fragment {} in enabled layers: {}".format(f, layers)) 383 384def handleInclude(statements, filename, lineno, m, force): 385 statements.append(IncludeNode(filename, lineno, m.group(1), force)) 386 387def handleIncludeAll(statements, filename, lineno, m): 388 statements.append(IncludeAllNode(filename, lineno, m.group(1))) 389 390def handleExport(statements, filename, lineno, m): 391 statements.append(ExportNode(filename, lineno, m.group(1))) 392 393def handleUnset(statements, filename, lineno, m): 394 statements.append(UnsetNode(filename, lineno, m.group(1))) 395 396def handleUnsetFlag(statements, filename, lineno, m): 397 statements.append(UnsetFlagNode(filename, lineno, m.group(1), m.group(2))) 398 399def handleData(statements, filename, lineno, groupd): 400 statements.append(DataNode(filename, lineno, groupd)) 401 402def handleMethod(statements, filename, lineno, func_name, body, python, fakeroot): 403 statements.append(MethodNode(filename, lineno, func_name, body, python, fakeroot)) 404 405def handlePythonMethod(statements, filename, lineno, funcname, modulename, body): 406 statements.append(PythonMethodNode(filename, lineno, funcname, modulename, body)) 407 408def handleExportFuncs(statements, filename, lineno, m, classname): 409 statements.append(ExportFuncsNode(filename, lineno, m.group(1), classname)) 410 411def handleAddTask(statements, filename, lineno, tasks, before, after): 412 statements.append(AddTaskNode(filename, lineno, tasks, before, after)) 413 414def handleDelTask(statements, filename, lineno, tasks): 415 statements.append(DelTaskNode(filename, lineno, tasks)) 416 417def handleBBHandlers(statements, filename, lineno, m): 418 statements.append(BBHandlerNode(filename, lineno, m.group(1))) 419 420def handlePyLib(statements, filename, lineno, m): 421 statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2))) 422 423def handleInherit(statements, filename, lineno, m): 424 classes = m.group(1) 425 statements.append(InheritNode(filename, lineno, classes)) 426 427def handleInheritDeferred(statements, filename, lineno, m): 428 classes = m.group(1) 429 statements.append(InheritDeferredNode(filename, lineno, classes)) 430 431def handleAddFragments(statements, filename, lineno, m): 432 fragments_path_prefix = m.group(1) 433 fragments_variable = m.group(2) 434 flagged_variables_list_variable = m.group(3) 435 statements.append(AddFragmentsNode(filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable)) 436 437def runAnonFuncs(d): 438 code = [] 439 for funcname in d.getVar("__BBANONFUNCS", False) or []: 440 code.append("%s(d)" % funcname) 441 bb.utils.better_exec("\n".join(code), {"d": d}) 442 443# Handle recipe level PREFERRED_PROVIDERs 444def handleVirtRecipeProviders(tasklist, d): 445 depends = (d.getVar("DEPENDS") or "").split() 446 virtprovs = (d.getVar("BB_RECIPE_VIRTUAL_PROVIDERS") or "").split() 447 newdeps = [] 448 for dep in depends: 449 if dep in virtprovs: 450 newdep = d.getVar("PREFERRED_PROVIDER_" + dep) 451 if not newdep: 452 bb.fatal("Error, recipe virtual provider PREFERRED_PROVIDER_%s not set" % dep) 453 newdeps.append(newdep) 454 else: 455 newdeps.append(dep) 456 d.setVar("DEPENDS", " ".join(newdeps)) 457 for task in tasklist: 458 taskdeps = (d.getVarFlag(task, "depends") or "").split() 459 remapped = [] 460 for entry in taskdeps: 461 r, t = entry.split(":") 462 if r in virtprovs: 463 r = d.getVar("PREFERRED_PROVIDER_" + r) 464 remapped.append("%s:%s" % (r, t)) 465 d.setVarFlag(task, "depends", " ".join(remapped)) 466 467def finalize(fn, d, variant = None): 468 saved_handlers = bb.event.get_handlers().copy() 469 try: 470 # Found renamed variables. Exit immediately 471 if d.getVar("_FAILPARSINGERRORHANDLED", False) == True: 472 raise bb.BBHandledException() 473 474 for var in d.getVar('__BBHANDLERS', False) or []: 475 # try to add the handler 476 handlerfn = d.getVarFlag(var, "filename", False) 477 if not handlerfn: 478 bb.fatal("Undefined event handler function '%s'" % var) 479 handlerln = int(d.getVarFlag(var, "lineno", False)) 480 bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data=d) 481 482 bb.event.fire(bb.event.RecipePreFinalise(fn), d) 483 484 bb.data.expandKeys(d) 485 486 bb.event.fire(bb.event.RecipePostKeyExpansion(fn), d) 487 488 runAnonFuncs(d) 489 490 tasklist = d.getVar('__BBTASKS', False) or [] 491 bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d) 492 handleVirtRecipeProviders(tasklist, d) 493 bb.build.add_tasks(tasklist, d) 494 495 bb.parse.siggen.finalise(fn, d, variant) 496 497 d.setVar('BBINCLUDED', bb.parse.get_file_depends(d)) 498 499 if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"): 500 bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.") 501 502 bb.event.fire(bb.event.RecipeParsed(fn), d) 503 finally: 504 bb.event.set_handlers(saved_handlers) 505 506def _create_variants(datastores, names, function, onlyfinalise): 507 def create_variant(name, orig_d, arg = None): 508 if onlyfinalise and name not in onlyfinalise: 509 return 510 new_d = bb.data.createCopy(orig_d) 511 function(arg or name, new_d) 512 datastores[name] = new_d 513 514 for variant in list(datastores.keys()): 515 for name in names: 516 if not variant: 517 # Based on main recipe 518 create_variant(name, datastores[""]) 519 else: 520 create_variant("%s-%s" % (variant, name), datastores[variant], name) 521 522def multi_finalize(fn, d): 523 appends = (d.getVar("__BBAPPEND") or "").split() 524 for append in appends: 525 logger.debug("Appending .bbappend file %s to %s", append, fn) 526 bb.parse.BBHandler.handle(append, d, True) 527 528 while True: 529 inherits = d.getVar('__BBDEFINHERITS', False) or [] 530 if not inherits: 531 break 532 inherit, filename, lineno = inherits.pop(0) 533 d.setVar('__BBDEFINHERITS', inherits) 534 bb.parse.BBHandler.inherit(inherit, filename, lineno, d, deferred=True) 535 536 onlyfinalise = d.getVar("__ONLYFINALISE", False) 537 538 safe_d = d 539 d = bb.data.createCopy(safe_d) 540 try: 541 finalize(fn, d) 542 except bb.parse.SkipRecipe as e: 543 d.setVar("__SKIPPED", e.args[0]) 544 datastores = {"": safe_d} 545 546 extended = d.getVar("BBCLASSEXTEND") or "" 547 if extended: 548 # the following is to support bbextends with arguments, for e.g. multilib 549 # an example is as follows: 550 # BBCLASSEXTEND = "multilib:lib32" 551 # it will create foo-lib32, inheriting multilib.bbclass and set 552 # BBEXTENDCURR to "multilib" and BBEXTENDVARIANT to "lib32" 553 extendedmap = {} 554 variantmap = {} 555 556 for ext in extended.split(): 557 eext = ext.split(':', 2) 558 if len(eext) > 1: 559 extendedmap[ext] = eext[0] 560 variantmap[ext] = eext[1] 561 else: 562 extendedmap[ext] = ext 563 564 pn = d.getVar("PN") 565 def extendfunc(name, d): 566 if name != extendedmap[name]: 567 d.setVar("BBEXTENDCURR", extendedmap[name]) 568 d.setVar("BBEXTENDVARIANT", variantmap[name]) 569 else: 570 d.setVar("PN", "%s-%s" % (pn, name)) 571 bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d) 572 573 safe_d.setVar("BBCLASSEXTEND", extended) 574 _create_variants(datastores, extendedmap.keys(), extendfunc, onlyfinalise) 575 576 for variant in datastores.keys(): 577 if variant: 578 try: 579 if not onlyfinalise or variant in onlyfinalise: 580 finalize(fn, datastores[variant], variant) 581 except bb.parse.SkipRecipe as e: 582 datastores[variant].setVar("__SKIPPED", e.args[0]) 583 584 datastores[""] = d 585 return datastores 586