1#
2# Copyright (C) 2003, 2004  Chris Larson
3# Copyright (C) 2003, 2004  Phil Blundell
4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
5# Copyright (C) 2005        Holger Hans Peter Freyther
6# Copyright (C) 2005        ROAD GmbH
7# Copyright (C) 2006        Richard Purdie
8#
9# SPDX-License-Identifier: GPL-2.0-only
10#
11
12import logging
13import os
14import re
15import sys
16import hashlib
17from functools import wraps
18import bb
19from bb import data
20import bb.parse
21
22logger      = logging.getLogger("BitBake")
23parselog    = logging.getLogger("BitBake.Parsing")
24
25class ConfigParameters(object):
26    def __init__(self, argv=None):
27        self.options, targets = self.parseCommandLine(argv or sys.argv)
28        self.environment = self.parseEnvironment()
29
30        self.options.pkgs_to_build = targets or []
31
32        for key, val in self.options.__dict__.items():
33            setattr(self, key, val)
34
35    def parseCommandLine(self, argv=sys.argv):
36        raise Exception("Caller must implement commandline option parsing")
37
38    def parseEnvironment(self):
39        return os.environ.copy()
40
41    def updateFromServer(self, server):
42        if not self.options.cmd:
43            defaulttask, error = server.runCommand(["getVariable", "BB_DEFAULT_TASK"])
44            if error:
45                raise Exception("Unable to get the value of BB_DEFAULT_TASK from the server: %s" % error)
46            self.options.cmd = defaulttask or "build"
47        _, error = server.runCommand(["setConfig", "cmd", self.options.cmd])
48        if error:
49            raise Exception("Unable to set configuration option 'cmd' on the server: %s" % error)
50
51        if not self.options.pkgs_to_build:
52            bbpkgs, error = server.runCommand(["getVariable", "BBTARGETS"])
53            if error:
54                raise Exception("Unable to get the value of BBTARGETS from the server: %s" % error)
55            if bbpkgs:
56                self.options.pkgs_to_build.extend(bbpkgs.split())
57
58    def updateToServer(self, server, environment):
59        options = {}
60        for o in ["halt", "force", "invalidate_stamp",
61                  "dry_run", "dump_signatures",
62                  "extra_assume_provided", "profile",
63                  "prefile", "postfile", "server_timeout",
64                  "nosetscene", "setsceneonly", "skipsetscene",
65                  "runall", "runonly", "writeeventlog"]:
66            options[o] = getattr(self.options, o)
67
68        options['build_verbose_shell'] = self.options.verbose
69        options['build_verbose_stdout'] = self.options.verbose
70        options['default_loglevel'] = bb.msg.loggerDefaultLogLevel
71        options['debug_domains'] = bb.msg.loggerDefaultDomains
72
73        ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
74        if error:
75            raise Exception("Unable to update the server configuration with local parameters: %s" % error)
76
77    def parseActions(self):
78        # Parse any commandline into actions
79        action = {'action':None, 'msg':None}
80        if self.options.show_environment:
81            if 'world' in self.options.pkgs_to_build:
82                action['msg'] = "'world' is not a valid target for --environment."
83            elif 'universe' in self.options.pkgs_to_build:
84                action['msg'] = "'universe' is not a valid target for --environment."
85            elif len(self.options.pkgs_to_build) > 1:
86                action['msg'] = "Only one target can be used with the --environment option."
87            elif self.options.buildfile and len(self.options.pkgs_to_build) > 0:
88                action['msg'] = "No target should be used with the --environment and --buildfile options."
89            elif self.options.pkgs_to_build:
90                action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build]
91            else:
92                action['action'] = ["showEnvironment", self.options.buildfile]
93        elif self.options.buildfile is not None:
94            action['action'] = ["buildFile", self.options.buildfile, self.options.cmd]
95        elif self.options.revisions_changed:
96            action['action'] = ["compareRevisions"]
97        elif self.options.show_versions:
98            action['action'] = ["showVersions"]
99        elif self.options.parse_only:
100            action['action'] = ["parseFiles"]
101        elif self.options.dot_graph:
102            if self.options.pkgs_to_build:
103                action['action'] = ["generateDotGraph", self.options.pkgs_to_build, self.options.cmd]
104            else:
105                action['msg'] = "Please specify a package name for dependency graph generation."
106        else:
107            if self.options.pkgs_to_build:
108                action['action'] = ["buildTargets", self.options.pkgs_to_build, self.options.cmd]
109            else:
110                #action['msg'] = "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information."
111                action = None
112        self.options.initialaction = action
113        return action
114
115class CookerConfiguration(object):
116    """
117    Manages build options and configurations for one run
118    """
119
120    def __init__(self):
121        self.debug_domains = bb.msg.loggerDefaultDomains
122        self.default_loglevel = bb.msg.loggerDefaultLogLevel
123        self.extra_assume_provided = []
124        self.prefile = []
125        self.postfile = []
126        self.cmd = None
127        self.halt = True
128        self.force = False
129        self.profile = False
130        self.nosetscene = False
131        self.setsceneonly = False
132        self.skipsetscene = False
133        self.invalidate_stamp = False
134        self.dump_signatures = []
135        self.build_verbose_shell = False
136        self.build_verbose_stdout = False
137        self.dry_run = False
138        self.tracking = False
139        self.writeeventlog = False
140        self.limited_deps = False
141        self.runall = []
142        self.runonly = []
143
144        self.env = {}
145
146    def __getstate__(self):
147        state = {}
148        for key in self.__dict__.keys():
149            state[key] = getattr(self, key)
150        return state
151
152    def __setstate__(self,state):
153        for k in state:
154            setattr(self, k, state[k])
155
156
157def catch_parse_error(func):
158    """Exception handling bits for our parsing"""
159    @wraps(func)
160    def wrapped(fn, *args):
161        try:
162            return func(fn, *args)
163        except IOError as exc:
164            import traceback
165            parselog.critical(traceback.format_exc())
166            parselog.critical("Unable to parse %s: %s" % (fn, exc))
167            raise bb.BBHandledException()
168        except bb.data_smart.ExpansionError as exc:
169            import traceback
170
171            bbdir = os.path.dirname(__file__) + os.sep
172            exc_class, exc, tb = sys.exc_info()
173            for tb in iter(lambda: tb.tb_next, None):
174                # Skip frames in bitbake itself, we only want the metadata
175                fn, _, _, _ = traceback.extract_tb(tb, 1)[0]
176                if not fn.startswith(bbdir):
177                    break
178            parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
179            raise bb.BBHandledException()
180        except bb.parse.ParseError as exc:
181            parselog.critical(str(exc))
182            raise bb.BBHandledException()
183    return wrapped
184
185@catch_parse_error
186def parse_config_file(fn, data, include=True):
187    return bb.parse.handle(fn, data, include)
188
189@catch_parse_error
190def _inherit(bbclass, data):
191    bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
192    return data
193
194def findConfigFile(configfile, data):
195    search = []
196    bbpath = data.getVar("BBPATH")
197    if bbpath:
198        for i in bbpath.split(":"):
199            search.append(os.path.join(i, "conf", configfile))
200    path = os.getcwd()
201    while path != "/":
202        search.append(os.path.join(path, "conf", configfile))
203        path, _ = os.path.split(path)
204
205    for i in search:
206        if os.path.exists(i):
207            return i
208
209    return None
210
211#
212# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
213# up to /. If that fails, bitbake would fall back to cwd.
214#
215
216def findTopdir():
217    d = bb.data.init()
218    bbpath = None
219    if 'BBPATH' in os.environ:
220        bbpath = os.environ['BBPATH']
221        d.setVar('BBPATH', bbpath)
222
223    layerconf = findConfigFile("bblayers.conf", d)
224    if layerconf:
225        return os.path.dirname(os.path.dirname(layerconf))
226
227    return os.path.abspath(os.getcwd())
228
229class CookerDataBuilder(object):
230
231    def __init__(self, cookercfg, worker = False):
232
233        self.prefiles = cookercfg.prefile
234        self.postfiles = cookercfg.postfile
235        self.tracking = cookercfg.tracking
236
237        bb.utils.set_context(bb.utils.clean_context())
238        bb.event.set_class_handlers(bb.event.clean_class_handlers())
239        self.basedata = bb.data.init()
240        if self.tracking:
241            self.basedata.enableTracking()
242
243        # Keep a datastore of the initial environment variables and their
244        # values from when BitBake was launched to enable child processes
245        # to use environment variables which have been cleaned from the
246        # BitBake processes env
247        self.savedenv = bb.data.init()
248        for k in cookercfg.env:
249            self.savedenv.setVar(k, cookercfg.env[k])
250            if k in bb.data_smart.bitbake_renamed_vars:
251                bb.error('Shell environment variable %s has been renamed to %s' % (k, bb.data_smart.bitbake_renamed_vars[k]))
252                bb.fatal("Exiting to allow enviroment variables to be corrected")
253
254        filtered_keys = bb.utils.approved_variables()
255        bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
256        self.basedata.setVar("BB_ORIGENV", self.savedenv)
257        self.basedata.setVar("__bbclasstype", "global")
258
259        if worker:
260            self.basedata.setVar("BB_WORKERCONTEXT", "1")
261
262        self.data = self.basedata
263        self.mcdata = {}
264
265    def parseBaseConfiguration(self, worker=False):
266        data_hash = hashlib.sha256()
267        try:
268            self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
269
270            if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker:
271                bb.fetch.fetcher_init(self.data)
272            bb.parse.init_parser(self.data)
273            bb.codeparser.parser_cache_init(self.data)
274
275            bb.event.fire(bb.event.ConfigParsed(), self.data)
276
277            reparse_cnt = 0
278            while self.data.getVar("BB_INVALIDCONF", False) is True:
279                if reparse_cnt > 20:
280                    logger.error("Configuration has been re-parsed over 20 times, "
281                                 "breaking out of the loop...")
282                    raise Exception("Too deep config re-parse loop. Check locations where "
283                                    "BB_INVALIDCONF is being set (ConfigParsed event handlers)")
284                self.data.setVar("BB_INVALIDCONF", False)
285                self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
286                reparse_cnt += 1
287                bb.event.fire(bb.event.ConfigParsed(), self.data)
288
289            bb.parse.init_parser(self.data)
290            data_hash.update(self.data.get_hash().encode('utf-8'))
291            self.mcdata[''] = self.data
292
293            multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
294            for config in multiconfig:
295                if config[0].isdigit():
296                    bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config)
297                mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
298                bb.event.fire(bb.event.ConfigParsed(), mcdata)
299                self.mcdata[config] = mcdata
300                data_hash.update(mcdata.get_hash().encode('utf-8'))
301            if multiconfig:
302                bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
303
304            self.data_hash = data_hash.hexdigest()
305        except (SyntaxError, bb.BBHandledException):
306            raise bb.BBHandledException()
307        except bb.data_smart.ExpansionError as e:
308            logger.error(str(e))
309            raise bb.BBHandledException()
310        except Exception:
311            logger.exception("Error parsing configuration files")
312            raise bb.BBHandledException()
313
314
315        # Handle obsolete variable names
316        d = self.data
317        renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {}
318        renamedvars.update(bb.data_smart.bitbake_renamed_vars)
319        issues = False
320        for v in renamedvars:
321            if d.getVar(v) != None or d.hasOverrides(v):
322                issues = True
323                loginfo = {}
324                history = d.varhistory.get_variable_refs(v)
325                for h in history:
326                    for line in history[h]:
327                        loginfo = {'file' : h, 'line' : line}
328                        bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
329                if not history:
330                    bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
331        if issues:
332            raise bb.BBHandledException()
333
334        # Create a copy so we can reset at a later date when UIs disconnect
335        self.origdata = self.data
336        self.data = bb.data.createCopy(self.origdata)
337        self.mcdata[''] = self.data
338
339    def reset(self):
340        # We may not have run parseBaseConfiguration() yet
341        if not hasattr(self, 'origdata'):
342            return
343        self.data = bb.data.createCopy(self.origdata)
344        self.mcdata[''] = self.data
345
346    def _findLayerConf(self, data):
347        return findConfigFile("bblayers.conf", data)
348
349    def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"):
350        data = bb.data.createCopy(self.basedata)
351        data.setVar("BB_CURRENT_MC", mc)
352
353        # Parse files for loading *before* bitbake.conf and any includes
354        for f in prefiles:
355            data = parse_config_file(f, data)
356
357        layerconf = self._findLayerConf(data)
358        if layerconf:
359            parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
360            # By definition bblayers.conf is in conf/ of TOPDIR.
361            # We may have been called with cwd somewhere else so reset TOPDIR
362            data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
363            data = parse_config_file(layerconf, data)
364
365            layers = (data.getVar('BBLAYERS') or "").split()
366            broken_layers = []
367
368            if not layers:
369                bb.fatal("The bblayers.conf file doesn't contain any BBLAYERS definition")
370
371            data = bb.data.createCopy(data)
372            approved = bb.utils.approved_variables()
373
374            # Check whether present layer directories exist
375            for layer in layers:
376                if not os.path.isdir(layer):
377                    broken_layers.append(layer)
378
379            if broken_layers:
380                parselog.critical("The following layer directories do not exist:")
381                for layer in broken_layers:
382                    parselog.critical("   %s", layer)
383                parselog.critical("Please check BBLAYERS in %s" % (layerconf))
384                raise bb.BBHandledException()
385
386            for layer in layers:
387                parselog.debug(2, "Adding layer %s", layer)
388                if 'HOME' in approved and '~' in layer:
389                    layer = os.path.expanduser(layer)
390                if layer.endswith('/'):
391                    layer = layer.rstrip('/')
392                data.setVar('LAYERDIR', layer)
393                data.setVar('LAYERDIR_RE', re.escape(layer))
394                data = parse_config_file(os.path.join(layer, "conf", "layer.conf"), data)
395                data.expandVarref('LAYERDIR')
396                data.expandVarref('LAYERDIR_RE')
397
398            data.delVar('LAYERDIR_RE')
399            data.delVar('LAYERDIR')
400
401            bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
402            collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
403            invalid = []
404            for entry in bbfiles_dynamic:
405                parts = entry.split(":", 1)
406                if len(parts) != 2:
407                    invalid.append(entry)
408                    continue
409                l, f = parts
410                invert = l[0] == "!"
411                if invert:
412                    l = l[1:]
413                if (l in collections and not invert) or (l not in collections and invert):
414                    data.appendVar("BBFILES", " " + f)
415            if invalid:
416                bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
417
418            layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
419            collections_tmp = collections[:]
420            for c in collections:
421                collections_tmp.remove(c)
422                if c in collections_tmp:
423                    bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
424                compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
425                if compat and not layerseries:
426                    bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c)
427                if compat and not (compat & layerseries):
428                    bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
429                              % (c, " ".join(layerseries), " ".join(compat)))
430                elif not compat and not data.getVar("BB_WORKERCONTEXT"):
431                    bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
432
433        if not data.getVar("BBPATH"):
434            msg = "The BBPATH variable is not set"
435            if not layerconf:
436                msg += (" and bitbake did not find a conf/bblayers.conf file in"
437                        " the expected location.\nMaybe you accidentally"
438                        " invoked bitbake from the wrong directory?")
439            raise SystemExit(msg)
440
441        if not data.getVar("TOPDIR"):
442            data.setVar("TOPDIR", os.path.abspath(os.getcwd()))
443
444        data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
445
446        # Parse files for loading *after* bitbake.conf and any includes
447        for p in postfiles:
448            data = parse_config_file(p, data)
449
450        # Handle any INHERITs and inherit the base class
451        bbclasses  = ["base"] + (data.getVar('INHERIT') or "").split()
452        for bbclass in bbclasses:
453            data = _inherit(bbclass, data)
454
455        # Normally we only register event handlers at the end of parsing .bb files
456        # We register any handlers we've found so far here...
457        for var in data.getVar('__BBHANDLERS', False) or []:
458            handlerfn = data.getVarFlag(var, "filename", False)
459            if not handlerfn:
460                parselog.critical("Undefined event handler function '%s'" % var)
461                raise bb.BBHandledException()
462            handlerln = int(data.getVarFlag(var, "lineno", False))
463            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data)
464
465        data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
466
467        return data
468
469