1#
2# Copyright (C) 2003, 2004  Chris Larson
3# Copyright (C) 2003, 2004  Phil Blundell
4# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
5# Copyright (C) 2005        Holger Hans Peter Freyther
6# Copyright (C) 2005        ROAD GmbH
7# Copyright (C) 2006        Richard Purdie
8#
9# SPDX-License-Identifier: GPL-2.0-only
10#
11
12import logging
13import os
14import re
15import sys
16import hashlib
17from functools import wraps
18import bb
19from bb import data
20import bb.parse
21
22logger      = logging.getLogger("BitBake")
23parselog    = logging.getLogger("BitBake.Parsing")
24
25class ConfigParameters(object):
26    def __init__(self, argv=None):
27        self.options, targets = self.parseCommandLine(argv or sys.argv)
28        self.environment = self.parseEnvironment()
29
30        self.options.pkgs_to_build = targets or []
31
32        for key, val in self.options.__dict__.items():
33            setattr(self, key, val)
34
35    def parseCommandLine(self, argv=sys.argv):
36        raise Exception("Caller must implement commandline option parsing")
37
38    def parseEnvironment(self):
39        return os.environ.copy()
40
41    def updateFromServer(self, server):
42        if not self.options.cmd:
43            defaulttask, error = server.runCommand(["getVariable", "BB_DEFAULT_TASK"])
44            if error:
45                raise Exception("Unable to get the value of BB_DEFAULT_TASK from the server: %s" % error)
46            self.options.cmd = defaulttask or "build"
47        _, error = server.runCommand(["setConfig", "cmd", self.options.cmd])
48        if error:
49            raise Exception("Unable to set configuration option 'cmd' on the server: %s" % error)
50
51        if not self.options.pkgs_to_build:
52            bbpkgs, error = server.runCommand(["getVariable", "BBTARGETS"])
53            if error:
54                raise Exception("Unable to get the value of BBTARGETS from the server: %s" % error)
55            if bbpkgs:
56                self.options.pkgs_to_build.extend(bbpkgs.split())
57
58    def updateToServer(self, server, environment):
59        options = {}
60        for o in ["halt", "force", "invalidate_stamp",
61                  "dry_run", "dump_signatures",
62                  "extra_assume_provided", "profile",
63                  "prefile", "postfile", "server_timeout",
64                  "nosetscene", "setsceneonly", "skipsetscene",
65                  "runall", "runonly", "writeeventlog"]:
66            options[o] = getattr(self.options, o)
67
68        options['build_verbose_shell'] = self.options.verbose
69        options['build_verbose_stdout'] = self.options.verbose
70        options['default_loglevel'] = bb.msg.loggerDefaultLogLevel
71        options['debug_domains'] = bb.msg.loggerDefaultDomains
72
73        ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
74        if error:
75            raise Exception("Unable to update the server configuration with local parameters: %s" % error)
76
77    def parseActions(self):
78        # Parse any commandline into actions
79        action = {'action':None, 'msg':None}
80        if self.options.show_environment:
81            if 'world' in self.options.pkgs_to_build:
82                action['msg'] = "'world' is not a valid target for --environment."
83            elif 'universe' in self.options.pkgs_to_build:
84                action['msg'] = "'universe' is not a valid target for --environment."
85            elif len(self.options.pkgs_to_build) > 1:
86                action['msg'] = "Only one target can be used with the --environment option."
87            elif self.options.buildfile and len(self.options.pkgs_to_build) > 0:
88                action['msg'] = "No target should be used with the --environment and --buildfile options."
89            elif self.options.pkgs_to_build:
90                action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build]
91            else:
92                action['action'] = ["showEnvironment", self.options.buildfile]
93        elif self.options.buildfile is not None:
94            action['action'] = ["buildFile", self.options.buildfile, self.options.cmd]
95        elif self.options.revisions_changed:
96            action['action'] = ["compareRevisions"]
97        elif self.options.show_versions:
98            action['action'] = ["showVersions"]
99        elif self.options.parse_only:
100            action['action'] = ["parseFiles"]
101        elif self.options.dot_graph:
102            if self.options.pkgs_to_build:
103                action['action'] = ["generateDotGraph", self.options.pkgs_to_build, self.options.cmd]
104            else:
105                action['msg'] = "Please specify a package name for dependency graph generation."
106        else:
107            if self.options.pkgs_to_build:
108                action['action'] = ["buildTargets", self.options.pkgs_to_build, self.options.cmd]
109            else:
110                #action['msg'] = "Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information."
111                action = None
112        self.options.initialaction = action
113        return action
114
115class CookerConfiguration(object):
116    """
117    Manages build options and configurations for one run
118    """
119
120    def __init__(self):
121        self.debug_domains = bb.msg.loggerDefaultDomains
122        self.default_loglevel = bb.msg.loggerDefaultLogLevel
123        self.extra_assume_provided = []
124        self.prefile = []
125        self.postfile = []
126        self.cmd = None
127        self.halt = True
128        self.force = False
129        self.profile = False
130        self.nosetscene = False
131        self.setsceneonly = False
132        self.skipsetscene = False
133        self.invalidate_stamp = False
134        self.dump_signatures = []
135        self.build_verbose_shell = False
136        self.build_verbose_stdout = False
137        self.dry_run = False
138        self.tracking = False
139        self.writeeventlog = False
140        self.limited_deps = False
141        self.runall = []
142        self.runonly = []
143
144        self.env = {}
145
146    def __getstate__(self):
147        state = {}
148        for key in self.__dict__.keys():
149            state[key] = getattr(self, key)
150        return state
151
152    def __setstate__(self,state):
153        for k in state:
154            setattr(self, k, state[k])
155
156
157def catch_parse_error(func):
158    """Exception handling bits for our parsing"""
159    @wraps(func)
160    def wrapped(fn, *args):
161        try:
162            return func(fn, *args)
163        except Exception as exc:
164            import traceback
165
166            bbdir = os.path.dirname(__file__) + os.sep
167            exc_class, exc, tb = sys.exc_info()
168            for tb in iter(lambda: tb.tb_next, None):
169                # Skip frames in bitbake itself, we only want the metadata
170                fn, _, _, _ = traceback.extract_tb(tb, 1)[0]
171                if not fn.startswith(bbdir):
172                    break
173            parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
174            raise bb.BBHandledException()
175    return wrapped
176
177@catch_parse_error
178def parse_config_file(fn, data, include=True):
179    return bb.parse.handle(fn, data, include, baseconfig=True)
180
181@catch_parse_error
182def _inherit(bbclass, data):
183    bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
184    return data
185
186def findConfigFile(configfile, data):
187    search = []
188    bbpath = data.getVar("BBPATH")
189    if bbpath:
190        for i in bbpath.split(":"):
191            search.append(os.path.join(i, "conf", configfile))
192    path = os.getcwd()
193    while path != "/":
194        search.append(os.path.join(path, "conf", configfile))
195        path, _ = os.path.split(path)
196
197    for i in search:
198        if os.path.exists(i):
199            return i
200
201    return None
202
203#
204# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
205# up to /. If that fails, bitbake would fall back to cwd.
206#
207
208def findTopdir():
209    d = bb.data.init()
210    bbpath = None
211    if 'BBPATH' in os.environ:
212        bbpath = os.environ['BBPATH']
213        d.setVar('BBPATH', bbpath)
214
215    layerconf = findConfigFile("bblayers.conf", d)
216    if layerconf:
217        return os.path.dirname(os.path.dirname(layerconf))
218
219    return os.path.abspath(os.getcwd())
220
221class CookerDataBuilder(object):
222
223    def __init__(self, cookercfg, worker = False):
224
225        self.prefiles = cookercfg.prefile
226        self.postfiles = cookercfg.postfile
227        self.tracking = cookercfg.tracking
228
229        bb.utils.set_context(bb.utils.clean_context())
230        bb.event.set_class_handlers(bb.event.clean_class_handlers())
231        self.basedata = bb.data.init()
232        if self.tracking:
233            self.basedata.enableTracking()
234
235        # Keep a datastore of the initial environment variables and their
236        # values from when BitBake was launched to enable child processes
237        # to use environment variables which have been cleaned from the
238        # BitBake processes env
239        self.savedenv = bb.data.init()
240        for k in cookercfg.env:
241            self.savedenv.setVar(k, cookercfg.env[k])
242            if k in bb.data_smart.bitbake_renamed_vars:
243                bb.error('Shell environment variable %s has been renamed to %s' % (k, bb.data_smart.bitbake_renamed_vars[k]))
244                bb.fatal("Exiting to allow enviroment variables to be corrected")
245
246        filtered_keys = bb.utils.approved_variables()
247        bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
248        self.basedata.setVar("BB_ORIGENV", self.savedenv)
249        self.basedata.setVar("__bbclasstype", "global")
250
251        if worker:
252            self.basedata.setVar("BB_WORKERCONTEXT", "1")
253
254        self.data = self.basedata
255        self.mcdata = {}
256
257    def parseBaseConfiguration(self, worker=False):
258        mcdata = {}
259        data_hash = hashlib.sha256()
260        try:
261            self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
262
263            if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker:
264                bb.fetch.fetcher_init(self.data)
265            bb.parse.init_parser(self.data)
266
267            bb.event.fire(bb.event.ConfigParsed(), self.data)
268
269            reparse_cnt = 0
270            while self.data.getVar("BB_INVALIDCONF", False) is True:
271                if reparse_cnt > 20:
272                    logger.error("Configuration has been re-parsed over 20 times, "
273                                 "breaking out of the loop...")
274                    raise Exception("Too deep config re-parse loop. Check locations where "
275                                    "BB_INVALIDCONF is being set (ConfigParsed event handlers)")
276                self.data.setVar("BB_INVALIDCONF", False)
277                self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
278                reparse_cnt += 1
279                bb.event.fire(bb.event.ConfigParsed(), self.data)
280
281            bb.parse.init_parser(self.data)
282            data_hash.update(self.data.get_hash().encode('utf-8'))
283            mcdata[''] = self.data
284
285            multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
286            for config in multiconfig:
287                if config[0].isdigit():
288                    bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config)
289                parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
290                bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata)
291                mcdata[config] = parsed_mcdata
292                data_hash.update(parsed_mcdata.get_hash().encode('utf-8'))
293            if multiconfig:
294                bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data)
295
296            self.data_hash = data_hash.hexdigest()
297        except bb.data_smart.ExpansionError as e:
298            logger.error(str(e))
299            raise bb.BBHandledException()
300
301        bb.codeparser.update_module_dependencies(self.data)
302
303        # Handle obsolete variable names
304        d = self.data
305        renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {}
306        renamedvars.update(bb.data_smart.bitbake_renamed_vars)
307        issues = False
308        for v in renamedvars:
309            if d.getVar(v) != None or d.hasOverrides(v):
310                issues = True
311                loginfo = {}
312                history = d.varhistory.get_variable_refs(v)
313                for h in history:
314                    for line in history[h]:
315                        loginfo = {'file' : h, 'line' : line}
316                        bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
317                if not history:
318                    bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
319        if issues:
320            raise bb.BBHandledException()
321
322        for mc in mcdata:
323            mcdata[mc].renameVar("__depends", "__base_depends")
324            mcdata[mc].setVar("__bbclasstype", "recipe")
325
326        # Create a copy so we can reset at a later date when UIs disconnect
327        self.mcorigdata = mcdata
328        for mc in mcdata:
329            self.mcdata[mc] = bb.data.createCopy(mcdata[mc])
330        self.data = self.mcdata['']
331
332    def reset(self):
333        # We may not have run parseBaseConfiguration() yet
334        if not hasattr(self, 'mcorigdata'):
335            return
336        for mc in self.mcorigdata:
337            self.mcdata[mc] = bb.data.createCopy(self.mcorigdata[mc])
338        self.data = self.mcdata['']
339
340    def _findLayerConf(self, data):
341        return findConfigFile("bblayers.conf", data)
342
343    def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"):
344        data = bb.data.createCopy(self.basedata)
345        data.setVar("BB_CURRENT_MC", mc)
346
347        # Parse files for loading *before* bitbake.conf and any includes
348        for f in prefiles:
349            data = parse_config_file(f, data)
350
351        layerconf = self._findLayerConf(data)
352        if layerconf:
353            parselog.debug2("Found bblayers.conf (%s)", layerconf)
354            # By definition bblayers.conf is in conf/ of TOPDIR.
355            # We may have been called with cwd somewhere else so reset TOPDIR
356            data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
357            data = parse_config_file(layerconf, data)
358
359            if not data.getVar("BB_CACHEDIR"):
360                data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
361
362            bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
363
364            layers = (data.getVar('BBLAYERS') or "").split()
365            broken_layers = []
366
367            if not layers:
368                bb.fatal("The bblayers.conf file doesn't contain any BBLAYERS definition")
369
370            data = bb.data.createCopy(data)
371            approved = bb.utils.approved_variables()
372
373            # Check whether present layer directories exist
374            for layer in layers:
375                if not os.path.isdir(layer):
376                    broken_layers.append(layer)
377
378            if broken_layers:
379                parselog.critical("The following layer directories do not exist:")
380                for layer in broken_layers:
381                    parselog.critical("   %s", layer)
382                parselog.critical("Please check BBLAYERS in %s" % (layerconf))
383                raise bb.BBHandledException()
384
385            layerseries = None
386            compat_entries = {}
387            for layer in layers:
388                parselog.debug2("Adding layer %s", layer)
389                if 'HOME' in approved and '~' in layer:
390                    layer = os.path.expanduser(layer)
391                if layer.endswith('/'):
392                    layer = layer.rstrip('/')
393                data.setVar('LAYERDIR', layer)
394                data.setVar('LAYERDIR_RE', re.escape(layer))
395                data = parse_config_file(os.path.join(layer, "conf", "layer.conf"), data)
396                data.expandVarref('LAYERDIR')
397                data.expandVarref('LAYERDIR_RE')
398
399                # Sadly we can't have nice things.
400                # Some layers think they're going to be 'clever' and copy the values from
401                # another layer, e.g. using ${LAYERSERIES_COMPAT_core}. The whole point of
402                # this mechanism is to make it clear which releases a layer supports and
403                # show when a layer master branch is bitrotting and is unmaintained.
404                # We therefore avoid people doing this here.
405                collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
406                for c in collections:
407                    compat_entry = data.getVar("LAYERSERIES_COMPAT_%s" % c)
408                    if compat_entry:
409                        compat_entries[c] = set(compat_entry.split())
410                        data.delVar("LAYERSERIES_COMPAT_%s" % c)
411                if not layerseries:
412                    layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
413                    if layerseries:
414                        data.delVar("LAYERSERIES_CORENAMES")
415
416            data.delVar('LAYERDIR_RE')
417            data.delVar('LAYERDIR')
418            for c in compat_entries:
419                data.setVar("LAYERSERIES_COMPAT_%s" % c, " ".join(sorted(compat_entries[c])))
420
421            bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
422            collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
423            invalid = []
424            for entry in bbfiles_dynamic:
425                parts = entry.split(":", 1)
426                if len(parts) != 2:
427                    invalid.append(entry)
428                    continue
429                l, f = parts
430                invert = l[0] == "!"
431                if invert:
432                    l = l[1:]
433                if (l in collections and not invert) or (l not in collections and invert):
434                    data.appendVar("BBFILES", " " + f)
435            if invalid:
436                bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
437
438            collections_tmp = collections[:]
439            for c in collections:
440                collections_tmp.remove(c)
441                if c in collections_tmp:
442                    bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
443
444                compat = set()
445                if c in compat_entries:
446                    compat = compat_entries[c]
447                if compat and not layerseries:
448                    bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c)
449                if compat and not (compat & layerseries):
450                    bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
451                              % (c, " ".join(layerseries), " ".join(compat)))
452                elif not compat and not data.getVar("BB_WORKERCONTEXT"):
453                    bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
454
455            data.setVar("LAYERSERIES_CORENAMES", " ".join(sorted(layerseries)))
456
457        if not data.getVar("BBPATH"):
458            msg = "The BBPATH variable is not set"
459            if not layerconf:
460                msg += (" and bitbake did not find a conf/bblayers.conf file in"
461                        " the expected location.\nMaybe you accidentally"
462                        " invoked bitbake from the wrong directory?")
463            bb.fatal(msg)
464
465        if not data.getVar("TOPDIR"):
466            data.setVar("TOPDIR", os.path.abspath(os.getcwd()))
467        if not data.getVar("BB_CACHEDIR"):
468            data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
469        bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
470
471        data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
472
473        # Parse files for loading *after* bitbake.conf and any includes
474        for p in postfiles:
475            data = parse_config_file(p, data)
476
477        # Handle any INHERITs and inherit the base class
478        bbclasses  = ["base"] + (data.getVar('INHERIT') or "").split()
479        for bbclass in bbclasses:
480            data = _inherit(bbclass, data)
481
482        # Normally we only register event handlers at the end of parsing .bb files
483        # We register any handlers we've found so far here...
484        for var in data.getVar('__BBHANDLERS', False) or []:
485            handlerfn = data.getVarFlag(var, "filename", False)
486            if not handlerfn:
487                parselog.critical("Undefined event handler function '%s'" % var)
488                raise bb.BBHandledException()
489            handlerln = int(data.getVarFlag(var, "lineno", False))
490            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data)
491
492        data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
493
494        return data
495
496    @staticmethod
497    def _parse_recipe(bb_data, bbfile, appends, mc, layername):
498        bb_data.setVar("__BBMULTICONFIG", mc)
499        bb_data.setVar("FILE_LAYERNAME", layername)
500
501        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
502        bb.parse.cached_mtime_noerror(bbfile_loc)
503
504        if appends:
505            bb_data.setVar('__BBAPPEND', " ".join(appends))
506
507        return bb.parse.handle(bbfile, bb_data)
508
509    def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None, layername=None):
510        """
511        Load and parse one .bb build file
512        Return the data and whether parsing resulted in the file being skipped
513        """
514
515        if virtonly:
516            (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile)
517            bb_data = self.mcdata[mc].createCopy()
518            bb_data.setVar("__ONLYFINALISE", virtual or "default")
519            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
520
521        if mc is not None:
522            bb_data = self.mcdata[mc].createCopy()
523            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
524
525        bb_data = self.data.createCopy()
526        datastores = self._parse_recipe(bb_data, bbfile, appends, '', layername)
527
528        for mc in self.mcdata:
529            if not mc:
530                continue
531            bb_data = self.mcdata[mc].createCopy()
532            newstores = self._parse_recipe(bb_data, bbfile, appends, mc, layername)
533            for ns in newstores:
534                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
535
536        return datastores
537
538    def parseRecipe(self, virtualfn, appends, layername):
539        """
540        Return a complete set of data for fn.
541        To do this, we need to parse the file.
542        """
543        logger.debug("Parsing %s (full)" % virtualfn)
544        (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn)
545        datastores = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername)
546        return datastores[virtual]
547