1# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5
6import logging
7import os
8
9from collections import defaultdict
10
11from urllib.parse import unquote, urlparse
12
13import layerindexlib
14
15import layerindexlib.plugin
16
17logger = logging.getLogger('BitBake.layerindexlib.cooker')
18
19import bb.utils
20
21def plugin_init(plugins):
22    return CookerPlugin()
23
24class CookerPlugin(layerindexlib.plugin.IndexPlugin):
25    def __init__(self):
26        self.type = "cooker"
27
28        self.server_connection = None
29        self.ui_module = None
30        self.server = None
31
32    def _run_command(self, command, path, default=None):
33        try:
34            result, _ = bb.process.run(command, cwd=path)
35            result = result.strip()
36        except bb.process.ExecutionError:
37            result = default
38        return result
39
40    def _handle_git_remote(self, remote):
41        if "://" not in remote:
42            if ':' in remote:
43                # This is assumed to be ssh
44                remote = "ssh://" + remote
45            else:
46                # This is assumed to be a file path
47                remote = "file://" + remote
48        return remote
49
50    def _get_bitbake_info(self):
51        """Return a tuple of bitbake information"""
52
53        # Our path SHOULD be .../bitbake/lib/layerindex/cooker.py
54        bb_path = os.path.dirname(__file__) # .../bitbake/lib/layerindex/cooker.py
55        bb_path = os.path.dirname(bb_path)  # .../bitbake/lib/layerindex
56        bb_path = os.path.dirname(bb_path)  # .../bitbake/lib
57        bb_path = os.path.dirname(bb_path)  # .../bitbake
58        bb_path = self._run_command('git rev-parse --show-toplevel', os.path.dirname(__file__), default=bb_path)
59        bb_branch = self._run_command('git rev-parse --abbrev-ref HEAD', bb_path, default="<unknown>")
60        bb_rev = self._run_command('git rev-parse HEAD', bb_path, default="<unknown>")
61        for remotes in self._run_command('git remote -v', bb_path, default="").split("\n"):
62            remote = remotes.split("\t")[1].split(" ")[0]
63            if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
64                bb_remote = self._handle_git_remote(remote)
65                break
66        else:
67            bb_remote = self._handle_git_remote(bb_path)
68
69        return (bb_remote, bb_branch, bb_rev, bb_path)
70
71    def _load_bblayers(self, branches=None):
72        """Load the BBLAYERS and related collection information"""
73
74        d = self.layerindex.data
75
76        if not branches:
77            raise layerindexlib.LayerIndexFetchError("No branches specified for _load_bblayers!")
78
79        index = layerindexlib.LayerIndexObj()
80
81        branchId = 0
82        index.branches = {}
83
84        layerItemId = 0
85        index.layerItems = {}
86
87        layerBranchId = 0
88        index.layerBranches = {}
89
90        bblayers = d.getVar('BBLAYERS').split()
91
92        if not bblayers:
93            # It's blank!  Nothing to process...
94            return index
95
96        collections = d.getVar('BBFILE_COLLECTIONS')
97        layerconfs = d.varhistory.get_variable_items_files('BBFILE_COLLECTIONS')
98        bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
99
100        (_, bb_branch, _, _) = self._get_bitbake_info()
101
102        for branch in branches:
103            branchId += 1
104            index.branches[branchId] = layerindexlib.Branch(index, None)
105            index.branches[branchId].define_data(branchId, branch, bb_branch)
106
107        for entry in collections.split():
108            layerpath = entry
109            if entry in bbfile_collections:
110                layerpath = bbfile_collections[entry]
111
112            layername = d.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % entry) or os.path.basename(layerpath)
113            layerversion = d.getVar('LAYERVERSION_%s' % entry) or ""
114            layerurl = self._handle_git_remote(layerpath)
115
116            layersubdir = ""
117            layerrev = "<unknown>"
118            layerbranch = "<unknown>"
119
120            if os.path.isdir(layerpath):
121                layerbasepath = self._run_command('git rev-parse --show-toplevel', layerpath, default=layerpath)
122                if os.path.abspath(layerpath) != os.path.abspath(layerbasepath):
123                    layersubdir = os.path.abspath(layerpath)[len(layerbasepath) + 1:]
124
125                layerbranch = self._run_command('git rev-parse --abbrev-ref HEAD', layerpath, default="<unknown>")
126                layerrev = self._run_command('git rev-parse HEAD', layerpath, default="<unknown>")
127
128                for remotes in self._run_command('git remote -v', layerpath, default="").split("\n"):
129                    if not remotes:
130                        layerurl = self._handle_git_remote(layerpath)
131                    else:
132                        remote = remotes.split("\t")[1].split(" ")[0]
133                        if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
134                            layerurl = self._handle_git_remote(remote)
135                            break
136
137            layerItemId += 1
138            index.layerItems[layerItemId] = layerindexlib.LayerItem(index, None)
139            index.layerItems[layerItemId].define_data(layerItemId, layername, description=layerpath, vcs_url=layerurl)
140
141            for branchId in index.branches:
142                layerBranchId += 1
143                index.layerBranches[layerBranchId] = layerindexlib.LayerBranch(index, None)
144                index.layerBranches[layerBranchId].define_data(layerBranchId, entry, layerversion, layerItemId, branchId,
145                                               vcs_subdir=layersubdir, vcs_last_rev=layerrev, actual_branch=layerbranch)
146
147        return index
148
149
150    def load_index(self, url, load):
151        """
152            Fetches layer information from a build configuration.
153
154            The return value is a dictionary containing API,
155            layer, branch, dependency, recipe, machine, distro, information.
156
157            url type should be 'cooker'.
158            url path is ignored
159        """
160
161        up = urlparse(url)
162
163        if up.scheme != 'cooker':
164            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
165
166        d = self.layerindex.data
167
168        params = self.layerindex._parse_params(up.params)
169
170        # Only reason to pass a branch is to emulate them...
171        if 'branch' in params:
172            branches = params['branch'].split(',')
173        else:
174            branches = ['HEAD']
175
176        logger.debug("Loading cooker data branches %s" % branches)
177
178        index = self._load_bblayers(branches=branches)
179
180        index.config = {}
181        index.config['TYPE'] = self.type
182        index.config['URL'] = url
183
184        if 'desc' in params:
185            index.config['DESCRIPTION'] = unquote(params['desc'])
186        else:
187            index.config['DESCRIPTION'] = 'local'
188
189        if 'cache' in params:
190            index.config['CACHE'] = params['cache']
191
192        index.config['BRANCH'] = branches
193
194        # ("layerDependencies", layerindexlib.LayerDependency)
195        layerDependencyId = 0
196        if "layerDependencies" in load:
197            index.layerDependencies = {}
198            for layerBranchId in index.layerBranches:
199                branchName = index.layerBranches[layerBranchId].branch.name
200                collection = index.layerBranches[layerBranchId].collection
201
202                def add_dependency(layerDependencyId, index, deps, required):
203                    try:
204                        depDict = bb.utils.explode_dep_versions2(deps)
205                    except bb.utils.VersionStringException as vse:
206                        bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (collection, str(vse)))
207
208                    for dep, oplist in list(depDict.items()):
209                        # We need to search ourselves, so use the _ version...
210                        depLayerBranch = index.find_collection(dep, branches=[branchName])
211                        if not depLayerBranch:
212                            # Missing dependency?!
213                            logger.error('Missing dependency %s (%s)' % (dep, branchName))
214                            continue
215
216                        # We assume that the oplist matches...
217                        layerDependencyId += 1
218                        layerDependency = layerindexlib.LayerDependency(index, None)
219                        layerDependency.define_data(id=layerDependencyId,
220                                        required=required, layerbranch=layerBranchId,
221                                        dependency=depLayerBranch.layer_id)
222
223                        logger.debug('%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
224                        index.add_element("layerDependencies", [layerDependency])
225
226                    return layerDependencyId
227
228                deps = d.getVar("LAYERDEPENDS_%s" % collection)
229                if deps:
230                    layerDependencyId = add_dependency(layerDependencyId, index, deps, True)
231
232                deps = d.getVar("LAYERRECOMMENDS_%s" % collection)
233                if deps:
234                    layerDependencyId = add_dependency(layerDependencyId, index, deps, False)
235
236        # Need to load recipes here (requires cooker access)
237        recipeId = 0
238        ## TODO: NOT IMPLEMENTED
239        # The code following this is an example of what needs to be
240        # implemented.  However, it does not work as-is.
241        if False and 'recipes' in load:
242            index.recipes = {}
243
244            ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
245
246            all_versions = self._run_command('allProviders')
247
248            all_versions_list = defaultdict(list, all_versions)
249            for pn in all_versions_list:
250                for ((pe, pv, pr), fpath) in all_versions_list[pn]:
251                    realfn = bb.cache.virtualfn2realfn(fpath)
252
253                    filepath = os.path.dirname(realfn[0])
254                    filename = os.path.basename(realfn[0])
255
256                    # This is all HORRIBLY slow, and likely unnecessary
257                    #dscon = self._run_command('parseRecipeFile', fpath, False, [])
258                    #connector = myDataStoreConnector(self, dscon.dsindex)
259                    #recipe_data = bb.data.init()
260                    #recipe_data.setVar('_remote_data', connector)
261
262                    #summary = recipe_data.getVar('SUMMARY')
263                    #description = recipe_data.getVar('DESCRIPTION')
264                    #section = recipe_data.getVar('SECTION')
265                    #license = recipe_data.getVar('LICENSE')
266                    #homepage = recipe_data.getVar('HOMEPAGE')
267                    #bugtracker = recipe_data.getVar('BUGTRACKER')
268                    #provides = recipe_data.getVar('PROVIDES')
269
270                    layer = bb.utils.get_file_layer(realfn[0], self.config_data)
271
272                    depBranchId = collection[layer]
273
274                    recipeId += 1
275                    recipe = layerindexlib.Recipe(index, None)
276                    recipe.define_data(id=recipeId,
277                                   filename=filename, filepath=filepath,
278                                   pn=pn, pv=pv,
279                                   summary=pn, description=pn, section='?',
280                                   license='?', homepage='?', bugtracker='?',
281                                   provides='?', bbclassextend='?', inherits='?',
282                                   disallowed='?', layerbranch=depBranchId)
283
284                    index = addElement("recipes", [recipe], index)
285
286        # ("machines", layerindexlib.Machine)
287        machineId = 0
288        if 'machines' in load:
289            index.machines = {}
290
291            for layerBranchId in index.layerBranches:
292                # load_bblayers uses the description to cache the actual path...
293                machine_path = index.layerBranches[layerBranchId].layer.description
294                machine_path = os.path.join(machine_path, 'conf/machine')
295                if os.path.isdir(machine_path):
296                    for (dirpath, _, filenames) in os.walk(machine_path):
297                        # Ignore subdirs...
298                        if not dirpath.endswith('conf/machine'):
299                            continue
300                        for fname in filenames:
301                            if fname.endswith('.conf'):
302                                machineId += 1
303                                machine = layerindexlib.Machine(index, None)
304                                machine.define_data(id=machineId, name=fname[:-5],
305                                                    description=fname[:-5],
306                                                    layerbranch=index.layerBranches[layerBranchId])
307
308                                index.add_element("machines", [machine])
309
310        # ("distros", layerindexlib.Distro)
311        distroId = 0
312        if 'distros' in load:
313            index.distros = {}
314
315            for layerBranchId in index.layerBranches:
316                # load_bblayers uses the description to cache the actual path...
317                distro_path = index.layerBranches[layerBranchId].layer.description
318                distro_path = os.path.join(distro_path, 'conf/distro')
319                if os.path.isdir(distro_path):
320                    for (dirpath, _, filenames) in os.walk(distro_path):
321                        # Ignore subdirs...
322                        if not dirpath.endswith('conf/distro'):
323                            continue
324                        for fname in filenames:
325                            if fname.endswith('.conf'):
326                                distroId += 1
327                                distro = layerindexlib.Distro(index, None)
328                                distro.define_data(id=distroId, name=fname[:-5],
329                                                    description=fname[:-5],
330                                                    layerbranch=index.layerBranches[layerBranchId])
331
332                                index.add_element("distros", [distro])
333
334        return index
335