1# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5
6import logging
7import json
8import os
9
10from urllib.parse import unquote
11from urllib.parse import urlparse
12
13import bb
14
15import layerindexlib
16import layerindexlib.plugin
17
18logger = logging.getLogger('BitBake.layerindexlib.restapi')
19
20def plugin_init(plugins):
21    return RestApiPlugin()
22
23class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
24    def __init__(self):
25        self.type = "restapi"
26
27    def load_index(self, url, load):
28        """
29            Fetches layer information from a local or remote layer index.
30
31            The return value is a LayerIndexObj.
32
33            url is the url to the rest api of the layer index, such as:
34            http://layers.openembedded.org/layerindex/api/
35
36            Or a local file...
37        """
38
39        up = urlparse(url)
40
41        if up.scheme == 'file':
42            return self.load_index_file(up, url, load)
43
44        if up.scheme == 'http' or up.scheme == 'https':
45            return self.load_index_web(up, url, load)
46
47        raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
48
49
50    def load_index_file(self, up, url, load):
51        """
52            Fetches layer information from a local file or directory.
53
54            The return value is a LayerIndexObj.
55
56            ud is the parsed url to the local file or directory.
57        """
58        if not os.path.exists(up.path):
59            raise FileNotFoundError(up.path)
60
61        index = layerindexlib.LayerIndexObj()
62
63        index.config = {}
64        index.config['TYPE'] = self.type
65        index.config['URL'] = url
66
67        params = self.layerindex._parse_params(up.params)
68
69        if 'desc' in params:
70            index.config['DESCRIPTION'] = unquote(params['desc'])
71        else:
72            index.config['DESCRIPTION'] = up.path
73
74        if 'cache' in params:
75            index.config['CACHE'] = params['cache']
76
77        if 'branch' in params:
78            branches = params['branch'].split(',')
79            index.config['BRANCH'] = branches
80        else:
81            branches = ['*']
82
83
84        def load_cache(path, index, branches=[]):
85            logger.debug('Loading json file %s' % path)
86            with open(path, 'rt', encoding='utf-8') as f:
87                pindex = json.load(f)
88
89            # Filter the branches on loaded files...
90            newpBranch = []
91            for branch in branches:
92                if branch != '*':
93                    if 'branches' in pindex:
94                        for br in pindex['branches']:
95                            if br['name'] == branch:
96                                newpBranch.append(br)
97                else:
98                    if 'branches' in pindex:
99                        for br in pindex['branches']:
100                            newpBranch.append(br)
101
102            if newpBranch:
103                index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
104            else:
105                logger.debug('No matching branches (%s) in index file(s)' % branches)
106                # No matching branches.. return nothing...
107                return
108
109            for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
110                                   ("layerBranches", layerindexlib.LayerBranch),
111                                   ("layerDependencies", layerindexlib.LayerDependency),
112                                   ("recipes", layerindexlib.Recipe),
113                                   ("machines", layerindexlib.Machine),
114                                   ("distros", layerindexlib.Distro)]:
115                if lName in pindex:
116                    index.add_raw_element(lName, lType, pindex[lName])
117
118
119        if not os.path.isdir(up.path):
120            load_cache(up.path, index, branches)
121            return index
122
123        logger.debug('Loading from dir %s...' % (up.path))
124        for (dirpath, _, filenames) in os.walk(up.path):
125            for filename in filenames:
126                if not filename.endswith('.json'):
127                    continue
128                fpath = os.path.join(dirpath, filename)
129                load_cache(fpath, index, branches)
130
131        return index
132
133
134    def load_index_web(self, up, url, load):
135        """
136            Fetches layer information from a remote layer index.
137
138            The return value is a LayerIndexObj.
139
140            ud is the parsed url to the rest api of the layer index, such as:
141            http://layers.openembedded.org/layerindex/api/
142        """
143
144        def _get_json_response(apiurl=None, username=None, password=None, retry=True):
145            assert apiurl is not None
146
147            logger.debug("fetching %s" % apiurl)
148
149            up = urlparse(apiurl)
150
151            username=up.username
152            password=up.password
153
154            # Strip username/password and params
155            if up.port:
156                up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
157            else:
158                up_stripped = up._replace(params="", netloc=up.hostname)
159
160            res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
161
162            try:
163                parsed = json.loads(res.read().decode('utf-8'))
164            except ConnectionResetError:
165                if retry:
166                    logger.debug("%s: Connection reset by peer.  Retrying..." % url)
167                    parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
168                    logger.debug("%s: retry successful.")
169                else:
170                    raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer.  Is there a firewall blocking your connection?' % apiurl)
171
172            return parsed
173
174        index = layerindexlib.LayerIndexObj()
175
176        index.config = {}
177        index.config['TYPE'] = self.type
178        index.config['URL'] = url
179
180        params = self.layerindex._parse_params(up.params)
181
182        if 'desc' in params:
183            index.config['DESCRIPTION'] = unquote(params['desc'])
184        else:
185            index.config['DESCRIPTION'] = up.hostname
186
187        if 'cache' in params:
188            index.config['CACHE'] = params['cache']
189
190        if 'branch' in params:
191            branches = params['branch'].split(',')
192            index.config['BRANCH'] = branches
193        else:
194            branches = ['*']
195
196        try:
197            index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
198        except Exception as e:
199            raise layerindexlib.LayerIndexFetchError(url, e)
200
201        # Local raw index set...
202        pindex = {}
203
204        # Load all the requested branches at the same time time,
205        # a special branch of '*' means load all branches
206        filter = ""
207        if "*" not in branches:
208            filter = "?filter=name:%s" % "OR".join(branches)
209
210        logger.debug("Loading %s from %s" % (branches, index.apilinks['branches']))
211
212        # The link won't include username/password, so pull it from the original url
213        pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
214                                                    username=up.username, password=up.password)
215        if not pindex['branches']:
216            logger.debug("No valid branches (%s) found at url %s." % (branch, url))
217            return index
218        index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
219
220        # Load all of the layerItems (these can not be easily filtered)
221        logger.debug("Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
222
223
224        # The link won't include username/password, so pull it from the original url
225        pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
226                                                  username=up.username, password=up.password)
227        if not pindex['layerItems']:
228            logger.debug("No layers were found at url %s." % (url))
229            return index
230        index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
231
232
233	# From this point on load the contents for each branch.  Otherwise we
234	# could run into a timeout.
235        for branch in index.branches:
236            filter = "?filter=branch__name:%s" % index.branches[branch].name
237
238            logger.debug("Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
239
240            # The link won't include username/password, so pull it from the original url
241            pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
242                                                  username=up.username, password=up.password)
243            if not pindex['layerBranches']:
244                logger.debug("No valid layer branches (%s) found at url %s." % (branches or "*", url))
245                return index
246            index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
247
248
249            # Load the rest, they all have a similar format
250            # Note: the layer index has a few more items, we can add them if necessary
251            # in the future.
252            filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
253            for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
254                                   ("recipes", layerindexlib.Recipe),
255                                   ("machines", layerindexlib.Machine),
256                                   ("distros", layerindexlib.Distro)]:
257                if lName not in load:
258                    continue
259                logger.debug("Loading %s from %s" % (lName, index.apilinks[lName]))
260
261                # The link won't include username/password, so pull it from the original url
262                pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
263                                            username=up.username, password=up.password)
264                index.add_raw_element(lName, lType, pindex[lName])
265
266        return index
267
268    def store_index(self, url, index):
269        """
270            Store layer information into a local file/dir.
271
272            The return value is a dictionary containing API,
273            layer, branch, dependency, recipe, machine, distro, information.
274
275            ud is a parsed url to a directory or file.  If the path is a
276            directory, we will split the files into one file per layer.
277            If the path is to a file (exists or not) the entire DB will be
278            dumped into that one file.
279        """
280
281        up = urlparse(url)
282
283        if up.scheme != 'file':
284            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
285
286        logger.debug("Storing to %s..." % up.path)
287
288        try:
289            layerbranches = index.layerBranches
290        except KeyError:
291            logger.error('No layerBranches to write.')
292            return
293
294
295        def filter_item(layerbranchid, objects):
296            filtered = []
297            for obj in getattr(index, objects, None):
298                try:
299                    if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
300                       filtered.append(getattr(index, objects)[obj]._data)
301                except AttributeError:
302                    logger.debug('No obj.layerbranch_id: %s' % objects)
303                    # No simple filter method, just include it...
304                    try:
305                        filtered.append(getattr(index, objects)[obj]._data)
306                    except AttributeError:
307                        logger.debug('No obj._data: %s %s' % (objects, type(obj)))
308                        filtered.append(obj)
309            return filtered
310
311
312        # Write out to a single file.
313        # Filter out unnecessary items, then sort as we write for determinism
314        if not os.path.isdir(up.path):
315            pindex = {}
316
317            pindex['branches'] = []
318            pindex['layerItems'] = []
319            pindex['layerBranches'] = []
320
321            for layerbranchid in layerbranches:
322                if layerbranches[layerbranchid].branch._data not in pindex['branches']:
323                    pindex['branches'].append(layerbranches[layerbranchid].branch._data)
324
325                if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
326                    pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
327
328                if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
329                    pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
330
331                for entry in index._index:
332                    # Skip local items, apilinks and items already processed
333                    if entry in index.config['local'] or \
334                       entry == 'apilinks' or \
335                       entry == 'branches' or \
336                       entry == 'layerBranches' or \
337                       entry == 'layerItems':
338                        continue
339                    if entry not in pindex:
340                        pindex[entry] = []
341                    pindex[entry].extend(filter_item(layerbranchid, entry))
342
343            bb.debug(1, 'Writing index to %s' % up.path)
344            with open(up.path, 'wt') as f:
345                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
346            return
347
348
349        # Write out to a directory one file per layerBranch
350        # Prepare all layer related items, to create a minimal file.
351        # We have to sort the entries as we write so they are deterministic
352        for layerbranchid in layerbranches:
353            pindex = {}
354
355            for entry in index._index:
356                # Skip local items, apilinks and items already processed
357                if entry in index.config['local'] or \
358                   entry == 'apilinks' or \
359                   entry == 'branches' or \
360                   entry == 'layerBranches' or \
361                   entry == 'layerItems':
362                    continue
363                pindex[entry] = filter_item(layerbranchid, entry)
364
365            # Add the layer we're processing as the first one...
366            pindex['branches'] = [layerbranches[layerbranchid].branch._data]
367            pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
368            pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
369
370            # We also need to include the layerbranch for any dependencies...
371            for layerdep in pindex['layerDependencies']:
372                layerdependency = layerindexlib.LayerDependency(index, layerdep)
373
374                layeritem = layerdependency.dependency
375                layerbranch = layerdependency.dependency_layerBranch
376
377                # We need to avoid duplicates...
378                if layeritem._data not in pindex['layerItems']:
379                    pindex['layerItems'].append(layeritem._data)
380
381                if layerbranch._data not in pindex['layerBranches']:
382                    pindex['layerBranches'].append(layerbranch._data)
383
384            # apply mirroring adjustments here....
385
386            fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
387            fname = fname.translate(str.maketrans('/ ', '__'))
388            fpath = os.path.join(up.path, fname)
389
390            bb.debug(1, 'Writing index to %s' % fpath + '.json')
391            with open(fpath + '.json', 'wt') as f:
392                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
393