1# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import logging
17import json
18from urllib.parse import unquote
19from urllib.parse import urlparse
20
21import layerindexlib
22import layerindexlib.plugin
23
24logger = logging.getLogger('BitBake.layerindexlib.restapi')
25
26def plugin_init(plugins):
27    return RestApiPlugin()
28
29class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
30    def __init__(self):
31        self.type = "restapi"
32
33    def load_index(self, url, load):
34        """
35            Fetches layer information from a local or remote layer index.
36
37            The return value is a LayerIndexObj.
38
39            url is the url to the rest api of the layer index, such as:
40            http://layers.openembedded.org/layerindex/api/
41
42            Or a local file...
43        """
44
45        up = urlparse(url)
46
47        if up.scheme == 'file':
48            return self.load_index_file(up, url, load)
49
50        if up.scheme == 'http' or up.scheme == 'https':
51            return self.load_index_web(up, url, load)
52
53        raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
54
55
56    def load_index_file(self, up, url, load):
57        """
58            Fetches layer information from a local file or directory.
59
60            The return value is a LayerIndexObj.
61
62            ud is the parsed url to the local file or directory.
63        """
64        if not os.path.exists(up.path):
65            raise FileNotFoundError(up.path)
66
67        index = layerindexlib.LayerIndexObj()
68
69        index.config = {}
70        index.config['TYPE'] = self.type
71        index.config['URL'] = url
72
73        params = self.layerindex._parse_params(up.params)
74
75        if 'desc' in params:
76            index.config['DESCRIPTION'] = unquote(params['desc'])
77        else:
78            index.config['DESCRIPTION'] = up.path
79
80        if 'cache' in params:
81            index.config['CACHE'] = params['cache']
82
83        if 'branch' in params:
84            branches = params['branch'].split(',')
85            index.config['BRANCH'] = branches
86        else:
87            branches = ['*']
88
89
90        def load_cache(path, index, branches=[]):
91            logger.debug(1, 'Loading json file %s' % path)
92            with open(path, 'rt', encoding='utf-8') as f:
93                pindex = json.load(f)
94
95            # Filter the branches on loaded files...
96            newpBranch = []
97            for branch in branches:
98                if branch != '*':
99                    if 'branches' in pindex:
100                        for br in pindex['branches']:
101                            if br['name'] == branch:
102                                newpBranch.append(br)
103                else:
104                    if 'branches' in pindex:
105                        for br in pindex['branches']:
106                            newpBranch.append(br)
107
108            if newpBranch:
109                index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
110            else:
111                logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
112                # No matching branches.. return nothing...
113                return
114
115            for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
116                                   ("layerBranches", layerindexlib.LayerBranch),
117                                   ("layerDependencies", layerindexlib.LayerDependency),
118                                   ("recipes", layerindexlib.Recipe),
119                                   ("machines", layerindexlib.Machine),
120                                   ("distros", layerindexlib.Distro)]:
121                if lName in pindex:
122                    index.add_raw_element(lName, lType, pindex[lName])
123
124
125        if not os.path.isdir(up.path):
126            load_cache(up.path, index, branches)
127            return index
128
129        logger.debug(1, 'Loading from dir %s...' % (up.path))
130        for (dirpath, _, filenames) in os.walk(up.path):
131            for filename in filenames:
132                if not filename.endswith('.json'):
133                    continue
134                fpath = os.path.join(dirpath, filename)
135                load_cache(fpath, index, branches)
136
137        return index
138
139
140    def load_index_web(self, up, url, load):
141        """
142            Fetches layer information from a remote layer index.
143
144            The return value is a LayerIndexObj.
145
146            ud is the parsed url to the rest api of the layer index, such as:
147            http://layers.openembedded.org/layerindex/api/
148        """
149
150        def _get_json_response(apiurl=None, username=None, password=None, retry=True):
151            assert apiurl is not None
152
153            logger.debug(1, "fetching %s" % apiurl)
154
155            up = urlparse(apiurl)
156
157            username=up.username
158            password=up.password
159
160            # Strip username/password and params
161            if up.port:
162                up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
163            else:
164                up_stripped = up._replace(params="", netloc=up.hostname)
165
166            res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
167
168            try:
169                parsed = json.loads(res.read().decode('utf-8'))
170            except ConnectionResetError:
171                if retry:
172                    logger.debug(1, "%s: Connection reset by peer.  Retrying..." % url)
173                    parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
174                    logger.debug(1, "%s: retry successful.")
175                else:
176                    raise LayerIndexFetchError('%s: Connection reset by peer.  Is there a firewall blocking your connection?' % apiurl)
177
178            return parsed
179
180        index = layerindexlib.LayerIndexObj()
181
182        index.config = {}
183        index.config['TYPE'] = self.type
184        index.config['URL'] = url
185
186        params = self.layerindex._parse_params(up.params)
187
188        if 'desc' in params:
189            index.config['DESCRIPTION'] = unquote(params['desc'])
190        else:
191            index.config['DESCRIPTION'] = up.hostname
192
193        if 'cache' in params:
194            index.config['CACHE'] = params['cache']
195
196        if 'branch' in params:
197            branches = params['branch'].split(',')
198            index.config['BRANCH'] = branches
199        else:
200            branches = ['*']
201
202        try:
203            index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
204        except Exception as e:
205            raise layerindexlib.LayerIndexFetchError(url, e)
206
207        # Local raw index set...
208        pindex = {}
209
210        # Load all the requested branches at the same time time,
211        # a special branch of '*' means load all branches
212        filter = ""
213        if "*" not in branches:
214            filter = "?filter=name:%s" % "OR".join(branches)
215
216        logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
217
218        # The link won't include username/password, so pull it from the original url
219        pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
220                                                    username=up.username, password=up.password)
221        if not pindex['branches']:
222            logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
223            return index
224        index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
225
226        # Load all of the layerItems (these can not be easily filtered)
227        logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
228
229
230        # The link won't include username/password, so pull it from the original url
231        pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
232                                                  username=up.username, password=up.password)
233        if not pindex['layerItems']:
234            logger.debug(1, "No layers were found at url %s." % (url))
235            return index
236        index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
237
238
239	# From this point on load the contents for each branch.  Otherwise we
240	# could run into a timeout.
241        for branch in index.branches:
242            filter = "?filter=branch__name:%s" % index.branches[branch].name
243
244            logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
245
246            # The link won't include username/password, so pull it from the original url
247            pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
248                                                  username=up.username, password=up.password)
249            if not pindex['layerBranches']:
250                logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
251                return index
252            index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
253
254
255            # Load the rest, they all have a similar format
256            # Note: the layer index has a few more items, we can add them if necessary
257            # in the future.
258            filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
259            for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
260                                   ("recipes", layerindexlib.Recipe),
261                                   ("machines", layerindexlib.Machine),
262                                   ("distros", layerindexlib.Distro)]:
263                if lName not in load:
264                    continue
265                logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
266
267                # The link won't include username/password, so pull it from the original url
268                pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
269                                            username=up.username, password=up.password)
270                index.add_raw_element(lName, lType, pindex[lName])
271
272        return index
273
274    def store_index(self, url, index):
275        """
276            Store layer information into a local file/dir.
277
278            The return value is a dictionary containing API,
279            layer, branch, dependency, recipe, machine, distro, information.
280
281            ud is a parsed url to a directory or file.  If the path is a
282            directory, we will split the files into one file per layer.
283            If the path is to a file (exists or not) the entire DB will be
284            dumped into that one file.
285        """
286
287        up = urlparse(url)
288
289        if up.scheme != 'file':
290            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
291
292        logger.debug(1, "Storing to %s..." % up.path)
293
294        try:
295            layerbranches = index.layerBranches
296        except KeyError:
297            logger.error('No layerBranches to write.')
298            return
299
300
301        def filter_item(layerbranchid, objects):
302            filtered = []
303            for obj in getattr(index, objects, None):
304                try:
305                    if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
306                       filtered.append(getattr(index, objects)[obj]._data)
307                except AttributeError:
308                    logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
309                    # No simple filter method, just include it...
310                    try:
311                        filtered.append(getattr(index, objects)[obj]._data)
312                    except AttributeError:
313                        logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
314                        filtered.append(obj)
315            return filtered
316
317
318        # Write out to a single file.
319        # Filter out unnecessary items, then sort as we write for determinism
320        if not os.path.isdir(up.path):
321            pindex = {}
322
323            pindex['branches'] = []
324            pindex['layerItems'] = []
325            pindex['layerBranches'] = []
326
327            for layerbranchid in layerbranches:
328                if layerbranches[layerbranchid].branch._data not in pindex['branches']:
329                    pindex['branches'].append(layerbranches[layerbranchid].branch._data)
330
331                if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
332                    pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
333
334                if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
335                    pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
336
337                for entry in index._index:
338                    # Skip local items, apilinks and items already processed
339                    if entry in index.config['local'] or \
340                       entry == 'apilinks' or \
341                       entry == 'branches' or \
342                       entry == 'layerBranches' or \
343                       entry == 'layerItems':
344                        continue
345                    if entry not in pindex:
346                        pindex[entry] = []
347                    pindex[entry].extend(filter_item(layerbranchid, entry))
348
349            bb.debug(1, 'Writing index to %s' % up.path)
350            with open(up.path, 'wt') as f:
351                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
352            return
353
354
355        # Write out to a directory one file per layerBranch
356        # Prepare all layer related items, to create a minimal file.
357        # We have to sort the entries as we write so they are deterministic
358        for layerbranchid in layerbranches:
359            pindex = {}
360
361            for entry in index._index:
362                # Skip local items, apilinks and items already processed
363                if entry in index.config['local'] or \
364                   entry == 'apilinks' or \
365                   entry == 'branches' or \
366                   entry == 'layerBranches' or \
367                   entry == 'layerItems':
368                    continue
369                pindex[entry] = filter_item(layerbranchid, entry)
370
371            # Add the layer we're processing as the first one...
372            pindex['branches'] = [layerbranches[layerbranchid].branch._data]
373            pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
374            pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
375
376            # We also need to include the layerbranch for any dependencies...
377            for layerdep in pindex['layerDependencies']:
378                layerdependency = layerindexlib.LayerDependency(index, layerdep)
379
380                layeritem = layerdependency.dependency
381                layerbranch = layerdependency.dependency_layerBranch
382
383                # We need to avoid duplicates...
384                if layeritem._data not in pindex['layerItems']:
385                    pindex['layerItems'].append(layeritem._data)
386
387                if layerbranch._data not in pindex['layerBranches']:
388                    pindex['layerBranches'].append(layerbranch._data)
389
390            # apply mirroring adjustments here....
391
392            fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
393            fname = fname.translate(str.maketrans('/ ', '__'))
394            fpath = os.path.join(up.path, fname)
395
396            bb.debug(1, 'Writing index to %s' % fpath + '.json')
397            with open(fpath + '.json', 'wt') as f:
398                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
399