1# Copyright (C) 2016-2018 Wind River Systems, Inc. 2# 3# This program is free software; you can redistribute it and/or modify 4# it under the terms of the GNU General Public License version 2 as 5# published by the Free Software Foundation. 6# 7# This program is distributed in the hope that it will be useful, 8# but WITHOUT ANY WARRANTY; without even the implied warranty of 9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10# See the GNU General Public License for more details. 11# 12# You should have received a copy of the GNU General Public License 13# along with this program; if not, write to the Free Software 14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 15 16import logging 17import json 18from urllib.parse import unquote 19from urllib.parse import urlparse 20 21import layerindexlib 22import layerindexlib.plugin 23 24logger = logging.getLogger('BitBake.layerindexlib.restapi') 25 26def plugin_init(plugins): 27 return RestApiPlugin() 28 29class RestApiPlugin(layerindexlib.plugin.IndexPlugin): 30 def __init__(self): 31 self.type = "restapi" 32 33 def load_index(self, url, load): 34 """ 35 Fetches layer information from a local or remote layer index. 36 37 The return value is a LayerIndexObj. 38 39 url is the url to the rest api of the layer index, such as: 40 http://layers.openembedded.org/layerindex/api/ 41 42 Or a local file... 43 """ 44 45 up = urlparse(url) 46 47 if up.scheme == 'file': 48 return self.load_index_file(up, url, load) 49 50 if up.scheme == 'http' or up.scheme == 'https': 51 return self.load_index_web(up, url, load) 52 53 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) 54 55 56 def load_index_file(self, up, url, load): 57 """ 58 Fetches layer information from a local file or directory. 59 60 The return value is a LayerIndexObj. 61 62 ud is the parsed url to the local file or directory. 63 """ 64 if not os.path.exists(up.path): 65 raise FileNotFoundError(up.path) 66 67 index = layerindexlib.LayerIndexObj() 68 69 index.config = {} 70 index.config['TYPE'] = self.type 71 index.config['URL'] = url 72 73 params = self.layerindex._parse_params(up.params) 74 75 if 'desc' in params: 76 index.config['DESCRIPTION'] = unquote(params['desc']) 77 else: 78 index.config['DESCRIPTION'] = up.path 79 80 if 'cache' in params: 81 index.config['CACHE'] = params['cache'] 82 83 if 'branch' in params: 84 branches = params['branch'].split(',') 85 index.config['BRANCH'] = branches 86 else: 87 branches = ['*'] 88 89 90 def load_cache(path, index, branches=[]): 91 logger.debug(1, 'Loading json file %s' % path) 92 with open(path, 'rt', encoding='utf-8') as f: 93 pindex = json.load(f) 94 95 # Filter the branches on loaded files... 96 newpBranch = [] 97 for branch in branches: 98 if branch != '*': 99 if 'branches' in pindex: 100 for br in pindex['branches']: 101 if br['name'] == branch: 102 newpBranch.append(br) 103 else: 104 if 'branches' in pindex: 105 for br in pindex['branches']: 106 newpBranch.append(br) 107 108 if newpBranch: 109 index.add_raw_element('branches', layerindexlib.Branch, newpBranch) 110 else: 111 logger.debug(1, 'No matching branches (%s) in index file(s)' % branches) 112 # No matching branches.. return nothing... 113 return 114 115 for (lName, lType) in [("layerItems", layerindexlib.LayerItem), 116 ("layerBranches", layerindexlib.LayerBranch), 117 ("layerDependencies", layerindexlib.LayerDependency), 118 ("recipes", layerindexlib.Recipe), 119 ("machines", layerindexlib.Machine), 120 ("distros", layerindexlib.Distro)]: 121 if lName in pindex: 122 index.add_raw_element(lName, lType, pindex[lName]) 123 124 125 if not os.path.isdir(up.path): 126 load_cache(up.path, index, branches) 127 return index 128 129 logger.debug(1, 'Loading from dir %s...' % (up.path)) 130 for (dirpath, _, filenames) in os.walk(up.path): 131 for filename in filenames: 132 if not filename.endswith('.json'): 133 continue 134 fpath = os.path.join(dirpath, filename) 135 load_cache(fpath, index, branches) 136 137 return index 138 139 140 def load_index_web(self, up, url, load): 141 """ 142 Fetches layer information from a remote layer index. 143 144 The return value is a LayerIndexObj. 145 146 ud is the parsed url to the rest api of the layer index, such as: 147 http://layers.openembedded.org/layerindex/api/ 148 """ 149 150 def _get_json_response(apiurl=None, username=None, password=None, retry=True): 151 assert apiurl is not None 152 153 logger.debug(1, "fetching %s" % apiurl) 154 155 up = urlparse(apiurl) 156 157 username=up.username 158 password=up.password 159 160 # Strip username/password and params 161 if up.port: 162 up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port)) 163 else: 164 up_stripped = up._replace(params="", netloc=up.hostname) 165 166 res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password) 167 168 try: 169 parsed = json.loads(res.read().decode('utf-8')) 170 except ConnectionResetError: 171 if retry: 172 logger.debug(1, "%s: Connection reset by peer. Retrying..." % url) 173 parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False) 174 logger.debug(1, "%s: retry successful.") 175 else: 176 raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) 177 178 return parsed 179 180 index = layerindexlib.LayerIndexObj() 181 182 index.config = {} 183 index.config['TYPE'] = self.type 184 index.config['URL'] = url 185 186 params = self.layerindex._parse_params(up.params) 187 188 if 'desc' in params: 189 index.config['DESCRIPTION'] = unquote(params['desc']) 190 else: 191 index.config['DESCRIPTION'] = up.hostname 192 193 if 'cache' in params: 194 index.config['CACHE'] = params['cache'] 195 196 if 'branch' in params: 197 branches = params['branch'].split(',') 198 index.config['BRANCH'] = branches 199 else: 200 branches = ['*'] 201 202 try: 203 index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password) 204 except Exception as e: 205 raise layerindexlib.LayerIndexFetchError(url, e) 206 207 # Local raw index set... 208 pindex = {} 209 210 # Load all the requested branches at the same time time, 211 # a special branch of '*' means load all branches 212 filter = "" 213 if "*" not in branches: 214 filter = "?filter=name:%s" % "OR".join(branches) 215 216 logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches'])) 217 218 # The link won't include username/password, so pull it from the original url 219 pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter, 220 username=up.username, password=up.password) 221 if not pindex['branches']: 222 logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url)) 223 return index 224 index.add_raw_element("branches", layerindexlib.Branch, pindex['branches']) 225 226 # Load all of the layerItems (these can not be easily filtered) 227 logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems'])) 228 229 230 # The link won't include username/password, so pull it from the original url 231 pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'], 232 username=up.username, password=up.password) 233 if not pindex['layerItems']: 234 logger.debug(1, "No layers were found at url %s." % (url)) 235 return index 236 index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems']) 237 238 239 # From this point on load the contents for each branch. Otherwise we 240 # could run into a timeout. 241 for branch in index.branches: 242 filter = "?filter=branch__name:%s" % index.branches[branch].name 243 244 logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches'])) 245 246 # The link won't include username/password, so pull it from the original url 247 pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter, 248 username=up.username, password=up.password) 249 if not pindex['layerBranches']: 250 logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url)) 251 return index 252 index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches']) 253 254 255 # Load the rest, they all have a similar format 256 # Note: the layer index has a few more items, we can add them if necessary 257 # in the future. 258 filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name 259 for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency), 260 ("recipes", layerindexlib.Recipe), 261 ("machines", layerindexlib.Machine), 262 ("distros", layerindexlib.Distro)]: 263 if lName not in load: 264 continue 265 logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName])) 266 267 # The link won't include username/password, so pull it from the original url 268 pindex[lName] = _get_json_response(index.apilinks[lName] + filter, 269 username=up.username, password=up.password) 270 index.add_raw_element(lName, lType, pindex[lName]) 271 272 return index 273 274 def store_index(self, url, index): 275 """ 276 Store layer information into a local file/dir. 277 278 The return value is a dictionary containing API, 279 layer, branch, dependency, recipe, machine, distro, information. 280 281 ud is a parsed url to a directory or file. If the path is a 282 directory, we will split the files into one file per layer. 283 If the path is to a file (exists or not) the entire DB will be 284 dumped into that one file. 285 """ 286 287 up = urlparse(url) 288 289 if up.scheme != 'file': 290 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) 291 292 logger.debug(1, "Storing to %s..." % up.path) 293 294 try: 295 layerbranches = index.layerBranches 296 except KeyError: 297 logger.error('No layerBranches to write.') 298 return 299 300 301 def filter_item(layerbranchid, objects): 302 filtered = [] 303 for obj in getattr(index, objects, None): 304 try: 305 if getattr(index, objects)[obj].layerbranch_id == layerbranchid: 306 filtered.append(getattr(index, objects)[obj]._data) 307 except AttributeError: 308 logger.debug(1, 'No obj.layerbranch_id: %s' % objects) 309 # No simple filter method, just include it... 310 try: 311 filtered.append(getattr(index, objects)[obj]._data) 312 except AttributeError: 313 logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj))) 314 filtered.append(obj) 315 return filtered 316 317 318 # Write out to a single file. 319 # Filter out unnecessary items, then sort as we write for determinism 320 if not os.path.isdir(up.path): 321 pindex = {} 322 323 pindex['branches'] = [] 324 pindex['layerItems'] = [] 325 pindex['layerBranches'] = [] 326 327 for layerbranchid in layerbranches: 328 if layerbranches[layerbranchid].branch._data not in pindex['branches']: 329 pindex['branches'].append(layerbranches[layerbranchid].branch._data) 330 331 if layerbranches[layerbranchid].layer._data not in pindex['layerItems']: 332 pindex['layerItems'].append(layerbranches[layerbranchid].layer._data) 333 334 if layerbranches[layerbranchid]._data not in pindex['layerBranches']: 335 pindex['layerBranches'].append(layerbranches[layerbranchid]._data) 336 337 for entry in index._index: 338 # Skip local items, apilinks and items already processed 339 if entry in index.config['local'] or \ 340 entry == 'apilinks' or \ 341 entry == 'branches' or \ 342 entry == 'layerBranches' or \ 343 entry == 'layerItems': 344 continue 345 if entry not in pindex: 346 pindex[entry] = [] 347 pindex[entry].extend(filter_item(layerbranchid, entry)) 348 349 bb.debug(1, 'Writing index to %s' % up.path) 350 with open(up.path, 'wt') as f: 351 json.dump(layerindexlib.sort_entry(pindex), f, indent=4) 352 return 353 354 355 # Write out to a directory one file per layerBranch 356 # Prepare all layer related items, to create a minimal file. 357 # We have to sort the entries as we write so they are deterministic 358 for layerbranchid in layerbranches: 359 pindex = {} 360 361 for entry in index._index: 362 # Skip local items, apilinks and items already processed 363 if entry in index.config['local'] or \ 364 entry == 'apilinks' or \ 365 entry == 'branches' or \ 366 entry == 'layerBranches' or \ 367 entry == 'layerItems': 368 continue 369 pindex[entry] = filter_item(layerbranchid, entry) 370 371 # Add the layer we're processing as the first one... 372 pindex['branches'] = [layerbranches[layerbranchid].branch._data] 373 pindex['layerItems'] = [layerbranches[layerbranchid].layer._data] 374 pindex['layerBranches'] = [layerbranches[layerbranchid]._data] 375 376 # We also need to include the layerbranch for any dependencies... 377 for layerdep in pindex['layerDependencies']: 378 layerdependency = layerindexlib.LayerDependency(index, layerdep) 379 380 layeritem = layerdependency.dependency 381 layerbranch = layerdependency.dependency_layerBranch 382 383 # We need to avoid duplicates... 384 if layeritem._data not in pindex['layerItems']: 385 pindex['layerItems'].append(layeritem._data) 386 387 if layerbranch._data not in pindex['layerBranches']: 388 pindex['layerBranches'].append(layerbranch._data) 389 390 # apply mirroring adjustments here.... 391 392 fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] 393 fname = fname.translate(str.maketrans('/ ', '__')) 394 fpath = os.path.join(up.path, fname) 395 396 bb.debug(1, 'Writing index to %s' % fpath + '.json') 397 with open(fpath + '.json', 'wt') as f: 398 json.dump(layerindexlib.sort_entry(pindex), f, indent=4) 399