1# Copyright (C) 2016-2018 Wind River Systems, Inc. 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5 6import logging 7import json 8import os 9 10from urllib.parse import unquote 11from urllib.parse import urlparse 12 13import bb 14 15import layerindexlib 16import layerindexlib.plugin 17 18logger = logging.getLogger('BitBake.layerindexlib.restapi') 19 20def plugin_init(plugins): 21 return RestApiPlugin() 22 23class RestApiPlugin(layerindexlib.plugin.IndexPlugin): 24 def __init__(self): 25 self.type = "restapi" 26 27 def load_index(self, url, load): 28 """ 29 Fetches layer information from a local or remote layer index. 30 31 The return value is a LayerIndexObj. 32 33 url is the url to the rest api of the layer index, such as: 34 https://layers.openembedded.org/layerindex/api/ 35 36 Or a local file... 37 """ 38 39 up = urlparse(url) 40 41 if up.scheme == 'file': 42 return self.load_index_file(up, url, load) 43 44 if up.scheme == 'http' or up.scheme == 'https': 45 return self.load_index_web(up, url, load) 46 47 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) 48 49 50 def load_index_file(self, up, url, load): 51 """ 52 Fetches layer information from a local file or directory. 53 54 The return value is a LayerIndexObj. 55 56 ud is the parsed url to the local file or directory. 57 """ 58 if not os.path.exists(up.path): 59 raise FileNotFoundError(up.path) 60 61 index = layerindexlib.LayerIndexObj() 62 63 index.config = {} 64 index.config['TYPE'] = self.type 65 index.config['URL'] = url 66 67 params = self.layerindex._parse_params(up.params) 68 69 if 'desc' in params: 70 index.config['DESCRIPTION'] = unquote(params['desc']) 71 else: 72 index.config['DESCRIPTION'] = up.path 73 74 if 'cache' in params: 75 index.config['CACHE'] = params['cache'] 76 77 if 'branch' in params: 78 branches = params['branch'].split(',') 79 index.config['BRANCH'] = branches 80 else: 81 branches = ['*'] 82 83 84 def load_cache(path, index, branches=[]): 85 logger.debug('Loading json file %s' % path) 86 with open(path, 'rt', encoding='utf-8') as f: 87 pindex = json.load(f) 88 89 # Filter the branches on loaded files... 90 newpBranch = [] 91 for branch in branches: 92 if branch != '*': 93 if 'branches' in pindex: 94 for br in pindex['branches']: 95 if br['name'] == branch: 96 newpBranch.append(br) 97 else: 98 if 'branches' in pindex: 99 for br in pindex['branches']: 100 newpBranch.append(br) 101 102 if newpBranch: 103 index.add_raw_element('branches', layerindexlib.Branch, newpBranch) 104 else: 105 logger.debug('No matching branches (%s) in index file(s)' % branches) 106 # No matching branches.. return nothing... 107 return 108 109 for (lName, lType) in [("layerItems", layerindexlib.LayerItem), 110 ("layerBranches", layerindexlib.LayerBranch), 111 ("layerDependencies", layerindexlib.LayerDependency), 112 ("recipes", layerindexlib.Recipe), 113 ("machines", layerindexlib.Machine), 114 ("distros", layerindexlib.Distro)]: 115 if lName in pindex: 116 index.add_raw_element(lName, lType, pindex[lName]) 117 118 119 if not os.path.isdir(up.path): 120 load_cache(up.path, index, branches) 121 return index 122 123 logger.debug('Loading from dir %s...' % (up.path)) 124 for (dirpath, _, filenames) in os.walk(up.path): 125 for filename in filenames: 126 if not filename.endswith('.json'): 127 continue 128 fpath = os.path.join(dirpath, filename) 129 load_cache(fpath, index, branches) 130 131 return index 132 133 134 def load_index_web(self, up, url, load): 135 """ 136 Fetches layer information from a remote layer index. 137 138 The return value is a LayerIndexObj. 139 140 ud is the parsed url to the rest api of the layer index, such as: 141 https://layers.openembedded.org/layerindex/api/ 142 """ 143 144 def _get_json_response(apiurl=None, username=None, password=None, retry=True): 145 assert apiurl is not None 146 147 logger.debug("fetching %s" % apiurl) 148 149 up = urlparse(apiurl) 150 151 username=up.username 152 password=up.password 153 154 # Strip username/password and params 155 if up.port: 156 up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port)) 157 else: 158 up_stripped = up._replace(params="", netloc=up.hostname) 159 160 res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password) 161 162 try: 163 parsed = json.loads(res.read().decode('utf-8')) 164 except ConnectionResetError: 165 if retry: 166 logger.debug("%s: Connection reset by peer. Retrying..." % url) 167 parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False) 168 logger.debug("%s: retry successful.") 169 else: 170 raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) 171 172 return parsed 173 174 index = layerindexlib.LayerIndexObj() 175 176 index.config = {} 177 index.config['TYPE'] = self.type 178 index.config['URL'] = url 179 180 params = self.layerindex._parse_params(up.params) 181 182 if 'desc' in params: 183 index.config['DESCRIPTION'] = unquote(params['desc']) 184 else: 185 index.config['DESCRIPTION'] = up.hostname 186 187 if 'cache' in params: 188 index.config['CACHE'] = params['cache'] 189 190 if 'branch' in params: 191 branches = params['branch'].split(',') 192 index.config['BRANCH'] = branches 193 else: 194 branches = ['*'] 195 196 try: 197 index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password) 198 except Exception as e: 199 raise layerindexlib.LayerIndexFetchError(url, e) 200 201 # Local raw index set... 202 pindex = {} 203 204 # Load all the requested branches at the same time time, 205 # a special branch of '*' means load all branches 206 filter = "" 207 if "*" not in branches: 208 filter = "?filter=name:%s" % "OR".join(branches) 209 210 logger.debug("Loading %s from %s" % (branches, index.apilinks['branches'])) 211 212 # The link won't include username/password, so pull it from the original url 213 pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter, 214 username=up.username, password=up.password) 215 if not pindex['branches']: 216 logger.debug("No valid branches (%s) found at url %s." % (branch, url)) 217 return index 218 index.add_raw_element("branches", layerindexlib.Branch, pindex['branches']) 219 220 # Load all of the layerItems (these can not be easily filtered) 221 logger.debug("Loading %s from %s" % ('layerItems', index.apilinks['layerItems'])) 222 223 224 # The link won't include username/password, so pull it from the original url 225 pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'], 226 username=up.username, password=up.password) 227 if not pindex['layerItems']: 228 logger.debug("No layers were found at url %s." % (url)) 229 return index 230 index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems']) 231 232 233 # From this point on load the contents for each branch. Otherwise we 234 # could run into a timeout. 235 for branch in index.branches: 236 filter = "?filter=branch__name:%s" % index.branches[branch].name 237 238 logger.debug("Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches'])) 239 240 # The link won't include username/password, so pull it from the original url 241 pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter, 242 username=up.username, password=up.password) 243 if not pindex['layerBranches']: 244 logger.debug("No valid layer branches (%s) found at url %s." % (branches or "*", url)) 245 return index 246 index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches']) 247 248 249 # Load the rest, they all have a similar format 250 # Note: the layer index has a few more items, we can add them if necessary 251 # in the future. 252 filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name 253 for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency), 254 ("recipes", layerindexlib.Recipe), 255 ("machines", layerindexlib.Machine), 256 ("distros", layerindexlib.Distro)]: 257 if lName not in load: 258 continue 259 logger.debug("Loading %s from %s" % (lName, index.apilinks[lName])) 260 261 # The link won't include username/password, so pull it from the original url 262 pindex[lName] = _get_json_response(index.apilinks[lName] + filter, 263 username=up.username, password=up.password) 264 index.add_raw_element(lName, lType, pindex[lName]) 265 266 return index 267 268 def store_index(self, url, index): 269 """ 270 Store layer information into a local file/dir. 271 272 The return value is a dictionary containing API, 273 layer, branch, dependency, recipe, machine, distro, information. 274 275 ud is a parsed url to a directory or file. If the path is a 276 directory, we will split the files into one file per layer. 277 If the path is to a file (exists or not) the entire DB will be 278 dumped into that one file. 279 """ 280 281 up = urlparse(url) 282 283 if up.scheme != 'file': 284 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) 285 286 logger.debug("Storing to %s..." % up.path) 287 288 try: 289 layerbranches = index.layerBranches 290 except KeyError: 291 logger.error('No layerBranches to write.') 292 return 293 294 295 def filter_item(layerbranchid, objects): 296 filtered = [] 297 for obj in getattr(index, objects, None): 298 try: 299 if getattr(index, objects)[obj].layerbranch_id == layerbranchid: 300 filtered.append(getattr(index, objects)[obj]._data) 301 except AttributeError: 302 logger.debug('No obj.layerbranch_id: %s' % objects) 303 # No simple filter method, just include it... 304 try: 305 filtered.append(getattr(index, objects)[obj]._data) 306 except AttributeError: 307 logger.debug('No obj._data: %s %s' % (objects, type(obj))) 308 filtered.append(obj) 309 return filtered 310 311 312 # Write out to a single file. 313 # Filter out unnecessary items, then sort as we write for determinism 314 if not os.path.isdir(up.path): 315 pindex = {} 316 317 pindex['branches'] = [] 318 pindex['layerItems'] = [] 319 pindex['layerBranches'] = [] 320 321 for layerbranchid in layerbranches: 322 if layerbranches[layerbranchid].branch._data not in pindex['branches']: 323 pindex['branches'].append(layerbranches[layerbranchid].branch._data) 324 325 if layerbranches[layerbranchid].layer._data not in pindex['layerItems']: 326 pindex['layerItems'].append(layerbranches[layerbranchid].layer._data) 327 328 if layerbranches[layerbranchid]._data not in pindex['layerBranches']: 329 pindex['layerBranches'].append(layerbranches[layerbranchid]._data) 330 331 for entry in index._index: 332 # Skip local items, apilinks and items already processed 333 if entry in index.config['local'] or \ 334 entry == 'apilinks' or \ 335 entry == 'branches' or \ 336 entry == 'layerBranches' or \ 337 entry == 'layerItems': 338 continue 339 if entry not in pindex: 340 pindex[entry] = [] 341 pindex[entry].extend(filter_item(layerbranchid, entry)) 342 343 bb.debug(1, 'Writing index to %s' % up.path) 344 with open(up.path, 'wt') as f: 345 json.dump(layerindexlib.sort_entry(pindex), f, indent=4) 346 return 347 348 349 # Write out to a directory one file per layerBranch 350 # Prepare all layer related items, to create a minimal file. 351 # We have to sort the entries as we write so they are deterministic 352 for layerbranchid in layerbranches: 353 pindex = {} 354 355 for entry in index._index: 356 # Skip local items, apilinks and items already processed 357 if entry in index.config['local'] or \ 358 entry == 'apilinks' or \ 359 entry == 'branches' or \ 360 entry == 'layerBranches' or \ 361 entry == 'layerItems': 362 continue 363 pindex[entry] = filter_item(layerbranchid, entry) 364 365 # Add the layer we're processing as the first one... 366 pindex['branches'] = [layerbranches[layerbranchid].branch._data] 367 pindex['layerItems'] = [layerbranches[layerbranchid].layer._data] 368 pindex['layerBranches'] = [layerbranches[layerbranchid]._data] 369 370 # We also need to include the layerbranch for any dependencies... 371 for layerdep in pindex['layerDependencies']: 372 layerdependency = layerindexlib.LayerDependency(index, layerdep) 373 374 layeritem = layerdependency.dependency 375 layerbranch = layerdependency.dependency_layerBranch 376 377 # We need to avoid duplicates... 378 if layeritem._data not in pindex['layerItems']: 379 pindex['layerItems'].append(layeritem._data) 380 381 if layerbranch._data not in pindex['layerBranches']: 382 pindex['layerBranches'].append(layerbranch._data) 383 384 # apply mirroring adjustments here.... 385 386 fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] 387 fname = fname.translate(str.maketrans('/ ', '__')) 388 fpath = os.path.join(up.path, fname) 389 390 bb.debug(1, 'Writing index to %s' % fpath + '.json') 391 with open(fpath + '.json', 'wt') as f: 392 json.dump(layerindexlib.sort_entry(pindex), f, indent=4) 393