1# Copyright (C) 2016-2018 Wind River Systems, Inc. 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5 6import datetime 7 8import logging 9import os 10 11from collections import OrderedDict 12from layerindexlib.plugin import LayerIndexPluginUrlError 13 14logger = logging.getLogger('BitBake.layerindexlib') 15 16# Exceptions 17 18class LayerIndexException(Exception): 19 '''LayerIndex Generic Exception''' 20 def __init__(self, message): 21 self.msg = message 22 Exception.__init__(self, message) 23 24 def __str__(self): 25 return self.msg 26 27class LayerIndexUrlError(LayerIndexException): 28 '''Exception raised when unable to access a URL for some reason''' 29 def __init__(self, url, message=""): 30 if message: 31 msg = "Unable to access layerindex url %s: %s" % (url, message) 32 else: 33 msg = "Unable to access layerindex url %s" % url 34 self.url = url 35 LayerIndexException.__init__(self, msg) 36 37class LayerIndexFetchError(LayerIndexException): 38 '''General layerindex fetcher exception when something fails''' 39 def __init__(self, url, message=""): 40 if message: 41 msg = "Unable to fetch layerindex url %s: %s" % (url, message) 42 else: 43 msg = "Unable to fetch layerindex url %s" % url 44 self.url = url 45 LayerIndexException.__init__(self, msg) 46 47 48# Interface to the overall layerindex system 49# the layer may contain one or more individual indexes 50class LayerIndex(): 51 def __init__(self, d): 52 if not d: 53 raise LayerIndexException("Must be initialized with bb.data.") 54 55 self.data = d 56 57 # List of LayerIndexObj 58 self.indexes = [] 59 60 self.plugins = [] 61 62 import bb.utils 63 bb.utils.load_plugins(logger, self.plugins, os.path.dirname(__file__)) 64 for plugin in self.plugins: 65 if hasattr(plugin, 'init'): 66 plugin.init(self) 67 68 def __add__(self, other): 69 newIndex = LayerIndex(self.data) 70 71 if self.__class__ != newIndex.__class__ or \ 72 other.__class__ != newIndex.__class__: 73 raise TypeError("Can not add different types.") 74 75 for indexEnt in self.indexes: 76 newIndex.indexes.append(indexEnt) 77 78 for indexEnt in other.indexes: 79 newIndex.indexes.append(indexEnt) 80 81 return newIndex 82 83 def _parse_params(self, params): 84 '''Take a parameter list, return a dictionary of parameters. 85 86 Expected to be called from the data of urllib.parse.urlparse(url).params 87 88 If there are two conflicting parameters, last in wins... 89 ''' 90 91 param_dict = {} 92 for param in params.split(';'): 93 if not param: 94 continue 95 item = param.split('=', 1) 96 logger.debug(item) 97 param_dict[item[0]] = item[1] 98 99 return param_dict 100 101 def _fetch_url(self, url, username=None, password=None, debuglevel=0): 102 '''Fetch data from a specific URL. 103 104 Fetch something from a specific URL. This is specifically designed to 105 fetch data from a layerindex-web instance, but may be useful for other 106 raw fetch actions. 107 108 It is not designed to be used to fetch recipe sources or similar. the 109 regular fetcher class should used for that. 110 111 It is the responsibility of the caller to check BB_NO_NETWORK and related 112 BB_ALLOWED_NETWORKS. 113 ''' 114 115 if not url: 116 raise LayerIndexUrlError(url, "empty url") 117 118 import urllib 119 from urllib.request import urlopen, Request 120 from urllib.parse import urlparse 121 122 up = urlparse(url) 123 124 if username: 125 logger.debug("Configuring authentication for %s..." % url) 126 password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() 127 password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password) 128 handler = urllib.request.HTTPBasicAuthHandler(password_mgr) 129 opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel)) 130 else: 131 opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel)) 132 133 urllib.request.install_opener(opener) 134 135 logger.debug("Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)])) 136 137 try: 138 res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True)) 139 except urllib.error.HTTPError as e: 140 logger.debug("HTTP Error: %s: %s" % (e.code, e.reason)) 141 logger.debug(" Requested: %s" % (url)) 142 logger.debug(" Actual: %s" % (e.geturl())) 143 144 if e.code == 404: 145 logger.debug("Request not found.") 146 raise LayerIndexFetchError(url, e) 147 else: 148 logger.debug("Headers:\n%s" % (e.headers)) 149 raise LayerIndexFetchError(url, e) 150 except OSError as e: 151 error = 0 152 reason = "" 153 154 # Process base OSError first... 155 if hasattr(e, 'errno'): 156 error = e.errno 157 reason = e.strerror 158 159 # Process gaierror (socket error) subclass if available. 160 if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'): 161 error = e.reason.errno 162 reason = e.reason.strerror 163 if error == -2: 164 raise LayerIndexFetchError(url, "%s: %s" % (e, reason)) 165 166 if error and error != 0: 167 raise LayerIndexFetchError(url, "Unexpected exception: [Error %s] %s" % (error, reason)) 168 else: 169 raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e) 170 171 finally: 172 logger.debug("...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)])) 173 174 return res 175 176 177 def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False): 178 '''Load the layerindex. 179 180 indexURI - An index to load. (Use multiple calls to load multiple indexes) 181 182 reload - If reload is True, then any previously loaded indexes will be forgotten. 183 184 load - List of elements to load. Default loads all items. 185 Note: plugs may ignore this. 186 187The format of the indexURI: 188 189 <url>;branch=<branch>;cache=<cache>;desc=<description> 190 191 Note: the 'branch' parameter if set can select multiple branches by using 192 comma, such as 'branch=master,morty,pyro'. However, many operations only look 193 at the -first- branch specified! 194 195 The cache value may be undefined, in this case a network failure will 196 result in an error, otherwise the system will look for a file of the cache 197 name and load that instead. 198 199 For example: 200 201 https://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index 202 cooker:// 203''' 204 if reload: 205 self.indexes = [] 206 207 logger.debug('Loading: %s' % indexURI) 208 209 if not self.plugins: 210 raise LayerIndexException("No LayerIndex Plugins available") 211 212 for plugin in self.plugins: 213 # Check if the plugin was initialized 214 logger.debug('Trying %s' % plugin.__class__) 215 if not hasattr(plugin, 'type') or not plugin.type: 216 continue 217 try: 218 # TODO: Implement 'cache', for when the network is not available 219 indexEnt = plugin.load_index(indexURI, load) 220 break 221 except LayerIndexPluginUrlError as e: 222 logger.debug("%s doesn't support %s" % (plugin.type, e.url)) 223 except NotImplementedError: 224 pass 225 else: 226 logger.debug("No plugins support %s" % indexURI) 227 raise LayerIndexException("No plugins support %s" % indexURI) 228 229 # Mark CONFIG data as something we've added... 230 indexEnt.config['local'] = [] 231 indexEnt.config['local'].append('config') 232 233 # No longer permit changes.. 234 indexEnt.lockData() 235 236 self.indexes.append(indexEnt) 237 238 def store_layerindex(self, indexURI, index=None): 239 '''Store one layerindex 240 241Typically this will be used to create a local cache file of a remote index. 242 243 file://<path>;branch=<branch> 244 245We can write out in either the restapi or django formats. The split option 246will write out the individual elements split by layer and related components. 247''' 248 if not index: 249 logger.warning('No index to write, nothing to do.') 250 return 251 252 if not self.plugins: 253 raise LayerIndexException("No LayerIndex Plugins available") 254 255 for plugin in self.plugins: 256 # Check if the plugin was initialized 257 logger.debug('Trying %s' % plugin.__class__) 258 if not hasattr(plugin, 'type') or not plugin.type: 259 continue 260 try: 261 plugin.store_index(indexURI, index) 262 break 263 except LayerIndexPluginUrlError as e: 264 logger.debug("%s doesn't support %s" % (plugin.type, e.url)) 265 except NotImplementedError: 266 logger.debug("Store not implemented in %s" % plugin.type) 267 pass 268 else: 269 logger.debug("No plugins support %s" % indexURI) 270 raise LayerIndexException("No plugins support %s" % indexURI) 271 272 273 def is_empty(self): 274 '''Return True or False if the index has any usable data. 275 276We check the indexes entries to see if they have a branch set, as well as 277layerBranches set. If not, they are effectively blank.''' 278 279 found = False 280 for index in self.indexes: 281 if index.__bool__(): 282 found = True 283 break 284 return not found 285 286 287 def find_vcs_url(self, vcs_url, branch=None): 288 '''Return the first layerBranch with the given vcs_url 289 290 If a branch has not been specified, we will iterate over the branches in 291 the default configuration until the first vcs_url/branch match.''' 292 293 for index in self.indexes: 294 logger.debug(' searching %s' % index.config['DESCRIPTION']) 295 layerBranch = index.find_vcs_url(vcs_url, [branch]) 296 if layerBranch: 297 return layerBranch 298 return None 299 300 def find_collection(self, collection, version=None, branch=None): 301 '''Return the first layerBranch with the given collection name 302 303 If a branch has not been specified, we will iterate over the branches in 304 the default configuration until the first collection/branch match.''' 305 306 logger.debug('find_collection: %s (%s) %s' % (collection, version, branch)) 307 308 if branch: 309 branches = [branch] 310 else: 311 branches = None 312 313 for index in self.indexes: 314 logger.debug(' searching %s' % index.config['DESCRIPTION']) 315 layerBranch = index.find_collection(collection, version, branches) 316 if layerBranch: 317 return layerBranch 318 else: 319 logger.debug('Collection %s (%s) not found for branch (%s)' % (collection, version, branch)) 320 return None 321 322 def find_layerbranch(self, name, branch=None): 323 '''Return the layerBranch item for a given name and branch 324 325 If a branch has not been specified, we will iterate over the branches in 326 the default configuration until the first name/branch match.''' 327 328 if branch: 329 branches = [branch] 330 else: 331 branches = None 332 333 for index in self.indexes: 334 layerBranch = index.find_layerbranch(name, branches) 335 if layerBranch: 336 return layerBranch 337 return None 338 339 def find_dependencies(self, names=None, layerbranches=None, ignores=None): 340 '''Return a tuple of all dependencies and valid items for the list of (layer) names 341 342 The dependency scanning happens depth-first. The returned 343 dependencies should be in the best order to define bblayers. 344 345 names - list of layer names (searching layerItems) 346 branches - when specified (with names) only this list of branches are evaluated 347 348 layerbranches - list of layerbranches to resolve dependencies 349 350 ignores - list of layer names to ignore 351 352 return: (dependencies, invalid) 353 354 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ] 355 invalid = [ LayerItem.name1, LayerItem.name2, ... ] 356 ''' 357 358 invalid = [] 359 360 # Convert name/branch to layerbranches 361 if layerbranches is None: 362 layerbranches = [] 363 364 for name in names: 365 if ignores and name in ignores: 366 continue 367 368 for index in self.indexes: 369 layerbranch = index.find_layerbranch(name) 370 if not layerbranch: 371 # Not in this index, hopefully it's in another... 372 continue 373 layerbranches.append(layerbranch) 374 break 375 else: 376 invalid.append(name) 377 378 379 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid, processed=None): 380 for layerbranch in layerbranches: 381 if ignores and layerbranch.layer.name in ignores: 382 continue 383 384 # Get a list of dependencies and then recursively process them 385 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]: 386 try: 387 deplayerbranch = layerdependency.dependency_layerBranch 388 except AttributeError as e: 389 logger.error('LayerBranch does not exist for dependent layer {}:{}\n' \ 390 ' Cannot continue successfully.\n' \ 391 ' You might be able to resolve this by checking out the layer locally.\n' \ 392 ' Consider reaching out the to the layer maintainers or the layerindex admins' \ 393 .format(layerdependency.dependency.name, layerbranch.branch.name)) 394 395 if ignores and deplayerbranch.layer.name in ignores: 396 continue 397 398 # Since this is depth first, we need to know what we're currently processing 399 # in order to avoid infinite recursion on a loop. 400 if processed and deplayerbranch.layer.name in processed: 401 # We have found a recursion... 402 logger.warning('Circular layer dependency found: %s -> %s' % (processed, deplayerbranch.layer.name)) 403 continue 404 405 # This little block is why we can't re-use the LayerIndexObj version, 406 # we must be able to satisfy each dependencies across layer indexes and 407 # use the layer index order for priority. (r stands for replacement below) 408 409 # If this is the primary index, we can fast path and skip this 410 if deplayerbranch.index != self.indexes[0]: 411 # Is there an entry in a prior index for this collection/version? 412 rdeplayerbranch = self.find_collection( 413 collection=deplayerbranch.collection, 414 version=deplayerbranch.version 415 ) 416 if rdeplayerbranch != deplayerbranch: 417 logger.debug('Replaced %s:%s:%s with %s:%s:%s' % \ 418 (deplayerbranch.index.config['DESCRIPTION'], 419 deplayerbranch.branch.name, 420 deplayerbranch.layer.name, 421 rdeplayerbranch.index.config['DESCRIPTION'], 422 rdeplayerbranch.branch.name, 423 rdeplayerbranch.layer.name)) 424 deplayerbranch = rdeplayerbranch 425 426 # New dependency, we need to resolve it now... depth-first 427 if deplayerbranch.layer.name not in dependencies: 428 # Avoid recursion on this branch. 429 # We copy so we don't end up polluting the depth-first branch with other 430 # branches. Duplication between individual branches IS expected and 431 # handled by 'dependencies' processing. 432 if not processed: 433 local_processed = [] 434 else: 435 local_processed = processed.copy() 436 local_processed.append(deplayerbranch.layer.name) 437 438 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid, local_processed) 439 440 if deplayerbranch.layer.name not in dependencies: 441 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency] 442 else: 443 if layerdependency not in dependencies[deplayerbranch.layer.name]: 444 dependencies[deplayerbranch.layer.name].append(layerdependency) 445 446 return (dependencies, invalid) 447 448 # OK, resolve this one... 449 dependencies = OrderedDict() 450 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid) 451 452 for layerbranch in layerbranches: 453 if layerbranch.layer.name not in dependencies: 454 dependencies[layerbranch.layer.name] = [layerbranch] 455 456 return (dependencies, invalid) 457 458 459 def list_obj(self, object): 460 '''Print via the plain logger object information 461 462This function is used to implement debugging and provide the user info. 463''' 464 for lix in self.indexes: 465 if not hasattr(lix, object): 466 continue 467 468 logger.plain ('') 469 logger.plain ('Index: %s' % lix.config['DESCRIPTION']) 470 471 output = [] 472 473 if object == 'branches': 474 logger.plain ('%s %s %s' % ('{:26}'.format('branch'), '{:34}'.format('description'), '{:22}'.format('bitbake branch'))) 475 logger.plain ('{:-^80}'.format("")) 476 for branchid in lix.branches: 477 output.append('%s %s %s' % ( 478 '{:26}'.format(lix.branches[branchid].name), 479 '{:34}'.format(lix.branches[branchid].short_description), 480 '{:22}'.format(lix.branches[branchid].bitbake_branch) 481 )) 482 for line in sorted(output): 483 logger.plain (line) 484 485 continue 486 487 if object == 'layerItems': 488 logger.plain ('%s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'))) 489 logger.plain ('{:-^80}'.format("")) 490 for layerid in lix.layerItems: 491 output.append('%s %s' % ( 492 '{:26}'.format(lix.layerItems[layerid].name), 493 '{:34}'.format(lix.layerItems[layerid].summary) 494 )) 495 for line in sorted(output): 496 logger.plain (line) 497 498 continue 499 500 if object == 'layerBranches': 501 logger.plain ('%s %s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'), '{:19}'.format('collection:version'))) 502 logger.plain ('{:-^80}'.format("")) 503 for layerbranchid in lix.layerBranches: 504 output.append('%s %s %s' % ( 505 '{:26}'.format(lix.layerBranches[layerbranchid].layer.name), 506 '{:34}'.format(lix.layerBranches[layerbranchid].layer.summary), 507 '{:19}'.format("%s:%s" % 508 (lix.layerBranches[layerbranchid].collection, 509 lix.layerBranches[layerbranchid].version) 510 ) 511 )) 512 for line in sorted(output): 513 logger.plain (line) 514 515 continue 516 517 if object == 'layerDependencies': 518 logger.plain ('%s %s %s %s' % ('{:19}'.format('branch'), '{:26}'.format('layer'), '{:11}'.format('dependency'), '{:26}'.format('layer'))) 519 logger.plain ('{:-^80}'.format("")) 520 for layerDependency in lix.layerDependencies: 521 if not lix.layerDependencies[layerDependency].dependency_layerBranch: 522 continue 523 524 output.append('%s %s %s %s' % ( 525 '{:19}'.format(lix.layerDependencies[layerDependency].layerbranch.branch.name), 526 '{:26}'.format(lix.layerDependencies[layerDependency].layerbranch.layer.name), 527 '{:11}'.format('requires' if lix.layerDependencies[layerDependency].required else 'recommends'), 528 '{:26}'.format(lix.layerDependencies[layerDependency].dependency_layerBranch.layer.name) 529 )) 530 for line in sorted(output): 531 logger.plain (line) 532 533 continue 534 535 if object == 'recipes': 536 logger.plain ('%s %s %s' % ('{:20}'.format('recipe'), '{:10}'.format('version'), 'layer')) 537 logger.plain ('{:-^80}'.format("")) 538 output = [] 539 for recipe in lix.recipes: 540 output.append('%s %s %s' % ( 541 '{:30}'.format(lix.recipes[recipe].pn), 542 '{:30}'.format(lix.recipes[recipe].pv), 543 lix.recipes[recipe].layer.name 544 )) 545 for line in sorted(output): 546 logger.plain (line) 547 548 continue 549 550 if object == 'machines': 551 logger.plain ('%s %s %s' % ('{:24}'.format('machine'), '{:34}'.format('description'), '{:19}'.format('layer'))) 552 logger.plain ('{:-^80}'.format("")) 553 for machine in lix.machines: 554 output.append('%s %s %s' % ( 555 '{:24}'.format(lix.machines[machine].name), 556 '{:34}'.format(lix.machines[machine].description)[:34], 557 '{:19}'.format(lix.machines[machine].layerbranch.layer.name) 558 )) 559 for line in sorted(output): 560 logger.plain (line) 561 562 continue 563 564 if object == 'distros': 565 logger.plain ('%s %s %s' % ('{:24}'.format('distro'), '{:34}'.format('description'), '{:19}'.format('layer'))) 566 logger.plain ('{:-^80}'.format("")) 567 for distro in lix.distros: 568 output.append('%s %s %s' % ( 569 '{:24}'.format(lix.distros[distro].name), 570 '{:34}'.format(lix.distros[distro].description)[:34], 571 '{:19}'.format(lix.distros[distro].layerbranch.layer.name) 572 )) 573 for line in sorted(output): 574 logger.plain (line) 575 576 continue 577 578 logger.plain ('') 579 580 581# This class holds a single layer index instance 582# The LayerIndexObj is made up of dictionary of elements, such as: 583# index['config'] - configuration data for this index 584# index['branches'] - dictionary of Branch objects, by id number 585# index['layerItems'] - dictionary of layerItem objects, by id number 586# ...etc... (See: https://layers.openembedded.org/layerindex/api/) 587# 588# The class needs to manage the 'index' entries and allow easily adding 589# of new items, as well as simply loading of the items. 590class LayerIndexObj(): 591 def __init__(self): 592 super().__setattr__('_index', {}) 593 super().__setattr__('_lock', False) 594 595 def __bool__(self): 596 '''False if the index is effectively empty 597 598 We check the index to see if it has a branch set, as well as 599 layerbranches set. If not, it is effectively blank.''' 600 601 if not bool(self._index): 602 return False 603 604 try: 605 if self.branches and self.layerBranches: 606 return True 607 except AttributeError: 608 pass 609 610 return False 611 612 def __getattr__(self, name): 613 if name.startswith('_'): 614 return super().__getattribute__(name) 615 616 if name not in self._index: 617 raise AttributeError('%s not in index datastore' % name) 618 619 return self._index[name] 620 621 def __setattr__(self, name, value): 622 if self.isLocked(): 623 raise TypeError("Can not set attribute '%s': index is locked" % name) 624 625 if name.startswith('_'): 626 super().__setattr__(name, value) 627 return 628 629 self._index[name] = value 630 631 def __delattr__(self, name): 632 if self.isLocked(): 633 raise TypeError("Can not delete attribute '%s': index is locked" % name) 634 635 if name.startswith('_'): 636 super().__delattr__(name) 637 638 self._index.pop(name) 639 640 def lockData(self): 641 '''Lock data object (make it readonly)''' 642 super().__setattr__("_lock", True) 643 644 def unlockData(self): 645 '''unlock data object (make it readonly)''' 646 super().__setattr__("_lock", False) 647 648 # When the data is unlocked, we have to clear the caches, as 649 # modification is allowed! 650 del(self._layerBranches_layerId_branchId) 651 del(self._layerDependencies_layerBranchId) 652 del(self._layerBranches_vcsUrl) 653 654 def isLocked(self): 655 '''Is this object locked (readonly)?''' 656 return self._lock 657 658 def add_element(self, indexname, objs): 659 '''Add a layer index object to index.<indexname>''' 660 if indexname not in self._index: 661 self._index[indexname] = {} 662 663 for obj in objs: 664 if obj.id in self._index[indexname]: 665 if self._index[indexname][obj.id] == obj: 666 continue 667 raise LayerIndexException('Conflict adding object %s(%s) to index' % (indexname, obj.id)) 668 self._index[indexname][obj.id] = obj 669 670 def add_raw_element(self, indexname, objtype, rawobjs): 671 '''Convert a raw layer index data item to a layer index item object and add to the index''' 672 objs = [] 673 for entry in rawobjs: 674 objs.append(objtype(self, entry)) 675 self.add_element(indexname, objs) 676 677 # Quick lookup table for searching layerId and branchID combos 678 @property 679 def layerBranches_layerId_branchId(self): 680 def createCache(self): 681 cache = {} 682 for layerbranchid in self.layerBranches: 683 layerbranch = self.layerBranches[layerbranchid] 684 cache["%s:%s" % (layerbranch.layer_id, layerbranch.branch_id)] = layerbranch 685 return cache 686 687 if self.isLocked(): 688 cache = getattr(self, '_layerBranches_layerId_branchId', None) 689 else: 690 cache = None 691 692 if not cache: 693 cache = createCache(self) 694 695 if self.isLocked(): 696 super().__setattr__('_layerBranches_layerId_branchId', cache) 697 698 return cache 699 700 # Quick lookup table for finding all dependencies of a layerBranch 701 @property 702 def layerDependencies_layerBranchId(self): 703 def createCache(self): 704 cache = {} 705 # This ensures empty lists for all branchids 706 for layerbranchid in self.layerBranches: 707 cache[layerbranchid] = [] 708 709 for layerdependencyid in self.layerDependencies: 710 layerdependency = self.layerDependencies[layerdependencyid] 711 cache[layerdependency.layerbranch_id].append(layerdependency) 712 return cache 713 714 if self.isLocked(): 715 cache = getattr(self, '_layerDependencies_layerBranchId', None) 716 else: 717 cache = None 718 719 if not cache: 720 cache = createCache(self) 721 722 if self.isLocked(): 723 super().__setattr__('_layerDependencies_layerBranchId', cache) 724 725 return cache 726 727 # Quick lookup table for finding all instances of a vcs_url 728 @property 729 def layerBranches_vcsUrl(self): 730 def createCache(self): 731 cache = {} 732 for layerbranchid in self.layerBranches: 733 layerbranch = self.layerBranches[layerbranchid] 734 if layerbranch.layer.vcs_url not in cache: 735 cache[layerbranch.layer.vcs_url] = [layerbranch] 736 else: 737 cache[layerbranch.layer.vcs_url].append(layerbranch) 738 return cache 739 740 if self.isLocked(): 741 cache = getattr(self, '_layerBranches_vcsUrl', None) 742 else: 743 cache = None 744 745 if not cache: 746 cache = createCache(self) 747 748 if self.isLocked(): 749 super().__setattr__('_layerBranches_vcsUrl', cache) 750 751 return cache 752 753 754 def find_vcs_url(self, vcs_url, branches=None): 755 ''''Return the first layerBranch with the given vcs_url 756 757 If a list of branches has not been specified, we will iterate on 758 all branches until the first vcs_url is found.''' 759 760 if not self.__bool__(): 761 return None 762 763 for layerbranch in self.layerBranches_vcsUrl: 764 if branches and layerbranch.branch.name not in branches: 765 continue 766 767 return layerbranch 768 769 return None 770 771 772 def find_collection(self, collection, version=None, branches=None): 773 '''Return the first layerBranch with the given collection name 774 775 If a list of branches has not been specified, we will iterate on 776 all branches until the first collection is found.''' 777 778 if not self.__bool__(): 779 return None 780 781 for layerbranchid in self.layerBranches: 782 layerbranch = self.layerBranches[layerbranchid] 783 if branches and layerbranch.branch.name not in branches: 784 continue 785 786 if layerbranch.collection == collection and \ 787 (version is None or version == layerbranch.version): 788 return layerbranch 789 790 return None 791 792 793 def find_layerbranch(self, name, branches=None): 794 '''Return the first layerbranch whose layer name matches 795 796 If a list of branches has not been specified, we will iterate on 797 all branches until the first layer with that name is found.''' 798 799 if not self.__bool__(): 800 return None 801 802 for layerbranchid in self.layerBranches: 803 layerbranch = self.layerBranches[layerbranchid] 804 if branches and layerbranch.branch.name not in branches: 805 continue 806 807 if layerbranch.layer.name == name: 808 return layerbranch 809 810 return None 811 812 def find_dependencies(self, names=None, branches=None, layerBranches=None, ignores=None): 813 '''Return a tuple of all dependencies and valid items for the list of (layer) names 814 815 The dependency scanning happens depth-first. The returned 816 dependencies should be in the best order to define bblayers. 817 818 names - list of layer names (searching layerItems) 819 branches - when specified (with names) only this list of branches are evaluated 820 821 layerBranches - list of layerBranches to resolve dependencies 822 823 ignores - list of layer names to ignore 824 825 return: (dependencies, invalid) 826 827 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ] 828 invalid = [ LayerItem.name1, LayerItem.name2, ... ]''' 829 830 invalid = [] 831 832 # Convert name/branch to layerBranches 833 if layerbranches is None: 834 layerbranches = [] 835 836 for name in names: 837 if ignores and name in ignores: 838 continue 839 840 layerbranch = self.find_layerbranch(name, branches) 841 if not layerbranch: 842 invalid.append(name) 843 else: 844 layerbranches.append(layerbranch) 845 846 for layerbranch in layerbranches: 847 if layerbranch.index != self: 848 raise LayerIndexException("Can not resolve dependencies across indexes with this class function!") 849 850 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid): 851 for layerbranch in layerbranches: 852 if ignores and layerbranch.layer.name in ignores: 853 continue 854 855 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]: 856 deplayerbranch = layerdependency.dependency_layerBranch or None 857 858 if ignores and deplayerbranch.layer.name in ignores: 859 continue 860 861 # New dependency, we need to resolve it now... depth-first 862 if deplayerbranch.layer.name not in dependencies: 863 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid) 864 865 if deplayerbranch.layer.name not in dependencies: 866 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency] 867 else: 868 if layerdependency not in dependencies[deplayerbranch.layer.name]: 869 dependencies[deplayerbranch.layer.name].append(layerdependency) 870 871 return (dependencies, invalid) 872 873 # OK, resolve this one... 874 dependencies = OrderedDict() 875 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid) 876 877 # Is this item already in the list, if not add it 878 for layerbranch in layerbranches: 879 if layerbranch.layer.name not in dependencies: 880 dependencies[layerbranch.layer.name] = [layerbranch] 881 882 return (dependencies, invalid) 883 884 885# Define a basic LayerIndexItemObj. This object forms the basis for all other 886# objects. The raw Layer Index data is stored in the _data element, but we 887# do not want users to access data directly. So wrap this and protect it 888# from direct manipulation. 889# 890# It is up to the insantiators of the objects to fill them out, and once done 891# lock the objects to prevent further accidently manipulation. 892# 893# Using the getattr, setattr and properties we can access and manipulate 894# the data within the data element. 895class LayerIndexItemObj(): 896 def __init__(self, index, data=None, lock=False): 897 if data is None: 898 data = {} 899 900 if type(data) != type(dict()): 901 raise TypeError('data (%s) is not a dict' % type(data)) 902 903 super().__setattr__('_lock', lock) 904 super().__setattr__('index', index) 905 super().__setattr__('_data', data) 906 907 def __eq__(self, other): 908 if self.__class__ != other.__class__: 909 return False 910 res=(self._data == other._data) 911 return res 912 913 def __bool__(self): 914 return bool(self._data) 915 916 def __getattr__(self, name): 917 # These are internal to THIS class, and not part of data 918 if name == "index" or name.startswith('_'): 919 return super().__getattribute__(name) 920 921 if name not in self._data: 922 raise AttributeError('%s not in datastore' % name) 923 924 return self._data[name] 925 926 def _setattr(self, name, value, prop=True): 927 '''__setattr__ like function, but with control over property object behavior''' 928 if self.isLocked(): 929 raise TypeError("Can not set attribute '%s': Object data is locked" % name) 930 931 if name.startswith('_'): 932 super().__setattr__(name, value) 933 return 934 935 # Since __setattr__ runs before properties, we need to check if 936 # there is a setter property and then execute it 937 # ... or return self._data[name] 938 propertyobj = getattr(self.__class__, name, None) 939 if prop and isinstance(propertyobj, property): 940 if propertyobj.fset: 941 propertyobj.fset(self, value) 942 else: 943 raise AttributeError('Attribute %s is readonly, and may not be set' % name) 944 else: 945 self._data[name] = value 946 947 def __setattr__(self, name, value): 948 self._setattr(name, value, prop=True) 949 950 def _delattr(self, name, prop=True): 951 # Since __delattr__ runs before properties, we need to check if 952 # there is a deleter property and then execute it 953 # ... or we pop it ourselves.. 954 propertyobj = getattr(self.__class__, name, None) 955 if prop and isinstance(propertyobj, property): 956 if propertyobj.fdel: 957 propertyobj.fdel(self) 958 else: 959 raise AttributeError('Attribute %s is readonly, and may not be deleted' % name) 960 else: 961 self._data.pop(name) 962 963 def __delattr__(self, name): 964 self._delattr(name, prop=True) 965 966 def lockData(self): 967 '''Lock data object (make it readonly)''' 968 super().__setattr__("_lock", True) 969 970 def unlockData(self): 971 '''unlock data object (make it readonly)''' 972 super().__setattr__("_lock", False) 973 974 def isLocked(self): 975 '''Is this object locked (readonly)?''' 976 return self._lock 977 978# Branch object 979class Branch(LayerIndexItemObj): 980 def define_data(self, id, name, bitbake_branch, 981 short_description=None, sort_priority=1, 982 updates_enabled=True, updated=None, 983 update_environment=None): 984 self.id = id 985 self.name = name 986 self.bitbake_branch = bitbake_branch 987 self.short_description = short_description or name 988 self.sort_priority = sort_priority 989 self.updates_enabled = updates_enabled 990 self.updated = updated or datetime.datetime.today().isoformat() 991 self.update_environment = update_environment 992 993 @property 994 def name(self): 995 return self.__getattr__('name') 996 997 @name.setter 998 def name(self, value): 999 self._data['name'] = value 1000 1001 if self.bitbake_branch == value: 1002 self.bitbake_branch = "" 1003 1004 @name.deleter 1005 def name(self): 1006 self._delattr('name', prop=False) 1007 1008 @property 1009 def bitbake_branch(self): 1010 try: 1011 return self.__getattr__('bitbake_branch') 1012 except AttributeError: 1013 return self.name 1014 1015 @bitbake_branch.setter 1016 def bitbake_branch(self, value): 1017 if self.name == value: 1018 self._data['bitbake_branch'] = "" 1019 else: 1020 self._data['bitbake_branch'] = value 1021 1022 @bitbake_branch.deleter 1023 def bitbake_branch(self): 1024 self._delattr('bitbake_branch', prop=False) 1025 1026 1027class LayerItem(LayerIndexItemObj): 1028 def define_data(self, id, name, status='P', 1029 layer_type='A', summary=None, 1030 description=None, 1031 vcs_url=None, vcs_web_url=None, 1032 vcs_web_tree_base_url=None, 1033 vcs_web_file_base_url=None, 1034 usage_url=None, 1035 mailing_list_url=None, 1036 index_preference=1, 1037 classic=False, 1038 updated=None): 1039 self.id = id 1040 self.name = name 1041 self.status = status 1042 self.layer_type = layer_type 1043 self.summary = summary or name 1044 self.description = description or summary or name 1045 self.vcs_url = vcs_url 1046 self.vcs_web_url = vcs_web_url 1047 self.vcs_web_tree_base_url = vcs_web_tree_base_url 1048 self.vcs_web_file_base_url = vcs_web_file_base_url 1049 self.index_preference = index_preference 1050 self.classic = classic 1051 self.updated = updated or datetime.datetime.today().isoformat() 1052 1053 1054class LayerBranch(LayerIndexItemObj): 1055 def define_data(self, id, collection, version, layer, branch, 1056 vcs_subdir="", vcs_last_fetch=None, 1057 vcs_last_rev=None, vcs_last_commit=None, 1058 actual_branch="", 1059 updated=None): 1060 self.id = id 1061 self.collection = collection 1062 self.version = version 1063 if isinstance(layer, LayerItem): 1064 self.layer = layer 1065 else: 1066 self.layer_id = layer 1067 1068 if isinstance(branch, Branch): 1069 self.branch = branch 1070 else: 1071 self.branch_id = branch 1072 1073 self.vcs_subdir = vcs_subdir 1074 self.vcs_last_fetch = vcs_last_fetch 1075 self.vcs_last_rev = vcs_last_rev 1076 self.vcs_last_commit = vcs_last_commit 1077 self.actual_branch = actual_branch 1078 self.updated = updated or datetime.datetime.today().isoformat() 1079 1080 # This is a little odd, the _data attribute is 'layer', but it's really 1081 # referring to the layer id.. so lets adjust this to make it useful 1082 @property 1083 def layer_id(self): 1084 return self.__getattr__('layer') 1085 1086 @layer_id.setter 1087 def layer_id(self, value): 1088 self._setattr('layer', value, prop=False) 1089 1090 @layer_id.deleter 1091 def layer_id(self): 1092 self._delattr('layer', prop=False) 1093 1094 @property 1095 def layer(self): 1096 try: 1097 return self.index.layerItems[self.layer_id] 1098 except KeyError: 1099 raise AttributeError('Unable to find layerItems in index to map layer_id %s' % self.layer_id) 1100 except IndexError: 1101 raise AttributeError('Unable to find layer_id %s in index layerItems' % self.layer_id) 1102 1103 @layer.setter 1104 def layer(self, value): 1105 if not isinstance(value, LayerItem): 1106 raise TypeError('value is not a LayerItem') 1107 if self.index != value.index: 1108 raise AttributeError('Object and value do not share the same index and thus key set.') 1109 self.layer_id = value.id 1110 1111 @layer.deleter 1112 def layer(self): 1113 del self.layer_id 1114 1115 @property 1116 def branch_id(self): 1117 return self.__getattr__('branch') 1118 1119 @branch_id.setter 1120 def branch_id(self, value): 1121 self._setattr('branch', value, prop=False) 1122 1123 @branch_id.deleter 1124 def branch_id(self): 1125 self._delattr('branch', prop=False) 1126 1127 @property 1128 def branch(self): 1129 try: 1130 logger.debug("Get branch object from branches[%s]" % (self.branch_id)) 1131 return self.index.branches[self.branch_id] 1132 except KeyError: 1133 raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id) 1134 except IndexError: 1135 raise AttributeError('Unable to find branch_id %s in index branches' % self.branch_id) 1136 1137 @branch.setter 1138 def branch(self, value): 1139 if not isinstance(value, LayerItem): 1140 raise TypeError('value is not a LayerItem') 1141 if self.index != value.index: 1142 raise AttributeError('Object and value do not share the same index and thus key set.') 1143 self.branch_id = value.id 1144 1145 @branch.deleter 1146 def branch(self): 1147 del self.branch_id 1148 1149 @property 1150 def actual_branch(self): 1151 if self.__getattr__('actual_branch'): 1152 return self.__getattr__('actual_branch') 1153 else: 1154 return self.branch.name 1155 1156 @actual_branch.setter 1157 def actual_branch(self, value): 1158 logger.debug("Set actual_branch to %s .. name is %s" % (value, self.branch.name)) 1159 if value != self.branch.name: 1160 self._setattr('actual_branch', value, prop=False) 1161 else: 1162 self._setattr('actual_branch', '', prop=False) 1163 1164 @actual_branch.deleter 1165 def actual_branch(self): 1166 self._delattr('actual_branch', prop=False) 1167 1168# Extend LayerIndexItemObj with common LayerBranch manipulations 1169# All of the remaining LayerIndex objects refer to layerbranch, and it is 1170# up to the user to follow that back through the LayerBranch object into 1171# the layer object to get various attributes. So add an intermediate set 1172# of attributes that can easily get us the layerbranch as well as layer. 1173 1174class LayerIndexItemObj_LayerBranch(LayerIndexItemObj): 1175 @property 1176 def layerbranch_id(self): 1177 return self.__getattr__('layerbranch') 1178 1179 @layerbranch_id.setter 1180 def layerbranch_id(self, value): 1181 self._setattr('layerbranch', value, prop=False) 1182 1183 @layerbranch_id.deleter 1184 def layerbranch_id(self): 1185 self._delattr('layerbranch', prop=False) 1186 1187 @property 1188 def layerbranch(self): 1189 try: 1190 return self.index.layerBranches[self.layerbranch_id] 1191 except KeyError: 1192 raise AttributeError('Unable to find layerBranches in index to map layerbranch_id %s' % self.layerbranch_id) 1193 except IndexError: 1194 raise AttributeError('Unable to find layerbranch_id %s in index branches' % self.layerbranch_id) 1195 1196 @layerbranch.setter 1197 def layerbranch(self, value): 1198 if not isinstance(value, LayerBranch): 1199 raise TypeError('value (%s) is not a layerBranch' % type(value)) 1200 if self.index != value.index: 1201 raise AttributeError('Object and value do not share the same index and thus key set.') 1202 self.layerbranch_id = value.id 1203 1204 @layerbranch.deleter 1205 def layerbranch(self): 1206 del self.layerbranch_id 1207 1208 @property 1209 def layer_id(self): 1210 return self.layerbranch.layer_id 1211 1212 # Doesn't make sense to set or delete layer_id 1213 1214 @property 1215 def layer(self): 1216 return self.layerbranch.layer 1217 1218 # Doesn't make sense to set or delete layer 1219 1220 1221class LayerDependency(LayerIndexItemObj_LayerBranch): 1222 def define_data(self, id, layerbranch, dependency, required=True): 1223 self.id = id 1224 if isinstance(layerbranch, LayerBranch): 1225 self.layerbranch = layerbranch 1226 else: 1227 self.layerbranch_id = layerbranch 1228 if isinstance(dependency, LayerDependency): 1229 self.dependency = dependency 1230 else: 1231 self.dependency_id = dependency 1232 self.required = required 1233 1234 @property 1235 def dependency_id(self): 1236 return self.__getattr__('dependency') 1237 1238 @dependency_id.setter 1239 def dependency_id(self, value): 1240 self._setattr('dependency', value, prop=False) 1241 1242 @dependency_id.deleter 1243 def dependency_id(self): 1244 self._delattr('dependency', prop=False) 1245 1246 @property 1247 def dependency(self): 1248 try: 1249 return self.index.layerItems[self.dependency_id] 1250 except KeyError: 1251 raise AttributeError('Unable to find layerItems in index to map layerbranch_id %s' % self.dependency_id) 1252 except IndexError: 1253 raise AttributeError('Unable to find dependency_id %s in index layerItems' % self.dependency_id) 1254 1255 @dependency.setter 1256 def dependency(self, value): 1257 if not isinstance(value, LayerDependency): 1258 raise TypeError('value (%s) is not a dependency' % type(value)) 1259 if self.index != value.index: 1260 raise AttributeError('Object and value do not share the same index and thus key set.') 1261 self.dependency_id = value.id 1262 1263 @dependency.deleter 1264 def dependency(self): 1265 self._delattr('dependency', prop=False) 1266 1267 @property 1268 def dependency_layerBranch(self): 1269 layerid = self.dependency_id 1270 branchid = self.layerbranch.branch_id 1271 1272 try: 1273 return self.index.layerBranches_layerId_branchId["%s:%s" % (layerid, branchid)] 1274 except IndexError: 1275 # layerBranches_layerId_branchId -- but not layerId:branchId 1276 raise AttributeError('Unable to find layerId:branchId %s:%s in index layerBranches_layerId_branchId' % (layerid, branchid)) 1277 except KeyError: 1278 raise AttributeError('Unable to find layerId:branchId %s:%s in layerItems and layerBranches' % (layerid, branchid)) 1279 1280 # dependency_layerBranch doesn't make sense to set or del 1281 1282 1283class Recipe(LayerIndexItemObj_LayerBranch): 1284 def define_data(self, id, 1285 filename, filepath, pn, pv, layerbranch, 1286 summary="", description="", section="", license="", 1287 homepage="", bugtracker="", provides="", bbclassextend="", 1288 inherits="", disallowed="", updated=None): 1289 self.id = id 1290 self.filename = filename 1291 self.filepath = filepath 1292 self.pn = pn 1293 self.pv = pv 1294 self.summary = summary 1295 self.description = description 1296 self.section = section 1297 self.license = license 1298 self.homepage = homepage 1299 self.bugtracker = bugtracker 1300 self.provides = provides 1301 self.bbclassextend = bbclassextend 1302 self.inherits = inherits 1303 self.updated = updated or datetime.datetime.today().isoformat() 1304 self.disallowed = disallowed 1305 if isinstance(layerbranch, LayerBranch): 1306 self.layerbranch = layerbranch 1307 else: 1308 self.layerbranch_id = layerbranch 1309 1310 @property 1311 def fullpath(self): 1312 return os.path.join(self.filepath, self.filename) 1313 1314 # Set would need to understand how to split it 1315 # del would we del both parts? 1316 1317 @property 1318 def inherits(self): 1319 if 'inherits' not in self._data: 1320 # Older indexes may not have this, so emulate it 1321 if '-image-' in self.pn: 1322 return 'image' 1323 return self.__getattr__('inherits') 1324 1325 @inherits.setter 1326 def inherits(self, value): 1327 return self._setattr('inherits', value, prop=False) 1328 1329 @inherits.deleter 1330 def inherits(self): 1331 return self._delattr('inherits', prop=False) 1332 1333 1334class Machine(LayerIndexItemObj_LayerBranch): 1335 def define_data(self, id, 1336 name, description, layerbranch, 1337 updated=None): 1338 self.id = id 1339 self.name = name 1340 self.description = description 1341 if isinstance(layerbranch, LayerBranch): 1342 self.layerbranch = layerbranch 1343 else: 1344 self.layerbranch_id = layerbranch 1345 self.updated = updated or datetime.datetime.today().isoformat() 1346 1347class Distro(LayerIndexItemObj_LayerBranch): 1348 def define_data(self, id, 1349 name, description, layerbranch, 1350 updated=None): 1351 self.id = id 1352 self.name = name 1353 self.description = description 1354 if isinstance(layerbranch, LayerBranch): 1355 self.layerbranch = layerbranch 1356 else: 1357 self.layerbranch_id = layerbranch 1358 self.updated = updated or datetime.datetime.today().isoformat() 1359 1360# When performing certain actions, we may need to sort the data. 1361# This will allow us to keep it consistent from run to run. 1362def sort_entry(item): 1363 newitem = item 1364 try: 1365 if type(newitem) == type(dict()): 1366 newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0])) 1367 for index in newitem: 1368 newitem[index] = sort_entry(newitem[index]) 1369 elif type(newitem) == type(list()): 1370 newitem.sort(key=lambda obj: obj['id']) 1371 for index, _ in enumerate(newitem): 1372 newitem[index] = sort_entry(newitem[index]) 1373 except: 1374 logger.error('Sort failed for item %s' % type(item)) 1375 pass 1376 1377 return newitem 1378