1# Copyright (C) 2016-2018 Wind River Systems, Inc. 2# 3# SPDX-License-Identifier: GPL-2.0-only 4# 5 6import datetime 7 8import logging 9import os 10 11from collections import OrderedDict 12from layerindexlib.plugin import LayerIndexPluginUrlError 13 14logger = logging.getLogger('BitBake.layerindexlib') 15 16# Exceptions 17 18class LayerIndexException(Exception): 19 '''LayerIndex Generic Exception''' 20 def __init__(self, message): 21 self.msg = message 22 Exception.__init__(self, message) 23 24 def __str__(self): 25 return self.msg 26 27class LayerIndexUrlError(LayerIndexException): 28 '''Exception raised when unable to access a URL for some reason''' 29 def __init__(self, url, message=""): 30 if message: 31 msg = "Unable to access layerindex url %s: %s" % (url, message) 32 else: 33 msg = "Unable to access layerindex url %s" % url 34 self.url = url 35 LayerIndexException.__init__(self, msg) 36 37class LayerIndexFetchError(LayerIndexException): 38 '''General layerindex fetcher exception when something fails''' 39 def __init__(self, url, message=""): 40 if message: 41 msg = "Unable to fetch layerindex url %s: %s" % (url, message) 42 else: 43 msg = "Unable to fetch layerindex url %s" % url 44 self.url = url 45 LayerIndexException.__init__(self, msg) 46 47 48# Interface to the overall layerindex system 49# the layer may contain one or more individual indexes 50class LayerIndex(): 51 def __init__(self, d): 52 if not d: 53 raise LayerIndexException("Must be initialized with bb.data.") 54 55 self.data = d 56 57 # List of LayerIndexObj 58 self.indexes = [] 59 60 self.plugins = [] 61 62 import bb.utils 63 bb.utils.load_plugins(logger, self.plugins, os.path.dirname(__file__)) 64 for plugin in self.plugins: 65 if hasattr(plugin, 'init'): 66 plugin.init(self) 67 68 def __add__(self, other): 69 newIndex = LayerIndex(self.data) 70 71 if self.__class__ != newIndex.__class__ or \ 72 other.__class__ != newIndex.__class__: 73 raise TypeError("Can not add different types.") 74 75 for indexEnt in self.indexes: 76 newIndex.indexes.append(indexEnt) 77 78 for indexEnt in other.indexes: 79 newIndex.indexes.append(indexEnt) 80 81 return newIndex 82 83 def _parse_params(self, params): 84 '''Take a parameter list, return a dictionary of parameters. 85 86 Expected to be called from the data of urllib.parse.urlparse(url).params 87 88 If there are two conflicting parameters, last in wins... 89 ''' 90 91 param_dict = {} 92 for param in params.split(';'): 93 if not param: 94 continue 95 item = param.split('=', 1) 96 logger.debug(item) 97 param_dict[item[0]] = item[1] 98 99 return param_dict 100 101 def _fetch_url(self, url, username=None, password=None, debuglevel=0): 102 '''Fetch data from a specific URL. 103 104 Fetch something from a specific URL. This is specifically designed to 105 fetch data from a layerindex-web instance, but may be useful for other 106 raw fetch actions. 107 108 It is not designed to be used to fetch recipe sources or similar. the 109 regular fetcher class should used for that. 110 111 It is the responsibility of the caller to check BB_NO_NETWORK and related 112 BB_ALLOWED_NETWORKS. 113 ''' 114 115 if not url: 116 raise LayerIndexUrlError(url, "empty url") 117 118 import urllib 119 from urllib.request import urlopen, Request 120 from urllib.parse import urlparse 121 122 up = urlparse(url) 123 124 if username: 125 logger.debug("Configuring authentication for %s..." % url) 126 password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() 127 password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password) 128 handler = urllib.request.HTTPBasicAuthHandler(password_mgr) 129 opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel)) 130 else: 131 opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel)) 132 133 urllib.request.install_opener(opener) 134 135 logger.debug("Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)])) 136 137 try: 138 res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True)) 139 except urllib.error.HTTPError as e: 140 logger.debug("HTTP Error: %s: %s" % (e.code, e.reason)) 141 logger.debug(" Requested: %s" % (url)) 142 logger.debug(" Actual: %s" % (e.geturl())) 143 144 if e.code == 404: 145 logger.debug("Request not found.") 146 raise LayerIndexFetchError(url, e) 147 else: 148 logger.debug("Headers:\n%s" % (e.headers)) 149 raise LayerIndexFetchError(url, e) 150 except OSError as e: 151 error = 0 152 reason = "" 153 154 # Process base OSError first... 155 if hasattr(e, 'errno'): 156 error = e.errno 157 reason = e.strerror 158 159 # Process gaierror (socket error) subclass if available. 160 if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'): 161 error = e.reason.errno 162 reason = e.reason.strerror 163 if error == -2: 164 raise LayerIndexFetchError(url, "%s: %s" % (e, reason)) 165 166 if error and error != 0: 167 raise LayerIndexFetchError(url, "Unexpected exception: [Error %s] %s" % (error, reason)) 168 else: 169 raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e) 170 171 finally: 172 logger.debug("...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)])) 173 174 return res 175 176 177 def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False): 178 '''Load the layerindex. 179 180 indexURI - An index to load. (Use multiple calls to load multiple indexes) 181 182 reload - If reload is True, then any previously loaded indexes will be forgotten. 183 184 load - List of elements to load. Default loads all items. 185 Note: plugs may ignore this. 186 187The format of the indexURI: 188 189 <url>;branch=<branch>;cache=<cache>;desc=<description> 190 191 Note: the 'branch' parameter if set can select multiple branches by using 192 comma, such as 'branch=master,morty,pyro'. However, many operations only look 193 at the -first- branch specified! 194 195 The cache value may be undefined, in this case a network failure will 196 result in an error, otherwise the system will look for a file of the cache 197 name and load that instead. 198 199 For example: 200 201 https://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index 202 cooker:// 203''' 204 if reload: 205 self.indexes = [] 206 207 logger.debug('Loading: %s' % indexURI) 208 209 if not self.plugins: 210 raise LayerIndexException("No LayerIndex Plugins available") 211 212 for plugin in self.plugins: 213 # Check if the plugin was initialized 214 logger.debug('Trying %s' % plugin.__class__) 215 if not hasattr(plugin, 'type') or not plugin.type: 216 continue 217 try: 218 # TODO: Implement 'cache', for when the network is not available 219 indexEnt = plugin.load_index(indexURI, load) 220 break 221 except LayerIndexPluginUrlError as e: 222 logger.debug("%s doesn't support %s" % (plugin.type, e.url)) 223 except NotImplementedError: 224 pass 225 else: 226 logger.debug("No plugins support %s" % indexURI) 227 raise LayerIndexException("No plugins support %s" % indexURI) 228 229 # Mark CONFIG data as something we've added... 230 indexEnt.config['local'] = [] 231 indexEnt.config['local'].append('config') 232 233 # No longer permit changes.. 234 indexEnt.lockData() 235 236 self.indexes.append(indexEnt) 237 238 def store_layerindex(self, indexURI, index=None): 239 '''Store one layerindex 240 241Typically this will be used to create a local cache file of a remote index. 242 243 file://<path>;branch=<branch> 244 245We can write out in either the restapi or django formats. The split option 246will write out the individual elements split by layer and related components. 247''' 248 if not index: 249 logger.warning('No index to write, nothing to do.') 250 return 251 252 if not self.plugins: 253 raise LayerIndexException("No LayerIndex Plugins available") 254 255 for plugin in self.plugins: 256 # Check if the plugin was initialized 257 logger.debug('Trying %s' % plugin.__class__) 258 if not hasattr(plugin, 'type') or not plugin.type: 259 continue 260 try: 261 plugin.store_index(indexURI, index) 262 break 263 except LayerIndexPluginUrlError as e: 264 logger.debug("%s doesn't support %s" % (plugin.type, e.url)) 265 except NotImplementedError: 266 logger.debug("Store not implemented in %s" % plugin.type) 267 pass 268 else: 269 logger.debug("No plugins support %s" % indexURI) 270 raise LayerIndexException("No plugins support %s" % indexURI) 271 272 273 def is_empty(self): 274 '''Return True or False if the index has any usable data. 275 276We check the indexes entries to see if they have a branch set, as well as 277layerBranches set. If not, they are effectively blank.''' 278 279 found = False 280 for index in self.indexes: 281 if index.__bool__(): 282 found = True 283 break 284 return not found 285 286 287 def find_vcs_url(self, vcs_url, branch=None): 288 '''Return the first layerBranch with the given vcs_url 289 290 If a branch has not been specified, we will iterate over the branches in 291 the default configuration until the first vcs_url/branch match.''' 292 293 for index in self.indexes: 294 logger.debug(' searching %s' % index.config['DESCRIPTION']) 295 layerBranch = index.find_vcs_url(vcs_url, [branch]) 296 if layerBranch: 297 return layerBranch 298 return None 299 300 def find_collection(self, collection, version=None, branch=None): 301 '''Return the first layerBranch with the given collection name 302 303 If a branch has not been specified, we will iterate over the branches in 304 the default configuration until the first collection/branch match.''' 305 306 logger.debug('find_collection: %s (%s) %s' % (collection, version, branch)) 307 308 if branch: 309 branches = [branch] 310 else: 311 branches = None 312 313 for index in self.indexes: 314 logger.debug(' searching %s' % index.config['DESCRIPTION']) 315 layerBranch = index.find_collection(collection, version, branches) 316 if layerBranch: 317 return layerBranch 318 else: 319 logger.debug('Collection %s (%s) not found for branch (%s)' % (collection, version, branch)) 320 return None 321 322 def find_layerbranch(self, name, branch=None): 323 '''Return the layerBranch item for a given name and branch 324 325 If a branch has not been specified, we will iterate over the branches in 326 the default configuration until the first name/branch match.''' 327 328 if branch: 329 branches = [branch] 330 else: 331 branches = None 332 333 for index in self.indexes: 334 layerBranch = index.find_layerbranch(name, branches) 335 if layerBranch: 336 return layerBranch 337 return None 338 339 def find_dependencies(self, names=None, layerbranches=None, ignores=None): 340 '''Return a tuple of all dependencies and valid items for the list of (layer) names 341 342 The dependency scanning happens depth-first. The returned 343 dependencies should be in the best order to define bblayers. 344 345 names - list of layer names (searching layerItems) 346 branches - when specified (with names) only this list of branches are evaluated 347 348 layerbranches - list of layerbranches to resolve dependencies 349 350 ignores - list of layer names to ignore 351 352 return: (dependencies, invalid) 353 354 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ] 355 invalid = [ LayerItem.name1, LayerItem.name2, ... ] 356 ''' 357 358 invalid = [] 359 360 # Convert name/branch to layerbranches 361 if layerbranches is None: 362 layerbranches = [] 363 364 for name in names: 365 if ignores and name in ignores: 366 continue 367 368 for index in self.indexes: 369 layerbranch = index.find_layerbranch(name) 370 if not layerbranch: 371 # Not in this index, hopefully it's in another... 372 continue 373 layerbranches.append(layerbranch) 374 break 375 else: 376 invalid.append(name) 377 378 379 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid, processed=None): 380 for layerbranch in layerbranches: 381 if ignores and layerbranch.layer.name in ignores: 382 continue 383 384 # Get a list of dependencies and then recursively process them 385 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]: 386 deplayerbranch = layerdependency.dependency_layerBranch 387 388 if ignores and deplayerbranch.layer.name in ignores: 389 continue 390 391 # Since this is depth first, we need to know what we're currently processing 392 # in order to avoid infinite recursion on a loop. 393 if processed and deplayerbranch.layer.name in processed: 394 # We have found a recursion... 395 logger.warning('Circular layer dependency found: %s -> %s' % (processed, deplayerbranch.layer.name)) 396 continue 397 398 # This little block is why we can't re-use the LayerIndexObj version, 399 # we must be able to satisfy each dependencies across layer indexes and 400 # use the layer index order for priority. (r stands for replacement below) 401 402 # If this is the primary index, we can fast path and skip this 403 if deplayerbranch.index != self.indexes[0]: 404 # Is there an entry in a prior index for this collection/version? 405 rdeplayerbranch = self.find_collection( 406 collection=deplayerbranch.collection, 407 version=deplayerbranch.version 408 ) 409 if rdeplayerbranch != deplayerbranch: 410 logger.debug('Replaced %s:%s:%s with %s:%s:%s' % \ 411 (deplayerbranch.index.config['DESCRIPTION'], 412 deplayerbranch.branch.name, 413 deplayerbranch.layer.name, 414 rdeplayerbranch.index.config['DESCRIPTION'], 415 rdeplayerbranch.branch.name, 416 rdeplayerbranch.layer.name)) 417 deplayerbranch = rdeplayerbranch 418 419 # New dependency, we need to resolve it now... depth-first 420 if deplayerbranch.layer.name not in dependencies: 421 # Avoid recursion on this branch. 422 # We copy so we don't end up polluting the depth-first branch with other 423 # branches. Duplication between individual branches IS expected and 424 # handled by 'dependencies' processing. 425 if not processed: 426 local_processed = [] 427 else: 428 local_processed = processed.copy() 429 local_processed.append(deplayerbranch.layer.name) 430 431 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid, local_processed) 432 433 if deplayerbranch.layer.name not in dependencies: 434 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency] 435 else: 436 if layerdependency not in dependencies[deplayerbranch.layer.name]: 437 dependencies[deplayerbranch.layer.name].append(layerdependency) 438 439 return (dependencies, invalid) 440 441 # OK, resolve this one... 442 dependencies = OrderedDict() 443 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid) 444 445 for layerbranch in layerbranches: 446 if layerbranch.layer.name not in dependencies: 447 dependencies[layerbranch.layer.name] = [layerbranch] 448 449 return (dependencies, invalid) 450 451 452 def list_obj(self, object): 453 '''Print via the plain logger object information 454 455This function is used to implement debugging and provide the user info. 456''' 457 for lix in self.indexes: 458 if not hasattr(lix, object): 459 continue 460 461 logger.plain ('') 462 logger.plain ('Index: %s' % lix.config['DESCRIPTION']) 463 464 output = [] 465 466 if object == 'branches': 467 logger.plain ('%s %s %s' % ('{:26}'.format('branch'), '{:34}'.format('description'), '{:22}'.format('bitbake branch'))) 468 logger.plain ('{:-^80}'.format("")) 469 for branchid in lix.branches: 470 output.append('%s %s %s' % ( 471 '{:26}'.format(lix.branches[branchid].name), 472 '{:34}'.format(lix.branches[branchid].short_description), 473 '{:22}'.format(lix.branches[branchid].bitbake_branch) 474 )) 475 for line in sorted(output): 476 logger.plain (line) 477 478 continue 479 480 if object == 'layerItems': 481 logger.plain ('%s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'))) 482 logger.plain ('{:-^80}'.format("")) 483 for layerid in lix.layerItems: 484 output.append('%s %s' % ( 485 '{:26}'.format(lix.layerItems[layerid].name), 486 '{:34}'.format(lix.layerItems[layerid].summary) 487 )) 488 for line in sorted(output): 489 logger.plain (line) 490 491 continue 492 493 if object == 'layerBranches': 494 logger.plain ('%s %s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'), '{:19}'.format('collection:version'))) 495 logger.plain ('{:-^80}'.format("")) 496 for layerbranchid in lix.layerBranches: 497 output.append('%s %s %s' % ( 498 '{:26}'.format(lix.layerBranches[layerbranchid].layer.name), 499 '{:34}'.format(lix.layerBranches[layerbranchid].layer.summary), 500 '{:19}'.format("%s:%s" % 501 (lix.layerBranches[layerbranchid].collection, 502 lix.layerBranches[layerbranchid].version) 503 ) 504 )) 505 for line in sorted(output): 506 logger.plain (line) 507 508 continue 509 510 if object == 'layerDependencies': 511 logger.plain ('%s %s %s %s' % ('{:19}'.format('branch'), '{:26}'.format('layer'), '{:11}'.format('dependency'), '{:26}'.format('layer'))) 512 logger.plain ('{:-^80}'.format("")) 513 for layerDependency in lix.layerDependencies: 514 if not lix.layerDependencies[layerDependency].dependency_layerBranch: 515 continue 516 517 output.append('%s %s %s %s' % ( 518 '{:19}'.format(lix.layerDependencies[layerDependency].layerbranch.branch.name), 519 '{:26}'.format(lix.layerDependencies[layerDependency].layerbranch.layer.name), 520 '{:11}'.format('requires' if lix.layerDependencies[layerDependency].required else 'recommends'), 521 '{:26}'.format(lix.layerDependencies[layerDependency].dependency_layerBranch.layer.name) 522 )) 523 for line in sorted(output): 524 logger.plain (line) 525 526 continue 527 528 if object == 'recipes': 529 logger.plain ('%s %s %s' % ('{:20}'.format('recipe'), '{:10}'.format('version'), 'layer')) 530 logger.plain ('{:-^80}'.format("")) 531 output = [] 532 for recipe in lix.recipes: 533 output.append('%s %s %s' % ( 534 '{:30}'.format(lix.recipes[recipe].pn), 535 '{:30}'.format(lix.recipes[recipe].pv), 536 lix.recipes[recipe].layer.name 537 )) 538 for line in sorted(output): 539 logger.plain (line) 540 541 continue 542 543 if object == 'machines': 544 logger.plain ('%s %s %s' % ('{:24}'.format('machine'), '{:34}'.format('description'), '{:19}'.format('layer'))) 545 logger.plain ('{:-^80}'.format("")) 546 for machine in lix.machines: 547 output.append('%s %s %s' % ( 548 '{:24}'.format(lix.machines[machine].name), 549 '{:34}'.format(lix.machines[machine].description)[:34], 550 '{:19}'.format(lix.machines[machine].layerbranch.layer.name) 551 )) 552 for line in sorted(output): 553 logger.plain (line) 554 555 continue 556 557 if object == 'distros': 558 logger.plain ('%s %s %s' % ('{:24}'.format('distro'), '{:34}'.format('description'), '{:19}'.format('layer'))) 559 logger.plain ('{:-^80}'.format("")) 560 for distro in lix.distros: 561 output.append('%s %s %s' % ( 562 '{:24}'.format(lix.distros[distro].name), 563 '{:34}'.format(lix.distros[distro].description)[:34], 564 '{:19}'.format(lix.distros[distro].layerbranch.layer.name) 565 )) 566 for line in sorted(output): 567 logger.plain (line) 568 569 continue 570 571 logger.plain ('') 572 573 574# This class holds a single layer index instance 575# The LayerIndexObj is made up of dictionary of elements, such as: 576# index['config'] - configuration data for this index 577# index['branches'] - dictionary of Branch objects, by id number 578# index['layerItems'] - dictionary of layerItem objects, by id number 579# ...etc... (See: https://layers.openembedded.org/layerindex/api/) 580# 581# The class needs to manage the 'index' entries and allow easily adding 582# of new items, as well as simply loading of the items. 583class LayerIndexObj(): 584 def __init__(self): 585 super().__setattr__('_index', {}) 586 super().__setattr__('_lock', False) 587 588 def __bool__(self): 589 '''False if the index is effectively empty 590 591 We check the index to see if it has a branch set, as well as 592 layerbranches set. If not, it is effectively blank.''' 593 594 if not bool(self._index): 595 return False 596 597 try: 598 if self.branches and self.layerBranches: 599 return True 600 except AttributeError: 601 pass 602 603 return False 604 605 def __getattr__(self, name): 606 if name.startswith('_'): 607 return super().__getattribute__(name) 608 609 if name not in self._index: 610 raise AttributeError('%s not in index datastore' % name) 611 612 return self._index[name] 613 614 def __setattr__(self, name, value): 615 if self.isLocked(): 616 raise TypeError("Can not set attribute '%s': index is locked" % name) 617 618 if name.startswith('_'): 619 super().__setattr__(name, value) 620 return 621 622 self._index[name] = value 623 624 def __delattr__(self, name): 625 if self.isLocked(): 626 raise TypeError("Can not delete attribute '%s': index is locked" % name) 627 628 if name.startswith('_'): 629 super().__delattr__(name) 630 631 self._index.pop(name) 632 633 def lockData(self): 634 '''Lock data object (make it readonly)''' 635 super().__setattr__("_lock", True) 636 637 def unlockData(self): 638 '''unlock data object (make it readonly)''' 639 super().__setattr__("_lock", False) 640 641 # When the data is unlocked, we have to clear the caches, as 642 # modification is allowed! 643 del(self._layerBranches_layerId_branchId) 644 del(self._layerDependencies_layerBranchId) 645 del(self._layerBranches_vcsUrl) 646 647 def isLocked(self): 648 '''Is this object locked (readonly)?''' 649 return self._lock 650 651 def add_element(self, indexname, objs): 652 '''Add a layer index object to index.<indexname>''' 653 if indexname not in self._index: 654 self._index[indexname] = {} 655 656 for obj in objs: 657 if obj.id in self._index[indexname]: 658 if self._index[indexname][obj.id] == obj: 659 continue 660 raise LayerIndexException('Conflict adding object %s(%s) to index' % (indexname, obj.id)) 661 self._index[indexname][obj.id] = obj 662 663 def add_raw_element(self, indexname, objtype, rawobjs): 664 '''Convert a raw layer index data item to a layer index item object and add to the index''' 665 objs = [] 666 for entry in rawobjs: 667 objs.append(objtype(self, entry)) 668 self.add_element(indexname, objs) 669 670 # Quick lookup table for searching layerId and branchID combos 671 @property 672 def layerBranches_layerId_branchId(self): 673 def createCache(self): 674 cache = {} 675 for layerbranchid in self.layerBranches: 676 layerbranch = self.layerBranches[layerbranchid] 677 cache["%s:%s" % (layerbranch.layer_id, layerbranch.branch_id)] = layerbranch 678 return cache 679 680 if self.isLocked(): 681 cache = getattr(self, '_layerBranches_layerId_branchId', None) 682 else: 683 cache = None 684 685 if not cache: 686 cache = createCache(self) 687 688 if self.isLocked(): 689 super().__setattr__('_layerBranches_layerId_branchId', cache) 690 691 return cache 692 693 # Quick lookup table for finding all dependencies of a layerBranch 694 @property 695 def layerDependencies_layerBranchId(self): 696 def createCache(self): 697 cache = {} 698 # This ensures empty lists for all branchids 699 for layerbranchid in self.layerBranches: 700 cache[layerbranchid] = [] 701 702 for layerdependencyid in self.layerDependencies: 703 layerdependency = self.layerDependencies[layerdependencyid] 704 cache[layerdependency.layerbranch_id].append(layerdependency) 705 return cache 706 707 if self.isLocked(): 708 cache = getattr(self, '_layerDependencies_layerBranchId', None) 709 else: 710 cache = None 711 712 if not cache: 713 cache = createCache(self) 714 715 if self.isLocked(): 716 super().__setattr__('_layerDependencies_layerBranchId', cache) 717 718 return cache 719 720 # Quick lookup table for finding all instances of a vcs_url 721 @property 722 def layerBranches_vcsUrl(self): 723 def createCache(self): 724 cache = {} 725 for layerbranchid in self.layerBranches: 726 layerbranch = self.layerBranches[layerbranchid] 727 if layerbranch.layer.vcs_url not in cache: 728 cache[layerbranch.layer.vcs_url] = [layerbranch] 729 else: 730 cache[layerbranch.layer.vcs_url].append(layerbranch) 731 return cache 732 733 if self.isLocked(): 734 cache = getattr(self, '_layerBranches_vcsUrl', None) 735 else: 736 cache = None 737 738 if not cache: 739 cache = createCache(self) 740 741 if self.isLocked(): 742 super().__setattr__('_layerBranches_vcsUrl', cache) 743 744 return cache 745 746 747 def find_vcs_url(self, vcs_url, branches=None): 748 ''''Return the first layerBranch with the given vcs_url 749 750 If a list of branches has not been specified, we will iterate on 751 all branches until the first vcs_url is found.''' 752 753 if not self.__bool__(): 754 return None 755 756 for layerbranch in self.layerBranches_vcsUrl: 757 if branches and layerbranch.branch.name not in branches: 758 continue 759 760 return layerbranch 761 762 return None 763 764 765 def find_collection(self, collection, version=None, branches=None): 766 '''Return the first layerBranch with the given collection name 767 768 If a list of branches has not been specified, we will iterate on 769 all branches until the first collection is found.''' 770 771 if not self.__bool__(): 772 return None 773 774 for layerbranchid in self.layerBranches: 775 layerbranch = self.layerBranches[layerbranchid] 776 if branches and layerbranch.branch.name not in branches: 777 continue 778 779 if layerbranch.collection == collection and \ 780 (version is None or version == layerbranch.version): 781 return layerbranch 782 783 return None 784 785 786 def find_layerbranch(self, name, branches=None): 787 '''Return the first layerbranch whose layer name matches 788 789 If a list of branches has not been specified, we will iterate on 790 all branches until the first layer with that name is found.''' 791 792 if not self.__bool__(): 793 return None 794 795 for layerbranchid in self.layerBranches: 796 layerbranch = self.layerBranches[layerbranchid] 797 if branches and layerbranch.branch.name not in branches: 798 continue 799 800 if layerbranch.layer.name == name: 801 return layerbranch 802 803 return None 804 805 def find_dependencies(self, names=None, branches=None, layerBranches=None, ignores=None): 806 '''Return a tuple of all dependencies and valid items for the list of (layer) names 807 808 The dependency scanning happens depth-first. The returned 809 dependencies should be in the best order to define bblayers. 810 811 names - list of layer names (searching layerItems) 812 branches - when specified (with names) only this list of branches are evaluated 813 814 layerBranches - list of layerBranches to resolve dependencies 815 816 ignores - list of layer names to ignore 817 818 return: (dependencies, invalid) 819 820 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ] 821 invalid = [ LayerItem.name1, LayerItem.name2, ... ]''' 822 823 invalid = [] 824 825 # Convert name/branch to layerBranches 826 if layerbranches is None: 827 layerbranches = [] 828 829 for name in names: 830 if ignores and name in ignores: 831 continue 832 833 layerbranch = self.find_layerbranch(name, branches) 834 if not layerbranch: 835 invalid.append(name) 836 else: 837 layerbranches.append(layerbranch) 838 839 for layerbranch in layerbranches: 840 if layerbranch.index != self: 841 raise LayerIndexException("Can not resolve dependencies across indexes with this class function!") 842 843 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid): 844 for layerbranch in layerbranches: 845 if ignores and layerbranch.layer.name in ignores: 846 continue 847 848 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]: 849 deplayerbranch = layerdependency.dependency_layerBranch 850 851 if ignores and deplayerbranch.layer.name in ignores: 852 continue 853 854 # New dependency, we need to resolve it now... depth-first 855 if deplayerbranch.layer.name not in dependencies: 856 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid) 857 858 if deplayerbranch.layer.name not in dependencies: 859 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency] 860 else: 861 if layerdependency not in dependencies[deplayerbranch.layer.name]: 862 dependencies[deplayerbranch.layer.name].append(layerdependency) 863 864 return (dependencies, invalid) 865 866 # OK, resolve this one... 867 dependencies = OrderedDict() 868 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid) 869 870 # Is this item already in the list, if not add it 871 for layerbranch in layerbranches: 872 if layerbranch.layer.name not in dependencies: 873 dependencies[layerbranch.layer.name] = [layerbranch] 874 875 return (dependencies, invalid) 876 877 878# Define a basic LayerIndexItemObj. This object forms the basis for all other 879# objects. The raw Layer Index data is stored in the _data element, but we 880# do not want users to access data directly. So wrap this and protect it 881# from direct manipulation. 882# 883# It is up to the insantiators of the objects to fill them out, and once done 884# lock the objects to prevent further accidently manipulation. 885# 886# Using the getattr, setattr and properties we can access and manipulate 887# the data within the data element. 888class LayerIndexItemObj(): 889 def __init__(self, index, data=None, lock=False): 890 if data is None: 891 data = {} 892 893 if type(data) != type(dict()): 894 raise TypeError('data (%s) is not a dict' % type(data)) 895 896 super().__setattr__('_lock', lock) 897 super().__setattr__('index', index) 898 super().__setattr__('_data', data) 899 900 def __eq__(self, other): 901 if self.__class__ != other.__class__: 902 return False 903 res=(self._data == other._data) 904 return res 905 906 def __bool__(self): 907 return bool(self._data) 908 909 def __getattr__(self, name): 910 # These are internal to THIS class, and not part of data 911 if name == "index" or name.startswith('_'): 912 return super().__getattribute__(name) 913 914 if name not in self._data: 915 raise AttributeError('%s not in datastore' % name) 916 917 return self._data[name] 918 919 def _setattr(self, name, value, prop=True): 920 '''__setattr__ like function, but with control over property object behavior''' 921 if self.isLocked(): 922 raise TypeError("Can not set attribute '%s': Object data is locked" % name) 923 924 if name.startswith('_'): 925 super().__setattr__(name, value) 926 return 927 928 # Since __setattr__ runs before properties, we need to check if 929 # there is a setter property and then execute it 930 # ... or return self._data[name] 931 propertyobj = getattr(self.__class__, name, None) 932 if prop and isinstance(propertyobj, property): 933 if propertyobj.fset: 934 propertyobj.fset(self, value) 935 else: 936 raise AttributeError('Attribute %s is readonly, and may not be set' % name) 937 else: 938 self._data[name] = value 939 940 def __setattr__(self, name, value): 941 self._setattr(name, value, prop=True) 942 943 def _delattr(self, name, prop=True): 944 # Since __delattr__ runs before properties, we need to check if 945 # there is a deleter property and then execute it 946 # ... or we pop it ourselves.. 947 propertyobj = getattr(self.__class__, name, None) 948 if prop and isinstance(propertyobj, property): 949 if propertyobj.fdel: 950 propertyobj.fdel(self) 951 else: 952 raise AttributeError('Attribute %s is readonly, and may not be deleted' % name) 953 else: 954 self._data.pop(name) 955 956 def __delattr__(self, name): 957 self._delattr(name, prop=True) 958 959 def lockData(self): 960 '''Lock data object (make it readonly)''' 961 super().__setattr__("_lock", True) 962 963 def unlockData(self): 964 '''unlock data object (make it readonly)''' 965 super().__setattr__("_lock", False) 966 967 def isLocked(self): 968 '''Is this object locked (readonly)?''' 969 return self._lock 970 971# Branch object 972class Branch(LayerIndexItemObj): 973 def define_data(self, id, name, bitbake_branch, 974 short_description=None, sort_priority=1, 975 updates_enabled=True, updated=None, 976 update_environment=None): 977 self.id = id 978 self.name = name 979 self.bitbake_branch = bitbake_branch 980 self.short_description = short_description or name 981 self.sort_priority = sort_priority 982 self.updates_enabled = updates_enabled 983 self.updated = updated or datetime.datetime.today().isoformat() 984 self.update_environment = update_environment 985 986 @property 987 def name(self): 988 return self.__getattr__('name') 989 990 @name.setter 991 def name(self, value): 992 self._data['name'] = value 993 994 if self.bitbake_branch == value: 995 self.bitbake_branch = "" 996 997 @name.deleter 998 def name(self): 999 self._delattr('name', prop=False) 1000 1001 @property 1002 def bitbake_branch(self): 1003 try: 1004 return self.__getattr__('bitbake_branch') 1005 except AttributeError: 1006 return self.name 1007 1008 @bitbake_branch.setter 1009 def bitbake_branch(self, value): 1010 if self.name == value: 1011 self._data['bitbake_branch'] = "" 1012 else: 1013 self._data['bitbake_branch'] = value 1014 1015 @bitbake_branch.deleter 1016 def bitbake_branch(self): 1017 self._delattr('bitbake_branch', prop=False) 1018 1019 1020class LayerItem(LayerIndexItemObj): 1021 def define_data(self, id, name, status='P', 1022 layer_type='A', summary=None, 1023 description=None, 1024 vcs_url=None, vcs_web_url=None, 1025 vcs_web_tree_base_url=None, 1026 vcs_web_file_base_url=None, 1027 usage_url=None, 1028 mailing_list_url=None, 1029 index_preference=1, 1030 classic=False, 1031 updated=None): 1032 self.id = id 1033 self.name = name 1034 self.status = status 1035 self.layer_type = layer_type 1036 self.summary = summary or name 1037 self.description = description or summary or name 1038 self.vcs_url = vcs_url 1039 self.vcs_web_url = vcs_web_url 1040 self.vcs_web_tree_base_url = vcs_web_tree_base_url 1041 self.vcs_web_file_base_url = vcs_web_file_base_url 1042 self.index_preference = index_preference 1043 self.classic = classic 1044 self.updated = updated or datetime.datetime.today().isoformat() 1045 1046 1047class LayerBranch(LayerIndexItemObj): 1048 def define_data(self, id, collection, version, layer, branch, 1049 vcs_subdir="", vcs_last_fetch=None, 1050 vcs_last_rev=None, vcs_last_commit=None, 1051 actual_branch="", 1052 updated=None): 1053 self.id = id 1054 self.collection = collection 1055 self.version = version 1056 if isinstance(layer, LayerItem): 1057 self.layer = layer 1058 else: 1059 self.layer_id = layer 1060 1061 if isinstance(branch, Branch): 1062 self.branch = branch 1063 else: 1064 self.branch_id = branch 1065 1066 self.vcs_subdir = vcs_subdir 1067 self.vcs_last_fetch = vcs_last_fetch 1068 self.vcs_last_rev = vcs_last_rev 1069 self.vcs_last_commit = vcs_last_commit 1070 self.actual_branch = actual_branch 1071 self.updated = updated or datetime.datetime.today().isoformat() 1072 1073 # This is a little odd, the _data attribute is 'layer', but it's really 1074 # referring to the layer id.. so lets adjust this to make it useful 1075 @property 1076 def layer_id(self): 1077 return self.__getattr__('layer') 1078 1079 @layer_id.setter 1080 def layer_id(self, value): 1081 self._setattr('layer', value, prop=False) 1082 1083 @layer_id.deleter 1084 def layer_id(self): 1085 self._delattr('layer', prop=False) 1086 1087 @property 1088 def layer(self): 1089 try: 1090 return self.index.layerItems[self.layer_id] 1091 except KeyError: 1092 raise AttributeError('Unable to find layerItems in index to map layer_id %s' % self.layer_id) 1093 except IndexError: 1094 raise AttributeError('Unable to find layer_id %s in index layerItems' % self.layer_id) 1095 1096 @layer.setter 1097 def layer(self, value): 1098 if not isinstance(value, LayerItem): 1099 raise TypeError('value is not a LayerItem') 1100 if self.index != value.index: 1101 raise AttributeError('Object and value do not share the same index and thus key set.') 1102 self.layer_id = value.id 1103 1104 @layer.deleter 1105 def layer(self): 1106 del self.layer_id 1107 1108 @property 1109 def branch_id(self): 1110 return self.__getattr__('branch') 1111 1112 @branch_id.setter 1113 def branch_id(self, value): 1114 self._setattr('branch', value, prop=False) 1115 1116 @branch_id.deleter 1117 def branch_id(self): 1118 self._delattr('branch', prop=False) 1119 1120 @property 1121 def branch(self): 1122 try: 1123 logger.debug("Get branch object from branches[%s]" % (self.branch_id)) 1124 return self.index.branches[self.branch_id] 1125 except KeyError: 1126 raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id) 1127 except IndexError: 1128 raise AttributeError('Unable to find branch_id %s in index branches' % self.branch_id) 1129 1130 @branch.setter 1131 def branch(self, value): 1132 if not isinstance(value, LayerItem): 1133 raise TypeError('value is not a LayerItem') 1134 if self.index != value.index: 1135 raise AttributeError('Object and value do not share the same index and thus key set.') 1136 self.branch_id = value.id 1137 1138 @branch.deleter 1139 def branch(self): 1140 del self.branch_id 1141 1142 @property 1143 def actual_branch(self): 1144 if self.__getattr__('actual_branch'): 1145 return self.__getattr__('actual_branch') 1146 else: 1147 return self.branch.name 1148 1149 @actual_branch.setter 1150 def actual_branch(self, value): 1151 logger.debug("Set actual_branch to %s .. name is %s" % (value, self.branch.name)) 1152 if value != self.branch.name: 1153 self._setattr('actual_branch', value, prop=False) 1154 else: 1155 self._setattr('actual_branch', '', prop=False) 1156 1157 @actual_branch.deleter 1158 def actual_branch(self): 1159 self._delattr('actual_branch', prop=False) 1160 1161# Extend LayerIndexItemObj with common LayerBranch manipulations 1162# All of the remaining LayerIndex objects refer to layerbranch, and it is 1163# up to the user to follow that back through the LayerBranch object into 1164# the layer object to get various attributes. So add an intermediate set 1165# of attributes that can easily get us the layerbranch as well as layer. 1166 1167class LayerIndexItemObj_LayerBranch(LayerIndexItemObj): 1168 @property 1169 def layerbranch_id(self): 1170 return self.__getattr__('layerbranch') 1171 1172 @layerbranch_id.setter 1173 def layerbranch_id(self, value): 1174 self._setattr('layerbranch', value, prop=False) 1175 1176 @layerbranch_id.deleter 1177 def layerbranch_id(self): 1178 self._delattr('layerbranch', prop=False) 1179 1180 @property 1181 def layerbranch(self): 1182 try: 1183 return self.index.layerBranches[self.layerbranch_id] 1184 except KeyError: 1185 raise AttributeError('Unable to find layerBranches in index to map layerbranch_id %s' % self.layerbranch_id) 1186 except IndexError: 1187 raise AttributeError('Unable to find layerbranch_id %s in index branches' % self.layerbranch_id) 1188 1189 @layerbranch.setter 1190 def layerbranch(self, value): 1191 if not isinstance(value, LayerBranch): 1192 raise TypeError('value (%s) is not a layerBranch' % type(value)) 1193 if self.index != value.index: 1194 raise AttributeError('Object and value do not share the same index and thus key set.') 1195 self.layerbranch_id = value.id 1196 1197 @layerbranch.deleter 1198 def layerbranch(self): 1199 del self.layerbranch_id 1200 1201 @property 1202 def layer_id(self): 1203 return self.layerbranch.layer_id 1204 1205 # Doesn't make sense to set or delete layer_id 1206 1207 @property 1208 def layer(self): 1209 return self.layerbranch.layer 1210 1211 # Doesn't make sense to set or delete layer 1212 1213 1214class LayerDependency(LayerIndexItemObj_LayerBranch): 1215 def define_data(self, id, layerbranch, dependency, required=True): 1216 self.id = id 1217 if isinstance(layerbranch, LayerBranch): 1218 self.layerbranch = layerbranch 1219 else: 1220 self.layerbranch_id = layerbranch 1221 if isinstance(dependency, LayerDependency): 1222 self.dependency = dependency 1223 else: 1224 self.dependency_id = dependency 1225 self.required = required 1226 1227 @property 1228 def dependency_id(self): 1229 return self.__getattr__('dependency') 1230 1231 @dependency_id.setter 1232 def dependency_id(self, value): 1233 self._setattr('dependency', value, prop=False) 1234 1235 @dependency_id.deleter 1236 def dependency_id(self): 1237 self._delattr('dependency', prop=False) 1238 1239 @property 1240 def dependency(self): 1241 try: 1242 return self.index.layerItems[self.dependency_id] 1243 except KeyError: 1244 raise AttributeError('Unable to find layerItems in index to map layerbranch_id %s' % self.dependency_id) 1245 except IndexError: 1246 raise AttributeError('Unable to find dependency_id %s in index layerItems' % self.dependency_id) 1247 1248 @dependency.setter 1249 def dependency(self, value): 1250 if not isinstance(value, LayerDependency): 1251 raise TypeError('value (%s) is not a dependency' % type(value)) 1252 if self.index != value.index: 1253 raise AttributeError('Object and value do not share the same index and thus key set.') 1254 self.dependency_id = value.id 1255 1256 @dependency.deleter 1257 def dependency(self): 1258 self._delattr('dependency', prop=False) 1259 1260 @property 1261 def dependency_layerBranch(self): 1262 layerid = self.dependency_id 1263 branchid = self.layerbranch.branch_id 1264 1265 try: 1266 return self.index.layerBranches_layerId_branchId["%s:%s" % (layerid, branchid)] 1267 except IndexError: 1268 # layerBranches_layerId_branchId -- but not layerId:branchId 1269 raise AttributeError('Unable to find layerId:branchId %s:%s in index layerBranches_layerId_branchId' % (layerid, branchid)) 1270 except KeyError: 1271 raise AttributeError('Unable to find layerId:branchId %s:%s in layerItems and layerBranches' % (layerid, branchid)) 1272 1273 # dependency_layerBranch doesn't make sense to set or del 1274 1275 1276class Recipe(LayerIndexItemObj_LayerBranch): 1277 def define_data(self, id, 1278 filename, filepath, pn, pv, layerbranch, 1279 summary="", description="", section="", license="", 1280 homepage="", bugtracker="", provides="", bbclassextend="", 1281 inherits="", blacklisted="", updated=None): 1282 self.id = id 1283 self.filename = filename 1284 self.filepath = filepath 1285 self.pn = pn 1286 self.pv = pv 1287 self.summary = summary 1288 self.description = description 1289 self.section = section 1290 self.license = license 1291 self.homepage = homepage 1292 self.bugtracker = bugtracker 1293 self.provides = provides 1294 self.bbclassextend = bbclassextend 1295 self.inherits = inherits 1296 self.updated = updated or datetime.datetime.today().isoformat() 1297 self.blacklisted = blacklisted 1298 if isinstance(layerbranch, LayerBranch): 1299 self.layerbranch = layerbranch 1300 else: 1301 self.layerbranch_id = layerbranch 1302 1303 @property 1304 def fullpath(self): 1305 return os.path.join(self.filepath, self.filename) 1306 1307 # Set would need to understand how to split it 1308 # del would we del both parts? 1309 1310 @property 1311 def inherits(self): 1312 if 'inherits' not in self._data: 1313 # Older indexes may not have this, so emulate it 1314 if '-image-' in self.pn: 1315 return 'image' 1316 return self.__getattr__('inherits') 1317 1318 @inherits.setter 1319 def inherits(self, value): 1320 return self._setattr('inherits', value, prop=False) 1321 1322 @inherits.deleter 1323 def inherits(self): 1324 return self._delattr('inherits', prop=False) 1325 1326 1327class Machine(LayerIndexItemObj_LayerBranch): 1328 def define_data(self, id, 1329 name, description, layerbranch, 1330 updated=None): 1331 self.id = id 1332 self.name = name 1333 self.description = description 1334 if isinstance(layerbranch, LayerBranch): 1335 self.layerbranch = layerbranch 1336 else: 1337 self.layerbranch_id = layerbranch 1338 self.updated = updated or datetime.datetime.today().isoformat() 1339 1340class Distro(LayerIndexItemObj_LayerBranch): 1341 def define_data(self, id, 1342 name, description, layerbranch, 1343 updated=None): 1344 self.id = id 1345 self.name = name 1346 self.description = description 1347 if isinstance(layerbranch, LayerBranch): 1348 self.layerbranch = layerbranch 1349 else: 1350 self.layerbranch_id = layerbranch 1351 self.updated = updated or datetime.datetime.today().isoformat() 1352 1353# When performing certain actions, we may need to sort the data. 1354# This will allow us to keep it consistent from run to run. 1355def sort_entry(item): 1356 newitem = item 1357 try: 1358 if type(newitem) == type(dict()): 1359 newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0])) 1360 for index in newitem: 1361 newitem[index] = sort_entry(newitem[index]) 1362 elif type(newitem) == type(list()): 1363 newitem.sort(key=lambda obj: obj['id']) 1364 for index, _ in enumerate(newitem): 1365 newitem[index] = sort_entry(newitem[index]) 1366 except: 1367 logger.error('Sort failed for item %s' % type(item)) 1368 pass 1369 1370 return newitem 1371