1""" 2BitBake 'Fetch' implementations 3 4Classes for obtaining upstream sources for the 5BitBake build tools. 6""" 7 8# Copyright (C) 2003, 2004 Chris Larson 9# Copyright (C) 2012 Intel Corporation 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os, re 16import signal 17import logging 18import urllib.request, urllib.parse, urllib.error 19if 'git' not in urllib.parse.uses_netloc: 20 urllib.parse.uses_netloc.append('git') 21import operator 22import collections 23import subprocess 24import pickle 25import errno 26import bb.persist_data, bb.utils 27import bb.checksum 28import bb.process 29import bb.event 30 31__version__ = "2" 32_checksum_cache = bb.checksum.FileChecksumCache() 33 34logger = logging.getLogger("BitBake.Fetcher") 35 36CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ] 37SHOWN_CHECKSUM_LIST = ["sha256"] 38 39class BBFetchException(Exception): 40 """Class all fetch exceptions inherit from""" 41 def __init__(self, message): 42 self.msg = message 43 Exception.__init__(self, message) 44 45 def __str__(self): 46 return self.msg 47 48class UntrustedUrl(BBFetchException): 49 """Exception raised when encountering a host not listed in BB_ALLOWED_NETWORKS""" 50 def __init__(self, url, message=''): 51 if message: 52 msg = message 53 else: 54 msg = "The URL: '%s' is not trusted and cannot be used" % url 55 self.url = url 56 BBFetchException.__init__(self, msg) 57 self.args = (url,) 58 59class MalformedUrl(BBFetchException): 60 """Exception raised when encountering an invalid url""" 61 def __init__(self, url, message=''): 62 if message: 63 msg = message 64 else: 65 msg = "The URL: '%s' is invalid and cannot be interpreted" % url 66 self.url = url 67 BBFetchException.__init__(self, msg) 68 self.args = (url,) 69 70class FetchError(BBFetchException): 71 """General fetcher exception when something happens incorrectly""" 72 def __init__(self, message, url = None): 73 if url: 74 msg = "Fetcher failure for URL: '%s'. %s" % (url, message) 75 else: 76 msg = "Fetcher failure: %s" % message 77 self.url = url 78 BBFetchException.__init__(self, msg) 79 self.args = (message, url) 80 81class ChecksumError(FetchError): 82 """Exception when mismatched checksum encountered""" 83 def __init__(self, message, url = None, checksum = None): 84 self.checksum = checksum 85 FetchError.__init__(self, message, url) 86 87class NoChecksumError(FetchError): 88 """Exception when no checksum is specified, but BB_STRICT_CHECKSUM is set""" 89 90class UnpackError(BBFetchException): 91 """General fetcher exception when something happens incorrectly when unpacking""" 92 def __init__(self, message, url): 93 msg = "Unpack failure for URL: '%s'. %s" % (url, message) 94 self.url = url 95 BBFetchException.__init__(self, msg) 96 self.args = (message, url) 97 98class NoMethodError(BBFetchException): 99 """Exception raised when there is no method to obtain a supplied url or set of urls""" 100 def __init__(self, url): 101 msg = "Could not find a fetcher which supports the URL: '%s'" % url 102 self.url = url 103 BBFetchException.__init__(self, msg) 104 self.args = (url,) 105 106class MissingParameterError(BBFetchException): 107 """Exception raised when a fetch method is missing a critical parameter in the url""" 108 def __init__(self, missing, url): 109 msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing) 110 self.url = url 111 self.missing = missing 112 BBFetchException.__init__(self, msg) 113 self.args = (missing, url) 114 115class ParameterError(BBFetchException): 116 """Exception raised when a url cannot be processed due to invalid parameters.""" 117 def __init__(self, message, url): 118 msg = "URL: '%s' has invalid parameters. %s" % (url, message) 119 self.url = url 120 BBFetchException.__init__(self, msg) 121 self.args = (message, url) 122 123class NetworkAccess(BBFetchException): 124 """Exception raised when network access is disabled but it is required.""" 125 def __init__(self, url, cmd): 126 msg = "Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command %s (for url %s)" % (cmd, url) 127 self.url = url 128 self.cmd = cmd 129 BBFetchException.__init__(self, msg) 130 self.args = (url, cmd) 131 132class NonLocalMethod(Exception): 133 def __init__(self): 134 Exception.__init__(self) 135 136class MissingChecksumEvent(bb.event.Event): 137 def __init__(self, url, **checksums): 138 self.url = url 139 self.checksums = checksums 140 bb.event.Event.__init__(self) 141 142 143class URI(object): 144 """ 145 A class representing a generic URI, with methods for 146 accessing the URI components, and stringifies to the 147 URI. 148 149 It is constructed by calling it with a URI, or setting 150 the attributes manually: 151 152 uri = URI("http://example.com/") 153 154 uri = URI() 155 uri.scheme = 'http' 156 uri.hostname = 'example.com' 157 uri.path = '/' 158 159 It has the following attributes: 160 161 * scheme (read/write) 162 * userinfo (authentication information) (read/write) 163 * username (read/write) 164 * password (read/write) 165 166 Note, password is deprecated as of RFC 3986. 167 168 * hostname (read/write) 169 * port (read/write) 170 * hostport (read only) 171 "hostname:port", if both are set, otherwise just "hostname" 172 * path (read/write) 173 * path_quoted (read/write) 174 A URI quoted version of path 175 * params (dict) (read/write) 176 * query (dict) (read/write) 177 * relative (bool) (read only) 178 True if this is a "relative URI", (e.g. file:foo.diff) 179 180 It stringifies to the URI itself. 181 182 Some notes about relative URIs: while it's specified that 183 a URI beginning with <scheme>:// should either be directly 184 followed by a hostname or a /, the old URI handling of the 185 fetch2 library did not conform to this. Therefore, this URI 186 class has some kludges to make sure that URIs are parsed in 187 a way comforming to bitbake's current usage. This URI class 188 supports the following: 189 190 file:relative/path.diff (IETF compliant) 191 git:relative/path.git (IETF compliant) 192 git:///absolute/path.git (IETF compliant) 193 file:///absolute/path.diff (IETF compliant) 194 195 file://relative/path.diff (not IETF compliant) 196 197 But it does not support the following: 198 199 file://hostname/absolute/path.diff (would be IETF compliant) 200 201 Note that the last case only applies to a list of 202 explicitly allowed schemes (currently only file://), that requires 203 its URIs to not have a network location. 204 """ 205 206 _relative_schemes = ['file', 'git'] 207 _netloc_forbidden = ['file'] 208 209 def __init__(self, uri=None): 210 self.scheme = '' 211 self.userinfo = '' 212 self.hostname = '' 213 self.port = None 214 self._path = '' 215 self.params = {} 216 self.query = {} 217 self.relative = False 218 219 if not uri: 220 return 221 222 # We hijack the URL parameters, since the way bitbake uses 223 # them are not quite RFC compliant. 224 uri, param_str = (uri.split(";", 1) + [None])[:2] 225 226 urlp = urllib.parse.urlparse(uri) 227 self.scheme = urlp.scheme 228 229 reparse = 0 230 231 # Coerce urlparse to make URI scheme use netloc 232 if not self.scheme in urllib.parse.uses_netloc: 233 urllib.parse.uses_params.append(self.scheme) 234 reparse = 1 235 236 # Make urlparse happy(/ier) by converting local resources 237 # to RFC compliant URL format. E.g.: 238 # file://foo.diff -> file:foo.diff 239 if urlp.scheme in self._netloc_forbidden: 240 uri = re.sub("(?<=:)//(?!/)", "", uri, 1) 241 reparse = 1 242 243 if reparse: 244 urlp = urllib.parse.urlparse(uri) 245 246 # Identify if the URI is relative or not 247 if urlp.scheme in self._relative_schemes and \ 248 re.compile(r"^\w+:(?!//)").match(uri): 249 self.relative = True 250 251 if not self.relative: 252 self.hostname = urlp.hostname or '' 253 self.port = urlp.port 254 255 self.userinfo += urlp.username or '' 256 257 if urlp.password: 258 self.userinfo += ':%s' % urlp.password 259 260 self.path = urllib.parse.unquote(urlp.path) 261 262 if param_str: 263 self.params = self._param_str_split(param_str, ";") 264 if urlp.query: 265 self.query = self._param_str_split(urlp.query, "&") 266 267 def __str__(self): 268 userinfo = self.userinfo 269 if userinfo: 270 userinfo += '@' 271 272 return "%s:%s%s%s%s%s%s" % ( 273 self.scheme, 274 '' if self.relative else '//', 275 userinfo, 276 self.hostport, 277 self.path_quoted, 278 self._query_str(), 279 self._param_str()) 280 281 def _param_str(self): 282 return ( 283 ''.join([';', self._param_str_join(self.params, ";")]) 284 if self.params else '') 285 286 def _query_str(self): 287 return ( 288 ''.join(['?', self._param_str_join(self.query, "&")]) 289 if self.query else '') 290 291 def _param_str_split(self, string, elmdelim, kvdelim="="): 292 ret = collections.OrderedDict() 293 for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]: 294 ret[k] = v 295 return ret 296 297 def _param_str_join(self, dict_, elmdelim, kvdelim="="): 298 return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()]) 299 300 @property 301 def hostport(self): 302 if not self.port: 303 return self.hostname 304 return "%s:%d" % (self.hostname, self.port) 305 306 @property 307 def path_quoted(self): 308 return urllib.parse.quote(self.path) 309 310 @path_quoted.setter 311 def path_quoted(self, path): 312 self.path = urllib.parse.unquote(path) 313 314 @property 315 def path(self): 316 return self._path 317 318 @path.setter 319 def path(self, path): 320 self._path = path 321 322 if not path or re.compile("^/").match(path): 323 self.relative = False 324 else: 325 self.relative = True 326 327 @property 328 def username(self): 329 if self.userinfo: 330 return (self.userinfo.split(":", 1))[0] 331 return '' 332 333 @username.setter 334 def username(self, username): 335 password = self.password 336 self.userinfo = username 337 if password: 338 self.userinfo += ":%s" % password 339 340 @property 341 def password(self): 342 if self.userinfo and ":" in self.userinfo: 343 return (self.userinfo.split(":", 1))[1] 344 return '' 345 346 @password.setter 347 def password(self, password): 348 self.userinfo = "%s:%s" % (self.username, password) 349 350def decodeurl(url): 351 """Decodes an URL into the tokens (scheme, network location, path, 352 user, password, parameters). 353 """ 354 355 m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) 356 if not m: 357 raise MalformedUrl(url) 358 359 type = m.group('type') 360 location = m.group('location') 361 if not location: 362 raise MalformedUrl(url) 363 user = m.group('user') 364 parm = m.group('parm') 365 366 locidx = location.find('/') 367 if locidx != -1 and type.lower() != 'file': 368 host = location[:locidx] 369 path = location[locidx:] 370 elif type.lower() == 'file': 371 host = "" 372 path = location 373 else: 374 host = location 375 path = "/" 376 if user: 377 m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user) 378 if m: 379 user = m.group('user') 380 pswd = m.group('pswd') 381 else: 382 user = '' 383 pswd = '' 384 385 p = collections.OrderedDict() 386 if parm: 387 for s in parm.split(';'): 388 if s: 389 if not '=' in s: 390 raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) 391 s1, s2 = s.split('=', 1) 392 p[s1] = s2 393 394 return type, host, urllib.parse.unquote(path), user, pswd, p 395 396def encodeurl(decoded): 397 """Encodes a URL from tokens (scheme, network location, path, 398 user, password, parameters). 399 """ 400 401 type, host, path, user, pswd, p = decoded 402 403 if not type: 404 raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) 405 url = ['%s://' % type] 406 if user and type != "file": 407 url.append("%s" % user) 408 if pswd: 409 url.append(":%s" % pswd) 410 url.append("@") 411 if host and type != "file": 412 url.append("%s" % host) 413 if path: 414 # Standardise path to ensure comparisons work 415 while '//' in path: 416 path = path.replace("//", "/") 417 url.append("%s" % urllib.parse.quote(path)) 418 if p: 419 for parm in p: 420 url.append(";%s=%s" % (parm, p[parm])) 421 422 return "".join(url) 423 424def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): 425 if not ud.url or not uri_find or not uri_replace: 426 logger.error("uri_replace: passed an undefined value, not replacing") 427 return None 428 uri_decoded = list(decodeurl(ud.url)) 429 uri_find_decoded = list(decodeurl(uri_find)) 430 uri_replace_decoded = list(decodeurl(uri_replace)) 431 logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) 432 result_decoded = ['', '', '', '', '', {}] 433 # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params 434 for loc, i in enumerate(uri_find_decoded): 435 result_decoded[loc] = uri_decoded[loc] 436 regexp = i 437 if loc == 0 and regexp and not regexp.endswith("$"): 438 # Leaving the type unanchored can mean "https" matching "file" can become "files" 439 # which is clearly undesirable. 440 regexp += "$" 441 if loc == 5: 442 # Handle URL parameters 443 if i: 444 # Any specified URL parameters must match 445 for k in uri_find_decoded[loc]: 446 if uri_decoded[loc][k] != uri_find_decoded[loc][k]: 447 return None 448 # Overwrite any specified replacement parameters 449 for k in uri_replace_decoded[loc]: 450 for l in replacements: 451 uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) 452 result_decoded[loc][k] = uri_replace_decoded[loc][k] 453 elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]: 454 # User/password in the replacement is just a straight replacement 455 result_decoded[loc] = uri_replace_decoded[loc] 456 elif (re.match(regexp, uri_decoded[loc])): 457 if not uri_replace_decoded[loc]: 458 result_decoded[loc] = "" 459 else: 460 for k in replacements: 461 uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) 462 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) 463 result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) 464 if loc == 2: 465 # Handle path manipulations 466 basename = None 467 if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball: 468 # If the source and destination url types differ, must be a mirrortarball mapping 469 basename = os.path.basename(mirrortarball) 470 # Kill parameters, they make no sense for mirror tarballs 471 uri_decoded[5] = {} 472 uri_find_decoded[5] = {} 473 elif ud.localpath and ud.method.supports_checksum(ud): 474 basename = os.path.basename(ud.localpath) 475 if basename: 476 uri_basename = os.path.basename(uri_decoded[loc]) 477 # Prefix with a slash as a sentinel in case 478 # result_decoded[loc] does not contain one. 479 path = "/" + result_decoded[loc] 480 if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename): 481 result_decoded[loc] = path[1:-len(uri_basename)] + basename 482 elif not path.endswith("/" + basename): 483 result_decoded[loc] = os.path.join(path[1:], basename) 484 else: 485 return None 486 result = encodeurl(result_decoded) 487 if result == ud.url: 488 return None 489 logger.debug2("For url %s returning %s" % (ud.url, result)) 490 return result 491 492methods = [] 493urldata_cache = {} 494saved_headrevs = {} 495 496def fetcher_init(d): 497 """ 498 Called to initialize the fetchers once the configuration data is known. 499 Calls before this must not hit the cache. 500 """ 501 502 revs = bb.persist_data.persist('BB_URI_HEADREVS', d) 503 try: 504 # fetcher_init is called multiple times, so make sure we only save the 505 # revs the first time it is called. 506 if not bb.fetch2.saved_headrevs: 507 bb.fetch2.saved_headrevs = dict(revs) 508 except: 509 pass 510 511 # When to drop SCM head revisions controlled by user policy 512 srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear" 513 if srcrev_policy == "cache": 514 logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) 515 elif srcrev_policy == "clear": 516 logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) 517 revs.clear() 518 else: 519 raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) 520 521 _checksum_cache.init_cache(d.getVar("BB_CACHEDIR")) 522 523 for m in methods: 524 if hasattr(m, "init"): 525 m.init(d) 526 527def fetcher_parse_save(): 528 _checksum_cache.save_extras() 529 530def fetcher_parse_done(): 531 _checksum_cache.save_merge() 532 533def fetcher_compare_revisions(d): 534 """ 535 Compare the revisions in the persistent cache with the saved values from 536 when bitbake was started and return true if they have changed. 537 """ 538 539 headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d)) 540 return headrevs != bb.fetch2.saved_headrevs 541 542def mirror_from_string(data): 543 mirrors = (data or "").replace('\\n',' ').split() 544 # Split into pairs 545 if len(mirrors) % 2 != 0: 546 bb.warn('Invalid mirror data %s, should have paired members.' % data) 547 return list(zip(*[iter(mirrors)]*2)) 548 549def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True): 550 """ 551 verify the MD5 and SHA256 checksum for downloaded src 552 553 Raises a FetchError if one or both of the SRC_URI checksums do not match 554 the downloaded file, or if BB_STRICT_CHECKSUM is set and there are no 555 checksums specified. 556 557 Returns a dict of checksums that can be stored in a done stamp file and 558 passed in as precomputed parameter in a later call to avoid re-computing 559 the checksums from the file. This allows verifying the checksums of the 560 file against those in the recipe each time, rather than only after 561 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. 562 """ 563 if ud.ignore_checksums or not ud.method.supports_checksum(ud): 564 return {} 565 566 if localpath is None: 567 localpath = ud.localpath 568 569 def compute_checksum_info(checksum_id): 570 checksum_name = getattr(ud, "%s_name" % checksum_id) 571 572 if checksum_id in precomputed: 573 checksum_data = precomputed[checksum_id] 574 else: 575 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath) 576 577 checksum_expected = getattr(ud, "%s_expected" % checksum_id) 578 579 if checksum_expected == '': 580 checksum_expected = None 581 582 return { 583 "id": checksum_id, 584 "name": checksum_name, 585 "data": checksum_data, 586 "expected": checksum_expected 587 } 588 589 checksum_infos = [] 590 for checksum_id in CHECKSUM_LIST: 591 checksum_infos.append(compute_checksum_info(checksum_id)) 592 593 checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos} 594 checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos} 595 596 for ci in checksum_infos: 597 if ci["id"] in SHOWN_CHECKSUM_LIST: 598 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] 599 600 # If no checksum has been provided 601 if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): 602 messages = [] 603 strict = d.getVar("BB_STRICT_CHECKSUM") or "0" 604 605 # If strict checking enabled and neither sum defined, raise error 606 if strict == "1": 607 raise NoChecksumError("\n".join(checksum_lines)) 608 609 bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) 610 611 if strict == "ignore": 612 return checksum_dict 613 614 # Log missing sums so user can more easily add them 615 messages.append("Missing checksum for '%s', consider adding at " \ 616 "least one to the recipe:" % ud.localpath) 617 messages.extend(checksum_lines) 618 logger.warning("\n".join(messages)) 619 620 # We want to alert the user if a checksum is defined in the recipe but 621 # it does not match. 622 messages = [] 623 messages.append("Checksum mismatch!") 624 bad_checksum = None 625 626 for ci in checksum_infos: 627 if ci["expected"] and ci["expected"] != ci["data"]: 628 messages.append("File: '%s' has %s checksum '%s' when '%s' was " \ 629 "expected" % (localpath, ci["id"], ci["data"], ci["expected"])) 630 bad_checksum = ci["data"] 631 632 if bad_checksum: 633 messages.append("If this change is expected (e.g. you have upgraded " \ 634 "to a new version without updating the checksums) " \ 635 "then you can use these lines within the recipe:") 636 messages.extend(checksum_lines) 637 messages.append("Otherwise you should retry the download and/or " \ 638 "check with upstream to determine if the file has " \ 639 "become corrupted or otherwise unexpectedly modified.") 640 raise ChecksumError("\n".join(messages), ud.url, bad_checksum) 641 642 return checksum_dict 643 644def verify_donestamp(ud, d, origud=None): 645 """ 646 Check whether the done stamp file has the right checksums (if the fetch 647 method supports them). If it doesn't, delete the done stamp and force 648 a re-download. 649 650 Returns True, if the donestamp exists and is valid, False otherwise. When 651 returning False, any existing done stamps are removed. 652 """ 653 if not ud.needdonestamp or (origud and not origud.needdonestamp): 654 return True 655 656 if not os.path.exists(ud.localpath): 657 # local path does not exist 658 if os.path.exists(ud.donestamp): 659 # done stamp exists, but the downloaded file does not; the done stamp 660 # must be incorrect, re-trigger the download 661 bb.utils.remove(ud.donestamp) 662 return False 663 664 if (not ud.method.supports_checksum(ud) or 665 (origud and not origud.method.supports_checksum(origud))): 666 # if done stamp exists and checksums not supported; assume the local 667 # file is current 668 return os.path.exists(ud.donestamp) 669 670 precomputed_checksums = {} 671 # Only re-use the precomputed checksums if the donestamp is newer than the 672 # file. Do not rely on the mtime of directories, though. If ud.localpath is 673 # a directory, there will probably not be any checksums anyway. 674 if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or 675 os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)): 676 try: 677 with open(ud.donestamp, "rb") as cachefile: 678 pickled = pickle.Unpickler(cachefile) 679 precomputed_checksums.update(pickled.load()) 680 except Exception as e: 681 # Avoid the warnings on the upgrade path from emtpy done stamp 682 # files to those containing the checksums. 683 if not isinstance(e, EOFError): 684 # Ignore errors, they aren't fatal 685 logger.warning("Couldn't load checksums from donestamp %s: %s " 686 "(msg: %s)" % (ud.donestamp, type(e).__name__, 687 str(e))) 688 689 try: 690 checksums = verify_checksum(ud, d, precomputed_checksums) 691 # If the cache file did not have the checksums, compute and store them 692 # as an upgrade path from the previous done stamp file format. 693 if checksums != precomputed_checksums: 694 with open(ud.donestamp, "wb") as cachefile: 695 p = pickle.Pickler(cachefile, 2) 696 p.dump(checksums) 697 return True 698 except ChecksumError as e: 699 # Checksums failed to verify, trigger re-download and remove the 700 # incorrect stamp file. 701 logger.warning("Checksum mismatch for local file %s\n" 702 "Cleaning and trying again." % ud.localpath) 703 if os.path.exists(ud.localpath): 704 rename_bad_checksum(ud, e.checksum) 705 bb.utils.remove(ud.donestamp) 706 return False 707 708 709def update_stamp(ud, d): 710 """ 711 donestamp is file stamp indicating the whole fetching is done 712 this function update the stamp after verifying the checksum 713 """ 714 if not ud.needdonestamp: 715 return 716 717 if os.path.exists(ud.donestamp): 718 # Touch the done stamp file to show active use of the download 719 try: 720 os.utime(ud.donestamp, None) 721 except: 722 # Errors aren't fatal here 723 pass 724 else: 725 try: 726 checksums = verify_checksum(ud, d) 727 # Store the checksums for later re-verification against the recipe 728 with open(ud.donestamp, "wb") as cachefile: 729 p = pickle.Pickler(cachefile, 2) 730 p.dump(checksums) 731 except ChecksumError as e: 732 # Checksums failed to verify, trigger re-download and remove the 733 # incorrect stamp file. 734 logger.warning("Checksum mismatch for local file %s\n" 735 "Cleaning and trying again." % ud.localpath) 736 if os.path.exists(ud.localpath): 737 rename_bad_checksum(ud, e.checksum) 738 bb.utils.remove(ud.donestamp) 739 raise 740 741def subprocess_setup(): 742 # Python installs a SIGPIPE handler by default. This is usually not what 743 # non-Python subprocesses expect. 744 # SIGPIPE errors are known issues with gzip/bash 745 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 746 747def mark_recipe_nocache(d): 748 if d.getVar('BB_SRCREV_POLICY') != "cache": 749 d.setVar('BB_DONT_CACHE', '1') 750 751def get_autorev(d): 752 mark_recipe_nocache(d) 753 d.setVar("__BBAUTOREV_SEEN", True) 754 return "AUTOINC" 755 756def _get_srcrev(d, method_name='sortable_revision'): 757 """ 758 Return the revision string, usually for use in the version string (PV) of the current package 759 Most packages usually only have one SCM so we just pass on the call. 760 In the multi SCM case, we build a value based on SRCREV_FORMAT which must 761 have been set. 762 763 The idea here is that we put the string "AUTOINC+" into return value if the revisions are not 764 incremental, other code is then responsible for turning that into an increasing value (if needed) 765 766 A method_name can be supplied to retrieve an alternatively formatted revision from a fetcher, if 767 that fetcher provides a method with the given name and the same signature as sortable_revision. 768 """ 769 770 d.setVar("__BBSRCREV_SEEN", "1") 771 recursion = d.getVar("__BBINSRCREV") 772 if recursion: 773 raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI") 774 d.setVar("__BBINSRCREV", True) 775 776 scms = [] 777 revs = [] 778 fetcher = Fetch(d.getVar('SRC_URI').split(), d) 779 urldata = fetcher.ud 780 for u in urldata: 781 if urldata[u].method.supports_srcrev(): 782 scms.append(u) 783 784 if not scms: 785 d.delVar("__BBINSRCREV") 786 return "", revs 787 788 789 if len(scms) == 1 and len(urldata[scms[0]].names) == 1: 790 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) 791 revs.append(rev) 792 if len(rev) > 10: 793 rev = rev[:10] 794 d.delVar("__BBINSRCREV") 795 if autoinc: 796 return "AUTOINC+" + rev, revs 797 return rev, revs 798 799 # 800 # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT 801 # 802 format = d.getVar('SRCREV_FORMAT') 803 if not format: 804 raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.\n"\ 805 "The SCMs are:\n%s" % '\n'.join(scms)) 806 807 name_to_rev = {} 808 seenautoinc = False 809 for scm in scms: 810 ud = urldata[scm] 811 for name in ud.names: 812 autoinc, rev = getattr(ud.method, method_name)(ud, d, name) 813 revs.append(rev) 814 seenautoinc = seenautoinc or autoinc 815 if len(rev) > 10: 816 rev = rev[:10] 817 name_to_rev[name] = rev 818 # Replace names by revisions in the SRCREV_FORMAT string. The approach used 819 # here can handle names being prefixes of other names and names appearing 820 # as substrings in revisions (in which case the name should not be 821 # expanded). The '|' regular expression operator tries matches from left to 822 # right, so we need to sort the names with the longest ones first. 823 names_descending_len = sorted(name_to_rev, key=len, reverse=True) 824 name_to_rev_re = "|".join(re.escape(name) for name in names_descending_len) 825 format = re.sub(name_to_rev_re, lambda match: name_to_rev[match.group(0)], format) 826 827 if seenautoinc: 828 format = "AUTOINC+" + format 829 830 d.delVar("__BBINSRCREV") 831 return format, revs 832 833def get_hashvalue(d, method_name='sortable_revision'): 834 pkgv, revs = _get_srcrev(d, method_name=method_name) 835 return " ".join(revs) 836 837def get_pkgv_string(d, method_name='sortable_revision'): 838 pkgv, revs = _get_srcrev(d, method_name=method_name) 839 return pkgv 840 841def get_srcrev(d, method_name='sortable_revision'): 842 pkgv, revs = _get_srcrev(d, method_name=method_name) 843 if not pkgv: 844 raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") 845 return pkgv 846 847def localpath(url, d): 848 fetcher = bb.fetch2.Fetch([url], d) 849 return fetcher.localpath(url) 850 851# Need to export PATH as binary could be in metadata paths 852# rather than host provided 853# Also include some other variables. 854FETCH_EXPORT_VARS = ['HOME', 'PATH', 855 'HTTP_PROXY', 'http_proxy', 856 'HTTPS_PROXY', 'https_proxy', 857 'FTP_PROXY', 'ftp_proxy', 858 'FTPS_PROXY', 'ftps_proxy', 859 'NO_PROXY', 'no_proxy', 860 'ALL_PROXY', 'all_proxy', 861 'GIT_PROXY_COMMAND', 862 'GIT_SSH', 863 'GIT_SSH_COMMAND', 864 'GIT_SSL_CAINFO', 865 'GIT_SMART_HTTP', 866 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 867 'SOCKS5_USER', 'SOCKS5_PASSWD', 868 'DBUS_SESSION_BUS_ADDRESS', 869 'P4CONFIG', 870 'SSL_CERT_FILE', 871 'NODE_EXTRA_CA_CERTS', 872 'AWS_PROFILE', 873 'AWS_ACCESS_KEY_ID', 874 'AWS_SECRET_ACCESS_KEY', 875 'AWS_ROLE_ARN', 876 'AWS_WEB_IDENTITY_TOKEN_FILE', 877 'AWS_DEFAULT_REGION', 878 'GIT_CACHE_PATH', 879 'REMOTE_CONTAINERS_IPC', 880 'SSL_CERT_DIR'] 881 882def get_fetcher_environment(d): 883 newenv = {} 884 origenv = d.getVar("BB_ORIGENV") 885 for name in bb.fetch2.FETCH_EXPORT_VARS: 886 value = d.getVar(name) 887 if not value and origenv: 888 value = origenv.getVar(name) 889 if value: 890 newenv[name] = value 891 return newenv 892 893def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): 894 """ 895 Run cmd returning the command output 896 Raise an error if interrupted or cmd fails 897 Optionally echo command output to stdout 898 Optionally remove the files/directories listed in cleanup upon failure 899 """ 900 901 exportvars = FETCH_EXPORT_VARS 902 903 if not cleanup: 904 cleanup = [] 905 906 # If PATH contains WORKDIR which contains PV-PR which contains SRCPV we 907 # can end up in circular recursion here so give the option of breaking it 908 # in a data store copy. 909 try: 910 d.getVar("PV") 911 d.getVar("PR") 912 except bb.data_smart.ExpansionError: 913 d = bb.data.createCopy(d) 914 d.setVar("PV", "fetcheravoidrecurse") 915 d.setVar("PR", "fetcheravoidrecurse") 916 917 origenv = d.getVar("BB_ORIGENV", False) 918 for var in exportvars: 919 val = d.getVar(var) or (origenv and origenv.getVar(var)) 920 if val: 921 cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) 922 923 # Disable pseudo as it may affect ssh, potentially causing it to hang. 924 cmd = 'export PSEUDO_DISABLED=1; ' + cmd 925 926 if workdir: 927 logger.debug("Running '%s' in %s" % (cmd, workdir)) 928 else: 929 logger.debug("Running %s", cmd) 930 931 success = False 932 error_message = "" 933 934 try: 935 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) 936 success = True 937 except bb.process.NotFoundError as e: 938 error_message = "Fetch command %s not found" % (e.command) 939 except bb.process.ExecutionError as e: 940 if e.stdout: 941 output = "output:\n%s\n%s" % (e.stdout, e.stderr) 942 elif e.stderr: 943 output = "output:\n%s" % e.stderr 944 else: 945 output = "no output" 946 error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) 947 except bb.process.CmdError as e: 948 error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) 949 if not success: 950 for f in cleanup: 951 try: 952 bb.utils.remove(f, True) 953 except OSError: 954 pass 955 956 raise FetchError(error_message) 957 958 return output 959 960def check_network_access(d, info, url): 961 """ 962 log remote network access, and error if BB_NO_NETWORK is set or the given 963 URI is untrusted 964 """ 965 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): 966 raise NetworkAccess(url, info) 967 elif not trusted_network(d, url): 968 raise UntrustedUrl(url, info) 969 else: 970 logger.debug("Fetcher accessed the network with the command %s" % info) 971 972def build_mirroruris(origud, mirrors, ld): 973 uris = [] 974 uds = [] 975 976 replacements = {} 977 replacements["TYPE"] = origud.type 978 replacements["HOST"] = origud.host 979 replacements["PATH"] = origud.path 980 replacements["BASENAME"] = origud.path.split("/")[-1] 981 replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') 982 983 def adduri(ud, uris, uds, mirrors, tarballs): 984 for line in mirrors: 985 try: 986 (find, replace) = line 987 except ValueError: 988 continue 989 990 for tarball in tarballs: 991 newuri = uri_replace(ud, find, replace, replacements, ld, tarball) 992 if not newuri or newuri in uris or newuri == origud.url: 993 continue 994 995 if not trusted_network(ld, newuri): 996 logger.debug("Mirror %s not in the list of trusted networks, skipping" % (newuri)) 997 continue 998 999 # Create a local copy of the mirrors minus the current line 1000 # this will prevent us from recursively processing the same line 1001 # as well as indirect recursion A -> B -> C -> A 1002 localmirrors = list(mirrors) 1003 localmirrors.remove(line) 1004 1005 try: 1006 newud = FetchData(newuri, ld) 1007 newud.ignore_checksums = True 1008 newud.setup_localpath(ld) 1009 except bb.fetch2.BBFetchException as e: 1010 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) 1011 logger.debug(str(e)) 1012 try: 1013 # setup_localpath of file:// urls may fail, we should still see 1014 # if mirrors of the url exist 1015 adduri(newud, uris, uds, localmirrors, tarballs) 1016 except UnboundLocalError: 1017 pass 1018 continue 1019 uris.append(newuri) 1020 uds.append(newud) 1021 1022 adduri(newud, uris, uds, localmirrors, tarballs) 1023 1024 adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None]) 1025 1026 return uris, uds 1027 1028def rename_bad_checksum(ud, suffix): 1029 """ 1030 Renames files to have suffix from parameter 1031 """ 1032 1033 if ud.localpath is None: 1034 return 1035 1036 new_localpath = "%s_bad-checksum_%s" % (ud.localpath, suffix) 1037 bb.warn("Renaming %s to %s" % (ud.localpath, new_localpath)) 1038 if not bb.utils.movefile(ud.localpath, new_localpath): 1039 bb.warn("Renaming %s to %s failed, grep movefile in log.do_fetch to see why" % (ud.localpath, new_localpath)) 1040 1041 1042def try_mirror_url(fetch, origud, ud, ld, check = False): 1043 # Return of None or a value means we're finished 1044 # False means try another url 1045 1046 if ud.lockfile and ud.lockfile != origud.lockfile: 1047 lf = bb.utils.lockfile(ud.lockfile) 1048 1049 try: 1050 if check: 1051 found = ud.method.checkstatus(fetch, ud, ld) 1052 if found: 1053 return found 1054 return False 1055 1056 if not verify_donestamp(ud, ld, origud) or ud.method.need_update(ud, ld): 1057 ud.method.download(ud, ld) 1058 if hasattr(ud.method,"build_mirror_data"): 1059 ud.method.build_mirror_data(ud, ld) 1060 1061 if not ud.localpath or not os.path.exists(ud.localpath): 1062 return False 1063 1064 if ud.localpath == origud.localpath: 1065 return ud.localpath 1066 1067 # We may be obtaining a mirror tarball which needs further processing by the real fetcher 1068 # If that tarball is a local file:// we need to provide a symlink to it 1069 dldir = ld.getVar("DL_DIR") 1070 1071 if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): 1072 # Create donestamp in old format to avoid triggering a re-download 1073 if ud.donestamp: 1074 bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) 1075 open(ud.donestamp, 'w').close() 1076 dest = os.path.join(dldir, os.path.basename(ud.localpath)) 1077 if not os.path.exists(dest): 1078 # In case this is executing without any file locks held (as is 1079 # the case for file:// URLs), two tasks may end up here at the 1080 # same time, in which case we do not want the second task to 1081 # fail when the link has already been created by the first task. 1082 try: 1083 os.symlink(ud.localpath, dest) 1084 except FileExistsError: 1085 pass 1086 if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): 1087 origud.method.download(origud, ld) 1088 if hasattr(origud.method, "build_mirror_data"): 1089 origud.method.build_mirror_data(origud, ld) 1090 return origud.localpath 1091 # Otherwise the result is a local file:// and we symlink to it 1092 ensure_symlink(ud.localpath, origud.localpath) 1093 update_stamp(origud, ld) 1094 return ud.localpath 1095 1096 except bb.fetch2.NetworkAccess: 1097 raise 1098 1099 except IOError as e: 1100 if e.errno in [errno.ESTALE]: 1101 logger.warning("Stale Error Observed %s." % ud.url) 1102 return False 1103 raise 1104 1105 except bb.fetch2.BBFetchException as e: 1106 if isinstance(e, ChecksumError): 1107 logger.warning("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url)) 1108 logger.warning(str(e)) 1109 if os.path.exists(ud.localpath): 1110 rename_bad_checksum(ud, e.checksum) 1111 elif isinstance(e, NoChecksumError): 1112 raise 1113 else: 1114 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) 1115 logger.debug(str(e)) 1116 try: 1117 ud.method.clean(ud, ld) 1118 except UnboundLocalError: 1119 pass 1120 return False 1121 finally: 1122 if ud.lockfile and ud.lockfile != origud.lockfile: 1123 bb.utils.unlockfile(lf) 1124 1125 1126def ensure_symlink(target, link_name): 1127 if not os.path.exists(link_name): 1128 dirname = os.path.dirname(link_name) 1129 bb.utils.mkdirhier(dirname) 1130 if os.path.islink(link_name): 1131 # Broken symbolic link 1132 os.unlink(link_name) 1133 1134 # In case this is executing without any file locks held (as is 1135 # the case for file:// URLs), two tasks may end up here at the 1136 # same time, in which case we do not want the second task to 1137 # fail when the link has already been created by the first task. 1138 try: 1139 os.symlink(target, link_name) 1140 except FileExistsError: 1141 pass 1142 1143 1144def try_mirrors(fetch, d, origud, mirrors, check = False): 1145 """ 1146 Try to use a mirrored version of the sources. 1147 This method will be automatically called before the fetchers go. 1148 1149 d Is a bb.data instance 1150 uri is the original uri we're trying to download 1151 mirrors is the list of mirrors we're going to try 1152 """ 1153 ld = d.createCopy() 1154 1155 uris, uds = build_mirroruris(origud, mirrors, ld) 1156 1157 for index, uri in enumerate(uris): 1158 ret = try_mirror_url(fetch, origud, uds[index], ld, check) 1159 if ret: 1160 return ret 1161 return None 1162 1163def trusted_network(d, url): 1164 """ 1165 Use a trusted url during download if networking is enabled and 1166 BB_ALLOWED_NETWORKS is set globally or for a specific recipe. 1167 Note: modifies SRC_URI & mirrors. 1168 """ 1169 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): 1170 return True 1171 1172 pkgname = d.expand(d.getVar('PN', False)) 1173 trusted_hosts = None 1174 if pkgname: 1175 trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False) 1176 1177 if not trusted_hosts: 1178 trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS') 1179 1180 # Not enabled. 1181 if not trusted_hosts: 1182 return True 1183 1184 scheme, network, path, user, passwd, param = decodeurl(url) 1185 1186 if not network: 1187 return True 1188 1189 network = network.split(':')[0] 1190 network = network.lower() 1191 1192 for host in trusted_hosts.split(" "): 1193 host = host.lower() 1194 if host.startswith("*.") and ("." + network).endswith(host[1:]): 1195 return True 1196 if host == network: 1197 return True 1198 1199 return False 1200 1201def srcrev_internal_helper(ud, d, name): 1202 """ 1203 Return: 1204 a) a source revision if specified 1205 b) latest revision if SRCREV="AUTOINC" 1206 c) None if not specified 1207 """ 1208 1209 srcrev = None 1210 pn = d.getVar("PN") 1211 attempts = [] 1212 if name != '' and pn: 1213 attempts.append("SRCREV_%s:pn-%s" % (name, pn)) 1214 if name != '': 1215 attempts.append("SRCREV_%s" % name) 1216 if pn: 1217 attempts.append("SRCREV:pn-%s" % pn) 1218 attempts.append("SRCREV") 1219 1220 for a in attempts: 1221 srcrev = d.getVar(a) 1222 if srcrev and srcrev != "INVALID": 1223 break 1224 1225 if 'rev' in ud.parm and 'tag' in ud.parm: 1226 raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url)) 1227 1228 if 'rev' in ud.parm or 'tag' in ud.parm: 1229 if 'rev' in ud.parm: 1230 parmrev = ud.parm['rev'] 1231 else: 1232 parmrev = ud.parm['tag'] 1233 if srcrev == "INVALID" or not srcrev: 1234 return parmrev 1235 if srcrev != parmrev: 1236 raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev)) 1237 return parmrev 1238 1239 if srcrev == "INVALID" or not srcrev: 1240 raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) 1241 if srcrev == "AUTOINC": 1242 d.setVar("__BBAUTOREV_ACTED_UPON", True) 1243 srcrev = ud.method.latest_revision(ud, d, name) 1244 1245 return srcrev 1246 1247def get_checksum_file_list(d): 1248 """ Get a list of files checksum in SRC_URI 1249 1250 Returns the resolved local paths of all local file entries in 1251 SRC_URI as a space-separated string 1252 """ 1253 fetch = Fetch([], d, cache = False, localonly = True) 1254 filelist = [] 1255 for u in fetch.urls: 1256 ud = fetch.ud[u] 1257 if ud and isinstance(ud.method, local.Local): 1258 found = False 1259 paths = ud.method.localfile_searchpaths(ud, d) 1260 for f in paths: 1261 pth = ud.decodedurl 1262 if os.path.exists(f): 1263 found = True 1264 filelist.append(f + ":" + str(os.path.exists(f))) 1265 if not found: 1266 bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found" 1267 "\nThe following paths were searched:" 1268 "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths))) 1269 1270 return " ".join(filelist) 1271 1272def get_file_checksums(filelist, pn, localdirsexclude): 1273 """Get a list of the checksums for a list of local files 1274 1275 Returns the checksums for a list of local files, caching the results as 1276 it proceeds 1277 1278 """ 1279 return _checksum_cache.get_checksums(filelist, pn, localdirsexclude) 1280 1281 1282class FetchData(object): 1283 """ 1284 A class which represents the fetcher state for a given URI. 1285 """ 1286 def __init__(self, url, d, localonly = False): 1287 # localpath is the location of a downloaded result. If not set, the file is local. 1288 self.donestamp = None 1289 self.needdonestamp = True 1290 self.localfile = "" 1291 self.localpath = None 1292 self.lockfile = None 1293 self.mirrortarballs = [] 1294 self.basename = None 1295 self.basepath = None 1296 (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url)) 1297 self.date = self.getSRCDate(d) 1298 self.url = url 1299 if not self.user and "user" in self.parm: 1300 self.user = self.parm["user"] 1301 if not self.pswd and "pswd" in self.parm: 1302 self.pswd = self.parm["pswd"] 1303 self.setup = False 1304 1305 def configure_checksum(checksum_id): 1306 if "name" in self.parm: 1307 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id) 1308 else: 1309 checksum_name = "%ssum" % checksum_id 1310 1311 setattr(self, "%s_name" % checksum_id, checksum_name) 1312 1313 if checksum_name in self.parm: 1314 checksum_expected = self.parm[checksum_name] 1315 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]: 1316 checksum_expected = None 1317 else: 1318 checksum_expected = d.getVarFlag("SRC_URI", checksum_name) 1319 1320 setattr(self, "%s_expected" % checksum_id, checksum_expected) 1321 1322 self.names = self.parm.get("name",'default').split(',') 1323 1324 self.method = None 1325 for m in methods: 1326 if m.supports(self, d): 1327 self.method = m 1328 break 1329 1330 if not self.method: 1331 raise NoMethodError(url) 1332 1333 if localonly and not isinstance(self.method, local.Local): 1334 raise NonLocalMethod() 1335 1336 if self.parm.get("proto", None) and "protocol" not in self.parm: 1337 logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN')) 1338 self.parm["protocol"] = self.parm.get("proto", None) 1339 1340 if hasattr(self.method, "urldata_init"): 1341 self.method.urldata_init(self, d) 1342 1343 for checksum_id in CHECKSUM_LIST: 1344 configure_checksum(checksum_id) 1345 1346 self.ignore_checksums = False 1347 1348 if "localpath" in self.parm: 1349 # if user sets localpath for file, use it instead. 1350 self.localpath = self.parm["localpath"] 1351 self.basename = os.path.basename(self.localpath) 1352 elif self.localfile: 1353 self.localpath = self.method.localpath(self, d) 1354 1355 dldir = d.getVar("DL_DIR") 1356 1357 if not self.needdonestamp: 1358 return 1359 1360 # Note: .done and .lock files should always be in DL_DIR whereas localpath may not be. 1361 if self.localpath and self.localpath.startswith(dldir): 1362 basepath = self.localpath 1363 elif self.localpath: 1364 basepath = dldir + os.sep + os.path.basename(self.localpath) 1365 elif self.basepath or self.basename: 1366 basepath = dldir + os.sep + (self.basepath or self.basename) 1367 else: 1368 bb.fatal("Can't determine lock path for url %s" % url) 1369 1370 self.donestamp = basepath + '.done' 1371 self.lockfile = basepath + '.lock' 1372 1373 def setup_revisions(self, d): 1374 self.revisions = {} 1375 for name in self.names: 1376 self.revisions[name] = srcrev_internal_helper(self, d, name) 1377 1378 # add compatibility code for non name specified case 1379 if len(self.names) == 1: 1380 self.revision = self.revisions[self.names[0]] 1381 1382 def setup_localpath(self, d): 1383 if not self.localpath: 1384 self.localpath = self.method.localpath(self, d) 1385 1386 def getSRCDate(self, d): 1387 """ 1388 Return the SRC Date for the component 1389 1390 d the bb.data module 1391 """ 1392 if "srcdate" in self.parm: 1393 return self.parm['srcdate'] 1394 1395 pn = d.getVar("PN") 1396 1397 if pn: 1398 return d.getVar("SRCDATE_%s" % pn) or d.getVar("SRCDATE") or d.getVar("DATE") 1399 1400 return d.getVar("SRCDATE") or d.getVar("DATE") 1401 1402class FetchMethod(object): 1403 """Base class for 'fetch'ing data""" 1404 1405 def __init__(self, urls=None): 1406 self.urls = [] 1407 1408 def supports(self, urldata, d): 1409 """ 1410 Check to see if this fetch class supports a given url. 1411 """ 1412 return 0 1413 1414 def localpath(self, urldata, d): 1415 """ 1416 Return the local filename of a given url assuming a successful fetch. 1417 Can also setup variables in urldata for use in go (saving code duplication 1418 and duplicate code execution) 1419 """ 1420 return os.path.join(d.getVar("DL_DIR"), urldata.localfile) 1421 1422 def supports_checksum(self, urldata): 1423 """ 1424 Is localpath something that can be represented by a checksum? 1425 """ 1426 1427 # We cannot compute checksums for None 1428 if urldata.localpath is None: 1429 return False 1430 # We cannot compute checksums for directories 1431 if os.path.isdir(urldata.localpath): 1432 return False 1433 return True 1434 1435 def recommends_checksum(self, urldata): 1436 """ 1437 Is the backend on where checksumming is recommended (should warnings 1438 be displayed if there is no checksum)? 1439 """ 1440 return False 1441 1442 def verify_donestamp(self, ud, d): 1443 """ 1444 Verify the donestamp file 1445 """ 1446 return verify_donestamp(ud, d) 1447 1448 def update_donestamp(self, ud, d): 1449 """ 1450 Update the donestamp file 1451 """ 1452 update_stamp(ud, d) 1453 1454 def _strip_leading_slashes(self, relpath): 1455 """ 1456 Remove leading slash as os.path.join can't cope 1457 """ 1458 while os.path.isabs(relpath): 1459 relpath = relpath[1:] 1460 return relpath 1461 1462 def setUrls(self, urls): 1463 self.__urls = urls 1464 1465 def getUrls(self): 1466 return self.__urls 1467 1468 urls = property(getUrls, setUrls, None, "Urls property") 1469 1470 def need_update(self, ud, d): 1471 """ 1472 Force a fetch, even if localpath exists? 1473 """ 1474 if os.path.exists(ud.localpath): 1475 return False 1476 return True 1477 1478 def supports_srcrev(self): 1479 """ 1480 The fetcher supports auto source revisions (SRCREV) 1481 """ 1482 return False 1483 1484 def download(self, urldata, d): 1485 """ 1486 Fetch urls 1487 Assumes localpath was called first 1488 """ 1489 raise NoMethodError(urldata.url) 1490 1491 def unpack(self, urldata, rootdir, data): 1492 iterate = False 1493 file = urldata.localpath 1494 1495 try: 1496 unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True) 1497 except ValueError as exc: 1498 bb.fatal("Invalid value for 'unpack' parameter for %s: %s" % 1499 (file, urldata.parm.get('unpack'))) 1500 1501 base, ext = os.path.splitext(file) 1502 if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']: 1503 efile = os.path.join(rootdir, os.path.basename(base)) 1504 else: 1505 efile = file 1506 cmd = None 1507 1508 if unpack: 1509 tar_cmd = 'tar --extract --no-same-owner' 1510 if 'striplevel' in urldata.parm: 1511 tar_cmd += ' --strip-components=%s' % urldata.parm['striplevel'] 1512 if file.endswith('.tar'): 1513 cmd = '%s -f %s' % (tar_cmd, file) 1514 elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 1515 cmd = '%s -z -f %s' % (tar_cmd, file) 1516 elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): 1517 cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd) 1518 elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): 1519 cmd = 'gzip -dc %s > %s' % (file, efile) 1520 elif file.endswith('.bz2'): 1521 cmd = 'bzip2 -dc %s > %s' % (file, efile) 1522 elif file.endswith('.txz') or file.endswith('.tar.xz'): 1523 cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd) 1524 elif file.endswith('.xz'): 1525 cmd = 'xz -dc %s > %s' % (file, efile) 1526 elif file.endswith('.tar.lz'): 1527 cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd) 1528 elif file.endswith('.lz'): 1529 cmd = 'lzip -dc %s > %s' % (file, efile) 1530 elif file.endswith('.tar.7z'): 1531 cmd = '7z x -so %s | %s -f -' % (file, tar_cmd) 1532 elif file.endswith('.7z'): 1533 cmd = '7za x -y %s 1>/dev/null' % file 1534 elif file.endswith('.tzst') or file.endswith('.tar.zst'): 1535 cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd) 1536 elif file.endswith('.zst'): 1537 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) 1538 elif file.endswith('.zip') or file.endswith('.jar'): 1539 try: 1540 dos = bb.utils.to_boolean(urldata.parm.get('dos'), False) 1541 except ValueError as exc: 1542 bb.fatal("Invalid value for 'dos' parameter for %s: %s" % 1543 (file, urldata.parm.get('dos'))) 1544 cmd = 'unzip -q -o' 1545 if dos: 1546 cmd = '%s -a' % cmd 1547 cmd = "%s '%s'" % (cmd, file) 1548 elif file.endswith('.rpm') or file.endswith('.srpm'): 1549 if 'extract' in urldata.parm: 1550 unpack_file = urldata.parm.get('extract') 1551 cmd = 'rpm2cpio.sh %s | cpio -id %s' % (file, unpack_file) 1552 iterate = True 1553 iterate_file = unpack_file 1554 else: 1555 cmd = 'rpm2cpio.sh %s | cpio -id' % (file) 1556 elif file.endswith('.deb') or file.endswith('.ipk'): 1557 output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup) 1558 datafile = None 1559 if output: 1560 for line in output.decode().splitlines(): 1561 if line.startswith('data.tar.'): 1562 datafile = line 1563 break 1564 else: 1565 raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url) 1566 else: 1567 raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) 1568 cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile) 1569 1570 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd 1571 if 'subdir' in urldata.parm: 1572 subdir = urldata.parm.get('subdir') 1573 if os.path.isabs(subdir): 1574 if not os.path.realpath(subdir).startswith(os.path.realpath(rootdir)): 1575 raise UnpackError("subdir argument isn't a subdirectory of unpack root %s" % rootdir, urldata.url) 1576 unpackdir = subdir 1577 else: 1578 unpackdir = os.path.join(rootdir, subdir) 1579 bb.utils.mkdirhier(unpackdir) 1580 else: 1581 unpackdir = rootdir 1582 1583 if not unpack or not cmd: 1584 urldata.unpack_tracer.unpack("file-copy", unpackdir) 1585 # If file == dest, then avoid any copies, as we already put the file into dest! 1586 dest = os.path.join(unpackdir, os.path.basename(file)) 1587 if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): 1588 destdir = '.' 1589 # For file:// entries all intermediate dirs in path must be created at destination 1590 if urldata.type == "file": 1591 # Trailing '/' does a copying to wrong place 1592 urlpath = urldata.path.rstrip('/') 1593 # Want files places relative to cwd so no leading '/' 1594 urlpath = urlpath.lstrip('/') 1595 if urlpath.find("/") != -1: 1596 destdir = urlpath.rsplit("/", 1)[0] + '/' 1597 bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) 1598 cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) 1599 else: 1600 urldata.unpack_tracer.unpack("archive-extract", unpackdir) 1601 1602 if not cmd: 1603 return 1604 1605 path = data.getVar('PATH') 1606 if path: 1607 cmd = "PATH=\"%s\" %s" % (path, cmd) 1608 bb.note("Unpacking %s to %s/" % (file, unpackdir)) 1609 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=unpackdir) 1610 1611 if ret != 0: 1612 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url) 1613 1614 if iterate is True: 1615 iterate_urldata = urldata 1616 iterate_urldata.localpath = "%s/%s" % (rootdir, iterate_file) 1617 self.unpack(urldata, rootdir, data) 1618 1619 return 1620 1621 def clean(self, urldata, d): 1622 """ 1623 Clean any existing full or partial download 1624 """ 1625 bb.utils.remove(urldata.localpath) 1626 1627 def try_premirror(self, urldata, d): 1628 """ 1629 Should premirrors be used? 1630 """ 1631 return True 1632 1633 def try_mirrors(self, fetch, urldata, d, mirrors, check=False): 1634 """ 1635 Try to use a mirror 1636 """ 1637 return bool(try_mirrors(fetch, d, urldata, mirrors, check)) 1638 1639 def checkstatus(self, fetch, urldata, d): 1640 """ 1641 Check the status of a URL 1642 Assumes localpath was called first 1643 """ 1644 logger.info("URL %s could not be checked for status since no method exists.", urldata.url) 1645 return True 1646 1647 def latest_revision(self, ud, d, name): 1648 """ 1649 Look in the cache for the latest revision, if not present ask the SCM. 1650 """ 1651 if not hasattr(self, "_latest_revision"): 1652 raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) 1653 1654 revs = bb.persist_data.persist('BB_URI_HEADREVS', d) 1655 key = self.generate_revision_key(ud, d, name) 1656 try: 1657 return revs[key] 1658 except KeyError: 1659 revs[key] = rev = self._latest_revision(ud, d, name) 1660 return rev 1661 1662 def sortable_revision(self, ud, d, name): 1663 latest_rev = self._build_revision(ud, d, name) 1664 return True, str(latest_rev) 1665 1666 def generate_revision_key(self, ud, d, name): 1667 return self._revision_key(ud, d, name) 1668 1669 def latest_versionstring(self, ud, d): 1670 """ 1671 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH" 1672 by searching through the tags output of ls-remote, comparing 1673 versions and returning the highest match as a (version, revision) pair. 1674 """ 1675 return ('', '') 1676 1677 def done(self, ud, d): 1678 """ 1679 Is the download done ? 1680 """ 1681 if os.path.exists(ud.localpath): 1682 return True 1683 return False 1684 1685 def implicit_urldata(self, ud, d): 1686 """ 1687 Get a list of FetchData objects for any implicit URLs that will also 1688 be downloaded when we fetch the given URL. 1689 """ 1690 return [] 1691 1692 1693class DummyUnpackTracer(object): 1694 """ 1695 Abstract API definition for a class that traces unpacked source files back 1696 to their respective upstream SRC_URI entries, for software composition 1697 analysis, license compliance and detailed SBOM generation purposes. 1698 User may load their own unpack tracer class (instead of the dummy 1699 one) by setting the BB_UNPACK_TRACER_CLASS config parameter. 1700 """ 1701 def start(self, unpackdir, urldata_dict, d): 1702 """ 1703 Start tracing the core Fetch.unpack process, using an index to map 1704 unpacked files to each SRC_URI entry. 1705 This method is called by Fetch.unpack and it may receive nested calls by 1706 gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit 1707 URLs and by recursively calling Fetch.unpack from new (nested) Fetch 1708 instances. 1709 """ 1710 return 1711 def start_url(self, url): 1712 """Start tracing url unpack process. 1713 This method is called by Fetch.unpack before the fetcher-specific unpack 1714 method starts, and it may receive nested calls by gitsm and npmsw 1715 fetchers. 1716 """ 1717 return 1718 def unpack(self, unpack_type, destdir): 1719 """ 1720 Set unpack_type and destdir for current url. 1721 This method is called by the fetcher-specific unpack method after url 1722 tracing started. 1723 """ 1724 return 1725 def finish_url(self, url): 1726 """Finish tracing url unpack process and update the file index. 1727 This method is called by Fetch.unpack after the fetcher-specific unpack 1728 method finished its job, and it may receive nested calls by gitsm 1729 and npmsw fetchers. 1730 """ 1731 return 1732 def complete(self): 1733 """ 1734 Finish tracing the Fetch.unpack process, and check if all nested 1735 Fecth.unpack calls (if any) have been completed; if so, save collected 1736 metadata. 1737 """ 1738 return 1739 1740 1741class Fetch(object): 1742 def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): 1743 if localonly and cache: 1744 raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") 1745 1746 if not urls: 1747 urls = d.getVar("SRC_URI").split() 1748 self.urls = urls 1749 self.d = d 1750 self.ud = {} 1751 self.connection_cache = connection_cache 1752 1753 fn = d.getVar('FILE') 1754 mc = d.getVar('__BBMULTICONFIG') or "" 1755 key = None 1756 if cache and fn: 1757 key = mc + fn + str(id(d)) 1758 if key in urldata_cache: 1759 self.ud = urldata_cache[key] 1760 1761 # the unpack_tracer object needs to be made available to possible nested 1762 # Fetch instances (when those are created by gitsm and npmsw fetchers) 1763 # so we set it as a global variable 1764 global unpack_tracer 1765 try: 1766 unpack_tracer 1767 except NameError: 1768 class_path = d.getVar("BB_UNPACK_TRACER_CLASS") 1769 if class_path: 1770 # use user-defined unpack tracer class 1771 import importlib 1772 module_name, _, class_name = class_path.rpartition(".") 1773 module = importlib.import_module(module_name) 1774 class_ = getattr(module, class_name) 1775 unpack_tracer = class_() 1776 else: 1777 # fall back to the dummy/abstract class 1778 unpack_tracer = DummyUnpackTracer() 1779 1780 for url in urls: 1781 if url not in self.ud: 1782 try: 1783 self.ud[url] = FetchData(url, d, localonly) 1784 self.ud[url].unpack_tracer = unpack_tracer 1785 except NonLocalMethod: 1786 if localonly: 1787 self.ud[url] = None 1788 pass 1789 1790 if key: 1791 urldata_cache[key] = self.ud 1792 1793 def localpath(self, url): 1794 if url not in self.urls: 1795 self.ud[url] = FetchData(url, self.d) 1796 1797 self.ud[url].setup_localpath(self.d) 1798 return self.d.expand(self.ud[url].localpath) 1799 1800 def localpaths(self): 1801 """ 1802 Return a list of the local filenames, assuming successful fetch 1803 """ 1804 local = [] 1805 1806 for u in self.urls: 1807 ud = self.ud[u] 1808 ud.setup_localpath(self.d) 1809 local.append(ud.localpath) 1810 1811 return local 1812 1813 def download(self, urls=None): 1814 """ 1815 Fetch all urls 1816 """ 1817 if not urls: 1818 urls = self.urls 1819 1820 network = self.d.getVar("BB_NO_NETWORK") 1821 premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) 1822 1823 checksum_missing_messages = [] 1824 for u in urls: 1825 ud = self.ud[u] 1826 ud.setup_localpath(self.d) 1827 m = ud.method 1828 done = False 1829 1830 if ud.lockfile: 1831 lf = bb.utils.lockfile(ud.lockfile) 1832 1833 try: 1834 self.d.setVar("BB_NO_NETWORK", network) 1835 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): 1836 done = True 1837 elif m.try_premirror(ud, self.d): 1838 logger.debug("Trying PREMIRRORS") 1839 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) 1840 done = m.try_mirrors(self, ud, self.d, mirrors) 1841 if done: 1842 try: 1843 # early checksum verification so that if the checksum of the premirror 1844 # contents mismatch the fetcher can still try upstream and mirrors 1845 m.update_donestamp(ud, self.d) 1846 except ChecksumError as e: 1847 logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u) 1848 logger.debug(str(e)) 1849 done = False 1850 1851 if premirroronly: 1852 self.d.setVar("BB_NO_NETWORK", "1") 1853 1854 firsterr = None 1855 verified_stamp = False 1856 if done: 1857 verified_stamp = m.verify_donestamp(ud, self.d) 1858 if not done and (not verified_stamp or m.need_update(ud, self.d)): 1859 try: 1860 if not trusted_network(self.d, ud.url): 1861 raise UntrustedUrl(ud.url) 1862 logger.debug("Trying Upstream") 1863 m.download(ud, self.d) 1864 if hasattr(m, "build_mirror_data"): 1865 m.build_mirror_data(ud, self.d) 1866 done = True 1867 # early checksum verify, so that if checksum mismatched, 1868 # fetcher still have chance to fetch from mirror 1869 m.update_donestamp(ud, self.d) 1870 1871 except bb.fetch2.NetworkAccess: 1872 raise 1873 1874 except BBFetchException as e: 1875 if isinstance(e, ChecksumError): 1876 logger.warning("Checksum failure encountered with download of %s - will attempt other sources if available" % u) 1877 logger.debug(str(e)) 1878 if os.path.exists(ud.localpath): 1879 rename_bad_checksum(ud, e.checksum) 1880 elif isinstance(e, NoChecksumError): 1881 raise 1882 else: 1883 logger.warning('Failed to fetch URL %s, attempting MIRRORS if available' % u) 1884 logger.debug(str(e)) 1885 firsterr = e 1886 # Remove any incomplete fetch 1887 if not verified_stamp: 1888 m.clean(ud, self.d) 1889 logger.debug("Trying MIRRORS") 1890 mirrors = mirror_from_string(self.d.getVar('MIRRORS')) 1891 done = m.try_mirrors(self, ud, self.d, mirrors) 1892 1893 if not done or not m.done(ud, self.d): 1894 if firsterr: 1895 logger.error(str(firsterr)) 1896 raise FetchError("Unable to fetch URL from any source.", u) 1897 1898 m.update_donestamp(ud, self.d) 1899 1900 except IOError as e: 1901 if e.errno in [errno.ESTALE]: 1902 logger.error("Stale Error Observed %s." % u) 1903 raise ChecksumError("Stale Error Detected") 1904 1905 except BBFetchException as e: 1906 if isinstance(e, NoChecksumError): 1907 (message, _) = e.args 1908 checksum_missing_messages.append(message) 1909 continue 1910 elif isinstance(e, ChecksumError): 1911 logger.error("Checksum failure fetching %s" % u) 1912 raise 1913 1914 finally: 1915 if ud.lockfile: 1916 bb.utils.unlockfile(lf) 1917 if checksum_missing_messages: 1918 logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages)) 1919 raise BBFetchException("There was some missing checksums in the recipe") 1920 1921 def checkstatus(self, urls=None): 1922 """ 1923 Check all URLs exist upstream. 1924 1925 Returns None if the URLs exist, raises FetchError if the check wasn't 1926 successful but there wasn't an error (such as file not found), and 1927 raises other exceptions in error cases. 1928 """ 1929 1930 if not urls: 1931 urls = self.urls 1932 1933 for u in urls: 1934 ud = self.ud[u] 1935 ud.setup_localpath(self.d) 1936 m = ud.method 1937 logger.debug("Testing URL %s", u) 1938 # First try checking uri, u, from PREMIRRORS 1939 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) 1940 ret = m.try_mirrors(self, ud, self.d, mirrors, True) 1941 if not ret: 1942 # Next try checking from the original uri, u 1943 ret = m.checkstatus(self, ud, self.d) 1944 if not ret: 1945 # Finally, try checking uri, u, from MIRRORS 1946 mirrors = mirror_from_string(self.d.getVar('MIRRORS')) 1947 ret = m.try_mirrors(self, ud, self.d, mirrors, True) 1948 1949 if not ret: 1950 raise FetchError("URL doesn't work", u) 1951 1952 def unpack(self, root, urls=None): 1953 """ 1954 Unpack urls to root 1955 """ 1956 1957 if not urls: 1958 urls = self.urls 1959 1960 unpack_tracer.start(root, self.ud, self.d) 1961 1962 for u in urls: 1963 ud = self.ud[u] 1964 ud.setup_localpath(self.d) 1965 1966 if ud.lockfile: 1967 lf = bb.utils.lockfile(ud.lockfile) 1968 1969 unpack_tracer.start_url(u) 1970 ud.method.unpack(ud, root, self.d) 1971 unpack_tracer.finish_url(u) 1972 1973 if ud.lockfile: 1974 bb.utils.unlockfile(lf) 1975 1976 unpack_tracer.complete() 1977 1978 def clean(self, urls=None): 1979 """ 1980 Clean files that the fetcher gets or places 1981 """ 1982 1983 if not urls: 1984 urls = self.urls 1985 1986 for url in urls: 1987 if url not in self.ud: 1988 self.ud[url] = FetchData(url, self.d) 1989 ud = self.ud[url] 1990 ud.setup_localpath(self.d) 1991 1992 if not ud.localfile and ud.localpath is None: 1993 continue 1994 1995 if ud.lockfile: 1996 lf = bb.utils.lockfile(ud.lockfile) 1997 1998 ud.method.clean(ud, self.d) 1999 if ud.donestamp: 2000 bb.utils.remove(ud.donestamp) 2001 2002 if ud.lockfile: 2003 bb.utils.unlockfile(lf) 2004 2005 def expanded_urldata(self, urls=None): 2006 """ 2007 Get an expanded list of FetchData objects covering both the given 2008 URLS and any additional implicit URLs that are added automatically by 2009 the appropriate FetchMethod. 2010 """ 2011 2012 if not urls: 2013 urls = self.urls 2014 2015 urldata = [] 2016 for url in urls: 2017 ud = self.ud[url] 2018 urldata.append(ud) 2019 urldata += ud.method.implicit_urldata(ud, self.d) 2020 2021 return urldata 2022 2023class FetchConnectionCache(object): 2024 """ 2025 A class which represents an container for socket connections. 2026 """ 2027 def __init__(self): 2028 self.cache = {} 2029 2030 def get_connection_name(self, host, port): 2031 return host + ':' + str(port) 2032 2033 def add_connection(self, host, port, connection): 2034 cn = self.get_connection_name(host, port) 2035 2036 if cn not in self.cache: 2037 self.cache[cn] = connection 2038 2039 def get_connection(self, host, port): 2040 connection = None 2041 2042 cn = self.get_connection_name(host, port) 2043 if cn in self.cache: 2044 connection = self.cache[cn] 2045 2046 return connection 2047 2048 def remove_connection(self, host, port): 2049 cn = self.get_connection_name(host, port) 2050 if cn in self.cache: 2051 self.cache[cn].close() 2052 del self.cache[cn] 2053 2054 def close_connections(self): 2055 for cn in list(self.cache.keys()): 2056 self.cache[cn].close() 2057 del self.cache[cn] 2058 2059from . import cvs 2060from . import git 2061from . import gitsm 2062from . import gitannex 2063from . import local 2064from . import svn 2065from . import wget 2066from . import ssh 2067from . import sftp 2068from . import s3 2069from . import perforce 2070from . import bzr 2071from . import hg 2072from . import osc 2073from . import repo 2074from . import clearcase 2075from . import npm 2076from . import npmsw 2077from . import az 2078from . import crate 2079from . import gcp 2080 2081methods.append(local.Local()) 2082methods.append(wget.Wget()) 2083methods.append(svn.Svn()) 2084methods.append(git.Git()) 2085methods.append(gitsm.GitSM()) 2086methods.append(gitannex.GitANNEX()) 2087methods.append(cvs.Cvs()) 2088methods.append(ssh.SSH()) 2089methods.append(sftp.SFTP()) 2090methods.append(s3.S3()) 2091methods.append(perforce.Perforce()) 2092methods.append(bzr.Bzr()) 2093methods.append(hg.Hg()) 2094methods.append(osc.Osc()) 2095methods.append(repo.Repo()) 2096methods.append(clearcase.ClearCase()) 2097methods.append(npm.Npm()) 2098methods.append(npmsw.NpmShrinkWrap()) 2099methods.append(az.Az()) 2100methods.append(crate.Crate()) 2101methods.append(gcp.GCP()) 2102