1""" 2BitBake 'Fetch' implementations 3 4Classes for obtaining upstream sources for the 5BitBake build tools. 6""" 7 8# Copyright (C) 2003, 2004 Chris Larson 9# Copyright (C) 2012 Intel Corporation 10# 11# SPDX-License-Identifier: GPL-2.0-only 12# 13# Based on functions from the base bb module, Copyright 2003 Holger Schurig 14 15import os, re 16import signal 17import logging 18import urllib.request, urllib.parse, urllib.error 19if 'git' not in urllib.parse.uses_netloc: 20 urllib.parse.uses_netloc.append('git') 21import operator 22import collections 23import subprocess 24import pickle 25import errno 26import bb.utils 27import bb.checksum 28import bb.process 29import bb.event 30 31__version__ = "2" 32_checksum_cache = bb.checksum.FileChecksumCache() 33_revisions_cache = bb.checksum.RevisionsCache() 34 35logger = logging.getLogger("BitBake.Fetcher") 36 37CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ] 38SHOWN_CHECKSUM_LIST = ["sha256"] 39 40class BBFetchException(Exception): 41 """Class all fetch exceptions inherit from""" 42 def __init__(self, message): 43 self.msg = message 44 Exception.__init__(self, message) 45 46 def __str__(self): 47 return self.msg 48 49class UntrustedUrl(BBFetchException): 50 """Exception raised when encountering a host not listed in BB_ALLOWED_NETWORKS""" 51 def __init__(self, url, message=''): 52 if message: 53 msg = message 54 else: 55 msg = "The URL: '%s' is not trusted and cannot be used" % url 56 self.url = url 57 BBFetchException.__init__(self, msg) 58 self.args = (url,) 59 60class MalformedUrl(BBFetchException): 61 """Exception raised when encountering an invalid url""" 62 def __init__(self, url, message=''): 63 if message: 64 msg = message 65 else: 66 msg = "The URL: '%s' is invalid and cannot be interpreted" % url 67 self.url = url 68 BBFetchException.__init__(self, msg) 69 self.args = (url,) 70 71class FetchError(BBFetchException): 72 """General fetcher exception when something happens incorrectly""" 73 def __init__(self, message, url = None): 74 if url: 75 msg = "Fetcher failure for URL: '%s'. %s" % (url, message) 76 else: 77 msg = "Fetcher failure: %s" % message 78 self.url = url 79 BBFetchException.__init__(self, msg) 80 self.args = (message, url) 81 82class ChecksumError(FetchError): 83 """Exception when mismatched checksum encountered""" 84 def __init__(self, message, url = None, checksum = None): 85 self.checksum = checksum 86 FetchError.__init__(self, message, url) 87 88class NoChecksumError(FetchError): 89 """Exception when no checksum is specified, but BB_STRICT_CHECKSUM is set""" 90 91class UnpackError(BBFetchException): 92 """General fetcher exception when something happens incorrectly when unpacking""" 93 def __init__(self, message, url): 94 msg = "Unpack failure for URL: '%s'. %s" % (url, message) 95 self.url = url 96 BBFetchException.__init__(self, msg) 97 self.args = (message, url) 98 99class NoMethodError(BBFetchException): 100 """Exception raised when there is no method to obtain a supplied url or set of urls""" 101 def __init__(self, url): 102 msg = "Could not find a fetcher which supports the URL: '%s'" % url 103 self.url = url 104 BBFetchException.__init__(self, msg) 105 self.args = (url,) 106 107class MissingParameterError(BBFetchException): 108 """Exception raised when a fetch method is missing a critical parameter in the url""" 109 def __init__(self, missing, url): 110 msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing) 111 self.url = url 112 self.missing = missing 113 BBFetchException.__init__(self, msg) 114 self.args = (missing, url) 115 116class ParameterError(BBFetchException): 117 """Exception raised when a url cannot be processed due to invalid parameters.""" 118 def __init__(self, message, url): 119 msg = "URL: '%s' has invalid parameters. %s" % (url, message) 120 self.url = url 121 BBFetchException.__init__(self, msg) 122 self.args = (message, url) 123 124class NetworkAccess(BBFetchException): 125 """Exception raised when network access is disabled but it is required.""" 126 def __init__(self, url, cmd): 127 msg = "Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command %s (for url %s)" % (cmd, url) 128 self.url = url 129 self.cmd = cmd 130 BBFetchException.__init__(self, msg) 131 self.args = (url, cmd) 132 133class NonLocalMethod(Exception): 134 def __init__(self): 135 Exception.__init__(self) 136 137class MissingChecksumEvent(bb.event.Event): 138 def __init__(self, url, **checksums): 139 self.url = url 140 self.checksums = checksums 141 bb.event.Event.__init__(self) 142 143 144class URI(object): 145 """ 146 A class representing a generic URI, with methods for 147 accessing the URI components, and stringifies to the 148 URI. 149 150 It is constructed by calling it with a URI, or setting 151 the attributes manually: 152 153 uri = URI("http://example.com/") 154 155 uri = URI() 156 uri.scheme = 'http' 157 uri.hostname = 'example.com' 158 uri.path = '/' 159 160 It has the following attributes: 161 162 * scheme (read/write) 163 * userinfo (authentication information) (read/write) 164 * username (read/write) 165 * password (read/write) 166 167 Note, password is deprecated as of RFC 3986. 168 169 * hostname (read/write) 170 * port (read/write) 171 * hostport (read only) 172 "hostname:port", if both are set, otherwise just "hostname" 173 * path (read/write) 174 * path_quoted (read/write) 175 A URI quoted version of path 176 * params (dict) (read/write) 177 * query (dict) (read/write) 178 * relative (bool) (read only) 179 True if this is a "relative URI", (e.g. file:foo.diff) 180 181 It stringifies to the URI itself. 182 183 Some notes about relative URIs: while it's specified that 184 a URI beginning with <scheme>:// should either be directly 185 followed by a hostname or a /, the old URI handling of the 186 fetch2 library did not conform to this. Therefore, this URI 187 class has some kludges to make sure that URIs are parsed in 188 a way comforming to bitbake's current usage. This URI class 189 supports the following: 190 191 file:relative/path.diff (IETF compliant) 192 git:relative/path.git (IETF compliant) 193 git:///absolute/path.git (IETF compliant) 194 file:///absolute/path.diff (IETF compliant) 195 196 file://relative/path.diff (not IETF compliant) 197 198 But it does not support the following: 199 200 file://hostname/absolute/path.diff (would be IETF compliant) 201 202 Note that the last case only applies to a list of 203 explicitly allowed schemes (currently only file://), that requires 204 its URIs to not have a network location. 205 """ 206 207 _relative_schemes = ['file', 'git'] 208 _netloc_forbidden = ['file'] 209 210 def __init__(self, uri=None): 211 self.scheme = '' 212 self.userinfo = '' 213 self.hostname = '' 214 self.port = None 215 self._path = '' 216 self.params = {} 217 self.query = {} 218 self.relative = False 219 220 if not uri: 221 return 222 223 # We hijack the URL parameters, since the way bitbake uses 224 # them are not quite RFC compliant. 225 uri, param_str = (uri.split(";", 1) + [None])[:2] 226 227 urlp = urllib.parse.urlparse(uri) 228 self.scheme = urlp.scheme 229 230 reparse = 0 231 232 # Coerce urlparse to make URI scheme use netloc 233 if not self.scheme in urllib.parse.uses_netloc: 234 urllib.parse.uses_params.append(self.scheme) 235 reparse = 1 236 237 # Make urlparse happy(/ier) by converting local resources 238 # to RFC compliant URL format. E.g.: 239 # file://foo.diff -> file:foo.diff 240 if urlp.scheme in self._netloc_forbidden: 241 uri = re.sub(r"(?<=:)//(?!/)", "", uri, count=1) 242 reparse = 1 243 244 if reparse: 245 urlp = urllib.parse.urlparse(uri) 246 247 # Identify if the URI is relative or not 248 if urlp.scheme in self._relative_schemes and \ 249 re.compile(r"^\w+:(?!//)").match(uri): 250 self.relative = True 251 252 if not self.relative: 253 self.hostname = urlp.hostname or '' 254 self.port = urlp.port 255 256 self.userinfo += urlp.username or '' 257 258 if urlp.password: 259 self.userinfo += ':%s' % urlp.password 260 261 self.path = urllib.parse.unquote(urlp.path) 262 263 if param_str: 264 self.params = self._param_str_split(param_str, ";") 265 if urlp.query: 266 self.query = self._param_str_split(urlp.query, "&") 267 268 def __str__(self): 269 userinfo = self.userinfo 270 if userinfo: 271 userinfo += '@' 272 273 return "%s:%s%s%s%s%s%s" % ( 274 self.scheme, 275 '' if self.relative else '//', 276 userinfo, 277 self.hostport, 278 self.path_quoted, 279 self._query_str(), 280 self._param_str()) 281 282 def _param_str(self): 283 return ( 284 ''.join([';', self._param_str_join(self.params, ";")]) 285 if self.params else '') 286 287 def _query_str(self): 288 return ( 289 ''.join(['?', self._param_str_join(self.query, "&")]) 290 if self.query else '') 291 292 def _param_str_split(self, string, elmdelim, kvdelim="="): 293 ret = collections.OrderedDict() 294 for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]: 295 ret[k] = v 296 return ret 297 298 def _param_str_join(self, dict_, elmdelim, kvdelim="="): 299 return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()]) 300 301 @property 302 def hostport(self): 303 if not self.port: 304 return self.hostname 305 return "%s:%d" % (self.hostname, self.port) 306 307 @property 308 def path_quoted(self): 309 return urllib.parse.quote(self.path) 310 311 @path_quoted.setter 312 def path_quoted(self, path): 313 self.path = urllib.parse.unquote(path) 314 315 @property 316 def path(self): 317 return self._path 318 319 @path.setter 320 def path(self, path): 321 self._path = path 322 323 if not path or re.compile("^/").match(path): 324 self.relative = False 325 else: 326 self.relative = True 327 328 @property 329 def username(self): 330 if self.userinfo: 331 return (self.userinfo.split(":", 1))[0] 332 return '' 333 334 @username.setter 335 def username(self, username): 336 password = self.password 337 self.userinfo = username 338 if password: 339 self.userinfo += ":%s" % password 340 341 @property 342 def password(self): 343 if self.userinfo and ":" in self.userinfo: 344 return (self.userinfo.split(":", 1))[1] 345 return '' 346 347 @password.setter 348 def password(self, password): 349 self.userinfo = "%s:%s" % (self.username, password) 350 351def decodeurl(url): 352 """Decodes an URL into the tokens (scheme, network location, path, 353 user, password, parameters). 354 """ 355 356 m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url) 357 if not m: 358 raise MalformedUrl(url) 359 360 type = m.group('type') 361 location = m.group('location') 362 if not location: 363 raise MalformedUrl(url) 364 user = m.group('user') 365 parm = m.group('parm') 366 367 locidx = location.find('/') 368 if locidx != -1 and type.lower() != 'file': 369 host = location[:locidx] 370 path = location[locidx:] 371 elif type.lower() == 'file': 372 host = "" 373 path = location 374 else: 375 host = location 376 path = "/" 377 if user: 378 m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user) 379 if m: 380 user = m.group('user') 381 pswd = m.group('pswd') 382 else: 383 user = '' 384 pswd = '' 385 386 p = collections.OrderedDict() 387 if parm: 388 for s in parm.split(';'): 389 if s: 390 if not '=' in s: 391 raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s)) 392 s1, s2 = s.split('=', 1) 393 p[s1] = s2 394 395 return type, host, urllib.parse.unquote(path), user, pswd, p 396 397def encodeurl(decoded): 398 """Encodes a URL from tokens (scheme, network location, path, 399 user, password, parameters). 400 """ 401 402 type, host, path, user, pswd, p = decoded 403 404 if not type: 405 raise MissingParameterError('type', "encoded from the data %s" % str(decoded)) 406 url = ['%s://' % type] 407 if user and type != "file": 408 url.append("%s" % user) 409 if pswd: 410 url.append(":%s" % pswd) 411 url.append("@") 412 if host and type != "file": 413 url.append("%s" % host) 414 if path: 415 # Standardise path to ensure comparisons work 416 while '//' in path: 417 path = path.replace("//", "/") 418 url.append("%s" % urllib.parse.quote(path)) 419 if p: 420 for parm in p: 421 url.append(";%s=%s" % (parm, p[parm])) 422 423 return "".join(url) 424 425def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): 426 if not ud.url or not uri_find or not uri_replace: 427 logger.error("uri_replace: passed an undefined value, not replacing") 428 return None 429 uri_decoded = list(decodeurl(ud.url)) 430 uri_find_decoded = list(decodeurl(uri_find)) 431 uri_replace_decoded = list(decodeurl(uri_replace)) 432 logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded)) 433 result_decoded = ['', '', '', '', '', {}] 434 # 0 - type, 1 - host, 2 - path, 3 - user, 4- pswd, 5 - params 435 for loc, i in enumerate(uri_find_decoded): 436 result_decoded[loc] = uri_decoded[loc] 437 regexp = i 438 if loc == 0 and regexp and not regexp.endswith("$"): 439 # Leaving the type unanchored can mean "https" matching "file" can become "files" 440 # which is clearly undesirable. 441 regexp += "$" 442 if loc == 5: 443 # Handle URL parameters 444 if i: 445 # Any specified URL parameters must match 446 for k in uri_find_decoded[loc]: 447 if uri_decoded[loc][k] != uri_find_decoded[loc][k]: 448 return None 449 # Overwrite any specified replacement parameters 450 for k in uri_replace_decoded[loc]: 451 for l in replacements: 452 uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l]) 453 result_decoded[loc][k] = uri_replace_decoded[loc][k] 454 elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]: 455 # User/password in the replacement is just a straight replacement 456 result_decoded[loc] = uri_replace_decoded[loc] 457 elif (re.match(regexp, uri_decoded[loc])): 458 if not uri_replace_decoded[loc]: 459 result_decoded[loc] = "" 460 else: 461 for k in replacements: 462 uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k]) 463 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc])) 464 result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1) 465 if loc == 2: 466 # Handle path manipulations 467 basename = None 468 if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball: 469 # If the source and destination url types differ, must be a mirrortarball mapping 470 basename = os.path.basename(mirrortarball) 471 # Kill parameters, they make no sense for mirror tarballs 472 uri_decoded[5] = {} 473 uri_find_decoded[5] = {} 474 elif ud.localpath and ud.method.supports_checksum(ud): 475 basename = os.path.basename(ud.localpath) 476 if basename: 477 uri_basename = os.path.basename(uri_decoded[loc]) 478 # Prefix with a slash as a sentinel in case 479 # result_decoded[loc] does not contain one. 480 path = "/" + result_decoded[loc] 481 if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename): 482 result_decoded[loc] = path[1:-len(uri_basename)] + basename 483 elif not path.endswith("/" + basename): 484 result_decoded[loc] = os.path.join(path[1:], basename) 485 else: 486 return None 487 result = encodeurl(result_decoded) 488 if result == ud.url: 489 return None 490 logger.debug2("For url %s returning %s" % (ud.url, result)) 491 return result 492 493methods = [] 494urldata_cache = {} 495saved_headrevs = {} 496 497def fetcher_init(d, servercontext=True): 498 """ 499 Called to initialize the fetchers once the configuration data is known. 500 Calls before this must not hit the cache. 501 """ 502 503 _checksum_cache.init_cache(d.getVar("BB_CACHEDIR")) 504 _revisions_cache.init_cache(d.getVar("BB_CACHEDIR")) 505 506 if not servercontext: 507 return 508 509 try: 510 # fetcher_init is called multiple times, so make sure we only save the 511 # revs the first time it is called. 512 if not bb.fetch2.saved_headrevs: 513 bb.fetch2.saved_headrevs = _revisions_cache.get_revs() 514 except: 515 pass 516 517 # When to drop SCM head revisions controlled by user policy 518 srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear" 519 if srcrev_policy == "cache": 520 logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) 521 elif srcrev_policy == "clear": 522 logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) 523 _revisions_cache.clear_cache() 524 else: 525 raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) 526 527 528 for m in methods: 529 if hasattr(m, "init"): 530 m.init(d) 531 532def fetcher_parse_save(): 533 _checksum_cache.save_extras() 534 _revisions_cache.save_extras() 535 536def fetcher_parse_done(): 537 _checksum_cache.save_merge() 538 _revisions_cache.save_merge() 539 540def fetcher_compare_revisions(d): 541 """ 542 Compare the revisions in the persistent cache with the saved values from 543 when bitbake was started and return true if they have changed. 544 """ 545 546 headrevs = _revisions_cache.get_revs() 547 return headrevs != bb.fetch2.saved_headrevs 548 549def mirror_from_string(data): 550 mirrors = (data or "").replace('\\n',' ').split() 551 # Split into pairs 552 if len(mirrors) % 2 != 0: 553 bb.warn('Invalid mirror data %s, should have paired members.' % data) 554 return list(zip(*[iter(mirrors)]*2)) 555 556def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True): 557 """ 558 verify the MD5 and SHA256 checksum for downloaded src 559 560 Raises a FetchError if one or both of the SRC_URI checksums do not match 561 the downloaded file, or if BB_STRICT_CHECKSUM is set and there are no 562 checksums specified. 563 564 Returns a dict of checksums that can be stored in a done stamp file and 565 passed in as precomputed parameter in a later call to avoid re-computing 566 the checksums from the file. This allows verifying the checksums of the 567 file against those in the recipe each time, rather than only after 568 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. 569 """ 570 if ud.ignore_checksums or not ud.method.supports_checksum(ud): 571 return {} 572 573 if localpath is None: 574 localpath = ud.localpath 575 576 def compute_checksum_info(checksum_id): 577 checksum_name = getattr(ud, "%s_name" % checksum_id) 578 579 if checksum_id in precomputed: 580 checksum_data = precomputed[checksum_id] 581 else: 582 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath) 583 584 checksum_expected = getattr(ud, "%s_expected" % checksum_id) 585 586 if checksum_expected == '': 587 checksum_expected = None 588 589 return { 590 "id": checksum_id, 591 "name": checksum_name, 592 "data": checksum_data, 593 "expected": checksum_expected 594 } 595 596 checksum_infos = [] 597 for checksum_id in CHECKSUM_LIST: 598 checksum_infos.append(compute_checksum_info(checksum_id)) 599 600 checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos} 601 checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos} 602 603 for ci in checksum_infos: 604 if ci["id"] in SHOWN_CHECKSUM_LIST: 605 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] 606 607 # If no checksum has been provided 608 if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): 609 messages = [] 610 strict = d.getVar("BB_STRICT_CHECKSUM") or "0" 611 612 # If strict checking enabled and neither sum defined, raise error 613 if strict == "1": 614 raise NoChecksumError("\n".join(checksum_lines)) 615 616 bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) 617 618 if strict == "ignore": 619 return checksum_dict 620 621 # Log missing sums so user can more easily add them 622 messages.append("Missing checksum for '%s', consider adding at " \ 623 "least one to the recipe:" % ud.localpath) 624 messages.extend(checksum_lines) 625 logger.warning("\n".join(messages)) 626 627 # We want to alert the user if a checksum is defined in the recipe but 628 # it does not match. 629 messages = [] 630 messages.append("Checksum mismatch!") 631 bad_checksum = None 632 633 for ci in checksum_infos: 634 if ci["expected"] and ci["expected"] != ci["data"]: 635 messages.append("File: '%s' has %s checksum '%s' when '%s' was " \ 636 "expected" % (localpath, ci["id"], ci["data"], ci["expected"])) 637 bad_checksum = ci["data"] 638 639 if bad_checksum: 640 messages.append("If this change is expected (e.g. you have upgraded " \ 641 "to a new version without updating the checksums) " \ 642 "then you can use these lines within the recipe:") 643 messages.extend(checksum_lines) 644 messages.append("Otherwise you should retry the download and/or " \ 645 "check with upstream to determine if the file has " \ 646 "become corrupted or otherwise unexpectedly modified.") 647 raise ChecksumError("\n".join(messages), ud.url, bad_checksum) 648 649 return checksum_dict 650 651def verify_donestamp(ud, d, origud=None): 652 """ 653 Check whether the done stamp file has the right checksums (if the fetch 654 method supports them). If it doesn't, delete the done stamp and force 655 a re-download. 656 657 Returns True, if the donestamp exists and is valid, False otherwise. When 658 returning False, any existing done stamps are removed. 659 """ 660 if not ud.needdonestamp or (origud and not origud.needdonestamp): 661 return True 662 663 if not os.path.exists(ud.localpath): 664 # local path does not exist 665 if os.path.exists(ud.donestamp): 666 # done stamp exists, but the downloaded file does not; the done stamp 667 # must be incorrect, re-trigger the download 668 bb.utils.remove(ud.donestamp) 669 return False 670 671 if (not ud.method.supports_checksum(ud) or 672 (origud and not origud.method.supports_checksum(origud))): 673 # if done stamp exists and checksums not supported; assume the local 674 # file is current 675 return os.path.exists(ud.donestamp) 676 677 precomputed_checksums = {} 678 # Only re-use the precomputed checksums if the donestamp is newer than the 679 # file. Do not rely on the mtime of directories, though. If ud.localpath is 680 # a directory, there will probably not be any checksums anyway. 681 if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or 682 os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)): 683 try: 684 with open(ud.donestamp, "rb") as cachefile: 685 pickled = pickle.Unpickler(cachefile) 686 precomputed_checksums.update(pickled.load()) 687 except Exception as e: 688 # Avoid the warnings on the upgrade path from emtpy done stamp 689 # files to those containing the checksums. 690 if not isinstance(e, EOFError): 691 # Ignore errors, they aren't fatal 692 logger.warning("Couldn't load checksums from donestamp %s: %s " 693 "(msg: %s)" % (ud.donestamp, type(e).__name__, 694 str(e))) 695 696 try: 697 checksums = verify_checksum(ud, d, precomputed_checksums) 698 # If the cache file did not have the checksums, compute and store them 699 # as an upgrade path from the previous done stamp file format. 700 if checksums != precomputed_checksums: 701 with open(ud.donestamp, "wb") as cachefile: 702 p = pickle.Pickler(cachefile, 2) 703 p.dump(checksums) 704 return True 705 except ChecksumError as e: 706 # Checksums failed to verify, trigger re-download and remove the 707 # incorrect stamp file. 708 logger.warning("Checksum mismatch for local file %s\n" 709 "Cleaning and trying again." % ud.localpath) 710 if os.path.exists(ud.localpath): 711 rename_bad_checksum(ud, e.checksum) 712 bb.utils.remove(ud.donestamp) 713 return False 714 715 716def update_stamp(ud, d): 717 """ 718 donestamp is file stamp indicating the whole fetching is done 719 this function update the stamp after verifying the checksum 720 """ 721 if not ud.needdonestamp: 722 return 723 724 if os.path.exists(ud.donestamp): 725 # Touch the done stamp file to show active use of the download 726 try: 727 os.utime(ud.donestamp, None) 728 except: 729 # Errors aren't fatal here 730 pass 731 else: 732 try: 733 checksums = verify_checksum(ud, d) 734 # Store the checksums for later re-verification against the recipe 735 with open(ud.donestamp, "wb") as cachefile: 736 p = pickle.Pickler(cachefile, 2) 737 p.dump(checksums) 738 except ChecksumError as e: 739 # Checksums failed to verify, trigger re-download and remove the 740 # incorrect stamp file. 741 logger.warning("Checksum mismatch for local file %s\n" 742 "Cleaning and trying again." % ud.localpath) 743 if os.path.exists(ud.localpath): 744 rename_bad_checksum(ud, e.checksum) 745 bb.utils.remove(ud.donestamp) 746 raise 747 748def subprocess_setup(): 749 # Python installs a SIGPIPE handler by default. This is usually not what 750 # non-Python subprocesses expect. 751 # SIGPIPE errors are known issues with gzip/bash 752 signal.signal(signal.SIGPIPE, signal.SIG_DFL) 753 754def mark_recipe_nocache(d): 755 if d.getVar('BB_SRCREV_POLICY') != "cache": 756 d.setVar('BB_DONT_CACHE', '1') 757 758def get_autorev(d): 759 mark_recipe_nocache(d) 760 d.setVar("__BBAUTOREV_SEEN", True) 761 return "AUTOINC" 762 763def _get_srcrev(d, method_name='sortable_revision'): 764 """ 765 Return the revision string, usually for use in the version string (PV) of the current package 766 Most packages usually only have one SCM so we just pass on the call. 767 In the multi SCM case, we build a value based on SRCREV_FORMAT which must 768 have been set. 769 770 The idea here is that we put the string "AUTOINC+" into return value if the revisions are not 771 incremental, other code is then responsible for turning that into an increasing value (if needed) 772 773 A method_name can be supplied to retrieve an alternatively formatted revision from a fetcher, if 774 that fetcher provides a method with the given name and the same signature as sortable_revision. 775 """ 776 777 d.setVar("__BBSRCREV_SEEN", "1") 778 recursion = d.getVar("__BBINSRCREV") 779 if recursion: 780 raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI") 781 d.setVar("__BBINSRCREV", True) 782 783 scms = [] 784 revs = [] 785 fetcher = Fetch(d.getVar('SRC_URI').split(), d) 786 urldata = fetcher.ud 787 for u in urldata: 788 if urldata[u].method.supports_srcrev(): 789 scms.append(u) 790 791 if not scms: 792 d.delVar("__BBINSRCREV") 793 return "", revs 794 795 796 if len(scms) == 1 and len(urldata[scms[0]].names) == 1: 797 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) 798 revs.append(rev) 799 if len(rev) > 10: 800 rev = rev[:10] 801 d.delVar("__BBINSRCREV") 802 if autoinc: 803 return "AUTOINC+" + rev, revs 804 return rev, revs 805 806 # 807 # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT 808 # 809 format = d.getVar('SRCREV_FORMAT') 810 if not format: 811 raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.\n"\ 812 "The SCMs are:\n%s" % '\n'.join(scms)) 813 814 name_to_rev = {} 815 seenautoinc = False 816 for scm in scms: 817 ud = urldata[scm] 818 for name in ud.names: 819 autoinc, rev = getattr(ud.method, method_name)(ud, d, name) 820 revs.append(rev) 821 seenautoinc = seenautoinc or autoinc 822 if len(rev) > 10: 823 rev = rev[:10] 824 name_to_rev[name] = rev 825 # Replace names by revisions in the SRCREV_FORMAT string. The approach used 826 # here can handle names being prefixes of other names and names appearing 827 # as substrings in revisions (in which case the name should not be 828 # expanded). The '|' regular expression operator tries matches from left to 829 # right, so we need to sort the names with the longest ones first. 830 names_descending_len = sorted(name_to_rev, key=len, reverse=True) 831 name_to_rev_re = "|".join(re.escape(name) for name in names_descending_len) 832 format = re.sub(name_to_rev_re, lambda match: name_to_rev[match.group(0)], format) 833 834 if seenautoinc: 835 format = "AUTOINC+" + format 836 837 d.delVar("__BBINSRCREV") 838 return format, revs 839 840def get_hashvalue(d, method_name='sortable_revision'): 841 pkgv, revs = _get_srcrev(d, method_name=method_name) 842 return " ".join(revs) 843 844def get_pkgv_string(d, method_name='sortable_revision'): 845 pkgv, revs = _get_srcrev(d, method_name=method_name) 846 return pkgv 847 848def get_srcrev(d, method_name='sortable_revision'): 849 pkgv, revs = _get_srcrev(d, method_name=method_name) 850 if not pkgv: 851 raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI") 852 return pkgv 853 854def localpath(url, d): 855 fetcher = bb.fetch2.Fetch([url], d) 856 return fetcher.localpath(url) 857 858# Need to export PATH as binary could be in metadata paths 859# rather than host provided 860# Also include some other variables. 861FETCH_EXPORT_VARS = ['HOME', 'PATH', 862 'HTTP_PROXY', 'http_proxy', 863 'HTTPS_PROXY', 'https_proxy', 864 'FTP_PROXY', 'ftp_proxy', 865 'FTPS_PROXY', 'ftps_proxy', 866 'NO_PROXY', 'no_proxy', 867 'ALL_PROXY', 'all_proxy', 868 'GIT_PROXY_COMMAND', 869 'GIT_SSH', 870 'GIT_SSH_COMMAND', 871 'GIT_SSL_CAINFO', 872 'GIT_SMART_HTTP', 873 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 874 'SOCKS5_USER', 'SOCKS5_PASSWD', 875 'DBUS_SESSION_BUS_ADDRESS', 876 'P4CONFIG', 877 'SSL_CERT_FILE', 878 'NODE_EXTRA_CA_CERTS', 879 'AWS_PROFILE', 880 'AWS_ACCESS_KEY_ID', 881 'AWS_SECRET_ACCESS_KEY', 882 'AWS_ROLE_ARN', 883 'AWS_WEB_IDENTITY_TOKEN_FILE', 884 'AWS_DEFAULT_REGION', 885 'AWS_SESSION_TOKEN', 886 'GIT_CACHE_PATH', 887 'REMOTE_CONTAINERS_IPC', 888 'GITHUB_TOKEN', 889 'SSL_CERT_DIR'] 890 891def get_fetcher_environment(d): 892 newenv = {} 893 origenv = d.getVar("BB_ORIGENV") 894 for name in bb.fetch2.FETCH_EXPORT_VARS: 895 value = d.getVar(name) 896 if not value and origenv: 897 value = origenv.getVar(name) 898 if value: 899 newenv[name] = value 900 return newenv 901 902def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): 903 """ 904 Run cmd returning the command output 905 Raise an error if interrupted or cmd fails 906 Optionally echo command output to stdout 907 Optionally remove the files/directories listed in cleanup upon failure 908 """ 909 910 exportvars = FETCH_EXPORT_VARS 911 912 if not cleanup: 913 cleanup = [] 914 915 # If PATH contains WORKDIR which contains PV-PR which contains SRCPV we 916 # can end up in circular recursion here so give the option of breaking it 917 # in a data store copy. 918 try: 919 d.getVar("PV") 920 d.getVar("PR") 921 except bb.data_smart.ExpansionError: 922 d = bb.data.createCopy(d) 923 d.setVar("PV", "fetcheravoidrecurse") 924 d.setVar("PR", "fetcheravoidrecurse") 925 926 origenv = d.getVar("BB_ORIGENV", False) 927 for var in exportvars: 928 val = d.getVar(var) or (origenv and origenv.getVar(var)) 929 if val: 930 cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) 931 932 # Disable pseudo as it may affect ssh, potentially causing it to hang. 933 cmd = 'export PSEUDO_DISABLED=1; ' + cmd 934 935 if workdir: 936 logger.debug("Running '%s' in %s" % (cmd, workdir)) 937 else: 938 logger.debug("Running %s", cmd) 939 940 success = False 941 error_message = "" 942 943 try: 944 (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir) 945 success = True 946 except bb.process.NotFoundError as e: 947 error_message = "Fetch command %s not found" % (e.command) 948 except bb.process.ExecutionError as e: 949 if e.stdout: 950 output = "output:\n%s\n%s" % (e.stdout, e.stderr) 951 elif e.stderr: 952 output = "output:\n%s" % e.stderr 953 else: 954 if log: 955 output = "see logfile for output" 956 else: 957 output = "no output" 958 error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output) 959 except bb.process.CmdError as e: 960 error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg) 961 if not success: 962 for f in cleanup: 963 try: 964 bb.utils.remove(f, True) 965 except OSError: 966 pass 967 968 raise FetchError(error_message) 969 970 return output 971 972def check_network_access(d, info, url): 973 """ 974 log remote network access, and error if BB_NO_NETWORK is set or the given 975 URI is untrusted 976 """ 977 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): 978 raise NetworkAccess(url, info) 979 elif not trusted_network(d, url): 980 raise UntrustedUrl(url, info) 981 else: 982 logger.debug("Fetcher accessed the network with the command %s" % info) 983 984def build_mirroruris(origud, mirrors, ld): 985 uris = [] 986 uds = [] 987 988 replacements = {} 989 replacements["TYPE"] = origud.type 990 replacements["HOST"] = origud.host 991 replacements["PATH"] = origud.path 992 replacements["BASENAME"] = origud.path.split("/")[-1] 993 replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') 994 995 def adduri(ud, uris, uds, mirrors, tarballs): 996 for line in mirrors: 997 try: 998 (find, replace) = line 999 except ValueError: 1000 continue 1001 1002 for tarball in tarballs: 1003 newuri = uri_replace(ud, find, replace, replacements, ld, tarball) 1004 if not newuri or newuri in uris or newuri == origud.url: 1005 continue 1006 1007 if not trusted_network(ld, newuri): 1008 logger.debug("Mirror %s not in the list of trusted networks, skipping" % (newuri)) 1009 continue 1010 1011 # Create a local copy of the mirrors minus the current line 1012 # this will prevent us from recursively processing the same line 1013 # as well as indirect recursion A -> B -> C -> A 1014 localmirrors = list(mirrors) 1015 localmirrors.remove(line) 1016 1017 try: 1018 newud = FetchData(newuri, ld) 1019 newud.ignore_checksums = True 1020 newud.setup_localpath(ld) 1021 except bb.fetch2.BBFetchException as e: 1022 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) 1023 logger.debug(str(e)) 1024 try: 1025 # setup_localpath of file:// urls may fail, we should still see 1026 # if mirrors of the url exist 1027 adduri(newud, uris, uds, localmirrors, tarballs) 1028 except UnboundLocalError: 1029 pass 1030 continue 1031 uris.append(newuri) 1032 uds.append(newud) 1033 1034 adduri(newud, uris, uds, localmirrors, tarballs) 1035 1036 adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None]) 1037 1038 return uris, uds 1039 1040def rename_bad_checksum(ud, suffix): 1041 """ 1042 Renames files to have suffix from parameter 1043 """ 1044 1045 if ud.localpath is None: 1046 return 1047 1048 new_localpath = "%s_bad-checksum_%s" % (ud.localpath, suffix) 1049 bb.warn("Renaming %s to %s" % (ud.localpath, new_localpath)) 1050 if not bb.utils.movefile(ud.localpath, new_localpath): 1051 bb.warn("Renaming %s to %s failed, grep movefile in log.do_fetch to see why" % (ud.localpath, new_localpath)) 1052 1053 1054def try_mirror_url(fetch, origud, ud, ld, check = False): 1055 # Return of None or a value means we're finished 1056 # False means try another url 1057 1058 if ud.lockfile and ud.lockfile != origud.lockfile: 1059 lf = bb.utils.lockfile(ud.lockfile) 1060 1061 try: 1062 if check: 1063 found = ud.method.checkstatus(fetch, ud, ld) 1064 if found: 1065 return found 1066 return False 1067 1068 if not verify_donestamp(ud, ld, origud) or ud.method.need_update(ud, ld): 1069 ud.method.download(ud, ld) 1070 if hasattr(ud.method,"build_mirror_data"): 1071 ud.method.build_mirror_data(ud, ld) 1072 1073 if not ud.localpath or not os.path.exists(ud.localpath): 1074 return False 1075 1076 if ud.localpath == origud.localpath: 1077 return ud.localpath 1078 1079 # We may be obtaining a mirror tarball which needs further processing by the real fetcher 1080 # If that tarball is a local file:// we need to provide a symlink to it 1081 dldir = ld.getVar("DL_DIR") 1082 1083 if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): 1084 # Create donestamp in old format to avoid triggering a re-download 1085 if ud.donestamp: 1086 bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) 1087 open(ud.donestamp, 'w').close() 1088 dest = os.path.join(dldir, os.path.basename(ud.localpath)) 1089 if not os.path.exists(dest): 1090 # In case this is executing without any file locks held (as is 1091 # the case for file:// URLs), two tasks may end up here at the 1092 # same time, in which case we do not want the second task to 1093 # fail when the link has already been created by the first task. 1094 try: 1095 os.symlink(ud.localpath, dest) 1096 except FileExistsError: 1097 pass 1098 if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): 1099 origud.method.download(origud, ld) 1100 if hasattr(origud.method, "build_mirror_data"): 1101 origud.method.build_mirror_data(origud, ld) 1102 return origud.localpath 1103 # Otherwise the result is a local file:// and we symlink to it 1104 ensure_symlink(ud.localpath, origud.localpath) 1105 update_stamp(origud, ld) 1106 return ud.localpath 1107 1108 except bb.fetch2.NetworkAccess: 1109 raise 1110 1111 except IOError as e: 1112 if e.errno in [errno.ESTALE]: 1113 logger.warning("Stale Error Observed %s." % ud.url) 1114 return False 1115 raise 1116 1117 except bb.fetch2.BBFetchException as e: 1118 if isinstance(e, ChecksumError): 1119 logger.warning("Mirror checksum failure for url %s (original url: %s)\nCleaning and trying again." % (ud.url, origud.url)) 1120 logger.warning(str(e)) 1121 if os.path.exists(ud.localpath): 1122 rename_bad_checksum(ud, e.checksum) 1123 elif isinstance(e, NoChecksumError): 1124 raise 1125 else: 1126 logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url)) 1127 logger.debug(str(e)) 1128 try: 1129 if ud.method.cleanup_upon_failure(): 1130 ud.method.clean(ud, ld) 1131 except UnboundLocalError: 1132 pass 1133 return False 1134 finally: 1135 if ud.lockfile and ud.lockfile != origud.lockfile: 1136 bb.utils.unlockfile(lf) 1137 1138 1139def ensure_symlink(target, link_name): 1140 if not os.path.exists(link_name): 1141 dirname = os.path.dirname(link_name) 1142 bb.utils.mkdirhier(dirname) 1143 if os.path.islink(link_name): 1144 # Broken symbolic link 1145 os.unlink(link_name) 1146 1147 # In case this is executing without any file locks held (as is 1148 # the case for file:// URLs), two tasks may end up here at the 1149 # same time, in which case we do not want the second task to 1150 # fail when the link has already been created by the first task. 1151 try: 1152 os.symlink(target, link_name) 1153 except FileExistsError: 1154 pass 1155 1156 1157def try_mirrors(fetch, d, origud, mirrors, check = False): 1158 """ 1159 Try to use a mirrored version of the sources. 1160 This method will be automatically called before the fetchers go. 1161 1162 d Is a bb.data instance 1163 uri is the original uri we're trying to download 1164 mirrors is the list of mirrors we're going to try 1165 """ 1166 ld = d.createCopy() 1167 1168 uris, uds = build_mirroruris(origud, mirrors, ld) 1169 1170 for index, uri in enumerate(uris): 1171 ret = try_mirror_url(fetch, origud, uds[index], ld, check) 1172 if ret: 1173 return ret 1174 return None 1175 1176def trusted_network(d, url): 1177 """ 1178 Use a trusted url during download if networking is enabled and 1179 BB_ALLOWED_NETWORKS is set globally or for a specific recipe. 1180 Note: modifies SRC_URI & mirrors. 1181 """ 1182 if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): 1183 return True 1184 1185 pkgname = d.expand(d.getVar('PN', False)) 1186 trusted_hosts = None 1187 if pkgname: 1188 trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False) 1189 1190 if not trusted_hosts: 1191 trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS') 1192 1193 # Not enabled. 1194 if not trusted_hosts: 1195 return True 1196 1197 scheme, network, path, user, passwd, param = decodeurl(url) 1198 1199 if not network: 1200 return True 1201 1202 network = network.split(':')[0] 1203 network = network.lower() 1204 1205 for host in trusted_hosts.split(" "): 1206 host = host.lower() 1207 if host.startswith("*.") and ("." + network).endswith(host[1:]): 1208 return True 1209 if host == network: 1210 return True 1211 1212 return False 1213 1214def srcrev_internal_helper(ud, d, name): 1215 """ 1216 Return: 1217 a) a source revision if specified 1218 b) latest revision if SRCREV="AUTOINC" 1219 c) None if not specified 1220 """ 1221 1222 srcrev = None 1223 pn = d.getVar("PN") 1224 attempts = [] 1225 if name != '' and pn: 1226 attempts.append("SRCREV_%s:pn-%s" % (name, pn)) 1227 if name != '': 1228 attempts.append("SRCREV_%s" % name) 1229 if pn: 1230 attempts.append("SRCREV:pn-%s" % pn) 1231 attempts.append("SRCREV") 1232 1233 for a in attempts: 1234 srcrev = d.getVar(a) 1235 if srcrev and srcrev != "INVALID": 1236 break 1237 1238 if 'rev' in ud.parm and 'tag' in ud.parm: 1239 raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url)) 1240 1241 if 'rev' in ud.parm or 'tag' in ud.parm: 1242 if 'rev' in ud.parm: 1243 parmrev = ud.parm['rev'] 1244 else: 1245 parmrev = ud.parm['tag'] 1246 if srcrev == "INVALID" or not srcrev: 1247 return parmrev 1248 if srcrev != parmrev: 1249 raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev)) 1250 return parmrev 1251 1252 if srcrev == "INVALID" or not srcrev: 1253 raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url) 1254 if srcrev == "AUTOINC": 1255 d.setVar("__BBAUTOREV_ACTED_UPON", True) 1256 srcrev = ud.method.latest_revision(ud, d, name) 1257 1258 return srcrev 1259 1260def get_checksum_file_list(d): 1261 """ Get a list of files checksum in SRC_URI 1262 1263 Returns the resolved local paths of all local file entries in 1264 SRC_URI as a space-separated string 1265 """ 1266 fetch = Fetch([], d, cache = False, localonly = True) 1267 filelist = [] 1268 for u in fetch.urls: 1269 ud = fetch.ud[u] 1270 if ud and isinstance(ud.method, local.Local): 1271 found = False 1272 paths = ud.method.localfile_searchpaths(ud, d) 1273 for f in paths: 1274 pth = ud.decodedurl 1275 if os.path.exists(f): 1276 found = True 1277 filelist.append(f + ":" + str(os.path.exists(f))) 1278 if not found: 1279 bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found" 1280 "\nThe following paths were searched:" 1281 "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths))) 1282 1283 return " ".join(filelist) 1284 1285def get_file_checksums(filelist, pn, localdirsexclude): 1286 """Get a list of the checksums for a list of local files 1287 1288 Returns the checksums for a list of local files, caching the results as 1289 it proceeds 1290 1291 """ 1292 return _checksum_cache.get_checksums(filelist, pn, localdirsexclude) 1293 1294 1295class FetchData(object): 1296 """ 1297 A class which represents the fetcher state for a given URI. 1298 """ 1299 def __init__(self, url, d, localonly = False): 1300 # localpath is the location of a downloaded result. If not set, the file is local. 1301 self.donestamp = None 1302 self.needdonestamp = True 1303 self.localfile = "" 1304 self.localpath = None 1305 self.lockfile = None 1306 self.mirrortarballs = [] 1307 self.basename = None 1308 self.basepath = None 1309 (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url)) 1310 self.date = self.getSRCDate(d) 1311 self.url = url 1312 if not self.user and "user" in self.parm: 1313 self.user = self.parm["user"] 1314 if not self.pswd and "pswd" in self.parm: 1315 self.pswd = self.parm["pswd"] 1316 self.setup = False 1317 1318 def configure_checksum(checksum_id): 1319 if "name" in self.parm: 1320 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id) 1321 else: 1322 checksum_name = "%ssum" % checksum_id 1323 1324 setattr(self, "%s_name" % checksum_id, checksum_name) 1325 1326 if checksum_name in self.parm: 1327 checksum_expected = self.parm[checksum_name] 1328 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod"]: 1329 checksum_expected = None 1330 else: 1331 checksum_expected = d.getVarFlag("SRC_URI", checksum_name) 1332 1333 setattr(self, "%s_expected" % checksum_id, checksum_expected) 1334 1335 self.names = self.parm.get("name",'default').split(',') 1336 1337 self.method = None 1338 for m in methods: 1339 if m.supports(self, d): 1340 self.method = m 1341 break 1342 1343 if not self.method: 1344 raise NoMethodError(url) 1345 1346 if localonly and not isinstance(self.method, local.Local): 1347 raise NonLocalMethod() 1348 1349 if self.parm.get("proto", None) and "protocol" not in self.parm: 1350 logger.warning('Consider updating %s recipe to use "protocol" not "proto" in SRC_URI.', d.getVar('PN')) 1351 self.parm["protocol"] = self.parm.get("proto", None) 1352 1353 if hasattr(self.method, "urldata_init"): 1354 self.method.urldata_init(self, d) 1355 1356 for checksum_id in CHECKSUM_LIST: 1357 configure_checksum(checksum_id) 1358 1359 self.ignore_checksums = False 1360 1361 if "localpath" in self.parm: 1362 # if user sets localpath for file, use it instead. 1363 self.localpath = self.parm["localpath"] 1364 self.basename = os.path.basename(self.localpath) 1365 elif self.localfile: 1366 self.localpath = self.method.localpath(self, d) 1367 1368 dldir = d.getVar("DL_DIR") 1369 1370 if not self.needdonestamp: 1371 return 1372 1373 # Note: .done and .lock files should always be in DL_DIR whereas localpath may not be. 1374 if self.localpath and self.localpath.startswith(dldir): 1375 basepath = self.localpath 1376 elif self.localpath: 1377 basepath = dldir + os.sep + os.path.basename(self.localpath) 1378 elif self.basepath or self.basename: 1379 basepath = dldir + os.sep + (self.basepath or self.basename) 1380 else: 1381 bb.fatal("Can't determine lock path for url %s" % url) 1382 1383 self.donestamp = basepath + '.done' 1384 self.lockfile = basepath + '.lock' 1385 1386 def setup_revisions(self, d): 1387 self.revisions = {} 1388 for name in self.names: 1389 self.revisions[name] = srcrev_internal_helper(self, d, name) 1390 1391 # add compatibility code for non name specified case 1392 if len(self.names) == 1: 1393 self.revision = self.revisions[self.names[0]] 1394 1395 def setup_localpath(self, d): 1396 if not self.localpath: 1397 self.localpath = self.method.localpath(self, d) 1398 1399 def getSRCDate(self, d): 1400 """ 1401 Return the SRC Date for the component 1402 1403 d the bb.data module 1404 """ 1405 if "srcdate" in self.parm: 1406 return self.parm['srcdate'] 1407 1408 pn = d.getVar("PN") 1409 1410 if pn: 1411 return d.getVar("SRCDATE_%s" % pn) or d.getVar("SRCDATE") or d.getVar("DATE") 1412 1413 return d.getVar("SRCDATE") or d.getVar("DATE") 1414 1415class FetchMethod(object): 1416 """Base class for 'fetch'ing data""" 1417 1418 def __init__(self, urls=None): 1419 self.urls = [] 1420 1421 def supports(self, urldata, d): 1422 """ 1423 Check to see if this fetch class supports a given url. 1424 """ 1425 return 0 1426 1427 def localpath(self, urldata, d): 1428 """ 1429 Return the local filename of a given url assuming a successful fetch. 1430 Can also setup variables in urldata for use in go (saving code duplication 1431 and duplicate code execution) 1432 """ 1433 return os.path.join(d.getVar("DL_DIR"), urldata.localfile) 1434 1435 def supports_checksum(self, urldata): 1436 """ 1437 Is localpath something that can be represented by a checksum? 1438 """ 1439 1440 # We cannot compute checksums for None 1441 if urldata.localpath is None: 1442 return False 1443 # We cannot compute checksums for directories 1444 if os.path.isdir(urldata.localpath): 1445 return False 1446 return True 1447 1448 def recommends_checksum(self, urldata): 1449 """ 1450 Is the backend on where checksumming is recommended (should warnings 1451 be displayed if there is no checksum)? 1452 """ 1453 return False 1454 1455 def cleanup_upon_failure(self): 1456 """ 1457 When a fetch fails, should clean() be called? 1458 """ 1459 return True 1460 1461 def verify_donestamp(self, ud, d): 1462 """ 1463 Verify the donestamp file 1464 """ 1465 return verify_donestamp(ud, d) 1466 1467 def update_donestamp(self, ud, d): 1468 """ 1469 Update the donestamp file 1470 """ 1471 update_stamp(ud, d) 1472 1473 def _strip_leading_slashes(self, relpath): 1474 """ 1475 Remove leading slash as os.path.join can't cope 1476 """ 1477 while os.path.isabs(relpath): 1478 relpath = relpath[1:] 1479 return relpath 1480 1481 def setUrls(self, urls): 1482 self.__urls = urls 1483 1484 def getUrls(self): 1485 return self.__urls 1486 1487 urls = property(getUrls, setUrls, None, "Urls property") 1488 1489 def need_update(self, ud, d): 1490 """ 1491 Force a fetch, even if localpath exists? 1492 """ 1493 if os.path.exists(ud.localpath): 1494 return False 1495 return True 1496 1497 def supports_srcrev(self): 1498 """ 1499 The fetcher supports auto source revisions (SRCREV) 1500 """ 1501 return False 1502 1503 def download(self, urldata, d): 1504 """ 1505 Fetch urls 1506 Assumes localpath was called first 1507 """ 1508 raise NoMethodError(urldata.url) 1509 1510 def unpack(self, urldata, rootdir, data): 1511 iterate = False 1512 file = urldata.localpath 1513 1514 try: 1515 unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True) 1516 except ValueError as exc: 1517 bb.fatal("Invalid value for 'unpack' parameter for %s: %s" % 1518 (file, urldata.parm.get('unpack'))) 1519 1520 base, ext = os.path.splitext(file) 1521 if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz', '.zst']: 1522 efile = os.path.join(rootdir, os.path.basename(base)) 1523 else: 1524 efile = file 1525 cmd = None 1526 1527 if unpack: 1528 tar_cmd = 'tar --extract --no-same-owner' 1529 if 'striplevel' in urldata.parm: 1530 tar_cmd += ' --strip-components=%s' % urldata.parm['striplevel'] 1531 if file.endswith('.tar'): 1532 cmd = '%s -f %s' % (tar_cmd, file) 1533 elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): 1534 cmd = '%s -z -f %s' % (tar_cmd, file) 1535 elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'): 1536 cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd) 1537 elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'): 1538 cmd = 'gzip -dc %s > %s' % (file, efile) 1539 elif file.endswith('.bz2'): 1540 cmd = 'bzip2 -dc %s > %s' % (file, efile) 1541 elif file.endswith('.txz') or file.endswith('.tar.xz'): 1542 cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd) 1543 elif file.endswith('.xz'): 1544 cmd = 'xz -dc %s > %s' % (file, efile) 1545 elif file.endswith('.tar.lz'): 1546 cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd) 1547 elif file.endswith('.lz'): 1548 cmd = 'lzip -dc %s > %s' % (file, efile) 1549 elif file.endswith('.tar.7z'): 1550 cmd = '7z x -so %s | %s -f -' % (file, tar_cmd) 1551 elif file.endswith('.7z'): 1552 cmd = '7za x -y %s 1>/dev/null' % file 1553 elif file.endswith('.tzst') or file.endswith('.tar.zst'): 1554 cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd) 1555 elif file.endswith('.zst'): 1556 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile) 1557 elif file.endswith('.zip') or file.endswith('.jar'): 1558 try: 1559 dos = bb.utils.to_boolean(urldata.parm.get('dos'), False) 1560 except ValueError as exc: 1561 bb.fatal("Invalid value for 'dos' parameter for %s: %s" % 1562 (file, urldata.parm.get('dos'))) 1563 cmd = 'unzip -q -o' 1564 if dos: 1565 cmd = '%s -a' % cmd 1566 cmd = "%s '%s'" % (cmd, file) 1567 elif file.endswith('.rpm') or file.endswith('.srpm'): 1568 if 'extract' in urldata.parm: 1569 unpack_file = urldata.parm.get('extract') 1570 cmd = 'rpm2cpio.sh %s | cpio -id %s' % (file, unpack_file) 1571 iterate = True 1572 iterate_file = unpack_file 1573 else: 1574 cmd = 'rpm2cpio.sh %s | cpio -id' % (file) 1575 elif file.endswith('.deb') or file.endswith('.ipk'): 1576 output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup) 1577 datafile = None 1578 if output: 1579 for line in output.decode().splitlines(): 1580 if line.startswith('data.tar.'): 1581 datafile = line 1582 break 1583 else: 1584 raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url) 1585 else: 1586 raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url) 1587 cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile) 1588 1589 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd 1590 if 'subdir' in urldata.parm: 1591 subdir = urldata.parm.get('subdir') 1592 if os.path.isabs(subdir): 1593 if not os.path.realpath(subdir).startswith(os.path.realpath(rootdir)): 1594 raise UnpackError("subdir argument isn't a subdirectory of unpack root %s" % rootdir, urldata.url) 1595 unpackdir = subdir 1596 else: 1597 unpackdir = os.path.join(rootdir, subdir) 1598 bb.utils.mkdirhier(unpackdir) 1599 else: 1600 unpackdir = rootdir 1601 1602 if not unpack or not cmd: 1603 urldata.unpack_tracer.unpack("file-copy", unpackdir) 1604 # If file == dest, then avoid any copies, as we already put the file into dest! 1605 dest = os.path.join(unpackdir, os.path.basename(file)) 1606 if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): 1607 destdir = '.' 1608 # For file:// entries all intermediate dirs in path must be created at destination 1609 if urldata.type == "file": 1610 # Trailing '/' does a copying to wrong place 1611 urlpath = urldata.path.rstrip('/') 1612 # Want files places relative to cwd so no leading '/' 1613 urlpath = urlpath.lstrip('/') 1614 if urlpath.find("/") != -1: 1615 destdir = urlpath.rsplit("/", 1)[0] + '/' 1616 bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) 1617 cmd = 'cp --force --preserve=timestamps --no-dereference --recursive -H "%s" "%s"' % (file, destdir) 1618 else: 1619 urldata.unpack_tracer.unpack("archive-extract", unpackdir) 1620 1621 if not cmd: 1622 return 1623 1624 path = data.getVar('PATH') 1625 if path: 1626 cmd = "PATH=\"%s\" %s" % (path, cmd) 1627 bb.note("Unpacking %s to %s/" % (file, unpackdir)) 1628 ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=unpackdir) 1629 1630 if ret != 0: 1631 raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url) 1632 1633 if iterate is True: 1634 iterate_urldata = urldata 1635 iterate_urldata.localpath = "%s/%s" % (rootdir, iterate_file) 1636 self.unpack(urldata, rootdir, data) 1637 1638 return 1639 1640 def clean(self, urldata, d): 1641 """ 1642 Clean any existing full or partial download 1643 """ 1644 bb.utils.remove(urldata.localpath) 1645 1646 def try_premirror(self, urldata, d): 1647 """ 1648 Should premirrors be used? 1649 """ 1650 return True 1651 1652 def try_mirrors(self, fetch, urldata, d, mirrors, check=False): 1653 """ 1654 Try to use a mirror 1655 """ 1656 return bool(try_mirrors(fetch, d, urldata, mirrors, check)) 1657 1658 def checkstatus(self, fetch, urldata, d): 1659 """ 1660 Check the status of a URL 1661 Assumes localpath was called first 1662 """ 1663 logger.info("URL %s could not be checked for status since no method exists.", urldata.url) 1664 return True 1665 1666 def latest_revision(self, ud, d, name): 1667 """ 1668 Look in the cache for the latest revision, if not present ask the SCM. 1669 """ 1670 if not hasattr(self, "_latest_revision"): 1671 raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) 1672 1673 key = self.generate_revision_key(ud, d, name) 1674 1675 rev = _revisions_cache.get_rev(key) 1676 if rev is None: 1677 rev = self._latest_revision(ud, d, name) 1678 _revisions_cache.set_rev(key, rev) 1679 return rev 1680 1681 def sortable_revision(self, ud, d, name): 1682 latest_rev = self._build_revision(ud, d, name) 1683 return True, str(latest_rev) 1684 1685 def generate_revision_key(self, ud, d, name): 1686 return self._revision_key(ud, d, name) 1687 1688 def latest_versionstring(self, ud, d): 1689 """ 1690 Compute the latest release name like "x.y.x" in "x.y.x+gitHASH" 1691 by searching through the tags output of ls-remote, comparing 1692 versions and returning the highest match as a (version, revision) pair. 1693 """ 1694 return ('', '') 1695 1696 def done(self, ud, d): 1697 """ 1698 Is the download done ? 1699 """ 1700 if os.path.exists(ud.localpath): 1701 return True 1702 return False 1703 1704 def implicit_urldata(self, ud, d): 1705 """ 1706 Get a list of FetchData objects for any implicit URLs that will also 1707 be downloaded when we fetch the given URL. 1708 """ 1709 return [] 1710 1711 1712class DummyUnpackTracer(object): 1713 """ 1714 Abstract API definition for a class that traces unpacked source files back 1715 to their respective upstream SRC_URI entries, for software composition 1716 analysis, license compliance and detailed SBOM generation purposes. 1717 User may load their own unpack tracer class (instead of the dummy 1718 one) by setting the BB_UNPACK_TRACER_CLASS config parameter. 1719 """ 1720 def start(self, unpackdir, urldata_dict, d): 1721 """ 1722 Start tracing the core Fetch.unpack process, using an index to map 1723 unpacked files to each SRC_URI entry. 1724 This method is called by Fetch.unpack and it may receive nested calls by 1725 gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit 1726 URLs and by recursively calling Fetch.unpack from new (nested) Fetch 1727 instances. 1728 """ 1729 return 1730 def start_url(self, url): 1731 """Start tracing url unpack process. 1732 This method is called by Fetch.unpack before the fetcher-specific unpack 1733 method starts, and it may receive nested calls by gitsm and npmsw 1734 fetchers. 1735 """ 1736 return 1737 def unpack(self, unpack_type, destdir): 1738 """ 1739 Set unpack_type and destdir for current url. 1740 This method is called by the fetcher-specific unpack method after url 1741 tracing started. 1742 """ 1743 return 1744 def finish_url(self, url): 1745 """Finish tracing url unpack process and update the file index. 1746 This method is called by Fetch.unpack after the fetcher-specific unpack 1747 method finished its job, and it may receive nested calls by gitsm 1748 and npmsw fetchers. 1749 """ 1750 return 1751 def complete(self): 1752 """ 1753 Finish tracing the Fetch.unpack process, and check if all nested 1754 Fecth.unpack calls (if any) have been completed; if so, save collected 1755 metadata. 1756 """ 1757 return 1758 1759 1760class Fetch(object): 1761 def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): 1762 if localonly and cache: 1763 raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time") 1764 1765 if not urls: 1766 urls = d.getVar("SRC_URI").split() 1767 self.urls = urls 1768 self.d = d 1769 self.ud = {} 1770 self.connection_cache = connection_cache 1771 1772 fn = d.getVar('FILE') 1773 mc = d.getVar('__BBMULTICONFIG') or "" 1774 key = None 1775 if cache and fn: 1776 key = mc + fn + str(id(d)) 1777 if key in urldata_cache: 1778 self.ud = urldata_cache[key] 1779 1780 # the unpack_tracer object needs to be made available to possible nested 1781 # Fetch instances (when those are created by gitsm and npmsw fetchers) 1782 # so we set it as a global variable 1783 global unpack_tracer 1784 try: 1785 unpack_tracer 1786 except NameError: 1787 class_path = d.getVar("BB_UNPACK_TRACER_CLASS") 1788 if class_path: 1789 # use user-defined unpack tracer class 1790 import importlib 1791 module_name, _, class_name = class_path.rpartition(".") 1792 module = importlib.import_module(module_name) 1793 class_ = getattr(module, class_name) 1794 unpack_tracer = class_() 1795 else: 1796 # fall back to the dummy/abstract class 1797 unpack_tracer = DummyUnpackTracer() 1798 1799 for url in urls: 1800 if url not in self.ud: 1801 try: 1802 self.ud[url] = FetchData(url, d, localonly) 1803 self.ud[url].unpack_tracer = unpack_tracer 1804 except NonLocalMethod: 1805 if localonly: 1806 self.ud[url] = None 1807 pass 1808 1809 if key: 1810 urldata_cache[key] = self.ud 1811 1812 def localpath(self, url): 1813 if url not in self.urls: 1814 self.ud[url] = FetchData(url, self.d) 1815 1816 self.ud[url].setup_localpath(self.d) 1817 return self.d.expand(self.ud[url].localpath) 1818 1819 def localpaths(self): 1820 """ 1821 Return a list of the local filenames, assuming successful fetch 1822 """ 1823 local = [] 1824 1825 for u in self.urls: 1826 ud = self.ud[u] 1827 ud.setup_localpath(self.d) 1828 local.append(ud.localpath) 1829 1830 return local 1831 1832 def download(self, urls=None): 1833 """ 1834 Fetch all urls 1835 """ 1836 if not urls: 1837 urls = self.urls 1838 1839 network = self.d.getVar("BB_NO_NETWORK") 1840 premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) 1841 1842 checksum_missing_messages = [] 1843 for u in urls: 1844 ud = self.ud[u] 1845 ud.setup_localpath(self.d) 1846 m = ud.method 1847 done = False 1848 1849 if ud.lockfile: 1850 lf = bb.utils.lockfile(ud.lockfile) 1851 1852 try: 1853 self.d.setVar("BB_NO_NETWORK", network) 1854 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): 1855 done = True 1856 elif m.try_premirror(ud, self.d): 1857 logger.debug("Trying PREMIRRORS") 1858 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) 1859 done = m.try_mirrors(self, ud, self.d, mirrors) 1860 if done: 1861 try: 1862 # early checksum verification so that if the checksum of the premirror 1863 # contents mismatch the fetcher can still try upstream and mirrors 1864 m.update_donestamp(ud, self.d) 1865 except ChecksumError as e: 1866 logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u) 1867 logger.debug(str(e)) 1868 done = False 1869 1870 if premirroronly: 1871 self.d.setVar("BB_NO_NETWORK", "1") 1872 1873 firsterr = None 1874 verified_stamp = False 1875 if done: 1876 verified_stamp = m.verify_donestamp(ud, self.d) 1877 if not done and (not verified_stamp or m.need_update(ud, self.d)): 1878 try: 1879 if not trusted_network(self.d, ud.url): 1880 raise UntrustedUrl(ud.url) 1881 logger.debug("Trying Upstream") 1882 m.download(ud, self.d) 1883 if hasattr(m, "build_mirror_data"): 1884 m.build_mirror_data(ud, self.d) 1885 done = True 1886 # early checksum verify, so that if checksum mismatched, 1887 # fetcher still have chance to fetch from mirror 1888 m.update_donestamp(ud, self.d) 1889 1890 except bb.fetch2.NetworkAccess: 1891 raise 1892 1893 except BBFetchException as e: 1894 if isinstance(e, ChecksumError): 1895 logger.warning("Checksum failure encountered with download of %s - will attempt other sources if available" % u) 1896 logger.debug(str(e)) 1897 if os.path.exists(ud.localpath): 1898 rename_bad_checksum(ud, e.checksum) 1899 elif isinstance(e, NoChecksumError): 1900 raise 1901 else: 1902 logger.warning('Failed to fetch URL %s, attempting MIRRORS if available' % u) 1903 logger.debug(str(e)) 1904 firsterr = e 1905 # Remove any incomplete fetch 1906 if not verified_stamp and m.cleanup_upon_failure(): 1907 m.clean(ud, self.d) 1908 logger.debug("Trying MIRRORS") 1909 mirrors = mirror_from_string(self.d.getVar('MIRRORS')) 1910 done = m.try_mirrors(self, ud, self.d, mirrors) 1911 1912 if not done or not m.done(ud, self.d): 1913 if firsterr: 1914 logger.error(str(firsterr)) 1915 raise FetchError("Unable to fetch URL from any source.", u) 1916 1917 m.update_donestamp(ud, self.d) 1918 1919 except IOError as e: 1920 if e.errno in [errno.ESTALE]: 1921 logger.error("Stale Error Observed %s." % u) 1922 raise ChecksumError("Stale Error Detected") 1923 1924 except BBFetchException as e: 1925 if isinstance(e, NoChecksumError): 1926 (message, _) = e.args 1927 checksum_missing_messages.append(message) 1928 continue 1929 elif isinstance(e, ChecksumError): 1930 logger.error("Checksum failure fetching %s" % u) 1931 raise 1932 1933 finally: 1934 if ud.lockfile: 1935 bb.utils.unlockfile(lf) 1936 if checksum_missing_messages: 1937 logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages)) 1938 raise BBFetchException("There was some missing checksums in the recipe") 1939 1940 def checkstatus(self, urls=None): 1941 """ 1942 Check all URLs exist upstream. 1943 1944 Returns None if the URLs exist, raises FetchError if the check wasn't 1945 successful but there wasn't an error (such as file not found), and 1946 raises other exceptions in error cases. 1947 """ 1948 1949 if not urls: 1950 urls = self.urls 1951 1952 for u in urls: 1953 ud = self.ud[u] 1954 ud.setup_localpath(self.d) 1955 m = ud.method 1956 logger.debug("Testing URL %s", u) 1957 # First try checking uri, u, from PREMIRRORS 1958 mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) 1959 ret = m.try_mirrors(self, ud, self.d, mirrors, True) 1960 if not ret: 1961 # Next try checking from the original uri, u 1962 ret = m.checkstatus(self, ud, self.d) 1963 if not ret: 1964 # Finally, try checking uri, u, from MIRRORS 1965 mirrors = mirror_from_string(self.d.getVar('MIRRORS')) 1966 ret = m.try_mirrors(self, ud, self.d, mirrors, True) 1967 1968 if not ret: 1969 raise FetchError("URL doesn't work", u) 1970 1971 def unpack(self, root, urls=None): 1972 """ 1973 Unpack urls to root 1974 """ 1975 1976 if not urls: 1977 urls = self.urls 1978 1979 unpack_tracer.start(root, self.ud, self.d) 1980 1981 for u in urls: 1982 ud = self.ud[u] 1983 ud.setup_localpath(self.d) 1984 1985 if ud.lockfile: 1986 lf = bb.utils.lockfile(ud.lockfile) 1987 1988 unpack_tracer.start_url(u) 1989 ud.method.unpack(ud, root, self.d) 1990 unpack_tracer.finish_url(u) 1991 1992 if ud.lockfile: 1993 bb.utils.unlockfile(lf) 1994 1995 unpack_tracer.complete() 1996 1997 def clean(self, urls=None): 1998 """ 1999 Clean files that the fetcher gets or places 2000 """ 2001 2002 if not urls: 2003 urls = self.urls 2004 2005 for url in urls: 2006 if url not in self.ud: 2007 self.ud[url] = FetchData(url, self.d) 2008 ud = self.ud[url] 2009 ud.setup_localpath(self.d) 2010 2011 if not ud.localfile and ud.localpath is None: 2012 continue 2013 2014 if ud.lockfile: 2015 lf = bb.utils.lockfile(ud.lockfile) 2016 2017 ud.method.clean(ud, self.d) 2018 if ud.donestamp: 2019 bb.utils.remove(ud.donestamp) 2020 2021 if ud.lockfile: 2022 bb.utils.unlockfile(lf) 2023 2024 def expanded_urldata(self, urls=None): 2025 """ 2026 Get an expanded list of FetchData objects covering both the given 2027 URLS and any additional implicit URLs that are added automatically by 2028 the appropriate FetchMethod. 2029 """ 2030 2031 if not urls: 2032 urls = self.urls 2033 2034 urldata = [] 2035 for url in urls: 2036 ud = self.ud[url] 2037 urldata.append(ud) 2038 urldata += ud.method.implicit_urldata(ud, self.d) 2039 2040 return urldata 2041 2042class FetchConnectionCache(object): 2043 """ 2044 A class which represents an container for socket connections. 2045 """ 2046 def __init__(self): 2047 self.cache = {} 2048 2049 def get_connection_name(self, host, port): 2050 return host + ':' + str(port) 2051 2052 def add_connection(self, host, port, connection): 2053 cn = self.get_connection_name(host, port) 2054 2055 if cn not in self.cache: 2056 self.cache[cn] = connection 2057 2058 def get_connection(self, host, port): 2059 connection = None 2060 2061 cn = self.get_connection_name(host, port) 2062 if cn in self.cache: 2063 connection = self.cache[cn] 2064 2065 return connection 2066 2067 def remove_connection(self, host, port): 2068 cn = self.get_connection_name(host, port) 2069 if cn in self.cache: 2070 self.cache[cn].close() 2071 del self.cache[cn] 2072 2073 def close_connections(self): 2074 for cn in list(self.cache.keys()): 2075 self.cache[cn].close() 2076 del self.cache[cn] 2077 2078from . import cvs 2079from . import git 2080from . import gitsm 2081from . import gitannex 2082from . import local 2083from . import svn 2084from . import wget 2085from . import ssh 2086from . import sftp 2087from . import s3 2088from . import perforce 2089from . import bzr 2090from . import hg 2091from . import osc 2092from . import repo 2093from . import clearcase 2094from . import npm 2095from . import npmsw 2096from . import az 2097from . import crate 2098from . import gcp 2099from . import gomod 2100 2101methods.append(local.Local()) 2102methods.append(wget.Wget()) 2103methods.append(svn.Svn()) 2104methods.append(git.Git()) 2105methods.append(gitsm.GitSM()) 2106methods.append(gitannex.GitANNEX()) 2107methods.append(cvs.Cvs()) 2108methods.append(ssh.SSH()) 2109methods.append(sftp.SFTP()) 2110methods.append(s3.S3()) 2111methods.append(perforce.Perforce()) 2112methods.append(bzr.Bzr()) 2113methods.append(hg.Hg()) 2114methods.append(osc.Osc()) 2115methods.append(repo.Repo()) 2116methods.append(clearcase.ClearCase()) 2117methods.append(npm.Npm()) 2118methods.append(npmsw.NpmShrinkWrap()) 2119methods.append(az.Az()) 2120methods.append(crate.Crate()) 2121methods.append(gcp.GCP()) 2122methods.append(gomod.GoMod()) 2123methods.append(gomod.GoModGit()) 2124