xref: /openbmc/openbmc/poky/bitbake/lib/bb/fetch2/wget.py (revision 8460358c3d24c71d9d38fd126c745854a6301564)
1"""
2BitBake 'Fetch' implementations
3
4Classes for obtaining upstream sources for the
5BitBake build tools.
6
7"""
8
9# Copyright (C) 2003, 2004  Chris Larson
10#
11# SPDX-License-Identifier: GPL-2.0-only
12#
13# Based on functions from the base bb module, Copyright 2003 Holger Schurig
14
15import shlex
16import re
17import tempfile
18import os
19import errno
20import bb
21import bb.progress
22import socket
23import http.client
24import urllib.request, urllib.parse, urllib.error
25from   bb.fetch2 import FetchMethod
26from   bb.fetch2 import FetchError
27from   bb.fetch2 import logger
28from   bb.fetch2 import runfetchcmd
29from   bs4 import BeautifulSoup
30from   bs4 import SoupStrainer
31
32class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
33    """
34    Extract progress information from wget output.
35    Note: relies on --progress=dot (with -v or without -q/-nv) being
36    specified on the wget command line.
37    """
38    def __init__(self, d):
39        super(WgetProgressHandler, self).__init__(d)
40        # Send an initial progress event so the bar gets shown
41        self._fire_progress(0)
42
43    def writeline(self, line):
44        percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
45        if percs:
46            progress = int(percs[-1][0])
47            rate = percs[-1][1] + '/s'
48            self.update(progress, rate)
49            return False
50        return True
51
52
53class Wget(FetchMethod):
54    """Class to fetch urls via 'wget'"""
55
56    def check_certs(self, d):
57        """
58        Should certificates be checked?
59        """
60        return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0"
61
62    def supports(self, ud, d):
63        """
64        Check to see if a given url can be fetched with wget.
65        """
66        return ud.type in ['http', 'https', 'ftp', 'ftps']
67
68    def recommends_checksum(self, urldata):
69        return True
70
71    def urldata_init(self, ud, d):
72        if 'protocol' in ud.parm:
73            if ud.parm['protocol'] == 'git':
74                raise bb.fetch2.ParameterError("Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http", ud.url)
75
76        if 'downloadfilename' in ud.parm:
77            ud.basename = ud.parm['downloadfilename']
78        else:
79            ud.basename = os.path.basename(ud.path)
80
81        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
82        if not ud.localfile:
83            ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
84
85        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 100"
86
87        if ud.type == 'ftp' or ud.type == 'ftps':
88            self.basecmd += " --passive-ftp"
89
90        if not self.check_certs(d):
91            self.basecmd += " --no-check-certificate"
92
93    def _runwget(self, ud, d, command, quiet, workdir=None):
94
95        progresshandler = WgetProgressHandler(d)
96
97        logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
98        bb.fetch2.check_network_access(d, command, ud.url)
99        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
100
101    def download(self, ud, d):
102        """Fetch urls"""
103
104        fetchcmd = self.basecmd
105
106        dldir = os.path.realpath(d.getVar("DL_DIR"))
107        localpath = os.path.join(dldir, ud.localfile) + ".tmp"
108        bb.utils.mkdirhier(os.path.dirname(localpath))
109        fetchcmd += " -O %s" % shlex.quote(localpath)
110
111        if ud.user and ud.pswd:
112            fetchcmd += " --auth-no-challenge"
113            if ud.parm.get("redirectauth", "1") == "1":
114                # An undocumented feature of wget is that if the
115                # username/password are specified on the URI, wget will only
116                # send the Authorization header to the first host and not to
117                # any hosts that it is redirected to.  With the increasing
118                # usage of temporary AWS URLs, this difference now matters as
119                # AWS will reject any request that has authentication both in
120                # the query parameters (from the redirect) and in the
121                # Authorization header.
122                fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
123
124        uri = ud.url.split(";")[0]
125        if os.path.exists(ud.localpath):
126            # file exists, but we didnt complete it.. trying again..
127            fetchcmd += " -c -P " + dldir + " '" + uri + "'"
128        else:
129            fetchcmd += " -P " + dldir + " '" + uri + "'"
130
131        self._runwget(ud, d, fetchcmd, False)
132
133        # Sanity check since wget can pretend it succeed when it didn't
134        # Also, this used to happen if sourceforge sent us to the mirror page
135        if not os.path.exists(localpath):
136            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri)
137
138        if os.path.getsize(localpath) == 0:
139            os.remove(localpath)
140            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
141
142        # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
143        # original file, which might be a race (imagine two recipes referencing the same
144        # source, one with an incorrect checksum)
145        bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
146
147        # Remove the ".tmp" and move the file into position atomically
148        # Our lock prevents multiple writers but mirroring code may grab incomplete files
149        os.rename(localpath, localpath[:-4])
150
151        return True
152
153    def checkstatus(self, fetch, ud, d, try_again=True):
154        class HTTPConnectionCache(http.client.HTTPConnection):
155            if fetch.connection_cache:
156                def connect(self):
157                    """Connect to the host and port specified in __init__."""
158
159                    sock = fetch.connection_cache.get_connection(self.host, self.port)
160                    if sock:
161                        self.sock = sock
162                    else:
163                        self.sock = socket.create_connection((self.host, self.port),
164                                    self.timeout, self.source_address)
165                        fetch.connection_cache.add_connection(self.host, self.port, self.sock)
166
167                    if self._tunnel_host:
168                        self._tunnel()
169
170        class CacheHTTPHandler(urllib.request.HTTPHandler):
171            def http_open(self, req):
172                return self.do_open(HTTPConnectionCache, req)
173
174            def do_open(self, http_class, req):
175                """Return an addinfourl object for the request, using http_class.
176
177                http_class must implement the HTTPConnection API from httplib.
178                The addinfourl return value is a file-like object.  It also
179                has methods and attributes including:
180                    - info(): return a mimetools.Message object for the headers
181                    - geturl(): return the original request URL
182                    - code: HTTP status code
183                """
184                host = req.host
185                if not host:
186                    raise urllib.error.URLError('no host given')
187
188                h = http_class(host, timeout=req.timeout) # will parse host:port
189                h.set_debuglevel(self._debuglevel)
190
191                headers = dict(req.unredirected_hdrs)
192                headers.update(dict((k, v) for k, v in list(req.headers.items())
193                            if k not in headers))
194
195                # We want to make an HTTP/1.1 request, but the addinfourl
196                # class isn't prepared to deal with a persistent connection.
197                # It will try to read all remaining data from the socket,
198                # which will block while the server waits for the next request.
199                # So make sure the connection gets closed after the (only)
200                # request.
201
202                # Don't close connection when connection_cache is enabled,
203                if fetch.connection_cache is None:
204                    headers["Connection"] = "close"
205                else:
206                    headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
207
208                headers = dict(
209                    (name.title(), val) for name, val in list(headers.items()))
210
211                if req._tunnel_host:
212                    tunnel_headers = {}
213                    proxy_auth_hdr = "Proxy-Authorization"
214                    if proxy_auth_hdr in headers:
215                        tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
216                        # Proxy-Authorization should not be sent to origin
217                        # server.
218                        del headers[proxy_auth_hdr]
219                    h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
220
221                try:
222                    h.request(req.get_method(), req.selector, req.data, headers)
223                except socket.error as err: # XXX what error?
224                    # Don't close connection when cache is enabled.
225                    # Instead, try to detect connections that are no longer
226                    # usable (for example, closed unexpectedly) and remove
227                    # them from the cache.
228                    if fetch.connection_cache is None:
229                        h.close()
230                    elif isinstance(err, OSError) and err.errno == errno.EBADF:
231                        # This happens when the server closes the connection despite the Keep-Alive.
232                        # Apparently urllib then uses the file descriptor, expecting it to be
233                        # connected, when in reality the connection is already gone.
234                        # We let the request fail and expect it to be
235                        # tried once more ("try_again" in check_status()),
236                        # with the dead connection removed from the cache.
237                        # If it still fails, we give up, which can happen for bad
238                        # HTTP proxy settings.
239                        fetch.connection_cache.remove_connection(h.host, h.port)
240                    raise urllib.error.URLError(err)
241                else:
242                    try:
243                        r = h.getresponse()
244                    except TimeoutError as e:
245                        if fetch.connection_cache:
246                            fetch.connection_cache.remove_connection(h.host, h.port)
247                        raise TimeoutError(e)
248
249                # Pick apart the HTTPResponse object to get the addinfourl
250                # object initialized properly.
251
252                # Wrap the HTTPResponse object in socket's file object adapter
253                # for Windows.  That adapter calls recv(), so delegate recv()
254                # to read().  This weird wrapping allows the returned object to
255                # have readline() and readlines() methods.
256
257                # XXX It might be better to extract the read buffering code
258                # out of socket._fileobject() and into a base class.
259                r.recv = r.read
260
261                # no data, just have to read
262                r.read()
263                class fp_dummy(object):
264                    def read(self):
265                        return ""
266                    def readline(self):
267                        return ""
268                    def close(self):
269                        pass
270                    closed = False
271
272                resp = urllib.response.addinfourl(fp_dummy(), r.msg, req.get_full_url())
273                resp.code = r.status
274                resp.msg = r.reason
275
276                # Close connection when server request it.
277                if fetch.connection_cache is not None:
278                    if 'Connection' in r.msg and r.msg['Connection'] == 'close':
279                        fetch.connection_cache.remove_connection(h.host, h.port)
280
281                return resp
282
283        class HTTPMethodFallback(urllib.request.BaseHandler):
284            """
285            Fallback to GET if HEAD is not allowed (405 HTTP error)
286            """
287            def http_error_405(self, req, fp, code, msg, headers):
288                fp.read()
289                fp.close()
290
291                if req.get_method() != 'GET':
292                    newheaders = dict((k, v) for k, v in list(req.headers.items())
293                                      if k.lower() not in ("content-length", "content-type"))
294                    return self.parent.open(urllib.request.Request(req.get_full_url(),
295                                                            headers=newheaders,
296                                                            origin_req_host=req.origin_req_host,
297                                                            unverifiable=True))
298
299                raise urllib.request.HTTPError(req, code, msg, headers, None)
300
301            # Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
302            # Forbidden when they actually mean 405 Method Not Allowed.
303            http_error_403 = http_error_405
304
305
306        class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
307            """
308            urllib2.HTTPRedirectHandler resets the method to GET on redirect,
309            when we want to follow redirects using the original method.
310            """
311            def redirect_request(self, req, fp, code, msg, headers, newurl):
312                newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
313                newreq.get_method = req.get_method
314                return newreq
315
316        # We need to update the environment here as both the proxy and HTTPS
317        # handlers need variables set. The proxy needs http_proxy and friends to
318        # be set, and HTTPSHandler ends up calling into openssl to load the
319        # certificates. In buildtools configurations this will be looking at the
320        # wrong place for certificates by default: we set SSL_CERT_FILE to the
321        # right location in the buildtools environment script but as BitBake
322        # prunes prunes the environment this is lost. When binaries are executed
323        # runfetchcmd ensures these values are in the environment, but this is
324        # pure Python so we need to update the environment.
325        #
326        # Avoid tramping the environment too much by using bb.utils.environment
327        # to scope the changes to the build_opener request, which is when the
328        # environment lookups happen.
329        newenv = bb.fetch2.get_fetcher_environment(d)
330
331        with bb.utils.environment(**newenv):
332            import ssl
333
334            if self.check_certs(d):
335                context = ssl.create_default_context()
336            else:
337                context = ssl._create_unverified_context()
338
339            handlers = [FixedHTTPRedirectHandler,
340                        HTTPMethodFallback,
341                        urllib.request.ProxyHandler(),
342                        CacheHTTPHandler(),
343                        urllib.request.HTTPSHandler(context=context)]
344            opener = urllib.request.build_opener(*handlers)
345
346            try:
347                uri_base = ud.url.split(";")[0]
348                uri = "{}://{}{}".format(urllib.parse.urlparse(uri_base).scheme, ud.host, ud.path)
349                r = urllib.request.Request(uri)
350                r.get_method = lambda: "HEAD"
351                # Some servers (FusionForge, as used on Alioth) require that the
352                # optional Accept header is set.
353                r.add_header("Accept", "*/*")
354                r.add_header("User-Agent", "bitbake/{}".format(bb.__version__))
355                def add_basic_auth(login_str, request):
356                    '''Adds Basic auth to http request, pass in login:password as string'''
357                    import base64
358                    encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
359                    authheader = "Basic %s" % encodeuser
360                    r.add_header("Authorization", authheader)
361
362                if ud.user and ud.pswd:
363                    add_basic_auth(ud.user + ':' + ud.pswd, r)
364
365                try:
366                    import netrc
367                    auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname)
368                    if auth_data:
369                        login, _, password = auth_data
370                        add_basic_auth("%s:%s" % (login, password), r)
371                except (FileNotFoundError, netrc.NetrcParseError):
372                    pass
373
374                with opener.open(r, timeout=100) as response:
375                    pass
376            except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
377                if try_again:
378                    logger.debug2("checkstatus: trying again")
379                    return self.checkstatus(fetch, ud, d, False)
380                else:
381                    # debug for now to avoid spamming the logs in e.g. remote sstate searches
382                    logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e))
383                    return False
384
385        return True
386
387    def _parse_path(self, regex, s):
388        """
389        Find and group name, version and archive type in the given string s
390        """
391
392        m = regex.search(s)
393        if m:
394            pname = ''
395            pver = ''
396            ptype = ''
397
398            mdict = m.groupdict()
399            if 'name' in mdict.keys():
400                pname = mdict['name']
401            if 'pver' in mdict.keys():
402                pver = mdict['pver']
403            if 'type' in mdict.keys():
404                ptype = mdict['type']
405
406            bb.debug(3, "_parse_path: %s, %s, %s" % (pname, pver, ptype))
407
408            return (pname, pver, ptype)
409
410        return None
411
412    def _modelate_version(self, version):
413        if version[0] in ['.', '-']:
414            if version[1].isdigit():
415                version = version[1] + version[0] + version[2:len(version)]
416            else:
417                version = version[1:len(version)]
418
419        version = re.sub('-', '.', version)
420        version = re.sub('_', '.', version)
421        version = re.sub('(rc)+', '.1000.', version)
422        version = re.sub('(beta)+', '.100.', version)
423        version = re.sub('(alpha)+', '.10.', version)
424        if version[0] == 'v':
425            version = version[1:len(version)]
426        return version
427
428    def _vercmp(self, old, new):
429        """
430        Check whether 'new' is newer than 'old' version. We use existing vercmp() for the
431        purpose. PE is cleared in comparison as it's not for build, and PR is cleared too
432        for simplicity as it's somehow difficult to get from various upstream format
433        """
434
435        (oldpn, oldpv, oldsuffix) = old
436        (newpn, newpv, newsuffix) = new
437
438        # Check for a new suffix type that we have never heard of before
439        if newsuffix:
440            m = self.suffix_regex_comp.search(newsuffix)
441            if not m:
442                bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix))
443                return False
444
445        # Not our package so ignore it
446        if oldpn != newpn:
447            return False
448
449        oldpv = self._modelate_version(oldpv)
450        newpv = self._modelate_version(newpv)
451
452        return bb.utils.vercmp(("0", oldpv, ""), ("0", newpv, ""))
453
454    def _fetch_index(self, uri, ud, d):
455        """
456        Run fetch checkstatus to get directory information
457        """
458        f = tempfile.NamedTemporaryFile()
459        with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
460            fetchcmd = self.basecmd
461            fetchcmd += " -O " + f.name + " '" + uri + "'"
462            try:
463                self._runwget(ud, d, fetchcmd, True, workdir=workdir)
464                fetchresult = f.read()
465            except bb.fetch2.BBFetchException:
466                fetchresult = ""
467
468        return fetchresult
469
470    def _check_latest_version(self, url, package, package_regex, current_version, ud, d):
471        """
472        Return the latest version of a package inside a given directory path
473        If error or no version, return ""
474        """
475        valid = 0
476        version = ['', '', '']
477
478        bb.debug(3, "VersionURL: %s" % (url))
479        soup = BeautifulSoup(self._fetch_index(url, ud, d), "html.parser", parse_only=SoupStrainer("a"))
480        if not soup:
481            bb.debug(3, "*** %s NO SOUP" % (url))
482            return ""
483
484        for line in soup.find_all('a', href=True):
485            bb.debug(3, "line['href'] = '%s'" % (line['href']))
486            bb.debug(3, "line = '%s'" % (str(line)))
487
488            newver = self._parse_path(package_regex, line['href'])
489            if not newver:
490                newver = self._parse_path(package_regex, str(line))
491
492            if newver:
493                bb.debug(3, "Upstream version found: %s" % newver[1])
494                if valid == 0:
495                    version = newver
496                    valid = 1
497                elif self._vercmp(version, newver) < 0:
498                    version = newver
499
500        pupver = re.sub('_', '.', version[1])
501
502        bb.debug(3, "*** %s -> UpstreamVersion = %s (CurrentVersion = %s)" %
503                (package, pupver or "N/A", current_version[1]))
504
505        if valid:
506            return pupver
507
508        return ""
509
510    def _check_latest_version_by_dir(self, dirver, package, package_regex, current_version, ud, d):
511        """
512        Scan every directory in order to get upstream version.
513        """
514        version_dir = ['', '', '']
515        version = ['', '', '']
516
517        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
518        s = dirver_regex.search(dirver)
519        if s:
520            version_dir[1] = s.group('ver')
521        else:
522            version_dir[1] = dirver
523
524        dirs_uri = bb.fetch.encodeurl([ud.type, ud.host,
525                ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
526        bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
527
528        soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a"))
529        if not soup:
530            return version[1]
531
532        for line in soup.find_all('a', href=True):
533            s = dirver_regex.search(line['href'].strip("/"))
534            if s:
535                sver = s.group('ver')
536
537                # When prefix is part of the version directory it need to
538                # ensure that only version directory is used so remove previous
539                # directories if exists.
540                #
541                # Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected
542                # result is v2.5.
543                spfx = s.group('pfx').split('/')[-1]
544
545                version_dir_new = ['', sver, '']
546                if self._vercmp(version_dir, version_dir_new) <= 0:
547                    dirver_new = spfx + sver
548                    path = ud.path.replace(dirver, dirver_new, True) \
549                        .split(package)[0]
550                    uri = bb.fetch.encodeurl([ud.type, ud.host, path,
551                        ud.user, ud.pswd, {}])
552
553                    pupver = self._check_latest_version(uri,
554                            package, package_regex, current_version, ud, d)
555                    if pupver:
556                        version[1] = pupver
557
558                    version_dir = version_dir_new
559
560        return version[1]
561
562    def _init_regexes(self, package, ud, d):
563        """
564        Match as many patterns as possible such as:
565                gnome-common-2.20.0.tar.gz (most common format)
566                gtk+-2.90.1.tar.gz
567                xf86-input-synaptics-12.6.9.tar.gz
568                dri2proto-2.3.tar.gz
569                blktool_4.orig.tar.gz
570                libid3tag-0.15.1b.tar.gz
571                unzip552.tar.gz
572                icu4c-3_6-src.tgz
573                genext2fs_1.3.orig.tar.gz
574                gst-fluendo-mp3
575        """
576        # match most patterns which uses "-" as separator to version digits
577        pn_prefix1 = r"[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]"
578        # a loose pattern such as for unzip552.tar.gz
579        pn_prefix2 = r"[a-zA-Z]+"
580        # a loose pattern such as for 80325-quicky-0.4.tar.gz
581        pn_prefix3 = r"[0-9]+[-]?[a-zA-Z]+"
582        # Save the Package Name (pn) Regex for use later
583        pn_regex = r"(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3)
584
585        # match version
586        pver_regex = r"(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)"
587
588        # match arch
589        parch_regex = "-source|_all_"
590
591        # src.rpm extension was added only for rpm package. Can be removed if the rpm
592        # packaged will always be considered as having to be manually upgraded
593        psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)"
594
595        # match name, version and archive type of a package
596        package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
597                                                    % (pn_regex, pver_regex, parch_regex, psuffix_regex))
598        self.suffix_regex_comp = re.compile(psuffix_regex)
599
600        # compile regex, can be specific by package or generic regex
601        pn_regex = d.getVar('UPSTREAM_CHECK_REGEX')
602        if pn_regex:
603            package_custom_regex_comp = re.compile(pn_regex)
604        else:
605            version = self._parse_path(package_regex_comp, package)
606            if version:
607                package_custom_regex_comp = re.compile(
608                    r"(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" %
609                    (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex))
610            else:
611                package_custom_regex_comp = None
612
613        return package_custom_regex_comp
614
615    def latest_versionstring(self, ud, d):
616        """
617        Manipulate the URL and try to obtain the latest package version
618
619        sanity check to ensure same name and type.
620        """
621        package = ud.path.split("/")[-1]
622        current_version = ['', d.getVar('PV'), '']
623
624        """possible to have no version in pkg name, such as spectrum-fw"""
625        if not re.search(r"\d+", package):
626            current_version[1] = re.sub('_', '.', current_version[1])
627            current_version[1] = re.sub('-', '.', current_version[1])
628            return (current_version[1], '')
629
630        package_regex = self._init_regexes(package, ud, d)
631        if package_regex is None:
632            bb.warn("latest_versionstring: package %s don't match pattern" % (package))
633            return ('', '')
634        bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
635
636        uri = ""
637        regex_uri = d.getVar("UPSTREAM_CHECK_URI")
638        if not regex_uri:
639            path = ud.path.split(package)[0]
640
641            # search for version matches on folders inside the path, like:
642            # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
643            dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
644            m = dirver_regex.findall(path)
645            if m:
646                pn = d.getVar('PN')
647                dirver = m[-1][0]
648
649                dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
650                if not dirver_pn_regex.search(dirver):
651                    return (self._check_latest_version_by_dir(dirver,
652                        package, package_regex, current_version, ud, d), '')
653
654            uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}])
655        else:
656            uri = regex_uri
657
658        return (self._check_latest_version(uri, package, package_regex,
659                current_version, ud, d), '')
660